package shuffle

import (
	"bytes"
	"context"
	"crypto/md5"
	"crypto/sha1"
	"crypto/tls"
	"encoding/hex"
	"encoding/json"

	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"strconv"

	"crypto/sha256"
	"math"
	"math/rand"
	"sort"
	"strings"
	"sync"
	"time"

	//"github.com/goccy/go-json"

	runtimeDebug "runtime/debug"

	"cloud.google.com/go/datastore"
	"github.com/Masterminds/semver"
	"github.com/bradfitz/slice"
	uuid "github.com/satori/go.uuid"

	//"github.com/frikky/kin-openapi/openapi3"
	"github.com/patrickmn/go-cache"
	"google.golang.org/api/iterator"

	"cloud.google.com/go/storage"
	gomemcache "github.com/bradfitz/gomemcache/memcache"
	"google.golang.org/appengine/memcache"

	opensearch "github.com/shuffle/opensearch-go/v4"
	//opensearch "github.com/opensearch-project/opensearch-go"
	//elasticsearch "github.com/elastic/go-elasticsearch/v8"
	//"github.com/opensearch-project/opensearch-go/v2/opensearchapi"
	"github.com/shuffle/opensearch-go/v4/opensearchapi"
)

var requestCache = cache.New(60*time.Minute, 60*time.Minute)
var memcached = os.Getenv("SHUFFLE_MEMCACHED")
var mc = gomemcache.New(memcached)
var gceProject = os.Getenv("SHUFFLE_GCEPROJECT")
var propagateUrl = os.Getenv("SHUFFLE_PROPAGATE_URL")
var propagateToken = os.Getenv("SHUFFLE_PROPAGATE_TOKEN")

var maxCacheSize = 1020000

// Dumps data from cache to DB for every {dbInterval} action (tried 5, 10, 25)
type ShuffleStorage struct {
	GceProject    string
	Dbclient      datastore.Client
	StorageClient storage.Client
	Environment   string
	CacheDb       bool
	Es            opensearchapi.Client
	DbType        string
	CloudUrl      string
	BucketName    string
}

// Create ElasticSearch/OpenSearch index prefix
// It is used where a single cluster of ElasticSearch/OpenSearch utilized by several
// Shuffle instance
// E.g. Instance1_Workflowapp
func GetESIndexPrefix(index string) string {
	prefix := os.Getenv("SHUFFLE_OPENSEARCH_INDEX_PREFIX")
	if len(prefix) > 0 {
		return fmt.Sprintf("%s_%s", prefix, index)
	}

	return index
}

func GetOpensearchBaseIndexes() []string {
	return []string{
		"workflowexecution",
		"datastore_ngram",
		"org_cache",
		"org_cache_revisions",
		"notifications",
		"shuffle_logs",
		"environments",
		"org_statistics",
		"workflowapp",
		"workflow",
		"workflow_revisions",
		"datastore_category",
	}
}

func SetOrgStatistics(ctx context.Context, stats ExecutionInfo, id string) error {
	nameKey := "org_statistics"

	// dedup based on date
	if stats.OrgId == "" {
		_, err := GetOrgStatistics(ctx, id)
		if err == nil {
			log.Printf("[ERROR] Org statistics already exists for org %s, skipping initialization with user stats.", id)
			return nil
		}

		log.Printf("[WARNING] Initializing org stats for org %s as org ID wasn't set", id)
		stats.OrgId = id
	}

	allDates := []string{}
	newDaily := []DailyStatistics{}
	for _, stat := range stats.OnpremStats {
		if stat.Date.IsZero() {
			continue
		}

		stat.Date = stat.Date.UTC()
		statdate := stat.Date.Format("2006-12-30")
		if !ArrayContains(allDates, statdate) {
			newDaily = append(newDaily, stat)
			allDates = append(allDates, statdate)
		}
	}

	if len(newDaily) < len(stats.OnpremStats) {
		if debug {
			log.Printf("[DEBUG] Deduped %d stats for org %s", len(stats.OnpremStats)-len(newDaily), id)
		}
	}

	stats.OnpremStats = newDaily

	data, err := json.Marshal(stats)
	if err != nil {
		log.Printf("[ERROR] Failed marshalling in set stats: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err := indexEs(ctx, nameKey, id, data)
		if err != nil {
			log.Printf("[ERROR] Failed indexing in set stats: %s", err)
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if _, putErr := project.Dbclient.Put(ctx, key, &stats); putErr != nil {
			log.Printf("[ERROR] Failed adding stats with ID %s: %s", id, putErr)

			if strings.Contains(fmt.Sprintf("%s", putErr), "entity is too big") {
				log.Printf("[WARNING] SetOrgStatistics: entity too big for org %s – archiving to GCS and trimming", id)

				if archiveErr := archiveOldStatsToGCSBucket(ctx, id, &stats); archiveErr != nil {
					log.Printf("[WARNING] SetOrgStatistics: GCS archive failed for org %s: %s – trimming anyway", id, archiveErr)
				}

				if len(stats.DailyStatistics) > 60 {
					sort.Slice(stats.DailyStatistics, func(a, b int) bool {
						return stats.DailyStatistics[a].Date.Before(stats.DailyStatistics[b].Date)
					})
					stats.DailyStatistics = stats.DailyStatistics[len(stats.DailyStatistics)-60:]
				}

				if _, retryErr := project.Dbclient.Put(ctx, key, &stats); retryErr != nil {
					log.Printf("[ERROR] SetOrgStatistics: retry put failed for org %s: %s", id, retryErr)
					return retryErr
				}

				log.Printf("[INFO] SetOrgStatistics: saved trimmed stats (last 60 days) for org %s", id)
			} else {
				return putErr
			}
		}

	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
		data, err := json.Marshal(data)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in set org stats: %s", err)
			return nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for org stats '%s': %s", cacheKey, err)
		}
	}

	return nil
}

// Cache handlers
func DeleteCache(ctx context.Context, name string) error {
	if len(memcached) > 0 {
		return mc.Delete(name)
	}

	//if project.Environment == "cloud" {
	if false {
		return memcache.Delete(ctx, name)

	} else if project.Environment == "onprem" {
		requestCache.Delete(name)
		return nil
	} else {
		requestCache.Delete(name)
		return nil
	}

	return errors.New(fmt.Sprintf("No cache found for %s when DELETING cache", name))
}

// Cache handlers
func GetCache(ctx context.Context, name string) (interface{}, error) {
	if len(name) == 0 {
		log.Printf("[ERROR] No name provided for cache")
		return "", nil
	}

	name = strings.Replace(name, " ", "_", -1)

	if len(memcached) > 0 {
		item, err := mc.Get(name)
		if err == gomemcache.ErrCacheMiss {
			//log.Printf("[DEBUG] Cache miss for %s: %s", name, err)
		} else if err != nil {
			//log.Printf("[DEBUG] Failed to find cache for key %s: %s", name, err)
		} else {
			//log.Printf("[INFO] Got new cache: %s", item)

			if len(item.Value) == maxCacheSize {
				totalData := item.Value
				keyCount := 1
				keyname := fmt.Sprintf("%s_%d", name, keyCount)
				for {
					if item, err := mc.Get(keyname); err != nil {
						break
					} else {
						if totalData != nil && item != nil && item.Value != nil {
							totalData = append(totalData, item.Value...)
						}

						//log.Printf("%d - %d = ", len(item.Value), maxCacheSize)
						if len(item.Value) != maxCacheSize {
							break
						}
					}

					keyCount += 1
					keyname = fmt.Sprintf("%s_%d", name, keyCount)
				}

				// Random~ high number
				if len(totalData) > 10062147 {
					//log.Printf("[WARNING] CACHE: TOTAL SIZE FOR %s: %d", name, len(totalData))
				}

				if len(totalData) == 0 {
					log.Printf("[ERROR] Cache payload invalid for key %s", name)
					return "", fmt.Errorf("Cache payload invalid for %s", name)
				}

				return totalData, nil
			} else {
				if len(item.Value) == 0 {
					log.Printf("[ERROR] Cache payload invalid for %s", name)
					return "", fmt.Errorf("Cache payload invalid for %s", name)
				}

				return item.Value, nil
			}
		}

		return "", errors.New(fmt.Sprintf("No cache found in SHUFFLE_MEMCACHED for %s", name))
	}

	if false {

		if item, err := memcache.Get(ctx, name); err != nil {

		} else if err != nil {
			return "", errors.New(fmt.Sprintf("Failed getting CLOUD cache for %s: %s", name, err))
		} else {
			// Loops if cachesize is more than max allowed in memcache (multikey)
			if len(item.Value) == maxCacheSize {
				totalData := item.Value
				keyCount := 1
				keyname := fmt.Sprintf("%s_%d", name, keyCount)
				for {
					if item, err := memcache.Get(ctx, keyname); err != nil {
						break
					} else {
						totalData = append(totalData, item.Value...)

						//log.Printf("%d - %d = ", len(item.Value), maxCacheSize)
						if len(item.Value) != maxCacheSize {
							break
						}
					}

					keyCount += 1
					keyname = fmt.Sprintf("%s_%d", name, keyCount)
				}

				// Random~ high number
				if len(totalData) > 10062147 {
					//log.Printf("[WARNING] CACHE: TOTAL SIZE FOR %s: %d", name, len(totalData))
				}
				return totalData, nil
			} else {
				return item.Value, nil
			}
		}
	} else if project.Environment == "onprem" {
		//log.Printf("[INFO] GETTING CACHE FOR %s ONPREM", name)
		if value, found := requestCache.Get(name); found {
			return value, nil
		} else {
			return "", errors.New(fmt.Sprintf("Failed getting ONPREM cache for %s", name))
		}
	} else {
		if value, found := requestCache.Get(name); found {
			return value, nil
		} else {
			return "", errors.New(fmt.Sprintf("Failed getting cache for %s", name))
		}
		//return "", errors.New(fmt.Sprintf("No cache handler for environment %s yet", project.Environment))
	}

	return "", errors.New(fmt.Sprintf("No cache found for %s", name))
}

// Sets a key in cache. Expiration is in minutes, unless you pass in useMilliseconds=true
// Added Millisecond timeout because some things like execution results may need more precise timing. Use by adding a true boolean as the last parameter.
func SetCache(ctx context.Context, name string, data []byte, expiration int32, useMillisecondsInput ...bool) error {
	// Set cache verbose
	//if strings.Contains(name, "execution") || strings.Contains(name, "action") && len(data) > 1 {
	//}

	if len(name) == 0 {
		log.Printf("[WARNING] Key '%s' is empty with value length %d and expiration %d. Skipping cache.", name, len(data), expiration)
		return nil
	}

	if len(data) == 0 {
		log.Printf("[WARNING] Data is empty with key %s and expiration %d. Skipping cache", name, expiration)
	}

	useMilliseconds := false
	if len(useMillisecondsInput) > 0 {
		if useMillisecondsInput[0] {
			useMilliseconds = true
		}
	}

	// Maxsize ish~
	name = strings.Replace(name, " ", "_", -1)

	// Splitting into multiple cache items
	//if project.Environment == "cloud" || len(memcached) > 0 {
	if len(memcached) > 0 {
		//	comparisonNumber := 100
		//	if len(data) > maxCacheSize*comparisonNumber {
		//		return errors.New(fmt.Sprintf("Couldn't set cache for %s - too large: %d > %d", name, len(data), maxCacheSize*comparisonNumber))
		//	}

		loop := false
		if len(data) > maxCacheSize {
			loop = true
			//log.Printf("Should make multiple cache items for %s", name)
		}

		// Custom for larger sizes. Max is maxSize*10 when being set
		if loop {
			currentChunk := 0
			keyAmount := 0
			totalAdded := 0
			chunkSize := maxCacheSize
			nextStep := chunkSize
			keyname := name

			for {
				if len(data) < nextStep {
					nextStep = len(data)
				}

				parsedData := data[currentChunk:nextStep]
				item := &memcache.Item{
					Key:        keyname,
					Value:      parsedData,
					Expiration: time.Minute * time.Duration(expiration),
				}

				if useMilliseconds {
					item.Expiration = time.Millisecond * time.Duration(expiration)
				}

				var err error
				if len(memcached) > 0 {
					newitem := &gomemcache.Item{
						Key:        keyname,
						Value:      parsedData,
						Expiration: expiration * 60,
					}

					err = mc.Set(newitem)
				} else {
					err = memcache.Set(ctx, item)
				}

				if err != nil {
					if !strings.Contains(fmt.Sprintf("%s", err), "App Engine context") {
						log.Printf("[ERROR] Failed setting cache for '%s' (1): %s", keyname, err)
					}
					break
				} else {
					totalAdded += chunkSize
					currentChunk = nextStep
					nextStep += chunkSize

					keyAmount += 1
					//log.Printf("%s: %d: %d", keyname, totalAdded, len(data))

					keyname = fmt.Sprintf("%s_%d", name, keyAmount)
					if totalAdded > len(data) {
						break
					}
				}
			}

			//log.Printf("[INFO] Set app cache with length %d and %d keys", len(data), keyAmount)
		} else {
			item := &memcache.Item{
				Key:        name,
				Value:      data,
				Expiration: time.Minute * time.Duration(expiration),
			}

			if useMilliseconds {
				item.Expiration = time.Millisecond * time.Duration(expiration)
			}

			var err error
			if len(memcached) > 0 {
				newitem := &gomemcache.Item{
					Key:        name,
					Value:      data,
					Expiration: expiration * 60,
				}

				err = mc.Set(newitem)
			} else {
				err = memcache.Set(ctx, item)
			}

			if err != nil {
				if !strings.Contains(fmt.Sprintf("%s", err), "App Engine context") {
					log.Printf("[ERROR] Failed setting memcache for key '%s' with data size %d (2): %s", name, len(data), err)
				} else {
					log.Printf("[ERROR] Something bad with App Engine context for memcache (key: %s): %s", name, err)
				}
			}
		}

		return nil
	} else if project.Environment == "onprem" {
		if useMilliseconds {
			requestCache.Set(name, data, time.Millisecond*time.Duration(expiration))
		} else {
			requestCache.Set(name, data, time.Minute*time.Duration(expiration))
		}
	} else {
		if useMilliseconds {
			requestCache.Set(name, data, time.Millisecond*time.Duration(expiration))
		} else {
			requestCache.Set(name, data, time.Minute*time.Duration(expiration))
		}

	}

	return nil
}

func GetDatastoreClient(ctx context.Context, projectID string) (datastore.Client, error) {
	client, err := datastore.NewClient(ctx, projectID)
	if err != nil {
		return datastore.Client{}, err
	}

	return *client, nil
}

func SetWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error {
	nameKey := "workflowapp"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	timeNow := int64(time.Now().Unix())
	workflowapp.Edited = timeNow

	if workflowapp.Created == 0 {
		workflowapp.Created = timeNow
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(workflowapp)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in setapp: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, workflowapp.ID, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &workflowapp); err != nil {
			if strings.Contains(fmt.Sprintf("%s", err), "entity is too big") || strings.Contains(fmt.Sprintf("%s", err), "is longer than") {
				workflowapp, err = UploadAppSpecFiles(ctx, &project.StorageClient, workflowapp, ParsedOpenApi{})
				if err != nil {
					log.Printf("[WARNING] Failed uploading app spec file in set workflow app: %s", err)
				} else {
					if _, err = project.Dbclient.Put(ctx, key, &workflowapp); err != nil {
						log.Printf("[ERROR] Failed second upload of app %s (%s): %s", workflowapp.Name, workflowapp.ID, err)
					} else {
						log.Printf("[DEBUG] Successfully updated app %s (%s)!", workflowapp.Name, workflowapp.ID)
					}
				}
			} else {
				log.Printf("[WARNING] Error adding workflow app: %s", err)
			}

			if err != nil {
				return err
			}
		}
	}

	if project.CacheDb {
		// Don't want to overwrite this part.
		//data, err := json.Marshal(workflowapp)
		//if err != nil {
		//	log.Printf("[WARNING] Failed marshalling in setapp: %s", err)
		//	return nil
		//}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[ERROR] Failed setting cache for 'setapp' key %s: %s", cacheKey, err)

		}

		DeleteCache(ctx, fmt.Sprintf("openapi3_%s", id))
	}

	return nil
}

func SetWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error {
	nameKey := "workflowexecution"
	if len(workflowExecution.ExecutionId) == 0 {
		log.Printf("[ERROR] Workflowexecution executionId can't be empty.")

		// Generate it on the fly?
		//workflowExecution.ExecutionId = uuid.NewV4().String()
		return errors.New("ExecutionId can't be empty.")
	}

	if len(workflowExecution.WorkflowId) == 0 {
		log.Printf("[ERROR][%s] Workflowexecution workflowId can't be empty.", workflowExecution.ExecutionId)
		workflowExecution.WorkflowId = workflowExecution.Workflow.ID
	}

	if len(workflowExecution.Authorization) == 0 {
		log.Printf("[ERROR][%s] Workflowexecution authorization can't be empty.", workflowExecution.ExecutionId)
		//workflowExecution.Authorization = uuid.NewV4().String()
		return errors.New("Authorization can't be empty.")
	}

	// Fixes missing pieces
	workflowExecution, newDbSave := Fixexecution(ctx, workflowExecution)
	workflowExecution = cleanupExecutionNodes(ctx, workflowExecution)
	if newDbSave {
		dbSave = true
	}

	cacheKey := fmt.Sprintf("%s_%s", nameKey, workflowExecution.ExecutionId)
	executionData, err := json.Marshal(workflowExecution)
	if err == nil {
		err = SetCache(ctx, cacheKey, executionData, 31)
		if err != nil {
			//log.Printf("[WARNING] Failed updating execution cache. Setting DB! %s", err)
			dbSave = true
		} else {

		}
	} else {
		//log.Printf("[ERROR] Failed marshalling execution for cache: %s", err)
		//log.Printf("[INFO] Set execution cache for workflowexecution %s", cacheKey)
	}

	// Weird workaround that only applies during local development
	hostname, err := os.Hostname()
	if err != nil || hostname == "debian" {
		hostname = "shuffle-backend"
	}

	// FIXME: This right here has caused more problems during dev than anything
	if (os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || project.Environment == "worker") && !strings.Contains(strings.ToLower(hostname), "backend") {
		if debug {
			log.Printf("[DEBUG] Not saving execution to DB (just cache), since we are running in swarm mode (SHUFFLE_SWARM_CONFIG=run).")
		}

		return nil
	}

	// This may get data from cache, hence we need to continuously set things in the database. Mainly as a precaution.
	newexec, err := GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
	if err != nil {
		return fmt.Errorf("[ERROR] Failed to get new execution(%s): %s", workflowExecution.ExecutionId, err)
	}

	HandleExecutionCacheIncrement(ctx, *newexec)
	if !dbSave && err == nil && (newexec.Status == "FINISHED" || newexec.Status == "ABORTED") {
		log.Printf("[INFO][%s] Already finished (set workflow) with status %s! Stopping the rest of the request for execution.", workflowExecution.ExecutionId, newexec.Status)
		return nil
	}

	// Deleting cache so that listing can work well
	DeleteCache(ctx, fmt.Sprintf("%s_%s", nameKey, workflowExecution.WorkflowId))
	DeleteCache(ctx, fmt.Sprintf("%s_%s_50", nameKey, workflowExecution.WorkflowId))
	DeleteCache(ctx, fmt.Sprintf("%s_%s_100", nameKey, workflowExecution.WorkflowId))
	DeleteCache(ctx, fmt.Sprintf("%s__%s", nameKey, workflowExecution.WorkflowId))
	if !dbSave && workflowExecution.Status == "EXECUTING" && len(workflowExecution.Results) > 1 {
		//log.Printf("[WARNING][%s] SHOULD skip DB saving for execution. Status: %s", workflowExecution.ExecutionId, workflowExecution.Status)

		if project.Environment != "cloud" {
			return nil
		}

		// Randomly saving once every 5 times
		// Just making sure results are saved
		if rand.Intn(5) != 1 {
			return nil
		}
	}

	if newexec.Status == "FINISHED" || newexec.Status == "ABORTED" {
		// Handles stat updates. Upgrading status to prevent timeouts for first iter of this
		ctx = context.Background()
		newexec = checkExecutionStatus(ctx, newexec)
	}

	// New struct, to not add body, author etc
	//log.Printf("[DEBUG][%s] Adding execution to database, not just cache. Workflow: %s (%s)", workflowExecution.ExecutionId, workflowExecution.Workflow.Name, workflowExecution.Workflow.ID)
	if project.DbType == "opensearch" {
		// Need to fix an indexing problem?
		// "mapper [workflow.actions.position.x] cannot be changed from type [float] to [long]"

		// Position doesn't matter in execution. Maybe just set all to 0?
		for actionIndex, _ := range workflowExecution.Workflow.Actions {
			workflowExecution.Workflow.Actions[actionIndex].Position.X = float64(0)
			workflowExecution.Workflow.Actions[actionIndex].Position.Y = float64(0)
		}

		for actionIndex, _ := range workflowExecution.Workflow.Triggers {
			workflowExecution.Workflow.Triggers[actionIndex].Position.X = float64(0)
			workflowExecution.Workflow.Triggers[actionIndex].Position.Y = float64(0)
		}

		for actionIndex, _ := range workflowExecution.Workflow.Comments {
			workflowExecution.Workflow.Comments[actionIndex].Position.X = float64(0)
			workflowExecution.Workflow.Comments[actionIndex].Position.Y = float64(0)
		}

		// Compresses and removes unecessary things
		workflowExecution, _ := compressExecution(ctx, workflowExecution, "db-connector save")

		executionData, err = json.Marshal(workflowExecution)
		if err != nil {
			log.Printf("[ERROR] Failed marshalling execution for ES: %s", err)
			return err
		}

		if debug {
			log.Printf("[DEBUG] Final string size of execution is: %d", len(executionData))
		}

		err = indexEs(ctx, nameKey, workflowExecution.ExecutionId, executionData)
		if err != nil {
			log.Printf("[ERROR] Failed saving new execution %s: %s", workflowExecution.ExecutionId, err)
			return err
		}

		//log.Printf("[INFO] Successfully saved new execution %s. Timestamp: %d!", workflowExecution.ExecutionId, workflowExecution.StartedAt)
	} else {

		// Compresses and removes unecessary things
		workflowExecution, _ := compressExecution(ctx, workflowExecution, "db-connector save")

		// Setting to nothing as this is realtime calculated anyway
		workflowExecution.Result = ""

		// Print 1 out of X times as a debug mode
		if rand.Intn(20) == 1 {
			log.Printf("[INFO][%s] Saving execution with status %s and %d/%d results (not including subflows) - 2", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
		}

		key := datastore.NameKey(nameKey, strings.ToLower(workflowExecution.ExecutionId), nil)
		if _, err := project.Dbclient.Put(ctx, key, &workflowExecution); err != nil {
			if strings.Contains(fmt.Sprintf("%s", err), "context deadline exceeded") {
				log.Printf("[ERROR][%s] Context deadline exceeded. Retrying...", workflowExecution.ExecutionId)
				ctx := context.Background()
				if _, err := project.Dbclient.Put(ctx, key, &workflowExecution); err != nil {
					log.Printf("[ERROR] Workflow execution Error number 1: %s", err)
				}
			} else if strings.Contains(fmt.Sprintf("%s", err), "context canceled") {
				log.Printf("[ERROR][%s] Context canceled, most likely with manual timeout: %s", workflowExecution.ExecutionId, err)
			} else {
				log.Printf("[ERROR][%s] Problem adding workflow_execution to datastore: %s", workflowExecution.ExecutionId, err)
			}

			// Has to do with certain data coming back in parameters where it shouldn't, causing saving to be impossible
			if strings.Contains(fmt.Sprintf("%s", err), "contains an invalid nested") {
				//log.Printf("[DEBUG] RETRYING WITHOUT WORKFLOW AND PARAMS?")
				//workflowExecution.Workflow = Workflow{}
				//newParams = []WorkflowAppActionParameters{}
				newResults := []ActionResult{}
				for _, result := range workflowExecution.Results {
					result.Action.Parameters = []WorkflowAppActionParameter{}
					newResults = append(newResults, result)
				}

				workflowExecution.Results = newResults

				key := datastore.NameKey(nameKey, workflowExecution.ExecutionId, nil)
				if _, err := project.Dbclient.Put(ctx, key, &workflowExecution); err != nil {
					log.Printf("[ERROR] Workflow execution Error number 2: %s", err)
				} else {
					return nil
				}
			}
			return err
		}
	}

	return nil
}

func GetEsConfig(defaultCreds bool) *opensearchapi.Client {
	esUrl := os.Getenv("SHUFFLE_OPENSEARCH_URL")
	if len(esUrl) == 0 {
		esUrl = "https://shuffle-opensearch:9200"
	}

	username := os.Getenv("SHUFFLE_OPENSEARCH_USERNAME")
	if len(username) == 0 {
		username = "admin"
	}

	password := os.Getenv("SHUFFLE_OPENSEARCH_PASSWORD")
	if len(password) == 0 {
		// New password that is set by default.
		// Security Audit points to changing this during onboarding.
		password = "StrongShufflePassword321!"
	}

	if defaultCreds {
		log.Printf("[DEBUG] Using default credentials for Opensearch (previous versions)")

		username = "admin"
		password = "admin"
	}

	log.Printf("[DEBUG] Using custom opensearch url '%s'", esUrl)

	// https://github.com/elastic/go-opensearch/blob/f741c073f324c15d3d401d945ee05b0c410bd06d/opensearch.go#L98
	config := opensearch.Config{
		Addresses:     strings.Split(esUrl, ","),
		Username:      username,
		Password:      password,
		MaxRetries:    5,
		RetryOnStatus: []int{500, 502, 503, 504, 429, 403},
	}

	if len(os.Getenv("SHUFFLE_OPENSEARCH_APIKEY")) > 0 {
		config.Username = ""
		config.Password = ""
		if config.Header == nil {
			config.Header = make(http.Header)
		}

		config.Header["Authorization"] = []string{"ApiKey " + os.Getenv("SHUFFLE_OPENSEARCH_APIKEY")}

	}

	//APIKey:        os.Getenv("SHUFFLE_OPENSEARCH_APIKEY"),
	//CloudID:       os.Getenv("SHUFFLE_OPENSEARCH_CLOUDID"),

	//config.Transport.TLSClientConfig
	//transport := http.DefaultTransport.(*http.Transport).Clone()
	transport := http.DefaultTransport.(*http.Transport).Clone()
	transport.MaxIdleConnsPerHost = 100
	transport.ResponseHeaderTimeout = time.Second * 10
	transport.Proxy = nil

	if len(os.Getenv("SHUFFLE_OPENSEARCH_PROXY")) > 0 {
		httpProxy := os.Getenv("SHUFFLE_OPENSEARCH_PROXY")

		url_i := url.URL{}
		url_proxy, err := url_i.Parse(httpProxy)
		if err == nil {
			log.Printf("[DEBUG] Setting Opensearch proxy to %s", httpProxy)
			transport.Proxy = http.ProxyURL(url_proxy)
		} else {
			log.Printf("[ERROR] Failed setting proxy for %s", httpProxy)
		}
	}

	skipSSLVerify := false
	if strings.ToLower(os.Getenv("SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY")) == "true" {
		//log.Printf("[DEBUG] SKIPPING SSL verification with Opensearch")
		skipSSLVerify = true
	}

	transport.TLSClientConfig = &tls.Config{
		MinVersion:         tls.VersionTLS11,
		InsecureSkipVerify: skipSSLVerify,
	}

	//https://github.com/elastic/go-opensearch/blob/master/_examples/security/opensearch-cluster.yml
	certificateLocation := os.Getenv("SHUFFLE_OPENSEARCH_CERTIFICATE_FILE")
	if len(certificateLocation) > 0 {
		cert, err := ioutil.ReadFile(certificateLocation)
		if err != nil {
			log.Fatalf("[WARNING] Failed configuring certificates: %s not found", err)
		} else {
			config.CACert = cert

			//if transport.TLSClientConfig.RootCAs, err = x509.SystemCertPool(); err != nil {
			//	log.Fatalf("[ERROR] Problem adding system CA: %s", err)
			//}

			//// --> Add the custom certificate authority
			//if ok := transport.TLSClientConfig.RootCAs.AppendCertsFromPEM(cert); !ok {
			//	log.Fatalf("[ERROR] Problem adding CA from file %q", *cert)
			//}
		}

		log.Printf("[INFO] Added certificate %s elastic client.", certificateLocation)
	}

	config.Transport = transport
	es, err := opensearchapi.NewClient(
		opensearchapi.Config{
			Client: config,
		},
	)

	if err != nil {
		log.Fatalf("[ERROR] Database client for ELASTICSEARCH error during init (fatal): %s", err)
	}

	return es
}

func GetWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) {
	nameKey := "workflowexecution"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)

	// Loads of cache management to ensure we have the latest version of the execution no matter what
	workflowExecution := &WorkflowExecution{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, workflowExecution)

			if (err == nil && workflowExecution != nil) && len(workflowExecution.ExecutionId) > 0 {
				//log.Printf("[DEBUG] Checking individual execution cache with %d results", len(workflowExecution.Results))
				if strings.Contains(workflowExecution.ExecutionArgument, "Result too large to handle") {
					baseArgument := &ActionResult{
						Result: workflowExecution.ExecutionArgument,
						Action: Action{ID: "execution_argument"},
					}

					newValue, err := getExecutionFileValue(ctx, *workflowExecution, *baseArgument)
					if err != nil {
						log.Printf("[DEBUG][%s] Failed to parse in execution file value for exec argument: %s (3)", workflowExecution.ExecutionId, err)
					} else {
						//log.Printf("[DEBUG][%s] Found a new value to parse with exec argument", workflowExecution.ExecutionId)
						workflowExecution.ExecutionArgument = newValue
					}
				}

				if strings.Contains(workflowExecution.Result, "Result too large to handle") {
					baseResult := &ActionResult{
						Result: workflowExecution.Result,
						Action: Action{ID: "execution_result"},
					}

					newValue, err := getExecutionFileValue(ctx, *workflowExecution, *baseResult)
					if err != nil {
						log.Printf("[DEBUG][%s] Failed to parse in execution file value for Result: %s", workflowExecution.ExecutionId, err)
					} else {
						log.Printf("[DEBUG][%s] Found a new value to parse with Result field", workflowExecution.ExecutionId)
						workflowExecution.Result = newValue
					}
				}

				for valueIndex, value := range workflowExecution.Results {
					if strings.Contains(value.Result, "Result too large to handle") {
						newValue, err := getExecutionFileValue(ctx, *workflowExecution, value)
						if err != nil {
							continue
						}

						workflowExecution.Results[valueIndex].Result = newValue
					}
				}

				// Fixes missing pieces
				newexec, _ := Fixexecution(ctx, *workflowExecution)
				workflowExecution = &newexec

				return workflowExecution, nil
			} else {
				if debug {
					log.Printf("[DEBUG] Failed mapping workflowexecution cache for '%s': %s", id, err)
				}
			}
		} else {
		}
	}

	if (os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || project.Environment == "worker") && project.Environment != "cloud" {
		return workflowExecution, errors.New("ExecutionId doesn't exist in cache")
	}

	var getErr error = nil
	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})

		if err != nil {
			if strings.Contains(err.Error(), "has more than one index associated with it") {
				fallbackExec, fallbackErr := getWorkflowExecutionByAliasSearch(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), id)
				if fallbackErr != nil {
					log.Printf("[WARNING][%s] Error for %s: %s", workflowExecution.ExecutionId, cacheKey, err)
					log.Printf("[WARNING][%s] WorkflowExecution alias fallback failed for %s: %s", workflowExecution.ExecutionId, cacheKey, fallbackErr)
					return workflowExecution, fallbackErr
				}

				workflowExecution = fallbackExec
			} else {
				log.Printf("[WARNING][%s] Error for %s: %s", workflowExecution.ExecutionId, cacheKey, err)
				return workflowExecution, err
			}
		}

		if err == nil {
			res := resp.Inspect().Response
			defer res.Body.Close()
			if res.StatusCode == 404 {
				return workflowExecution, errors.New("execution doesn't exist")
			}

			respBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return workflowExecution, err
			}

			wrapped := ExecWrapper{}
			err = json.Unmarshal(respBody, &wrapped)
			//err = gojson.Unmarshal(respBody, &wrapped)
			if err != nil && len(wrapped.Source.ExecutionId) == 0 {
				return workflowExecution, err
			}

			workflowExecution = &wrapped.Source
		}
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		if getErr = project.Dbclient.Get(ctx, key, workflowExecution); getErr != nil {
			if strings.Contains(getErr.Error(), `cannot load field`) {
				getErr = nil
			} else {
				//return workflowExecution, err
			}
		}

		// A workaround for large bits of information for execution argument
		if strings.Contains(workflowExecution.ExecutionArgument, "Result too large to handle") {
			//log.Printf("[DEBUG] Found prefix %s to be replaced for exec argument (3)", workflowExecution.ExecutionArgument)
			baseArgument := &ActionResult{
				Result: workflowExecution.ExecutionArgument,
				Action: Action{ID: "execution_argument"},
			}

			newValue, err := getExecutionFileValue(ctx, *workflowExecution, *baseArgument)
			if err != nil {
				log.Printf("[DEBUG] Failed to parse in execution file value for exec argument: %s (4)", err)
			} else {
				//log.Printf("[DEBUG] Found a new value to parse with exec argument")
				workflowExecution.ExecutionArgument = newValue
			}
		}

		// Parsing as file.
		//log.Printf("[DEBUG] Got execution %s. Results: ~%d/%d", id, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
		for valueIndex, value := range workflowExecution.Results {
			if strings.Contains(value.Result, "Result too large to handle") {
				//log.Printf("[DEBUG] Found prefix %s to be replaced (2)", value.Result)
				newValue, err := getExecutionFileValue(ctx, *workflowExecution, value)
				if err != nil {
					log.Printf("[DEBUG] Failed to parse in execution file value %s (5)", err)
					continue
				}

				workflowExecution.Results[valueIndex].Result = newValue
			}
		}
	}

	//log.Printf("[DEBUG] Returned execution %s with %d results (1)", id, len(workflowExecution.Results))

	// Fixes missing pieces
	newexec, _ := Fixexecution(ctx, *workflowExecution)
	workflowExecution = &newexec

	//log.Printf("[DEBUG] Returned execution %s with %d results (2)", id, len(workflowExecution.Results))

	if project.CacheDb && workflowExecution.Authorization != "" {
		newexecution, err := json.Marshal(workflowExecution)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling execution: %s", err)
			return workflowExecution, getErr
		}

		err = SetCache(ctx, id, newexecution, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating execution: %s", err)
		}
	}

	return workflowExecution, getErr
}

func getWorkflowExecutionByAliasSearch(ctx context.Context, aliasName, id string) (*WorkflowExecution, error) {
	var buf bytes.Buffer
	query := map[string]interface{}{
		"size": 1,
		"query": map[string]interface{}{
			"ids": map[string]interface{}{
				"values": []string{id},
			},
		},
		"sort": []map[string]interface{}{
			{
				"edited": map[string]interface{}{
					"order":         "desc",
					"unmapped_type": "long",
				},
			},
			{
				"created": map[string]interface{}{
					"order":         "desc",
					"unmapped_type": "long",
				},
			},
		},
	}

	if err := json.NewEncoder(&buf).Encode(query); err != nil {
		return nil, err
	}

	resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
		Indices: []string{aliasName},
		Body:    &buf,
		Params: opensearchapi.SearchParams{
			TrackTotalHits: true,
		},
	})
	if err != nil {
		return nil, err
	}

	res := resp.Inspect().Response
	defer res.Body.Close()

	if res.StatusCode == 404 {
		return nil, errors.New("execution doesn't exist")
	}

	respBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	if res.StatusCode != 200 && res.StatusCode != 201 {
		return nil, fmt.Errorf("failed workflowexecution alias lookup. status=%d body=%s", res.StatusCode, string(respBody))
	}

	wrapped := ExecutionSearchWrapper{}
	err = json.Unmarshal(respBody, &wrapped)
	if err != nil {
		return nil, err
	}

	if len(wrapped.Hits.Hits) == 0 {
		return nil, errors.New("execution doesn't exist")
	}

	found := wrapped.Hits.Hits[0].Source
	return &found, nil
}

// archiveOldStatsToGCSBucket offloads DailyStatistics entries older than 60 days to a GCS
// bucket so they are not lost when the Datastore entity grows too large.
//
// Bucket : shuffle_org_files
// Object : org_statistics/{orgId}/stats.json
func archiveOldStatsToGCSBucket(ctx context.Context, orgId string, stats *ExecutionInfo) error {
	if project.Environment != "cloud" {
		return nil
	}

	if len(orgId) == 0 {
		return errors.New("archiveOldStatsToGCSBucket: orgId must not be empty")
	}

	// Skip if a concurrent archive is already running for this org.
	archiveCacheKey := fmt.Sprintf("gcs_archive_%s", orgId)
	if cacheVal, cacheErr := GetCache(ctx, archiveCacheKey); cacheErr == nil {
		log.Printf("[DEBUG] archiveOldStatsToGCSBucket: skipping org %s – archive in progress (key=%s val=%v)", orgId, archiveCacheKey, cacheVal)
		return nil
	} else {
		log.Printf("[DEBUG] archiveOldStatsToGCSBucket: proceeding for org %s (key=%s not set)", orgId, archiveCacheKey)
	}
	_ = SetCache(ctx, archiveCacheKey, []byte("1"), 5)

	cutoff := time.Now().UTC().AddDate(0, 0, -60)
	overflowStats := []DailyStatistics{}
	for _, d := range stats.DailyStatistics {
		if d.Date.UTC().Before(cutoff) {
			overflowStats = append(overflowStats, d)
		}
	}

	if len(overflowStats) == 0 {
		log.Printf("[DEBUG] archiveOldStatsToGCSBucket: no entries older than 60 days for org %s", orgId)
		return nil
	}

	bucketPath := fmt.Sprintf("org_statistics/%s/stats.json", orgId)
	obj := project.StorageClient.Bucket(orgFileBucket).Object(bucketPath)

	// Read existing GCS file to merge without losing older entries.
	existingStats := []DailyStatistics{}
	reader, readerErr := obj.NewReader(ctx)
	if readerErr == nil {
		existingBytes, readErr := ioutil.ReadAll(reader)
		reader.Close()
		if readErr == nil && len(existingBytes) > 0 {
			if unmarshalErr := json.Unmarshal(existingBytes, &existingStats); unmarshalErr != nil {
				log.Printf("[WARNING] archiveOldStatsToGCSBucket: could not parse existing GCS stats for org %s (will overwrite): %s", orgId, unmarshalErr)
				existingStats = []DailyStatistics{}
			}
		}
	}

	// Deduplicate by date; new overflow entries win on conflict.
	dateMapCap := len(existingStats)
	if len(overflowStats) > dateMapCap {
		dateMapCap = len(overflowStats)
	}
	dateMap := make(map[string]DailyStatistics, dateMapCap)
	for _, d := range existingStats {
		dateMap[d.Date.UTC().Format("2006-01-02")] = d
	}
	for _, d := range overflowStats {
		dateMap[d.Date.UTC().Format("2006-01-02")] = d
	}

	merged := make([]DailyStatistics, 0, len(dateMap))
	for _, d := range dateMap {
		merged = append(merged, d)
	}
	sort.Slice(merged, func(i, j int) bool {
		return merged[i].Date.Before(merged[j].Date)
	})

	mergedBytes, err := json.Marshal(merged)
	if err != nil {
		return fmt.Errorf("archiveOldStatsToGCSBucket: failed to marshal overflow stats for org %s: %w", orgId, err)
	}

	gcsWriter := obj.NewWriter(ctx)
	if _, writeErr := gcsWriter.Write(mergedBytes); writeErr != nil {
		_ = gcsWriter.Close()
		return fmt.Errorf("archiveOldStatsToGCSBucket: failed to write to GCS for org %s: %w", orgId, writeErr)
	}
	if closeErr := gcsWriter.Close(); closeErr != nil {
		return fmt.Errorf("archiveOldStatsToGCSBucket: failed to close GCS writer for org %s: %w", orgId, closeErr)
	}

	log.Printf("[INFO] archiveOldStatsToGCSBucket: archived %d entries (>60 days old) to %s/%s for org %s",
		len(overflowStats), orgFileBucket, bucketPath, orgId)
	return nil
}

func IncrementCacheDump(ctx context.Context, orgId, dataType string, amount ...int) error {

	nameKey := "org_statistics"
	orgStatistics := &ExecutionInfo{}

	dbDumpInterval := uint(dbInterval)
	if len(amount) > 0 {
		if amount[0] > 0 {
			dbDumpInterval = uint(amount[0])
		}
	}

	// Get the org
	tmpOrgDetail, err := GetOrg(ctx, orgId)
	if err != nil {
		log.Printf("[ERROR] Failed getting org in increment: %s", err)
		return err
	}

	// Ensuring we at least have one.
	if len(tmpOrgDetail.ManagerOrgs) == 0 && len(tmpOrgDetail.CreatorOrg) > 0 {
		tmpOrgDetail.ManagerOrgs = append(tmpOrgDetail.ManagerOrgs, OrgMini{
			Id: tmpOrgDetail.CreatorOrg,
		})
	}

	// FIXME: Can look for childorg_app_executions here as well which
	// would make tracking app runs at scale recursively work
	// The problem is... recursion (:

	if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "app_executions") {
		for _, managerOrg := range tmpOrgDetail.ManagerOrgs {
			if len(managerOrg.Id) == 36 {
				IncrementCache(ctx, managerOrg.Id, "childorg_app_executions", int(dbDumpInterval))
			}
		}
	}

	if len(tmpOrgDetail.ManagerOrgs) > 0 && (dataType == "workflow_executions") {
		for _, managerOrg := range tmpOrgDetail.ManagerOrgs {
			if len(managerOrg.Id) == 36 {
				IncrementCache(ctx, managerOrg.Id, "childorg_workflow_executions", int(dbDumpInterval))
			}
		}
	}

	concurrentTxn := false
	errMsg := ""

	if project.DbType == "opensearch" {
		// Get it from opensearch (may be prone to more issues at scale (thousands/second) due to no transactional locking)

		id := strings.ToLower(orgId)
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})

		if err != nil {
			if debug {
				log.Printf("[WARNING] Error in org STATS get: %s", err)
			}
			//return err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		respBody, bodyErr := ioutil.ReadAll(res.Body)
		if err != nil || bodyErr != nil || res.StatusCode >= 300 {
			log.Printf("[WARNING] Failed getting org STATS body: %s. Resp: %d. Body err: %s", err, res.StatusCode, bodyErr)

			// Init the org stats if it doesn't exist
			if res.StatusCode == 404 {
				orgStatistics.OrgId = orgId
				orgStatistics = HandleIncrement(dataType, orgStatistics, dbDumpInterval)
				orgStatistics = handleDailyCacheUpdate(orgStatistics)

				marshalledData, err := json.Marshal(orgStatistics)
				if err != nil {
					log.Printf("[ERROR] Failed marshalling org STATS body: %s", err)
				} else {
					err := indexEs(ctx, nameKey, id, marshalledData)
					if err != nil {
						log.Printf("[ERROR] Failed indexing org STATS body: %s", err)
					} else {
						log.Printf("[DEBUG] Indexed org STATS body for %s", orgId)
					}
				}
			}

			return err
		}

		orgStatsWrapper := &ExecutionInfoWrapper{}
		err = json.Unmarshal(respBody, &orgStatsWrapper)
		if err != nil {
			log.Printf("[ERROR] Failed unmarshalling org STATS body: %s", err)
			return err
		}

		orgStatistics = &orgStatsWrapper.Source
		if orgStatistics.OrgName == "" || orgStatistics.OrgName == orgStatistics.OrgId {
			org, err := GetOrg(ctx, orgId)
			if err == nil {
				orgStatistics.OrgName = org.Name
			}

			orgStatistics.OrgId = orgId
		}

		orgStatistics = HandleIncrement(dataType, orgStatistics, dbDumpInterval)
		orgStatistics = handleDailyCacheUpdate(orgStatistics)

		// Set the data back in the database
		marshalledData, err := json.Marshal(orgStatistics)
		if err != nil {
			log.Printf("[ERROR] Failed marshalling org STATS body (2): %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, id, marshalledData)
		if err != nil {
			log.Printf("[ERROR] Failed indexing org STATS body (2): %s", err)
		}

		//log.Printf("[DEBUG] Incremented org stats for %s", orgId)
	} else {
		maxRetries := 3
		for i := 0; i < maxRetries; i++ {
			concurrentTxn = false

			tx, err := project.Dbclient.NewTransaction(ctx)
			if err != nil {
				log.Printf("[WARNING] Error in cache dump: %s", err)
				return err
			}

			key := datastore.NameKey(nameKey, strings.ToLower(orgId), nil)
			if err := tx.Get(key, orgStatistics); err != nil {
				if strings.Contains(fmt.Sprintf("%s", err), "no such entity") {
					log.Printf("[DEBUG] Continuing by creating entity for org %s", orgId)
				} else {
					if !strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
						log.Printf("[ERROR] Failed getting stats in increment: %s", err)
						tx.Rollback()
						return err
					}
				}
			}

			if orgStatistics.OrgName == "" || orgStatistics.OrgName == orgStatistics.OrgId {
				org, err := GetOrg(ctx, orgId)
				if err == nil {
					orgStatistics.OrgName = org.Name
				}

				orgStatistics.OrgId = orgId
			}

			orgStatistics = HandleIncrement(dataType, orgStatistics, dbDumpInterval)
			orgStatistics = handleDailyCacheUpdate(orgStatistics)

			// Transaction control
			if _, err := tx.Put(key, orgStatistics); err != nil {
				log.Printf("[WARNING] Failed setting stats: %s", err)
				tx.Rollback()
				return err
			}

			if _, err = tx.Commit(); err != nil {
				log.Printf("[ERROR] Failed commiting stats for %s: %s", orgStatistics.OrgId, err)
				if strings.Contains(fmt.Sprintf("%s", err), "concurrent transaction") {
					concurrentTxn = true
					errMsg = fmt.Sprintf("%s", err)
					time.Sleep(time.Duration(200*(i+1)) * time.Millisecond)
					continue
				}
				return err
			}

			break
		}

		if concurrentTxn {
			log.Printf("[ERROR] Failed to update stats for org %s after %d retries: concurrent transaction error: %s", orgId, maxRetries, errMsg)
			return errors.New(errMsg)
		}

	}

	// Could use cache for everything, really
	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)
		data, err := json.Marshal(orgStatistics)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in set org stats: %s", err)
			return err
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for org stats '%s': %s", cacheKey, err)
		}
	}

	if concurrentTxn {
		return errors.New(errMsg)
	}

	return nil
}

func GetLiveWorkflowExecutionData(ctx context.Context, beforeTimestamp int, afterTimestamp int, limit int, mode string) ([]LiveExecutionStatus, error) {
	nameKey := "live_execution_status"
	liveExecs := []LiveExecutionStatus{}

	modes := []string{"1h", "7h", "1d", "7d"}
	if !ArrayContains(modes, mode) {
		mode = ""
	} else {
		beforeTimestamp = 0
		if mode == "1h" {
			afterTimestamp = int(time.Now().Unix()) - 3600
		} else if mode == "1d" {
			afterTimestamp = int(time.Now().Unix()) - 86400
		} else if mode == "7h" {
			afterTimestamp = int(time.Now().Unix()) - 25200
		} else if mode == "7d" {
			afterTimestamp = int(time.Now().Unix()) - 604800
		}
	}

	if mode != "" {
		cacheKey := fmt.Sprintf("%s-%s", nameKey, mode)
		if project.CacheDb {
			cache, err := GetCache(ctx, cacheKey)
			if err == nil {
				cacheData := []byte(cache.([]uint8))
				err = json.Unmarshal(cacheData, &liveExecs)
				if err == nil {
					return liveExecs, nil
				}
			}
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"sort": map[string]interface{}{
				"created_at": map[string]interface{}{
					"order": "desc",
				},
			},
		}

		if limit != 0 {
			query["size"] = limit
		}

		if beforeTimestamp > 0 || afterTimestamp > 0 {
			query["query"] = map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{},
				},
			}
		}

		if beforeTimestamp > 0 {
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(
				query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]map[string]interface{}),
				map[string]interface{}{
					"range": map[string]interface{}{
						"created_at": map[string]interface{}{
							"gt": beforeTimestamp,
						},
					},
				},
			)
		}

		if afterTimestamp > 0 {
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(
				query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]map[string]interface{}),
				map[string]interface{}{
					"range": map[string]interface{}{
						"created_at": map[string]interface{}{
							"lt": afterTimestamp,
						},
					},
				},
			)
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding live execution status query: %s", err)
			return liveExecs, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return liveExecs, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get live execution status): %s", err)
			return liveExecs, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode != 200 && res.StatusCode != 201 {
			return liveExecs, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return liveExecs, err
			} else {
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return liveExecs, err
		}

		wrapped := struct {
			Hits struct {
				Hits []struct {
					Source LiveExecutionStatus `json:"_source"`
				} `json:"hits"`
			} `json:"hits"`
		}{}

		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return liveExecs, err
		}

		for _, hit := range wrapped.Hits.Hits {
			liveExecs = append(liveExecs, hit.Source)
		}

	} else {
		q := datastore.NewQuery(nameKey)

		if beforeTimestamp != 0 {
			q = q.Filter("CreatedAt <", beforeTimestamp)
		}

		if afterTimestamp != 0 {
			q = q.Filter("CreatedAt >", afterTimestamp)
		}

		if limit != 0 {
			q = q.Limit(limit)
		}

		q = q.Order("-CreatedAt")

		_, err := project.Dbclient.GetAll(ctx, q, &liveExecs)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Error getting live execution status: %s", err)
				return liveExecs, err
			}
		}
	}

	if mode != "" {
		cacheKey := fmt.Sprintf("%s-%s", nameKey, mode)
		if project.CacheDb {
			data, err := json.Marshal(liveExecs)
			if err != nil {
				log.Printf("[WARNING] Failed marshalling live execution status: %s", err)
				return liveExecs, nil
			}

			var ttl int32
			ttl = 5
			if mode == "7h" {
				ttl = 60
			} else if mode == "7d" {
				ttl = 300
			} else if mode == "1d" {
				ttl = 120
			}

			err = SetCache(ctx, cacheKey, data, ttl)
			if err != nil {
				log.Printf("[WARNING] Failed updating live execution status cache: %s", err)
			}
		}
	}

	return liveExecs, nil
}

func SetLiveWorkflowExecutionData(ctx context.Context, liveExec LiveExecutionStatus) error {
	nameKey := "live_execution_status"
	// Generate random ID if not already set
	if liveExec.ID == "" {
		liveExec.ID = uuid.NewV4().String()
	}

	data, err := json.Marshal(liveExec)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set live workflow execution data: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, liveExec.ID, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, liveExec.ID, nil)
		if _, err := project.Dbclient.Put(ctx, key, &liveExec); err != nil {
			log.Printf("[WARNING] Error adding live workflow execution data: %s", err)
			return err
		}
	}

	return nil
}

// Initializes an execution's extra variables
func SetInitExecutionVariables(ctx context.Context, workflowExecution WorkflowExecution) {
	environments := []string{}
	nextActions := []string{}
	startAction := ""
	extra := 0
	parents := map[string][]string{}
	children := map[string][]string{}

	// Hmm
	triggersHandled := []string{}

	for _, action := range workflowExecution.Workflow.Actions {
		if !ArrayContains(environments, action.Environment) {
			environments = append(environments, action.Environment)
		}

		if action.ID == workflowExecution.Start {
			/*
				functionName = fmt.Sprintf("%s-%s", action.AppName, action.AppVersion)

				if !action.Sharing {
					functionName = fmt.Sprintf("%s-%s", action.AppName, action.PrivateID)
				}
			*/

			startAction = action.ID
		}
	}

	nextActions = append(nextActions, startAction)
	for _, branch := range workflowExecution.Workflow.Branches {
		// Check what the parent is first. If it's trigger - skip
		sourceFound := false
		destinationFound := false
		for _, action := range workflowExecution.Workflow.Actions {
			if action.ID == branch.SourceID {
				sourceFound = true
			}

			if action.ID == branch.DestinationID {
				destinationFound = true
			}
		}

		continueCount := true
		if extra > 0 {
			continueCount = false
		}

		for _, trigger := range workflowExecution.Workflow.Triggers {
			//log.Printf("Appname trigger (0): %s", trigger.AppName)
			if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" {
				//log.Printf("%s is a special trigger. Checking where.", trigger.AppName)

				found := false
				for _, check := range triggersHandled {
					if check == trigger.ID {
						found = true
						break
					}
				}

				if !found {
					if continueCount {
						extra += 1
					}
				} else {
					triggersHandled = append(triggersHandled, trigger.ID)
				}

				if trigger.ID == branch.SourceID {
					//log.Printf("[INFO] Trigger %s is the source!", trigger.AppName)
					sourceFound = true
				} else if trigger.ID == branch.DestinationID {
					//log.Printf("[INFO] Trigger %s is the destination!", trigger.AppName)
					destinationFound = true
				}
			}
		}

		if sourceFound {
			parents[branch.DestinationID] = append(parents[branch.DestinationID], branch.SourceID)
		} else {
			//log.Printf("[WARNING] Action ID %s was not found in actions! Skipping parent. (TRIGGER?)", branch.SourceID)
		}

		if destinationFound {
			children[branch.SourceID] = append(children[branch.SourceID], branch.DestinationID)
		} else {
			//log.Printf("[WARNING] Action ID %s was not found in actions! Skipping child. (TRIGGER?)", branch.SourceID)
		}
	}

	UpdateExecutionVariables(ctx, workflowExecution.ExecutionId, startAction, children, parents, []string{startAction}, []string{startAction}, nextActions, environments, extra)
}

func UpdateExecutionVariables(ctx context.Context, executionId, startnode string, children, parents map[string][]string, visited, executed, nextActions, environments []string, extra int) error {
	cacheKey := fmt.Sprintf("%s-actions", executionId)

	// Get first and check if too many changes
	_, _, oldchildren, oldparents, _, _, _, _ := GetExecutionVariables(ctx, executionId)

	// Don't allow certain parts to update
	if len(oldchildren) > 0 {
		children = oldchildren
	}

	if len(oldparents) > 0 {
		parents = oldparents
	}

	newVariableWrapper := ExecutionVariableWrapper{
		StartNode:    startnode,
		Children:     children,
		Parents:      parents,
		NextActions:  nextActions,
		Environments: environments,
		Extra:        extra,
		Visited:      visited,
		Executed:     visited,
	}

	variableWrapperData, err := json.Marshal(newVariableWrapper)
	if err != nil {
		log.Printf("[ERROR] Failed marshalling execution: %s", err)
		return err
	}

	err = SetCache(ctx, cacheKey, variableWrapperData, 30)
	if err != nil {
		log.Printf("[ERROR] Failed updating execution variables: %s", err)
		return err
	}

	return nil
}

func GetExecutionVariables(ctx context.Context, executionId string) (string, int, map[string][]string, map[string][]string, []string, []string, []string, []string) {

	cacheKey := fmt.Sprintf("%s-actions", executionId)
	wrapper := &ExecutionVariableWrapper{}
	cache, err := GetCache(ctx, cacheKey)
	if err == nil {
		cacheData := []byte(cache.([]uint8))
		err = json.Unmarshal(cacheData, &wrapper)
		if err == nil {

			return wrapper.StartNode, wrapper.Extra, wrapper.Children, wrapper.Parents, wrapper.Visited, wrapper.Executed, wrapper.NextActions, wrapper.Environments
		}
	} else {
		//log.Printf("[WARNING][%s] Failed getting cache for execution variables data %s: %s", executionId, executionId, err)
	}

	return "", 0, map[string][]string{}, map[string][]string{}, []string{}, []string{}, []string{}, []string{}
}

func getExecutionFileValue(ctx context.Context, workflowExecution WorkflowExecution, action ActionResult) (string, error) {
	fullParsedPath := fmt.Sprintf("large_executions/%s/%s_%s", workflowExecution.ExecutionOrg, workflowExecution.ExecutionId, action.Action.ID)

	cacheKey := fmt.Sprintf("%s_%s_action_replace", workflowExecution.ExecutionId, action.Action.ID)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := string(cache.([]uint8))
			return cacheData, nil
		}
	}

	var data []byte
	var err error

	if project.DbType == "opensearch" {
		// On-premise: read from local filesystem
		basepath := os.Getenv("SHUFFLE_FILE_LOCATION")
		if len(basepath) == 0 {
			basepath = "files"
		}

		localPath := fmt.Sprintf("%s/%s", basepath, fullParsedPath)
		data, err = ioutil.ReadFile(localPath)
		if err != nil {
			// Use DEBUG for file not found (expected on first save), ERROR for other issues
			if os.IsNotExist(err) {
				log.Printf("[DEBUG] File '%s' does not exist yet (expected on first save): %s", localPath, err)
			} else {
				log.Printf("[ERROR] Failed reading file '%s' from local storage: %s", localPath, err)
			}
			return "", err
		}
	} else {
		// Cloud: read from bucket
		projectName := os.Getenv("SHUFFLE_GCEPROJECT")
		bucketName := project.BucketName

		bucket := project.StorageClient.Bucket(bucketName)
		obj := bucket.Object(fullParsedPath)
		fileReader, err := obj.NewReader(ctx)
		if err != nil {
			log.Printf("[ERROR] Failed reading file '%s' from bucket %s: %s. Will try with alternative solution.", fullParsedPath, bucketName, err)

			if projectName != "shuffler" {
				bucketName = fmt.Sprintf("%s.appspot.com", projectName)
				bucket = project.StorageClient.Bucket(bucketName)
				obj = bucket.Object(fullParsedPath)
				fileReader, err = obj.NewReader(ctx)
				if err != nil {
					log.Printf("[ERROR] Failed reading file '%s' again from bucket %s: %s", fullParsedPath, bucketName, err)
					return "", err
				}
			} else {
				return "", err
			}
		}

		data, err = ioutil.ReadAll(fileReader)
		if err != nil {
			return "", err
		}
	}

	if project.CacheDb {
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating execution file value: %s", err)
		}
	}

	return string(data), nil
}

func SanitizeExecution(workflowExecution WorkflowExecution) WorkflowExecution {
	// New form REQUIRES sanitization no matter what
	//if workflowExecution.Workflow.Sharing != "form" {
	sanitizeLiquid := os.Getenv("LIQUID_SANITIZE_INPUT")
	if sanitizeLiquid == "" {
		sanitizeLiquid = "true" // Set default value to "true" if not set
	}

	if project.Environment == "cloud" || sanitizeLiquid != "true" {
		if sanitizeLiquid != "true" {
			log.Printf("[WARNING] Liquid sanitization is disabled. Skipping sanitization.")
		}

		return workflowExecution
	}

	workflowExecution.ExecutionArgument = sanitizeString(workflowExecution.ExecutionArgument)
	for i := range workflowExecution.Results {
		workflowExecution.Results[i].Result = sanitizeString(workflowExecution.Results[i].Result)
	}

	// Sanitize ExecutionVariables
	for i := range workflowExecution.ExecutionVariables {
		workflowExecution.ExecutionVariables[i].Value = sanitizeString(workflowExecution.ExecutionVariables[i].Value)
	}

	return workflowExecution
}

// Sanitizes Liquid formatting to ensure it can't run retroactively
func sanitizeString(input string) string {
	// Sanitize instances of {{...}}
	for strings.Contains(input, "{{") && strings.Contains(input, "}}") {
		startIndex := strings.Index(input, "{{")
		endIndex := strings.Index(input, "}}") + 2

		if startIndex >= 0 && endIndex > startIndex {
			input = input[:startIndex] + input[endIndex:]
		} else {
			break // Exit the loop if opening and closing tags don't exist for each other
		}
	}

	// Sanitize instances of {%...%}
	for strings.Contains(input, "{%") && strings.Contains(input, "%}") {
		startIndex := strings.Index(input, "{%")
		endIndex := strings.Index(input, "%}") + 2

		if startIndex >= 0 && endIndex > startIndex {
			input = input[:startIndex] + input[endIndex:]
		} else {
			break // Same here
		}
	}

	return input
}

func GetExecutionValidation(ctx context.Context, executionId string) (TypeValidation, error) {
	validation := TypeValidation{}

	cacheKey := fmt.Sprintf("validation_%s", executionId)
	validationData, err := GetCache(ctx, cacheKey)
	if err == nil {
		//log.Printf("\n\nFound cachekey for %#v\n\n", cacheKey)

		cacheData := []byte(validationData.([]uint8))
		err = json.Unmarshal(cacheData, &validation)
		if err != nil {
			log.Printf("[ERROR] Failed unmarshalling cache data for execution status (2): %s", err)
			return validation, err
		}
	} else {
		//log.Printf("\n\n Can't find cachekey for %#v\n\n", cacheKey)
	}

	return validation, nil
}

func Fixexecution(ctx context.Context, workflowExecution WorkflowExecution) (WorkflowExecution, bool) {
	dbsave := false
	workflowExecution.Workflow.Image = ""

	workflowExecution = cleanupProtectedKeys(workflowExecution)
	validation, err := GetExecutionValidation(ctx, workflowExecution.ExecutionId)
	if err == nil {
		if workflowExecution.NotificationsCreated > 0 {
			validation.NotificationsCreated = workflowExecution.NotificationsCreated
		}

		workflowExecution.Workflow.Validation = validation
	}

	// Make sure to not having missing items in the execution
	lastexecVar := map[string]ActionResult{}
	for actionIndex, action := range workflowExecution.Workflow.Actions {
		found := false
		result := ActionResult{}

		workflowExecution.Workflow.Actions[actionIndex].LargeImage = ""
		workflowExecution.Workflow.Actions[actionIndex].SmallImage = ""
		for resultIndex, innerresult := range workflowExecution.Results {
			if innerresult.Action.ID != action.ID {
				continue
			}

			// There was some WAITING issue here. This is a hotfix from agent issues.
			if innerresult.Status == "WAITING" && innerresult.Action.AppName == "Shuffle Tools" && innerresult.CompletedAt > 0 {
				workflowExecution.Results[resultIndex].Status = "SUCCESS"
			}

			// Forcing it to become agent
			if innerresult.Action.AppName == "AI Agent" || innerresult.Action.AppName == "Shuffle Agent" {
				workflowExecution.Type = "AGENT"
			}

			if innerresult.Status != "WAITING" && innerresult.Status != "SUCCESS" {
				found = true
				result = innerresult
				break

				//} else if innerresult.Status == "WAITING" || innerresult.Status == "SUCCESS" && (action.AppName == "AI Agent" || action.AppName == "Shuffle Agent") {
			} else if (innerresult.Status == "WAITING" || innerresult.Status == "SUCCESS") && (innerresult.Action.AppName == "AI Agent" || innerresult.Action.AppName == "Shuffle Agent") {
				if workflowExecution.Results[resultIndex].StartedAt == 0 {
					workflowExecution.Results[resultIndex].StartedAt = time.Now().UnixMilli()
				}

				// Somehow possible to get Nano()
				if workflowExecution.Results[resultIndex].StartedAt > 17769710273568 {
					workflowExecution.Results[resultIndex].StartedAt = time.Now().UnixMilli()
				}

				// Auto fixing decision data based on cache for better decisionmaking
				// Map the result into AgentOutput to check decisions
				decisionsUpdated := false

				mappedOutput := AgentOutput{}
				err = json.Unmarshal([]byte(innerresult.Result), &mappedOutput)
				if err != nil {
					log.Printf("[WARNING] Agent mapping: Failed in mapped output mapping: %s", err)
				} else {
					// Handles "stuck" cases
					if innerresult.Status == "WAITING" {
						decisionFailedCheck := ResultChecker{}
						err = json.Unmarshal([]byte(mappedOutput.DecisionString), &decisionFailedCheck)
						if err == nil && len(decisionFailedCheck.Reason) > 0 && decisionFailedCheck.Success == false {
							//if strings.Contains(decisionFailedCheck.Reason
							//mappedOutput.Status = "SKIPPED"
							mappedOutput.Status = "FINISHED"

							innerresult.Status = "SKIPPED"
							workflowExecution.Results[resultIndex].Status = "SKIPPED"
							decisionsUpdated = true
						}
					}
				}

				finishedDecisions := []string{}
				failedFound := false
				finishDecisionFound := false
				for decisionIndex, decision := range mappedOutput.Decisions {
					if decision.Action == "finish" {
						finishDecisionFound = true
					}

					decisionId := fmt.Sprintf("agent-%s-%s", workflowExecution.ExecutionId, decision.RunDetails.Id)
					if decision.RunDetails.Status == "FINISHED" || decision.RunDetails.Status == "IGNORED" {
						finishedDecisions = append(finishedDecisions, decision.RunDetails.Id)
						continue
					} else if decision.RunDetails.Status == "FAILURE" {
						//finishedDecisions = append(finishedDecisions, decision.RunDetails.Id)
						failedFound = true
						continue
					} else if decision.RunDetails.Status == "RUNNING" && decision.Action != "ask" {

						// Max runtime of a decision at 5 minutes
						if decision.RunDetails.StartedAt > 0 && time.Now().UnixMilli()-decision.RunDetails.StartedAt > 300000 {
							if debug { 
								log.Printf("[DEBUG] AI_AGENT_DECISION_TIMEOUT: execution_id=%s tool=%s action=%s duration=%ds", workflowExecution.ExecutionId, decision.Tool, decision.Action, time.Now().UnixMilli()-decision.RunDetails.StartedAt)
							}

							decisionsUpdated = true
							mappedOutput.Decisions[decisionIndex].RunDetails.Status = "FAILURE"
							mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = time.Now().UnixMilli()
							mappedOutput.Decisions[decisionIndex].RunDetails.RawResponse += "\n[ERROR] Decision marked as FAILURE due to 5 minute timeout."

							// Count this as finished + failed so recovery triggers in the same Fixexecution run
							// finishedDecisions = append(finishedDecisions, decision.RunDetails.Id)
							// failedFound = true
						}
					} else {
						if decision.RunDetails.CompletedAt > 0 {
							if debug {
								log.Printf("[DEBUG] Rewriting decision %s to FINISHED based on completed at timestamp.", decision.RunDetails.Id)
							}

							mappedOutput.Decisions[decisionIndex].RunDetails.Status = "FINISHED"
							finishedDecisions = append(finishedDecisions, decision.RunDetails.Id)
							decisionsUpdated = true

							marshalledDecision, err := json.Marshal(mappedOutput.Decisions[decisionIndex])
							if err == nil {
								err = SetCache(ctx, decisionId, marshalledDecision, 60)
							}
							continue
						} else {
							if decision.Action == "finish" && decision.RunDetails.Status == "" {
								mappedOutput.Decisions[decisionIndex].RunDetails.Status = "FINISHED"
								if mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt == 0 {
									mappedOutput.Decisions[decisionIndex].RunDetails.StartedAt = time.Now().UnixMilli()
								}

								finishedDecisions = append(finishedDecisions, decision.RunDetails.Id)
								mappedOutput.Decisions[decisionIndex].RunDetails.CompletedAt = time.Now().UnixMilli()
								decisionsUpdated = true

								marshalledDecision, err := json.Marshal(mappedOutput.Decisions[decisionIndex])
								if err == nil {
									err = SetCache(ctx, decisionId, marshalledDecision, 60)
								}
							}

							if debug {
								log.Printf("[DEBUG][%s] Decision %s for agent action %s is still RUNNING but no completed at timestamp. Checking cache for updates.", workflowExecution.ExecutionId, decision.RunDetails.Id, action.ID)
							}
						}
					}

					//log.Printf("[DEBUG] Check cache for %s with status %s", decision.RunDetails.Id, decision.RunDetails.Status)
					cache, err := GetCache(ctx, decisionId)
					if err == nil {
						foundDecision := AgentDecision{}
						cacheData := []byte(cache.([]uint8))
						err = json.Unmarshal(cacheData, &foundDecision)
						if err != nil {
							log.Printf("[ERROR][%s] Faled mapping foundDecision: %s", workflowExecution.ExecutionId, foundDecision.RunDetails.Id)
						} else {
							if foundDecision.RunDetails.Status != "" {
								decisionsUpdated = true
								mappedOutput.Decisions[decisionIndex] = foundDecision
							}
						}
					}
				}

				// FIXME: Is failure hadnling here necessary?
				// Changed it to do failure handling better in the agent itself
				// due to having a 'finish' action that should handle it properly
				if failedFound {
					decisionsUpdated = true
					//if debug {
					//	log.Printf("[DEBUG][%s] Failure found for agent %s. Should we exit?", workflowExecution.ExecutionId, action.ID)
					//}

					/*
						mappedOutput.Status = "FAILURE"
						mappedOutput.CompletedAt = time.Now().UnixMilli()
						workflowExecution.Results[resultIndex].Status = "ABORTED"

						go sendAgentActionSelfRequest("FAILURE", workflowExecution, workflowExecution.Results[resultIndex])
					*/

				}

				if len(finishedDecisions) == len(mappedOutput.Decisions) && mappedOutput.Status != "FINISHED" && mappedOutput.Status != "FAILURE" && mappedOutput.Status != "ABORTED" {

					// Check if requests was recently sent or not
					cacheId := fmt.Sprintf("agent-%s-%s-fixexec-finished-check", workflowExecution.ExecutionId, action.ID)
					if _, err := GetCache(ctx, cacheId); err == nil {
						// Recently sent, skip
						//log.Printf("[INFO][%s] Recently handled all decisions finished for agent action %s - skipping.", workflowExecution.ExecutionId, action.ID)
						continue
					}

					// Set cache to prevent multiple sends
					SetCache(ctx, cacheId, []byte("handled"), 1)

					decisionsUpdated = true
					if finishDecisionFound {
						log.Printf("[INFO][%s] All decisions finished for agent action %s - marking as FINISHED.", workflowExecution.ExecutionId, action.ID)

						mappedOutput.Status = "FINISHED"
						mappedOutput.CompletedAt = time.Now().UnixMilli()

						workflowExecution.Results[resultIndex].Status = "SUCCESS"

						go func() {
							time.Sleep(1 * time.Second)
							go sendAgentActionSelfRequest("SUCCESS", workflowExecution, workflowExecution.Results[resultIndex])
						}()
					} else {
						log.Printf("[INFO][%s] All decisions finished for agent action %s - but no finish action found, marking as WAITING.", workflowExecution.ExecutionId, action.ID)
						//log.Printf("[INFO][%s] All decisions finished for agent action %s - but no finish action found. Re-invoking agent to finalize (failedFound: %t).", workflowExecution.ExecutionId, action.ID, failedFound)

						mappedOutput.Status = "RUNNING"
						mappedOutput.CompletedAt = 0
						workflowExecution.Results[resultIndex].Status = "WAITING"

						if workflowExecution.Status == "FINISHED" {
							workflowExecution.Status = "EXECUTING"
						}
						// To ensure the execution is actually updated
						// Re-invoke the agent so the LLM can see the failure and produce a proper "finish" decision.

						// capturedExec := workflowExecution
						// capturedAction := action
						go func() {
							time.Sleep(1 * time.Second)
							sendAgentActionSelfRequest("WAITING", workflowExecution, workflowExecution.Results[resultIndex])
							// time.Sleep(2 * time.Second)
							// _, err := HandleAiAgentExecutionStart(capturedExec, capturedAction, true)
							// if err != nil {
							// 	log.Printf("[ERROR][%s] Failed re-invoking agent after decisions completed for action %s: %s", capturedExec.ExecutionId, capturedAction.ID, err)
							// }
						}()
					}
				} else if (result.Status == "" || result.Status == "WAITING") && mappedOutput.Status == "FINISHED" {
					workflowExecution.Results[resultIndex].Status = "SUCCESS"
					go sendAgentActionSelfRequest("SUCCESS", workflowExecution, workflowExecution.Results[resultIndex])
				}

				if decisionsUpdated {
					marshalledResult, err := json.Marshal(mappedOutput)
					if err == nil {
						workflowExecution.Results[resultIndex].Result = string(marshalledResult)
					} else {
						log.Printf("[DEBUG] Failed unmarshalling agent decision: %s", err)
					}
				}
			}
		}

		if found {
			// Handles execution vars
			result.Action = action
			if setExecutionVariable(result) {

				// Check if key in lastexecVar
				if _, ok := lastexecVar[result.Action.ExecutionVariable.Name]; ok {

					if lastexecVar[result.Action.ExecutionVariable.Name].CompletedAt > result.CompletedAt {
						lastexecVar[result.Action.ExecutionVariable.Name] = result
					}
				} else {
					lastexecVar[result.Action.ExecutionVariable.Name] = result
				}
			}

			continue
		}

		cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, action.ID)
		cache, err := GetCache(ctx, cacheId)
		if err != nil {
			//log.Printf("[WARNING] Couldn't find in fix exec %s (2): %s", cacheId, err)
			continue
		}

		cacheData := []byte(cache.([]uint8))

		// Just ensuring the data is good
		err = json.Unmarshal(cacheData, &result)
		if err == nil {
			workflowExecution.Results = append(workflowExecution.Results, result)
			result.Action = action
			if setExecutionVariable(result) {

				// Check if key in lastexecVar
				if _, ok := lastexecVar[result.Action.ExecutionVariable.Name]; ok {

					if lastexecVar[result.Action.ExecutionVariable.Name].CompletedAt > result.CompletedAt {
						lastexecVar[result.Action.ExecutionVariable.Name] = result
					}
				} else {
					lastexecVar[result.Action.ExecutionVariable.Name] = result
				}
			}

		} else {
			log.Printf("[ERROR] Failed unmarshalling in fix exec for ID %s (1): %s", cacheId, err)
		}
	}

	// Don't forget any!!
	extra := 0
	for triggerIndex, trigger := range workflowExecution.Workflow.Triggers {
		if trigger.TriggerType != "SUBFLOW" && trigger.TriggerType != "USERINPUT" {
			continue
		}

		workflowExecution.Workflow.Triggers[triggerIndex].LargeImage = ""
		workflowExecution.Workflow.Triggers[triggerIndex].SmallImage = ""

		workflowExecution.Workflow.Triggers[triggerIndex] = trigger

		extra += 1

		found := false
		for _, result := range workflowExecution.Results {
			if result.Action.ID == trigger.ID {
				found = true
				break
			}
		}

		if found {
			continue
		}

		cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, trigger.ID)
		cache, err := GetCache(ctx, cacheId)
		if err != nil {
			//log.Printf("[WARNING] Couldn't find in fix exec %s (2): %s", cacheId, err)
			continue
		}

		actionResult := ActionResult{}
		cacheData := []byte(cache.([]uint8))

		// Just ensuring the data is good
		err = json.Unmarshal(cacheData, &actionResult)
		if err == nil {
			workflowExecution.Results = append(workflowExecution.Results, actionResult)
		} else {
			log.Printf("[ERROR] Failed unmarshalling in fix exec for ID %s (2): %s", cacheId, err)
		}
	}

	// Deduplicat the results
	handled := []string{}
	newResults := []ActionResult{}
	for _, result := range workflowExecution.Results {
		if result.Action.ID == "" && result.Action.Name == "" && result.Result == "" {
			//log.Printf("[WARNING][%s] Removing empty result started at '%d' and finished at '%d'. ID: %#v, Name: %#v.", workflowExecution.ExecutionId, result.StartedAt, result.CompletedAt, result.Action.ID, result.Action.Name)
			continue
		}

		if ArrayContains(handled, result.Action.ID) {
			continue
		}

		// Checking if results are correct or not
		if project.Environment != "worker" {
			if result.Status != "WAITING" && result.Status != "SKIPPED" && (result.Action.AppName == "User Input" || result.Action.AppName == "Shuffle Workflow" || result.Action.AppName == "shuffle-subflow") {
				tmpResult, _ := parseSubflowResults(ctx, result)

				if result.Status == "SUCCESS" {
					result.Result = tmpResult.Result
				}
			}

			// Checks for subflows in waiting status
			// May also work for user input in the future
			if result.Status == "WAITING" {
				tmpResult, changed := parseSubflowResults(ctx, result)
				//log.Printf("HANDLE HERE: %s", tmpResult.Status)

				if changed && (tmpResult.Status == "SUCCESS" || tmpResult.Status == "FAILURE") {
					// Making sure we don't infinite loop :)
					// Keeping for 1 minute, as that's the rerun period
					cacheKey := fmt.Sprintf("%s_%s_sent", workflowExecution.ExecutionId, tmpResult.Action.ID)
					cache, err := GetCache(ctx, cacheKey)
					if err == nil && cache != nil {
						//SetCache(ctx, cacheKey, []byte("1"), 1)

						result = tmpResult
					} else {
						SetCache(ctx, cacheKey, []byte("1"), 1)

						log.Printf("[DEBUG][%s] Found waiting result for %s, now with status %s. Sending request to self for the full response of it", workflowExecution.ExecutionId, result.Action.ID, tmpResult.Status)

						// Forcing a resend to handle transaction normally
						actionData, err := json.Marshal(tmpResult)
						if err == nil {
							ResendActionResult(actionData, 4)
						} else {
							//result = tmpResult
						}
					}

				} else {
					//result = tmpResult
				}
			}
		}

		handled = append(handled, result.Action.ID)
		newResults = append(newResults, result)

	}

	workflowExecution.Results = newResults

	// Sort results based on CompletedAt
	sort.Slice(workflowExecution.Results, func(i, j int) bool {
		return workflowExecution.Results[i].CompletedAt < workflowExecution.Results[j].CompletedAt
	})

	for varKey, variable := range workflowExecution.Workflow.ExecutionVariables {
		for key, value := range lastexecVar {
			if key != variable.Name {
				continue
			}

			if workflowExecution.Workflow.ExecutionVariables[varKey].Value != value.Result {
				//log.Printf("\n\n\n[DEBUG][%s] Updating execution variable '%s' from len %d to %d (%s)\n\n", workflowExecution.ExecutionId, variable.Name, len(workflowExecution.Workflow.ExecutionVariables[varKey].Value), len(value.Result), value.Action.Label)
			}

			workflowExecution.Workflow.ExecutionVariables[varKey].Value = value.Result
			break
		}
	}

	workflowExecution.ExecutionVariables = workflowExecution.Workflow.ExecutionVariables

	// Check for failures before setting to finished
	// Update execution parent
	if workflowExecution.Status == "EXECUTING" {

		for _, result := range workflowExecution.Results {
			if result.Status == "FAILURE" || result.Status == "ABORTED" {
				// Only log once per execution to avoid spam
				cacheKey := fmt.Sprintf("abort_log_%s", workflowExecution.ExecutionId)
				if _, err := GetCache(ctx, cacheKey); err != nil {
					log.Printf("[DEBUG][%s] Setting execution to aborted because of result %s (%s) with status '%s'. Should update execution parent if it exists (not implemented).", workflowExecution.ExecutionId, result.Action.Name, result.Action.ID, result.Status)
					SetCache(ctx, cacheKey, []byte("logged"), 5) // 5 minute TTL
				}

				workflowExecution.Status = "ABORTED"
				dbsave = true
				if workflowExecution.CompletedAt == 0 {
					workflowExecution.CompletedAt = time.Now().Unix()
				}

				break
			}
		}
	}

	// Check if finished too?
	finalWorkflowExecution := SanitizeExecution(workflowExecution)
	if (workflowExecution.Status == "WAITING" || workflowExecution.Status == "EXECUTING") && len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra {
		skipFinished := false
		for _, result := range workflowExecution.Results {
			if result.Status == "WAITING" {
				skipFinished = true
				break
			}
		}

		// Has to do with rerun systems from April 2025
		for _, action := range workflowExecution.Workflow.Actions {
			if action.Category == "rerun" {
				skipFinished = true
				break
			}
		}

		if !skipFinished {
			// FIXME: Is this subflow result (not implemented) valid? I think it should have been added? Hmm.
			//log.Printf("[DEBUG][%s] Setting execution to finished because all results are in and it was still in EXECUTING mode. Should set subflow parent result as well (not implemented) - just returning for now for parent function to handle.", workflowExecution.ExecutionId)
			finalWorkflowExecution.Status = "FINISHED"
			dbsave = true
			if finalWorkflowExecution.CompletedAt == 0 {
				finalWorkflowExecution.CompletedAt = time.Now().Unix()
			}
		}
	}

	// Cleaning up values as they shouldn't exist anymore in actions
	// after a result has been found for it.
	for resIndex, result := range finalWorkflowExecution.Results {
		if result.Status != "FINISHED" && result.Status != "SUCCESS" && result.Status != "ABORTED" {
			continue
		}

		cleaned := false
		for paramIndex, param := range result.Action.Parameters {
			if param.Configuration {
				finalWorkflowExecution.Results[resIndex].Action.Parameters[paramIndex].Value = ""
			}

			finalWorkflowExecution.Results[resIndex].Action.Parameters[paramIndex].Example = ""
			finalWorkflowExecution.Results[resIndex].Action.Parameters[paramIndex].Description = ""
		}

		if cleaned {
			for actionIndex, action := range finalWorkflowExecution.Workflow.Actions {
				if action.ID != result.Action.ID {
					continue
				}

				for paramIndex, param := range action.Parameters {
					if param.Configuration {
						finalWorkflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Value = ""
					}

					finalWorkflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Example = ""
					finalWorkflowExecution.Workflow.Actions[actionIndex].Parameters[paramIndex].Description = ""
				}
			}
		}
	}

	// Update WorkflowExecution.Result to be correct, as to return correct for:
	// - Subflows with wait for response
	// - Webhooks v2 with for response
	if finalWorkflowExecution.Status == "ABORTED" {
		finalWorkflowExecution.Result = finalWorkflowExecution.Workflow.DefaultReturnValue
	} else if (len(finalWorkflowExecution.Result) == 0 || finalWorkflowExecution.Result == finalWorkflowExecution.Workflow.DefaultReturnValue) && finalWorkflowExecution.Status == "FINISHED" {
		lastResult := ""
		lastCompleted := int64(-1)
		for _, result := range finalWorkflowExecution.Results {
			if result.Status == "SUCCESS" && result.CompletedAt > lastCompleted {
				lastResult = result.Result
				lastCompleted = result.CompletedAt
			}
		}

		if len(lastResult) > 0 {
			finalWorkflowExecution.Result = lastResult
		} else {
			if len(finalWorkflowExecution.Result) == 0 && len(finalWorkflowExecution.Workflow.DefaultReturnValue) > 0 {
				finalWorkflowExecution.Result = finalWorkflowExecution.Workflow.DefaultReturnValue
			}
		}
	}

	return finalWorkflowExecution, dbsave
}

func GetWorkflowExecutionByAuth(ctx context.Context, authId string) (*WorkflowExecution, error) {
	nameKey := "workflowexecution"
	cacheKey := fmt.Sprintf("%s_auth_%s", nameKey, authId)

	workflowExecution := &WorkflowExecution{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &workflowExecution)
			if err == nil || len(workflowExecution.ExecutionId) > 0 {
				return workflowExecution, nil
			}
		}
	}

	if project.DbType == "opensearch" {
		return workflowExecution, errors.New("Not implemented")
	} else {
		// Google datastore search based on "authorization ="
		allExecutions := []*WorkflowExecution{}
		q := datastore.NewQuery(nameKey).Filter("authorization =", authId).Limit(1)
		_, err := project.Dbclient.GetAll(ctx, q, &allExecutions)
		if err != nil {
			log.Printf("[WARNING] Failed getting workflow execution by auth: %s", err)
			if strings.Contains(err.Error(), `cannot load field`) {
				err = nil
			} else {
				return nil, err
			}
		} else {
			if len(allExecutions) > 0 {
				workflowExecution = allExecutions[0]
			}
		}
	}

	if project.CacheDb {
		//log.Printf("[DEBUG] Caching workflow execution %s", cacheKey)
		workflowExecutionJson, err := json.Marshal(workflowExecution)
		if err == nil {
			err := SetCache(ctx, cacheKey, workflowExecutionJson, 10)
			if err != nil {
				log.Printf("[WARNING] Failed caching workflow execution %s: %s", cacheKey, err)
			}
		}
	}

	return workflowExecution, nil
}

func getCloudFileApp(ctx context.Context, workflowApp WorkflowApp, id string) (WorkflowApp, error) {
	if len(workflowApp.Name) == 0 {
		return workflowApp, nil
	}
	//project.BucketName := project.BucketName

	if strings.HasSuffix(id, ".") {
		id = id[:len(id)-1]
	}

	fullParsedPath := fmt.Sprintf("extra_specs/%s/appspec.json", id)
	//log.Printf("[DEBUG] Couldn't find working app for app with ID %s. Checking filepath gs://%s/%s (size too big)", id, project.BucketName, fullParsedPath)
	//gs://shuffler.appspot.com/extra_specs/0373ed696a3a2cba0a2b6838068f2b80

	cacheKey := fmt.Sprintf("cloud_file_app_%s", id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &workflowApp)
			if err == nil {
				return workflowApp, nil
			}
		}
	}

	client, err := storage.NewClient(ctx)
	if err != nil {
		log.Printf("[WARNING] Failed to create client (storage - algolia img): %s", err)
		return workflowApp, err
	}

	bucket := client.Bucket(project.BucketName)
	obj := bucket.Object(fullParsedPath)
	fileReader, err := obj.NewReader(ctx)
	if err != nil {
		// Set cache anyway
		if project.CacheDb {
			data, err := json.Marshal(workflowApp)
			if err != nil {
				log.Printf("[WARNING] Failed marshalling app: %s", err)
				return workflowApp, nil
			}

			err = SetCache(ctx, cacheKey, data, 30)
			if err != nil {
				log.Printf("[WARNING] Failed updating app: %s", err)
			}
		}

		//log.Printf("[ERROR] Failed making App reader for %s: %s", fullParsedPath, err)
		return workflowApp, err
	}

	data, err := ioutil.ReadAll(fileReader)
	if err != nil {
		log.Printf("[WARNING] Failed reading from filereader: %s", err)
		return workflowApp, err
	}

	err = json.Unmarshal(data, &workflowApp)
	if err != nil {
		log.Printf("[WARNING] Failed unmarshaling from remote store: %s", err)
		return workflowApp, err
	}

	//log.Printf("[DEBUG] Got new file data for app with ID %s from filepath gs://%s/%s with %d actions", id, project.BucketName, fullParsedPath, len(workflowApp.Actions))
	if project.CacheDb {
		data, err := json.Marshal(workflowApp)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in get cloud app cache: %s", err)
			return workflowApp, nil
		}

		err = SetCache(ctx, cacheKey, data, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for get cloud app cache key '%s': %s", cacheKey, err)
		}
	}

	defer fileReader.Close()
	return workflowApp, nil
}

func GetApp(ctx context.Context, id string, user User, skipCache bool) (*WorkflowApp, error) {
	workflowApp := &WorkflowApp{}
	if len(id) == 0 {
		return workflowApp, errors.New("No ID provided to get an app")
	}

	if id == "integration" {
		return workflowApp, errors.New("App ID 'integration' is for Singul. Uses the Shuffle-AI app. This error is from GetApp(integration) which does not work. Contact support@shuffler.io if this persists.")
	}

	nameKey := "workflowapp"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)

	if !skipCache && project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)

		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, workflowApp)
			if err == nil {

				// Grabbing extra files necessary
				if (len(workflowApp.ID) == 0 || len(workflowApp.Actions) == 0) && project.Environment == "cloud" {
					tmpApp, err := getCloudFileApp(ctx, *workflowApp, id)

					if err == nil {
						log.Printf("[DEBUG] Got app '%s' (%s) with %d actions from file (cache)", workflowApp.Name, workflowApp.ID, len(tmpApp.Actions))
						workflowApp = &tmpApp
						return workflowApp, nil
					} else {
						//log.Printf("[DEBUG] Failed remote loading app '%s' (%s) from file (cache): %s", workflowApp.Name, workflowApp.ID, err)
					}
				} else {
					return workflowApp, nil
				}
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for org: %s", err)
		}
	} else {
		//log.Printf("[DEBUG] Skipping cache check in get app for ID %s", id)
	}

	if project.DbType == "opensearch" {
		indexAlias := strings.ToLower(GetESIndexPrefix(nameKey))
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      indexAlias,
			DocumentID: id,
		})
		if err != nil {
			if strings.Contains(err.Error(), "has more than one index associated with it") {
				var buf bytes.Buffer
				query := map[string]interface{}{
					"size": 1,
					"query": map[string]interface{}{
						"ids": map[string]interface{}{
							"values": []string{id},
						},
					},
					"sort": []map[string]interface{}{
						{
							"edited": map[string]interface{}{
								"order":         "desc",
								"unmapped_type": "long",
							},
						},
						{
							"created": map[string]interface{}{
								"order":         "desc",
								"unmapped_type": "long",
							},
						},
					},
				}

				if err := json.NewEncoder(&buf).Encode(query); err != nil {
					return workflowApp, err
				}

				searchResp, serr := project.Es.Search(ctx, &opensearchapi.SearchReq{
					Indices: []string{indexAlias},
					Body:    &buf,
				})
				if serr != nil {
					return workflowApp, serr
				}

				searchRes := searchResp.Inspect().Response
				defer searchRes.Body.Close()
				searchBody, serr := ioutil.ReadAll(searchRes.Body)
				if serr != nil {
					return workflowApp, serr
				}

				if searchRes.StatusCode != 200 && searchRes.StatusCode != 201 {
					return workflowApp, errors.New(fmt.Sprintf("Bad statuscode: %d, error: %s", searchRes.StatusCode, string(searchBody)))
				}

				wrappedSearch := AppSearchWrapper{}
				if serr := json.Unmarshal(searchBody, &wrappedSearch); serr != nil {
					return workflowApp, serr
				}

				if len(wrappedSearch.Hits.Hits) == 0 {
					return workflowApp, errors.New("App doesn't exist")
				}

				workflowApp = &wrappedSearch.Hits.Hits[0].Source
			} else {
				log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
				return workflowApp, err
			}
		} else {
			res := resp.Inspect().Response
			defer res.Body.Close()
			if res.StatusCode == 404 {
				return workflowApp, errors.New("App doesn't exist")
			}

			respBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return workflowApp, err
			}

			if res.StatusCode != 200 && res.StatusCode != 201 {
				return workflowApp, errors.New(fmt.Sprintf("Bad statuscode: %d, error: %s", res.StatusCode, string(respBody)))
			}

			wrapped := AppWrapper{}
			err = json.Unmarshal(respBody, &wrapped)
			if err != nil {
				return workflowApp, err
			}

			workflowApp = &wrapped.Source
		}
	} else {
		//log.Printf("[DEBUG] Getting app from datastore for ID %s", id)

		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		err := project.Dbclient.Get(ctx, key, workflowApp)

		//log.Printf("\n\n[DEBUG] Actions in %s (%s): %d. Err: %s", workflowApp.Name, strings.ToLower(id), len(workflowApp.Actions), err)

		if err != nil || len(workflowApp.Actions) == 0 {
			if strings.Contains(fmt.Sprintf("%s", err), "no such entity") {
				return workflowApp, errors.New("App doesn't exist")
			}

			//log.Printf("[WARNING] Failed getting app in GetApp with name %s and ID %s. Actions: %d. Getting if EITHER is bad or 0. Err: %s", workflowApp.Name, id, len(workflowApp.Actions), err)
			for _, app := range user.PrivateApps {
				if app.ID == id {
					workflowApp = &app
					break
				}
			}

			// Exists in case of "too large" issues.
			if (len(workflowApp.ID) == 0 || len(workflowApp.Actions) == 0) && project.Environment == "cloud" {
				tmpApp, err := getCloudFileApp(ctx, *workflowApp, id)

				if err == nil {
					//log.Printf("[DEBUG] Got app %s (%s) with %d actions from file", workflowApp.Name, workflowApp.ID, len(tmpApp.Actions))
					workflowApp = &tmpApp
				} else {
					//log.Printf("[DEBUG] Failed remote loading app  %s (%s) from file: %s", workflowApp.Name, workflowApp.ID, err)
				}

			} else {
				//log.Printf("[DEBUG] Returning %s (%s) normally", workflowApp.Name, id)
			}
		}
	}
	if project.CacheDb {
		data, err := json.Marshal(workflowApp)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getapp: %s", err)
			return workflowApp, nil
		}

		err = SetCache(ctx, cacheKey, data, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getapp key '%s': %s", cacheKey, err)
		}
	}

	if workflowApp.ID == "" {
		return workflowApp, errors.New(fmt.Sprintf("Couldn't find app %s", id))
	}

	return workflowApp, nil
}

func SetSubscriptionRecipient(ctx context.Context, sub SubscriptionRecipient, id string) error {
	nameKey := "gmail_subscription"
	sub.Edited = int(time.Now().Unix())

	// New struct, to not add body, author etc
	data, err := json.Marshal(sub)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in setGmailSub: %s", err)
		return nil
	}
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, id, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &sub); err != nil {
			log.Printf("\n\n[WARNING] Error adding gmail sub: %s\n\n", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for setworkflow key '%s': %s", cacheKey, err)
		}
	}

	return nil
}

func GetSubscriptionRecipient(ctx context.Context, id string) (*SubscriptionRecipient, error) {
	sub := &SubscriptionRecipient{}
	nameKey := "gmail_subscription"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &sub)
			if err == nil {
				return sub, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for sub: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return sub, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return sub, errors.New("HistoryId doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return sub, err
		}

		wrapped := SubWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return sub, err
		}

		sub = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		if err := project.Dbclient.Get(ctx, key, sub); err != nil {
			return &SubscriptionRecipient{}, err
			//if strings.Contains(err.Error(), `cannot load field`) {
			//	log.Printf("[INFO] Error in sub loading. Migrating sub to new sub handler.")
			//	err = nil
			//} else {
			//	return &SubscriptionRecipient{}, err
			//}
		}
	}

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for sub %s", cacheKey)
		data, err := json.Marshal(sub)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getsub: %s", err)
			return sub, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getsub key '%s': %s", cacheKey, err)
		}
	}

	return sub, nil
}

// No deduplication for popular files
func FindSimilarFilename(ctx context.Context, filename, orgId string) ([]File, error) {
	//log.Printf("\n\n[DEBUG] Getting query %s for orgId %s\n\n", id, orgId)
	files := []File{}
	nameKey := "Files"

	cacheKey := fmt.Sprintf("%s_%s_%s", nameKey, orgId, filename)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &files)
			if err == nil {
				return files, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for file: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer

		// Or search?
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"filename": filename,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return files, nil
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return files, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (find file filename): %s", err)
			return files, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return files, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return files, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return files, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return files, err
		}

		wrapped := FileSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return files, err
		}

		if len(wrapped.Hits.Hits) == 1 && len(orgId) == 0 && wrapped.Hits.Hits[0].Source.Status == "active" && wrapped.Hits.Hits[0].Source.Md5sum == filename {
			files = append(files, wrapped.Hits.Hits[0].Source)
		} else {
			//file = []Environment{}
			for _, hit := range wrapped.Hits.Hits {
				if hit.Source.Md5sum != filename {
					continue
				}

				if hit.Source.OrgId == orgId && hit.Source.Status == "active" {
					files = append(files, hit.Source)
				}

			}
		}
	} else {
		query := datastore.NewQuery(nameKey).Filter("filename =", filename).Limit(25)
		_, err := project.Dbclient.GetAll(ctx, query, &files)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting deals for org: %s", orgId)
				return files, err
			}
		} else {
			//log.Printf("[INFO] Got %d files for filename: %s", len(files), filename)
			parsedFiles := []File{}
			for _, newfile := range files {
				if newfile.OrgId == orgId && newfile.Status == "active" {
					parsedFiles = append(parsedFiles, newfile)
				}
			}

			//log.Printf("[INFO] Got %d PARSD files for filename: %s", len(parsedFiles), md5)

			if len(parsedFiles) == 0 {
				return parsedFiles, errors.New(fmt.Sprintf("No file found for filename: %s", filename))
				//log.Printf("[INFO] Couldn't find file with md5 %s for org %s", md5, orgId)
			}

			files = parsedFiles
		}
	}

	//log.Printf("[DEBUG] Got hit: %s", file)

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for workflow %s", cacheKey)
		data, err := json.Marshal(files)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in find file md5 : %s", err)
			return files, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for find file md5 %s: %s", cacheKey, err)
		}
	}

	return files, nil
}

// Check OrgId later
// No deduplication for popular files
func FindSimilarFile(ctx context.Context, md5, orgId string) ([]File, error) {
	//log.Printf("\n\n[DEBUG] Getting query %s for orgId %s\n\n", id, orgId)
	files := []File{}
	nameKey := "Files"

	cacheKey := fmt.Sprintf("%s_%s_%s", nameKey, orgId, md5)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &files)
			if err == nil || len(files) > 0 {
				return files, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for file: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer

		// Or search?
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"md5_sum": md5,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return files, nil
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return files, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (find file md5): %s", err)
			return files, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return files, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return files, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return files, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return files, err
		}

		wrapped := FileSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return files, err
		}

		if len(wrapped.Hits.Hits) == 1 && len(orgId) == 0 && wrapped.Hits.Hits[0].Source.Status == "active" && wrapped.Hits.Hits[0].Source.Md5sum == md5 {
			files = append(files, wrapped.Hits.Hits[0].Source)
		} else {
			//file = []Environment{}
			for _, hit := range wrapped.Hits.Hits {
				if hit.Source.Md5sum != md5 {
					continue
				}

				if hit.Source.OrgId == orgId && hit.Source.Status == "active" {
					files = append(files, hit.Source)
				}

			}
		}
	} else {
		query := datastore.NewQuery(nameKey).Filter("md5_sum =", md5).Limit(250)
		_, err := project.Dbclient.GetAll(ctx, query, &files)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting deals for org: %s", orgId)
				//return files, err
			}
		} else {
			//log.Printf("[INFO] Got %d files for md5: %s", len(files), md5)
			parsedFiles := []File{}
			for _, newfile := range files {
				if newfile.OrgId == orgId && newfile.Status == "active" {
					parsedFiles = append(parsedFiles, newfile)
				}
			}

			if len(parsedFiles) == 0 {
				return parsedFiles, errors.New(fmt.Sprintf("No file found for md5: %s", md5))
				//log.Printf("[INFO] Couldn't find file with md5 %s for org %s", md5, orgId)
			}

			files = parsedFiles
		}
	}

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for workflow %s", cacheKey)
		data, err := json.Marshal(files)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in find file md5 : %s", err)
			return files, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for find file md5 %s: %s", cacheKey, err)
		}
	}

	return files, nil
}

func GetEnvironment(ctx context.Context, id, orgId string) (*Environment, error) {
	//log.Printf("\n\n[DEBUG] Getting query %s for orgId %s\n\n", id, orgId)
	env := &Environment{}
	nameKey := "Environments"

	cacheKey := fmt.Sprintf("%s_%s_%s", nameKey, orgId, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &env)
			if err == nil {
				return env, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for env: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer

		// "should" -> "must"?
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"should": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"Name": id,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"id": id,
							},
						},
					},
				},
			},
			"sort": map[string]interface{}{
				"created": map[string]interface{}{
					"order": "desc",
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return env, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return env, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get environment): %s", err)
			return env, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return env, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return env, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return env, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return env, err
		}

		wrapped := EnvironmentSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return env, err
		}

		//log.Printf("[DEBUG] Got %d environments for id: %s", len(wrapped.Hits.Hits), id)

		if len(wrapped.Hits.Hits) == 1 && len(orgId) == 0 {
			env = &wrapped.Hits.Hits[0].Source
		} else {
			//environments = []Environment{}
			for _, hit := range wrapped.Hits.Hits {
				//log.Printf("[DEBUG] Hit: %s", hit)
				//if hit.ID == id {
				//	env = &hit.Source
				//	break
				//}
				if hit.Source.OrgId == orgId {
					env = &hit.Source
					break
				}

				//environments = append(environments, hit.Source)
			}

			//if len(environments) != 1 {
			//	return env, errors.New(fmt.Sprintf("Found %d environments. Want 1 only.", len(environments)))
			//}
		}
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		if err := project.Dbclient.Get(ctx, key, env); err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[INFO] Error in environment loading of %s", id)
				err = nil
			} else {
				return env, err
			}
		}
	}

	//log.Printf("[DEBUG] Got hit: %s", env)

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for workflow %s", cacheKey)
		data, err := json.Marshal(env)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getenv: %s", err)
			return env, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getenv '%s: %s", cacheKey, err)
		}
	}

	return env, nil
}

func GetWorkflowRunCount(ctx context.Context, id string, start int64, end int64) (int, error) {
	var err error
	nameKey := "workflowexecution"
	cacheKey := fmt.Sprintf("%s_count_%s_%s_%s", nameKey, id, strconv.FormatInt(start, 10), strconv.FormatInt(end, 10))

	count := 0

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			count, err = strconv.Atoi(string(cacheData))
			if err == nil {
				//log.Printf("[DEBUG] Got count %d from cache for workflow id %s", count, id)
				return count, nil
			}
		}
		//log.Printf("[DEBUG] Failed getting count cache for workflow id %s: %s", id, err)
	}

	if project.DbType == "opensearch" {
		// count WorkflowExecution where workflowId = id
		query := map[string]interface{}{
			"size": 0,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"workflow_id": id,
							},
						},
						map[string]interface{}{
							"range": map[string]interface{}{
								"started_at": map[string]interface{}{
									"gte": start,
									"lte": end,
								},
							},
						},
					},
				},
			},
		}

		var buf bytes.Buffer
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding get workflow run count query: %s", err)
			return 0, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return 0, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflow run count): %s", err)
			return 0, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return 0, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return 0, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return 0, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return 0, err
		}

		wrapped := ExecutionSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return 0, err
		}

		count = wrapped.Hits.Total.Value
	} else {
		// count WorkflowExecution where workflowId = id
		//query := datastore.NewQuery(nameKey).Filter("workflow_id =", strings.ToLower(id))

		query := datastore.NewQuery(nameKey).Filter("workflow_id =", strings.ToLower(id)).Filter("started_at >=", start).Filter("started_at <=", end)
		count, err = project.Dbclient.Count(ctx, query)
		if err != nil {
			log.Printf("[WARNING] Failed getting count for workflow %s : %s", id, err)
			return 0, err
		}
	}

	// count int to []byte
	countStr := strconv.Itoa(count)
	countBytes := []byte(countStr)
	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache count for workflow id %s count: %s", id, countStr)

		err := SetCache(ctx, cacheKey, countBytes, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for workflow id %s count: %s", id, err)
		}
	}

	return count, nil
}

// Doesn't get ALL anymore. Max 100 by default (cloud)
func GetAllChildOrgs(ctx context.Context, orgId string, cursorInput ...string) ([]Org, string, error) {
	cursor := ""
	if len(cursorInput) > 0 {
		cursor = cursorInput[0]
	}

	orgs := []Org{}
	nameKey := "Organizations"

	cacheKey := fmt.Sprintf("%s_%s_childorgs", orgId, cursor)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &orgs)
			//if err == nil && len(orgs) > 0 {
			if err == nil {
				return orgs, cursor, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for workflow (7): %s", err)
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"creator_org": orgId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return orgs, cursor, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return orgs, cursor, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (Get workflows 2): %s", err)
			return orgs, cursor, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return orgs, cursor, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return orgs, cursor, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return orgs, cursor, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return orgs, cursor, err
		}

		wrapped := OrgSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return orgs, cursor, err
		}

		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.CreatorOrg != orgId {
				continue
			}

			orgs = append(orgs, hit.Source)
		}
	} else {
		// Cloud database
		//log.Printf("Pre running creator org search for %s", orgId)
		//_, err := project.Dbclient.GetAll(ctx, query, &orgs)
		//if err != nil {
		//	if !strings.Contains(err.Error(), `cannot load field`) {
		//	}
		//}

		maxAmount := 100
		query := datastore.NewQuery(nameKey).Filter("creator_org =", orgId).Limit(100)

		if cursor != "" {
			outputcursor, err := datastore.DecodeCursor(cursor)
			if err != nil {
				log.Printf("[ERROR] Error decoding cursor in creator org load: %s", err)
				//return orgs, "", err
			}

			query = query.Start(outputcursor)
		}

		iterCount := 0
		//cursorStr := ""
		var err error
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerOrg := Org{}
				_, err = it.Next(&innerOrg)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
					} else {
						//log.Printf("[WARNING] Workflow iterator issue: %s", err)
						break
					}
				}

				if debug {
					log.Printf("[DEBUG] SUBORG LOADER: %d", len(orgs))
				}

				iterCount++
				orgs = append(orgs, innerOrg)
				if iterCount >= maxAmount {
					break
				}
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor (childorg): %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursor == nextStr {
					break
				}

				cursor = nextStr
				query = query.Start(nextCursor)
			}

			if iterCount >= maxAmount {
				cursor = fmt.Sprintf("%s", nextCursor)
				break
			}
		}
	}

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for workflow %s", cacheKey)
		data, err := json.Marshal(orgs)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getchildorgs: %s", err)
			return orgs, cursor, nil
		}

		err = SetCache(ctx, cacheKey, data, 10)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getworkflow '%s': %s", cacheKey, err)
		}
	}

	return orgs, cursor, nil
}

func GetWorkflow(ctx context.Context, id string, skipHealth ...bool) (*Workflow, error) {
	workflow := &Workflow{}
	nameKey := "workflow"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, workflow)
			if err == nil && workflow.ID != "" {
				validationData, err := GetCache(ctx, fmt.Sprintf("validation_workflow_%s", workflow.ID))
				if err == nil {
					cacheData := []byte(validationData.([]uint8))
					err = json.Unmarshal(cacheData, &workflow.Validation)
					if err != nil {
						log.Printf("[ERROR] Failed unmarshalling cache data for execution status (4): %s", err)
					}
				}

				// Somehow this can happen. Reverting to LATEST revision
				if len(workflow.Actions) > 0 && len(workflow.Triggers) == 0 {
					revisions, err := ListWorkflowRevisions(ctx, workflow.ID, 2)
					if err != nil {
						log.Printf("[WARNING] Failed getting revisions during trigger load for workflow %s: %s", workflow.ID, err)
					} else {
						if len(revisions) > 0 {
							for _, revision := range revisions {
								if revision.ID != workflow.ID {
									continue
								}

								if len(revision.Triggers) > 0 {
									workflow.Triggers = revision.Triggers
									break
								}
							}

							//log.Printf("[INFO] Reverting to revision triggers for workflow %s from 0 triggers to %d triggers", workflow.ID, len(revisions[0].Triggers))
							workflow.Triggers = revisions[0].Triggers
						}
					}
				}

				if len(skipHealth) == 0 || (len(skipHealth) > 0 && !skipHealth[0]) {
					healthWorkflow, _, err := GetStaticWorkflowHealth(ctx, *workflow)
					if err != nil {
						if !strings.Contains(err.Error(), "Org ID not set") {
							log.Printf("[ERROR] Failed getting static workflow health for workflow %s: %s (2)", workflow.ID, err)
						}

					} else {
						workflow = &healthWorkflow
					}
				}

				if len(workflow.Actions) > 1 || len(workflow.Triggers) > 0 {
					return workflow, nil
				}
			}
		} else {
			if debug {
				//log.Printf("[DEBUG] Failed getting cache for workflow (2): %s", err)
			}
		}
	}

	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			if strings.Contains(err.Error(), "has more than one index associated with it") {
				fallbackWorkflow, fallbackErr := getWorkflowByAliasSearch(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), id)
				if fallbackErr != nil {
					log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
					log.Printf("[WARNING] Workflow alias fallback failed for %s: %s", cacheKey, fallbackErr)
					return workflow, fallbackErr
				}

				workflow = fallbackWorkflow
			} else {
				log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
				return workflow, err
			}
		}

		if err == nil {
			res := resp.Inspect().Response
			defer res.Body.Close()
			if res.StatusCode == 404 {
				return workflow, errors.New("Workflow doesn't exist")
			}

			respBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return workflow, err
			}

			wrapped := WorkflowWrapper{}
			err = json.Unmarshal(respBody, &wrapped)
			if err != nil {
				return workflow, err
			}

			workflow = &wrapped.Source
		}
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		if err := project.Dbclient.Get(ctx, key, workflow); err != nil {
			if strings.Contains(err.Error(), `no such entity`) {
				query := datastore.NewQuery(nameKey).Filter("id =", strings.ToLower(id)).Limit(1)
				var workflows []Workflow
				if _, err := project.Dbclient.GetAll(ctx, query, &workflows); err != nil {
					if !strings.Contains(err.Error(), `cannot load field`) {
						return &Workflow{}, err
					}
				}

				if len(workflows) == 1 {
					workflow = &workflows[0]
				}
			} else if strings.Contains(err.Error(), `cannot load field`) {
				// Due to form migration
				if !strings.Contains(err.Error(), `input_markdown`) {
					log.Printf("[ERROR] Error in workflow loading. Migrating workflow to new workflow handler (5): %s", err)
				}

				err = nil
			} else {
				return &Workflow{}, err
			}
		}
	}

	validationData, err := GetCache(ctx, fmt.Sprintf("validation_workflow_%s", workflow.ID))
	if err == nil {
		cacheData := []byte(validationData.([]uint8))
		err = json.Unmarshal(cacheData, &workflow.Validation)
		if err != nil {
			log.Printf("[ERROR] Failed unmarshalling cache data for execution status (4): %s", err)
		}
	}

	// Somehow this can happen. Reverting to LATEST revision
	if len(workflow.Actions) > 0 && len(workflow.Triggers) == 0 {
		revisions, err := ListWorkflowRevisions(ctx, workflow.ID, 2)
		if err != nil {
			log.Printf("[WARNING] Failed getting revisions during trigger load for workflow %s: %s", workflow.ID, err)
		} else {
			if len(revisions) > 0 {
				for _, revision := range revisions {
					if revision.ID != workflow.ID {
						continue
					}

					if len(revision.Triggers) > 0 {
						workflow.Triggers = revision.Triggers
						break
					}
				}

				//log.Printf("[INFO] Reverting to revision triggers for workflow %s from 0 triggers to %d triggers", workflow.ID, len(revisions[0].Triggers))
				workflow.Triggers = revisions[0].Triggers
			}
		}
	}

	newWorkflow := FixWorkflowPosition(ctx, *workflow)
	workflow = &newWorkflow

	if len(skipHealth) == 0 || (len(skipHealth) > 0 && !skipHealth[0]) {

		healthWorkflow, _, err := GetStaticWorkflowHealth(ctx, *workflow)
		if err != nil {
			if !strings.Contains(err.Error(), "Org ID not set") {
				log.Printf("[ERROR] Failed getting static workflow health for workflow %s: %s (2)", workflow.ID, err)
			}
		} else {
			workflow = &healthWorkflow
		}
	} else {
		//log.Printf("[DEBUG] Skipping healthcheck during exec.")
	}

	if project.CacheDb && workflow.ID != "" && (len(workflow.Actions) > 1 || len(workflow.Triggers) > 0) {
		//log.Printf("[DEBUG] Setting cache for workflow %s", cacheKey)
		data, err := json.Marshal(workflow)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getworkflow: %s", err)
			return workflow, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getworkflow '%s': %s", cacheKey, err)
		}
	}

	return workflow, nil
}

func getWorkflowByAliasSearch(ctx context.Context, aliasName, id string) (*Workflow, error) {
	var buf bytes.Buffer
	query := map[string]interface{}{
		"size": 1,
		"query": map[string]interface{}{
			"ids": map[string]interface{}{
				"values": []string{id},
			},
		},
		"sort": []map[string]interface{}{
			{
				"edited": map[string]interface{}{
					"order":         "desc",
					"unmapped_type": "long",
				},
			},
			{
				"created": map[string]interface{}{
					"order":         "desc",
					"unmapped_type": "long",
				},
			},
		},
	}

	if err := json.NewEncoder(&buf).Encode(query); err != nil {
		return nil, err
	}

	resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
		Indices: []string{aliasName},
		Body:    &buf,
		Params:  opensearchapi.SearchParams{TrackTotalHits: true},
	})
	if err != nil {
		return nil, err
	}

	res := resp.Inspect().Response
	defer res.Body.Close()

	if res.StatusCode == 404 {
		return nil, errors.New("Workflow doesn't exist")
	}

	respBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	if res.StatusCode != 200 && res.StatusCode != 201 {
		return nil, fmt.Errorf("failed workflow alias lookup. status=%d body=%s", res.StatusCode, string(respBody))
	}

	wrapped := WorkflowSearchWrapper{}
	err = json.Unmarshal(respBody, &wrapped)
	if err != nil {
		return nil, err
	}

	if len(wrapped.Hits.Hits) == 0 {
		return nil, errors.New("Workflow doesn't exist")
	}

	found := wrapped.Hits.Hits[0].Source
	return &found, nil
}

func GetOrgStatistics(ctx context.Context, orgId string) (*ExecutionInfo, error) {
	nameKey := "org_statistics"
	stats := &ExecutionInfo{}
	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, stats)
			if err == nil {
				return stats, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for stats: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		shouldInitializeStats := false

		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: orgId,
		})

		if err != nil && !strings.Contains(err.Error(), "status: 404") {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return stats, err
		}

		if err != nil && strings.Contains(err.Error(), "status: 404") {
			shouldInitializeStats = true
		}

		if !shouldInitializeStats {
			res := resp.Inspect().Response
			defer res.Body.Close()
			if res.StatusCode == 404 {
				shouldInitializeStats = true
			} else {
				respBody, err := ioutil.ReadAll(res.Body)
				if err != nil {
					return stats, err
				}

				wrapped := ExecutionInfoWrapper{}
				err = json.Unmarshal(respBody, &wrapped)
				if err != nil {
					return stats, err
				}

				if !wrapped.Found {
					shouldInitializeStats = true
				} else {
					stats = &wrapped.Source
				}
			}
		}

		if shouldInitializeStats {
			org, err := GetOrg(ctx, orgId)
			if err != nil {
				log.Printf("[ERROR] Failed to get org(%s) for org_stats: %s", orgId, err)
				return stats, err
			}

			stats.OrgId = orgId
			stats.OrgName = org.Name
			if err := SetOrgStatistics(ctx, *stats, orgId); err != nil {
				log.Printf("[ERROR] Failed to set org(%s) stats after 404: %s", orgId, err)
				return stats, err
			}

			return stats, nil
		}
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(orgId), nil)
		if err := project.Dbclient.Get(ctx, key, stats); err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[INFO] Error in org stats loading (1). Migrating org to new org and user handler (3): %s", err)
				err = nil
			} else {
				return stats, err
			}
		}
	}

	if (stats.OrgId != orgId) && (len(orgId) > 0) {
		log.Printf("[WARNING] Org stats data corruption detected. Fixing org stats for org %s (was %s)", orgId, stats.OrgId)
		stats.OrgId = orgId
		org, err := GetOrg(ctx, orgId)
		if err == nil {
			stats.OrgName = org.Name
			err = SetOrgStatistics(ctx, *stats, orgId)
			if err != nil {
				log.Printf("[WARNING] Failed fixing org stats for org %s: %s", orgId, err)
			} else {
				log.Printf("[INFO] Fixed org stats for org %s", orgId)
			}
		} else {
			log.Printf("[WARNING] Failed getting org during org stats fix for org %s: %s", orgId, err)
		}
	}

	for dailyStatIndex, _ := range stats.DailyStatistics {
		for additionIndex, _ := range stats.DailyStatistics[dailyStatIndex].Additions {
			stats.DailyStatistics[dailyStatIndex].Additions[additionIndex].Date = stats.DailyStatistics[dailyStatIndex].Date
		}
	}

	// Sort stats.DailyStatistics by date. It's time.Time
	sort.Slice(stats.DailyStatistics, func(i, j int) bool {
		return stats.DailyStatistics[i].Date.Before(stats.DailyStatistics[j].Date)
	})

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for stats %s", cacheKey)
		data, err := json.Marshal(stats)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in get stats: %s", err)
			return stats, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for get stats'%s': %s", cacheKey, err)
		}
	}

	return stats, nil
}

func GetAllWorkflowsByQuery(ctx context.Context, user User, maxAmount int, cursor string) ([]Workflow, error) {
	var workflows []Workflow
	var err error
	limit := 30

	if user.Role == "org-reader" {
		log.Printf("[DEBUG] Giving org-reader %s (%s) access to all workflows in their active org.", user.Username, user.Id)
		user.Role = "admin"
	}

	if user.Role == "user" {
		log.Printf("[DEBUG] Giving org-user %s (%s) access to all workflows in their active org.", user.Username, user.Id)
		user.Role = "admin"
	}

	// Cache
	if maxAmount == 0 || maxAmount > 250 {
		maxAmount = 250
	}

	cacheKey := fmt.Sprintf("%s_%s_workflows", cursor, user.ActiveOrg.Id)
	if len(cursor) == 0 {
		cacheKey = fmt.Sprintf("%s_workflows", user.ActiveOrg.Id)
	}

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)

		if err == nil {

			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &workflows)
			if err == nil {
				//if debug { 
				//	log.Printf("\n\n[DEBUG] Cache FOUND for key '%s': %d workflows\n\n", cacheKey, len(workflows))
				//}

				return workflows, nil
			}
		}
	}

	// Appending the users' workflows
	nameKey := "workflow"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		// increased the maxAmount for onprem user on May 15th
		if maxAmount <= 250 {
			maxAmount = 600
		}

		query := map[string]interface{}{
			"size": maxAmount,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"owner": user.Id,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"owner": "",
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return workflows, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return workflows, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflows): %s", err)
			return workflows, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return workflows, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return workflows, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return workflows, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return workflows, err
		}

		wrapped := WorkflowSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return workflows, err
		}

		//log.Printf("Found workflows: %d", len(wrapped.Hits.Hits))
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.ID == "" {
				continue
			}

			if hit.Source.Owner == user.Id || hit.Source.OrgId == user.ActiveOrg.Id {
				workflows = append(workflows, hit.Source)
			} else {
				//log.Printf("bad workflow owner: %s", hit.Source.Owner)
			}
		}

		if user.Role == "admin" {
			var buf bytes.Buffer
			query = map[string]interface{}{
				"size": maxAmount,
				"query": map[string]interface{}{
					"match": map[string]interface{}{
						"org_id": user.ActiveOrg.Id,
					},
				},
			}
			if err := json.NewEncoder(&buf).Encode(query); err != nil {
				log.Printf("[WARNING] Error encoding find user query: %s", err)
				return workflows, err
			}

			resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
				Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
				Body:    &buf,
				Params: opensearchapi.SearchParams{
					TrackTotalHits: true,
				},
			})
			if err != nil {
				if strings.Contains(err.Error(), "index_not_found_exception") {
					return workflows, nil
				}

				log.Printf("[ERROR] Error getting response from Opensearch (Get workflows 2): %s", err)
				return workflows, err
			}

			res := resp.Inspect().Response
			defer res.Body.Close()
			if res.StatusCode == 404 {
				return workflows, nil
			}

			if res.IsError() {
				var e map[string]interface{}
				if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
					log.Printf("[WARNING] Error parsing the response body: %s", err)
					return workflows, err
				} else {
					// Print the response status and error information.
					log.Printf("[%s] %s: %s",
						res.Status(),
						e["error"].(map[string]interface{})["type"],
						e["error"].(map[string]interface{})["reason"],
					)
				}
			}

			if res.StatusCode != 200 && res.StatusCode != 201 {
				return workflows, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
			}

			respBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return workflows, err
			}

			wrapped := WorkflowSearchWrapper{}
			err = json.Unmarshal(respBody, &wrapped)
			if err != nil {
				return workflows, err
			}

			userWorkflowLen := len(workflows)
			for _, hit := range wrapped.Hits.Hits {
				if hit.Source.ID == "" {
					continue
				}

				found := false
				for _, workflow := range workflows {
					if workflow.ID == hit.ID {
						found = true
						break
					}
				}

				if !found {
					workflows = append(workflows, hit.Source)
				}
			}

			if debug {
				log.Printf("[DEBUG] Appending workflows (ADMIN + suborg distribution) for organization %s. Already have %d workflows for the user. Found %d (%d new) for org. New unique amount: %d (1)", user.ActiveOrg.Id, userWorkflowLen, len(wrapped.Hits.Hits), len(workflows)-userWorkflowLen, len(workflows))
			}
		}

	} else {
		//log.Printf("[INFO] Appending workflows (ADMIN) for organization %s (2)", user.ActiveOrg.Id)

		if len(user.ActiveOrg.Id) == 0 {
			return workflows, errors.New("No active org to find workflows for found")
		}

		//log.Printf("\n\n\nLooking for workflows for org %s with user %s (%s)\n\n\n", user.ActiveOrg.Id, user.Username, user.Id)

		cursorStr := ""
		query := datastore.NewQuery(nameKey).Filter("org_id =", user.ActiveOrg.Id).Limit(limit)
		for {
			it := project.Dbclient.Run(ctx, query)
			if len(workflows) >= maxAmount {
				break
			}

			for {
				innerWorkflow := Workflow{}
				_, err = it.Next(&innerWorkflow)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {

					} else {
						if !strings.Contains(fmt.Sprintf("%s", err), "no more items in iterator") {
							//log.Printf("[WARNING] Workflow iterator issue: %s", err)
						}

						break
					}
				}

				if innerWorkflow.Public {
					continue
				}

				if innerWorkflow.Hidden {
					continue
				}

				found := false
				for _, loopedWorkflow := range workflows {
					if loopedWorkflow.ID == innerWorkflow.ID {
						found = true
						break
					}
				}

				if !found {
					workflows = append(workflows, innerWorkflow)
				}

				if len(workflows) >= maxAmount {
					break
				}
			}

			if err != iterator.Done {
				log.Printf("[INFO] Failed fetching workflow results: %v", err)
				break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	//log.Printf("Found %d workflows for user %s (%s) in org %s", len(workflows), user.Username, user.Id, user.ActiveOrg.Id)

	if len(workflows) > maxAmount {
		workflows = workflows[:maxAmount]
	}

	fixedWorkflows := []Workflow{}
	for _, workflow := range workflows {
		if workflow.Hidden {
			continue
		}

		if len(workflow.Name) == 0 && len(workflow.Actions) <= 1 {
			continue
		}

		if len(workflow.OrgId) == 0 && len(workflow.Owner) == 0 {
			log.Printf("[ERROR] Workflow %s (%s) has no org or owner", workflow.Name, workflow.ID)
			continue
		}

		fixedWorkflows = append(fixedWorkflows, workflow)
	}

	slice.Sort(fixedWorkflows[:], func(i, j int) bool {
		return fixedWorkflows[i].Edited > fixedWorkflows[j].Edited
	})

	if project.CacheDb {
		newjson, err := json.Marshal(fixedWorkflows)
		if err != nil {
			return fixedWorkflows, nil
		}

		err = SetCache(ctx, cacheKey, newjson, 5)
		if err != nil {
			log.Printf("[WARNING] Failed updating workflow cache: %s", err)
		}
	}

	return fixedWorkflows, nil
}

func GetOrgByCreatorId(ctx context.Context, id string) (*Org, error) {
	nameKey := "Organizations"
	cacheKey := fmt.Sprintf("creator_%s_%s", nameKey, id)

	curOrg := &Org{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, curOrg)
			if err == nil {
				return curOrg, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for org %s (1): %s", id, err)
		}
	}

	setOrg := false
	if project.DbType == "opensearch" {
	} else {
		query := datastore.NewQuery(nameKey).Filter("creator_id =", id).Limit(1)

		allOrgs := []Org{}
		_, err := project.Dbclient.GetAll(ctx, query, &allOrgs)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return curOrg, err
			}
		}

		if len(allOrgs) > 0 {
			curOrg = &allOrgs[0]
		}
	}

	// How does this happen?
	if len(curOrg.Id) == 0 {
		curOrg.Id = id
		return curOrg, errors.New(fmt.Sprintf("Couldn't find creator org with ID %s", curOrg.Id))
	}

	newUsers := []User{}
	for _, user := range curOrg.Users {
		user.Password = ""
		user.Session = ""
		user.ResetReference = ""
		user.PrivateApps = []WorkflowApp{}
		user.VerificationToken = ""
		user.ApiKey = ""
		newUsers = append(newUsers, user)
	}

	curOrg.Users = newUsers
	if len(curOrg.Tutorials) == 0 {
		curOrg = GetTutorials(ctx, *curOrg, true)
	}

	// Making sure to skip old irrelevant priorities
	newPriorities := []Priority{}
	for _, priority := range curOrg.Priorities {
		if priority.Type == "usecases" {
			continue
		}

		newPriorities = append(newPriorities, priority)
	}

	curOrg.Priorities = newPriorities
	if project.CacheDb {
		neworg, err := json.Marshal(curOrg)
		if err != nil {
			log.Printf("[ERROR] Failed marshalling org for cache: %s", err)
			return curOrg, nil
		}

		err = SetCache(ctx, cacheKey, neworg, 1440)
		if err != nil {
			log.Printf("[ERROR] Failed updating org cache: %s", err)
		}

		if setOrg {
			log.Printf("[INFO] UPDATING ORG %s!!", curOrg.Id)
			SetOrg(ctx, *curOrg, curOrg.Id)
		}
	}

	return curOrg, nil
}

// ListBooks returns a list of books, ordered by title.
// Handles org grabbing and user / org migrations
func GetOrg(ctx context.Context, id string) (*Org, error) {
	if id == "public" {
		//return &Org{}, errors.New("'public' org is used for Singul action without being logged in. Not relevant.")
		return &Org{
			Id:   "public",
			Name: "Public",
		}, nil
	}

	// Clean the ID: remove whitespace, quotes, and backslashes
	originalId := id
	id = strings.TrimSpace(id)
	id = strings.ReplaceAll(id, "\"", "")
	id = strings.ReplaceAll(id, "'", "")
	id = strings.ReplaceAll(id, "\\", "")
	if len(id) == 0 {
		return &Org{}, errors.New("Empty org id after cleaning")
	}
	if id != originalId {
		log.Printf("[WARNING] GetOrg ID was cleaned from '%s' to '%s' - check data source", originalId, id)
	}

	nameKey := "Organizations"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	curOrg := &Org{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, curOrg)
			if err == nil {
				if curOrg.Id == "" {
					return curOrg, errors.New("Org doesn't exist")
				} else {
					return curOrg, nil
				}
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for org %s (2): %s", id, err)
		}
	}

	setOrg := false
	if project.DbType == "opensearch" {
		if len(id) == 0 {
			return &Org{}, errors.New("Empty org id")
		}

		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error in org get: %s", err)
			return &Org{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Printf("[WARNING] Failed getting org body: %s", err)
			return &Org{}, err
		}

		if res.StatusCode == 404 {
			log.Printf("[WARNING] Failed getting org '%s' - status: 404 - %s", id, string(respBody))
			return &Org{}, errors.New("Org doesn't exist")
		}

		wrapped := OrgWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			log.Printf("[WARNING] Failed unmarshaling org: %s", err)
			return &Org{}, err
		}

		curOrg = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, curOrg); err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Error in org loading (4), but returning without warning: %s", err)
				err = nil
			} else {
				if strings.Contains(err.Error(), `no such entity`) && project.CacheDb {
					neworg, err := json.Marshal(curOrg)
					if err != nil {
						return &Org{}, err
					}

					// Set cache for it
					err = SetCache(ctx, cacheKey, neworg, 30)
					if err != nil {
						log.Printf("[ERROR] Failed updating org cache (3): %s", err)
					}
				} else {
					log.Printf("[ERROR] Problem in org loading (2) for %s: %s", key, err)
				}

				//orgErr = err
				return &Org{}, err
			}
		}
	}

	// How does this happen?
	if len(curOrg.Id) == 0 {
		curOrg.Id = id
		//return curOrg, errors.New(fmt.Sprintf("Couldn't find org with ID '%s'", curOrg.Id))
	}

	newUsers := []User{}
	for _, user := range curOrg.Users {
		user.Password = ""
		user.Session = ""
		user.ResetReference = ""
		user.PrivateApps = []WorkflowApp{}
		user.VerificationToken = ""
		user.ApiKey = ""
		newUsers = append(newUsers, user)
	}

	curOrg.Users = newUsers
	if len(curOrg.Tutorials) == 0 {
		curOrg = GetTutorials(ctx, *curOrg, true)
	}

	// Making sure to skip old irrelevant priorities
	newPriorities := []Priority{}
	for _, priority := range curOrg.Priorities {
		if priority.Type == "usecases" {
			continue
		}

		newPriorities = append(newPriorities, priority)
	}

	// Check if Subscription is from BEFORE November 4th 2023

	eulaSigned := false
	if len(curOrg.Subscriptions) > 1 {
		replicas := map[string]int64{}
		for orgIndex, sub := range curOrg.Subscriptions {
			if sub.EulaSigned {
				eulaSigned = true
			}

			if sub.Startdate == 0 || sub.Startdate < 1699053459 {
				curOrg.Subscriptions[orgIndex].EulaSigned = true
			}

			if _, ok := replicas[sub.Name]; ok {
				if replicas[sub.Name] > sub.Startdate {
					log.Printf("[DEBUG] Removing subscription %s from org %s", sub.Name, curOrg.Id)

					replicas[sub.Name] = sub.Startdate
				}
			} else {
				replicas[sub.Name] = sub.Startdate
			}
		}

		newsubs := []PaymentSubscription{}
		for key, value := range replicas {
			foundsub := PaymentSubscription{}
			for _, sub := range curOrg.Subscriptions {
				if sub.Name == key && sub.Startdate == value {
					foundsub = sub
					break
				}
			}

			if foundsub.Name != "" {
				foundsub.EulaSigned = eulaSigned
				newsubs = append(newsubs, foundsub)
			}
		}

		if len(newsubs) > 0 {
			curOrg.Subscriptions = newsubs
			//log.Printf("[DEBUG] New subscriptions for org %s: %d", curOrg.Id, len(newsubs))
		}
	}

	curOrg.Priorities = newPriorities
	if project.CacheDb {
		neworg, err := json.Marshal(curOrg)
		if err != nil {
			log.Printf("[ERROR] Failed marshalling org for cache: %s", err)
			return curOrg, nil
		}

		err = SetCache(ctx, cacheKey, neworg, 1440)
		if err != nil {
			log.Printf("[ERROR] Failed updating org cache: %s", err)
		}

		if setOrg {
			log.Printf("[INFO] AUTO UPDATING ORG %s!!", curOrg.Id)
			SetOrg(ctx, *curOrg, curOrg.Id)
		}
	}

	/*
		if orgErr {
			return curOrg, orgErr
		}
	*/

	return curOrg, nil
}

func init() {

	isValid := checkImportPath()
	if !isValid {
		time.Sleep(time.Duration(600+rand.Intn(600)) * time.Second)
		os.Exit(3)
	}
}

func GetFirstOrg(ctx context.Context) (*Org, error) {
	nameKey := "Organizations"

	curOrg := &Org{}
	if project.DbType == "opensearch" {
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			//Body: true,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return curOrg, err
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get first org): %s", err)

			return curOrg, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode != 200 && res.StatusCode != 201 {
			return curOrg, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return curOrg, err
		}

		wrapped := OrgSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return curOrg, err
		}

		if len(wrapped.Hits.Hits) > 0 {
			for _, hit := range wrapped.Hits.Hits {
				if len(hit.Source.Id) > 0 && len(hit.Source.Users) > 0 {
					curOrg = &hit.Source
					break
				}
			}

			if curOrg.Id == "" {
				log.Printf("[ERROR] No orgs found with users & an ID, returning first org")
				curOrg = &wrapped.Hits.Hits[0].Source
			}
		} else {
			return curOrg, errors.New("No orgs found")
		}

	} else {
		query := datastore.NewQuery(nameKey).Limit(1)
		allOrgs := []Org{}
		_, err := project.Dbclient.GetAll(ctx, query, &allOrgs)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return curOrg, err
			}
		}

		if len(allOrgs) > 0 {
			curOrg = &allOrgs[0]
		} else {
			return curOrg, errors.New("No orgs found")
		}
	}

	return curOrg, nil
}

func indexEs(ctx context.Context, nameKey, id string, bytes []byte) error {
	//req := esapi.IndexRequest{
	req := opensearchapi.IndexReq{
		Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
		DocumentID: id,
		Body:       strings.NewReader(string(bytes)),
		Params: opensearchapi.IndexParams{
			Refresh: "true",
			Pretty:  true,
		},
	}

	//res, err := req.Do(ctx, &project.Es)
	resp, err := project.Es.Index(ctx, req)
	if err != nil {
		// Usually due to goroutines
		if strings.Contains(err.Error(), "context deadline exceeded") {
			resp, err = project.Es.Index(context.Background(), req)
			if err != nil {
				if strings.Contains(err.Error(), "index_not_found_exception") {
					log.Printf("[ERROR] Error getting response from Opensearch (index ES) - 2: %s", err)
				}
			}
		} else {
			log.Printf("[ERROR] Error getting response from Opensearch (index ES) - 1: %s", err)
		}

		return err
	}

	res := resp.Inspect().Response
	defer res.Body.Close()
	respBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		respBody = []byte("Failed to parse body")
	}

	if res.StatusCode != 200 && res.StatusCode != 201 {
		return errors.New(fmt.Sprintf("Bad statuscode from database: %d. Reason: %s", res.StatusCode, string(respBody)))
	}

	var r map[string]interface{}
	err = json.Unmarshal(respBody, &r)
	if err != nil {
		log.Printf("[WARNING] Error parsing the response body from Opensearch: %s. Raw: %s", err, respBody)
		//return err
	}
	return nil
}

func GetTutorials(ctx context.Context, org Org, updateOrg bool) *Org {
	log.Printf("[DEBUG] Getting init tutorials for org %s (%s)", org.Name, org.Id)

	allSteps := []Tutorial{
		Tutorial{
			Name:        "Find relevant apps",
			Description: "0 out of 8 apps configured",
			Done:        false,
			Link:        "/welcome?tab=2",
			Active:      true,
		},
		Tutorial{
			Name:        "Discover Usecases",
			Description: "0 workflows created. Create from Workflow Templates! Additional usecases: /usecases",
			Done:        false,
			Link:        "/welcome?tab=3",
			Active:      true,
		},
		Tutorial{
			Name:        "Invite teammates",
			Description: "Configure org name, image, and invite teammates",
			Done:        false,
			Link:        "/admin?tab=users",
			Active:      true,
		},
		Tutorial{
			Name:        "Security & Stability",
			Description: "Configure MFA or SAML/SSO, new Environments & a Notification workflow",
			Done:        false,
			Link:        "/admin?tab=organization",
			Active:      true,
		},
	}

	have := []string{}
	missing := []string{}
	if len(org.SecurityFramework.SIEM.Name) > 0 {
		have = append(have, "SIEM")
	} else {
		missing = append(missing, "SIEM")
	}
	if len(org.SecurityFramework.Communication.Name) > 0 {
		have = append(have, "Communication")
	} else {
		missing = append(missing, "Communication")
	}
	if len(org.SecurityFramework.Assets.Name) > 0 {
		have = append(have, "Assets")
	} else {
		missing = append(missing, "Assets")
	}
	if len(org.SecurityFramework.Cases.Name) > 0 {
		have = append(have, "Cases")
	} else {
		missing = append(missing, "Cases")
	}
	if len(org.SecurityFramework.Network.Name) > 0 {
		have = append(have, "Network")
	} else {
		missing = append(missing, "Network")
	}
	if len(org.SecurityFramework.Intel.Name) > 0 {
		have = append(have, "Intel")
	} else {
		missing = append(missing, "Intel")
	}
	if len(org.SecurityFramework.EDR.Name) > 0 {
		have = append(have, "EDR")
	} else {
		missing = append(missing, "EDR")
	}
	if len(org.SecurityFramework.IAM.Name) > 0 {
		have = append(have, "IAM")
	} else {
		missing = append(missing, "IAM")
	}

	if len(have) > 1 {
		allSteps[0].Done = true
		allSteps[0].Description = fmt.Sprintf("%d out of %d apps configured", len(have), len(have)+len(missing))
	}

	selectedUser := User{}
	for _, inputUser := range org.Users {
		user, err := GetUser(ctx, inputUser.Id)
		if user.Role == "admin" && user.ActiveOrg.Id == org.Id {
			if err == nil {
				selectedUser = *user
				break
			}
		}
	}

	if len(org.Users) > 1 {
		allSteps[2].Description = fmt.Sprintf("%d users invited and org name changed.", len(org.Users))
		if strings.ToLower(org.Org) == strings.ToLower(org.Name) {
			allSteps[2].Description = "Edit your org name and image, and invite your teammates to build together"
			allSteps[2].Link = "/admin?tab=users"
		} else {
			allSteps[2].Done = true
		}
	}

	if len(selectedUser.Id) > 0 {
		workflows, _ := GetAllWorkflowsByQuery(ctx, selectedUser, 250, "")
		if len(workflows) > 1 {
			allSteps[1].Done = true
			allSteps[1].Description = fmt.Sprintf("%d workflows created. Find more workflows in the searchbar or on /usecases", len(workflows))
			allSteps[1].Link = "/usecases"
		}
	}

	if org.SSOConfig.SSORequired {
		allSteps[3].Done = true
	} else {
		allSteps[3].Link = "/admin?admin_tab=organization"
	}

	org.Tutorials = allSteps

	if updateOrg {
		SetOrg(ctx, org, org.Id)
	}
	return &org
}

func propagateOrg(org Org, reverse bool) error {
	// the philosophy here is that, usually, we propagate only
	// from the main region to the other regions. However, "reverse"
	// makes propagation go from the other regions to the main region.

	if len(org.Id) == 0 {
		return errors.New("no ID provided for org")
	}

	if len(propagateUrl) == 0 || len(propagateToken) == 0 {
		return errors.New("no SHUFFLE_PROPAGATE_URL or SHUFFLE_PROPAGATE_TOKEN provided")
	}

	log.Printf("[INFO] Asking %s to propagate org %s", propagateUrl, org.Id)

	data := map[string]string{"mode": "org", "orgId": org.Id}

	if reverse {
		data["region"] = os.Getenv("SHUFFLE_GCEPROJECT_REGION")
	}

	reqBody, err := json.Marshal(data)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", propagateUrl, bytes.NewBuffer(reqBody))
	if err != nil {
		return err
	}

	// Set headers
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", propagateToken)

	// Send the request via a client
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	defer resp.Body.Close()

	// Check the response
	if resp.StatusCode != 200 {
		log.Printf("[WARNING] Error in propagation: %s for org %s", resp.Status, org.Id)
		return errors.New(fmt.Sprintf("bad statuscode: %d", resp.StatusCode))
	}

	return nil
}

func propagateApp(appId string, delete bool) error {
	if len(appId) == 0 {
		return errors.New("no ID provided for app")
	}

	if delete {
		log.Printf("[INFO] Deletion propagation is disabled right now.")
		return nil
	}

	if len(propagateUrl) == 0 || len(propagateToken) == 0 {
		return errors.New("no SHUFFLE_PROPAGATE_URL or SHUFFLE_PROPAGATE_TOKEN provided")
	}
	// SHUFFLE_GCE_LOCATION
	gceRegion := os.Getenv("SHUFFLE_GCEPROJECT_REGION")

	log.Printf("[INFO] Asking %s to propagate app %s", propagateUrl, appId)
	data := map[string]string{"mode": "app", "appId": appId, "region": gceRegion}

	reqBody, err := json.Marshal(data)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling propagation data %s: %s", appId, err)
		return err
	}

	req, err := http.NewRequest("POST", propagateUrl, bytes.NewBuffer(reqBody))
	if err != nil {
		log.Printf("[WARNING] Failed creating request for app %s: %s", appId, err)
		return err
	}

	// Set headers
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", propagateToken)

	// Send the request via a client
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("[WARNING] Failed sending request for app %s: %s", appId, err)
		return err
	}

	defer resp.Body.Close()

	// Check the response
	if resp.StatusCode != 200 {
		log.Printf("[WARNING] Error in propagation: %s for app %s", resp.Status, appId)
		return errors.New(fmt.Sprintf("bad statuscode: %d", resp.StatusCode))
	}

	log.Printf("[INFO] Propagation successful for app %s", appId)

	return nil
}

func propagateUser(user User, delete bool) error {
	if len(user.Id) == 0 {
		return errors.New("no ID provided for user")
	}

	if len(propagateUrl) == 0 || len(propagateToken) == 0 {
		return errors.New("no SHUFFLE_PROPAGATE_URL or SHUFFLE_PROPAGATE_TOKEN provided")
	}

	log.Printf("[INFO] Asking %s to propagate user %s", propagateUrl, user.Id)

	data := map[string]string{"mode": "user", "userId": user.Id}
	if delete {
		log.Printf("[INFO] Deletion propagation is disabled right now.")
		// data["delete"] = "true"
	}

	reqBody, err := json.Marshal(data)
	if err != nil {
		return err
	}

	req, err := http.NewRequest("POST", propagateUrl, bytes.NewBuffer(reqBody))
	if err != nil {
		return err
	}

	// Set headers
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Authorization", propagateToken)

	// Send the request via a client
	client := &http.Client{}
	resp, err := client.Do(req)

	if err != nil {
		return err
	}

	defer resp.Body.Close()

	// Check the response
	if resp.StatusCode != 200 {
		log.Printf("[WARNING] Error in propagation: %s for user %s", resp.Status, user.Id)
		return errors.New(fmt.Sprintf("bad statuscode: %d", resp.StatusCode))
	}

	return nil
}

func GetUsersByOrg(ctx context.Context, orgId string) ([]User, error) {
	nameKey := "Users"

	users := []User{}
	cacheKey := fmt.Sprintf("%s_orgusers_%s", nameKey, orgId)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &users)
			if err == nil {
				return users, nil
			}
		}
	}

	if project.DbType == "opensearch" {
		return users, errors.New("Not implemented")
	} else {
		query := datastore.NewQuery(nameKey).Filter("orgs =", orgId)

		_, err := project.Dbclient.GetAll(ctx, query, &users)
		if err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				return users, nil
			}

			log.Printf("[ERROR] Problem in user loading for org %s: %s", orgId, err)
			return users, err
		}
	}

	if project.CacheDb {
		marshaled, err := json.Marshal(users)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling users for cache: %s", err)
			return users, nil
		}

		err = SetCache(ctx, cacheKey, marshaled, 1)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for users by org '%s': %s", cacheKey, err)
		}
	}

	return users, nil
}

func SetOrg(ctx context.Context, data Org, id string) error {
	if len(id) == 0 {
		return errors.New(fmt.Sprintf("No ID provided for org %s", data.Name))
	}

	if len(data.Users) == 0 {
		// Where do users go sometimes? wtf.
		if project.Environment == "cloud" {
			orgUsers, err := GetUsersByOrg(ctx, id)
			if err != nil {
				log.Printf("[ERROR] Error loading users during org autocorrecting: %s", err)
			}

			if len(orgUsers) > 0 {
				log.Printf("[ERROR] Found 0 users for org %s. Autocorrected it to %d (reloaded). FIX: Why did the org LOSE users?", data.Id, len(orgUsers))
				data.Users = orgUsers
			}
		}

		if len(data.Users) == 0 {
			return errors.New("Not allowed to update an org without any users in the organization. Need AT LEAST one user to update")
		}
	}

	if id != data.Id && len(data.Id) > 0 {
		log.Printf("[ERROR] Org ID mismatch: %s != %s. Resetting ID", id, data.Id)
		id = data.Id
	}

	data.Id = id
	if len(data.Name) == 0 {
		data.Name = "tmp"

		if len(data.Org) > 0 {
			data.Name = data.Org
		} else {
			data.Org = data.Name
		}
	}

	if len(data.ManagerOrgs) == 0 && len(data.CreatorOrg) > 0 {
		data.ManagerOrgs = []OrgMini{
			OrgMini{
				Id: data.CreatorOrg,
			},
		}
	}

	nameKey := "Organizations"
	timeNow := int64(time.Now().Unix())
	if data.Created == 0 {
		data.Created = timeNow
	}

	data.Edited = timeNow
	newUsers := []User{}
	for _, user := range data.Users {
		user.Password = ""
		user.Session = ""
		user.ApiKey = ""
		user.PrivateApps = []WorkflowApp{}
		user.MFA = MFAInfo{}
		user.Authentication = []UserAuth{}

		user.PublicProfile = PublicProfile{}
		user.LoginInfo = []LoginInfo{}
		user.PersonalInfo = PersonalInfo{}

		//user.Orgs = []string{}

		newUsers = append(newUsers, user)
	}

	data.Users = newUsers
	if len(data.Tutorials) == 0 {
		data = *GetTutorials(ctx, data, false)
	}

	if len(data.Users) == 0 {
		return errors.New("Not allowed to update an org without any users in the organization. Add at least one user to update")
	}

	// clear session_token and API_token for user
	if project.DbType == "opensearch" {
		b, err := json.Marshal(data)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling %s - %s: %s", id, nameKey, err)
			return err
		}

		err = indexEs(ctx, nameKey, id, b)
		if err != nil {
			return err
		}
	} else {
		k := datastore.NameKey(nameKey, id, nil)
		if _, err := project.Dbclient.Put(ctx, k, &data); err != nil {
			log.Println(err)
			return err
		}

		if data.Region != "" && data.Region != "europe-west2" && gceProject == "shuffler" {
			go func() {
				err := propagateOrg(data, false)
				if err != nil {
					if !strings.Contains(fmt.Sprintf("%s", err), "no SHUFFLE_PROPAGATE_URL") {
						log.Printf("[ERROR] Failed propagating org %s for region %#v: %s", data.Id, data.Region, err)
					}
				} else {
					//log.Printf("[INFO] Successfully propagated org %s to region %#v", data.Id, data.Region)
				}
			}()
		}
	}

	if project.CacheDb {
		newUsers := []User{}
		for _, user := range data.Users {
			user.Password = ""
			user.Session = ""
			user.ResetReference = ""
			user.PrivateApps = []WorkflowApp{}
			user.VerificationToken = ""
			newUsers = append(newUsers, user)
		}

		data.Users = newUsers

		neworg, err := json.Marshal(data)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in setorg: %s", err)
			return nil
		}

		cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
		err = SetCache(ctx, cacheKey, neworg, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for org '%s': %s", cacheKey, err)
		}

		for _, user := range data.Users {
			DeleteCache(ctx, fmt.Sprintf("user_orgs_%s", user.Id))
		}
	}

	return nil
}

// Index = Username
func DeleteKey(ctx context.Context, entity string, value string, orgIdList ...string) error {

	orgId := ""
	if len(orgIdList) > 0 && len(orgIdList[0]) > 0 {
		orgId = orgIdList[0]
	}

	// Non indexed User data
	if entity == "workflowexecution" {
		log.Printf("[WARNING][%s] DELETING workflowexecution in org '%s'", value, orgId)
	}

	if entity == "org_cache" {
		// FIXME: Add check in ngram to clean up correlations after deletions
	}

	if entity == "workflow" && len(orgId) > 0 {
		DeleteCache(ctx, fmt.Sprintf("%s_workflows", orgId))
		DeleteCache(ctx, fmt.Sprintf("%s_%s_workflows", "", orgId))
	}

	DeleteCache(ctx, fmt.Sprintf("%s_%s", entity, value))
	if len(value) == 0 {
		//log.Printf("[WARNING] Couldn't delete %s because value (id) must be longer than 0", entity)
		return errors.New("Value to delete must be larger than 0")
	}

	if project.DbType == "opensearch" {
		//log.Printf("[DEBUG] Deleting from index '%s' with item '%s' from opensearch", entity, value)

		resp, err := project.Es.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{
			Index:      strings.ToLower(GetESIndexPrefix(entity)),
			DocumentID: value,
		})

		if err != nil {
			if strings.Contains(err.Error(), "has more than one index associated with it") {
				deleteErr := deleteDocumentByQueryAcrossAlias(ctx, strings.ToLower(GetESIndexPrefix(entity)), value)
				if deleteErr == nil {
					return nil
				}

				log.Printf("[WARNING] Fallback delete by query failed for %s/%s: %s", entity, value, deleteErr)
				return deleteErr
			}

			if strings.Contains(err.Error(), "not_found") {
				return nil
			}

			log.Printf("[WARNING] Error in DELETE (2): %s", err)
			return err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			//log.Printf("[WARNING] Couldn't delete %s:%s. Status: %d", entity, value, res.StatusCode)
			return nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body (DELETE): %s", err)
				return err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		//log.Printf("[DEBUG] Deleted %s (%s)", strings.ToLower(entity), value)
	} else {
		key1 := datastore.NameKey(entity, value, nil)
		err := project.Dbclient.Delete(ctx, key1)
		if err != nil {
			log.Printf("[WARNING] Error deleting %s from %s: %s", value, entity, err)
			return err
		}
	}

	return nil
}

func deleteDocumentByQueryAcrossAlias(ctx context.Context, aliasName, documentID string) error {
	query := map[string]interface{}{
		"query": map[string]interface{}{
			"ids": map[string]interface{}{
				"values": []string{documentID},
			},
		},
	}

	queryBytes, err := json.Marshal(query)
	if err != nil {
		return err
	}

	resp, err := project.Es.Document.DeleteByQuery(ctx, opensearchapi.DocumentDeleteByQueryReq{
		Indices: []string{aliasName},
		Body:    bytes.NewReader(queryBytes),
	})
	if err != nil {
		if strings.Contains(err.Error(), "not_found") {
			return nil
		}

		return err
	}

	res := resp.Inspect().Response
	defer res.Body.Close()

	if res.StatusCode == 404 {
		return nil
	}

	if res.IsError() {
		responseData, readErr := ioutil.ReadAll(res.Body)
		if readErr != nil {
			return readErr
		}

		return fmt.Errorf("delete by query failed with status %d: %s", res.StatusCode, string(responseData))
	}

	return nil
}

// Index = Username
func SetApikey(ctx context.Context, Userdata User) error {

	// Non indexed User data
	newapiUser := new(Userapi)
	newapiUser.ApiKey = Userdata.ApiKey
	newapiUser.Username = strings.ToLower(Userdata.Username)
	nameKey := "apikey"

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		data, err := json.Marshal(Userdata)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling user in set apikey: %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, newapiUser.ApiKey, data)
		if err != nil {
			return err
		}
	} else {
		key1 := datastore.NameKey(nameKey, newapiUser.ApiKey, nil)
		if _, err := project.Dbclient.Put(ctx, key1, newapiUser); err != nil {
			log.Printf("Error adding apikey: %s", err)
			return err
		}
	}

	return nil
}

func SetOpenApiDatastore(ctx context.Context, id string, openapi ParsedOpenApi) error {
	nameKey := "openapi3"
	if project.DbType == "opensearch" {
		data, err := json.Marshal(openapi)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling user: %s", err)
			return err
		}
		err = indexEs(ctx, nameKey, id, data)
		if err != nil {
			return err
		}
	} else {
		k := datastore.NameKey(nameKey, id, nil)
		if _, err := project.Dbclient.Put(ctx, k, &openapi); err != nil {

			if strings.Contains(fmt.Sprintf("%s", err), "entity is too big") || strings.Contains(fmt.Sprintf("%s", err), "is longer than") {
				_, err = UploadAppSpecFiles(ctx, &project.StorageClient, WorkflowApp{}, openapi)
				if err != nil {
					log.Printf("[WARNING] Failed uploading app spec file in set openapi app: %s", err)
				} else {
					oldBody := openapi.Body
					openapi.Body = ""
					if _, err = project.Dbclient.Put(ctx, k, &openapi); err != nil {
						log.Printf("[ERROR] Failed second upload of openapi app %s: %s", openapi.ID, err)
					} else {
						log.Printf("[DEBUG] Successfully updated openapi app with no body!")

						// Ensuring cache is in order
						openapi.Body = oldBody
					}
				}
			} else {
				//log.Printf("[WARNING] Error adding workflow app: %s", err)
				log.Printf("[WARNING] Failed setting openapi for ID %s in datastore: %s", id, err)
			}
			return err
		}

	}

	if project.CacheDb {
		data, err := json.Marshal(openapi)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling openapi3 in set: %s", err)
			return nil
		}

		cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating openapi cache in set: %s", err)
		}
	}

	return nil
}

func GetOpenApiDatastore(ctx context.Context, id string) (ParsedOpenApi, error) {
	nameKey := "openapi3"
	api := &ParsedOpenApi{}

	if strings.HasSuffix(id, ".") {
		id = id[:len(id)-1]
	}

	if len(id) > 32 {
		log.Printf("[ERROR] ID %s is too long for datastore. Reducing to 32", id)
		id = id[:32]
	}

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &api)
			if err == nil {
				return *api, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for user: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return *api, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return *api, errors.New("OpenAPI spec doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return *api, err
		}

		wrapped := ParsedApiWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return *api, err
		}

		api = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		err := project.Dbclient.Get(ctx, key, api)
		//if (err != nil || len(api.Body) == 0) && !strings.Contains(fmt.Sprintf("%s", err), "no such") {
		if err != nil || len(api.Body) == 0 {
			if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
				return *api, nil
			}

			log.Printf("[ERROR] Some OpenAPI refissue for ID '%s': %s", id, err)

			//project.BucketName := project.BucketName
			fullParsedPath := fmt.Sprintf("extra_specs/%s/openapi.json", id)
			//gs://shuffler.appspot.com/extra_specs/0373ed696a3a2cba0a2b6838068f2b80
			//log.Printf("[DEBUG] Couldn't find openapi for %s. Checking filepath gs://%s/%s (size too big). Error: %s", id, project.BucketName, fullParsedPath, err)

			client, err := storage.NewClient(ctx)
			if err != nil {
				log.Printf("[WARNING] Failed to create client (storage - algolia img): %s", err)
				return *api, err
			}

			bucket := client.Bucket(project.BucketName)
			obj := bucket.Object(fullParsedPath)
			fileReader, err := obj.NewReader(ctx)
			if err != nil {
				//log.Printf("[ERROR] Failed making OpenAPI reader for %s: %s", fullParsedPath, err)
				return *api, err
			}

			data, err := ioutil.ReadAll(fileReader)
			if err != nil {
				log.Printf("[WARNING] Failed reading from filereader: %s", err)
				return *api, err
			}

			err = json.Unmarshal(data, &api)
			if err != nil {
				log.Printf("[WARNING] Failed unmarshaling from remote store: %s", err)
				return *api, err
			}

			defer fileReader.Close()
		}
	}

	// Can we diff here? Otherwise we may miss items hmm
	// Check if we recently cached the ID. Don't run updates more often than once a day for an app
	checkCacheId := fmt.Sprintf("openapi_updatecheck_%s", id)
	if _, err := GetCache(ctx, checkCacheId); err != nil {
		api = syncAppContentLabels(ctx, id, api)

		// Set a cache to not do this again for a day
		SetCache(ctx, checkCacheId, []byte("1"), 1440)
	}

	if project.CacheDb {
		data, err := json.Marshal(api)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling openapi: %s", err)
			return *api, nil
		}

		err = SetCache(ctx, cacheKey, data, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed updating openapi cache: %s", err)
		}
	}

	return *api, nil
}

// Index = Username
func SetSession(ctx context.Context, user User, value string) error {
	//parsedKey := strings.ToLower(user.Username)
	// Non indexed User data
	parsedKey := user.Id
	user.Session = value

	nameKey := "Users"
	if project.DbType == "opensearch" {
		data, err := json.Marshal(user)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling user: %s", err)
			return err
		}

		//log.Printf("SESSION RES: %s", res)
		err = indexEs(ctx, nameKey, parsedKey, data)
		if err != nil {
			log.Printf("[WARNING] Failed updating user with session: %s", err)
			return err
		}
	} else {
		key1 := datastore.NameKey(nameKey, parsedKey, nil)
		if _, err := project.Dbclient.Put(ctx, key1, &user); err != nil {
			log.Printf("[WARNING] Error adding Usersession: %s", err)
			return err
		}
	}

	if len(user.Session) > 0 {
		// Indexed session data
		sessiondata := new(Session)
		sessiondata.UserId = strings.ToLower(user.Id)
		sessiondata.Username = strings.ToLower(user.Username)
		sessiondata.Session = user.Session
		sessiondata.Id = user.Id
		nameKey = "sessions"

		if project.DbType == "opensearch" {
			data, err := json.Marshal(sessiondata)
			if err != nil {
				log.Printf("[WARNING] Failed marshalling session %s", err)
				return err
			}

			err = indexEs(ctx, nameKey, sessiondata.Session, data)
			if err != nil {
				return err
			}
		} else {
			key2 := datastore.NameKey(nameKey, sessiondata.Session, nil)
			if _, err := project.Dbclient.Put(ctx, key2, sessiondata); err != nil {
				log.Printf("Error adding session: %s", err)
				return err
			}
		}
	}

	return nil
}

func FindWorkflowByName(ctx context.Context, name string) ([]Workflow, error) {
	var workflows []Workflow

	if project.DbType == "opensearch" {
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"name": name,
				},
			},
		}

		var buf bytes.Buffer
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return workflows, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix("workflow"))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return workflows, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflows named): %s", err)
			return workflows, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return workflows, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return workflows, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return workflows, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return workflows, err
		}

		wrapped := WorkflowSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return workflows, err
		}

		for _, hit := range wrapped.Hits.Hits {
			workflows = append(workflows, hit.Source)
		}
	} else {
		q := datastore.NewQuery("workflow").Filter("name =", name).Limit(100)

		_, err := project.Dbclient.GetAll(ctx, q, &workflows)
		if err != nil && len(workflows) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return []Workflow{}, err
			}
		}
	}

	return workflows, nil
}

func FindWorkflowAppByName(ctx context.Context, appName string) ([]WorkflowApp, error) {
	var apps []WorkflowApp

	nameKey := "workflowapp"
	cacheKey := fmt.Sprintf("%s_appname_%s", nameKey, appName)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &apps)
			if err == nil {
				return apps, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for user: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"name": appName,
				},
			},
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find app query: %s", err)
			return apps, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return apps, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (find app by name): %s", err)
			return apps, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return apps, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return apps, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return apps, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return apps, err
		}

		wrapped := AppSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return apps, err
		}

		apps = []WorkflowApp{}
		for _, hit := range wrapped.Hits.Hits {
			apps = append(apps, hit.Source)
		}
	} else {
		//log.Printf("Looking for name %s in %s", appName, nameKey)
		q := datastore.NewQuery(nameKey).Filter("Name =", appName).Limit(6)
		_, err := project.Dbclient.GetAll(ctx, q, &apps)
		if err != nil && len(apps) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting apps for name: %s", appName)
				return apps, err
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(apps)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling apps for appname %s: %s", appName, err)
			return apps, nil
		}

		err = SetCache(ctx, cacheKey, data, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed updating cache: %s", err)
		}
	}

	log.Printf("[INFO] Found %d apps for name '%s' in db-connector", len(apps), appName)
	return apps, nil
}

// FindUserBySSOIdentity finds a user by their SSO identity using efficient database queries
// Also validates that the clientID matches the org's configured SSO
func FindUserBySSOIdentity(ctx context.Context, sub, clientID, orgID, email string) (User, error) {
	var emptyUser User

	// Check if Sub is empty - user hasn't connected SSO yet
	if sub == "" {
		return emptyUser, errors.New("connect user account with SSO first")
	}

	if clientID == "" || orgID == "" || email == "" {
		return emptyUser, errors.New("clientID, orgID, and email are all required")
	}

	// Verify the clientID actually matches the org's SSO configuration
	org, err := GetOrg(ctx, orgID)
	if err != nil {
		return emptyUser, fmt.Errorf("failed to get org %s: %w", orgID, err)
	}

	if org.SSOConfig.OpenIdClientId != clientID {
		return emptyUser, fmt.Errorf("clientID %s does not match org's configured SSO client ID %s", clientID, org.SSOConfig.OpenIdClientId)
	}

	// Normalize email for comparison
	normalizedEmail := strings.ToLower(strings.TrimSpace(email))

	nameKey := "Users"
	var users []User

	if project.DbType == "opensearch" {
		// OpenSearch query to find users with matching SSO info
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 10,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						{
							"term": map[string]interface{}{
								"username.keyword": normalizedEmail,
							},
						},
						{
							"nested": map[string]interface{}{
								"path": "sso_infos",
								"query": map[string]interface{}{
									"bool": map[string]interface{}{
										"must": []map[string]interface{}{
											{
												"term": map[string]interface{}{
													"sso_infos.sub.keyword": sub,
												},
											},
											{
												"term": map[string]interface{}{
													"sso_infos.client_id.keyword": clientID,
												},
											},
											{
												"term": map[string]interface{}{
													"sso_infos.org_id.keyword": orgID,
												},
											},
										},
									},
								},
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			return emptyUser, fmt.Errorf("failed to encode opensearch query: %w", err)
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			return emptyUser, fmt.Errorf("opensearch query failed: %w", err)
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode != 200 && res.StatusCode != 201 {
			return emptyUser, fmt.Errorf("opensearch error response: %d", res.StatusCode)
		}

		var r map[string]interface{}
		if err := json.NewDecoder(res.Body).Decode(&r); err != nil {
			return emptyUser, fmt.Errorf("failed to parse opensearch response: %w", err)
		}

		hits, ok := r["hits"].(map[string]interface{})["hits"].([]interface{})
		if !ok {
			return emptyUser, errors.New("no matching user found")
		}

		for _, hit := range hits {
			if source, ok := hit.(map[string]interface{})["_source"]; ok {
				data, _ := json.Marshal(source)
				var user User
				if err := json.Unmarshal(data, &user); err == nil {
					users = append(users, user)
				}
			}
		}
	} else {
		// Datastore query - need to get by email first then validate SSO info
		// (Datastore doesn't support nested queries efficiently)
		q := datastore.NewQuery(nameKey).Filter("Username =", normalizedEmail).Limit(10)
		_, err := project.Dbclient.GetAll(ctx, q, &users)
		if err != nil {
			return emptyUser, fmt.Errorf("datastore query failed: %w", err)
		}

		// Filter users to find exact SSO match
		var matchingUsers []User
		for _, user := range users {
			for _, ssoInfo := range user.SSOInfos {
				if ssoInfo.Sub == sub &&
					ssoInfo.ClientID == clientID &&
					ssoInfo.OrgID == orgID {
					matchingUsers = append(matchingUsers, user)
					break
				}
			}
		}
		users = matchingUsers
	}

	if len(users) == 0 {
		return emptyUser, fmt.Errorf("no user found with Sub=%s, ClientID=%s, OrgID=%s, Email=%s", sub, clientID, orgID, normalizedEmail)
	}

	if len(users) > 1 {
		log.Printf("[CRITICAL] Multiple users found with same SSO identity: Sub=%s, ClientID=%s, OrgID=%s, Email=%s",
			sub, clientID, orgID, normalizedEmail)
		return emptyUser, errors.New("multiple users found with same SSO identity - data integrity issue")
	}

	return users[0], nil
}

func FindGeneratedUser(ctx context.Context, username string) ([]User, error) {
	var users []User

	nameKey := "Users"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"generated_username": username,
				},
			},
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return []User{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []User{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (find user): %s", err)
			return []User{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []User{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return []User{}, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []User{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []User{}, err
		}

		wrapped := UserSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return []User{}, err
		}

		users = []User{}
		for _, hit := range wrapped.Hits.Hits {
			users = append(users, hit.Source)
		}
	} else {
		q := datastore.NewQuery(nameKey).Filter("Username =", username)
		_, err := project.Dbclient.GetAll(ctx, q, &users)
		if err != nil && len(users) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting users for username: %s", username)
				return users, err
			}
		}
	}

	newUsers := []User{}
	parsedUsername := strings.ToLower(strings.TrimSpace(username))
	for _, user := range users {
		if strings.ToLower(strings.TrimSpace(user.GeneratedUsername)) != parsedUsername {
			continue
		}

		newUsers = append(newUsers, user)
	}

	log.Printf("[INFO] Found %d (%d) user(s) for username %s in db-connector", len(newUsers), len(users), username)
	return newUsers, nil
}

func FindUser(ctx context.Context, username string) ([]User, error) {
	var users []User

	nameKey := "Users"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": map[string]interface{}{
						"match": map[string]interface{}{
							"username": username,
						},
					},
				},
			},
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return []User{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []User{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (find user): %s", err)
			return []User{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []User{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return []User{}, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []User{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []User{}, err
		}

		wrapped := UserSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return []User{}, err
		}

		users = []User{}
		for _, hit := range wrapped.Hits.Hits {
			users = append(users, hit.Source)
		}
	} else {
		q := datastore.NewQuery(nameKey).Filter("Username =", username)
		_, err := project.Dbclient.GetAll(ctx, q, &users)
		if err != nil && len(users) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting users for username: %s", username)
				return users, err
			}
		}
	}

	newUsers := []User{}
	parsedUsername := strings.ToLower(strings.TrimSpace(username))
	for _, user := range users {
		if strings.ToLower(strings.TrimSpace(user.Username)) != parsedUsername {
			continue
		}

		newUsers = append(newUsers, user)
	}

	log.Printf("[INFO] Found %d (%d) user(s) for username %s in db-connector", len(newUsers), len(users), username)
	return newUsers, nil
}

func GetUser(ctx context.Context, username string) (*User, error) {
	curUser := &User{}

	parsedKey := strings.ToLower(username)
	cacheKey := fmt.Sprintf("user_%s", parsedKey)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &curUser)
			if err == nil {
				return curUser, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for user: %s", err)
		}
	}

	nameKey := "Users"
	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: parsedKey,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return curUser, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return curUser, errors.New("User doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return curUser, err
		}

		wrapped := UserWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return curUser, err
		}

		curUser = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, parsedKey, nil)
		if err := project.Dbclient.Get(ctx, key, curUser); err != nil {
			// Handles migration of the user
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[DEBUG] Failed loading user %s (this is ok): %s", username, err)
			} else {
				log.Printf("[WARNING] Failed loading user %s - does it have to change? %s", username, err)
				return &User{}, err
			}
			//	curUser.ActiveOrg = OrgMini{
			//		Name: curUser.ActiveOrg.Name,
			//		Id:   curUser.ActiveOrg.Id,
			//		Role: "user",
			//	}

			//	// Updating the user and their org
			//	SetUser(ctx, curUser, false)
			//} else {
			//	log.Printf("[WARNING] Error in Get User: %s", err)
			//	return &User{}, err
			//}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(curUser)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling user: %s", err)
			return curUser, nil
		}

		err = SetCache(ctx, cacheKey, data, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed updating cache: %s", err)
		}
	}

	return curUser, nil
}

func (u *User) GetSSOInfo(orgID string) (SSOInfo, bool) {
	log.Printf("[DEBUG] Getting SSOInfo for user %s and org %s", u.Id, orgID)
	for _, sso := range u.SSOInfos {
		if sso.OrgID == orgID {
			return sso, true
		}
	}
	return SSOInfo{}, false
}

func (u *User) SetSSOInfo(orgID string, ssoInfo SSOInfo) {
	ssoInfo.OrgID = orgID
	for i, sso := range u.SSOInfos {
		if sso.OrgID == orgID {
			u.SSOInfos[i] = ssoInfo
			return
		}
	}
	u.SSOInfos = append(u.SSOInfos, ssoInfo)
}

func (u *User) InitSSOInfos() {
	if u.SSOInfos == nil {
		u.SSOInfos = []SSOInfo{}
	}
}

func SetUser(ctx context.Context, user *User, updateOrg bool) error {
	log.Printf("[INFO] Updating user %s (%s) that has the role %s with %d apps and %d orgs. Org updater: %t", user.Username, user.Id, user.Role, len(user.PrivateApps), len(user.Orgs), updateOrg)
	parsedKey := user.Id

	DeleteCache(ctx, user.ApiKey)
	DeleteCache(ctx, user.ApiKey+user.ActiveOrg.Id)
	DeleteCache(ctx, user.Session)
	DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session))

	if len(user.Username) == 0 {
		log.Printf("[ERROR] Setting user without username: %s. Is this expected?", user.Id)
	}

	if updateOrg {
		user = fixUserOrg(ctx, user)
	}

	nameKey := "Users"
	data, err := json.Marshal(user)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling user: %s", err)
		return nil
	}

	//log.Printf("[INFO] Updating user %s (%s) with data length %d", user.Username, user.Id, len(data))

	// This may cause issues huh
	if len(data) > 1000000 {
		user.PrivateApps = []WorkflowApp{}

		data, err = json.Marshal(user)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling user (2): %s", err)
			return nil
		}
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, parsedKey, data)
		if err != nil {
			return err
		}
	} else {
		if len(user.Regions) == 1 {
			if user.Regions[0] != "https://shuffler.io" {
				user.Regions = append(user.Regions, "https://shuffler.io")
			}
		}

		k := datastore.NameKey(nameKey, parsedKey, nil)
		if _, err := project.Dbclient.Put(ctx, k, user); err != nil {
			log.Printf("[WARNING] Error updating user: %s", err)
			return err
		}

		if len(user.Regions) > 1 {
			go func() {
				log.Printf("[INFO] Propagating user %s in org %s (%s) with region %#v", user.Username, user.ActiveOrg.Name, user.ActiveOrg.Id, user.Regions)
				err = propagateUser(*user, false)
				if err != nil {
					log.Printf("[ERROR] Failed propagating user %s (%s) with region %#v: %s", user.Username, user.Id, user.Regions, err)
				}
			}()
		}
	}

	DeleteCache(ctx, user.ApiKey)
	DeleteCache(ctx, user.Session)
	DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session))
	err = DeleteCache(ctx, fmt.Sprintf("Users_%s", user.ApiKey))
	if err != nil {
		log.Printf("[ERROR] Failed to delete cache for user apikey %s", err)
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("user_%s", parsedKey)

		err = SetCache(ctx, cacheKey, data, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed updating user cache (ID): %s", err)
		}

		cacheKey = fmt.Sprintf("user_%s", strings.ToLower(user.Username))
		err = SetCache(ctx, cacheKey, data, 1440)
		if err != nil {
			log.Printf("[WARNING] Failed updating user cache (username): %s", err)
		}
	}

	return nil
}

func DeleteUsersAccount(ctx context.Context, user *User) error {
	cacheKey := fmt.Sprintf("user_%s", user.Id)

	for _, orgId := range user.Orgs {
		org, err := GetOrg(ctx, orgId)
		if err != nil {
			log.Printf("[WARNING] Error getting org %s in delete user: %s", orgId, err)
			continue
		}

		newUsers := []User{}
		for _, orgUser := range org.Users {
			if orgUser.Id == user.Id {
				continue
			}

			newUsers = append(newUsers, orgUser)
		}
		org.Users = newUsers
		err = SetOrg(ctx, *org, org.Id)
		if err != nil {
			log.Printf("[WARNING] Failed setting org %s (1)", orgId)
		}
	}

	nameKey := "Users"
	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Delete(ctx, opensearchapi.DocumentDeleteReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: user.Id,
		})

		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if debug {
			log.Printf("[DEBUG] Response from OpenSearch deletion: StatusCode=%d", res.StatusCode)
		}

		if res.StatusCode == 404 {
			return errors.New("User doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return err
		}

		wrapped := UserWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, user.Id, nil)
		err := project.Dbclient.Delete(ctx, key)
		if err != nil {
			log.Printf("[Error] deleting from %s from %s: %s", nameKey, user.Id, err)
		}
	}

	DeleteCache(ctx, user.ApiKey)
	DeleteCache(ctx, user.Session)
	DeleteCache(ctx, fmt.Sprintf("session_%s", user.Session))

	return nil
}

// Partners functions
func SetPartner(ctx context.Context, partner *Partner) error {
	if partner == nil {
		return errors.New("partner cannot be nil")
	}

	nameKey := "Partners"
	timeNow := int64(time.Now().Unix())

	// Set created time for new partners
	if partner.Created == 0 {
		partner.Created = timeNow
	}
	// Always update edited time
	partner.Edited = timeNow

	// Create datastore key and save
	k := datastore.NameKey(nameKey, partner.Id, nil)
	_, err := project.Dbclient.Put(ctx, k, partner)
	if err != nil {
		return err
	}

	// Update cache
	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, partner.Id)
		orgCacheKey := fmt.Sprintf("%s_org_%s", nameKey, partner.OrgId)
		partnerData, err := json.Marshal(partner)
		if err == nil {
			SetCache(ctx, cacheKey, partnerData, 30)
			SetCache(ctx, orgCacheKey, partnerData, 30)
		}
	}

	return nil
}

func GetPartnerById(ctx context.Context, id string) (*Partner, error) {
	if id == "" {
		return nil, fmt.Errorf("partner ID cannot be empty")
	}

	nameKey := "Partners"
	partner := &Partner{}
	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cachedData, err := GetCache(ctx, cacheKey)
		if err == nil && cachedData != nil {
			partnerBytes, ok := cachedData.([]byte)
			if ok {
				err = json.Unmarshal(partnerBytes, partner)
				if err == nil {
					return partner, nil
				}
			}
		}
	}

	key := datastore.NameKey(nameKey, id, nil)
	if err := project.Dbclient.Get(ctx, key, partner); err != nil {

		if strings.Contains(err.Error(), `cannot load field`) {
			log.Printf("[ERROR] Error in getting partner (3): %s", err)
			err = nil
		} else {
			return partner, fmt.Errorf("Error getting partner %s: %s", partner.Id, err)
		}
	}

	if project.CacheDb {
		partnerData, err := json.Marshal(partner)
		if err == nil {
			SetCache(ctx, cacheKey, partnerData, 30)
		}
	}

	return partner, nil
}

func GetPartnerByOrgId(ctx context.Context, orgId string) (*Partner, error) {
	if orgId == "" {
		return nil, fmt.Errorf("organization ID cannot be empty")
	}

	nameKey := "Partners"
	partner := &Partner{}

	cacheKey := fmt.Sprintf("%s_org_%s", nameKey, orgId)
	if project.CacheDb {
		cachedData, err := GetCache(ctx, cacheKey)
		if err == nil && cachedData != nil {
			// Cache hit
			partnerBytes, ok := cachedData.([]byte)
			if ok {
				err = json.Unmarshal(partnerBytes, partner)
				if err == nil {
					return partner, nil
				}
			}
		}
	}

	q := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Limit(1)
	var partners []Partner
	_, err := project.Dbclient.GetAll(ctx, q, &partners)
	if err != nil {
		if strings.Contains(err.Error(), `cannot load field`) {
			log.Printf("[ERROR] Error in getting partner (3): %s", err)
			err = nil
		} else {
			return partner, fmt.Errorf("failed to get partner by org_id: %w", err)
		}
	}

	if len(partners) == 0 {
		return nil, fmt.Errorf("no partner found for org_id: %s", orgId)
	}

	partner = &partners[0]

	if project.CacheDb {
		// Cache the result
		partnerData, err := json.Marshal(partner)
		if err == nil {
			SetCache(ctx, cacheKey, partnerData, 30)
		}
	}

	return partner, nil
}

func GetAllPartners(ctx context.Context) ([]Partner, error) {
	nameKey := "Partners"

	// Try to get from cache first
	cacheKey := fmt.Sprintf("%s_all", nameKey)
	if project.CacheDb {
		cachedData, err := GetCache(ctx, cacheKey)
		if err == nil && cachedData != nil {
			// Cache hit
			partnersBytes, ok := cachedData.([]byte)
			if ok {
				var partners []Partner
				err = json.Unmarshal(partnersBytes, &partners)
				if err == nil {
					return partners, nil
				}
			}
		}
	}

	// Cache miss or error, get from datastore
	var partners []Partner
	q := datastore.NewQuery(nameKey)
	_, err := project.Dbclient.GetAll(ctx, q, &partners)
	if err != nil {
		if strings.Contains(err.Error(), `cannot load field`) {
			log.Printf("[ERROR] Error in getting partner (3): %s", err)
			err = nil
		} else {
			return partners, fmt.Errorf("failed to get all partners: %w", err)
		}
	}

	if project.CacheDb {
		// Cache the results
		if len(partners) > 0 {
			partnersData, err := json.Marshal(partners)
			if err == nil {
				SetCache(ctx, cacheKey, partnersData, 30)
			}
		}
	}

	return partners, nil
}

func getDatastoreClient(ctx context.Context, projectID string) (datastore.Client, error) {
	// FIXME - this doesn't work
	//client, err := datastore.NewClient(ctx, projectID, option.WithCredentialsFile(test"))
	client, err := datastore.NewClient(ctx, projectID)
	//client, err := datastore.NewClient(ctx, projectID, option.WithCredentialsFile("test"))
	if err != nil {
		return datastore.Client{}, err
	}

	return *client, nil
}

func fixUserOrg(ctx context.Context, user *User) *User {
	// Made it background due to potential timeouts if this is
	// used in API calls
	ctx = context.Background()

	found := false
	for _, id := range user.Orgs {
		if user.ActiveOrg.Id == id {
			found = true
			break
		}
	}

	if !found && !user.SupportAccess {
		user.Orgs = append(user.Orgs, user.ActiveOrg.Id)
	}

	innerUser := *user
	innerUser.PrivateApps = []WorkflowApp{}
	innerUser.Authentication = []UserAuth{}
	innerUser.Password = ""
	innerUser.Session = ""

	// Might be vulnerable to timing attacks.
	for _, orgId := range user.Orgs {
		if len(orgId) == 0 {
			continue
		}

		go func(orgId string) {
			org, err := GetOrg(ctx, orgId)
			if err != nil {
				if !strings.Contains(err.Error(), "doesn't exist") {
					log.Printf("[WARNING] Error getting org %s in fixUserOrg: %s", orgId, err)
				}

				return
			}

			orgIndex := 0
			userFound := false
			for index, orgUser := range org.Users {
				if orgUser.Id == user.Id {
					orgIndex = index
					userFound = true
					break
				}
			}

			if userFound {
				org.Users[orgIndex] = innerUser
			} else if !user.SupportAccess {
				org.Users = append(org.Users, innerUser)
			} else {
				log.Printf("[DEBUG] Skipping org.Users update for support user %s (%s) in org %s — not an official member", user.Username, user.Id, orgId)
				return
			}

			err = SetOrg(ctx, *org, org.Id)
			if err != nil {
				log.Printf("[WARNING] Failed setting org %s (2)", orgId)
			}
		}(orgId)
	}

	return user
}

func GetAllWorkflowAppAuth(ctx context.Context, orgId string) ([]AppAuthenticationStorage, error) {
	var allworkflowappAuths []AppAuthenticationStorage
	nameKey := "workflowappauth"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &allworkflowappAuths)
			if err == nil || len(allworkflowappAuths) > 0 {
				return allworkflowappAuths, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for app auth: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org_id": orgId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return allworkflowappAuths, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return allworkflowappAuths, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get app auth): %s", err)
			return allworkflowappAuths, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return allworkflowappAuths, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return allworkflowappAuths, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return allworkflowappAuths, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return allworkflowappAuths, err
		}

		wrapped := AppAuthSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return allworkflowappAuths, err
		}

		allworkflowappAuths = []AppAuthenticationStorage{}
		for _, hit := range wrapped.Hits.Hits {
			allworkflowappAuths = append(allworkflowappAuths, hit.Source)
		}
	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id = ", orgId)
		if orgId == "ALL" && project.Environment != "cloud" {
			q = datastore.NewQuery(nameKey)
		}

		_, err := project.Dbclient.GetAll(ctx, q, &allworkflowappAuths)
		if err != nil && len(allworkflowappAuths) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {

				if project.CacheDb {
					data, err := json.Marshal(allworkflowappAuths)
					if err != nil {
						log.Printf("[WARNING] Failed marshalling get app auth (2): %s", err)
						return allworkflowappAuths, nil
					}

					err = SetCache(ctx, cacheKey, data, 10)
					if err != nil {
						log.Printf("[WARNING] Failed updating get app auth cache (2): %s", err)
					}
				}

				return allworkflowappAuths, err
			}
		}
	}

	// Should check if it's a child org and get parent orgs app auths that are shared
	foundOrg, err := GetOrg(ctx, orgId)
	if err == nil && len(foundOrg.ChildOrgs) == 0 && len(foundOrg.CreatorOrg) > 0 && foundOrg.CreatorOrg != orgId {

		parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg)
		if err == nil {

			// No recursion as parents can't have parents
			parentAuths, err := GetAllWorkflowAppAuth(ctx, parentOrg.Id)
			if err == nil {
				for _, parentAuth := range parentAuths {
					if !parentAuth.SuborgDistributed && !ArrayContains(parentAuth.SuborgDistribution, orgId) {
						continue
					}

					allworkflowappAuths = append(allworkflowappAuths, parentAuth)
				}
			}
		}
	}

	// Deduplicate keys
	for _, auth := range allworkflowappAuths {
		allFields := []string{}
		newFields := []AuthenticationStore{}
		for _, field := range auth.Fields {
			if ArrayContains(allFields, field.Key) {
				continue
			}

			allFields = append(allFields, field.Key)
			newFields = append(newFields, field)
		}

		auth.Fields = newFields
	}

	if project.CacheDb {
		data, err := json.Marshal(allworkflowappAuths)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling get app auth: %s", err)
			return allworkflowappAuths, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating get app auth cache: %s", err)
		}
	}

	//for _, env := range allworkflowappAuths {
	//	for _, param := range env.Fields {
	//		log.Printf("ENV: %s", param)
	//	}
	//}

	return allworkflowappAuths, nil
}

func GetEnvironments(ctx context.Context, orgId string) ([]Environment, error) {
	//log.Printf("[DEBUG] Getting environments for orgId %s", orgId)
	nameKey := "Environments"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)
	environments := []Environment{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &environments)
			if err == nil {
				//if debug {
				//	log.Printf("[DEBUG] Got %d environments from cache for orgId '%s'", len(environments), orgId)
				//}

				return environments, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache in GET environments: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org_id": orgId,
				},
			},
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return environments, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return environments, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get environments): %s", err)
			return environments, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 && len(orgId) > 0 {
			item := Environment{
				Name:    "Shuffle",
				Type:    "onprem",
				OrgId:   orgId,
				Default: true,
				Id:      uuid.NewV4().String(),
			}

			err = SetEnvironment(ctx, &item)
			if err != nil {
				log.Printf("[WARNING] Failed setting up new environment")
			} else {
				environments = append(environments, item)
			}

			return environments, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return environments, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return environments, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return environments, err
		}

		wrapped := EnvironmentSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return environments, err
		}

		// Ensures we HAVE to match OrgId (somehow) :))
		environments = []Environment{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.OrgId != orgId {
				continue
			}

			environments = append(environments, hit.Source)
		}
	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Limit(10)
		_, err := project.Dbclient.GetAll(ctx, q, &environments)
		if err != nil && len(environments) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {

				if project.CacheDb {
					log.Printf("[INFO] Setting empty cache for environments in org %s", orgId)
					data, err := json.Marshal(environments)
					if err != nil {
						log.Printf("[WARNING] Failed marshalling environment cache (2): %s", err)
						return environments, nil
					}

					err = SetCache(ctx, cacheKey, data, 60)
					if err != nil {
						log.Printf("[WARNING] Failed updating environment cache (2): %s", err)
					}
				}

				return []Environment{}, err
			}
		}

		//log.Printf("Got %d environments for org: %s", len(environments), environments)
	}

	if len(environments) == 0 && len(orgId) > 0 {
		item := Environment{
			Name:    "Shuffle",
			Type:    "onprem",
			OrgId:   orgId,
			Default: true,
			Id:      uuid.NewV4().String(),
		}

		if project.Environment == "cloud" {
			item.Name = "Cloud"
			item.Type = "cloud"
		}

		err := SetEnvironment(ctx, &item)
		if err != nil {
			log.Printf("[WARNING] Failed setting up new environment")
		} else {
			environments = append(environments, item)
		}
	}

	//Check if this is suborg and get parent org environments if it distributed
	if len(orgId) > 0 {
		foundOrg, err := GetOrg(ctx, orgId)
		if err == nil && len(foundOrg.ChildOrgs) == 0 && len(foundOrg.CreatorOrg) > 0 && foundOrg.CreatorOrg != orgId {
			parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg)
			if err == nil {
				parentEnvs, err := GetEnvironments(ctx, parentOrg.Id)
				if err == nil {
					for _, parentEnv := range parentEnvs {
						if !ArrayContains(parentEnv.SuborgDistribution, orgId) {
							continue
						}
						environments = append(environments, parentEnv)
					}
				}
			}
		}
	}

	// Fixing environment return search problems
	timenow := time.Now().Unix()
	for envIndex, env := range environments {
		if env.Name == "Cloud" {
			environments[envIndex].Type = "cloud"
			environments[envIndex].RunType = "cloud"

		} else if env.Name == "Shuffle" {
			environments[envIndex].Type = "onprem"

			if env.RunType == "" {
				environments[envIndex].RunType = "docker"
			}
		} else {
			if environments[envIndex].Type == "" {
				environments[envIndex].Type = "onprem"
			}

			if env.RunType == "" {
				environments[envIndex].RunType = "docker"
			}
		}

		if environments[envIndex].Type == "onprem" {
			if env.Checkin > 0 && timenow-env.Checkin > 90 {
				environments[envIndex].RunningIp = ""
				//environments[envIndex].Licensed = false
			}
		}
	}

	hideEnvs := false
	multiEnvLimit := 0
	if project.Environment == "onprem" {
		if orgId == "" {
			if debug {
				log.Printf("[DEBUG] No orgId provided, skipping multi-env license check")
			}
			return environments, nil
		}

		currentOrg, err := GetOrg(ctx, orgId)
		if err != nil {
			log.Printf("[WARNING] Failed to get current org %s: %v", orgId, err)
			return environments, nil
		}

		parentOrg := currentOrg
		if len(currentOrg.CreatorOrg) > 0 {
			parentOrg, err = GetOrg(ctx, currentOrg.CreatorOrg)
			if err != nil {
				log.Printf("[WARNING] Failed to get parent org %s: %v", currentOrg.CreatorOrg, err)
				parentOrg = currentOrg
			}
		}

		licenseOrg := HandleCheckLicense(ctx, *parentOrg)
		multiEnvLimit = int(licenseOrg.SyncFeatures.MultiEnv.Limit)
		if !licenseOrg.SyncFeatures.MultiEnv.Active && int64(len(environments)) > int64(multiEnvLimit) {
			hideEnvs = true
		}
	}

	if hideEnvs && len(environments) > multiEnvLimit {
		sort.Slice(environments, func(i, j int) bool {
			return environments[i].Created < environments[j].Created
		})

		newEnvs := []Environment{}
		for i, env := range environments {
			if env.Default {
				env.Archived = false
			} else if i < multiEnvLimit {
				env.Archived = false
			} else {
				env.Archived = true
			}
			newEnvs = append(newEnvs, env)
		}
		environments = newEnvs
	}

	//log.Printf("\n\n[DEBUG2] Getting environments2 for orgId %s\n\n", orgId)

	if project.CacheDb {
		data, err := json.Marshal(environments)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling environment cache: %s", err)
			return environments, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating environment cache: %s", err)
		}
	}

	return environments, nil
}

// Gets apps based on a new schema instead of looping everything
// Primarily made for cloud. Load in this order:
// 1. Get ORGs' private apps
// 2. Get USERs' private apps
// 3. Get PUBLIC apps
func GetPrioritizedApps(ctx context.Context, user User) ([]WorkflowApp, error) {
	if project.Environment != "cloud" {
		// Make "body" field a required field if it exists
		allApps, err := GetAllWorkflowApps(ctx, 1000, 0)
		if err != nil {
			return allApps, err
		}

		for appIndex, app := range allApps {
			for actionIndex, action := range app.Actions {
				for paramIndex, param := range action.Parameters {
					if param.Name == "body" {
						allApps[appIndex].Actions[actionIndex].Parameters[paramIndex].Required = true

					}
				}
			}

			if app.Authentication.Type == "oauth2-app" && len(app.Authentication.RedirectUri) > 0 {
				allApps[appIndex].Authentication.Type = "oauth2"
			}
		}

		return allApps, nil
	}

	if user.Username != "HealthWorkflowFunction" {
		//log.Printf("[AUDIT] Getting apps for user '%s' with active org %s", user.Username, user.ActiveOrg.Id)
	}

	// 1. Caching apps locally
	// Make it based on org and not user :)
	allApps := []WorkflowApp{}
	cacheKey := fmt.Sprintf("apps_%s", user.ActiveOrg.Id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &allApps)
			if err == nil {
				return allApps, nil
			} else {
				//log.Println(string(cacheData))
				log.Printf("[ERROR] Failed unmarshaling apps (in cache). Is it stored or mapped together correctly?: %s", err)
				DeleteCache(ctx, cacheKey)
				//log.Printf("[ERROR] DATALEN: %d", len(cacheData))
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for apps with KEY %s: %s", cacheKey, err)
		}
	}

	maxLen := 200
	queryLimit := 25
	cursorStr := ""

	//allApps = user.PrivateApps
	allApps = []WorkflowApp{}
	org, orgErr := GetOrg(ctx, user.ActiveOrg.Id)
	if orgErr == nil && len(org.ActiveApps) > 150 {
		// No reason for it to be this big. Arbitrarily reducing.
		same := []string{}
		samecnt := 0
		for _, activeApp := range org.ActiveApps {
			if ArrayContains(same, activeApp) {
				samecnt += 1
				continue
			}

			same = append(same, activeApp)
		}

		org.ActiveApps = org.ActiveApps[len(org.ActiveApps)-100 : len(org.ActiveApps)-1]
		go SetOrg(ctx, *org, org.Id)
	}

	if len(user.PrivateApps) > 0 && orgErr == nil {
		if debug {
			log.Printf("[INFO] Migrating %d apps for user %s to org %s if they don't exist", len(user.PrivateApps), user.Username, user.ActiveOrg.Id)
		}

		orgChanged := false
		for _, app := range user.PrivateApps {
			if !ArrayContains(org.ActiveApps, app.ID) {
				orgChanged = true
				org.ActiveApps = append(org.ActiveApps, app.ID)
			}
		}

		if orgChanged {
			err := SetOrg(ctx, *org, org.Id)
			if err != nil {
				log.Printf("[WARNING] Failed setting org %s with %d apps: %s", org.Id, len(org.ActiveApps), err)

				if len(org.Users) > 10 {
					newUsers := []User{}
					for _, user := range org.Users {
						if len(user.Id) == 0 {
							continue
						}

						newUsers = append(newUsers, user)
					}

					if len(newUsers) > 0 {
						org.Users = newUsers

						err := SetOrg(ctx, *org, org.Id)
						if err != nil {
							log.Printf("[WARNING] (2) Failed setting org %s with %d apps after cleanup: %s", org.Id, len(org.ActiveApps), err)
						}
					}
				}
			}
		}
	}

	nameKey := "workflowapp"
	var err error
	if user.ActiveOrg.Id != "" {
		query := datastore.NewQuery(nameKey).Filter("reference_org =", user.ActiveOrg.Id).Limit(queryLimit)
		//log.Printf("[INFO] Before ref org search. Org: %s\n\n", user.ActiveOrg.Id)
		maxAmount := 100
		cnt := 0
		for {
			it := project.Dbclient.Run(ctx, query)
			if cnt > maxAmount {
				//log.Printf("[ERROR] Maximum try exceeded for workflowapp (1)")
				break
			}

			for {
				innerApp := WorkflowApp{}
				_, err := it.Next(&innerApp)
				cnt += 1
				if cnt > maxAmount {
					log.Printf("[ERROR] Maximum try exceeded for workflowapp (2)")
					break
				}

				if err != nil {
					//log.Printf("[INFO] Failed fetching results: %v", err)
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
						//log.Printf("[ERROR] Error in reference_org app load of %s (%s): %s.", innerApp.Name, innerApp.ID, err)
					} else {
						//log.Printf("[WARNING] No more apps for %s in org app load? Breaking: %s.", user.Username, err)

						break
					}
				}

				if innerApp.Name == "Shuffle Subflow" {
					continue
				}

				//if orgErr == nil && !ArrayContains(org.ActiveApps, innerApp.ID) {
				//	continue
				//}

				if len(innerApp.Actions) == 0 {
					//log.Printf("[INFO] App %s (%s) doesn't have actions (1) - check filepath", innerApp.Name, innerApp.ID)

					foundApp, err := getCloudFileApp(ctx, innerApp, innerApp.ID)
					if err == nil {
						innerApp = foundApp
					}
				}

				allApps, innerApp = fixAppAppend(allApps, innerApp)
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}

			if len(allApps) > maxLen {
				break
			}
		}
	}

	//for _, app := range allApps {
	//	if strings.Contains(strings.ToLower(app.Name), "tools") {
	//		log.Printf("APP-1: %s:%s (%s) - %s", app.Name, app.AppVersion, app.ID)
	//	}
	//}

	// Find public apps

	appsAdded := []string{}

	// Search for apps with these names, not all public ones
	importantApps := []string{"Shuffle Tools", "http"}

	publicApps := []WorkflowApp{}
	publicAppsKey := fmt.Sprintf("public_apps")
	if project.CacheDb {
		cache, err := GetCache(ctx, publicAppsKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &publicApps)
			if err != nil {
				log.Printf("[WARNING] Failed unmarshaling PUBLIC apps: %s", err)
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for PUBLIC apps: %s", err)
		}
	}

	// May be better to just list all, then set to true?
	// Is this the slow one?
	if len(publicApps) == 0 {
		for _, name := range importantApps {
			query := datastore.NewQuery(nameKey).Filter("Name =", name).Limit(queryLimit)
			//query := datastore.NewQuery(nameKey).Filter("public =", true).Limit(queryLimit)
			for {
				it := project.Dbclient.Run(ctx, query)

				for {
					innerApp := WorkflowApp{}
					_, err := it.Next(&innerApp)
					if err != nil {
						//log.Printf("[WARNING] No more apps (public). Amount found: %d", len(publicApps))

						if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
							//log.Printf("[WARNING] Error in public app load: %s", err)
							//continue
						} else {

							//log.Printf("[WARNING] No more apps (public) - Breaking: %s.", err)
							break
						}
					}

					if innerApp.Name == "Shuffle Subflow" {
						continue
					}

					// Special fix for other regions for these reserved apps
					if innerApp.Public == false {
						continue
					}

					/*
						if innerApp.Public == false && innerApp.Sharing == false && gceProject != "shuffler" && gceProject != sandboxProject && len(gceProject) > 0 {
							if ArrayContains(importantApps, innerApp.Name) {
								innerApp.Public = true
								innerApp.Sharing = true
							} else {
								log.Printf("[INFO] App %s is not public", innerApp.Name)
								continue
							}
						}
					*/

					if len(innerApp.Actions) == 0 {
						foundApp, err := getCloudFileApp(ctx, innerApp, innerApp.ID)
						if err == nil {
							innerApp = foundApp
						}
					}

					allApps, innerApp = fixAppAppend(allApps, innerApp)

					// Validating IF the right app is being appended/updated or not
					//for _, app := range allApps {
					//	if strings.Contains(strings.ToLower(app.Name), "tools") {
					//		log.Printf("APP-INNER: %s:%s (%s) - %s", app.Name, app.AppVersion, app.ID)
					//	}
					//}

				}

				if err != iterator.Done {
				}

				// Get the cursor for the next page of results.
				nextCursor, err := it.Cursor()
				if err != nil {
					log.Printf("[ERROR] Problem with cursor: %s", err)
					break
				} else {
					nextStr := fmt.Sprintf("%s", nextCursor)
					if cursorStr == nextStr {
						break
					}

					cursorStr = nextStr
					query = query.Start(nextCursor)
				}

				if len(allApps) > maxLen {
					break
				}
			}
		}

		newbody, err := json.Marshal(publicApps)
		if err != nil {
			return allApps, nil
		}

		err = SetCache(ctx, publicAppsKey, newbody, 1440)
		if err != nil {
			log.Printf("[INFO] Error setting app cache item for %s: %v", publicAppsKey, err)
		} else {
			//log.Printf("[INFO] Set app cache for %s. Next are private apps.", publicAppsKey)
		}
	}

	if orgErr == nil {
		for _, publicApp := range publicApps {
			if ArrayContains(org.ActiveApps, publicApp.ID) {
				appsAdded = append(appsAdded, publicApp.ID)
				allApps = append(allApps, publicApp)
			}
		}
	}

	//for _, app := range allApps {
	//	if strings.Contains(strings.ToLower(app.Name), "tools") {
	//		log.Printf("APP-2: %s:%s (%s) - %s", app.Name, app.AppVersion, app.ID)
	//	}
	//}

	// PS: If you think there's an error here, it's probably in the Algolia upload of CloudSpecific
	// Instead loading in all public apps which is shared between all orgs
	// This should make the request fast for everyone except that one
	// person who loads it first (or keeps it in cache?)
	if orgErr == nil && len(org.ActiveApps) > 0 {

		allKeys := []*datastore.Key{}
		for _, appId := range org.ActiveApps {
			if ArrayContains(appsAdded, appId) {
				continue
			}

			found := false
			for _, app := range allApps {
				if app.ID == appId {
					found = true
					break
				}
			}

			if !found && len(appId) > 0 {
				allKeys = append(allKeys, datastore.NameKey(nameKey, appId, nil))
			}
		}

		keyLists := [][]*datastore.Key{}
		// Split into 10 each
		for i := 0; i < len(allKeys); i += 5 {
			end := i + 5
			if end > len(allKeys) {
				end = len(allKeys)
			}

			keyLists = append(keyLists, allKeys[i:end])
		}

		// Goroutine each list, then put them back together
		newApps := []WorkflowApp{}
		allChannels := []chan []WorkflowApp{}
		for _, keyList := range keyLists {
			appChannel := make(chan []WorkflowApp)

			go func(keyList []*datastore.Key) {
				newAppsList := make([]WorkflowApp, len(keyList))
				err = project.Dbclient.GetMulti(ctx, keyList, newAppsList)
				if err != nil {
					//log.Printf("[ERROR] Problem getting org apps for %s: %s. Apps: %d. NOT FATAL", org.Id, err, len(newAppsList))
				}

				appChannel <- newAppsList
			}(keyList)

			allChannels = append(allChannels, appChannel)
		}

		parentOrg := &Org{}
		if len(org.CreatorOrg) > 0 && len(org.ManagerOrgs) == 0 {
			org.ManagerOrgs = []OrgMini{
				OrgMini{
					Id: org.CreatorOrg,
				},
			}
		}

		if len(org.ManagerOrgs) > 0 {
			parentOrg, err = GetOrg(ctx, org.ManagerOrgs[0].Id)
			if err != nil {
				log.Printf("[ERROR] Failed getting parent org %s during app load verification: %s", org.ManagerOrgs[0].Id, err)
			}
		}

		if len(parentOrg.Id) == 0 && len(org.ChildOrgs) > 0 {
			parentOrg = org
		}

		// Waiting until here, as org loading could take a bit too
		for _, appChannel := range allChannels {
			newApps = append(newApps, <-appChannel...)
		}

		notAppendedApps := []string{}
		parsedNewapps := []WorkflowApp{}
		for _, newApp := range newApps {
			if len(newApp.ID) == 0 || len(newApp.Name) == 0 {
				continue
			}

			//if user.SupportAccess {
			//	parsedNewapps = append(parsedNewapps, newApp)
			if newApp.Sharing || newApp.Public || newApp.SharingConfig == "everyone" || newApp.SharingConfig == "public" {
				parsedNewapps = append(parsedNewapps, newApp)
			} else if newApp.Owner == user.ActiveOrg.Id || newApp.Owner == user.Id {
				parsedNewapps = append(parsedNewapps, newApp)
			} else if newApp.ReferenceOrg == user.ActiveOrg.Id {
				parsedNewapps = append(parsedNewapps, newApp)

			} else {
				// FIXME: Parentorg <-> suborg access
				if len(newApp.ReferenceOrg) > 0 {
					orgFound := false
					for _, childOrg := range parentOrg.ChildOrgs {
						if childOrg.Id != newApp.ReferenceOrg {
							continue
						}

						orgFound = true
						//log.Printf("[DEBUG] Found matching org %s in parent org %s", newApp.ReferenceOrg, parentOrg.Id)
						break
					}

					if orgFound {
						parsedNewapps = append(parsedNewapps, newApp)
						continue
					}
				}

				notAppendedApps = append(notAppendedApps, fmt.Sprintf("%s - %s", newApp.Name, newApp.ID))
			}
		}

		if len(notAppendedApps) > 0 {
			//log.Printf("[INFO] Not appended apps (%d) for org %s (%s): %s", len(notAppendedApps), user.ActiveOrg.Name, user.ActiveOrg.Id, strings.Join(notAppendedApps, ", "))
			//log.Printf("[WARNING] %d non-allowed, but activated apps for org %s (%s). Removed.", len(notAppendedApps), user.ActiveOrg.Name, user.ActiveOrg.Id)
		}

		allApps = append(allApps, newApps...)
	}

	//for _, app := range allApps {
	//	if strings.Contains(strings.ToLower(app.Name), "tools") {
	//		log.Printf("APP-3: %s:%s (%s) - %s", app.Name, app.AppVersion, app.ID)
	//	}
	//}

	// Deduplicate (e.g. multiple gmail)
	dedupedApps := []WorkflowApp{}
	for _, app := range allApps {
		found := false
		replaceIndex := -1
		for dedupIndex, dedupApp := range dedupedApps {
			if len(strings.TrimSpace(dedupApp.Name)) == 0 {
				continue
			}

			// Name, owner, ID, parent ID
			if strings.ToLower(dedupApp.Name) == strings.ToLower(app.Name) {
				//log.Printf("[DEBUG] Found duplicate app: %s (%s). Dedup index: %d", app.Name, app.ID, dedupIndex)
				found = true
				replaceIndex = dedupIndex
			}
		}

		if !found {
			dedupedApps = append(dedupedApps, app)
			continue
		}

		// Check if one is referenceOrg not
		if dedupedApps[replaceIndex].ReferenceOrg == user.ActiveOrg.Id {
			continue
		}

		if app.ReferenceOrg == user.ActiveOrg.Id {
			dedupedApps[replaceIndex] = app
			continue
		}

		if app.Edited > dedupedApps[replaceIndex].Edited {
			dedupedApps[replaceIndex] = app
			continue
		}

		// Check if image, and other doesn't have
		if len(dedupedApps[replaceIndex].LargeImage) == 0 && len(app.LargeImage) > 0 {
			log.Printf("[INFO] Replacing deduped app with image in get apps (2): %s", app.Name)
			dedupedApps[replaceIndex] = app
		}
	}

	allApps = dedupedApps
	for appIndex, app := range allApps {
		requiredAuthFields := []WorkflowAppActionParameter{}
		if app.Authentication.Required {
			for _, param := range app.Authentication.Parameters {
				requiredAuthFields = append(requiredAuthFields, WorkflowAppActionParameter{
					Description: param.Description,
					ID:          param.ID,
					Name:        param.Name,
					Example:     param.Example,
					Value:       param.Value,
					Multiline:   param.Multiline,
					Required:    param.Required,
				})
			}
		}

		for actionIndex, action := range app.Actions {
			lastRequiredIndex := -1
			bodyIndex := -1

			authFields := []string{}
			for paramIndex, param := range action.Parameters {
				if param.Configuration {
					authFields = append(authFields, param.Name)
				}

				if param.Required {
					lastRequiredIndex = paramIndex
				}

				if param.Name == "body" {
					allApps[appIndex].Actions[actionIndex].Parameters[paramIndex].Required = true
					bodyIndex = paramIndex
				}

				if param.Name == "headers" {
					// Make a newline between all headers based on knownHeaders
					// or just rewrite because lol
					if strings.Count(strings.ToLower(param.Value), "content-type") > 1 {
						allApps[appIndex].Actions[actionIndex].Parameters[paramIndex].Value = "Content-Type=application/json\nAccept=application/json"
					}

					if strings.Contains(strings.ToLower(param.Value), "accept") && strings.Contains(strings.ToLower(param.Value), "application/json") && !strings.Contains(strings.ToLower(param.Value), "content-type") {
						allApps[appIndex].Actions[actionIndex].Parameters[paramIndex].Value = fmt.Sprintf("%s\nContent-Type=application/json", param.Value)
					}
				}
			}

			_ = lastRequiredIndex

			// Add bodyIndex parameter in the next index after lastRequiredIndex, but retain all fields
			if bodyIndex > -1 {
				//log.Printf("[INFO] Moving body parameter to index %d after %d", lastRequiredIndex+1, bodyIndex)
			}

			if len(authFields) < len(requiredAuthFields) {
				if app.Authentication.Type == "oauth2" || app.Authentication.Type == "oauth2-app" {
					continue
				}

				if action.Name == "custom_action" {
					continue
				}

				for _, requiredField := range requiredAuthFields {
					allApps[appIndex].Actions[actionIndex].Parameters = append(allApps[appIndex].Actions[actionIndex].Parameters, requiredField)
				}
			}
		}
	}

	// Also prioritize most used ones from app-framework on top?
	slice.Sort(allApps[:], func(i, j int) bool {
		return allApps[i].Edited > allApps[j].Edited
	})

	//for _, app := range allApps {
	//	if strings.Contains(strings.ToLower(app.Name), "tools") {
	//		log.Printf("APP-4: %s:%s (%s) - %s", app.Name, app.AppVersion, app.ID)
	//	}
	//}

	// Fix Oauth2 issues
	for appIndex, app := range allApps {
		if app.Authentication.Type != "oauth2-app" {
			continue
		}

		if len(app.Authentication.RedirectUri) > 0 {
			allApps[appIndex].Authentication.Type = "oauth2"
		}
	}

	if len(allApps) > 0 {
		// Finds references
		allApps = findReferenceAppDocs(ctx, allApps)

		newbody, err := json.Marshal(allApps)
		if err != nil {
			return allApps, nil
		}

		err = SetCache(ctx, cacheKey, newbody, 1440)
		if err != nil {
			log.Printf("[INFO] Error setting app cache item for %s: %v", cacheKey, err)
		} else {
			//log.Printf("[INFO] Set app cache for %s", cacheKey)
		}
	}

	return allApps, nil
}

func fixAppAppend(allApps []WorkflowApp, innerApp WorkflowApp) ([]WorkflowApp, WorkflowApp) {
	// Hardcoded for certain apps
	if innerApp.Name == "Shuffle Tools" || innerApp.Name == "http" || innerApp.Name == "Shuffle AI" {
		innerApp.Activated = true
	}

	newIndex := -1
	newApp := WorkflowApp{}
	found := false

	for appIndex, loopedApp := range allApps {

		// Check if shuffle subflow and skip
		if strings.ToLower(loopedApp.Name) == "shuffle tools" {
			//log.Printf("%s vs %s - %s vs %s", loopedApp.Name, innerApp.Name, loopedApp.AppVersion, innerApp.AppVersion)
			//continue
		}

		if loopedApp.Name != innerApp.Name {
			continue
		}

		//log.Printf("[DEBUG] Found app %s:%s on index %d", loopedApp.Name, loopedApp.AppVersion, appIndex)

		if ArrayContains(loopedApp.LoopVersions, innerApp.AppVersion) || loopedApp.AppVersion == innerApp.AppVersion {

			if innerApp.Activated && !loopedApp.Activated {
				newIndex = appIndex
				newApp = innerApp

				//newApp.Versions = append(newApp.Versions, AppVersion{
				//	Version: innerApp.AppVersion,
				//	ID:      innerApp.ID,
				//})
				//newApp.LoopVersions = append(newApp.LoopVersions, innerApp.AppVersion)

				//newApp.Versions = loopedApp.Versions
				//newApp.LoopVersions = loopedApp.Versions
				found = false
			} else {
				found = true
			}
		} else {
			//log.Printf("\n\nFound NEW version %s of app %s on index %d\n\n", innerApp.AppVersion, innerApp.Name, appIndex)

			v2, err := semver.NewVersion(innerApp.AppVersion)
			if err != nil {
				log.Printf("[ERROR] Failed parsing original app version %s: %s", innerApp.AppVersion, err)
				continue
			}

			appConstraint := fmt.Sprintf("> %s", loopedApp.AppVersion)
			c, err := semver.NewConstraint(appConstraint)
			if err != nil {
				log.Printf("[ERROR] Failed preparing constraint %s: %s", appConstraint, err)
				continue
			}

			// IF larger, change to this app
			// IF smaller, just append to versions
			if c.Check(v2) {
				newApp = innerApp
				newApp.Versions = loopedApp.Versions
				newApp.LoopVersions = loopedApp.LoopVersions

				//log.Printf("[DEBUG] New IS larger - changing app on index %d from %s to %s. Versions: %s", appIndex, loopedApp.AppVersion, innerApp.AppVersion, newApp.LoopVersions)
			} else {
				//log.Printf("[DEBUG] New is NOT larger: %s_%s (new) vs %s_%s - just appending", innerApp.Name, innerApp.AppVersion, loopedApp.Name, loopedApp.AppVersion)
				newApp = loopedApp
			}

			newApp.Versions = append(newApp.Versions, AppVersion{
				Version: innerApp.AppVersion,
				ID:      innerApp.ID,
			})
			newApp.LoopVersions = append(newApp.LoopVersions, innerApp.AppVersion)
			newIndex = appIndex
			//log.Printf("Versions for %s_%s: %s", newApp.Name, newApp.AppVersion, newApp.LoopVersions)
		}

		break
	}

	if newIndex >= 0 && newApp.ID != "" {
		//log.Printf("Updating app on index %d to be %s:%s instead of %s\n\n", newIndex, newApp.Name, newApp.AppVersion, allApps[newIndex].AppVersion)
		allApps[newIndex] = newApp
	} else {
		if !found {
			innerApp.Versions = append(innerApp.Versions, AppVersion{
				Version: innerApp.AppVersion,
				ID:      innerApp.ID,
			})
			innerApp.LoopVersions = append(innerApp.LoopVersions, innerApp.AppVersion)

			allApps = append(allApps, innerApp)
		}
	}

	return allApps, innerApp
}

func GetUserApps(ctx context.Context, userId string) ([]WorkflowApp, error) {
	wrapper := []WorkflowApp{}
	//var err error

	cacheKey := fmt.Sprintf("userapps-%s", userId)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &wrapper)
			if err == nil {
				return wrapper, nil
			}
		}
	}

	userApps := []WorkflowApp{}
	indexName := "workflowapp"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"should": []map[string]interface{}{
						{
							"match": map[string]interface{}{
								"owner": userId,
							},
						},
						{
							"match": map[string]interface{}{
								"contributors": userId,
							},
						},
					},
					"minimum_should_match": 1,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find workflowapp query: %s", err)
			return []WorkflowApp{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(indexName))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []WorkflowApp{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get apps): %s", err)
			return []WorkflowApp{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []WorkflowApp{}, err
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []WorkflowApp{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []WorkflowApp{}, err
		}

		wrapped := AppSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return []WorkflowApp{}, err
		}

		for _, hit := range wrapped.Hits.Hits {
			innerApp := hit.Source
			userApps = append(userApps, innerApp)
		}

		if len(userApps) > 0 {
			slice.Sort(userApps[:], func(i, j int) bool {
				return userApps[i].Edited > userApps[j].Edited
			})
		}
	} else {

		cursorStr := ""

		log.Printf("[DEBUG] Getting user apps for %s", userId)
		var err error

		queries := []datastore.Query{}
		q := datastore.NewQuery(indexName).Filter("contributors =", userId)

		queries = append(queries, *q)

		q = datastore.NewQuery(indexName).Filter("owner =", userId)
		queries = append(queries, *q)

		cnt := 0
		maxAmount := 100
		for _, tmpQuery := range queries {
			query := &tmpQuery

			if cnt > maxAmount {
				break
			}

			for {
				it := project.Dbclient.Run(ctx, query)
				if cnt > maxAmount {
					break
				}

				for {
					innerApp := WorkflowApp{}
					_, err = it.Next(&innerApp)
					alreadyExists := false
					//log.Printf("Got app: %s (%s)", innerApp.Name, innerApp.ID)
					cnt += 1
					if cnt > maxAmount {
						break
					}

					if err != nil {

						if !strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {

							if !strings.Contains(fmt.Sprintf("%s", err), "no more items") {
								log.Printf("[ERROR] Failed fetching user apps (1): %v", err)
							}

							if strings.Contains("no matching index found", fmt.Sprintf("%s", err)) {
								log.Printf("[ERROR] No more apps for %s in user app load? Breaking: %s.", userId, err)
							} else {
								if !strings.Contains(fmt.Sprintf("%s", err), "no more items") {
									log.Printf("[WARNING] Error in app loading: %s", err)
								}
							}

							break
						}
					}

					if !ArrayContains(innerApp.Contributors, userId) && innerApp.Owner != userId {
						continue
					}

					// Not sure if it actually make the API slower
					for _, app := range userApps {
						if app.ID == innerApp.ID {
							alreadyExists = true
						}
					}

					if !alreadyExists {
						userApps = append(userApps, innerApp)
					}
				}

				if err != nil {
					if !strings.Contains(fmt.Sprintf("%s", err), "no more items") {
						log.Printf("[ERROR] Failed fetching user apps (3): %v", err)
					}

					break
				}

				if err != iterator.Done && err != nil {
					log.Printf("[ERROR] Failed fetching user apps (2): %v", err)
				}

				// Get the cursor for the next page of results.
				nextCursor, err := it.Cursor()
				if err != nil {
					log.Printf("Cursor error: %s", err)
					break

				} else {
					nextStr := fmt.Sprintf("%s", nextCursor)
					if cursorStr == nextStr {
						// Break the loop if the cursor is the same as the previous one
						break
					}

					cursorStr = nextStr
					query = query.Start(nextCursor)
				}
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(userApps)
		if err == nil {
			err = SetCache(ctx, cacheKey, data, 30)
			if err != nil {
				log.Printf("[WARNING] Failed updating cache for execution: %s", err)
			}
		} else {
			log.Printf("[WARNING] Failed marshalling execution: %s", err)
		}
	}

	return userApps, nil
}

func GetAllWorkflowApps(ctx context.Context, maxLen int, depth int) ([]WorkflowApp, error) {
	var allApps []WorkflowApp
	var err error

	// Used for recursion and autocleanup
	if depth > 5 {
		return []WorkflowApp{}, errors.New(fmt.Sprintf("Too deep: max recursion at %d", depth))
	}

	wrapper := []WorkflowApp{}
	cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &wrapper)
			if err == nil {
				return wrapper, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for apps with KEY %s: %s", cacheKey, err)
		}
	}

	nameKey := "workflowapp"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer

		query := map[string]interface{}{
			"size": 1000,
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find workflowapp query: %s", err)
			return []WorkflowApp{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []WorkflowApp{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get apps): %s", err)
			return []WorkflowApp{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []WorkflowApp{}, err
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return []WorkflowApp{}, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []WorkflowApp{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []WorkflowApp{}, err
		}

		wrapped := AppSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return []WorkflowApp{}, err
		}

		allApps = []WorkflowApp{}
		duplicates := map[string][]string{}
		for _, hit := range wrapped.Hits.Hits {
			innerApp := hit.Source
			//if strings.Contains(strings.ToLower(innerApp.Name), "shuffle") {
			//	log.Printf("APP: %s", innerApp.Name)
			//}

			_, found := duplicates[innerApp.Name]
			if found {
				duplicates[innerApp.Name] = append(duplicates[innerApp.Name], innerApp.ID)
			} else {
				duplicates[innerApp.Name] = []string{innerApp.ID}
				//duplicates[innerApp.Name] = append(duplicates[innerApp.Name], innerApp.ID)
			}

			if innerApp.Name == "Shuffle Subflow" {
				continue
			}

			// This is used to validate with ALL apps
			if maxLen == 0 {
				allApps = append(allApps, innerApp)
				continue
			}

			if !innerApp.IsValid {
				log.Printf("[INFO] Skipping invalid app %s (%s)", innerApp.Name, innerApp.ID)
				continue
			}

			allApps, innerApp = fixAppAppend(allApps, innerApp)
		}

		if len(allApps) > 0 {
			slice.Sort(allApps[:], func(i, j int) bool {
				return allApps[i].Edited > allApps[j].Edited
			})
		}

		/*
			deletions := false
			for key, value := range duplicates {
				if len(value) <= 10 {
					continue
				}

				log.Printf("[WARNING] Should delete loads of %s (%d). Cleanup process starting (max 5 recursions)", key, len(value))
				err = DeleteKeys(ctx, "workflowapp", value[0:len(value)-10])
				if err == nil {
					deletions = true
				} else {
					log.Printf("[WARNING] App cleanup failed: %s", err)
				}
			}

			if deletions {
				newAllApps, err := GetAllWorkflowApps(ctx, maxLen, depth+1)
				if err != nil {
					log.Printf("[WARNING] Failed to get subapps after cleanup")
					allApps = newAllApps
				} else {
					allApps = newAllApps
				}
			}
		*/

	} else {
		cursorStr := ""
		query := datastore.NewQuery(nameKey).Order("-edited").Limit(10)
		for {
			it := project.Dbclient.Run(ctx, query)
			//innerApp := WorkflowApp{}
			//data, err := it.Next(&innerApp)

			for {
				innerApp := WorkflowApp{}
				_, err := it.Next(&innerApp)
				if err != nil {
					//log.Printf("No more apps? Breaking: %s.", err)
					break
				}

				if innerApp.Name == "Shuffle Subflow" {
					continue
				}

				if !innerApp.IsValid {
					continue
				}

				allApps, innerApp = fixAppAppend(allApps, innerApp)
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}

			if len(allApps) > maxLen && maxLen != 0 {
				break
			}
		}
	}

	slice.Sort(allApps[:], func(i, j int) bool {
		return allApps[i].Edited > allApps[j].Edited
	})

	if project.CacheDb {
		data, err := json.Marshal(allApps)
		if err == nil {
			err = SetCache(ctx, cacheKey, data, 30)
			if err != nil {
				log.Printf("[WARNING] Failed updating cache for execution: %s", err)
			}
		} else {
			log.Printf("[WARNING] Failed marshalling execution: %s", err)
		}
	}

	return allApps, nil
}

func SetWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, env string) error {
	env = strings.ReplaceAll(env, " ", "-")
	nameKey := fmt.Sprintf("workflowqueue-%s", env)

	// Onprem indexing: workflowqueue-%s -> workflowqueue-environmentname
	// Cloud: workflowqueue-%s-%s -> workflowqueue-environmentname-orgid
	if executionRequest.ExecutionId == "" {
		executionRequest.ExecutionId = uuid.NewV4().String()
	}

	if executionRequest.CreatedAt == 0 {
		executionRequest.CreatedAt = time.Now().Unix()
	}

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		data, err := json.Marshal(executionRequest)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in setworkflow: %s", err)
			return nil
		}

		nameKey = strings.ToLower(nameKey)
		err = indexEs(ctx, nameKey, executionRequest.ExecutionId, data)
		if err != nil {
			return err
		}
	} else {
		//log.Printf("[DEBUG] Adding execution to queue: %s", nameKey)

		key := datastore.NameKey(nameKey, executionRequest.ExecutionId, nil)
		if _, err := project.Dbclient.Put(ctx, key, &executionRequest); err != nil {
			log.Printf("[WARNING] Error adding workflow queue: %s", err)
			return err
		}
	}

	return nil
}

func GetWorkflowQueue(ctx context.Context, id string, limit int, inputEnv ...Environment) (ExecutionRequestWrapper, error) {
	id = strings.ReplaceAll(id, " ", "-")
	nameKey := fmt.Sprintf("workflowqueue-%s", id)
	executions := []ExecutionRequest{}

	// workflowqueue-new-service-test_7e9b9007-5df2-4b47-bca5-c4d267ef2943

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": limit,
			"sort": map[string]interface{}{
				"priority": map[string]interface{}{
					"order": "desc",
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return ExecutionRequestWrapper{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return ExecutionRequestWrapper{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflow queue): %s", err)
			return ExecutionRequestWrapper{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()

		// Here in case of older executions. Should work itself out long-term with
		// priority sorting
		if res.StatusCode == 400 {
			query = map[string]interface{}{
				"from": 0,
				"size": limit,
			}

			if err := json.NewEncoder(&buf).Encode(query); err != nil {
				log.Printf("[WARNING] Error encoding find user query: %s", err)
				return ExecutionRequestWrapper{}, err
			}

			resp, err = project.Es.Search(ctx, &opensearchapi.SearchReq{
				Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
				Body:    &buf,
				Params: opensearchapi.SearchParams{
					TrackTotalHits: true,
				},
			})
			if err != nil {
				if strings.Contains(err.Error(), "index_not_found_exception") {
					log.Printf("[ERROR] Error getting response from Opensearch (get workflow queue): %s", err)
					return ExecutionRequestWrapper{}, nil
				}

				return ExecutionRequestWrapper{}, err
			}

			defer res.Body.Close()
		}

		if res.StatusCode == 404 {
			return ExecutionRequestWrapper{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return ExecutionRequestWrapper{}, err
			} else {
				// Check if "error" key exists and is of the expected type
				if errInfo, ok := e["error"].(map[string]interface{}); ok {
					log.Printf("[%s] %s: %s",
						res.Status(),
						errInfo["type"],
						errInfo["reason"],
					)
				} else {
					log.Printf("[ERROR] Unexpected error format: %v", e["error"])
				}
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return ExecutionRequestWrapper{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))

		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return ExecutionRequestWrapper{}, err
		}

		wrapped := ExecRequestSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return ExecutionRequestWrapper{}, err
		}

		executions = []ExecutionRequest{}
		for _, hit := range wrapped.Hits.Hits {
			executions = append(executions, hit.Source)
		}
	} else {
		q := datastore.NewQuery(nameKey).Limit(limit)
		_, err := project.Dbclient.GetAll(ctx, q, &executions)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Error getting workflow queue: %s", err)
				return ExecutionRequestWrapper{
					Data: executions,
				}, err
			}
		}
	}

	if project.Environment != "cloud" && len(inputEnv) > 0 && len(executions) > 0 {
		env := inputEnv[0]

		orgId := env.OrgId
		org, err := GetOrg(ctx, orgId)
		if err != nil {
			log.Printf("[ERROR] Failed getting org %s for queue: %s", orgId, err)
			return ExecutionRequestWrapper{
				Data: executions,
			}, nil
		}

		parentOrg := org
		if len(org.CreatorOrg) > 0 {
			parentOrg, err = GetOrg(ctx, org.CreatorOrg)
			if err != nil {
				log.Printf("[ERROR] Failed getting parent org %s for queue: %s", org.CreatorOrg, err)
				return ExecutionRequestWrapper{
					Data: executions,
				}, nil
			}
		}

		licenseOrg := HandleCheckLicense(ctx, *parentOrg)
		stats, err := GetOrgStatistics(ctx, parentOrg.Id)
		if err != nil {
			log.Printf("[ERROR] Failed getting statistics for org %s: %s", parentOrg.Id, err)

			stats.MonthlyAppExecutions = 0
			stats.MonthlyChildAppExecutions = 0
		}

		limit := licenseOrg.SyncFeatures.AppExecutions.Limit
		totalAppExecutions := stats.MonthlyAppExecutions + stats.MonthlyChildAppExecutions

		license := checkNoInternet()
		if license.Valid {
			limit = limit * 2
		}

		shouldSkipRateLimit := false
		if licenseOrg.CloudSync && !license.Valid && licenseOrg.SyncFeatures.AppExecutions.Limit >= 300000 {
			shouldSkipRateLimit = true
		}

		if !shouldSkipRateLimit && totalAppExecutions > limit {
			cacheKey := fmt.Sprintf("org-%s-last-queue-send", orgId)
			currentTime := time.Now().Unix()
			lastSendCache, err := GetCache(ctx, cacheKey)
			if err == nil {
				var lastSendTime int64
				if timeBytes, ok := lastSendCache.([]byte); ok {
					if unmarshallErr := json.Unmarshal(timeBytes, &lastSendTime); unmarshallErr == nil {
						timeSinceLastSend := currentTime - lastSendTime

						if timeSinceLastSend < 60 {
							//log.Printf("[INFO] Rate limiting (1): Org %s exceeded the 10K workflow run quota for non-licensed users (current queued: %d, current month usage: %d). To increase scale, upgrade to an Enterprise license.", orgId, len(executions), totalWorkflowExecutions)
							//executionRequests.Data = []ExecutionRequest{}
							executions = []ExecutionRequest{}
						} else {
							if len(executions) > 1 {
								//log.Printf("[INFO] Rate limiting (2): Org %s exceeded the 10K workflow run quota for non-licensed users (current queued: %d, current month usage: %d). To increase scale, upgrade to an Enterprise license.", orgId, len(executions), totalWorkflowExecutions)
								executions = executions[0:1]
							}

							timeBytes, _ := json.Marshal(currentTime)
							if cacheErr := SetCache(ctx, cacheKey, timeBytes, 1); cacheErr != nil {
								log.Printf("[WARNING] Failed to set rate limiting cache for org %s: %s", orgId, cacheErr)
							}
						}
					}
				}
			} else {

				if len(executions) > 1 {
					log.Printf("[INFO] Rate limiting (3): Org %s exceeded the 25K app run quota for non-licensed users (current queued: %d, current month usage: %d). To increase scale, upgrade to an Enterprise license.", orgId, len(executions), totalAppExecutions)
					executions = executions[0:1]
				}

				timeBytes, _ := json.Marshal(currentTime)
				if cacheErr := SetCache(ctx, cacheKey, timeBytes, 1); cacheErr != nil {
					log.Printf("[WARNING] Failed to set initial rate limiting cache for org %s: %s", orgId, cacheErr)
				}
			}
		}
	}

	return ExecutionRequestWrapper{
		Data: executions,
	}, nil
}

func SetNewValue(ctx context.Context, newvalue NewValue) error {
	nameKey := fmt.Sprintf("app_execution_values")

	if newvalue.Created == 0 {
		newvalue.Created = int64(time.Now().Unix())
	}

	if newvalue.Id == "" {
		newvalue.Id = uuid.NewV4().String()
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(newvalue)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in newValue: %s", err)
		return nil
	}
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, newvalue.Id, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, newvalue.Id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &newvalue); err != nil {
			log.Printf("Error adding newvalue: %s", err)
			return err
		}

	}

	return nil
}

func GetPlatformHealth(ctx context.Context, beforeTimestamp int, afterTimestamp int, limit int) ([]HealthCheckDB, error) {
	nameKey := "platform_health"
	// sort by "updated", and get the first one

	health := []HealthCheckDB{}
	cacheKey := fmt.Sprintf("%s-%d-%d-%d", nameKey, beforeTimestamp, afterTimestamp, limit)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &health)
			if err == nil {
				return health, nil
			} else {
				//log.Printf("[WARNING] Failed collection: %s", err)
			}
		} else {
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"sort": map[string]interface{}{
				"updated": map[string]interface{}{
					"order": "desc",
				},
			},
		}

		if limit != 0 {
			query["size"] = limit
		}

		if beforeTimestamp > 0 || afterTimestamp > 0 {
			query["query"] = map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{},
				},
			}
		}

		if beforeTimestamp > 0 {
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(
				query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]map[string]interface{}),
				map[string]interface{}{
					"range": map[string]interface{}{
						"updated": map[string]interface{}{
							"gt": beforeTimestamp,
						},
					},
				},
			)
		}

		if afterTimestamp > 0 {
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(
				query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]map[string]interface{}),
				map[string]interface{}{
					"range": map[string]interface{}{
						"updated": map[string]interface{}{
							"lt": afterTimestamp,
						},
					},
				},
			)
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return health, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return health, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get latest platform health): %s", err)
			return health, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode != 200 && res.StatusCode != 201 {
			return health, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return health, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return health, err
		}

		wrapped := HealthCheckSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return health, err
		}

		for _, hit := range wrapped.Hits.Hits {
			health = append(health, hit.Source)
		}

	} else {
		q := datastore.NewQuery(nameKey)

		// Modify the query to filter for "before" timestamp.
		if beforeTimestamp != 0 {
			q = q.Filter("Updated >", beforeTimestamp)
		}

		// Modify the query to filter for "after" timestamp.
		if afterTimestamp != 0 {
			q = q.Filter("Updated <", afterTimestamp)
		}

		if limit != 0 {
			//log.Printf("[ERROR] Limiting platform health to %d", limit)
			q = q.Limit(limit)
		}

		q = q.Order("-Updated")

		_, err := project.Dbclient.GetAll(ctx, q, &health)
		if err != nil {
			if strings.Contains(err.Error(), "cannot load field") {
			} else {
				log.Printf("[WARNING] Error getting latest platform health: %s", err)
				return health, err
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(health)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling health: %s", err)
			return health, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating health cache: %s", err)
		}
	}

	return health, nil
}

func SetPlatformHealth(ctx context.Context, health HealthCheckDB) error {
	nameKey := "platform_health"

	// generate random ID
	health.ID = uuid.NewV4().String()

	data, err := json.Marshal(health)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set platform health: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, health.ID, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, health.ID, nil)
		if _, err := project.Dbclient.Put(ctx, key, &health); err != nil {
			log.Printf("[WARNING] Error adding platform health: %s", err)
			return err
		}
	}

	return nil
}

func ListChildWorkflows(ctx context.Context, originalId string) ([]Workflow, error) {
	var workflows []Workflow
	var err error

	nameKey := "workflow"
	cacheKey := fmt.Sprintf("%s_%s_childworkflows", nameKey, originalId)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &workflows)
			if err == nil || len(workflows) > 0 {

				sort.Slice(workflows, func(i, j int) bool {
					return workflows[i].Edited > workflows[j].Edited
				})

				return workflows, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for workflow (3): %s", err)
		}
	}

	parentWorkflow, err := GetWorkflow(ctx, originalId)
	if err != nil {
		//log.Printf("[WARNING] Failed getting parent workflow ID %s: %s. This means we SHOULDN'T load child IDs either.", originalId, err)
		return workflows, err
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"parentorg_workflow": originalId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return workflows, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return workflows, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (Get workflows 2): %s", err)
			return workflows, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return workflows, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return workflows, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return workflows, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return workflows, err
		}

		wrapped := WorkflowSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil && len(wrapped.Hits.Hits) == 0 {
			return workflows, err
		}

		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.ParentWorkflowId != originalId {
				continue
			}

			workflows = append(workflows, hit.Source)
		}
	} else {
		query := datastore.NewQuery(nameKey).Filter("parentorg_workflow =", originalId).Limit(50)
		//if project.Environment != "cloud" {
		//	query = query.Order("-edited")
		//}

		cursorStr := ""
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerWorkflow := Workflow{}
				_, err := it.Next(&innerWorkflow)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
					} else {
						//log.Printf("[WARNING] Workflow iterator issue: %s", err)
						break
					}
				}

				workflows = append(workflows, innerWorkflow)
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	// Sort by edited
	sort.Slice(workflows, func(i, j int) bool {
		return workflows[i].Edited > workflows[j].Edited
	})

	// Reduces it in case the distribution changes
	// Ensures suborg workflows can still exist, but not be shown
	if len(parentWorkflow.SuborgDistribution) > 0 {
		newFiltered := []Workflow{}

		for _, childWf := range workflows {
			found := false
			for _, subflowOrg := range parentWorkflow.SuborgDistribution {
				if childWf.OrgId == subflowOrg {
					found = true
					break
				}
			}

			if !found {
				//log.Printf("\n\n[ERROR] Failed to find child workflow %s org %s in parent %s. Should we delete them?\n\n", childWf.ID, childWf.OrgId, parentWorkflow.ID)
			} else {
				newFiltered = append(newFiltered, childWf)
			}
		}

		workflows = newFiltered
	}

	// Set cache
	if project.CacheDb {
		cacheData, err := json.Marshal(workflows)
		if err != nil {
			return workflows, nil
		}

		err = SetCache(ctx, cacheKey, cacheData, 60)
		if err != nil {
			log.Printf("[ERROR] Failed setting cache for workflow revisions: %s (not critical)", err)
		}
	}

	return workflows, nil
}

func ListWorkflowRevisions(ctx context.Context, originalId string, amount int) ([]Workflow, error) {
	var workflows []Workflow
	var err error

	if amount <= 0 {
		amount = 50
	}

	if amount >= 200 {
		amount = 200
	}

	nameKey := "workflow_revisions"
	cacheKey := fmt.Sprintf("%s_%s_%d", nameKey, originalId, amount)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &workflows)
			if err == nil {

				sort.Slice(workflows, func(i, j int) bool {
					return workflows[i].Edited > workflows[j].Edited
				})

				return workflows, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for workflow (4): %s", err)
		}
	}

	//log.Printf("[AUDIT] Getting workflow revisions for workflow %s.", originalId)
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": amount,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"id": originalId,
				},
			},
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return workflows, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return workflows, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (Get workflows 2): %s", err)
			return workflows, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return workflows, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return workflows, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return workflows, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return workflows, err
		}

		wrapped := WorkflowSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil && len(wrapped.Hits.Hits) == 0 {
			return workflows, err
		}

		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.ID != originalId {
				continue
			}

			workflows = append(workflows, hit.Source)
		}
	} else {
		queryAmount := 20
		if amount < queryAmount {
			queryAmount = amount
		}

		query := datastore.NewQuery(nameKey).Filter("id =", originalId).Limit(queryAmount)
		query = query.Order("-edited")

		iterCount := 0

		cursorStr := ""
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerWorkflow := Workflow{}
				_, err := it.Next(&innerWorkflow)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
					} else {
						//log.Printf("[WARNING] Workflow iterator issue: %s", err)
						break
					}
				}

				iterCount++
				workflows = append(workflows, innerWorkflow)
				if iterCount >= amount {
					break
				}
			}

			if iterCount >= amount {
				break
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	// Sort by edited
	sort.Slice(workflows, func(i, j int) bool {
		return workflows[i].Edited > workflows[j].Edited
	})

	// Deduplicate based on edited time
	filtered := []Workflow{}
	handled := []string{}
	for _, workflow := range workflows {
		if ArrayContains(handled, fmt.Sprintf("%d", workflow.Edited)) {
			continue
		}

		handled = append(handled, fmt.Sprintf("%d", workflow.Edited))
		filtered = append(filtered, workflow)
	}

	// Set cache
	if project.CacheDb {
		cacheData, err := json.Marshal(workflows)
		if err != nil {
			return workflows, nil
		}

		err = SetCache(ctx, cacheKey, cacheData, 60)
		if err != nil {
			log.Printf("[ERROR] Failed setting cache for workflow revisions: %s (not critical)", err)
		}
	}

	return workflows, nil
}

func SetAppRevision(ctx context.Context, app WorkflowApp) error {
	nameKey := "app_revisions"
	timeNow := int64(time.Now().Unix())
	app.Edited = timeNow
	if app.Created == 0 {
		app.Created = timeNow
	}

	actionNames := ""
	for _, action := range app.Actions {
		actionNames += fmt.Sprintf("%s-", action.Name)
	}

	appHashString := fmt.Sprintf("%s_%s_%s", app.Name, app.ID, actionNames)
	hasher := md5.New()
	hasher.Write([]byte(appHashString))
	appHash := hex.EncodeToString(hasher.Sum(nil))
	app.RevisionId = appHash

	// New struct, to not add body, author etc
	data, err := json.Marshal(app)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set app revision: %s", err)
		return nil
	}
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, app.RevisionId, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, app.RevisionId, nil)
		if _, err := project.Dbclient.Put(ctx, key, &app); err != nil {
			log.Printf("[ERROR] Error adding app revision: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, app.RevisionId)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for set app revision '%s': %s", cacheKey, err)
		}

		DeleteCache(ctx, fmt.Sprintf("%s_%s", nameKey, app.ID))
	}

	return nil
}

func SetWorkflowRevision(ctx context.Context, workflow Workflow) error {
	nameKey := "workflow_revisions"
	timeNow := int64(time.Now().Unix())
	workflow.Edited = timeNow
	if workflow.Created == 0 {
		workflow.Created = timeNow
	}

	trimOversizedWorkflowImages(&workflow)

	// Tet ID to be an md5 for name+ID+action+triggers+variables
	// this makes sure overwrites don't happen, and duplicates aren't kept
	// json marshal actions
	actionData, actionerr := json.Marshal(workflow.Actions)
	triggerData, triggererr := json.Marshal(workflow.Triggers)
	variableData, variableerr := json.Marshal(workflow.WorkflowVariables)

	if actionerr != nil || triggererr != nil || variableerr != nil {
		log.Printf("[WARNING] Failed marshalling in set workflow revision: %s", actionerr)
		return nil
	}

	workflowHashString := fmt.Sprintf("%s_%s_%s_%s_%s", workflow.Name, workflow.ID, string(actionData), string(triggerData), string(variableData))
	// md5 of workflowHashString
	hasher := md5.New()
	hasher.Write([]byte(workflowHashString))
	workflowHash := hex.EncodeToString(hasher.Sum(nil))
	workflow.RevisionId = workflowHash

	// New struct, to not add body, author etc
	data, err := json.Marshal(workflow)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set workflow revision: %s", err)
		return nil
	}
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, workflow.RevisionId, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, workflow.RevisionId, nil)
		if _, err := project.Dbclient.Put(ctx, key, &workflow); err != nil {
			log.Printf("[WARNING] Error adding workflow revision: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, workflow.RevisionId)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for set workflow revision '%s': %s", cacheKey, err)
		}

		DeleteCache(ctx, fmt.Sprintf("%s_%s", nameKey, workflow.ID))

		// For workflow revision backups
		go DeleteCache(ctx, fmt.Sprintf("%s_%s_1", nameKey, workflow.ID))
		go DeleteCache(ctx, fmt.Sprintf("%s_%s_200", nameKey, workflow.ID))
		// Actively used keys
		DeleteCache(ctx, fmt.Sprintf("%s_%s_2", nameKey, workflow.ID))
		DeleteCache(ctx, fmt.Sprintf("%s_%s_50", nameKey, workflow.ID))
	}

	return nil
}

func fixPosition(position float64) float64 {
	intValue := int(math.Round(position)) // Convert the float to the nearest integer

	difference := position - float64(intValue)
	difference = math.Abs(difference)

	if difference == 0 {
		//log.Printf("[DEBUG] Position fixed from %s to %s", position, position + 0.001)
		return position + 0.001
	}

	return position
}

func FixWorkflowPosition(ctx context.Context, workflow Workflow) Workflow {
	for index, action := range workflow.Actions {
		workflow.Actions[index].Position.X = fixPosition(float64(action.Position.X))
		workflow.Actions[index].Position.Y = fixPosition(float64(action.Position.Y))

		// Check if no ID
		if action.ID == "" {
			workflow.Actions[index].ID = uuid.NewV4().String()
		}
	}

	for index, comments := range workflow.Comments {
		workflow.Comments[index].Position.X = fixPosition(float64(comments.Position.X))
		workflow.Comments[index].Position.Y = fixPosition(float64(comments.Position.Y))

		if comments.ID == "" {
			workflow.Comments[index].ID = uuid.NewV4().String()
		}
	}

	// Fix branches & triggers
	scheduleNotStarted := ""
	for index, trigger := range workflow.Triggers {
		if trigger.TriggerType == "SCHEDULE" {
			if trigger.Status != "RUNNING" {
				scheduleNotStarted = trigger.ID
			}
		}

		if trigger.ID == "" {
			workflow.Triggers[index].ID = uuid.NewV4().String()
		}
	}

	for index, branch := range workflow.Branches {
		if branch.ID == "" {
			workflow.Branches[index].ID = uuid.NewV4().String()
		}

		if branch.DestinationID == branch.SourceID {
			workflow.Branches = append(workflow.Branches[:index], workflow.Branches[index+1:]...)
		}
	}

	// Check validation if Schedule is started (?)
	if len(scheduleNotStarted) > 0 {
		// Add validation problem
		found := false
		for _, problem := range workflow.Validation.Errors {
			if problem.Type == "SCHEDULE" {
				found = true
				break
			}
		}

		if !found {
			workflow.Validation.Errors = append(workflow.Validation.Errors, ValidationProblem{
				Order:    -1,
				Type:     "SCHEDULE",
				ActionId: scheduleNotStarted,
				Error:    "Schedule not started",
			})
		}
	}

	if len(workflow.Validation.Errors) == 0 {
		workflow.Validation.Errors = []ValidationProblem{}
	}

	if len(workflow.Validation.SubflowApps) == 0 {
		workflow.Validation.SubflowApps = []ValidationProblem{}
	}

	return workflow
}

func SetWorkflow(ctx context.Context, workflow Workflow, id string, optionalEditedSecondsOffset ...int) error {

	if len(workflow.Actions) == 0 && workflow.ExecutionEnvironment == "cloud" {
		log.Printf("[WARNING] No actions in workflow %s. Not saving.", id)
		return errors.New("At least one action required to save")
	}

	// FIXME: Due to a possibility of ID reusage on duplication, we re-randomize ID's IF the workflow is new
	// Due to caching, this is kind of fine.
	nameKey := "workflow"
	id = workflow.ID
	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	foundWorkflow, err := GetWorkflow(ctx, id)
	if (err != nil || foundWorkflow.ID == "") && !workflow.BackgroundProcessing {
		log.Printf("[INFO] Workflow %s doesn't exist, randomizing IDs for Triggers during init", id)

		// Old ID + Org ID as seed -> generate new uuid
		for triggerIndex, trigger := range workflow.Triggers {
			uuidSeed := fmt.Sprintf("%s_%s", trigger.ID, workflow.OrgId)
			newTriggerId := uuid.NewV5(uuid.NamespaceOID, uuidSeed).String()
			for branchIndex, branch := range workflow.Branches {
				if branch.SourceID == trigger.ID {
					workflow.Branches[branchIndex].SourceID = newTriggerId
				}

				if branch.DestinationID == trigger.ID {
					workflow.Branches[branchIndex].DestinationID = newTriggerId
				}
			}

			workflow.Triggers[triggerIndex].ID = newTriggerId
			workflow.Triggers[triggerIndex].Status = "stopped"
		}
	} 

	if err != nil || foundWorkflow.ID == "" {
		if debug { 
			log.Printf("[DEBUG] Creating new workflow with ID %s. Clearing workflow cache.", id)
		}

		DeleteCache(ctx, fmt.Sprintf("%s_%s_workflows", "", workflow.OrgId))
		DeleteCache(ctx, fmt.Sprintf("%s_workflows", workflow.OrgId))
	}

	// Overwriting to be sure these are matching
	// No real point in having id + workflow.ID anymore

	timeNow := int64(time.Now().Unix())
	workflow.Edited = timeNow
	if workflow.Created == 0 {
		workflow.Created = timeNow
	}

	if len(optionalEditedSecondsOffset) > 0 {
		workflow.Edited += int64(optionalEditedSecondsOffset[0])
	}

	// Used for exporting. Should NEVER be stored.
	workflow.Subflows = []Workflow{}

	// Clean up types in subflows
	if len(workflow.Validation.SubflowApps) > 0 {
		for index, _ := range workflow.Validation.SubflowApps {
			// Stops infinite recursion issue for self-contained subflows in export
			if len(workflow.Validation.SubflowApps[index].Type) > 20 {
				workflow.Validation.SubflowApps[index].Type = workflow.Validation.SubflowApps[index].Type[:20] + "_app"
			}
		}
	}

	workflow = FixWorkflowPosition(ctx, workflow)
	trimOversizedWorkflowImages(&workflow)

	// New struct, to not add body, author etc
	data, err := json.Marshal(workflow)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set workflow: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, id, data)
		if err != nil {
			return err
		}
	} else {
		//log.Printf("\n\n[INFO] Adding workflow with ID %s\n\n", id)
		key := datastore.NameKey(nameKey, id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &workflow); err != nil {
			log.Printf("[ERROR] Failed adding workflow with ID %s: %s", id, err)
			return err
		}
	}

	// Handles parent/child workflow relationships
	if len(workflow.ParentWorkflowId) > 0 {
		DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", workflow.ID))
		DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", workflow.ParentWorkflowId))
	}

	if len(workflow.ChildWorkflowIds) > 0 {
		DeleteCache(ctx, fmt.Sprintf("workflow_%s_childworkflows", workflow.ID))
	}

	if project.CacheDb {
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getworkflow '%s': %s", cacheKey, err)
		}

		// Find the key for "workflows_<workflow.org_id>" and update the cache for this one. If it doesn't exist, add it
		// Get the cache for the workflows
		DeleteCache(ctx, fmt.Sprintf("%s_workflows", workflow.OrgId))

		cacheKey = fmt.Sprintf("%s_workflows", workflow.OrgId)
		cache, err := GetCache(ctx, cacheKey)
		if err != nil {
			//log.Printf("[WARNING] Failed getting cache for getworkflow '%s': %s", cacheKey, err)
		} else {
			var workflows []Workflow

			cacheData := []byte(cache.([]uint8))
			//log.Printf("[INFO] Got cache for getworkflow '%s': %s", cacheKey, cacheData)
			DeleteCache(ctx, cacheKey)

			err = json.Unmarshal(cacheData, &workflows)
			if err != nil {
				log.Printf("[WARNING] Failed unmarshalling cache for getworkflow '%s': %s", cacheKey, err)
			} else {

				slice.Sort(workflows[:], func(i, j int) bool {
					return workflows[i].Edited > workflows[j].Edited
				})

				// Find the workflow in the cache
				found := false
				for i, w := range workflows {
					if w.ID == id {
						// Update the cache
						workflows[i] = workflow
						found = true
						break
					}
				}

				if !found {
					// Add it to the cache
					workflows = append(workflows, workflow)
				}

				// Marshal
				workflowsData, err := json.Marshal(workflows)
				if err != nil {
					log.Printf("[WARNING] Failed marshalling cache for getworkflow '%s': %s", cacheKey, err)
				} else {
					err = SetCache(ctx, cacheKey, workflowsData, 30)
					if err != nil {
						log.Printf("[WARNING] Failed setting cache for getworkflow '%s': %s", cacheKey, err)
					}
				}
			}
		}
	}

	return nil
}

func trimOversizedWorkflowImages(workflow *Workflow) {
	if workflow == nil {
		return
	}

	if len(workflow.Image) > 32766 {
		workflow.Image = ""
	}

	for index := range workflow.Actions {
		if shouldStripWorkflowImage(workflow.Actions[index].LargeImage) {
			workflow.Actions[index].LargeImage = ""
		}

		if shouldStripWorkflowImage(workflow.Actions[index].SmallImage) {
			workflow.Actions[index].SmallImage = ""
		}
	}

	for index := range workflow.Triggers {
		if shouldStripWorkflowImage(workflow.Triggers[index].LargeImage) {
			workflow.Triggers[index].LargeImage = ""
		}

		if shouldStripWorkflowImage(workflow.Triggers[index].SmallImage) {
			workflow.Triggers[index].SmallImage = ""
		}
	}
}

func shouldStripWorkflowImage(value string) bool {
	if value == "" {
		return false
	}

	if len(value) > 32766 {
		return true
	}

	return false
}

func SetWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error {
	nameKey := "workflowappauth"
	timeNow := int64(time.Now().Unix())
	if workflowappauth.Created == 0 {
		workflowappauth.Created = timeNow
	}

	workflowappauth.Edited = timeNow
	workflowappauth.App.Actions = []WorkflowAppAction{}

	if len(workflowappauth.Fields) > 500 {
		//log.Printf("[WARNING][%s] Too many fields for app auth: %d", id, len(workflowappauth.Fields))
		newfields := []AuthenticationStore{}

		// Rebuilds all fields
		addedFields := []string{}

		// Run loop backwards due to ordering, as to take last version of all parts
		for i := len(workflowappauth.Fields) - 1; i >= 0; i-- {
			field := workflowappauth.Fields[i]
			if ArrayContains(addedFields, field.Key) {
				continue
			}

			addedFields = append(addedFields, field.Key)
			newfields = append(newfields, field)
		}

		workflowappauth.Fields = newfields

		log.Printf("[INFO][%s] Reduced auth fields for app auth to %d", id, len(workflowappauth.Fields))
	}

	// Will ALWAYS encrypt the values when it's not done already
	// This makes it so just re-saving the auth will encrypt them (next run)

	// Uses OrgId (Database) + Backend (ENV) modifier for the keys.
	// Using created timestamp to ensure it's always unique, even if it's the same key of same app in same org.
	if !workflowappauth.Encrypted {
		setEncrypted := true
		newFields := []AuthenticationStore{}
		for _, field := range workflowappauth.Fields {
			// Custom skip for this
			//if field.Key == "url" {
			//	newFields = append(newFields, field)
			//	continue
			//}

			parsedKey := fmt.Sprintf("%s_%d_%s_%s", workflowappauth.OrgId, workflowappauth.Created, workflowappauth.Label, field.Key)
			newKey, err := HandleKeyEncryption([]byte(field.Value), parsedKey)
			if err != nil {
				//log.Printf("[WARNING] Failed encrypting key '%s': %s", field.Key, err)
				setEncrypted = false
				break
			}

			field.Value = string(newKey)
			newFields = append(newFields, field)
		}

		if setEncrypted {
			//log.Printf("[INFO] Encrypted authentication values as they weren't already encrypted")
			workflowappauth.Fields = newFields
			workflowappauth.Encrypted = true
		}
	}

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		data, err := json.Marshal(workflowappauth)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in set app auth: %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, id, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &workflowappauth); err != nil {
			log.Printf("[ERROR] Error adding workflow app AUTH %s (%s) with %d fields: %s", workflowappauth.Label, workflowappauth.Id, len(workflowappauth.Fields), err)

			return err
		}
	}

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	DeleteCache(ctx, cacheKey)
	cacheKey = fmt.Sprintf("%s_%s", nameKey, workflowappauth.OrgId)
	DeleteCache(ctx, cacheKey)

	for _, dorg := range workflowappauth.SuborgDistribution {
		cacheKey = fmt.Sprintf("%s_%s", nameKey, dorg)
		DeleteCache(ctx, cacheKey)
	}

	return nil
}

func GetAppAuthGroup(ctx context.Context, id string) (*AppAuthenticationGroup, error) {
	authGroup := &AppAuthenticationGroup{}
	nameKey := "workflowappauthgroup"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &authGroup)
			if err == nil && authGroup.Id != "" {
				return authGroup, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for authGroup: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return authGroup, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return authGroup, errors.New("Workflow doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return authGroup, err
		}

		wrapped := AuthGroupWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return authGroup, err
		}

		authGroup = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		if err := project.Dbclient.Get(ctx, key, authGroup); err != nil {
			log.Printf("[WARNING] Error getting workflow app auth group %s: %s", id, err)
			return authGroup, err
		}
	}

	if project.CacheDb && authGroup.Id != "" {
		data, err := json.Marshal(authGroup)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in get auth group: %s", err)
			return authGroup, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for authGroup '%s': %s", cacheKey, err)
		}
	}

	return authGroup, nil
}

func SetAuthGroupDatastore(ctx context.Context, workflowappauthgroup AppAuthenticationGroup, id string) error {
	nameKey := "workflowappauthgroup"
	timeNow := int64(time.Now().Unix())
	if workflowappauthgroup.Created == 0 {
		workflowappauthgroup.Created = timeNow
	}

	data, err := json.Marshal(workflowappauthgroup)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set app auth group: %s", err)
		return err
	}

	workflowappauthgroup.Edited = timeNow

	// Check for uniqueness and organization membership
	newAuth := []AppAuthenticationStorage{}
	removeIds := []string{}
	uniqueIds := make(map[string]bool)
	for _, auth := range workflowappauthgroup.AppAuths {
		// Check uniqueness
		if _, exists := uniqueIds[auth.Id]; exists {
			log.Printf("[WARNING] App auth group %s has duplicate app auth id %s", id, auth.Id)
			//return errors.New("Duplicate app auth id")
			removeIds = append(removeIds, auth.Id)
			continue
		}

		// Fetch real data
		uniqueIds[auth.Id] = true
		realAuth, err := GetWorkflowAppAuthDatastore(ctx, auth.Id)
		if err != nil {
			log.Printf("[WARNING] Failed getting app auth %s for app auth group %s: %s", auth.Id, id, err)
			removeIds = append(removeIds, auth.Id)

			// Remove the app auth from the slice
			//workflowappauthgroup.AppAuths = append(workflowappauthgroup.AppAuths[:index], workflowappauthgroup.AppAuths[index+1:]...)
			continue
		}

		// Update the slice with real data
		//workflowappauthgroup.AppAuths[index] = *realAuth
		auth = *realAuth

		// Check organization membership
		if realAuth.OrgId != workflowappauthgroup.OrgId {
			log.Printf("[WARNING] App auth group %s has app auth id %s that doesn't belong to the same org", id, auth.Id)
			removeIds = append(removeIds, auth.Id)
			continue
		}

		auth.App.SmallImage = ""
		auth.App.LargeImage = ""
		auth.App.Documentation = ""

		for authFieldIndex, _ := range auth.Fields {
			auth.Fields[authFieldIndex].Value = ""
		}

		newAuth = append(newAuth, auth)
	}

	workflowappauthgroup.AppAuths = newAuth

	// Remove the invalid app auths
	for _, removeId := range removeIds {
		for index, auth := range workflowappauthgroup.AppAuths {
			if auth.Id == removeId {
				log.Printf("[WARNING] Removed invalid app auth %s from app auth group %s", removeId, id)
				workflowappauthgroup.AppAuths = append(workflowappauthgroup.AppAuths[:index], workflowappauthgroup.AppAuths[index+1:]...)
				break
			}
		}
	}

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, id, data)
		if err != nil {
			log.Printf("[ERROR] Error adding workflow app AUTH group %s (%s) with %d apps: %s", workflowappauthgroup.Label, workflowappauthgroup.Id, len(workflowappauthgroup.AppAuths), err)
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &workflowappauthgroup); err != nil {
			log.Printf("[ERROR] Error adding workflow app AUTH group %s (%s) with %d apps: %s", workflowappauthgroup.Label, workflowappauthgroup.Id, len(workflowappauthgroup.AppAuths), err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
		err := SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for setusecase: %s", err)
		}

		cacheKey = fmt.Sprintf("%s_%s", nameKey, workflowappauthgroup.OrgId)
		DeleteCache(ctx, cacheKey)
	}

	return nil
}

func SetEnvironment(ctx context.Context, env *Environment) error {
	// clear session_token and API_token for user
	nameKey := "Environments"
	if env.Id == "" {
		env.Id = uuid.NewV4().String()
	}

	timeNow := time.Now().Unix()
	if env.Created == 0 {
		env.Created = timeNow
	}

	env.Edited = timeNow

	if debug {
		// Skip update for cloud env due to it not being necessary past creation
		//if env.Created != timeNow && (item.Name == "Cloud" || item.Type == "cloud") {
		//	return nil
		//}

		//log.Printf("[DEBUG] Setting environment %s (%s) for org '%s'. Checkin: %d", env.Name, env.Id, env.OrgId, env.Checkin)
	}

	data, err := json.Marshal(env)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set env: %s", err)
		return err
	}

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, env.Id, data)
		if err != nil {
			return err
		}
	} else {
		k := datastore.NameKey(nameKey, env.Id, nil)
		if _, err := project.Dbclient.Put(ctx, k, env); err != nil {
			log.Printf("[ERROR] Failed to update environment %s: %s", env.Id, err)
			return err
		}
	}

	// Update it in cache as well
	if project.CacheDb {
		// Both name & ID references are used for orgs
		cacheKey := fmt.Sprintf("%s_%s_%s", nameKey, env.OrgId, env.Id)
		err = SetCache(ctx, cacheKey, data, 10)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for set env '%s': %s", cacheKey, err)
		}

		cacheKey = fmt.Sprintf("%s_%s_%s", nameKey, env.OrgId, env.Name)

		err = SetCache(ctx, cacheKey, data, 10)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for set env '%s': %s", cacheKey, err)
		}

		// This ensures it works onprem WITHOUT an org
		if project.Environment != "cloud" {
			cacheKey2 := fmt.Sprintf("%s__%s", nameKey, env.Name)
			if cacheKey2 != cacheKey {
				err = SetCache(ctx, cacheKey2, data, 10)
				if err != nil {
					log.Printf("[WARNING] Failed setting cache for set env '%s': %s", cacheKey, err)
				}
			}
		}

		// Handles both no orgid AND id
		DeleteCache(ctx, fmt.Sprintf("%s_%s", nameKey, env.OrgId))
		DeleteCache(ctx, fmt.Sprintf("%s_", nameKey))
	}

	return nil
}

func GetScheduleByWorkflowId(ctx context.Context, workflowId string) (*ScheduleOld, error) {
	nameKey := "schedules"
	curSchedule := &ScheduleOld{}
	if project.DbType == "opensearch" {
		return curSchedule, errors.New("Not implemented")
	} else {
		q := datastore.NewQuery(nameKey).Filter("workflow_id =", workflowId).Limit(1)
		tmpSchedules := []ScheduleOld{}
		_, err := project.Dbclient.GetAll(ctx, q, &tmpSchedules)
		if err != nil && len(tmpSchedules) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Error getting schedules for workflow Id: %s", err)
				return curSchedule, err
			}
		}

		if len(tmpSchedules) > 0 {
			curSchedule = &tmpSchedules[0]
		}
	}

	return curSchedule, nil
}

func GetSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) {
	nameKey := "schedules"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, schedulename)
	curUser := &ScheduleOld{}

	schedulename = strings.ToLower(schedulename)
	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: schedulename,
		})

		if err != nil {
			if strings.Contains(err.Error(), "status: 404") || strings.Contains(err.Error(), "not_found") {
				return &ScheduleOld{}, errors.New("Schedule doesn't exist")
			}

			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return &ScheduleOld{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return &ScheduleOld{}, errors.New("Schedule doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return &ScheduleOld{}, err
		}

		wrapped := ScheduleWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return &ScheduleOld{}, err
		}

		curUser = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, schedulename, nil)
		if err := project.Dbclient.Get(ctx, key, curUser); err != nil {
			return &ScheduleOld{}, err
		}

	}

	return curUser, nil
}

func GetHooks(ctx context.Context, OrgId string) ([]Hook, error) {
	hooks := []Hook{}
	nameKey := "hooks"
	OrgId = strings.ToLower(OrgId)

	//FIXME: Implement caching

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org_id": OrgId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return []Hook{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []Hook{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get hooks): %s", err)
			return []Hook{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []Hook{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return []Hook{}, nil
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []Hook{}, fmt.Errorf("Bad statuscode: %d", res.StatusCode)
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []Hook{}, err
		}
		wrapper := AllHooksWrapper{}
		err = json.Unmarshal(respBody, &wrapper)

		if err != nil {
			return []Hook{}, err
		}

		for _, hit := range wrapper.Hits.Hits {
			hook := hit.Source
			hooks = append(hooks, hook)
		}
		return hooks, err

	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id = ", OrgId).Limit(1000)

		_, err := project.Dbclient.GetAll(ctx, q, &hooks)
		if err != nil && len(hooks) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return hooks, err
			}
		}
	}

	return hooks, nil
}

func GetPipelines(ctx context.Context, OrgId string) ([]Pipeline, error) {
	pipelines := []Pipeline{}
	nameKey := "pipelines"
	OrgId = strings.ToLower(OrgId)

	//FIXME: Implement caching

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org_id": OrgId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return []Pipeline{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []Pipeline{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get pipelines): %s", err)
			return []Pipeline{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []Pipeline{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return []Pipeline{}, nil
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []Pipeline{}, fmt.Errorf("bad statuscode: %d", res.StatusCode)

		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []Pipeline{}, err
		}
		wrapper := AllPipelinesWrapper{}
		err = json.Unmarshal(respBody, &wrapper)

		if err != nil {
			return []Pipeline{}, err
		}

		for _, hit := range wrapper.Hits.Hits {
			pipeline := hit.Source
			pipelines = append(pipelines, pipeline)
		}
		return pipelines, err

	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id = ", OrgId).Limit(1000)

		_, err := project.Dbclient.GetAll(ctx, q, &pipelines)
		if err != nil && len(pipelines) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return pipelines, err
			}
		}
	}

	return pipelines, nil
}

func GetSessionNew(ctx context.Context, sessionId string) (User, error) {
	cacheKey := fmt.Sprintf("session_%s", sessionId)
	user := &User{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &user)
			if err == nil && len(user.Id) > 0 {

				return *user, nil
			} else {
				log.Printf("[WARNING] Bad cache for %s: %s", sessionId, err)
				//return *user, errors.New(fmt.Sprintf("Bad cache for %s", sessionId))
			}
		} else {
		}
	}

	// Query for the specific API-key in users
	nameKey := "Users"
	var users []User
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"session": sessionId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return User{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return User{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get api keys): %s", err)
			return User{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return User{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return User{}, nil
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return User{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))

		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return User{}, err
		}

		wrapped := UserSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return User{}, err
		}

		users = []User{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.Session != sessionId {
				continue
			}

			users = append(users, hit.Source)
		}

	} else {
		//log.Printf("[DEBUG] Searching for session %s", sessionId)
		q := datastore.NewQuery(nameKey).Filter("session =", sessionId).Limit(1)
		_, err := project.Dbclient.GetAll(ctx, q, &users)
		if err != nil && len(users) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Error getting session: %s", err)
				return User{}, err
			}
		}
	}

	if len(users) == 0 {
		return User{}, errors.New("No users found for this apikey (1)")
	}

	if project.CacheDb {
		data, err := json.Marshal(users[0])
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getSession: %s", err)
			return User{}, err
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting session cache for user %s: %s", sessionId, err)
		}
	}

	return users[0], nil
}

func GetApikey(ctx context.Context, apikey string) (User, error) {

	// Query for the specific API-key in users
	nameKey := "Users"

	var users []User

	//	cacheKey := fmt.Sprintf("%s_%s", nameKey, apikey)
	//	if project.CacheDb {
	//		cache, err := GetCache(ctx, cacheKey)
	//		if err == nil {
	//			cacheData := []byte(cache.([]uint8))
	//			err = json.Unmarshal(cacheData, &users)
	//			if err == nil && len(users) > 0 {
	//				log.Printf("[DEBUG] Found user apikey cache %s", cacheKey)
	//				return users[0], nil
	//			}
	//		}
	//	}

	if debug {
		log.Printf("[DEBUG] Looking for the API Key pass the cache check %s", project.DbType)
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"apikey": apikey,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return User{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return User{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get api keys): %s", err)
			return User{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return User{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return User{}, nil
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return User{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))

		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return User{}, err
		}

		wrapped := UserSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return User{}, err
		}

		users = []User{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.ApiKey != apikey {
				continue
			}

			users = append(users, hit.Source)
		}

	} else {
		q := datastore.NewQuery(nameKey).Filter("apikey =", apikey).Limit(1)
		_, err := project.Dbclient.GetAll(ctx, q, &users)
		if err != nil && len(users) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Error getting apikey: %s", err)
				//return User{}, err
			}
		}
	}

	if len(users) != 0 {
		//if debug {
		//	log.Printf("[DEBUG] Moving away from getapikey '%s' (%s)", users[0].Username, users[0].Id)
		//}
	}

	//	if project.CacheDb {
	//		userData, err := json.Marshal(users)
	//		if err != nil {
	//			log.Printf("[WARNING] Failed marshalling in getusers apikey: %s", err)
	//			if len(users) > 0 {
	//				return users[0], nil
	//			} else {
	//				return User{}, err
	//			}
	//		}
	//
	//		err = SetCache(ctx, cacheKey, userData, 10)
	//		if err != nil {
	//			log.Printf("[WARNING] Failed setting cache for getusers apikey '%s': %s", cacheKey, err)
	//		}
	//	}

	if len(users) == 0 {
		return User{}, errors.New("No users found for this apikey (2)")
	}

	for _, user := range users {
		if len(user.Username) > 0 && len(user.Id) > 0 {
			return user, nil
		}
	}

	return users[0], nil
}

func savePipelineData(ctx context.Context, pipeline Pipeline) error {
	// assuming IndexRequest can be used as an upsert operation
	nameKey := "pipelines"

	pipelineData, err := json.Marshal(pipeline)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in savePipelineData: %s", err)
		return err
	}
	triggerId := strings.ToLower(pipeline.TriggerId)
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, triggerId, pipelineData)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, triggerId, nil)
		if _, err := project.Dbclient.Put(ctx, key, &pipeline); err != nil {
			log.Printf("[ERROR] failed to add pipeline: %s", err)
			return err
		}
	}

	return nil
}

func GetHook(ctx context.Context, hookId string) (*Hook, error) {
	nameKey := "hooks"
	hookId = strings.ToLower(hookId)
	cacheKey := fmt.Sprintf("%s_%s", nameKey, hookId)

	hook := &Hook{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &hook)
			if err == nil && len(hook.Id) > 0 {
				return hook, nil
			} else {
				if len(hook.Id) == 0 && len(cacheData) > 0 {
					return hook, errors.New(fmt.Sprintf("No good cache for hook %s", hookId))
				}
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for hook: %s", err)
		}
	}
	//log.Printf("DBTYPE: %s", project.DbType)

	var err error
	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: hookId,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return &Hook{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return &Hook{}, errors.New("Hook doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return &Hook{}, err
		}

		wrapped := HookWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return &Hook{}, err
		}

		hook = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, hookId, nil)
		err = project.Dbclient.Get(ctx, key, hook)
		if err != nil {
			//return &Hook{}, err
		}
	}

	if project.CacheDb {
		hookData, hookerr := json.Marshal(hook)
		if hookerr != nil {
			log.Printf("[WARNING] Failed marshalling in gethook: %s", err)
			return hook, err
		}

		cacheerr := SetCache(ctx, cacheKey, hookData, 30)
		if cacheerr != nil {
			log.Printf("[WARNING] Failed setting cache for gethook '%s': %s", cacheKey, err)
		}
	}

	return hook, err
}

func SetHook(ctx context.Context, hook Hook) error {
	nameKey := "hooks"

	// New struct, to not add body, author etc
	hookData, err := json.Marshal(hook)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in setHook: %s", err)
		return nil
	}

	hookId := strings.ToLower(hook.Id)
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, hookId, hookData)
		if err != nil {
			return err
		}
	} else {
		key1 := datastore.NameKey(nameKey, hookId, nil)
		if _, err := project.Dbclient.Put(ctx, key1, &hook); err != nil {
			log.Printf("Error adding hook: %s", err)
			return err
		}
	}

	if project.CacheDb {

		cacheKey := fmt.Sprintf("%s_%s", nameKey, hookId)
		err = SetCache(ctx, cacheKey, hookData, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for hook key '%s': %s", cacheKey, err)
		}
	}

	return nil
}

func GetPipeline(ctx context.Context, triggerId string) (*Pipeline, error) {
	pipeline := &Pipeline{}
	nameKey := "pipelines"

	triggerId = strings.ToLower(triggerId)

	if project.DbType == "opensearch" {

		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: triggerId,
		})
		if err != nil {
			return &Pipeline{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return &Pipeline{}, errors.New("pipeline doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return &Pipeline{}, err
		}

		wrapped := PipelineWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return &Pipeline{}, err
		}

		pipeline = &wrapped.Source
	} else {
		// key := datastore.NameKey(nameKey, triggerId, nil)
		// err := project.Dbclient.Get(ctx, key, pipeline)
		// if err != nil {
		// 	return &Pipeline{}, err
		// }
	}

	return pipeline, nil
}

func GetNotification(ctx context.Context, id string) (*Notification, error) {
	nameKey := "notifications"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	curFile := &Notification{}
	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return &Notification{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return &Notification{}, errors.New("Notification with that ID doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return &Notification{}, err
		}

		wrapped := NotificationWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return &Notification{}, err
		}

		curFile = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, curFile); err != nil {
			return &Notification{}, err
		}

	}

	return curFile, nil
}

func GetAutofixAppLabelsCache(ctx context.Context, app WorkflowApp, label string, keys []string) (WorkflowAppAction, error) {
	nameKey := "auto_fix_app_labels_cache_"
	cacheKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, app.Name, label, strings.Join(keys, "_"))

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			curAppAction := WorkflowAppAction{}
			err = json.Unmarshal(cacheData, &curAppAction)
			if err == nil {
				return curAppAction, nil
			}

			log.Printf("[WARNING] Failed unmarshalling in get autofix app labels cache: %s", err)
			return WorkflowAppAction{}, err
		}
	}

	return WorkflowAppAction{}, errors.New("No cache found")
}

func GetFile(ctx context.Context, id string) (*File, error) {
	nameKey := "Files"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			curFile := &File{}
			err = json.Unmarshal(cacheData, &curFile)
			if err == nil {
				return curFile, nil
			}
		}
	}

	curFile := &File{}
	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return &File{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return &File{}, errors.New("File doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return &File{}, err
		}

		wrapped := FileWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return &File{}, err
		}

		curFile = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, curFile); err != nil {
			return &File{}, err
		}
	}

	if project.CacheDb {
		fileData, err := json.Marshal(curFile)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getfile: %s", err)
			return curFile, nil
		}

		err = SetCache(ctx, cacheKey, fileData, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for file key '%s': %s", cacheKey, err)
		}
	}

	return curFile, nil
}

func SetAutofixAppLabelsCache(ctx context.Context, app WorkflowApp, appAction WorkflowAppAction, label string, keys []string) error {
	nameKey := "auto_fix_app_labels_cache_"
	cacheKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, app.Name, label, strings.Join(keys, "_"))

	if project.CacheDb {
		data, err := json.Marshal(appAction)
		if err != nil {
			log.Printf("[DEBUG] Failed marshalling in set autofix app labels cache: %s", err)
			return err
		}

		err = SetCache(ctx, cacheKey, data, 120)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for autofix app labels cache key '%s': %s", cacheKey, err)
			return err
		}
	}

	return errors.New("No cache found")
}

func SetNotification(ctx context.Context, notification Notification) error {
	// clear session_token and API_token for user
	timeNow := time.Now().Unix()
	if notification.CreatedAt == 0 {
		notification.CreatedAt = timeNow
	}

	notification.UpdatedAt = timeNow
	nameKey := "notifications"
	//log.Printf("SETTING NOTIFICATION: %s", notification)

	if project.DbType == "opensearch" {
		data, err := json.Marshal(notification)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling set notification: %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, notification.Id, data)
		if err != nil {
			return err
		}
	} else {
		k := datastore.NameKey(nameKey, notification.Id, nil)
		if _, err := project.Dbclient.Put(ctx, k, &notification); err != nil {
			log.Println(err)
			return err
		}
	}

	/*
		cacheKey := fmt.Sprintf("%s_%s", nameKey, notification.OrgId)
		DeleteCache(ctx, cacheKey)
		cacheKey = fmt.Sprintf("%s_%s", nameKey, notification.UserId)
		DeleteCache(ctx, cacheKey)
	*/

	return nil
}

func SetFile(ctx context.Context, file File) error {
	// clear session_token and API_token for user
	timeNow := time.Now().Unix()
	file.UpdatedAt = timeNow
	nameKey := "Files"

	if file.CreatedAt == 0 {
		file.CreatedAt = timeNow
	}

	/*
		if !strings.HasPrefix(file.Id, "file_") {
			return errors.New("Invalid file ID. Must start with file_")
		}
	*/

	cacheKey := fmt.Sprintf("%s_%s", nameKey, file.Id)

	if project.DbType == "opensearch" {
		data, err := json.Marshal(file)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling set file: %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, file.Id, data)
		if err != nil {
			return err
		}
	} else {
		k := datastore.NameKey(nameKey, file.Id, nil)
		if _, err := project.Dbclient.Put(ctx, k, &file); err != nil {
			log.Println(err)
			return err
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(file)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in setfile: %s", err)

		} else {
			err = SetCache(ctx, cacheKey, data, 30)
			if err != nil {
				log.Printf("[WARNING] Failed setting cache for set file '%s': %s", cacheKey, err)
			}
		}
	}

	DeleteCache(ctx, fmt.Sprintf("files_%s_%s", file.OrgId, file.Namespace))
	DeleteCache(ctx, fmt.Sprintf("files_%s_", file.OrgId))

	return nil
}

func StoreDisabledRules(ctx context.Context, file DisabledRules) error {

	nameKey := "disabled_rules"

	if project.DbType == "opensearch" {
		data, err := json.Marshal(file)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling set file: %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, "0", data)
		if err != nil {
			return err
		}
	} else {
		k := datastore.NameKey(nameKey, "0", nil)
		if _, err := project.Dbclient.Put(ctx, k, &file); err != nil {
			log.Println(err)
			return err
		}
	}

	return nil
}

func GetDisabledRules(ctx context.Context, orgId string) (*DisabledRules, error) {
	nameKey := "disabled_rules"
	disabledRules := &DisabledRules{}
	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: orgId,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", nameKey, err)
			return disabledRules, nil
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			// Index empty
			//log.Printf("[DEBUG] No disabled rules for org %s. Should auto-index?", orgId)

			return disabledRules, nil
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return disabledRules, err
		}

		wrapped := DisabledHookWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return disabledRules, err
		}

		disabledRules = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, orgId, nil)
		if err := project.Dbclient.Get(ctx, key, disabledRules); err != nil {
			if strings.Contains(err.Error(), "no such entity") {
				//log.Printf("[DEBUG] No disabled rules for org %s. Should auto-index?", orgId)
				return disabledRules, nil
			}

			log.Printf("[WARNING] Error getting disabled for org %s: %s", orgId, err)
			return disabledRules, err
		}
	}

	return disabledRules, nil
}

func StoreSelectedRules(ctx context.Context, TriggerId string, rules SelectedDetectionRules) error {

	nameKey := "selected_rules"

	if project.DbType == "opensearch" {
		data, err := json.Marshal(rules)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling set file: %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, TriggerId, data)
		if err != nil {
			return err
		}
	} else {
		k := datastore.NameKey(nameKey, TriggerId, nil)
		if _, err := project.Dbclient.Put(ctx, k, &rules); err != nil {
			log.Println(err)
			return err
		}
	}

	return nil
}

func GetSelectedRules(ctx context.Context, TriggerId string) (*SelectedDetectionRules, error) {
	nameKey := "selected_rules"
	selectedRules := &SelectedDetectionRules{}
	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: TriggerId,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", nameKey, err)
			return &SelectedDetectionRules{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return &SelectedDetectionRules{}, errors.New("rules doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return &SelectedDetectionRules{}, err
		}

		wrapped := SelectedRulesWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return &SelectedDetectionRules{}, err
		}

		selectedRules = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, TriggerId, nil)
		if err := project.Dbclient.Get(ctx, key, selectedRules); err != nil {
			return &SelectedDetectionRules{}, err
		}
	}

	return selectedRules, nil
}

func GetOrgNotifications(ctx context.Context, orgId string) ([]Notification, error) {
	nameKey := "notifications"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)

	var notifications []Notification
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &notifications)
			if err == nil {
				return notifications, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for org: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"sort": map[string]interface{}{
				"updated_at": map[string]interface{}{
					"order": "desc",
				},
			},
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org_id": orgId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return notifications, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return notifications, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get notifications): %s", err)
			return notifications, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return notifications, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return notifications, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return notifications, err
		}

		if res.StatusCode == 400 {
			//log.Printf("[WARNING] Bad request when getting notifications: %s. Is the index initialised?", respBody)
			return notifications, nil
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return notifications, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		wrapped := NotificationSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return notifications, err
		}

		notifications = []Notification{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.Personal {
				continue
			}

			if hit.Source.OrgId == orgId {
				notifications = append(notifications, hit.Source)
			}
		}

	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Order("-updated_at").Limit(250)
		_, err := project.Dbclient.GetAll(ctx, q, &notifications)

		if err != nil && len(notifications) == 0 {
			data, err := json.Marshal(notifications)
			if err != nil {
				log.Printf("[ERROR] Failed marshalling notification cache (2): %s", err)
				return notifications, nil
			}

			err = SetCache(ctx, cacheKey, data, 5)
			if err != nil {
				log.Printf("[ERROR] Failed updating notification cache (2): %s", err)
			}

			if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
				q = q.Limit(50)
				_, err := project.Dbclient.GetAll(ctx, q, &notifications)
				if err != nil && len(notifications) == 0 {
					return notifications, err
				}
			} else if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
				log.Printf("[INFO] Failed loading SOME notifications - skipping: %s", err)
			} else if strings.Contains(fmt.Sprintf("%s", err), "no matching index found") || strings.Contains(fmt.Sprintf("%s", err), "not ready to serve") {
				log.Printf("[ERROR] Failed loading notifications based on index: %s", err)

				q := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Limit(199)
				_, err := project.Dbclient.GetAll(ctx, q, &notifications)
				if err != nil && len(notifications) == 0 {
					if !strings.Contains(err.Error(), `cannot load field`) {
						return notifications, err
					}
				}

			} else {
				return notifications, err
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(notifications)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling notification cache: %s", err)
			return notifications, nil
		}

		// Set it low, because Notifications are very often being set
		// in certain cases. This means lowering this, will increase cache util
		// while not clearing it on every SetNotification()
		err = SetCache(ctx, cacheKey, data, 5)
		if err != nil {
			log.Printf("[WARNING] Failed updating notification cache: %s", err)
		}
	}

	return notifications, nil
}

func GetUserNotifications(ctx context.Context, userId string) ([]Notification, error) {
	var notifications []Notification

	nameKey := "notifications"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"user_id": userId,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"read": false,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return notifications, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return notifications, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get user notifications): %s", err)
			return notifications, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return notifications, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return notifications, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return notifications, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))

		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return notifications, err
		}

		wrapped := NotificationSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return notifications, err
		}

		//log.Printf("[DEBUG] Have %d notifications for user %s", len(wrapped.Hits.Hits), userId)

		notifications = []Notification{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.UserId == userId {
				notifications = append(notifications, hit.Source)
			}
		}

	} else {
		q := datastore.NewQuery(nameKey).Filter("user_id =", userId).Limit(25)

		_, err := project.Dbclient.GetAll(ctx, q, &notifications)
		if err != nil && len(notifications) == 0 {
			if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
				q = q.Limit(10)
				_, err := project.Dbclient.GetAll(ctx, q, &notifications)
				if err != nil && len(notifications) == 0 {
					return notifications, err
				}
			} else if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
				log.Printf("[INFO] Failed loading SOME notifications - skipping: %s", err)
			} else {
				return notifications, err
			}
		}
	}

	return notifications, nil
}

func GetAllFiles(ctx context.Context, orgId, namespace string) ([]File, error) {
	var files []File

	cacheKey := fmt.Sprintf("files_%s_%s", orgId, namespace)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &files)
			if err == nil {
				return files, nil
			}
		}
	}

	nameKey := "Files"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org_id": orgId,
				},
			},
		}

		if len(namespace) > 0 {
			query = map[string]interface{}{
				"from": 0,
				"size": 1000,
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": []map[string]interface{}{
							map[string]interface{}{
								"match": map[string]interface{}{
									"org_id": orgId,
								},
							},
							map[string]interface{}{
								"match": map[string]interface{}{
									"namespace": namespace,
								},
							},
						},
					},
				},
			}
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return files, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return files, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get files): %s", err)
			return files, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return files, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return files, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return files, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))

		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return files, err
		}

		wrapped := FileSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return files, err
		}

		files = []File{}
		for _, hit := range wrapped.Hits.Hits {
			files = append(files, hit.Source)
		}

	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Order("-created_at").Limit(200)
		if len(namespace) > 0 {
			q = datastore.NewQuery(nameKey).Filter("namespace =", namespace).Filter("org_id =", orgId).Order("-created_at").Limit(200)
		}

		_, err := project.Dbclient.GetAll(ctx, q, &files)
		if err != nil && len(files) == 0 {
			if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
				q = q.Limit(50)
				_, err := project.Dbclient.GetAll(ctx, q, &files)
				if err != nil && len(files) == 0 {
					return []File{}, err
				}
			} else if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
				log.Printf("[INFO] Failed loading SOME files - skipping: %s", err)
			} else {
				log.Printf("[ERROR] Failed loading files: %s", err)
				return []File{}, err
			}
		}

		// Finds extra namespaces in the db if none are specified
		if len(namespace) == 0 {
			foundNamespaces := []string{}
			for _, f := range files {
				if f.OrgId != orgId {
					continue
				}

				if !ArrayContains(foundNamespaces, f.Namespace) {
					foundNamespaces = append(foundNamespaces, f.Namespace)
				}
			}

			var namespaceFiles []File
			namespaceQuery := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Filter("namespace !=", "").Limit(1000)
			_, err = project.Dbclient.GetAll(ctx, namespaceQuery, &namespaceFiles)
			if err != nil {
				if !strings.Contains(err.Error(), `cannot load field`) {
					log.Printf("[ERROR] Failed loading namespace files: %s", err)
					return files, nil
				}
			}

			for _, f := range namespaceFiles {
				if f.OrgId != orgId {
					continue
				}

				if !ArrayContains(foundNamespaces, f.Namespace) {
					foundNamespaces = append(foundNamespaces, f.Namespace)

					files = append(files, f)
				}
			}
		}
	}

	// Should check if it's a child org and get parent orgs files if that is distributed to that child org
	foundOrg, err := GetOrg(ctx, orgId)
	if err == nil && len(foundOrg.ChildOrgs) == 0 && len(foundOrg.CreatorOrg) > 0 && foundOrg.CreatorOrg != orgId {
		parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg)
		if err == nil {
			parentFiles, err := GetAllFiles(ctx, parentOrg.Id, namespace)
			if err == nil {
				for _, f := range parentFiles {
					if !ArrayContains(f.SuborgDistribution, orgId) {
						continue
					}
					files = append(files, f)
				}
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(files)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling file cache: %s", err)
			return files, nil
		}

		err = SetCache(ctx, cacheKey, data, 2)
		if err != nil {
			log.Printf("[WARNING] Failed updating file cache: %s", err)
		}
	}

	return files, nil
}

// Gets a specific auth for an org
func GetWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) {
	nameKey := "workflowappauth"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)

	appAuth := &AppAuthenticationStorage{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &appAuth)
			if err == nil {
				return appAuth, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for org: %s", err)
		}
	}

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return appAuth, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return appAuth, errors.New("App auth doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return appAuth, nil
		}

		wrapped := AppAuthWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return appAuth, nil
		}

		appAuth = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, appAuth); err != nil {
			if !strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
				log.Printf("[ERROR] Failed loading app auth: %s", err)
				return &AppAuthenticationStorage{}, err
			}

			log.Printf("[ERROR] Failed loading app auth fields for auth %s (continue anyway): %s", appAuth.Id, err)
		}
	}

	allFields := []string{}
	newFields := []AuthenticationStore{}
	for _, field := range appAuth.Fields {
		if ArrayContains(allFields, field.Key) {
			continue
		}

		allFields = append(allFields, field.Key)
		newFields = append(newFields, field)
	}

	appAuth.Fields = newFields

	if project.CacheDb {
		data, err := json.Marshal(appAuth)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling app auth cache: %s", err)
			return appAuth, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating app auth cache: %s", err)
		}
	}

	return appAuth, nil
}

func GetAuthGroups(ctx context.Context, orgId string) ([]AppAuthenticationGroup, error) {
	nameKey := "workflowappauthgroup"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)

	appAuths := []AppAuthenticationGroup{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &appAuths)
			if err == nil {
				return appAuths, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for org: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org_id": orgId,
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return appAuths, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return appAuths, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get app auths): %s", err)
			return appAuths, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return appAuths, nil
		}

	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Limit(50)
		_, err := project.Dbclient.GetAll(ctx, q, &appAuths)
		if err != nil && len(appAuths) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return appAuths, err
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(appAuths)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling app auth cache: %s", err)
			return appAuths, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating app auth cache: %s", err)
		}
	}

	return appAuths, nil
}

func GetAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) {
	var schedules []ScheduleOld

	nameKey := "schedules"
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
			"query": map[string]interface{}{
				"match": map[string]interface{}{
					"org": orgId,
				},
			},
		}

		if orgId == "ALL" && project.Environment != "cloud" {
			query = map[string]interface{}{
				"from": 0,
				"size": 1000,
			}
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("Error encoding query: %s", err)
			return schedules, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return schedules, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get schedules): %s", err)
			return schedules, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return schedules, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return schedules, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return schedules, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return schedules, err
		}

		wrapped := ScheduleSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return schedules, err
		}

		schedules = []ScheduleOld{}
		for _, hit := range wrapped.Hits.Hits {
			schedules = append(schedules, hit.Source)
		}

		return schedules, err
	} else {
		q := datastore.NewQuery(nameKey).Filter("org = ", orgId).Limit(50)

		_, err := project.Dbclient.GetAll(ctx, q, &schedules)
		if err != nil && len(schedules) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return schedules, err
			}
		}
	}

	return schedules, nil
}

func GetTriggerAuth(ctx context.Context, id string) (*TriggerAuth, error) {
	nameKey := "trigger_auth"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	triggerauth := &TriggerAuth{}

	id = strings.ToLower(id)
	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return &TriggerAuth{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return &TriggerAuth{}, errors.New("Trigger auth doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return &TriggerAuth{}, err
		}

		wrapped := TriggerAuthWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return &TriggerAuth{}, err
		}

		triggerauth = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, triggerauth); err != nil {
			return &TriggerAuth{}, err
		}
	}

	return triggerauth, nil
}

func SetTriggerAuth(ctx context.Context, trigger TriggerAuth) error {
	nameKey := "trigger_auth"

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		data, err := json.Marshal(trigger)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in set trigger auth: %s", err)
			return err
		}

		err = indexEs(ctx, nameKey, strings.ToLower(trigger.Id), data)
		if err != nil {
			return err
		}
	} else {
		key1 := datastore.NameKey(nameKey, strings.ToLower(trigger.Id), nil)
		if _, err := project.Dbclient.Put(ctx, key1, &trigger); err != nil {
			log.Printf("[ERROR] Error adding trigger auth: %s", err)
			return err
		}
	}

	return nil
}

// Index = Username
func DeleteKeys(ctx context.Context, entity string, value []string) error {
	// Non indexed User data
	if project.DbType == "opensearch" {
		for _, item := range value {
			DeleteKey(ctx, entity, item)
		}
	} else {
		// Tons of helpers to ENSURE the key deletion happens properly
		// This especially prominent for custom "Datastore" keys
		keys := []*datastore.Key{}
		for _, item := range value {
			keys = append(keys, datastore.NameKey(entity, strings.ToLower(item), nil))
			keys = append(keys, datastore.NameKey(entity, item, nil))
			if len(item) > 127 {
				keys = append(keys, datastore.NameKey(entity, strings.ToLower(item[:127]), nil))
			}
		}
		
		// Max 500 at a time => total max keys = 5000
		prevStop := 0
		iter := 0
		finished := false

		maxAmount := 500
		for {
			if iter > 10 || finished {  
				break
			}

			iter += 1
			currentKeys := []*datastore.Key{}
			for cnt, key := range keys {
				if cnt < prevStop {
					continue
				}

				currentKeys = append(currentKeys, key)
				if len(currentKeys) >= 500 {
					prevStop = cnt
					break
				}

				if cnt == len(keys)-1 {
					finished = true
				}
			}

			if len(currentKeys) == 0 {
				break
			}

			err := project.Dbclient.DeleteMulti(ctx, currentKeys)
			if err != nil {
				log.Printf("[ERROR] Failed deleting %d values from '%s': %s", len(value), entity, err)
				return err
			}

			if len(currentKeys) < maxAmount { 
				break
			}
		}
	}

	return nil
}

func GetEnvironmentCount() (int, error) {
	ctx := context.Background()
	q := datastore.NewQuery("Environments").Limit(1)
	count, err := project.Dbclient.Count(ctx, q)
	if err != nil {
		return 0, err
	}

	return count, nil
}

// Used for onprem validation of workflow -> user -> org mapping
func GetAllWorkflows(ctx context.Context) ([]Workflow, error) {
	nameKey := "workflow"

	workflows := []Workflow{}
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding  %s", err)
			return workflows, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return workflows, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflows): %s", err)
			return workflows, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return workflows, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return workflows, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return workflows, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return workflows, err
		}

		wrapped := WorkflowSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return workflows, err
		}

		//log.Printf("Found workflows: %d", len(wrapped.Hits.Hits))
		for _, hit := range wrapped.Hits.Hits {
			workflows = append(workflows, hit.Source)
		}
		return workflows, nil
	}

	return workflows, nil
}

func GetAllUsers(ctx context.Context) ([]User, error) {
	nameKey := "Users"

	users := []User{}
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"from": 0,
			"size": 1000,
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find workflowapp query: %s", err)
			return []User{}, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []User{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get all users): %s", err)
			return []User{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []User{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return []User{}, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []User{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []User{}, err
		}

		wrapped := UserSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return []User{}, err
		}

		users = []User{}
		for _, hit := range wrapped.Hits.Hits {
			users = append(users, hit.Source)
		}

		return users, nil
	} else {
		q := datastore.NewQuery(nameKey).Limit(50)

		_, err := project.Dbclient.GetAll(ctx, q, &users)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return []User{}, err
			}
		}
	}

	return users, nil
}

func GetUnfinishedExecutionsCron(ctx context.Context) (map[string][]WorkflowExecution, int, error) {
	mappedExecutions := make(map[string][]WorkflowExecution)

	nameKey := "workflowexecution"
	var executions []WorkflowExecution
	var err error
	// FIXME: Sorting doesn't seem to work...
	//StartedAt          int64          `json:"started_at" datastore:"started_at"`
	var query *datastore.Query
	query = datastore.NewQuery(nameKey).Filter("started_at >", time.Now().Unix()-60).Order("-started_at").Limit(100000)

	max := 100000
	cursorStr := ""
	for {
		// it := project.dbclient.Run(ctx, query)
		it := project.Dbclient.Run(ctx, query)

		for {
			innerWorkflow := WorkflowExecution{}
			_, err := it.Next(&innerWorkflow)
			if err != nil {
				// log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
				if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
				} else {
					//log.Printf("[WARNING] Workflow iterator issue: %s", err)
					break
				}
			}

			executions = append(executions, innerWorkflow)
		}

		if err != iterator.Done {
			//log.Printf("[INFO] Failed fetching results: %v", err)
			//break
		}

		if len(executions) >= max {
			break
		}

		// Get the cursor for the next page of results.
		nextCursor, err := it.Cursor()
		if err != nil {
			log.Printf("[ERROR] Cursorerror: %s", err)
			break
		} else {
			nextStr := fmt.Sprintf("%s", nextCursor)
			if cursorStr == nextStr {
				break
			}

			cursorStr = nextStr
			query = query.Start(nextCursor)
			//cursorStr = nextCursor
			//break
		}
	}

	newExecutions := []WorkflowExecution{}
	for _, execution := range executions {
		if execution.Workflow.OrgId == "INTERNAL" && execution.Status != "FINISHED" {
			continue
		}

		newExecutions = append(newExecutions, execution)
	}
	executions = newExecutions

	slice.Sort(executions[:], func(i, j int) bool {
		return executions[i].StartedAt > executions[j].StartedAt
	})

	// Gets the correct one from cache to make it appear to be correct everywhere
	for execIndex, execution := range executions {
		if execution.Status != "EXECUTING" {
			continue
		}

		// Get the right one from cache
		newexec, err := GetWorkflowExecution(ctx, execution.ExecutionId)
		if err == nil {
			// Set the execution as well in the database
			// if newexec.Status != execution.Status {

			// 	if project.Environment == "cloud" {
			// 		go SetWorkflowExecution(ctx, *newexec, true)
			// 	} else {
			// 		SetWorkflowExecution(ctx, *newexec, false)
			// 	}
			// }
			if newexec.Status != "EXECUTING" {
				continue
			}

			executions[execIndex] = *newexec
			// mappedExecutions[newexec.Status] = append(mappedExecutions[newexec.Status], *newexec)
		}
	}

	for _, execution := range executions {
		mappedExecutions[execution.Status] = append(mappedExecutions[execution.Status], execution)
	}

	// now, make a COUNT query for the number of notifications
	query = datastore.NewQuery(nameKey).Filter("started_at >", time.Now().Unix()-60)
	notificationCount, err := project.Dbclient.Count(ctx, query)
	if err != nil {
		log.Printf("[ERROR] Failed counting executions: %s", err)
	}

	return mappedExecutions, notificationCount, nil
}

func GetUnfinishedExecutions(ctx context.Context, workflowId string) ([]WorkflowExecution, error) {
	nameKey := "workflowexecution"
	var executions []WorkflowExecution
	var err error
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"sort": map[string]interface{}{
				"started_at": map[string]interface{}{
					"order": "desc",
				},
			},
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"workflow_id": workflowId,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"status": "EXECUTING",
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("Error encoding query: %s", err)
			return executions, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return executions, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflow executions): %s", err)
			return executions, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return executions, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return executions, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return executions, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return executions, err
		}

		wrapped := ExecutionSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return executions, err
		}

		executions = []WorkflowExecution{}
		for _, hit := range wrapped.Hits.Hits {
			executions = append(executions, hit.Source)
		}

		return executions, nil
	} else {
		// FIXME: Sorting doesn't seem to work...
		//StartedAt          int64          `json:"started_at" datastore:"started_at"`
		query := datastore.NewQuery(nameKey).Filter("workflow_id =", workflowId).Limit(10)
		max := 100
		cursorStr := ""
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerWorkflow := WorkflowExecution{}
				_, err := it.Next(&innerWorkflow)
				if err != nil {
					// log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
					} else {
						//log.Printf("[WARNING] Workflow iterator issue: %s", err)
						break
					}
				}

				executions = append(executions, innerWorkflow)
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			if len(executions) >= max {
				break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Cursorerror: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
				//cursorStr = nextCursor
				//break
			}
		}

		slice.Sort(executions[:], func(i, j int) bool {
			return executions[i].StartedAt > executions[j].StartedAt
		})
	}

	newExecutions := []WorkflowExecution{}
	for _, execution := range executions {
		if execution.Workflow.OrgId == "INTERNAL" && execution.Status != "FINISHED" {
			continue
		}

		newExecutions = append(newExecutions, execution)
	}
	executions = newExecutions

	// Gets the correct one from cache to make it appear to be correct everywhere
	for execIndex, execution := range executions {
		if execution.Status != "EXECUTING" {
			continue
		}

		// Get the right one from cache
		newexec, err := GetWorkflowExecution(ctx, execution.ExecutionId)
		if err == nil {
			// Set the execution as well in the database
			if newexec.Status != execution.Status {

				if project.Environment == "cloud" {
					go SetWorkflowExecution(ctx, *newexec, true)
				} else {
					SetWorkflowExecution(ctx, *newexec, false)
				}
			}

			executions[execIndex] = *newexec
		}
	}

	return executions, nil
}

func GetAllWorkflowExecutionsV2(ctx context.Context, workflowId string, amount int, inputcursor string) ([]WorkflowExecution, string, error) {
	nameKey := "workflowexecution"

	var executions []WorkflowExecution
	cacheKey := fmt.Sprintf("%s_%s_%s", nameKey, inputcursor, workflowId)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &executions)
			//if err == nil && len(executions) > 0 {
			if err == nil {
				return executions, "", nil
			}
		}
	}

	var err error
	totalMaxSize := 11184810

	cursor := ""
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": amount,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						{
							"match": map[string]interface{}{
								"workflow_id": workflowId,
							},
						},
					},
				},
			},
			"sort": map[string]interface{}{
				"started_at": map[string]interface{}{
					"order": "desc",
				},
			},
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding executions query: %s", err)
			return executions, cursor, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return executions, cursor, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflow executions): %s", err)
			return executions, cursor, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return executions, cursor, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return executions, cursor, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return executions, cursor, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return executions, cursor, err
		}

		wrapped := ExecutionSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil && len(wrapped.Hits.Hits) == 0 {
			return executions, cursor, err
		}

		executions = []WorkflowExecution{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.WorkflowId == workflowId || hit.Source.Workflow.ID == workflowId {
				executions = append(executions, hit.Source)
			}
		}

	} else {
		query := datastore.NewQuery(nameKey).Filter("workflow_id =", workflowId).Order("-started_at").Limit(5)
		if inputcursor != "" {
			outputcursor, err := datastore.DecodeCursor(inputcursor)
			if err != nil {
				log.Printf("[WARNING] Error decoding cursor: %s", err)
				return executions, "", err
			}

			query = query.Start(outputcursor)
		}

		// Create a timeout to prevent the query from taking more than 5 seconds total

		cursorStr := ""
		maxAmount := 100
		cnt := 0
		for {
			it := project.Dbclient.Run(ctx, query)
			if cnt > maxAmount {
				log.Printf("[ERROR] Error getting workflow execution (4): reached maximum retries")
				break
			}

			breakOuter := false
			for {
				innerWorkflow := WorkflowExecution{}
				_, err := it.Next(&innerWorkflow)
				if cnt > maxAmount {
					log.Printf("[ERROR] Error getting workflow executions (3): reached maximum retries")
					break
				}

				if err != nil {
					if strings.Contains(err.Error(), "context deadline exceeded") {
						log.Printf("[WARNING] Error getting workflow executions (1): %s", err)
						cnt += 1
						breakOuter = true
						break

					} else {
						if strings.Contains(err.Error(), `cannot load field`) {
							// Bug with moving types
							err = nil
						} else if strings.Contains(err.Error(), `no more items`) {
							//breakOuter = true
							break
						} else {
							log.Printf("[WARNING] Error getting workflow executions (2): %s", err)
							break
						}
					}
				}

				executions = append(executions, innerWorkflow)
			}

			if breakOuter {
				break
			}

			if err != iterator.Done {
				//log.Printf("[DEBUG] Breaking due to no more iterator")
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// This is a way to load as much data as we want, and the frontend will load the actual result for us
			executionmarshal, err := json.Marshal(executions)
			if err == nil {
				if len(executionmarshal) > totalMaxSize {
					// Reducing size

					for execIndex, execution := range executions {
						// Making sure the first 5 are "always" proper
						if execIndex < 5 {
							continue
						}

						newResults := []ActionResult{}

						newActions := []Action{}
						for _, action := range execution.Workflow.Actions {
							newAction := Action{
								Name:    action.Name,
								ID:      action.ID,
								AppName: action.AppName,
								AppID:   action.AppID,
							}

							newActions = append(newActions, newAction)
						}

						executions[execIndex].Workflow = Workflow{
							Name:     execution.Workflow.Name,
							ID:       execution.Workflow.ID,
							Triggers: execution.Workflow.Triggers,
							Actions:  newActions,
						}

						for _, result := range execution.Results {
							result.Result = "Result was too large to load. Full Execution needs to be loaded individually for this execution. Click \"Explore execution\" in the UI to see it in detail."
							result.Action = Action{
								Name:       result.Action.Name,
								ID:         result.Action.ID,
								AppName:    result.Action.AppName,
								AppID:      result.Action.AppID,
								LargeImage: result.Action.LargeImage,
							}

							newResults = append(newResults, result)
						}

						executions[execIndex].ExecutionArgument = "too large"
						executions[execIndex].Results = newResults
					}

					executionmarshal, err = json.Marshal(executions)
					if err == nil && len(executionmarshal) > totalMaxSize {
						//log.Printf("Length breaking (2): %d", len(executionmarshal))
						break
					}
				}
			}

			// expected to get here
			if len(executions) >= amount {
				//log.Printf("[INFO] Breaking due to executions larger than amount (%d/%d)", len(executions), amount)
				// Get next cursor
				nextCursor, err := it.Cursor()
				if err != nil {
					log.Printf("[ERROR] Cursorerror: %s", err)
				} else {
					cursor = fmt.Sprintf("%s", nextCursor)
				}

				break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Cursorerror: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				cursor = nextStr
				if cursorStr == nextStr {
					//log.Printf("Breaking due to no new cursor")

					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	newExecutions := []WorkflowExecution{}
	for _, execution := range executions {
		if execution.Workflow.OrgId == "INTERNAL" && execution.Status != "FINISHED" {
			continue
		}

		newExecutions = append(newExecutions, execution)
	}
	executions = newExecutions

	// Find difference between what's in the list and what is in cache
	//log.Printf("\n\n[DEBUG] Checking local cache for executions. Got %d executions\n\n", len(executions))
	for execIndex, execution := range executions {
		if execution.Status == "EXECUTING" {
			//log.Printf("\n\n[DEBUG] Execution %s is executing, skipping cache\n\n", execution.ExecutionId)

			// Get the right one from cache
			newexec, err := GetWorkflowExecution(ctx, execution.ExecutionId)
			if err == nil {
				//log.Printf("[DEBUG] Got with status %s", newexec.Status)
				// Set the execution as well in the database
				if newexec.Status != execution.Status || len(newexec.Results) > len(execution.Results) {

					if project.Environment == "cloud" {
						go SetWorkflowExecution(ctx, *newexec, true)
					} else {
						SetWorkflowExecution(ctx, *newexec, true)
					}
				}

				executions[execIndex] = *newexec
			}
		} else {
			// Delete cache to clear up memory
			if project.Environment != "cloud" && (execution.Status == "ABORTED" || execution.Status == "FAILURE" || execution.Status == "FINISHED") {
				// Delete cache for it
				RunCacheCleanup(ctx, execution)
			}
		}
	}

	slice.Sort(executions[:], func(i, j int) bool {
		return executions[i].StartedAt > executions[j].StartedAt
	})

	executionmarshal, err := json.Marshal(executions)
	if err == nil {
		if len(executionmarshal) > totalMaxSize {
			// Reducing size

			for execIndex, execution := range executions {
				// Making sure the first 5 are "always" proper
				if execIndex < 5 {
					continue
				}

				newResults := []ActionResult{}

				newActions := []Action{}
				for _, action := range execution.Workflow.Actions {
					newAction := Action{
						Name:    action.Name,
						ID:      action.ID,
						AppName: action.AppName,
						AppID:   action.AppID,
					}

					newActions = append(newActions, newAction)
				}

				executions[execIndex].Workflow = Workflow{
					Name:     execution.Workflow.Name,
					ID:       execution.Workflow.ID,
					Triggers: execution.Workflow.Triggers,
					Actions:  newActions,
				}

				for _, result := range execution.Results {
					result.Result = "Result was too large to load. Full Execution needs to be loaded individually for this execution. Click \"Explore execution\" in the UI to see it in detail."
					result.Action = Action{
						Name:       result.Action.Name,
						ID:         result.Action.ID,
						AppName:    result.Action.AppName,
						AppID:      result.Action.AppID,
						LargeImage: result.Action.LargeImage,
					}

					newResults = append(newResults, result)
				}

				executions[execIndex].ExecutionArgument = "too large"
				executions[execIndex].Results = newResults
			}
		}
	}

	// Short-term caching
	if project.CacheDb {
		data, err := json.Marshal(executions)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling update execution cache: %s", err)
			return executions, cursor, nil
		}

		err = SetCache(ctx, cacheKey, data, 1)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache executions (%s): %s", workflowId, err)
			return executions, cursor, nil
		}
	}

	return executions, cursor, nil
}

func GetAllWorkflowExecutions(ctx context.Context, workflowId string, amount int) ([]WorkflowExecution, error) {
	nameKey := "workflowexecution"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, workflowId)
	var executions []WorkflowExecution
	var err error
	totalMaxSize := 11184810
	/*
		if project.CacheDb {
			cache, err := GetCache(ctx, cacheKey)
			if err == nil {
				cacheData := []byte(cache.([]uint8))
				err = json.Unmarshal(cacheData, &executions)
				if err == nil {
					if len(executions) > amount {
						executions = executions[:amount]
					}

					log.Printf("[DEBUG] Returned %d executions for workflow %s", len(executions), workflowId)

					return executions, nil
				} else {
					log.Printf("[WARNING] Failed getting workflowexecutions for %s: %s", workflowId, err)
				}
			} else {
				//log.Printf("[WARNING] Failed getting execution cache for workflow %s", workflowId)
			}
		}
	*/

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": amount,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						{
							"match": map[string]interface{}{
								"workflow_id": workflowId,
							},
						},
					},
				},
			},
			"sort": map[string]interface{}{
				"started_at": map[string]interface{}{
					"order": "desc",
				},
			},
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding executions query: %s", err)
			return executions, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return executions, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get workflow executions): %s", err)
			return executions, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return executions, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return executions, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return executions, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return executions, err
		}

		wrapped := ExecutionSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil && len(wrapped.Hits.Hits) == 0 {
			return executions, err
		}

		executions = []WorkflowExecution{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.WorkflowId == workflowId || hit.Source.Workflow.ID == workflowId {
				executions = append(executions, hit.Source)
			}
		}

		//return executions, nil
	} else {
		// FIXME: Sorting doesn't seem to work...
		//StartedAt          int64          `json:"started_at" datastore:"started_at"`
		//query := datastore.NewQuery(index).Filter("workflow_id =", workflowId).Limit(10)
		//totalMaxSize := 33554432
		//totalMaxSize := 22369621 // Total of App Engine max /3*2
		//totalMaxSize := 11184810
		query := datastore.NewQuery(nameKey).Filter("workflow_id =", workflowId).Order("-started_at").Limit(5)
		cursorStr := ""
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerWorkflow := WorkflowExecution{}
				_, err := it.Next(&innerWorkflow)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
					} else {
						log.Printf("[WARNING] CreateValue iterator issue (get executions): %s", err)
						break
					}
				}

				executions = append(executions, innerWorkflow)
			}

			if err != iterator.Done {
				//log.Printf("Breaking due to no more iterator")
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// This is a way to load as much data as we want, and the frontend will load the actual result for us
			executionmarshal, err := json.Marshal(executions)
			if err == nil {
				if len(executionmarshal) > totalMaxSize {
					// Reducing size

					for execIndex, execution := range executions {
						// Making sure the first 5 are "always" proper
						if execIndex < 5 {
							continue
						}

						newResults := []ActionResult{}

						newActions := []Action{}
						for _, action := range execution.Workflow.Actions {
							newAction := Action{
								Name:    action.Name,
								ID:      action.ID,
								AppName: action.AppName,
								AppID:   action.AppID,
							}

							newActions = append(newActions, newAction)
						}

						executions[execIndex].Workflow = Workflow{
							Name:     execution.Workflow.Name,
							ID:       execution.Workflow.ID,
							Triggers: execution.Workflow.Triggers,
							Actions:  newActions,
						}

						for _, result := range execution.Results {
							result.Result = "Result was too large to load. Full Execution needs to be loaded individually for this execution. Click \"Explore execution\" in the UI to see it in detail."
							result.Action = Action{
								Name:       result.Action.Name,
								ID:         result.Action.ID,
								AppName:    result.Action.AppName,
								AppID:      result.Action.AppID,
								LargeImage: result.Action.LargeImage,
							}

							newResults = append(newResults, result)
						}

						executions[execIndex].ExecutionArgument = "too large"
						executions[execIndex].Results = newResults
					}

					executionmarshal, err = json.Marshal(executions)
					if err == nil && len(executionmarshal) > totalMaxSize {
						//log.Printf("Length breaking (2): %d", len(executionmarshal))
						break
					}
				}
			}

			// expected to get here
			if len(executions) >= amount {
				//log.Printf("[INFO] Breaking due to executions larger than amount (%d/%d)", len(executions), amount)
				break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Cursorerror: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					//log.Printf("Breaking due to no new cursor")
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
				//cursorStr = nextCursor
				//break
			}
		}
	}

	newExecutions := []WorkflowExecution{}
	for _, execution := range executions {
		if execution.Workflow.OrgId == "INTERNAL" && execution.Status != "FINISHED" {
			continue
		}

		newExecutions = append(newExecutions, execution)
	}
	executions = newExecutions

	slice.Sort(executions[:], func(i, j int) bool {
		return executions[i].StartedAt > executions[j].StartedAt
	})

	executionmarshal, err := json.Marshal(executions)
	if err == nil {
		if len(executionmarshal) > totalMaxSize {
			// Reducing size

			for execIndex, execution := range executions {
				// Making sure the first 5 are "always" proper
				if execIndex < 5 {
					continue
				}

				newResults := []ActionResult{}

				newActions := []Action{}
				for _, action := range execution.Workflow.Actions {
					newAction := Action{
						Name:    action.Name,
						ID:      action.ID,
						AppName: action.AppName,
						AppID:   action.AppID,
					}

					newActions = append(newActions, newAction)
				}

				executions[execIndex].Workflow = Workflow{
					Name:     execution.Workflow.Name,
					ID:       execution.Workflow.ID,
					Triggers: execution.Workflow.Triggers,
					Actions:  newActions,
				}

				for _, result := range execution.Results {
					result.Result = "Result was too large to load. Full Execution needs to be loaded individually for this execution. Click \"Explore execution\" in the UI to see it in detail."
					result.Action = Action{
						Name:       result.Action.Name,
						ID:         result.Action.ID,
						AppName:    result.Action.AppName,
						AppID:      result.Action.AppID,
						LargeImage: result.Action.LargeImage,
					}

					newResults = append(newResults, result)
				}

				executions[execIndex].ExecutionArgument = "too large"
				executions[execIndex].Results = newResults
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(executions)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling update execution cache: %s", err)
			return executions, nil
		}

		err = SetCache(ctx, cacheKey, data, 10)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache executions (%s): %s", workflowId, err)
			return executions, nil
		}
	}

	return executions, nil
}

func GetOrgByField(ctx context.Context, fieldName, value string) ([]Org, error) {
	nameKey := "Organizations"

	var orgs []Org
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						{
							"match": map[string]interface{}{
								fieldName: value,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return orgs, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(nameKey)},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return orgs, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get app exec values): %s", err)
			return orgs, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return orgs, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return orgs, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return orgs, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return orgs, err
		}

		wrapped := OrgSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return orgs, err
		}

		orgs = []Org{}
		for _, hit := range wrapped.Hits.Hits {
			orgs = append(orgs, hit.Source)
		}
	} else {
		query := datastore.NewQuery(nameKey).Filter(fmt.Sprintf("%s =", fieldName), value).Limit(10)
		_, err := project.Dbclient.GetAll(ctx, query, &orgs)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting orgs for field %s: %s", fieldName, err)
				return orgs, err
			}
		}
	}

	return orgs, nil
}

func GetAllOrgs(ctx context.Context) ([]Org, error) {
	nameKey := "Organizations"

	var orgs []Org
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
		}
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find workflowapp query: %s", err)
			return []Org{}, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return []Org{}, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get org): %s", err)
			return []Org{}, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return []Org{}, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return []Org{}, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return []Org{}, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return []Org{}, err
		}

		wrapped := OrgSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return []Org{}, err
		}

		orgs = []Org{}
		for _, hit := range wrapped.Hits.Hits {
			orgs = append(orgs, hit.Source)
		}

		return orgs, nil
	} else {
		q := datastore.NewQuery(nameKey).Limit(400)

		_, err := project.Dbclient.GetAll(ctx, q, &orgs)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return []Org{}, err
			}
		}
	}

	return orgs, nil
}

func GetOrgMoveCache(ctx context.Context, orgId string) (RegionChangeHistory, error) {
	nameKey := "org_move_cache_" + orgId
	var err error
	var attempt RegionChangeHistory

	if project.CacheDb {
		cache, err := GetCache(ctx, nameKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &attempt)
			if err == nil {
				return attempt, nil
			}
		} else {
			log.Printf("[DEBUG] Failed getting cache for org %s (3): %s", orgId, err)
		}
	}

	return attempt, err
}

func GetSingulStatByExecutionId(ctx context.Context, executionId string) (SingulStats, error) {
	nameKey := "singul_stats"

	var stats SingulStats
	if project.DbType == "opensearch" {
		return SingulStats{}, errors.New("GetSingulStatByExecutionId not implemented for opensearch")
	} else {
		query := datastore.NewQuery(nameKey).Filter("execution_id =", executionId).Limit(1)
		_, err := project.Dbclient.GetAll(ctx, query, &stats)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting SingulStatByExecutionId: %s", err)
				return SingulStats{}, err
			}
		}
	}

	return stats, nil
}

func GetSingulStats(ctx context.Context) ([]SingulStats, error) {
	nameKey := "singul_stats"

	if project.DbType == "opensearch" {
		return []SingulStats{}, errors.New("GetSingulStats not implemented for opensearch")
	} else {
		query := datastore.NewQuery(nameKey).Limit(1000).Order("-created_at")
		var stats []SingulStats
		_, err := project.Dbclient.GetAll(ctx, query, &stats)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting SingulStats: %s", err)
				return []SingulStats{}, err
			}
		}

		if len(stats) == 0 {
			return []SingulStats{}, nil
		}

		return stats, nil
	}

	return []SingulStats{}, errors.New("GetSingulStats not implemented for this database type")
}

func SetSingulStats(ctx context.Context, stats SingulStats) error {
	nameKey := "singul_stats"

	if project.DbType == "opensearch" {
		// not implemented yet
		return errors.New("SetSingulStats not implemented for opensearch")
	} else {
		if stats.Id == "" {
			stats.Id = uuid.NewV4().String()
		}

		key := datastore.NameKey(nameKey, strings.ToLower(stats.Id), nil)
		if _, err := project.Dbclient.Put(ctx, key, &stats); err != nil {
			log.Printf("[WARNING] Error adding SingulStats: %s", err)
			return err
		}
	}

	return nil
}

func SetOrgMoveCache(ctx context.Context, orgId string) error {
	nameKey := "org_move_cache_" + orgId
	timeNow := int64(time.Now().Unix())

	attempt := RegionChangeHistory{
		OrgId:       orgId,
		LastAttempt: timeNow,
	}

	if project.CacheDb {
		attemptByte, err := json.Marshal(attempt)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in setorgmovecache: %s", err)
			return nil
		}

		err = SetCache(ctx, nameKey, attemptByte, 1440*15)
		if err != nil {
			log.Printf("[WARNING] Failed setting org move cache for %s: %s", orgId, err)
			return err
		}
	}

	return nil
}

// Index = Username
func SetSchedule(ctx context.Context, schedule ScheduleOld) error {
	nameKey := "schedules"

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		data, err := json.Marshal(schedule)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in setschedule: %s", err)
			return nil
		}

		err = indexEs(ctx, nameKey, strings.ToLower(schedule.Id), data)
		if err != nil {
			return err
		}
	} else {
		key1 := datastore.NameKey(nameKey, strings.ToLower(schedule.Id), nil)
		if _, err := project.Dbclient.Put(ctx, key1, &schedule); err != nil {
			log.Printf("Error adding schedule: %s", err)
			return err
		}
	}

	return nil
}

func GetAppExecutionValues(ctx context.Context, parameterNames, orgId, workflowId, value string) ([]NewValue, error) {
	nameKey := fmt.Sprintf("app_execution_values")
	var workflows []NewValue
	var err error

	// Appending the users' workflows
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
					},
				},
			},
		}

		//"workflow_id":    executionId,
		//"parameter_name": parameterNames,
		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return workflows, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return workflows, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get app exec values): %s", err)
			return workflows, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return workflows, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return workflows, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return workflows, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return workflows, err
		}

		wrapped := NewValueSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return workflows, err
		}

		workflows = []NewValue{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.Value == value && hit.Source.OrgId == orgId {
				workflows = append(workflows, hit.Source)
			}
		}
	} else {
		query := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Filter("workflow_id =", workflowId).Filter("parameter_name =", parameterNames).Filter("value =", value)
		//foundCount, err := project.Dbclient.Count(ctx, q)
		cursorStr := ""
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerWorkflow := NewValue{}
				_, err := it.Next(&innerWorkflow)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
					} else {
						log.Printf("[WARNING] CreateValue iterator issue (app execution values): %s", err)
						break
					}
				}

				workflows = append(workflows, innerWorkflow)
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	return workflows, nil
}

func GetDatastoreCategories(ctx context.Context, orgId string) ([]DatastoreCategoryUpdate, error) {
	nameKey := "datastore_category"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)
	categories := []DatastoreCategoryUpdate{}
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &categories)
			if err == nil {
				return categories, nil
			}
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find datastore categories query: %s", err)
			return categories, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return categories, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get datastore categories): %s", err)
			return categories, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.IsError() {
			if strings.Contains(res.String(), "index_not_found_exception") {
			} else {
				log.Printf("[WARNING] Failed datastore category query: %s", res.String())
				return categories, errors.New(res.String())
			}

			return categories, nil
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return categories, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return categories, err
		}

		wrapped := OrgDatastoreCategoryWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil && len(wrapped.Hits.Hits) == 0 {
			return categories, err
		}

		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.OrgId != orgId {
				continue
			}

			categories = append(categories, hit.Source)
		}
	} else {
		query := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Limit(50)

		_, err := project.Dbclient.GetAll(ctx, query, &categories)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting categories for %s: %s (1)", orgId, err)
				return categories, err
			}
		}
	}

	if len(categories) == 0 {
		if debug {
			log.Printf("[DEBUG] No categories found for org %s", orgId)
		}

		return categories, nil
	}

	if project.CacheDb {
		cacheDataByte, err := json.Marshal(categories)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in get datastore categories: %s", err)
			return categories, nil
		}

		err = SetCache(ctx, cacheKey, cacheDataByte, 60)
		if err != nil {
			log.Printf("[WARNING] Failed setting datastore categories for org %s: %s", orgId, err)
			return categories, nil
		}
	}

	return categories, nil

}

func GetDatastoreCategoryConfig(ctx context.Context, orgId, category string) (*DatastoreCategoryUpdate, error) {
	nameKey := "datastore_category"
	category = strings.ReplaceAll(strings.ToLower(category), " ", "_")

	categoryData := &DatastoreCategoryUpdate{}
	cacheKey := fmt.Sprintf("%s_%s_%s", nameKey, orgId, category)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &categoryData)
			if err == nil {
				return categoryData, nil
			}
		}
	}

	seedString := fmt.Sprintf("%s_%s", orgId, category)
	hash := sha1.New()
	hash.Write([]byte(seedString))
	hashBytes := hash.Sum(nil)
	uuidBytes := make([]byte, 16)
	copy(uuidBytes, hashBytes)
	id := uuid.Must(uuid.FromBytes(uuidBytes)).String()

	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return categoryData, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return categoryData, errors.New("Key doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return categoryData, err
		}

		wrapped := DatastoreCategoryKeyWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return categoryData, err
		}

		categoryData = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, id, nil)

		if err := project.Dbclient.Get(ctx, key, categoryData); err != nil {

			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[ERROR] Error in cache key loading. Migrating org cache to new handler (3): %s", err)
				err = nil
			} else {
				return categoryData, fmt.Errorf("Error getting datastore category config for org %s and category '%s': %w", orgId, category, err)
			}
		}
	}

	if len(categoryData.Id) == 0 {
		return categoryData, fmt.Errorf("No category found for org %s and category %s", orgId, category)
	}

	if project.CacheDb {
		cacheDataByte, err := json.Marshal(categoryData)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in get datastore category config: %s", err)
			return categoryData, nil
		}

		err = SetCache(ctx, cacheKey, cacheDataByte, 60)
		if err != nil {
			log.Printf("[WARNING] Failed setting datastore category for get category '%s' in org %s: %s", category, orgId, err)
			return categoryData, err
		}
	}

	return categoryData, nil
}

func SetDatastoreCategoryConfig(ctx context.Context, category DatastoreCategoryUpdate) error {
	nameKey := "datastore_category"
	if len(category.OrgId) == 0 {
		return errors.New("OrgId is required for SetSetDatastoreCategoryConfig")
	}

	category.Category = strings.ReplaceAll(strings.ToLower(category.Category), " ", "_")

	// Deterministic UUID based on OrgId and Category
	seedString := fmt.Sprintf("%s_%s", category.OrgId, category.Category)
	hash := sha1.New()
	hash.Write([]byte(seedString))
	hashBytes := hash.Sum(nil)
	uuidBytes := make([]byte, 16)
	copy(uuidBytes, hashBytes)

	category.Id = uuid.Must(uuid.FromBytes(uuidBytes)).String()
	if len(category.Id) != 36 {
		return errors.New(fmt.Sprintf("Failed to generate valid UUID for category with orgId %s and category %s", category.OrgId, category.Category))
	}

	// Clean up empty fields
	for automationIndex, automation := range category.Automations {
		newOptions := []DatastoreAutomationOption{}
		for _, option := range automation.Options {
			if len(option.Value) == 0 { 
				continue
			}

			newOptions = append(newOptions, option)
		}

		category.Automations[automationIndex].Options = newOptions
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(category)
	if err != nil {
		log.Printf("[ERROR] Failed marshalling in set datastore category key: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, category.Id, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, category.Id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &category); err != nil {
			log.Printf("[ERROR] Error setting datastore config: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s_%s", nameKey, category.OrgId, category.Category)
		err = SetCache(ctx, cacheKey, data, 62)
		if err != nil {
			log.Printf("[ERROR] Failed setting datastore category for set category '%s' in org %s: %s", category.Category, category.OrgId, err)
		}
	}

	return nil
}

// Used for cache for individual organizations
// Tracks key by key, and scales pretty well :3
func SetDatastoreKeyBulk(ctx context.Context, allKeys []CacheKeyData) ([]DatastoreKeyMini, error) {
	nameKey := "org_cache"
	timeNow := int64(time.Now().Unix())

	dbKeys := []*datastore.Key{}

	existingInfo := []DatastoreKeyMini{}

	mainCategory := ""
	wg := sync.WaitGroup{}

	// 1. Get the key first.
	// 2. Validate suborg distribution and other category configs
	cnt := 0
	for index, cacheData := range allKeys {
		// Disallowing setting of multiple categories at a time
		if index > 0 && len(cacheData.Category) > 0 {
			if mainCategory != cacheData.Category {
				continue
			}
		}

		mainCategory = cacheData.Category
		cnt += 1
	}

	cacheKeys := make(chan CacheKeyData, cnt)
	datastoreKeys := make(chan datastore.Key, cnt)
	orgId := ""

	for index, cacheData := range allKeys {
		// 1. Get the key first.
		// 2. Validate suborg distribution and other category configs

		// Disallowing setting of multiple categories at a time
		if index > 0 && len(cacheData.Category) > 0 {
			if mainCategory != cacheData.Category {
				continue
			}
		}

		orgId = cacheData.OrgId

		wg.Add(1)
		go func(cacheData CacheKeyData, index int) {
			defer wg.Done()

			cacheData.Existed = false
			cacheData.Changed = false
			cacheData.Created = timeNow
			cacheData.Edited = timeNow

			cacheData.Category = strings.ReplaceAll(strings.ToLower(cacheData.Category), " ", "_")
			datastoreId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
			if len(cacheData.Category) > 0 && cacheData.Category != "default" {
				// Adds category on the end
				datastoreId = fmt.Sprintf("%s_%s", datastoreId, cacheData.Category)
			}

			// Check for if the key already existed. Ok with
			// goroutine as we use heavy caching for this.
			sameValue := false
			config, getCacheError := GetDatastoreKey(ctx, datastoreId, cacheData.Category)

			cacheData.Changed = true
			if getCacheError == nil && config.Value == cacheData.Value {
				sameValue = true
			}

			if len(cacheData.Enrichments) > 0 && len(cacheData.Value) == 0 {
				if debug {
					log.Printf("[DEBUG] Having enrichments with empty value doesn't make sense, skipping enrichments for key %s in category %s", cacheData.Key, cacheData.Category)
				}

				cacheData.Value = config.Value
			}

			// Works on merging enrichments
			if len(cacheData.Enrichments) > 0 {
				timeNow := int64(time.Now().Unix())

				// Start with existing ones
				newObservables := config.Enrichments
				for _, observable := range cacheData.Enrichments {
					if len(observable.Value) == 0 {
						continue
					}

					existed := false
					for existingObsIndex, existingObs := range newObservables {
						if existingObs.Type == observable.Type && existingObs.Value == observable.Value {
							existed = true
							newObservables[existingObsIndex].LastSeen = timeNow
							if existingObs.FirstSeen == 0 {
								existingObs.FirstSeen = timeNow
							}

							continue
						}
					}

					if !existed {
						observable.FirstSeen = timeNow
						observable.LastSeen = timeNow
						newObservables = append(newObservables, observable)
					}
				}

				cacheData.Enrichments = newObservables
			}

			if getCacheError == nil && config.Created > 0 {

				// Compares old vs new, checks if allowed

				if cacheData.IgnoreSecurityRules == true {
					//if debug { 
					//	log.Printf("[DEBUG] Ignoring security rules for %s => %s", cacheData.Key, cacheData.Category)
					//}
					//os.Exit(3)
				} else {
					categoryConfig, err := GetDatastoreCategoryConfig(ctx, cacheData.OrgId, cacheData.Category)
					if err != nil {
						log.Printf("[WARNING] Failed getting category config for org %s and category %s: %s", orgId, mainCategory, err)
					}

					//if debug {
					//	log.Printf("[DEBUG] RULECHECK %#v -> %#v", getCacheError, config.Created)
					//}

					ruleValid := true
					for _, automation := range categoryConfig.Automations {
						if !automation.Enabled {
							continue
						}

						if automation.Name != "security_rules" && automation.Name != "Security Rules" {
							continue
						}

						foundRule := ""
						for _, option := range automation.Options {
							if option.Key == "rule" {
								foundRule = option.Value
								break
							}
						}

						if debug {
							log.Printf("[DEBUG] FOUND SECURITY RULES AUTOMATION FOR ORG %s AND CATEGORY %s: %#v", cacheData.OrgId, mainCategory, foundRule)
						}

						if len(foundRule) > 5 {
							oldDoc := config.Value
							newDoc := cacheData.Value
							mergedJSON, allowed, errString := EvalPolicyJSON(foundRule, oldDoc, newDoc)
							if debug {
								log.Printf("[DEBUG] RLS Security Rule OUTCOME (%s). Org: '%s', Key: '%s', Category: '%s': %#v. .\n\nError: %#v", foundRule, cacheData.OrgId, cacheData.Key, cacheData.Category, allowed, errString)
							}

							// Since merge happens, can we trust it 100% of the time?
							cacheData.Value = mergedJSON
							ruleValid = true

							//if allowed {
							//	ruleValid = true
							//	cacheData.Value = mergedJSON
							//} else {
							//	ruleValid = false
							//}

						}

						break
					}

					if !ruleValid {
						// Break out
						if debug {
							log.Printf("[WARNING] Rule is NOT valid! Skipping modification.")
						}

						return
					}
				}

				cacheData.Created = config.Created
				cacheData.Authorization = config.Authorization
				cacheData.SuborgDistribution = config.SuborgDistribution
				cacheData.PublicAuthorization = config.PublicAuthorization

				if len(cacheData.Enrichments) == 0 && len(config.Enrichments) > 0 {
					cacheData.Enrichments = config.Enrichments
				}

				if len(cacheData.Tags) == 0 {
					cacheData.Tags = config.Tags
				} else {
					sameValue = false
				}

				cacheData.Existed = true
			}

			if cacheData.Created == 0 {
				cacheData.Created = timeNow
			}

			if len(cacheData.Key) == 0 {
				cacheData.Key = datastoreId
			}

			// Makes sure
			cacheData.IgnoreSecurityRules = false

			// Sets new keys in cache so they can be queried fast next time
			marshalledEntry, err := json.Marshal(cacheData)
			if err == nil {
				newCacheId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
				if len(cacheData.Category) > 0 && cacheData.Category != "default" {
					newCacheId = fmt.Sprintf("%s_%s", newCacheId, cacheData.Category)
				}

				newCacheId = url.QueryEscape(newCacheId)
				if len(newCacheId) > 127 {
					newCacheId = newCacheId[0:127]
				}

				newCacheId = fmt.Sprintf("org_cache_%s", newCacheId)
				SetCache(ctx, newCacheId, marshalledEntry, 62)
			}

			// URL encode
			datastoreId = url.QueryEscape(datastoreId)
			if len(cacheData.PublicAuthorization) == 0 && cacheData.Category != "protected" {
				cacheData.PublicAuthorization = uuid.NewV4().String()
			}

			cacheData.Authorization = ""

			allKeys[index] = cacheData
			if len(datastoreId) > 127 {
				datastoreId = datastoreId[:127]
			}

			if cacheData.Category == "protected" {
				cacheData.Encrypted = true

				encryptionKey := fmt.Sprintf("%s_%d_%s_%s", cacheData.OrgId, cacheData.Created, cacheData.Category, cacheData.Key)
				//newValue, err := HandleKeyDecryption([]byte(field.Value), parsedKey)
				newValue, err := HandleKeyEncryption([]byte(cacheData.Value), encryptionKey)
				if err != nil {
					cacheData.Encrypted = false
				} else {
					cacheData.Value = string(newValue)
				}
			}

			if sameValue {
				// cacheData.Changed = false

				// FIXME: Should NOT be returning keys?
				// This would overwrite keys otherwise which is...
				// unnecessary. At least it makes edited => last seen
				// This may mean to sen nil to datastoreKeys & cacheKeys
				// It does however still have to take into account Existed, which means we need to pass along details :)
				//datastoreKeys <- *datastore.NameKey("", datastoreId, nil)
				//cacheKeys <- CacheKeyData{}

			}

			datastoreKeys <- *datastore.NameKey(nameKey, datastoreId, nil)
			cacheKeys <- cacheData
		}(cacheData, index)
		// Should set cache key here just in case? :thinking:
	}

	wg.Wait()
	close(cacheKeys)
	close(datastoreKeys)

	// Ensures no duplicates
	newArray := []CacheKeyData{}
	handledKeys := []string{}

	skippedKeys := []string{}
	for key := range cacheKeys {
		if key.Key == "" {
			//if debug { 
			//	log.Printf("[DEBUG] Skipping empty key in category %s", key.Category)
			//}
			continue
		}

		// Assumes duplicates
		checkKey := fmt.Sprintf("%s_%s", key.Key, key.Category)
		if ArrayContains(handledKeys, checkKey) {
			//if debug {
			//	log.Printf("[DEBUG] Skipping duplicate key %s in category %s", key.Key, key.Category)
			//}

			handledKeys = append(handledKeys, checkKey)
			continue
		}

		// Details to help with filtering old vs new
		// Built for the "is_in_datastore" shuffle tools action
		minKey := DatastoreKeyMini{
			Key:     key.Key,
			Existed: key.Existed,
		}

		existingInfo = append(existingInfo, minKey)
		if !key.Changed {
			parsedKey := fmt.Sprintf("%s_%s_%s", key.OrgId, key.Key, key.Category)
			skippedKeys = append(skippedKeys, parsedKey)
			//log.Printf("[DEBUG] Key %s did NOT change, skipping database", parsedKey)
			continue
		}

		key.Existed = false
		key.Changed = false
		newArray = append(newArray, key)

	}

	handledKeys = []string{}
	for key := range datastoreKeys {
		if key.Name == "" {
			//if debug { 
			//	log.Printf("[DEBUG] Skipping empty datastore key")
			//}

			continue
		}

		// Duplicate handler
		if ArrayContains(handledKeys, key.Name) {
			//if debug {
			//	log.Printf("[DEBUG] Skipping duplicate datastore key %s", key.Name)
			//}

			continue
		}

		if ArrayContains(skippedKeys, key.Name) {
			//if debug { 
			//	log.Printf("[DEBUG] Skipping datastore key %s as it was marked as skipped due to no changes", key.Name)
			//}

			continue
		}

		// Look for empty keys and continue if so:
		handledKeys = append(handledKeys, key.Name)
		dbKeys = append(dbKeys, &key)
	}

	// Autofixer on the fly
	if len(newArray) != len(dbKeys) {
		dbKeys = []*datastore.Key{}

		// FIXME: newArray backwards to ALWAYS have latest key? Is latest last
		// or first in the array? :thinking:
		handledKeys := []string{}
		skippedIndexes := []int{}
		for skipIndex, cacheData := range newArray {
			datastoreId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
			if len(cacheData.Category) > 0 && cacheData.Category != "default" {
				// Adds category on the end
				datastoreId = fmt.Sprintf("%s_%s", datastoreId, cacheData.Category)
			}

			if ArrayContains(handledKeys, datastoreId) {
				skippedIndexes = append(skippedIndexes, skipIndex)
				continue
			}

			handledKeys = append(handledKeys, datastoreId)

			dbKeys = append(dbKeys, datastore.NameKey(nameKey, strings.ToLower(datastoreId), nil))
		}

		// Cleanup newArray again due to transactional handler
		// Example where problems show up are nested items:
		// 		Multiple emails in the same thread
		if len(skippedIndexes) > 0 {
			newDeduped := []CacheKeyData{}
			for index, val := range newArray {
				if ArrayContainsInt(skippedIndexes, index) {
					continue
				}

				newDeduped = append(newDeduped, val)
			}

			newArray = newDeduped
		}
	}

	for _, cacheData := range newArray {
		go SetDatastoreKeyRevision(context.Background(), cacheData)

		// Only update stats on the first run (?) as changes are inevitable
		existed := false
		for _, existing := range existingInfo {
			if existing.Key == cacheData.Key && existing.Existed {
				existed = true
				break
			}
		}

		if !existed {
			UpdateDetectionStats(context.Background(), cacheData)
		}
	}

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		var buf bytes.Buffer

		// Bulk encoding them
		for _, cacheData := range newArray {
			cacheId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
			if len(cacheData.Category) > 0 && cacheData.Category != "default" {
				cacheId = fmt.Sprintf("%s_%s", cacheId, cacheData.Category)
			}

			// URL encode
			cacheId = url.QueryEscape(cacheId)
			if len(cacheId) > 127 {
				cacheId = cacheId[:127]
			}

			meta := map[string]map[string]string{
				"index": {
					"_index": strings.ToLower(GetESIndexPrefix(nameKey)),
					"_id":    cacheId,
				},
			}

			metaLine, err := json.Marshal(meta)
			if err != nil {
				log.Printf("[ERROR] Failed marshalling meta in SetDatastoreKeyBulk: %s", err)
				continue
			}

			buf.Write(metaLine)
			buf.WriteByte('\n')

			docLine, err := json.Marshal(cacheData)
			if err != nil {
				log.Printf("[ERROR] Failed marshalling doc in SetDatastoreKeyBulk: %s", err)
				continue
			}

			if debug {
				log.Printf("[DEBUG] Doc with key %s is being rolled", cacheId)
			}

			buf.Write(docLine)
			buf.WriteByte('\n')
		}

		resp, err := project.Es.Bulk(ctx, opensearchapi.BulkReq{
			Body:  bytes.NewReader(buf.Bytes()),
			Index: strings.ToLower(GetESIndexPrefix(nameKey)),
		})

		res := resp.Inspect().Response
		defer res.Body.Close()

		if err == nil {
			if debug {
				log.Printf("[DEBUG] There was no error sending bulk request to Opensearch for cache key")
				// print body
				body, err := ioutil.ReadAll(res.Body)
				if err != nil {
					log.Printf("[ERROR] Error reading response body: %s", err)
				} else {
					log.Printf("[DEBUG] Response body: %s", string(body))
				}
			}
		}

		if err != nil {
			log.Printf("[ERROR] Error sending bulk request to Opensearch: %s", err)

			body, err := ioutil.ReadAll(res.Body)
			if err != nil {
				if strings.Contains(err.Error(), "index_not_found_exception") {
					return existingInfo, nil
				}

				log.Printf("[ERROR] Error getting response from Opensearch (set datastore key bulk): %s", err)
				return existingInfo, err
			}

			log.Printf("[ERROR] Error getting response from Opensearch (set datastore key bulk): %s. Body: %s", err, body)
			return existingInfo, err
		}

		if debug {
			log.Printf("[DEBUG] Response status: %d", res.StatusCode)
		}

	} else {
		if len(newArray) != len(dbKeys) {
			log.Printf("[ERROR] SetDatastoreKeyBulk: Length of newArray (%d) and allKeys (%d) do not match", len(newArray), len(allKeys))

			return existingInfo, errors.New("SetDatastoreKeyBulk: Length of newArray and allKeys do not match")
		}

		if _, err := project.Dbclient.PutMulti(ctx, dbKeys, newArray); err != nil {
			log.Printf("[ERROR] Error setting bulk org datastore: %s", err)
			return existingInfo, err
		}
	}

	if len(newArray) > 0 {
		log.Printf("[INFO] SetDatastoreKeyBulk: Successfully set %d key(s) in category %s for org %s", len(newArray), mainCategory, orgId)
	}

	/*
		if project.CacheDb {
			cacheKey := fmt.Sprintf("%s_%s", nameKey, cacheId)
			err = SetCache(ctx, cacheKey, data, 30)
			if err != nil {
				log.Printf("[ERROR] Failed setting cache for set cache key '%s': %s", cacheKey, err)
			}

			// Delete cache in current org + category + child orgs
			cursor := ""
			currentKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, cursor, cacheData.OrgId, cacheData.Category)
			DeleteCache(ctx, currentKey)

			for _, suborg := range cacheData.SuborgDistribution {
				currentKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, cursor, suborg, cacheData.Category)
				DeleteCache(ctx, currentKey)
			}
		}
	*/

	for cnt, cacheData := range newArray {
		// Maxing out at 100 for now just in case
		if cnt > 100 {
			break
		}

		if len(cacheData.Category) == 0 || len(cacheData.OrgId) == 0 {
			if debug {
				log.Printf("[DEBUG] No category/orgid. Continue")
			}
			continue
		}

		found := false
		for _, existing := range existingInfo {
			if existing.Key != cacheData.Key {
				continue
			}

			if existing.Existed {
				found = true
			}

			break
		}

		// Only runs once per minute MAX except for enrichments.
		enrichmentsOnly := false
		if found {
			cacheKey := fmt.Sprintf("ngram_check_%s_%s_%s", cacheData.OrgId, cacheData.Category, cacheData.Key)
			data, err := GetCache(ctx, cacheKey)
			if err == nil && data != nil {
				if len(cacheData.Enrichments) > 0 {
					enrichmentsOnly = true
				}
			} else {
				enrichmentsOnly = false
				SetCache(ctx, cacheKey, []byte("1"), 1)
			}
		}

		go crossCorrelateNGrams(context.Background(), cacheData.OrgId, cacheData.Category, cacheData.Key, cacheData.Value, cacheData.Enrichments, enrichmentsOnly)
	}

	// Look for category triggers
	if len(mainCategory) > 0 && mainCategory != "default" && len(newArray) > 0 && len(newArray[0].OrgId) > 0 {

		orgId := newArray[0].OrgId

		categoryConfig, err := GetDatastoreCategoryConfig(ctx, orgId, mainCategory)
		if err != nil {
			// Set it in the DB
			categoryUpdate := DatastoreCategoryUpdate{
				Category: mainCategory,
				OrgId:    orgId,
				Id:       uuid.NewV4().String(),

				Settings: DatastoreCategorySettings{
					Public:  false,
					Timeout: 0,
				},
			}

			err := SetDatastoreCategoryConfig(ctx, categoryUpdate)
			if err != nil {
				log.Printf("[ERROR] Failed setting datastore category config for org %s and category %s: %s", orgId, mainCategory, err)
			}
		} else {
			for _, cacheData := range newArray {
				for _, automation := range categoryConfig.Automations {
					if !automation.Enabled {
						continue
					}

					if automation.Name == "security_rules" || automation.Name == "Security Rules" {
						continue
					}

					if len(automation.Options) == 0 {
						if debug {
							log.Printf("\n\n\n[ERROR] Debug: Automation '%s' in category '%s' has no options, skipping\n\n\n", automation.Name, categoryConfig.Category)
						}

						continue
					}

					//if debug {
					//	log.Printf("[DEBUG] Found automation '%s' to run (2). Value: '%s'", automation.Name, automation.Options[0].Value)
					//}

					// Run the automation
					// This should make a notification if it fails
					go func(cacheData CacheKeyData, automation DatastoreAutomation) {
						err := handleRunDatastoreAutomation(ctx, cacheData, automation)
						if err != nil {
							log.Printf("[ERROR] Failed running automation %s for cache key %s: %s", automation.Name, cacheData.Key, err)

							CreateOrgNotification(
								ctx,
								fmt.Sprintf("Problem with automation '%s' in category '%s'", automation.Name, cacheData.Category),
								fmt.Sprintf("Failed running automation '%s' for cache key '%s' in category '%s'. Error: %s", automation.Name, cacheData.Key, cacheData.Category, err),
								fmt.Sprintf("/admin?tab=datastore&category=%s", cacheData.Category),
								cacheData.OrgId,
								true,
								"MEDIUM",
								"Datastore_Automation_Error",
							)
						}
					}(cacheData, automation)
				}
			}
		}
	}

	if len(mainCategory) == 0 {
		DeleteCache(ctx, fmt.Sprintf("%s_%s_%s", nameKey, "", orgId))
	}

	cacheKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, "", orgId, mainCategory)
	DeleteCache(ctx, cacheKey)
	DeleteCache(ctx, fmt.Sprintf("datastore_category_%s", orgId))
	return existingInfo, nil
}

func GetDatastoreRevisions(ctx context.Context, key, category, orgId string) ([]CacheKeyData, error) {
	var datastoreKeys []CacheKeyData
	if len(orgId) == 0 {
		return datastoreKeys, errors.New("Org ID required for revisions")
	}

	var err error

	amount := 50
	if amount <= 0 {
		amount = 50
	}

	if amount >= 200 {
		amount = 200
	}

	key = url.QueryEscape(key)
	if len(key) > 127 {
		key = key[:127]
	}

	category = url.QueryEscape(category)
	if len(category) > 127 {
		category = category[:127]
	}

	nameKey := "org_cache_revisions"
	cacheKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, key, category, orgId)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &datastoreKeys)
			if err == nil {

				sort.Slice(datastoreKeys, func(i, j int) bool {
					return datastoreKeys[i].Edited > datastoreKeys[j].Edited
				})

				return datastoreKeys, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for workflow (5): %s", err)
		}
	}

	//log.Printf("[AUDIT] Getting workflow revisions for workflow %s.", originalId)
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": amount,
			"sort": map[string]interface{}{
				"edited": map[string]interface{}{
					"order": "desc",
				},
			},
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"key": key,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"category": category,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return datastoreKeys, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return datastoreKeys, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (Get datastoreKeys2 - revisions): %s", err)
			return datastoreKeys, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return datastoreKeys, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return datastoreKeys, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return datastoreKeys, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return datastoreKeys, err
		}

		wrapped := CacheKeySearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil && len(wrapped.Hits.Hits) == 0 {
			return datastoreKeys, err
		}

		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.Key != key {
				continue
			}

			datastoreKeys = append(datastoreKeys, hit.Source)
		}
	} else {
		queryAmount := 20
		if amount < queryAmount {
			queryAmount = amount
		}

		query := datastore.NewQuery(nameKey).Filter("Key =", key).Filter("category =", category).Filter("OrgId =", orgId).Limit(queryAmount)
		query = query.Order("-Edited")

		iterCount := 0

		cursorStr := ""
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerData := CacheKeyData{}
				_, err := it.Next(&innerData)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {
					} else {
						//log.Printf("[ERROR] Datastore revision iterator issue: %s", err)
						break
					}
				}

				iterCount++
				datastoreKeys = append(datastoreKeys, innerData)
				if iterCount >= amount {
					break
				}
			}

			if iterCount >= amount {
				break
			}

			if err != iterator.Done {
				//log.Printf("[INFO] Failed fetching datastore revisions: %v", err)
				//break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with datastore revisions cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	// Sort by edited
	sort.Slice(datastoreKeys, func(i, j int) bool {
		return datastoreKeys[i].Edited > datastoreKeys[j].Edited
	})

	// Deduplicate based on edited time
	filtered := []CacheKeyData{}
	handled := []string{}
	for _, datastoreKey := range datastoreKeys {
		if ArrayContains(handled, fmt.Sprintf("%d", datastoreKey.Edited)) {
			continue
		}

		handled = append(handled, fmt.Sprintf("%d", datastoreKey.Edited))
		filtered = append(filtered, datastoreKey)
	}

	// Set cache
	if project.CacheDb {
		cacheData, err := json.Marshal(datastoreKeys)
		if err != nil {
			return datastoreKeys, nil
		}

		err = SetCache(ctx, cacheKey, cacheData, 2)
		if err != nil {
			log.Printf("[ERROR] Failed setting cache for workflow revisions: %s (not critical)", err)
		}
	}

	return datastoreKeys, nil
}

func SetDatastoreKeyRevision(ctx context.Context, cacheData CacheKeyData) error {
	nameKey := "org_cache_revisions"
	timeNow := int64(time.Now().Unix())
	cacheData.Edited = timeNow
	cacheData.RevisionId = uuid.NewV4().String()
	if cacheData.Created == 0 {
		cacheData.Created = timeNow
	}

	cacheId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
	if len(cacheData.Category) > 0 && cacheData.Category != "default" {
		cacheId = fmt.Sprintf("%s_%s", cacheId, cacheData.Category)
	}

	cacheId = fmt.Sprintf("%s_%s", cacheId, cacheData.RevisionId)

	// URL encode
	cacheId = url.QueryEscape(cacheId)
	if len(cacheId) > 127 {
		cacheId = cacheId[:127]
	}

	cacheData.Authorization = ""
	if len(cacheData.PublicAuthorization) == 0 && cacheData.Category != "protected" {
		cacheData.PublicAuthorization = uuid.NewV4().String()
	}

	cacheData.Category = strings.ReplaceAll(strings.ToLower(cacheData.Category), " ", "_")

	// Test just for protected category (for now)
	if cacheData.Category == "protected" {
		return errors.New("Not storing revisions for protected category keys")
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(cacheData)
	if err != nil {
		log.Printf("[ERROR] Failed marshalling in set cache key: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, cacheId, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, cacheId, nil)
		if _, err := project.Dbclient.Put(ctx, key, &cacheData); err != nil {
			log.Printf("[ERROR] Error setting datastore key revision: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, cacheId)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[ERROR] Failed setting cache for set cache key '%s': %s", cacheKey, err)
		}
	}

	DeleteCache(ctx, fmt.Sprintf("datastore_category_revisions_%s", cacheData.OrgId))
	return nil
}

// Primarily used for updating tags and other metadata.
func SetDatastoreKeyMeta(ctx context.Context, cacheData CacheKeyData) error {
	nameKey := "org_cache"
	cacheId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
	if len(cacheData.Category) > 0 && cacheData.Category != "default" {
		cacheId = fmt.Sprintf("%s_%s", cacheId, cacheData.Category)
	}

	// URL encode
	cacheId = url.QueryEscape(cacheId)
	if len(cacheId) > 127 {
		cacheId = cacheId[:127]
	}

	if len(cacheData.Tags) > 1 {
		newTags := []string{}
		for _, cacheTag := range cacheData.Tags {
			if cacheTag == "none" {
				continue
			}

			newTags = append(newTags, cacheTag)
		}

		cacheData.Tags = newTags
	}

	data, err := json.Marshal(cacheData)
	if err != nil {
		log.Printf("[ERROR] Failed marshalling in set cache key: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, cacheId, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, cacheId, nil)
		if _, err := project.Dbclient.Put(ctx, key, &cacheData); err != nil {
			log.Printf("[ERROR] Error setting datastore key meta: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, cacheId)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[ERROR] Failed setting cache for set cache key '%s': %s", cacheKey, err)
		}
	}

	return nil
}

// Used for cache for individual organizations
func SetDatastoreKey(ctx context.Context, cacheData CacheKeyData) error {
	nameKey := "org_cache"
	timeNow := int64(time.Now().Unix())
	cacheData.Edited = timeNow

	if cacheData.Created == 0 {
		cacheData.Created = timeNow
	}

	//cacheId := fmt.Sprintf("%s_%s_%s", cacheData.OrgId, cacheData.WorkflowId, cacheData.Key)
	cacheId := fmt.Sprintf("%s_%s", cacheData.OrgId, cacheData.Key)
	if len(cacheData.Category) > 0 && cacheData.Category != "default" {
		cacheId = fmt.Sprintf("%s_%s", cacheId, cacheData.Category)
	}

	// URL encode
	cacheId = url.QueryEscape(cacheId)
	if len(cacheId) > 127 {
		cacheId = cacheId[:127]
	}

	cacheData.Authorization = ""
	if len(cacheData.PublicAuthorization) == 0 && cacheData.Category != "protected" {
		cacheData.PublicAuthorization = uuid.NewV4().String()
	}

	cacheData.Category = strings.ReplaceAll(strings.ToLower(cacheData.Category), " ", "_")

	// Test just for protected category (for now)
	if cacheData.Category == "protected" {
		cacheData.Encrypted = true

		encryptionKey := fmt.Sprintf("%s_%d_%s_%s", cacheData.OrgId, cacheData.Created, cacheData.Category, cacheData.Key)
		//newValue, err := HandleKeyDecryption([]byte(field.Value), parsedKey)
		newValue, err := HandleKeyEncryption([]byte(cacheData.Value), encryptionKey)
		if err != nil {
			cacheData.Encrypted = false
		} else {
			cacheData.Value = string(newValue)
		}
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(cacheData)
	if err != nil {
		log.Printf("[ERROR] Failed marshalling in set cache key: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, cacheId, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, cacheId, nil)
		if _, err := project.Dbclient.Put(ctx, key, &cacheData); err != nil {
			log.Printf("[ERROR] Error setting datastore key: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, cacheId)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[ERROR] Failed setting cache for set cache key '%s': %s", cacheKey, err)
		}

		// Delete cache in current org + category + child orgs
		cursor := ""
		currentKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, cursor, cacheData.OrgId, cacheData.Category)
		DeleteCache(ctx, currentKey)

		for _, suborg := range cacheData.SuborgDistribution {
			currentKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, cursor, suborg, cacheData.Category)
			DeleteCache(ctx, currentKey)
		}
	}

	// Look for category triggers
	if len(cacheData.Category) > 0 && cacheData.Category != "default" {
		categoryConfig, err := GetDatastoreCategoryConfig(ctx, cacheData.OrgId, cacheData.Category)
		if err != nil {
			// Set it in the DB
			categoryUpdate := DatastoreCategoryUpdate{
				Category: cacheData.Category,
				OrgId:    cacheData.OrgId,
				Id:       uuid.NewV4().String(),

				Settings: DatastoreCategorySettings{
					Public:  false,
					Timeout: 0,
				},
			}

			err := SetDatastoreCategoryConfig(ctx, categoryUpdate)
			if err != nil {
				log.Printf("[ERROR] Failed setting datastore category config for org %s and category %s: %s", cacheData.OrgId, cacheData.Category, err)
			}

		} else {
			for _, automation := range categoryConfig.Automations {
				if !automation.Enabled {
					continue
				}

				if len(automation.Options) == 0 {
					continue
				}

				if debug {
					log.Printf("[DEBUG] Found automation %s to run. Value: %s", automation.Name, automation.Options[0].Value)
				}

				// Run the automation
				// This should make a notification if it fails
				go func(cacheData CacheKeyData, automation DatastoreAutomation) {
					err := handleRunDatastoreAutomation(ctx, cacheData, automation)
					if err != nil {
						log.Printf("[ERROR] Failed running automation %s for cache key %s: %s", automation.Name, cacheData.Key, err)

						CreateOrgNotification(
							ctx,
							fmt.Sprintf("Problem with automation '%s' in category '%s'", automation.Name, cacheData.Category),
							fmt.Sprintf("Failed running automation '%s' for cache key '%s' in category '%s'. Error: %s", automation.Name, cacheData.Key, cacheData.Category, err),
							fmt.Sprintf("/admin?tab=datastore&category=%s", cacheData.Category),
							cacheData.OrgId,
							true,
							"MEDIUM",
							"Datastore_Automation_Error",
						)
					}
				}(cacheData, automation)
			}
		}
	}

	DeleteCache(ctx, fmt.Sprintf("datastore_category_%s", cacheData.OrgId))
	return nil
}

// Used for cache for individual organizations
func GetDatastoreKey(ctx context.Context, id string, category string) (*CacheKeyData, error) {
	cacheData := &CacheKeyData{}
	nameKey := "org_cache"

	category = strings.ReplaceAll(strings.ToLower(category), " ", "_")
	if len(category) > 0 && category != "default" {
		// FIXME: If they key itself is 'test_protected' and category
		// is 'protected' this breaks... Keeping it for now.
		if !strings.HasSuffix(id, fmt.Sprintf("_%s", category)) {
			id = fmt.Sprintf("%s_%s", id, category)
		}
	}

	id = url.QueryEscape(id)
	if len(id) > 127 {
		id = id[0:127]
	}

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)

	if debug {
		//log.Printf("[DEBUG] Getting datastore key '%s'", cacheKey)
	}

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			parsedCache := []byte(cache.([]uint8))
			err = json.Unmarshal(parsedCache, cacheData)
			if err == nil {
				return cacheData, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for cache key %s: %s", id, err)
		}
	}

	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			if strings.Contains(err.Error(), "has more than one index associated with it") {
				fallbackData, fallbackErr := getCacheKeyByAliasSearch(ctx, strings.ToLower(GetESIndexPrefix(nameKey)), id)
				if fallbackErr == nil {
					cacheData = fallbackData
				} else {
					log.Printf("[WARNING] Alias search fallback failed for %s: %s", cacheKey, fallbackErr)
					return cacheData, fallbackErr
				}
			} else {
				log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
				return cacheData, err
			}
		}

		if err == nil {
			res := resp.Inspect().Response
			defer res.Body.Close()
			if res.StatusCode == 404 {
				return cacheData, errors.New("Key doesn't exist")
			}

			respBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				return cacheData, err
			}

			wrapped := CacheKeyWrapper{}
			err = json.Unmarshal(respBody, &wrapped)
			if err != nil {
				return cacheData, err
			}

			cacheData = &wrapped.Source
		}
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, cacheData); err != nil {
			if project.CacheDb {
				data, err := json.Marshal(cacheData)
				if err != nil {
					log.Printf("[ERROR] Failed marshalling in getcachekey (2): %s", err)
				} else {
					err = SetCache(ctx, cacheKey, data, 30)
					if err != nil {
						log.Printf("[ERROR] Failed setting cache for get cache key (2): %s", err)
					}
				}
			}

			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[ERROR] Error in cache key loading. Migrating org cache to new handler (3): %s", err)
				err = nil
			} else {
				//log.Printf("[WARNING] Error in datastore key loading for %s: %s", id, err)

				if len(category) > 0 && category != "default" {
				} else {
					// Search for key by removing first uuid part
					newId := id
					orgId := ""
					newIdSplit := strings.Split(id, "_")
					if len(newIdSplit) > 1 {
						orgId = newIdSplit[0]
						newId = strings.Join(newIdSplit[1:], "_")
					} else {
						log.Printf("[ERROR] Failed splitting cache id %s", id)
						return cacheData, err
					}

					// 2e7b6a08-b63b-4fc2-bd70-718091509db1
					// b0ef85ff-353c-4dbf-9e47-b9d0474dc14e
					// Skipped+because+of+previous+node+-+1

					newId, err = url.QueryUnescape(newId)
					if err != nil {
						log.Printf("[ERROR] Failed unescaping cache id %s", newId)
					}

					// Search for it in datastore with key =
					cacheKeys := []CacheKeyData{}
					cacheData.FormattedKey = newId
					query := datastore.NewQuery(nameKey).Filter("Key =", newId).Limit(5)
					_, err := project.Dbclient.GetAll(ctx, query, &cacheKeys)
					if err != nil {
						if !strings.Contains(err.Error(), `cannot load field`) {
							log.Printf("[WARNING] Failed getting datastoreKey (2) %s: %s", newId, err)

							if project.CacheDb {
								data, err := json.Marshal(cacheData)
								if err != nil {
									log.Printf("[WARNING] Failed marshalling in getcachekey (3): %s", err)
									return cacheData, nil
								}

								err = SetCache(ctx, cacheKey, data, 30)
								if err != nil {
									log.Printf("[WARNING] Failed setting cache for get cache key (3): %s", err)
								}
							}

							return cacheData, err
						}
					}

					if len(cacheKeys) > 0 {
						for _, cacheKey := range cacheKeys {
							if cacheKey.OrgId == orgId {
								cacheData = &cacheKey
								break
							}

							for _, subOrg := range cacheKey.SuborgDistribution {
								if subOrg == orgId {
									cacheData = &cacheKey
									break
								}
							}
						}

						if cacheData.Key == "" {
							return cacheData, errors.New("Key doesn't exist")
						}
					} else {
						log.Printf("[WARNING] Failed getting datastoreKey '%s': %s", newId, err)

						return cacheData, errors.New("Key doesn't exist")
					}
				}
			}
		} else {
			cacheData.FormattedKey = id
		}
	}

	if cacheData.Encrypted {
		encryptionKey := fmt.Sprintf("%s_%d_%s_%s", cacheData.OrgId, cacheData.Created, cacheData.Category, cacheData.Key)
		newValue, err := HandleKeyDecryption([]byte(cacheData.Value), encryptionKey)
		if err == nil {
			cacheData.Value = string(newValue)

			// Not removing this as it just causes confusion
			//cacheData.Encrypted = false
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(cacheData)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getcachekey: %s", err)
			return cacheData, nil
		}

		err = SetCache(ctx, cacheKey, data, 62)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for get cache key: %s", err)
		}
	}

	return cacheData, nil
}

func getCacheKeyByAliasSearch(ctx context.Context, aliasName, id string) (*CacheKeyData, error) {
	var buf bytes.Buffer
	query := map[string]interface{}{
		"size": 1,
		"sort": map[string]interface{}{
			"edited": map[string]interface{}{
				"order": "desc",
			},
		},
		"query": map[string]interface{}{
			"ids": map[string]interface{}{
				"values": []string{id},
			},
		},
	}

	if err := json.NewEncoder(&buf).Encode(query); err != nil {
		return nil, err
	}

	resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
		Indices: []string{aliasName},
		Body:    &buf,
		Params: opensearchapi.SearchParams{
			TrackTotalHits: true,
		},
	})
	if err != nil {
		return nil, err
	}

	res := resp.Inspect().Response
	defer res.Body.Close()

	if res.StatusCode == 404 {
		return nil, errors.New("Key doesn't exist")
	}

	respBody, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return nil, err
	}

	if res.StatusCode != 200 && res.StatusCode != 201 {
		return nil, fmt.Errorf("failed alias fallback lookup. status=%d body=%s", res.StatusCode, string(respBody))
	}

	wrapped := CacheKeySearchWrapper{}
	err = json.Unmarshal(respBody, &wrapped)
	if err != nil {
		return nil, err
	}

	if len(wrapped.Hits.Hits) == 0 {
		return nil, errors.New("Key doesn't exist")
	}

	item := wrapped.Hits.Hits[0].Source
	return &item, nil
}

var retryCount int

func RunInit(dbclient datastore.Client, storageClient storage.Client, gceProject, environment string, cacheDb bool, dbType string, defaultCreds bool, count int) (ShuffleStorage, error) {
	if dbType == "elasticsearch" {
		dbType = "opensearch"
	}

	cloudRunUrl := os.Getenv("SHUFFLE_CLOUDRUN_URL")
	if cloudRunUrl == "" {
		cloudRunUrl = "https://shuffler.io"
	}

	project = ShuffleStorage{
		Dbclient:      dbclient,
		StorageClient: storageClient,
		GceProject:    gceProject,
		Environment:   environment,
		CacheDb:       cacheDb,
		DbType:        dbType,
		CloudUrl:      cloudRunUrl,
		BucketName:    fmt.Sprintf("%s.appspot.com", gceProject),
	}

	bucketName := os.Getenv("SHUFFLE_ORG_BUCKET")
	if len(bucketName) > 0 {
		log.Printf("[DEBUG] Using custom project bucketname: %s", bucketName)
		project.BucketName = bucketName
	}

	kmsDebugEnabled := os.Getenv("SHUFFLE_KMS_DEBUG")
	if strings.ToLower(kmsDebugEnabled) == "true" {
		kmsDebug = true
	}

	// docker run -p 11211:11211 --name memcache -d memcached -m 100
	log.Printf("[DEBUG] Starting with memcached address '%s' (SHUFFLE_MEMCACHED). If this is empty, fallback to default (appengine / local). Name: '%s'", memcached, environment)

	// In case of downtime / large requests
	if len(memcached) > 0 {
		if strings.Contains(memcached, ",") {

			newMemcached := []string{}
			for _, memcached := range strings.Split(memcached, ",") {
				memcached = strings.TrimSpace(memcached)
				if len(memcached) > 0 {
					newMemcached = append(newMemcached, memcached)
				}
			}

			log.Printf("[DEBUG] Multiple memcached servers detected. Split into %#v", newMemcached)
			mc = gomemcache.New(newMemcached...)
		} else {
			log.Printf("[DEBUG] Initializing single memcached client with memcached url: %s", memcached)
			mc = gomemcache.New(memcached)
		}

		mc.Timeout = 10 * time.Second
	}

	requestCache = cache.New(35*time.Minute, 35*time.Minute)
	if strings.ToLower(environment) != "worker" && (strings.ToLower(dbType) == "opensearch" || strings.ToLower(dbType) == "opensearch") {

		ctx := context.Background()
		project.Es = *GetEsConfig(defaultCreds)

		infoSearchReq := &opensearchapi.InfoReq{}
		resp, err := project.Es.Info(ctx, infoSearchReq)
		if err != nil {
			if strings.Contains(fmt.Sprintf("%s", err), "the client noticed that the server is not a supported distribution") {
				log.Printf("[ERROR] Version is not supported - most likely Elasticsearch >= 8.0.0: %#v -> %s", resp, err)
			}
		}

		res := resp.Inspect().Response
		if err != nil {
			if fmt.Sprintf("%s", err) == "EOF" {
				log.Printf("[ERROR] Database should be available soon. Retrying in 5 seconds: %s", err)
			} else {
				log.Printf("[WARNING] Failed setting up Opensearch: %s. Typically means the backend can't connect, or that there's a HTTPS vs HTTP problem. Is the SHUFFLE_OPENSEARCH_URL correct?", err)
			}

			return project, err
		}

		if res.StatusCode >= 300 {
			respBody, err := ioutil.ReadAll(res.Body)
			if err != nil {
				log.Printf("[ERROR] Failed handling ES setup: %s", res)
				return project, errors.New(fmt.Sprintf("Bad status code from ES: %d", res.StatusCode))
			}

			log.Printf("[ERROR] Bad Status from ES: %d", res.StatusCode)
			log.Printf("[ERROR] Bad Body from ES: %s", string(respBody))

			if count == 0 {
				count += 1
				log.Printf("[ERROR] Trying default creds for ES once before failing")
				return RunInit(dbclient, storageClient, gceProject, environment, cacheDb, dbType, true, count)
			}

			return project, errors.New(fmt.Sprintf("Bad status code from ES: %d", res.StatusCode))
		} else {
			//log.Printf("\n\n[INFO] Should check for SSO during setup - finding main org\n\n")
			/*
				orgs, err := GetAllOrgs(ctx)
				if err == nil {
					for _, org := range orgs {
						if len(org.ManagerOrgs) == 0 && len(org.SSOConfig.SSOEntrypoint) > 0 {
							log.Printf("[INFO] Set initial SSO url for logins to %s", org.SSOConfig.SSOEntrypoint)
							SSOUrl = org.SSOConfig.SSOEntrypoint
							break
						}
					}
				} else {
					log.Printf("[WARNING] Error loading orgs: %s", err)
				}
			*/
		}
	} else {
		// Fix potential cloud init problems here
	}

	return project, nil
}

func checkImportPath() bool {
	info, ok := runtimeDebug.ReadBuildInfo()
	if !ok {
		return false
	}

	for _, dep := range info.Deps {
		if strings.Contains(dep.Path, "shuffle-shared") && dep.Path != AllowedImportPath() {
			return false
		}

		if dep.Path == AllowedImportPath() {
			return true
		}
	}

	return false

}

type customTransport struct {
	apiKey string
	rt     http.RoundTripper
}

func (t *customTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	// Inject custom Authorization header
	req.Header.Set("Authorization", "ApiKey "+t.apiKey)

	// You can also inject other headers here, e.g. X-Custom-Header
	return t.rt.RoundTrip(req)
}

func checkNoInternet() OnpremLicense {

	license := OnpremLicense{
		Valid: false,
		Tenant: OnpremLimits{
			Active: false,
			Limit:  3,
		},
		Environment: OnpremLimits{
			Active: false,
			Limit:  1,
		},
		AppRuns: OnpremLimits{
			Active: false,
			Limit:  25000,
		},
		Timeout:  "",
		Branding: false,
	}

	// ==== LICENSE BYPASS PATCH ====
	license.Valid = true
	license.Environment.Active = true
	license.Environment.Limit = 1000000000
	license.Tenant.Active = true
	license.Tenant.Limit = 1000000000
	license.AppRuns.Active = true
	license.AppRuns.Limit = 1000000000
	license.Branding = true
	license.Timeout = "01-01-2100"
	return license
	// ==== END LICENSE BYPASS PATCH ====

	licenseKey := os.Getenv("SHUFFLE_LICENSE")
	if len(licenseKey) == 0 {
		return license
	}

	if len(licenseKey) < 32 {
		log.Printf("[ERROR] License key is too short")
		return license
	}

	// Split the license key into chunks of 32 characters
	licenseParts := []string{}
	for i := 0; i < len(licenseKey); i += 32 {
		end := i + 32

		if end > len(licenseKey) {
			end = len(licenseKey)
		}
		licenseParts = append(licenseParts, licenseKey[i:end])
	}

	licenseKeyPart := licenseParts[0]
	sum := sha256.Sum256([]byte(licenseKeyPart))
	encodedString := hex.EncodeToString(sum[:])

	appRunsLimitKey := ""
	if len(licenseParts) > 1 {
		appRunsLimitKey = licenseParts[1]
	}

	appRunsLimitHash := sha256.Sum256([]byte(appRunsLimitKey))
	encodedAppRunsLimit := hex.EncodeToString(appRunsLimitHash[:])

	tenantKey := ""
	if len(licenseParts) > 2 {
		tenantKey = licenseParts[2]
	}

	tenantHash := sha256.Sum256([]byte(tenantKey))
	encodedTenant := hex.EncodeToString(tenantHash[:])
	environmentKey := ""
	if len(licenseParts) > 3 {
		environmentKey = licenseParts[3]
	}

	environmentHash := sha256.Sum256([]byte(environmentKey))
	encodedEnvironment := hex.EncodeToString(environmentHash[:])

	branding := ""
	if len(licenseParts) > 4 {
		branding = licenseParts[4]
	}

	brandingHash := sha256.Sum256([]byte(branding))
	encodedBranding := hex.EncodeToString(brandingHash[:])
	// Returns a map[sha256]timeout string
	onpremKeys := GetOnpremKeys()
	if timeout, ok := onpremKeys[encodedString]; ok {
		// Check if current time is MORE than the encoded timeout. The timeout format
		parsedTimeout, err := time.Parse("02-01-2006", timeout)
		if err != nil {
			log.Printf("[ERROR] Failed parsing license timeout: %s", err)
		} else {
			if time.Now().Before(parsedTimeout) {

				license.Valid = true
				license.Timeout = timeout

				if len(tenantKey) > 0 && len(encodedTenant) > 0 {
					amount := GetTenantAmount(encodedTenant)
					license.Tenant.Limit = int64(amount)
					if amount > 3 {
						license.Tenant.Active = true
					} else {
						license.Tenant.Active = false
					}
				} else {
					license.Tenant.Limit = 3
					license.Tenant.Active = false
				}

				//check env limit
				if len(environmentKey) > 0 && len(encodedEnvironment) > 0 {
					amount := GetRuntimeLocationAmount(encodedEnvironment)
					license.Environment.Limit = int64(amount)
					if amount > 1 {
						license.Environment.Active = true
					} else {
						license.Environment.Active = false
					}

				} else {
					license.Environment.Limit = 1
					license.Environment.Active = false
				}

				//check branding enable
				if len(branding) > 0 && len(encodedBranding) > 0 {
					branding := GetBrandingAvailable(encodedBranding)
					license.Branding = branding
				} else {
					license.Branding = false
				}

				//check app runs limit
				if len(appRunsLimitKey) > 0 && len(encodedAppRunsLimit) > 0 {
					amount := GetWorkflowRunAmount(encodedAppRunsLimit)
					license.AppRuns.Limit = int64(amount)
					if amount > 25000 {
						license.AppRuns.Active = true
					} else {
						license.AppRuns.Active = false
					}
				}

				return license
			} else {
				log.Printf("[ERROR] License key has expired on %s", timeout)
				return license
			}
		}
	}

	log.Printf("[ERROR] No valid license key found based SHUFFLE_LICENSE %s", licenseKey)
	return license
}

func UploadAppSpecFiles(ctx context.Context, client *storage.Client, api WorkflowApp, parsed ParsedOpenApi) (WorkflowApp, error) {
	extraPath := fmt.Sprintf("extra_specs/%s/appspec.json", api.ID)
	openApiPath := fmt.Sprintf("extra_specs/%s/openapi.json", parsed.ID)
	//log.Printf("[WARNING] Should save actions as other part: %s", extraPath)

	appBytes, err := json.Marshal(api)
	if err != nil {
		log.Printf("[WARNING] Failed marshaling app during failure fix: %s", err)
		return api, err
	}

	openapiBytes, err := json.Marshal(parsed)
	if err != nil {
		log.Printf("[WARNING] Failed marshaling app's OpenAPI during failure fix: %s", err)
		return api, err
	}

	// Api.yaml
	bucket := client.Bucket(project.BucketName)

	if len(api.ID) > 0 {
		obj := bucket.Object(extraPath)
		w := obj.NewWriter(ctx)
		if _, err := fmt.Fprint(w, string(appBytes)); err != nil {
			log.Printf("[WARNING] Failed writing app file: %s", err)
			return api, err
		}

		// Close, just like writing a file.
		if err := w.Close(); err != nil {
			log.Printf("[WARNING] Failed closing app file: %s", err)
			return api, err
		}
	}

	// OpenAPI
	if len(parsed.ID) > 0 {
		obj := bucket.Object(openApiPath)
		w := obj.NewWriter(ctx)
		if _, err := fmt.Fprint(w, string(openapiBytes)); err != nil {
			log.Printf("[WARNING] Failed writing openapi file: %s", err)
			return api, err
		}

		// Close, just like writing a file.
		if err := w.Close(); err != nil {
			log.Printf("[WARNING] Failed closing openapi file: %s", err)
			return api, err
		}

		log.Printf("[DEBUG] Uploaded OpenAPI for api with ID '%s' to path: %s", api.ID, openApiPath)
	}

	fullParsedPath := fmt.Sprintf("gs://%s/extra_specs/%s", project.BucketName, api.ID)
	log.Printf("[DEBUG] Successfully uploaded app action data to path: %s. App ID: %s, OpenAPI ID: %s", fullParsedPath, api.ID, parsed.ID)
	api.Actions = []WorkflowAppAction{}
	api.ActionFilePath = fullParsedPath
	err = SetWorkflowAppDatastore(ctx, api, api.ID)
	if err != nil {
		log.Printf("[ERROR] Failed adding app to db: %s", err)
		return api, err
	}

	return api, nil
}

func SetUsecase(ctx context.Context, usecase Usecase, optionalEditedSecondsOffset ...int) error {
	var err error
	nameKey := "usecases"
	name := strings.ToLower(strings.Replace(usecase.Name, " ", "_", -1))

	timeNow := int64(time.Now().Unix())
	usecase.Edited = timeNow

	// New struct, to not add body, author etc
	data, err := json.Marshal(usecase)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in setapp: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, name, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, name, nil)
		if _, err := project.Dbclient.Put(ctx, key, &usecase); err != nil {
			log.Printf("[WARNING] Error adding usecase: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, name)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for setusecase: %s", err)
		}
	}

	return nil
}

func GetUsecase(ctx context.Context, name string) (*Usecase, error) {
	usecase := &Usecase{}
	nameKey := "usecases"
	id := strings.ToLower(strings.Replace(name, " ", "_", -1))

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &usecase)
			if err == nil {
				return usecase, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for usecase: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		//log.Printf("GETTING ES USER %s",
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: id,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return usecase, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return usecase, errors.New("Usecase doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return usecase, err
		}

		wrapped := UsecaseWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return usecase, err
		}

		usecase = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		if err := project.Dbclient.Get(ctx, key, usecase); err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[INFO] Error in usecase loading. Migrating usecase to new workflow handler.")
				err = nil
			} else {
				// Let it cache. No point in DB searching every time
				//return usecase, err
			}
		}
	}

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for usecase %s", cacheKey)
		data, err := json.Marshal(usecase)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getusecase: %s", err)
			return usecase, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getusecase: %s", err)
		}
	}

	return usecase, nil

}

func SetUsecaseNew(ctx context.Context, usecase *UsecaseInfo) error {
	if usecase == nil {
		return errors.New("usecase cannot be nil")
	}

	nameKey := "Usecases"
	timeNow := int64(time.Now().Unix())

	// Set created time for new usecase
	if usecase.Created == 0 {
		usecase.Created = timeNow
	}
	// Always update edited time
	usecase.Edited = timeNow

	// Marshal data for storage and caching
	data, err := json.Marshal(usecase)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in SetUsecaseNew: %s", err)
		return err
	}

	// Store in database based on type
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, usecase.Id, data)
		if err != nil {
			log.Printf("[ERROR] Failed indexing usecase in OpenSearch: %s", err)
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, usecase.Id, nil)
		if _, err := project.Dbclient.Put(ctx, key, usecase); err != nil {
			log.Printf("[ERROR] Error adding usecase: %s", err)
			return err
		}
	}

	// Update cache
	if project.CacheDb {
		// Cache the usecase by ID
		cacheKey := fmt.Sprintf("%s_%s", nameKey, usecase.Id)
		partnerCacheKey := fmt.Sprintf("%s_partner_%s", nameKey, usecase.CompanyInfo.Id)
		SetCache(ctx, partnerCacheKey, data, 30)
		SetCache(ctx, cacheKey, data, 30)
	}

	return nil
}

// GetIndividualUsecase retrieves a single usecase by its ID
func GetIndividualUsecase(ctx context.Context, id string) (UsecaseInfo, error) {
	nameKey := "Usecases"
	usecase := UsecaseInfo{}
	// Check cache first
	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
		cacheData, err := GetCache(ctx, cacheKey)
		if err == nil {
			// Cache hit
			var usecase UsecaseInfo
			cacheBytes, ok := cacheData.([]byte)
			if ok {
				err = json.Unmarshal(cacheBytes, &usecase)
				if err == nil {
					return usecase, nil
				}
			}
		}
	}

	// Get from datastore if not in cache
	k := datastore.NameKey(nameKey, id, nil)
	err := project.Dbclient.Get(ctx, k, &usecase)
	if err != nil {
		if strings.Contains(err.Error(), `cannot load field`) {
			log.Printf("[ERROR] Error in getting usecase (3): %s", err)
			err = nil
		} else {
			return usecase, fmt.Errorf("failed to get usecase by ID: %w", err)
		}
	}

	// Cache the result
	if project.CacheDb {
		data, err := json.Marshal(usecase)
		if err == nil {
			cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
			SetCache(ctx, cacheKey, data, 30)
		}
	}

	return usecase, nil
}

// GetUsecases retrieves multiple usecases by partner ID
func GetPartnerUsecases(ctx context.Context, partnerId string) ([]UsecaseInfo, error) {
	nameKey := "Usecases"
	var usecases []UsecaseInfo

	// Check cache first
	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_partner_%s", nameKey, partnerId)
		cacheData, err := GetCache(ctx, cacheKey)
		if err == nil {
			var cachedUsecases []UsecaseInfo
			cacheBytes, ok := cacheData.([]byte)
			if ok {
				err = json.Unmarshal(cacheBytes, &cachedUsecases)
				if err == nil {
					return cachedUsecases, nil
				}
			}
		}
	}

	// Get from datastore if not in cache
	q := datastore.NewQuery(nameKey).Filter("companyInfo.id=", partnerId)
	_, err := project.Dbclient.GetAll(ctx, q, &usecases)
	if err != nil {
		if strings.Contains(err.Error(), `cannot load field`) {
			log.Printf("[ERROR] Error in getting usecase (3): %s", err)
			err = nil
		} else {
			return usecases, fmt.Errorf("failed to get usecases by partner ID: %w", err)
		}
	}

	// Cache the results
	if project.CacheDb && len(usecases) > 0 {
		data, err := json.Marshal(usecases)
		if err == nil {
			cacheKey := fmt.Sprintf("%s_partner_%s", nameKey, partnerId)
			SetCache(ctx, cacheKey, data, 30)
		}
	}

	return usecases, nil
}

func SetNewDeal(ctx context.Context, deal ResellerDeal) error {
	nameKey := "reseller_deal"

	timeNow := int64(time.Now().Unix())
	deal.Edited = timeNow
	if deal.Created == 0 {
		deal.Created = timeNow
	}

	if len(deal.ID) == 0 {
		deal.ID = uuid.NewV4().String()
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(deal)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set deal: %s", err)
		return err
	}

	// FIXMe: Shouldn't really be possible, but may be useful for hybrid (?)
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, deal.ID, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, deal.ID, nil)
		if _, err := project.Dbclient.Put(ctx, key, &deal); err != nil {
			log.Printf("[WARNING] Error adding deal: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, deal.ID)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for deal: %s", err)
		}
	}

	return nil
}

func GetCacheKeyCount(ctx context.Context, orgId string, category string) (int, error) {
	nameKey := "org_cache"
	if category == "default" {
		category = ""
	}

	count := -1
	if len(orgId) == 0 {
		return count, errors.New("OrgId is required for GetCacheKeyCount")
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 10000,
			"sort": map[string]interface{}{
				"edited": map[string]interface{}{
					"order": "desc",
				},
			},
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
					},
				},
			},
		}

		if len(category) > 0 {
			// Change out the "must" part entirely to contain the workflow id as well
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = []map[string]interface{}{
				{
					"match": map[string]interface{}{
						"org_id": orgId,
					},
				},
				{
					"match": map[string]interface{}{
						"category": category,
					},
				},
			}
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[ERROR] Error encoding cache key count query: %s", err)
			return count, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return count, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get cache key count): %s", err)
			return count, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			log.Printf("[ERROR] Error reading response body for cache key count: %s", err)
			return count, err
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			if debug {
				log.Printf("[DEBUG] Body of cache key count is bad (1). Status: %d. This is fixed by adding an item. Body: %s", res.StatusCode, string(respBody))
			}

			if res.StatusCode == 404 {
				return count, nil // No keys found
			}

			return count, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		wrapped := CacheKeySearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			log.Printf("[ERROR] Error unmarshalling response body for cache key count: %s", err)
			return count, err
		}

		count = wrapped.Hits.Total.Value
	} else {
		query := datastore.NewQuery(nameKey).Filter("OrgId =", orgId)
		if len(category) > 0 {
			query = query.Filter("category =", category)
		}

		newCount, err := project.Dbclient.Count(ctx, query)
		if err != nil {
			//log.Printf("[ERROR] Error counting cache keys for org %s: %s", orgId, err)
			return count, err
		} else {
			count = newCount
		}
	}

	return count, nil
}

func GetAllCacheKeys(ctx context.Context, orgId string, category string, max int, inputcursor string, cleanupDepthParam ...int) ([]CacheKeyData, string, error) {
	if os.Getenv("SHUFFLE_SWARM_CONFIG") == "run" || project.Environment == "worker" {
		if debug && category != "protected" {
			log.Printf("[DEBUG] Disabled GetAllCacheKeys for '%s' in worker swarm mode", category)
		}

		return []CacheKeyData{}, "", errors.New("Not available in worker mode")
	}

	nameKey := "org_cache"
	cleanupDepth := 0
	if len(cleanupDepthParam) > 0 {
		if cleanupDepthParam[0] > 0 {
			cleanupDepth = cleanupDepthParam[0]
		}
	}

	if strings.ToLower(category) == "default" {
		category = ""
	}

	category = strings.ReplaceAll(strings.ToLower(category), " ", "_")
	cacheKey := fmt.Sprintf("%s_%s_%s_%s", nameKey, inputcursor, orgId, category)

	// Find cache and return instantly
	cacheKeys := []CacheKeyData{}
	//if project.CacheDb && category == "protected" {
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &cacheKeys)
			if err == nil {

				// Avoids an issue with bad caching
				if len(cacheKeys) > 1 {
					return cacheKeys, "", nil
				} 
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for appstats: %s", err)
		}
	}

	if max > 1000 {
		max = 1000
	}

	// Look for
	cursor := ""
	if project.DbType == "opensearch" {
		//log.Printf("[DEBUG] GETTING cachekeys for org %s in item %s", orgId, nameKey)
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": max,
			"sort": map[string]interface{}{
				"edited": map[string]interface{}{
					"order":         "desc",
					"unmapped_type": "date",
				},
			},
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
					},
				},
			},
		}

		if len(category) > 0 {
			// Change out the "must" part entirely to contain the workflow id as well
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = []map[string]interface{}{
				{
					"match": map[string]interface{}{
						"org_id": orgId,
					},
				},
				{
					"match": map[string]interface{}{
						"category": category,
					},
				},
			}
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("Error encoding deal query: %s", err)
			return cacheKeys, "", err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return cacheKeys, "", nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get cachekeys): %s", err)
			return cacheKeys, "", err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return cacheKeys, "", err
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			if debug {
				//log.Printf("[DEBUG] Body of cachekeys is bad (2). Status: %d. This is fixed by adding an item.", res.StatusCode)
			}

			if res.StatusCode == 404 {
				return cacheKeys, "", nil
			}

			return cacheKeys, "", errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		wrapped := CacheKeySearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return cacheKeys, "", err
		}

		newCacheKeys := []CacheKeyData{}
		deletedKeys := 0
		for _, hit := range wrapped.Hits.Hits {
			// Handles a bug from 2.1.0 where keys didn't get assigned properly
			if len(hit.ID) == 20 {
				err = DeleteKey(context.Background(), nameKey, hit.ID)
				if err != nil {
					log.Printf("[ERROR] Failed deleting bad datastore key %s: %s", hit.ID, err)
				}

				deletedKeys += 1
				continue
			}

			if hit.Source.OrgId != orgId {
				continue
			}

			newCacheKeys = append(newCacheKeys, hit.Source)
		}

		if deletedKeys > 0 {
			log.Printf("[WARNING] Removed %d bad datastore key(s) for org %s. This is an autofix for issues from 2.1.0.", deletedKeys, orgId)
		}

		//log.Printf("[INFO] Got %d cachekeys for org %s (es)", len(newCacheKeys), orgId)
		cacheKeys = newCacheKeys
	} else {

		// Query datastore with pages
		query := datastore.NewQuery(nameKey).Filter("OrgId =", orgId).Order("-Edited")
		if len(category) > 0 {
			query = query.Filter("category =", category)
		} else {
			query = query.Filter("category =", "")
		}

		query = query.Limit(max)
		if inputcursor != "" {
			outputcursor, err := datastore.DecodeCursor(inputcursor)
			if err != nil {
				log.Printf("[WARNING] Error decoding cursor: %s", err)
				return cacheKeys, "", err
			}

			query = query.Start(outputcursor)
		}

		// Skip page in query
		errcnt := 0
		cursorStr := inputcursor
		var err error
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerKey := CacheKeyData{}
				_, err := it.Next(&innerKey)
				if err != nil {
					//log.Printf("[WARNING] Workflow iterator issue: %s", err)
					break
				}

				cacheKeys = append(cacheKeys, innerKey)
			}

			if err != iterator.Done {
				//log.Printf("[ERROR] Failed fetching results for cache: %v", err)
				//break
			}

			if len(cacheKeys) >= max {
				// Get next cursor and set it as the new cursor

				nextCursor, err := it.Cursor()
				if err != nil {
					log.Printf("[ERROR] Cursorerror for cache: %s", err)
				} else {
					cursor = fmt.Sprintf("%s", nextCursor)
				}

				break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				if errcnt == 0 && (strings.Contains(err.Error(), "no matching index") || strings.Contains(err.Error(), "not ready to serve")) {
					log.Printf("[WARNING] No matching index for cache. Running without edit index: %s.", err)
					query = datastore.NewQuery(nameKey).Filter("OrgId =", orgId).Limit(max)
					errcnt += 1
					continue
				}

				log.Printf("[ERROR] Problem with cursor: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)

				cursor = cursorStr
				//cursorStr = nextCursor
				//break
			}

		}
	}

	categories := []string{}
	if len(category) > 0 && category != "default" {
		categories = []string{category}
	} else if len(category) == 0 || category == "default" {
		for _, cacheKey := range cacheKeys {
			if len(cacheKey.Category) > 0 && cacheKey.Category != "default" && !ArrayContains(categories, cacheKey.Category) {
				categories = append(categories, cacheKey.Category)
			}
		}
	}

	// Get category settings and do stuff
	skipCache := false
	if len(categories) > 0 {
		removedKeys := []string{}
		for _, category := range categories {
			categoryConfig, err := GetDatastoreCategoryConfig(ctx, orgId, category)
			if err != nil {
				continue
			}

			// Kind of arbitrary, but a good start
			if categoryConfig.Settings.Timeout >= 60 {
				// Check if any key is edited within this time
				editedTime := time.Now().Unix() - int64(categoryConfig.Settings.Timeout)
				backgroundCtx := context.Background()
				deleteKeys := []string{}
				newCacheKeys := []CacheKeyData{}
				for _, cacheKey := range cacheKeys {
					if cacheKey.Category != category {
						if debug {
							log.Printf("[WARNING] Cache key '%s' has category '%s' which doesn't match expected category '%s'. Skipping timeout check for this key.", cacheKey.Key, cacheKey.Category, category)
						}
						continue
					}

					if cacheKey.Edited >= editedTime {
						newCacheKeys = append(newCacheKeys, cacheKey)
					} else {
						if debug {
							//log.Printf("[DEBUG] Should delete cache key '%s' with edited time %d. Timed out!", cacheKey.Key, cacheKey.Edited)
						}

						// URL encode the key
						// FIXME: Not sure why SOMETIMES it isn't QueryEscaped 
						// and sometimes the Key doesn't match
						//parsedRawkey := url.QueryEscape(cacheKey.Key)
						parsedKey := fmt.Sprintf("%s_%s_%s", orgId, cacheKey.Key, category)

						deleteKeys = append(deleteKeys, parsedKey)
						removedKeys = append(removedKeys, cacheKey.Key+cacheKey.Category)
						skipCache = true
					}
				}

				//8aa779dd-773c-4e80-ac9d-e46944889777_<h1>index of /doc/misp/feed-osint</h1>_ioc_domain
				//8aa779dd-773c-4e80-ac9d-e46944889777_<h1>index of /doc/misp/feed-osint</h1>_ioc_domain

				if len(deleteKeys) > 0 {
					cursor = ""
					if debug {
						log.Printf("[DEBUG] Removing %d cache keys for category '%s' in org '%s' due to timeout settings. This is an auto-fix for stale cache keys. Running recursion.", len(deleteKeys), category, orgId)
					}

					err = DeleteKeys(backgroundCtx, nameKey, deleteKeys)
					if err != nil {
						log.Printf("[ERROR] Failed deleting cache keys for category '%s' in org '%s': %s", category, orgId, err)
					} else {
						if len(deleteKeys) == max {
							cleanupDepth += 1
							if cleanupDepth >= 5 {
								log.Printf("[WARNING] Cleanup depth for cache keys has reached %d. Stopping recursion to prevent potential infinite loop. Please investigate if there are many stale keys for category '%s' in org '%s'.", cleanupDepth, category, orgId)
							} else {
								// Makes sure we do a toooon of keys at once when cleanup is relevant
								newKeys, _, err := GetAllCacheKeys(ctx, orgId, category, 500, "", cleanupDepth)
								if err == nil { 
									cacheKeys = newKeys
								}
							}
						}
					}
				}
			}
		}

		// Find missing keys from
		allNewCacheKeys := []CacheKeyData{}
		for _, cacheKey := range cacheKeys {
			if cacheKey.Category == "default" || len(cacheKey.Category) == 0 {
				allNewCacheKeys = append(allNewCacheKeys, cacheKey)
				continue
			}

			parsedKey := cacheKey.Key + cacheKey.Category
			if !ArrayContains(removedKeys, parsedKey) {
				allNewCacheKeys = append(allNewCacheKeys, cacheKey)
			}
		}

		cacheKeys = allNewCacheKeys
	}

	// Sort by edited field
	slice.Sort(cacheKeys[:], func(i, j int) bool {
		return cacheKeys[i].Edited > cacheKeys[j].Edited
	})

	for index, newKey := range cacheKeys {

		// Only runs on specific loads of a category
		if newKey.Encrypted && newKey.Category == category {
			encryptionKey := fmt.Sprintf("%s_%d_%s_%s", newKey.OrgId, newKey.Created, newKey.Category, newKey.Key)
			newValue, err := HandleKeyDecryption([]byte(newKey.Value), encryptionKey)
			if err == nil {
				cacheKeys[index].Value = string(newValue)
				cacheKeys[index].Encrypted = false
			} else {
				log.Printf("[ERROR] Failed decrypting datastore key %s: %s. Category: %s", newKey.Key, err, newKey.Category)
			}
		}

	}

	foundOrg, err := GetOrg(ctx, orgId)
	if err == nil && len(foundOrg.CreatorOrg) > 0 && foundOrg.CreatorOrg != orgId {
		parentOrg, err := GetOrg(ctx, foundOrg.CreatorOrg)
		if err != nil {
			log.Printf("[ERROR] Failed finding parent org %s for org %s: %s", foundOrg.CreatorOrg, orgId, err)
		} else {
			parentOrgCache, _, err := GetAllCacheKeys(ctx, parentOrg.Id, "", max, inputcursor)
			if err != nil {
				log.Printf("[ERROR] Failed getting parent org cache keys for org %s: %s", parentOrg.Id, err)
			} else {
				if debug {
					//log.Printf("[DEBUG] Loaded %d parent org cache keys for org %s. Validating if child org %s should get the keys", len(parentOrgCache), parentOrg.Id, orgId)
				}

				for _, parentCache := range parentOrgCache {
					/*
						if debug && len(parentCache.SuborgDistribution) > 0 {
							log.Printf("[DEBUG] Parent org %s keys: %#v", parentOrg.Id, parentCache.SuborgDistribution)
						}
					*/

					if !ArrayContains(parentCache.SuborgDistribution, orgId) {
						continue
					}

					// Clean up just in case
					parentCache.PublicAuthorization = ""
					parentCache.SuborgDistribution = []string{orgId}
					cacheKeys = append(cacheKeys, parentCache)
				}
			}
		}
	}

	// Only cache if NO cursor at all.
	// Otherwise we need to track and clean up all cursors(?)
	if project.CacheDb && !skipCache {
		newcache, err := json.Marshal(cacheKeys)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling cacheKeys: %s", err)
			return cacheKeys, cursor, nil
		}

		err = SetCache(ctx, cacheKey, newcache, 5)
		if err != nil {
			log.Printf("[WARNING] Failed updating cache keys cache: %s", err)
		}
	}

	return cacheKeys, cursor, nil
}

func GetAllDeals(ctx context.Context, orgId string) ([]ResellerDeal, error) {
	nameKey := "reseller_deal"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)

	deals := []ResellerDeal{}
	if project.DbType == "opensearch" {
		log.Printf("GETTING deals for org %s in item %s", orgId, nameKey)
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": 1000,
			"sort": map[string]interface{}{
				"edited": map[string]interface{}{
					"order": "desc",
				},
			},
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"reseller_org": orgId,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("Error encoding deal query: %s", err)
			return deals, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return deals, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get deals): %s", err)
			return deals, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return deals, err
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			log.Printf("[WARNING] Body of deals is bad: %s", string(respBody))
			return deals, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		wrapped := DealSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return deals, err
		}

		newDeals := []ResellerDeal{}
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.ResellerOrg != orgId {
				continue
			}

			newDeals = append(newDeals, hit.Source)
		}

		log.Printf("[INFO] Got %d deals for org %s", len(newDeals), orgId)
		deals = newDeals
	} else {

		query := datastore.NewQuery(nameKey).Filter("reseller_org =", orgId).Limit(50)
		_, err := project.Dbclient.GetAll(ctx, query, &deals)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting deals for org: %s", orgId)
				return deals, err
			}
		}

		log.Printf("[INFO] Got %d deals for org %s", len(deals), orgId)
	}

	if project.CacheDb {
		newdeal, err := json.Marshal(deals)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling deals: %s", err)
			return deals, nil
		}

		err = SetCache(ctx, cacheKey, newdeal, 30)
		if err != nil {
			log.Printf("[WARNING] Failed updating deal cache: %s", err)
		}
	}

	return deals, nil
}

func GetAppStats(ctx context.Context, id string) (*Conversionevents, error) {
	stats := &Conversionevents{}

	nameKey := "app_stats"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &stats)
			if err == nil {
				return stats, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for appstats: %s", err)
		}
	}

	if project.DbType == "opensearch" {
		return &Conversionevents{}, errors.New("es api not supported yet")
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, stats); err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[ERROR] Error in appstats loading of %s: %s", id, err)
			}
		}
	}

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for workflow %s", cacheKey)
		data, err := json.Marshal(stats)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getappstats: %s", err)
			return stats, nil
		}

		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getappstats: %s", err)
		}
	}

	return stats, nil
}

// Finds custom oauth2 secret etc. based on
func GetHostedOAuth(ctx context.Context, id string) (*DataToSend, error) {
	stats := &DataToSend{}

	nameKey := "oauth2_storage"
	if project.DbType == "opensearch" {
		return &DataToSend{}, errors.New("es api not supported for custom oauth")
	} else {
		key := datastore.NameKey(nameKey, id, nil)
		if err := project.Dbclient.Get(ctx, key, stats); err != nil {
			log.Printf("[WARNING] Error in oauth2 key loading of ID %s: %s", id, err)
		}
	}

	return stats, nil
}

func GetCreatorStats(ctx context.Context, creatorName string, startDate string, endDate string) ([]CreatorStats, error) {
	stats := []CreatorStats{}
	nameKey := "creator_stats"

	log.Printf("[AUDIT] Looking for creator stats for name %s", creatorName)

	q := datastore.NewQuery(nameKey).Filter("creator =", creatorName).Limit(1)
	_, err := project.Dbclient.GetAll(ctx, q, &stats)
	if err != nil {
		if strings.Contains(err.Error(), `cannot load field`) {
			log.Printf("[INFO] error %s", err)
			err = nil
		} else {
			log.Printf("[INFO] error loading data for %s", creatorName)
		}
	}

	if len(stats) == 0 { // used to handle error when creator name is not valid
		return stats, nil
	}

	var parsedStartDate time.Time
	var parsedEndDate time.Time

	startPresent := false
	endPresent := false
	bothPresent := false

	if len(startDate) > 0 && len(endDate) > 0 {
		parsedStartDate, err = time.Parse("2006-01-02", startDate)
		if err != nil {
			log.Printf("[ERROR] Incorrect date format %s: %s", startDate, err)
			return stats, err
		}
		parsedEndDate, err = time.Parse("2006-01-02", endDate)
		if err != nil {
			log.Printf("[ERROR] Incorrect date format %s: %s", endDate, err)
			return stats, err
		}
		bothPresent = true
	} else if len(startDate) > 0 {
		parsedStartDate, err = time.Parse("2006-01-02", startDate)
		if err != nil {
			log.Printf("[ERROR] Incorrect date format %s: %s", startDate, err)
			return stats, err
		}
		startPresent = true
	} else if len(endDate) > 0 {
		parsedEndDate, err = time.Parse("2006-01-02", endDate)
		if err != nil {
			log.Printf("[ERROR] Incorrect date format %s: %s", endDate, err)
			return stats, err
		}
		endPresent = true
	}

	if startPresent == false && endPresent == false && bothPresent == false { // if no query parameter is provided
		if len(stats[0].AppStats) > 0 {
			// calculating most conversed app and sorts in order highest first
			sort.Slice(stats[0].AppStats, func(i, j int) bool {
				return len(stats[0].AppStats[i].Events[0].Data) > len(stats[0].AppStats[j].Events[0].Data)
			})
			stats[0].MostConversedApp = stats[0].AppStats[0].AppName

			// calculating most clicked app and sorts in order highest first
			sort.Slice(stats[0].AppStats, func(i, j int) bool {
				return len(stats[0].AppStats[i].Events[1].Data) > len(stats[0].AppStats[j].Events[1].Data)
			})
			stats[0].MostClickedApp = stats[0].AppStats[0].AppName
		}
		return stats, err
	}

	var updatedStats []AppStats

	for index, i := range stats[0].AppStats { // This is for filtering data by dates.
		var appData []WidgetPoint
		var totalConversions int64
		var totalClicks int64

		for eventIndex, j := range i.Events {
			var appEvents []WidgetPointData
			if len(j.Data) > 0 {
				for _, k := range j.Data {
					if len(k.Key) > 0 {
						parsedData, err := time.Parse("2006-01-02", k.Key)
						if err != nil {
							log.Printf("[ERROR] error parsing data %s: %s", k.Key, err)
							return stats, err
						}
						if bothPresent == true {
							if parsedData.After(parsedStartDate) && parsedData.Before(parsedEndDate) {
								appEvents = append(appEvents, k)
								if eventIndex == 0 {
									totalConversions += k.Data
								}
								if eventIndex == 1 {
									totalClicks += k.Data
								}
							}
						}
						if startPresent == true {
							if parsedData.After(parsedStartDate) {
								appEvents = append(appEvents, k)
								if eventIndex == 0 {
									totalConversions += k.Data
								}
								if eventIndex == 1 {
									totalClicks += k.Data
								}
							}
						}
						if endPresent == true {
							if parsedData.Before(parsedEndDate) {
								appEvents = append(appEvents, k)
								if eventIndex == 0 {
									totalConversions += k.Data
								}
								if eventIndex == 1 {
									totalClicks += k.Data
								}
							}
						}

					} else {
						// stats[0].AppStats[index] = AppStats{} // for discarding apps with no events
					}

				}
				if eventIndex == 0 {
					appData = append(appData, WidgetPoint{"conversion", appEvents})
					// appData[0].Key = "conversion"
					// appData[0].Data = appEvents
				}

				if eventIndex == 1 {
					appData = append(appData, WidgetPoint{"click", appEvents})
					// appData[1].Key = "click"
					// appData[1].Data = appEvents
				}
				//
			}

		}
		updatedStats = append(updatedStats, i) //fill in updatedStats with old data
		updatedStats[index].TotalConversions = int(totalConversions)
		updatedStats[index].TotalClicks = int(totalClicks)
		updatedStats[index].Events = appData // update events with filtered events
	}
	stats[0].AppStats = updatedStats // updating stats with updated values

	if len(stats[0].AppStats) > 1 {
		// calculating most conversed app and sorts in order highest first

		sort.Slice(stats[0].AppStats, func(i, j int) bool {
			if len(stats[0].AppStats[i].Events) > 1 {
				return len(stats[0].AppStats[i].Events[0].Data) > len(stats[0].AppStats[j].Events[0].Data)
			} else {
				return false
			}
		})
		stats[0].MostConversedApp = stats[0].AppStats[0].AppName

		// // calculating most clicked app and sorts in order highest first
		sort.Slice(stats[0].AppStats, func(i, j int) bool {
			if len(stats[0].AppStats[i].Events) > 1 {
				return len(stats[0].AppStats[i].Events[0].Data) > len(stats[0].AppStats[j].Events[0].Data)
			} else {
				return false
			}
		})
		stats[0].MostClickedApp = stats[0].AppStats[1].AppName
	}

	return stats, err
}

// Stopped clearing them out as the result from it is used in subsequent workflows as well (subflows). This means the 31 min timeout is default.
func RunCacheCleanup(ctx context.Context, workflowExecution WorkflowExecution) {
	// Keeping cache for 30-60 min due to rerun management
	if project.Environment == "cloud" {
		return
	}

	// As worker will be killed off anyway otherwise
	if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" {
		return
	}

	//log.Printf("[INFO][%s] Cleaning up cache for all %d results.", workflowExecution.ExecutionId, len(workflowExecution.Results))
	//for _, result := range workflowExecution.Results {
	//	cacheId := fmt.Sprintf("%s_%s_result", workflowExecution.ExecutionId, result.Action.ID)
	//	DeleteCache(ctx, cacheId)
	//}
	//DeleteCache(ctx, fmt.Sprintf("workflowexecution_%s", workflowExecution.ExecutionId))
}

func ValidateFinished(ctx context.Context, extra int, workflowExecution WorkflowExecution) bool {

	//log.Printf("\n\nVALIDATING FINISHED. STATUS: %s. Action: %d, Results: %d\n", workflowExecution.Status, len(workflowExecution.Workflow.Actions), len(workflowExecution.Results))

	// Validates RERUN of single actions  (new 2025)
	// Identified by:
	// 1. Predefined result from previous exec
	// 2. Only ONE action
	// 3. Every predefined result having result.Action.Category == "rerun"
	rerunFound := false
	if len(workflowExecution.Workflow.Actions) == 1 && len(workflowExecution.Results) > 0 {
		found := false
		for _, result := range workflowExecution.Results {
			if result.Action.Category == "rerun" {
				rerunFound = true
			}

			// Find if the result for the single action exists or not
			if result.Action.ID == workflowExecution.Workflow.Actions[0].ID {
				found = true
			}
		}

		//log.Printf("ACTIONS: %d, RESULTS: %d, FOUND: %t", len(workflowExecution.Workflow.Actions), len(workflowExecution.Results), found)

		if found {
			// Continue -> this means finished check is ok
		} else {
			return false
		}
	}

	// Print 1/5 times to
	// Should find it if it doesn't exist
	//if extra == -1 {
	extra = 0
	for _, trigger := range workflowExecution.Workflow.Triggers {
		if trigger.Name == "User Input" || trigger.AppName == "User Input" || trigger.Name == "Shuffle Workflow" || trigger.AppName == "Shuffle Workflow" {

			extra += 1
		}
	}

	for _, action := range workflowExecution.Workflow.Actions {
		if action.AppName == "User Input" || action.AppName == "Shuffle Workflow" {
			extra += 1
		}
	}

	workflowExecution, _ = Fixexecution(ctx, workflowExecution)
	//if rand.Intn(5) == 1 || len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions) {
	//log.Printf("[INFO][%s] Workflow Finished Check. Status: %s, Actions: %d, Extra: %d, Results: %d\n", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results))

	if len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra && len(workflowExecution.Workflow.Actions) > 0 {
		validResults := 0
		invalidResults := 0
		subflows := 0

		lastResult := ActionResult{}
		for _, result := range workflowExecution.Results {
			if result.Status == "EXECUTING" || result.Status == "WAITING" {
				//log.Printf("[DEBUG][%s] Waiting for action %s to finish", workflowExecution.ExecutionId, result.Action.ID)
				return false
			}

			if result.Status == "SUCCESS" && result.CompletedAt >= lastResult.CompletedAt {
				lastResult = result
			}

			if result.Status == "SUCCESS" {
				validResults += 1
			}

			if result.Status == "ABORTED" || result.Status == "FAILURE" {
				invalidResults += 1
			}

			if result.Action.AppName == "User Input" || result.Action.AppName == "Shuffle Workflow" {
				subflows += 1
			}
		}

		// Check if status is already set first from cache
		newexec, err := GetWorkflowExecution(ctx, workflowExecution.ExecutionId)
		if err == nil && (newexec.Status == "FINISHED" || newexec.Status == "ABORTED") {
			//log.Printf("[INFO][%s] Already finished from GetWorkflowExecution (validate)! Stopping the rest of the request for execution.", workflowExecution.ExecutionId)
			return true
		}

		if len(workflowExecution.Result) == 0 && len(lastResult.Result) > 0 {
			workflowExecution.Result = lastResult.Result
		}

		workflowExecution.CompletedAt = int64(time.Now().Unix())
		workflowExecution.Status = "FINISHED"
		HandleExecutionCacheIncrement(ctx, workflowExecution)

		func() {
			defer func() {
				if r := recover(); r != nil {
					log.Printf("[ERROR][%s] Panic in ValidateExecutionChronology: %v", workflowExecution.ExecutionId, r)
				}
			}()
			violations := ValidateExecutionChronology(ctx, &workflowExecution)
			if len(violations) > 0 {
				log.Printf("[WARNING][%s] Found %d execution chronology violation(s)", workflowExecution.ExecutionId, len(violations))
				for _, v := range violations {
					log.Printf("[WARNING][%s] %s started %.2fs before parent %s completed", workflowExecution.ExecutionId, v.ActionLabel, float64(v.GapMs)/1000.0, v.ParentID[:8])
				}
			}
		}()

		err = SetWorkflowExecution(ctx, workflowExecution, true)
		if err != nil {
			log.Printf("[ERROR] Failed to set execution during finalization %s: %s", workflowExecution.ExecutionId, err)
		} else {
			log.Printf("[INFO] Finalized execution %s for workflow %s with %d results and status %s", workflowExecution.ExecutionId, workflowExecution.Workflow.ID, len(workflowExecution.Results), workflowExecution.Status)

			// Validate text vs previous executions
			//RunTextClassifier(ctx, workflowExecution)
			if rerunFound {
				return true
			}

			comparisonTime := workflowExecution.CompletedAt - workflowExecution.StartedAt

			userInput := false
			for _, result := range workflowExecution.Results {
				if result.Action.AppName == "User Input" {
					userInput = true
				}
			}

			if comparisonTime > 600 && !userInput {
				// FIXME: Check if there are any actions with delays?

				err := CreateOrgNotification(
					ctx,
					fmt.Sprintf("Workflow %s took too long to run. Time taken: %d seconds", workflowExecution.Workflow.Name, comparisonTime),
					fmt.Sprintf("This notification is made when the execution takes more than 10 minutes."),
					fmt.Sprintf("/workflows/%s?execution_id=%s&view=executions", workflowExecution.Workflow.ID, workflowExecution.ExecutionId),
					workflowExecution.ExecutionOrg,
					true,
					"MEDIUM",
					"workflow_long_execution",
				)

				if err != nil {
					log.Printf("[ERROR] Failed to create notification for workflow %s: %s", workflowExecution.Workflow.ID, err)
				}
			}

			return true
		}
	}

	HandleExecutionCacheIncrement(ctx, workflowExecution)
	return false
}

func SetSuggestion(ctx context.Context, suggestion Suggestion) error {
	nameKey := "Suggestions"
	timeNow := int64(time.Now().Unix())
	suggestion.Edited = timeNow
	if suggestion.Created == 0 {
		suggestion.Created = timeNow
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(suggestion)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set suggestion: %s", err)
		return nil
	}
	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, suggestion.SuggestionID, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, suggestion.SuggestionID, nil)
		if _, err := project.Dbclient.Put(ctx, key, &suggestion); err != nil {
			log.Printf("[WARNING] Error adding suggestion: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, suggestion.SuggestionID)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for set suggestion '%s': %s", cacheKey, err)
		}
	}

	return nil
}

func GetSuggestions(ctx context.Context, creatorname string) ([]Suggestion, error) {
	var suggestions []Suggestion

	nameKey := "Suggestions"
	if project.DbType == "opensearch" {
		// Not implemented
		return []Suggestion{}, nil
	} else {
		//log.Printf("Looking for name %s in %s", appName, nameKey)
		q := datastore.NewQuery(nameKey).Filter("creator =", creatorname).Filter("status =", "")
		_, err := project.Dbclient.GetAll(ctx, q, &suggestions)
		if err != nil && len(suggestions) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting suggestion for: %s. Err: %s", creatorname, err)
				return suggestions, err
			}
		}
	}

	log.Printf("[INFO] Found %d suggestions for name %s in db-connector", len(suggestions), creatorname)

	slice.Sort(suggestions[:], func(i, j int) bool {
		return suggestions[i].Edited > suggestions[j].Edited
	})

	return suggestions, nil
}

func GetSuggestion(ctx context.Context, id string) (*Suggestion, error) {
	suggestion := &Suggestion{}
	nameKey := "Suggestions"

	cacheKey := fmt.Sprintf("%s_%s", nameKey, id)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &suggestion)
			if err == nil {
				return suggestion, nil
			}
		} else {
			//log.Printf("[DEBUG] Failed getting cache for workflow (6): %s", err)
		}
	}

	if project.DbType == "opensearch" {
		return suggestion, nil
	} else {
		key := datastore.NameKey(nameKey, strings.ToLower(id), nil)
		if err := project.Dbclient.Get(ctx, key, suggestion); err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[ERROR] Error in workflow loading. Migrating suggestions to new workflow handler (4): %s", err)
				err = nil
			} else {
				return suggestion, err
			}
		}
	}

	if project.CacheDb {
		//log.Printf("[DEBUG] Setting cache for suggestion %s", cacheKey)
		data, err := json.Marshal(suggestion)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in getsuggestion: %s", err)
			return suggestion, nil
		}

		err = SetCache(ctx, cacheKey, data, 60)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for getsuggestion'%s': %s", cacheKey, err)
		}
	}

	return suggestion, nil
}

func SetConversation(ctx context.Context, input QueryInput) error {
	nameKey := "conversations"

	if len(input.Id) == 0 {
		input.Id = uuid.NewV4().String()
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(input)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in conversation: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, input.Id, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, input.Id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &input); err != nil {
			log.Printf("[WARNING] Error adding conversation: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, input.Id)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for conversation '%s': %s", cacheKey, err)
		}
	}

	return nil
}

func GetConversationHistory(ctx context.Context, conversationId string, limit int) ([]ConversationMessage, error) {
	nameKey := "conversations"

	if conversationId == "" {
		return []ConversationMessage{}, errors.New("conversationId is empty")
	}

	if limit == 0 {
		limit = 100
	}

	cacheKey := fmt.Sprintf("%s_history_%s", nameKey, conversationId)
	conversationMessages := []ConversationMessage{}

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &conversationMessages)
			if err == nil {
				return conversationMessages, nil
			}
		}
	}

	queryInputs := []QueryInput{}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": limit,
			"query": map[string]interface{}{
				"term": map[string]interface{}{
					"conversation_id": conversationId,
				},
			},
			"sort": []map[string]interface{}{
				{
					"time_started": map[string]interface{}{
						"order": "asc",
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding conversation history query: %s", err)
			return conversationMessages, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return conversationMessages, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get conversation history): %s", err)
			return conversationMessages, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return conversationMessages, err
			} else {
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return conversationMessages, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return conversationMessages, err
		}

		type ConversationSearchWrapper struct {
			Hits struct {
				Hits []struct {
					Source QueryInput `json:"_source"`
				} `json:"hits"`
			} `json:"hits"`
		}

		wrapped := ConversationSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return conversationMessages, err
		}

		for _, hit := range wrapped.Hits.Hits {
			queryInputs = append(queryInputs, hit.Source)
		}
	} else {
		q := datastore.NewQuery(nameKey).Filter("conversation_id =", conversationId).Limit(limit)
		_, err := project.Dbclient.GetAll(ctx, q, &queryInputs)
		if err != nil && len(queryInputs) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return conversationMessages, err
			}
		}

		sort.Slice(queryInputs, func(i, j int) bool {
			return queryInputs[i].TimeStarted < queryInputs[j].TimeStarted
		})
	}

	for _, queryInput := range queryInputs {
		// Skip messages with invalid or empty role
		if queryInput.Role != "user" && queryInput.Role != "assistant" {
			log.Printf("[WARNING] Skipping message with invalid role: '%s'", queryInput.Role)
			continue
		}

		message := ConversationMessage{
			UserId:    queryInput.UserId,
			Role:      queryInput.Role,
			Timestamp: time.UnixMicro(queryInput.TimeStarted),
		}

		if queryInput.Role == "user" {
			message.Content = queryInput.Query
		} else if queryInput.Role == "assistant" {
			message.Content = queryInput.Response
		}

		// Skip messages with empty content
		if message.Content == "" {
			log.Printf("[WARNING] Skipping message with empty content for role: %s", queryInput.Role)
			continue
		}

		conversationMessages = append(conversationMessages, message)
	}

	if project.CacheDb && len(conversationMessages) > 0 {
		data, err := json.Marshal(conversationMessages)
		if err == nil {
			err = SetCache(ctx, cacheKey, data, 5)
			if err != nil {
				log.Printf("[WARNING] Failed setting cache for conversation history '%s': %s", cacheKey, err)
			}
		}
	}

	return conversationMessages, nil
}

func SetConversationMetadata(ctx context.Context, conversation Conversation) error {
	nameKey := "conversation_metadata"

	if len(conversation.Id) == 0 {
		conversation.Id = uuid.NewV4().String()
	}

	if conversation.CreatedAt == 0 {
		conversation.CreatedAt = time.Now().Unix()
	}

	conversation.UpdatedAt = time.Now().Unix()

	data, err := json.Marshal(conversation)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling conversation metadata: %s", err)
		return err
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, conversation.Id, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, conversation.Id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &conversation); err != nil {
			log.Printf("[WARNING] Error adding conversation metadata: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, conversation.Id)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for conversation metadata '%s': %s", cacheKey, err)
		}

		// Invalidate org conversations cache
		orgCacheKey := fmt.Sprintf("%s_org_%s", nameKey, conversation.OrgId)
		DeleteCache(ctx, orgCacheKey)
	}

	return nil
}

func GetOrgConversations(ctx context.Context, orgId string, limit int) ([]Conversation, error) {
	nameKey := "conversation_metadata"

	if orgId == "" {
		return []Conversation{}, errors.New("orgId is empty")
	}

	if limit == 0 {
		limit = 50
	}

	cacheKey := fmt.Sprintf("%s_org_%s", nameKey, orgId)
	conversations := []Conversation{}

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &conversations)
			if err == nil {
				return conversations, nil
			}
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": limit,
			"query": map[string]interface{}{
				"term": map[string]interface{}{
					"org_id": orgId,
				},
			},
			"sort": []map[string]interface{}{
				{
					"updated_at": map[string]interface{}{
						"order": "desc",
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding org conversations query: %s", err)
			return conversations, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return conversations, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get org conversations): %s", err)
			return conversations, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return conversations, err
			} else {
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return conversations, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return conversations, err
		}

		type ConversationMetadataSearchWrapper struct {
			Hits struct {
				Hits []struct {
					Source Conversation `json:"_source"`
				} `json:"hits"`
			} `json:"hits"`
		}

		wrapped := ConversationMetadataSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return conversations, err
		}

		for _, hit := range wrapped.Hits.Hits {
			conversations = append(conversations, hit.Source)
		}
	} else {
		q := datastore.NewQuery(nameKey).Filter("org_id =", orgId).Order("-updated_at").Limit(limit)
		_, err := project.Dbclient.GetAll(ctx, q, &conversations)
		if err != nil && len(conversations) == 0 {
			if !strings.Contains(err.Error(), `cannot load field`) {
				return conversations, err
			}
		}
	}

	// Cache the result
	if project.CacheDb && len(conversations) > 0 {
		data, err := json.Marshal(conversations)
		if err == nil {
			err = SetCache(ctx, cacheKey, data, 5)
			if err != nil {
				log.Printf("[WARNING] Failed setting cache for org conversations '%s': %s", cacheKey, err)
			}
		}
	}

	return conversations, nil
}

func GetConversationMetadata(ctx context.Context, conversationId string) (*Conversation, error) {
	nameKey := "conversation_metadata"
	conversation := &Conversation{}

	if conversationId == "" {
		return conversation, errors.New("conversationId is empty")
	}

	cacheKey := fmt.Sprintf("%s_%s", nameKey, conversationId)

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, conversation)
			if err == nil {
				return conversation, nil
			}
		}
	}

	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: conversationId,
		})

		if err != nil {
			log.Printf("[WARNING] Error getting conversation metadata %s: %s", conversationId, err)
			return conversation, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()

		if res.StatusCode == 404 {
			return conversation, errors.New("conversation not found")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return conversation, err
		}

		type ConversationWrapper struct {
			Source Conversation `json:"_source"`
		}

		wrapped := ConversationWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return conversation, err
		}

		conversation = &wrapped.Source
	} else {
		key := datastore.NameKey(nameKey, conversationId, nil)
		if err := project.Dbclient.Get(ctx, key, conversation); err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				err = nil
			} else {
				return conversation, err
			}
		}
	}

	if project.CacheDb && len(conversation.Id) > 0 {
		data, err := json.Marshal(conversation)
		if err == nil {
			err = SetCache(ctx, cacheKey, data, 30)
			if err != nil {
				log.Printf("[WARNING] Failed setting cache for conversation metadata '%s': %s", cacheKey, err)
			}
		}
	}

	return conversation, nil
}

func SetenvStats(ctx context.Context, input OrborusStats) error {
	nameKey := "environment_stats"

	if len(input.Id) == 0 {
		input.Id = uuid.NewV4().String()
	}

	if input.Timestamp == 0 {
		input.Timestamp = time.Now().Unix()
	}

	// New struct, to not add body, author etc
	data, err := json.Marshal(input)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in conversation: %s", err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, input.Id, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, input.Id, nil)
		if _, err := project.Dbclient.Put(ctx, key, &input); err != nil {
			log.Printf("[WARNING] Error adding stats: %s", err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, input.Id)
		err = SetCache(ctx, cacheKey, data, 30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for conversation '%s': %s", cacheKey, err)
		}
	}

	return nil
}

func GetNodeRelations(ctx context.Context) (map[string]NodeRelation, error) {
	// Check if we already have it in cache
	cacheKey := "workflow_node_relations"

	// Download a file
	allNodesRelations := make(map[string]NodeRelation)

	url := "https://storage.googleapis.com/shuffle_public/machine_learning/node_recs_2.json"
	resp, err := http.Get(url)
	if err != nil {
		log.Printf("\n\n[WARNING] Failed getting node relations: %s\n\n", err)
		return allNodesRelations, err
	}

	defer resp.Body.Close()
	// Unmarshal
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("[WARNING] Failed reading body: %s", err)
		return allNodesRelations, err
	}

	var nodeRelations map[string]NodeRelation
	err = json.Unmarshal(body, &nodeRelations)
	if err != nil {
		log.Printf("[WARNING] Failed unmarshalling body: %s", err)
		return allNodesRelations, err
	}

	// Set cache for it
	if project.CacheDb {
		err = SetCache(ctx, cacheKey, body, 60*60*24*30)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for node relations '%s': %s", cacheKey, err)
		}
	}

	return nodeRelations, nil
}

func GetDatastore() *datastore.Client {
	return &project.Dbclient
}

func GetStorage() *storage.Client {
	return &project.StorageClient
}

func GetWorkflowRunsBySearch(ctx context.Context, orgId string, search WorkflowSearch) ([]WorkflowExecution, string, error) {
	nameKey := "workflowexecution"

	var executions []WorkflowExecution
	totalMaxSize := 11184810

	inputcursor := search.Cursor
	maxLimit := 20
	if search.Limit > 0 {
		maxLimit = search.Limit
	}

	cursor := ""
	if project.DbType == "opensearch" {
		var buf bytes.Buffer
		query := map[string]interface{}{
			"size": maxLimit,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						{
							"match": map[string]interface{}{
								"execution_org": orgId,
							},
						},
					},
				},
			},
			"sort": map[string]interface{}{
				"started_at": map[string]interface{}{
					"order": "desc",
				},
			},
		}

		if len(search.WorkflowId) > 0 {
			if search.WorkflowId == "AGENT" {
				query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = []map[string]interface{}{
					{
						"match": map[string]interface{}{
							"execution_org": orgId,
						},
					},
					{
						"match": map[string]interface{}{
							"type": "AGENT",
						},
					},
				}
			} else if search.WorkflowId == "SENSOR_ACTION" {
				query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = []map[string]interface{}{
					{
						"match": map[string]interface{}{
							"execution_org": orgId,
						},
					},
					{
						"match": map[string]interface{}{
							"type": "SENSOR_ACTION",
						},
					},
				}
			} else {
				// Change out the "must" part entirely to contain the workflow id as well
				query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = []map[string]interface{}{
					{
						"match": map[string]interface{}{
							"execution_org": orgId,
						},
					},
					{
						"match": map[string]interface{}{
							"workflow_id": search.WorkflowId,
						},
					},
				}
			}
		}

		if len(search.Status) > 0 {

			// Change out the "must" part entirely to contain the workflow id as well
			// Append map[string]interface{} to the "must" part
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]map[string]interface{}), map[string]interface{}{
				"match": map[string]interface{}{
					"status": search.Status,
				},
			})
		}

		// String to timestamp for search.SearchFrom (string)
		startTimestamp, err := time.Parse(time.RFC3339, search.SearchFrom)
		if err != nil {
			//log.Printf("[WARNING] Failed parsing start time: %s", err)
		} else {
			// Make sure to add map[string]interface{} to the "must" part
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]map[string]interface{}), map[string]interface{}{
				"range": map[string]interface{}{
					"started_at": map[string]interface{}{
						"gte": startTimestamp.Unix(),
					},
				},
			})
		}

		// String to timestamp for search.SearchTo (string)
		endTimestamp, err := time.Parse(time.RFC3339, search.SearchUntil)
		if err != nil {
			//log.Printf("[WARNING] Failed parsing end time: %s", err)
		} else {
			query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"] = append(query["query"].(map[string]interface{})["bool"].(map[string]interface{})["must"].([]map[string]interface{}), map[string]interface{}{
				"range": map[string]interface{}{
					"started_at": map[string]interface{}{
						"lte": endTimestamp.Unix(),
					},
				},
			})
		}

		if len(inputcursor) > 0 {
			log.Printf("[DEBUG] Using cursor: %s", inputcursor)
			query["search_after"] = []interface{}{inputcursor}
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding executions query: %s", err)
			return executions, cursor, err
		}

		// Perform the search request.
		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})

		if err != nil {
			log.Printf("[WARNING] Failed executing query: %s", err)
			return executions, "", err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.IsError() {
			log.Printf("[WARNING] Failed executing query: %s", res.String())
			return executions, "", errors.New(res.String())
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return executions, "", errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return executions, "", err
		}

		wrapped := ExecutionSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil && len(wrapped.Hits.Hits) == 0 {
			return executions, "", err
		}

		executions = []WorkflowExecution{}
		for _, hit := range wrapped.Hits.Hits {
			executions = append(executions, hit.Source)
		}

		//return executions, "", errors.New("Not implemented yet")
	} else {
		query := datastore.NewQuery(nameKey).Filter("execution_org=", orgId).Order("-started_at").Limit(5)

		// This is a trick for SupportAccess users
		if len(orgId) == 0 {
			query = datastore.NewQuery(nameKey).Order("-started_at").Limit(5)
		}

		if len(search.WorkflowId) > 0 {
			if search.WorkflowId == "AGENT" {
				query = query.Filter("type =", "AGENT")
			} else if search.WorkflowId == "SENSOR_ACTION" {
				query = query.Filter("type =", "SENSOR_ACTION")
			} else {
				query = query.Filter("workflow_id =", search.WorkflowId)
			}
		}

		if len(search.Status) > 0 {
			query = query.Filter("status =", search.Status)
		}

		// String to timestamp for search.SearchFrom (string)
		startTimestamp, err := time.Parse(time.RFC3339, search.SearchFrom)
		endTimestamp, enderr := time.Parse(time.RFC3339, search.SearchUntil)
		if err != nil {
			if len(search.SearchFrom) > 0 {
				//log.Printf("[WARNING] Failed parsing start time: %s", err)

				// If there is no endTimestamp
				if enderr != nil {
					// FIXME: Set 3 months back in time
				}
			}
		} else {
			// Make it into a number instead of a string
			query = query.Filter("started_at >=", startTimestamp.Unix())
		}

		// String to timestamp for search.SearchUntil (string)
		if enderr != nil {
			if len(search.SearchFrom) > 0 {
				//log.Printf("[WARNING] Failed parsing end time: %s", err)
			}
		} else {
			// Make it into a number instead of a string
			query = query.Filter("started_at <=", endTimestamp.Unix())
		}

		if inputcursor != "" {
			outputcursor, err := datastore.DecodeCursor(inputcursor)
			if err != nil {
				log.Printf("[WARNING] Error decoding cursor: %s", err)
				return executions, "", err
			}

			query = query.Start(outputcursor)
		}

		cursorStr := ""
		for {
			it := project.Dbclient.Run(ctx, query)

			for {
				innerWorkflow := WorkflowExecution{}
				_, err := it.Next(&innerWorkflow)
				if err != nil {
					if strings.Contains(err.Error(), "context deadline exceeded") {
						log.Printf("[WARNING] Error getting workflow search executions (1): %s", err)
					} else {
						if strings.Contains(err.Error(), `cannot load field`) {
							// Bug with moving types
							err = nil
						} else if strings.Contains(err.Error(), `no more items`) {
							//breakOuter = true
							break
						} else {
							log.Printf("[WARNING] Error getting workflow search executions (2): %s", err)
							break
						}
					}
				}

				executions = append(executions, innerWorkflow)
			}

			if err != iterator.Done {
				//log.Printf("Breaking due to no more iterator")
				//log.Printf("[INFO] Failed fetching results: %v", err)
				//break
			}

			// This is a way to load as much data as we want, and the frontend will load the actual result for us
			executionmarshal, err := json.Marshal(executions)
			if err == nil {
				if len(executionmarshal) > totalMaxSize {
					// Reducing size

					for execIndex, execution := range executions {
						// Making sure the first 5 are "always" proper
						if execIndex < 5 {
							continue
						}

						newResults := []ActionResult{}

						newActions := []Action{}
						for _, action := range execution.Workflow.Actions {
							newAction := Action{
								Name:    action.Name,
								ID:      action.ID,
								AppName: action.AppName,
								AppID:   action.AppID,
							}

							newActions = append(newActions, newAction)
						}

						executions[execIndex].Workflow = Workflow{
							Name:     execution.Workflow.Name,
							ID:       execution.Workflow.ID,
							Triggers: execution.Workflow.Triggers,
							Actions:  newActions,
						}

						for _, result := range execution.Results {
							result.Result = "Result was too large to load. Full Execution needs to be loaded individually for this execution. Click \"Explore execution\" in the UI to see it in detail."
							result.Action = Action{
								Name:       result.Action.Name,
								ID:         result.Action.ID,
								AppName:    result.Action.AppName,
								AppID:      result.Action.AppID,
								LargeImage: result.Action.LargeImage,
							}

							newResults = append(newResults, result)
						}

						executions[execIndex].ExecutionArgument = "too large"
						executions[execIndex].Results = newResults
					}

					executionmarshal, err = json.Marshal(executions)
					if err == nil && len(executionmarshal) > totalMaxSize {
						//log.Printf("Length breaking (2): %d", len(executionmarshal))
						break
					}
				}
			}

			// expected to get here
			if len(executions) >= maxLimit {
				//log.Printf("[INFO] Breaking due to executions larger than amount (%d/%d)", len(executions), maxLimit)
				// Get next cursor
				nextCursor, err := it.Cursor()
				if err != nil {
					log.Printf("[ERROR] Cursorerror: %s", err)
				} else {
					cursor = fmt.Sprintf("%s", nextCursor)
				}

				break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Cursorerror: %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				cursor = nextStr
				if cursorStr == nextStr {
					//log.Printf("Breaking due to no new cursor")

					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	newExecutions := []WorkflowExecution{}
	for _, execution := range executions {
		if execution.Workflow.OrgId == "INTERNAL" && execution.Status != "FINISHED" {
			continue
		}

		newExecutions = append(newExecutions, execution)
	}
	executions = newExecutions

	// Find difference between what's in the list and what is in cache

	removeIndexes := []int{}
	for execIndex, execution := range executions {
		if execution.ExecutionOrg != orgId && len(orgId) > 0 {
			removeIndexes = append(removeIndexes, execIndex)
			continue
		}

		if execution.Status == "EXECUTING" {
			// Get the right one from cache
			newexec, err := GetWorkflowExecution(ctx, execution.ExecutionId)
			if err == nil {
				//log.Printf("[DEBUG] Got with status %s", newexec.Status)
				// Set the execution as well in the database
				if newexec.Status != execution.Status || len(newexec.Results) > len(execution.Results) {

					if project.Environment == "cloud" {
						go SetWorkflowExecution(ctx, *newexec, true)
					} else {
						SetWorkflowExecution(ctx, *newexec, true)
					}
				}

				executions[execIndex] = *newexec
			}
		} else {
			// Delete cache to clear up memory
			if project.Environment != "cloud" && (execution.Status == "ABORTED" || execution.Status == "FAILURE" || execution.Status == "FINISHED") {
				// Delete cache for it
				RunCacheCleanup(ctx, execution)
			}
		}

		parsedActions := []Action{}

		for _, action := range execution.Workflow.Actions {
			parsedActions = append(parsedActions, Action{
				Name:    action.Name,
				ID:      action.ID,
				AppName: action.AppName,
				AppID:   action.AppID,
			})
		}

		executions[execIndex].Workflow = Workflow{
			ID:       execution.Workflow.ID,
			Name:     execution.Workflow.Name,
			Triggers: execution.Workflow.Triggers,
			Actions:  parsedActions,
		}

		//execution.Result = ""
		if len(execution.Results) > 1000 {
			execution.Results = execution.Results[:1000]
		}

		/*
			for resIndex, _ := range execution.Results {
				if execIndex > len(executions) {
					continue
				}

				if resIndex > len(executions[execIndex].Results) {
					continue
				}

				executions[execIndex].Results[resIndex].Action = Action{}
				executions[execIndex].Results[resIndex].Result = ""
			}
		*/

		// Set action in all execution results to empty

	}

	// Loop through removeIndexes backwards and remove them
	for i := len(removeIndexes) - 1; i >= 0; i-- {
		executions = append(executions[:removeIndexes[i]], executions[removeIndexes[i]+1:]...)
	}

	slice.Sort(executions[:], func(i, j int) bool {
		return executions[i].StartedAt > executions[j].StartedAt
	})

	/*
		var err error
		executionmarshal, err := json.Marshal(executions)
		if err == nil {
			if len(executionmarshal) > totalMaxSize {
				// Reducing size

				for execIndex, execution := range executions {
					// Making sure the first 5 are "always" proper
					if execIndex < 5 {
						continue
					}

					newResults := []ActionResult{}

					newActions := []Action{}
					for _, action := range execution.Workflow.Actions {
						newAction := Action{
							Name:    action.Name,
							ID:      action.ID,
							AppName: action.AppName,
							AppID:   action.AppID,
						}

						newActions = append(newActions, newAction)
					}

					executions[execIndex].Workflow = Workflow{
						Name:     execution.Workflow.Name,
						ID:       execution.Workflow.ID,
						Triggers: execution.Workflow.Triggers,
						Actions:  newActions,
					}

					for _, result := range execution.Results {
						result.Result = "Result was too large to load. Full Execution needs to be loaded individually for this execution. Click \"Explore execution\" in the UI to see it in detail."
						result.Action = Action{
							Name:       result.Action.Name,
							ID:         result.Action.ID,
							AppName:    result.Action.AppName,
							AppID:      result.Action.AppID,
							LargeImage: result.Action.LargeImage,
						}

						newResults = append(newResults, result)
					}

					executions[execIndex].ExecutionArgument = "too large"
					executions[execIndex].Results = newResults
				}
			}
		}
	*/

	/*
		if project.CacheDb {
			data, err := json.Marshal(executions)
			if err != nil {
				log.Printf("[WARNING] Failed marshalling update execution cache: %s", err)
				return executions, cursor, nil
			}

			err = SetCache(ctx, cacheKey, data, 10)
			if err != nil {
				log.Printf("[WARNING] Failed setting cache executions (%s): %s", workflowId, err)
				return executions, cursor, nil
			}
		}
	*/

	return executions, cursor, nil

}

func DeleteDbIndex(ctx context.Context, index string) error {
	if !strings.HasPrefix(index, "workflowqueue-") {
		return errors.New("Not allowed to delete that index")
	}

	if project.Environment != "cloud" {
		// Send the Delete By Query request
		query := `{"query": {"match_all": {}}}`
		resp, err := project.Es.Document.DeleteByQuery(ctx, opensearchapi.DocumentDeleteByQueryReq{
			Indices: []string{index},                // Index name
			Body:    bytes.NewReader([]byte(query)), // Query body
		})

		if err != nil {
			if strings.Contains(err.Error(), "not_found") {
				return nil
			}

			log.Printf("[WARNING] Error in DELETE: %s", err)
			return err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			responseData, err := ioutil.ReadAll(res.Body)
			if err != nil {
				log.Printf("[WARNING] Error reading response data: %s", err)
				return err
			}

			log.Printf("[WARNING] Couldn't delete index %s:%s. Status: %d", index, query, res.StatusCode)
			return errors.New(fmt.Sprintf("Couldn't delete index %s:%s. Status: %d. Raw: %s", index, query, res.StatusCode, string(responseData)))
		}

		return nil
	}

	log.Printf("[WARNING] Deleting index %s entirely. This is normal behavior for workflowqueues", index)

	// Create a query to retrieve all items in the index
	var err error
	query := datastore.NewQuery(index).KeysOnly()
	it := project.Dbclient.Run(ctx, query)

	var keys []*datastore.Key
	for {
		var key *datastore.Key
		key, err = it.Next(nil)
		if err == iterator.Done {
			break
		}

		if err != nil {
			log.Printf("[ERROR] Error fetching next key: %v\n", err)
			break
		}

		keys = append(keys, key)
		if len(keys) == 500 {
			// Delete entities in batch
			err := project.Dbclient.DeleteMulti(ctx, keys)
			if err != nil {
				log.Printf("[WARNING] Failed deleting keys: %s", err)
				break
			}
			keys = nil
		}
	}

	// Delete remaining entities
	if len(keys) > 0 {
		err := project.Dbclient.DeleteMulti(ctx, keys)
		if err != nil {
			log.Printf("[WARNING] Failed deleting keys: %s", err)
		}
	}

	return nil
}

func SetTraining(ctx context.Context, training Training) error {
	if project.DbType == "opensearch" {
		return errors.New("Not implemented")
	}

	if training.ID == "" {
		training.ID = uuid.NewV4().String()
	}

	if training.SignupTime == 0 {
		training.SignupTime = time.Now().Unix()
	}

	// Overwriting to be sure these are matching
	// No real point in having id + workflow.ID anymore
	nameKey := "training"

	log.Printf("[INFO] Setting training with %d attendants", training.NumberOfAttendees)
	key := datastore.NameKey(nameKey, training.ID, nil)
	if _, err := project.Dbclient.Put(ctx, key, &training); err != nil {
		log.Printf("[ERROR] Failed adding training with ID %s: %s", training.ID, err)
		return err
	}

	return nil
}

func GetOrgAuth(ctx context.Context, session string) (User, error) {
	// Search the "org" index for the session in org.org_auth.token
	log.Printf("[DEBUG] Searching for session %#v", session)
	nameKey := "Organizations"

	if project.DbType == "opensearch" {
		return User{}, errors.New("Not implemented")
	} else {
		q := datastore.NewQuery(nameKey).Filter("org_auth.token =", session)
		var orgs []Org
		_, err := project.Dbclient.GetAll(ctx, q, &orgs)
		if err != nil {
			if !strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[WARNING] Failed getting org for session %#v: %s", session, err)
				return User{}, err
			}
		}

		if len(orgs) == 0 {
			return User{}, errors.New("No org found")
		}

		// Get the user from the org
		org := orgs[0]
		// Check if the token is expired. If it is, override and returns error
		if org.OrgAuth.Expires.Before(time.Now()) {
			org.OrgAuth.Token = uuid.NewV4().String()
			org.OrgAuth.Expires = time.Now().AddDate(0, 0, 1)

			SetOrg(ctx, org, org.Id)
			return User{}, errors.New("Token expired")
		}

		for _, user := range org.Users {
			if user.Role == "admin" {
				log.Printf("[DEBUG] Letting org auth token %#v impersonate admin user %s (%s) in org %s (%s)", session, user.Username, user.Id, org.Name, org.Id)
				return user, nil
			}
		}
	}

	// If found, return a sample admin user
	return User{}, nil
}

// Returns the orgid related to the key
func GetSyncApikeyByOrg(ctx context.Context, orgId string) (string, error) {
	nameKey := "SyncKey"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, orgId)
	newstring := []string{}
	var syncKeys []SyncKey
	cache, err := GetCache(ctx, cacheKey)
	if err == nil {
		cacheData := []byte(cache.([]uint8))
		//log.Printf("CACHEDATA: %s", cacheData)
		err = json.Unmarshal(cacheData, &syncKeys)
		if err == nil {
			for _, item := range syncKeys {
				newstring = append(newstring, item.Apikey)
			}

			return strings.Join(newstring, ","), nil
		}
	} else {
		//log.Printf("[INFO] Failed getting cache for synckeys: %s", err)
	}

	dbclient, err := GetDatastoreClient(ctx, gceProject)
	if err != nil {
		log.Println(err)
		return "", err
	}

	q := datastore.NewQuery(nameKey).Filter("OrgId =", orgId)
	_, err = dbclient.GetAll(ctx, q, &syncKeys)
	if err != nil && len(syncKeys) == 0 {
		if !strings.Contains(err.Error(), `cannot load field`) {
			log.Printf("[WARNING] Error getting cloudsync apikeys: %s", err)
			return "", err
		}
	}

	returnData := ""
	if len(syncKeys) == 1 {
		returnData = syncKeys[0].Apikey
	} else {
		log.Printf("[WARNING] Error: Found %d synckeys for org %s. Should be one..? Returning with comma.", len(syncKeys), orgId)

		for _, item := range syncKeys {
			newstring = append(newstring, item.Apikey)
		}

		returnData = strings.Join(newstring, ",")
	}

	data, err := json.Marshal(syncKeys)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in getSynckeys: %s", err)
		return returnData, nil
	}

	err = SetCache(ctx, cacheKey, data, 30)
	if err != nil {
		log.Printf("[WARNING] Failed setting cache for getSynckeys: %s", err)
	}

	return returnData, nil
	//errors.New(fmt.Sprintf("Found %d keys for org %s", len(syncKeys), orgId))
}

// Returns the orgid related to the key
func getSyncApikey(ctx context.Context, apikey string) (string, error) {
	nameKey := "SyncKey"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, apikey)

	synckey := &SyncKey{}
	cache, err := GetCache(ctx, cacheKey)
	if err == nil {
		cacheData := []byte(cache.([]uint8))
		//log.Printf("CACHEDATA: %s", cacheData)
		err = json.Unmarshal(cacheData, &synckey)
		if err == nil {
			//log.Printf("[INFO] Successfully got cache for synckey with orgid %s", synckey.OrgId)
			return synckey.OrgId, nil
		}
	} else {
		//log.Printf("[INFO] Failed getting cache for syncKEY: %s", err)
	}

	dbclient, err := GetDatastoreClient(ctx, gceProject)
	if err != nil {
		log.Println(err)
		return "", err
	}

	key := datastore.NameKey(nameKey, apikey, nil)
	if err := dbclient.Get(ctx, key, synckey); err != nil {
		return "", err
	}

	data, err := json.Marshal(synckey)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in getSynckeys: %s", err)
		return synckey.OrgId, nil
	}

	err = SetCache(ctx, cacheKey, data, 30)
	if err != nil {
		log.Printf("[WARNING] Failed setting cache for getSynckeys: %s", err)
	}

	return synckey.OrgId, nil
}

func SetSyncApikey(ctx context.Context, synckey *SyncKey) error {
	// clear session_token and API_token for user
	dbclient, err := GetDatastoreClient(ctx, gceProject)
	if err != nil {
		log.Println(err)
		return err
	}

	synckey.CreatedAt = time.Now().Unix()

	k := datastore.NameKey("SyncKey", synckey.Apikey, nil)
	if _, err := dbclient.Put(ctx, k, synckey); err != nil {
		return err
	}

	return nil
}

func SetDatastoreNGramItem(ctx context.Context, key string, ngramItem *NGramItem) error {
	// OrgId (uuid) + key
	if len(key) < 38 {
		return errors.New(fmt.Sprintf("Invalid key for ngram item. Must be at least 38 characters long. Got '%s'", key))
	}

	nameKey := "datastore_ngram"
	data, err := json.Marshal(ngramItem)
	if err != nil {
		log.Printf("[WARNING] Failed marshalling in set ngram %s: %s", key, err)
		return nil
	}

	if project.DbType == "opensearch" {
		err = indexEs(ctx, nameKey, key, data)
		if err != nil {
			return err
		}
	} else {
		key := datastore.NameKey(nameKey, key, nil)
		if _, err := project.Dbclient.Put(ctx, key, ngramItem); err != nil {
			log.Printf("[ERROR] Failed adding ngramkey with ID %s: %s", key, err)
			return err
		}
	}

	if project.CacheDb {
		cacheKey := fmt.Sprintf("%s_%s", nameKey, key)

		err = SetCache(ctx, cacheKey, data, 60)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for ngramitem '%s': %s", cacheKey, err)
		}
	}

	return nil
}

func GetDatastoreNgramItems(ctx context.Context, orgId, searchKey string, maxAmount int) ([]NGramItem, error) {
	var items []NGramItem
	var err error
	nameKey := "datastore_ngram"

	cacheKey := fmt.Sprintf("%s_%s_%s_%d", nameKey, orgId, searchKey, maxAmount)
	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &items)
			if err == nil {
				return items, nil
			}
		}
	}

	if project.DbType == "opensearch" {
		var buf bytes.Buffer

		query := map[string]interface{}{
			"size": maxAmount,
			"query": map[string]interface{}{
				"bool": map[string]interface{}{
					"must": []map[string]interface{}{
						map[string]interface{}{
							"match": map[string]interface{}{
								"org_id": orgId,
							},
						},
						map[string]interface{}{
							"match": map[string]interface{}{
								"ref": searchKey,
							},
						},
					},
				},
			},
		}

		if err := json.NewEncoder(&buf).Encode(query); err != nil {
			log.Printf("[WARNING] Error encoding find user query: %s", err)
			return items, err
		}

		resp, err := project.Es.Search(ctx, &opensearchapi.SearchReq{
			Indices: []string{strings.ToLower(GetESIndexPrefix(nameKey))},
			Body:    &buf,
			Params: opensearchapi.SearchParams{
				TrackTotalHits: true,
			},
		})
		if err != nil {
			if strings.Contains(err.Error(), "index_not_found_exception") {
				return items, nil
			}

			log.Printf("[ERROR] Error getting response from Opensearch (get ngram items): %s", err)
			return items, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return items, nil
		}

		if res.IsError() {
			var e map[string]interface{}
			if err := json.NewDecoder(res.Body).Decode(&e); err != nil {
				log.Printf("[WARNING] Error parsing the response body: %s", err)
				return items, err
			} else {
				// Print the response status and error information.
				log.Printf("[%s] %s: %s",
					res.Status(),
					e["error"].(map[string]interface{})["type"],
					e["error"].(map[string]interface{})["reason"],
				)
			}
		}

		if res.StatusCode != 200 && res.StatusCode != 201 {
			return items, errors.New(fmt.Sprintf("Bad statuscode: %d", res.StatusCode))
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return items, err
		}

		wrapped := NGramSearchWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return items, err
		}

		//log.Printf("Found items: %d", len(wrapped.Hits.Hits))
		for _, hit := range wrapped.Hits.Hits {
			if hit.Source.Key == "" {
				continue
			}

			if hit.Source.OrgId == orgId {
				items = append(items, hit.Source)
			}
		}

	} else {

		if len(orgId) == 0 {
			return items, errors.New("No org to find ngrams for found")
		}

		cursorStr := ""
		query := datastore.NewQuery(nameKey).Filter("OrgId =", orgId).Filter("Ref =", searchKey).Limit(maxAmount)
		for {
			it := project.Dbclient.Run(ctx, query)
			if len(items) >= maxAmount {
				break
			}

			for {
				innerItem := NGramItem{}
				_, err = it.Next(&innerItem)
				if err != nil {
					if strings.Contains(fmt.Sprintf("%s", err), "cannot load field") {

					} else {
						if !strings.Contains(fmt.Sprintf("%s", err), "no more items in iterator") {
							log.Printf("[WARNING] NGram iterator issue: %s", err)
						}

						break
					}
				}

				found := false
				for _, loopedItem := range items {
					if loopedItem.Key == innerItem.Key {
						found = true
						break
					}
				}

				if !found {
					items = append(items, innerItem)
				}

				if len(items) >= maxAmount {
					break
				}
			}

			if err != iterator.Done {
				log.Printf("[INFO] Failed fetching ngrams: %v", err)
				break
			}

			// Get the cursor for the next page of results.
			nextCursor, err := it.Cursor()
			if err != nil {
				log.Printf("[ERROR] Problem with cursor (ngram): %s", err)
				break
			} else {
				nextStr := fmt.Sprintf("%s", nextCursor)
				if cursorStr == nextStr {
					break
				}

				cursorStr = nextStr
				query = query.Start(nextCursor)
			}
		}
	}

	if len(items) > maxAmount {
		items = items[:maxAmount]
	}

	if project.CacheDb {
		data, err := json.Marshal(items)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in GetDatastoreNgramItems: %s", err)
			return items, nil
		}

		// Short caching due to possible rapid updates
		err = SetCache(ctx, cacheKey, data, 2)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for GetDatastoreNgramItems '%s': %s", cacheKey, err)
		}
	}

	return items, nil
}

// Key itself contains the orgId so this should "just work"
// To get ALL items matching a key, use GetDatastoreNgramItems()
func GetDatastoreNGramItem(ctx context.Context, key string) (*NGramItem, error) {

	nameKey := "datastore_ngram"
	cacheKey := fmt.Sprintf("%s_%s", nameKey, key)
	ngramItem := &NGramItem{}

	if project.CacheDb {
		cache, err := GetCache(ctx, cacheKey)
		if err == nil {
			cacheData := []byte(cache.([]uint8))
			err = json.Unmarshal(cacheData, &ngramItem)
			if err == nil {
				//log.Printf("[DEBUG] Successfully got cache for ngramitem with key %s", key)
				return ngramItem, nil
			}
		}
	}

	if project.DbType == "opensearch" {
		resp, err := project.Es.Document.Get(ctx, opensearchapi.DocumentGetReq{
			Index:      strings.ToLower(GetESIndexPrefix(nameKey)),
			DocumentID: key,
		})
		if err != nil {
			log.Printf("[WARNING] Error for %s: %s", cacheKey, err)
			return ngramItem, err
		}

		res := resp.Inspect().Response
		defer res.Body.Close()
		if res.StatusCode == 404 {
			return ngramItem, errors.New("Item doesn't exist")
		}

		respBody, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return ngramItem, err
		}

		wrapped := NgramItemWrapper{}
		err = json.Unmarshal(respBody, &wrapped)
		if err != nil {
			return ngramItem, err
		}

		ngramItem = &wrapped.Source
	} else {
		// Get the ngram item from the datastore
		getNgramKey := datastore.NameKey(nameKey, key, nil)
		if err := project.Dbclient.Get(ctx, getNgramKey, ngramItem); err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[ERROR] Error in ngramitem loading. Migrating ngramitems to new handler (1): %s", err)
				err = nil
			} else {
				return ngramItem, err
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(ngramItem)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in GetNGramItem: %s", err)
			return ngramItem, nil
		}

		err = SetCache(ctx, cacheKey, data, 15)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for GetNGramItem '%s': %s", cacheKey, err)
			return ngramItem, nil
		}

		//log.Printf("[DEBUG] Successfully set cache for ngramitem with key %s", key)
	}

	return ngramItem, nil
}

func HealthCheckHandler(resp http.ResponseWriter, request *http.Request) {
	ctx := GetContext(request)
	infoSearchReq := &opensearchapi.InfoReq{}
	healthResp, err := project.Es.Info(ctx, infoSearchReq)
	res := healthResp.Inspect().Response
	if err != nil {
		log.Printf("[ERROR] Failed connecting to ES: %s", err)
		resp.WriteHeader(res.StatusCode)
		resp.Write([]byte("Bad response from ES (1). Check logs for more details."))
		return
	}

	if res.StatusCode >= 300 {
		resp.WriteHeader(res.StatusCode)
		resp.Write([]byte(fmt.Sprintf("Bad response from ES - Status code %d", res.StatusCode)))
		return
	}

	fmt.Fprint(resp)
	//fmt.Fprint(res, "OK")
}

func InitOpensearchIndexes() {
	if project.DbType != "opensearch" {
		return
	}

	if os.Getenv("SHUFFLE_SKIP_OPENSEARCH_INDEX_INIT") == "true" {
		return
	}

	// Check if the "workflowexecution" index exists and configuring rollovers if possible
	log.Printf("[INFO] Configuring Opensearch indexes for scaling")

	ctx := context.Background()
	opensearchUrl := strings.TrimRight(os.Getenv("SHUFFLE_OPENSEARCH_URL"), "/")
	if len(opensearchUrl) == 0 {
		opensearchUrl = "https://shuffle-opensearch:9200"
	}

	relevantScaleIndexes := []string{}
	for _, baseIndex := range GetOpensearchBaseIndexes() {
		relevantScaleIndexes = append(relevantScaleIndexes, GetESIndexPrefix(baseIndex))
	}

	customConfig := os.Getenv("OPENSEARCH_INDEX_CONFIG")
	if len(customConfig) > 0 {
		checkValidJson := map[string]interface{}{}
		if err := json.Unmarshal([]byte(customConfig), &checkValidJson); err != nil {
			log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG: %s", err)
			customConfig = ""
		}

		log.Printf("[DEBUG] Using custom index config for relevant scale indexes: %s", customConfig)
	}

	customRollover := os.Getenv("OPENSEARCH_INDEX_ROLLOVER")
	if len(customRollover) > 0 {
		checkValidJson := map[string]interface{}{}
		if err := json.Unmarshal([]byte(customRollover), &checkValidJson); err != nil {
			log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_ROLLOVER: %s", err)
			customRollover = ""
		}

		log.Printf("[DEBUG] Using custom rollover config for relevant scale indexes: %s", customRollover)
	}

	rolloverConfig := []byte(fmt.Sprintf(`{
		"conditions": {
			"max_age": "90d",
			"max_size": "40gb",
			"max_docs": 1000000
		}
	}`))

	if len(customRollover) > 0 {
		rolloverConfig = []byte(customRollover)
	}

	ismEnabled := strings.ToLower(strings.TrimSpace(os.Getenv("OPENSEARCH_USE_ISM_ROLLOVER"))) != "false"
	ismPolicyName := strings.TrimSpace(os.Getenv("OPENSEARCH_ISM_POLICY_NAME"))
	if ismPolicyName == "" {
		ismPolicyName = "shuffle-rollover"
	}

	ismReady := false
	if ismEnabled {
		var err error
		ismReady, err = ensureOpensearchISMRolloverPolicy(ctx, opensearchUrl, relevantScaleIndexes, rolloverConfig, ismPolicyName)
		if err != nil {
			log.Printf("[WARNING] Failed ensuring ISM rollover policy '%s': %s", ismPolicyName, err)
		}
	}

	if fixResult, fixErr := FixOpensearchIndexPrefix(ctx); fixErr != nil {
		log.Printf("[WARNING] Prefix repair before init failed: %s", fixErr)
	} else if !fixResult.Success {
		log.Printf("[WARNING] Prefix repair before init completed with verification warnings: %s", fixResult.Reason)
	} else {
		log.Printf("[INFO] Prefix repair before init: expected aliases=%d found=%d", fixResult.ExpectedAliases, fixResult.FoundAliases)
	}

	for _, index := range relevantScaleIndexes {
		indexConfig := []byte(fmt.Sprintf(`{
			"aliases": {
				"%s": {
					"is_write_index": true
				}
			},
			"settings": {
				"number_of_shards": 3,
				"number_of_replicas": 1,
				"refresh_interval": "30s"
			},
			"mappings": {
				"dynamic_templates": [
					{
						"strings_as_keywords": {
							"match_mapping_type": "string",
							"mapping": {
								"type": "keyword"
							}
						}
					}
				]
			}
		}`, index))

		if len(customConfig) > 0 {
			indexConfig = []byte(customConfig)

			// Check if alias is in the index or not, otherwise inject it
			unmarshalled := map[string]interface{}{}
			if err := json.Unmarshal(indexConfig, &unmarshalled); err != nil {
				log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG (2): %s", err)
			} else {
				if _, ok := unmarshalled["aliases"]; !ok {
					// Inject it
					aliasPart := map[string]interface{}{
						index: map[string]bool{
							"is_write_index": true,
						},
					}
					unmarshalled["aliases"] = aliasPart
					newConfig, err := json.Marshal(unmarshalled)
					if err != nil {
						log.Printf("[ERROR] Invalid JSON in OPENSEARCH_INDEX_CONFIG (3): %s", err)
					} else {
						indexConfig = newConfig
						log.Printf("[INFO] Injected alias into OPENSEARCH_INDEX_CONFIG for index %s", index)
					}
				}
			}

		}

		index = strings.ToLower(index)
		initialIndexName := fmt.Sprintf("%s-000001", index)
		indexConfig = ensureOpensearchIndexRolloverAlias(indexConfig, index)
		// Directly try to force create it. Opensearch throws a 400 if it fails.

		resp, err := project.Es.Indices.Create(ctx, opensearchapi.IndicesCreateReq{
			Index: initialIndexName,
			Body:  bytes.NewReader(indexConfig),
		})

		res := resp.Inspect().Response
		defer res.Body.Close()
		if err != nil {
			if !strings.Contains(fmt.Sprintf("%s", err), "serverless mode") && !strings.Contains(fmt.Sprintf("%s", err), "resource_already_exists_exception") {
				log.Printf("[WARNING] Error creating index %s: %s", index, err)
			}

			// Make sure if the resource exist it is part of correct alias
			if strings.Contains(fmt.Sprintf("%s", err), "resource_already_exists_exception") {
				body := fmt.Sprintf(`{
				  "actions": [
				    {
				      "add": {
				        "index": "%s",
				        "alias": "%s",
				        "is_write_index": true
				      }
				    }
				  ]
				}`, initialIndexName, index)

				aliasResp, aerr := project.Es.Aliases(ctx, opensearchapi.AliasesReq{
					Body: strings.NewReader(body),
				})
				if aerr != nil {
					log.Printf("[WARNING] Failed to ensure alias %s for index %s: %s", index, initialIndexName, aerr)
					return
				}

				res := aliasResp.Inspect().Response
				defer res.Body.Close()

				if res.StatusCode >= 300 {
					log.Printf("[WARNING] Alias enforcement failed: %s", res.String())
					return
				}
			}
		} else {
			if res.IsError() {
				if !strings.Contains(res.String(), "resource_already_exists_exception") {
					log.Printf("[DEBUG] Error creating index %s with custom config: %s", index, res.String())
				}

			} else {
				log.Printf("[DEBUG] Successfully created index %s with custom config", index)
			}
		}

		if ismReady {
			if err := ensureOpensearchIndexRolloverAliasSetting(ctx, opensearchUrl, initialIndexName, index); err != nil {
				log.Printf("[WARNING] Failed ensuring rollover_alias on index %s: %s", initialIndexName, err)
			}

			if err := ensureOpensearchIndexISMPolicy(ctx, opensearchUrl, initialIndexName, ismPolicyName); err != nil {
				log.Printf("[WARNING] Failed attaching ISM policy '%s' to %s: %s", ismPolicyName, initialIndexName, err)
			}

			continue
		}

		rolloverResp, err := project.Es.Indices.Rollover(ctx, opensearchapi.IndicesRolloverReq{
			Alias: index,
			Body:  bytes.NewReader(rolloverConfig),
		})

		if err != nil {
			if !strings.Contains(fmt.Sprintf("%s", err), "serverless mode") && !strings.Contains(fmt.Sprintf("%s", err), "status: 404") {
				log.Printf("[WARNING] Problem during rollover config for %s: %s", index, err)
			}

			continue
		}

		rolloverRes := rolloverResp.Inspect().Response
		defer rolloverRes.Body.Close()
		if rolloverRes.IsError() {
			log.Printf("[ERROR] Rollover config failed for %s: %s", index, rolloverRes.String())
		} else {
			log.Printf("[INFO] Rollover executed successfully for %s", index)
		}

	}

	if fixResult, fixErr := FixOpensearchIndexPrefix(ctx); fixErr != nil {
		log.Printf("[WARNING] Alias verification after init failed: %s", fixErr)
	} else if !fixResult.Success {
		log.Printf("[WARNING] Alias verification after init completed with warnings: %s", fixResult.Reason)
	} else {
		log.Printf("[INFO] Alias verification after init passed: expected aliases=%d found=%d", fixResult.ExpectedAliases, fixResult.FoundAliases)
	}

}

func ensureOpensearchIndexRolloverAlias(indexConfig []byte, alias string) []byte {
	unmarshalled := map[string]interface{}{}
	if err := json.Unmarshal(indexConfig, &unmarshalled); err != nil {
		return indexConfig
	}

	settings, ok := unmarshalled["settings"].(map[string]interface{})
	if !ok || settings == nil {
		settings = map[string]interface{}{}
	}

	settings["plugins.index_state_management.rollover_alias"] = alias
	unmarshalled["settings"] = settings

	updated, err := json.Marshal(unmarshalled)
	if err != nil {
		return indexConfig
	}

	return updated
}

func getOpensearchISMRolloverConditions(rolloverConfig []byte) map[string]interface{} {
	defaultConditions := map[string]interface{}{
		"min_index_age": "90d",
		"min_size":      "40gb",
		"min_doc_count": 1000000,
	}

	parsed := struct {
		Conditions map[string]interface{} `json:"conditions"`
	}{}

	if err := json.Unmarshal(rolloverConfig, &parsed); err != nil {
		return defaultConditions
	}

	if len(parsed.Conditions) == 0 {
		return defaultConditions
	}

	conditions := map[string]interface{}{}
	if value, ok := parsed.Conditions["min_index_age"]; ok {
		conditions["min_index_age"] = value
	} else if value, ok := parsed.Conditions["max_age"]; ok {
		conditions["min_index_age"] = value
	}

	if value, ok := parsed.Conditions["min_size"]; ok {
		conditions["min_size"] = value
	} else if value, ok := parsed.Conditions["max_size"]; ok {
		conditions["min_size"] = value
	}

	if value, ok := parsed.Conditions["min_doc_count"]; ok {
		conditions["min_doc_count"] = value
	} else if value, ok := parsed.Conditions["max_docs"]; ok {
		conditions["min_doc_count"] = value
	}

	if len(conditions) == 0 {
		return defaultConditions
	}

	return conditions
}

func ensureOpensearchISMRolloverPolicy(ctx context.Context, opensearchUrl string, aliases []string, rolloverConfig []byte, policyName string) (bool, error) {
	conditions := getOpensearchISMRolloverConditions(rolloverConfig)

	patterns := []string{}
	for _, alias := range aliases {
		patterns = append(patterns, fmt.Sprintf("%s-*", alias))
	}

	policyBody := map[string]interface{}{
		"policy": map[string]interface{}{
			"description":   "Shuffle rollover policy",
			"default_state": "hot",
			"states": []map[string]interface{}{
				{
					"name": "hot",
					"actions": []map[string]interface{}{
						{
							"rollover": conditions,
						},
					},
					"transitions": []interface{}{},
				},
			},
			"ism_template": []map[string]interface{}{
				{
					"index_patterns": patterns,
					"priority":       100,
				},
			},
		},
	}

	policyData, err := json.Marshal(policyBody)
	if err != nil {
		return false, err
	}

	req, err := http.NewRequestWithContext(ctx, "PUT", fmt.Sprintf("%s/_plugins/_ism/policies/%s", opensearchUrl, policyName), bytes.NewReader(policyData))
	if err != nil {
		return false, err
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := project.Es.Client.Transport.Perform(req)
	if err != nil {
		return false, err
	}
	defer resp.Body.Close()

	body, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode >= 300 {
		if resp.StatusCode == 404 || resp.StatusCode == 400 {
			if strings.Contains(strings.ToLower(string(body)), "_plugins/_ism") || strings.Contains(strings.ToLower(string(body)), "no handler found") {
				log.Printf("[INFO] ISM plugin not available. Falling back to direct rollover")
				return false, nil
			}
		}

		return false, fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(body))
	}

	log.Printf("[INFO] Ensured ISM rollover policy '%s' for %d index patterns", policyName, len(patterns))
	return true, nil
}

func ensureOpensearchIndexRolloverAliasSetting(ctx context.Context, opensearchUrl, indexName, alias string) error {
	settingsBody := map[string]interface{}{
		"index": map[string]interface{}{
			"plugins.index_state_management.rollover_alias": alias,
		},
	}

	body, err := json.Marshal(settingsBody)
	if err != nil {
		return err
	}

	req, err := http.NewRequestWithContext(ctx, "PUT", fmt.Sprintf("%s/%s/_settings", opensearchUrl, indexName), bytes.NewReader(body))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := project.Es.Client.Transport.Perform(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	respBody, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode >= 300 {
		if resp.StatusCode == 404 && strings.Contains(strings.ToLower(string(respBody)), "index_not_found_exception") {
			return nil
		}

		return fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(respBody))
	}

	return nil
}

func ensureOpensearchIndexISMPolicy(ctx context.Context, opensearchUrl, indexName, policyName string) error {
	policyBody := map[string]interface{}{
		"policy_id": policyName,
	}

	body, err := json.Marshal(policyBody)
	if err != nil {
		return err
	}

	req, err := http.NewRequestWithContext(ctx, "POST", fmt.Sprintf("%s/_plugins/_ism/add/%s", opensearchUrl, indexName), bytes.NewReader(body))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	resp, err := project.Es.Client.Transport.Perform(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	respBody, _ := ioutil.ReadAll(resp.Body)
	if resp.StatusCode >= 300 {
		lowerResp := strings.ToLower(string(respBody))
		if strings.Contains(lowerResp, "already has a policy") {
			return nil
		}

		if resp.StatusCode == 404 && strings.Contains(lowerResp, "index_not_found_exception") {
			return nil
		}

		return fmt.Errorf("status: %d, body: %s", resp.StatusCode, string(respBody))
	}

	return nil
}

func ListVulnerabilities(ctx context.Context, ecosystem string, inputcursor string) ([]OSVVulnerability, string, error) {
	nameKey := "vulnerabilities"

	var vulns []OSVVulnerability
	cacheKey := fmt.Sprintf("%s_list_%s_%s", nameKey, ecosystem, inputcursor)
	cache, err := GetCache(ctx, cacheKey)
	if err == nil {
		cacheData := []byte(cache.([]uint8))
		err = json.Unmarshal(cacheData, &vulns)
		if err == nil {
			return vulns, "", nil
		}
	}

	if project.DbType == "opensearch" {
		return nil, "", errors.New("Not implemented for opensearch. Use shuffler.io/api/v1/vulnerabilities")
	} else {
		q := datastore.NewQuery(nameKey)
		if len(ecosystem) > 0 {
			log.Printf("[DEBUG] Filtering vulnerabilities for ecosystem: '%s'", ecosystem)
			q = q.Filter("Affected.Package.Ecosystem = ", ecosystem)
		}

		q = q.Order("-CreatedAt").Limit(100)

		if len(inputcursor) > 0 {
			cursor, err := datastore.DecodeCursor(inputcursor)
			if err != nil {
				log.Printf("[WARNING] Invalid cursor provided to ListVulnerabilities: %s", err)
			} else {
				q = q.Start(cursor)
			}
		}

		// Not sure if cursor works this way but ok
		_, err := project.Dbclient.GetAll(ctx, q, &vulns)
		if err != nil {
			if strings.Contains(err.Error(), `cannot load field`) {
				log.Printf("[ERROR] Error in vulnerability loading. Migrating vulnerabilities to new handler (1): %s", err)
				return vulns, "", nil
			}
		}
	}

	if project.CacheDb {
		data, err := json.Marshal(vulns)
		if err != nil {
			log.Printf("[WARNING] Failed marshalling in ListVulnerabilities: %s", err)
			return vulns, "", nil
		}

		err = SetCache(ctx, cacheKey, data, 60)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for ListVulnerabilities '%s': %s", cacheKey, err)
		}
	}

	return vulns, "", nil
}

func SetVulnerability(ctx context.Context, vuln OSVVulnerability) error {
	if vuln.ID == "" {
		log.Printf("[WARNING] No ID provided for GET vulnerability. Cannot set without ID.")
		return errors.New("ID is required for vulnerability subscription")
	}

	nameKey := "vulnerabilities"

	// Check if it's in cache already
	cacheKey := fmt.Sprintf("%s_%s", nameKey, vuln.ID)
	cached, err := GetCache(ctx, cacheKey)
	if err == nil && len(cached.([]uint8)) > 0 {
		return nil
	}

	if vuln.CreatedAt == 0 {
		vuln.CreatedAt = time.Now().Unix()
	}

	// New struct, to not add body, author etc
	if project.DbType == "opensearch" {
		return errors.New("Not implemented for opensearch. Use shuffler.io/api/v1/vulnerabilities")
	} else {
		key := datastore.NameKey(nameKey, vuln.ID, nil)
		if _, err := project.Dbclient.Put(ctx, key, &vuln); err != nil {
			log.Printf("\n\n[WARNING] Failed adding vulnerability with ID %s: %s", vuln.ID, err)
			return err
		}
	}

	if project.CacheDb {
		// 1 month~
		// Just a check for exists or not to not use db writes too much (?)
		err = SetCache(ctx, cacheKey, []byte("1"), 525960)
		if err != nil {
			log.Printf("[WARNING] Failed setting cache for setworkflow key '%s': %s", cacheKey, err)
		}
	}

	return nil
}