Updated version with migration built-in

.github/install-guide.md

@@ -28,7 +28,7 @@ docker-compose up -d
 
 When you're done, skip to the "After installation" step below.
 
-## Windows Docker setup 
+## Windows with WSL  
 This step is for setting up with Docker on windows from scratch.
 
 1. Make sure you have [Docker](https://docs.docker.com/docker-for-windows/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed. WSL2 may be required.

backend/app_sdk/app_base.py

@@ -514,7 +514,7 @@ class AppBase:
             #    return
 
             print("[INFO] Multiplier length: %d" % len(param_multiplier))
-            tmp = ""
+            #tmp = ""
             for subparams in param_multiplier:
                 print(f"SUBPARAMS IN MULTI: {subparams}")
                 try:
@@ -2307,12 +2307,13 @@ class AppBase:
 
                             #newres = await func(**params)
                             #print("PARAMS: %s" % params)
-                            newres = ""
+                            #newres = ""
                             while True:
                                 try:
                                     newres = await func(**params)
                                     break
                                 except TypeError as e:
+                                    newres = ""
                                     errorstring = "%s" % e
                                     if "got an unexpected keyword argument" in errorstring:
                                         fieldsplit = errorstring.split("'")

backend/app_sdk/build.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 NAME=shuffle-app_sdk
-VERSION=0.8.100
+VERSION=0.8.103
 
 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
 docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION

backend/go-app/go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/elastic/go-elasticsearch v0.0.0 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect
 	github.com/frikky/kin-openapi v0.39.0
-	github.com/frikky/shuffle-shared v0.0.66
+	github.com/frikky/shuffle-shared v0.0.68
 	github.com/fsouza/go-dockerclient v1.7.2 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-billy/v5 v5.0.0

backend/go-app/go.sum

@@ -181,6 +181,10 @@ github.com/frikky/shuffle-shared v0.0.63 h1:btn7V7s98eZmx/9qyapGE3V1TyxftvjMmrzk
 github.com/frikky/shuffle-shared v0.0.63/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
 github.com/frikky/shuffle-shared v0.0.65 h1:TvISX6WE1Y7M2UJ3YuMv0PtgRPWtMj5iGiArVmv6SI8=
 github.com/frikky/shuffle-shared v0.0.65/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
+github.com/frikky/shuffle-shared v0.0.66 h1:1vIcG5ZirIVQ+Li10y9PBHRUOHwpL45L0Nbmpnmpnrw=
+github.com/frikky/shuffle-shared v0.0.66/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
+github.com/frikky/shuffle-shared v0.0.67 h1:Zl09+6rFnlpeOagDXvagC661bdhVm4DC0dylsiZ8dn4=
+github.com/frikky/shuffle-shared v0.0.67/go.mod h1:oPDyGFBuurPtE6UQLUrHU/JtbOJftqDljRfEdLlCTI4=
 github.com/fsouza/go-dockerclient v1.7.2 h1:bBEAcqLTkpq205jooP5RVroUKiVEWgGecHyeZc4OFjo=
 github.com/fsouza/go-dockerclient v1.7.2/go.mod h1:+ugtMCVRwnPfY7d8/baCzZ3uwB0BrG5DB8OzbtxaRz8=
 github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE=

backend/go-app/main.go

@@ -615,13 +615,13 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini)
 	// Use this for register
 	err := shuffle.CheckPasswordStrength(password)
 	if err != nil {
-		log.Printf("Bad password strength: %s", err)
+		log.Printf("[WARNING] Bad password strength: %s", err)
 		return err
 	}
 
 	err = checkUsername(username)
 	if err != nil {
-		log.Printf("Bad Username strength: %s", err)
+		log.Printf("[WARNING] Bad Username strength: %s", err)
 		return err
 	}
 
@@ -5272,7 +5272,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
 	resp.Write(respBody)
 }
 
-/*
+// Runs DB migration from Datastore to Opensearch
+// If the function has "ALL" in it, that means it's intended to be used for Orgs
+// but that we've added a function to grab everything
 func migrateDatabase(resp http.ResponseWriter, request *http.Request) {
 	cors := shuffle.HandleCors(resp, request)
 	if cors {
@@ -5294,72 +5296,210 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
-	type dbSetup struct {
+	if strings.ToLower(os.Getenv("SHUFFLE_ELASTIC")) != "false" {
-		dbUrl string
+		log.Printf("[WARNING] Failed to migrate because main DB is Elastic. Set SHUFFLE_ELASTIC=false in .env")
-	}
-
-	config := elasticsearch.Config{
-		Addresses: []string{
-			"https://192.168.3.8:9200",
-		},
-		Username: "",
-		Password: "",
-	}
-
-	//es, err := elasticsearch.NewDefaultClient()
-	es, err := elasticsearch.NewClient(config)
-	if err != nil {
-		log.Printf("[WARNING] Failed connecting with es7: %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
 	}
-	_ = es
 
-	/*
+	ctx := context.Background()
-		q := datastore.NewQuery(indexType)
+	es := getEsConfig()
-		var users []shuffle.User
+	_, err := shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", false, "")
-		_, err = dbclient.GetAll(ctx, q, &users)
+	if err != nil {
-		if err == nil && len(users) > 0 {
+		log.Printf("[WARNING] Failed to start migration because of init issues: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	log.Printf("\n\n------- STARTING MIGRATION TO OPENSEARCH --------")
+	users, err := shuffle.GetAllUsers(ctx)
+	if err != nil {
+		log.Printf("[ERROR] Failed getting users: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d user(s) to be migrated", len(users))
+	}
+
+	orgs, err := shuffle.GetAllOrgs(ctx)
+	if err != nil {
+		log.Printf("[ERROR] Failed getting orgs: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d org(s) to be migrated", len(orgs))
+	}
+
+	workflows, err := shuffle.GetAllWorkflows(ctx, "ALL")
+	if err != nil {
+		log.Printf("[ERROR] Failed getting workflows: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d workflows(s) to be migrated", len(workflows))
+	}
+
+	apps, err := shuffle.GetAllWorkflowApps(ctx, 0)
+	if err != nil {
+		log.Printf("[ERROR] Failed getting apps: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d app(s) to be migrated", len(apps))
+	}
+
+	openapiApps, err := shuffle.GetAllOpenApi(ctx)
+	if err != nil {
+		log.Printf("[ERROR] Failed getting openapi apps: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d openapi(s) to be migrated", len(openapiApps))
+	}
+
+	workflowappauth, err := shuffle.GetAllWorkflowAppAuth(ctx, "ALL")
+	if err != nil {
+		log.Printf("[ERROR] Failed getting app auth: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d appauth(s) to be migrated", len(workflowappauth))
+	}
+
+	environments, err := shuffle.GetEnvironments(ctx, "ALL")
+	if err != nil {
+		log.Printf("[ERROR] Failed getting environments: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d environment(s) to be migrated", len(environments))
+	}
+
+	hooks, err := shuffle.GetAllHooks(ctx)
+	if err != nil {
+		log.Printf("[ERROR] Failed getting hooks: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d hook(s) to be migrated", len(hooks))
+	}
+
+	schedules, err := shuffle.GetAllSchedules(ctx, "ALL")
+	if err != nil {
+		log.Printf("[ERROR] Failed getting schedules: %#v", err)
+	} else {
+		log.Printf("[DEBUG] Found %d schedule(s) to be migrated", len(schedules))
+	}
+
+	log.Printf("\n\n------- SWAPPING TO OPENSEARCH DB WITH ACQUIRED INFO ---------")
+	userSuccess := 0
+	orgSuccess := 0
+	workflowSuccess := 0
+	appSuccess := 0
+	openapiSuccess := 0
+	authSuccess := 0
+	envSuccess := 0
+	hookSuccess := 0
+	scheduleSuccess := 0
+	_, err = shuffle.RunInit(*dbclient, *es, storage.Client{}, gceProject, "onprem", false, "elasticsearch")
+
+	for _, item := range orgs {
+		err = shuffle.SetOrg(ctx, item, item.Id)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update org in opensearch: %s", err)
 		} else {
-			log.Printf("[WARNING] Failed getting USERS: %s", err)
+			//log.Printf("[DEBUG] Set org %s (%s) in opensearch", item.Name, item.Id)
-
+			orgSuccess += 1
-			for _, user := range users {
-				id := user.Id
-				log.Printf("Indexing %s (%s)", user.Username, id)
-
-				b, err := json.Marshal(user)
-				if err != nil {
-					log.Printf("[WARNING] Failed marshalling %s - %s: %s", id, indexType, err)
-					return
-				}
-
-				req := esapi.IndexRequest{
-					Index:      indexType,
-					DocumentID: id,
-					Body:       strings.NewReader(string(b)),
-					Refresh:    "true",
-				}
-
-				res, err := req.Do(context.Background(), es)
-				if err != nil {
-					log.Printf("Error getting response: %s", err)
-				}
-
-				defer res.Body.Close()
-				if res.IsError() {
-					log.Printf("[%s] Error indexing document ID=%d", res.Status(), id)
-				} else {
-					log.Printf("Successfully indexed %s of ID %s", indexType, id)
-				}
-
-				break
-			}
 		}
+	}
 
+	log.Printf("----- ORGS FOUND: %d - success: %d - failed: %d", len(orgs), orgSuccess, len(orgs)-orgSuccess)
+
+	for _, item := range workflowappauth {
+		err = shuffle.SetWorkflowAppAuthDatastore(ctx, item, item.Id)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update app auth in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set app auth %s in opensearch", item.Id)
+			authSuccess += 1
+		}
+	}
+
+	log.Printf("----- AUTH FOUND: %d - success: %d - failed: %d", len(workflowappauth), authSuccess, len(workflowappauth)-authSuccess)
+
+	for _, item := range environments {
+		err = shuffle.SetEnvironment(ctx, &item)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update env in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set env %s in opensearch", item.Id)
+			envSuccess += 1
+		}
+	}
+
+	log.Printf("----- ENVS FOUND: %d - success: %d - failed: %d", len(environments), envSuccess, len(environments)-envSuccess)
+
+	for _, item := range hooks {
+		err = shuffle.SetHook(ctx, item)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update hooks in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set hook %s in opensearch", item.Id)
+			hookSuccess += 1
+		}
+	}
+
+	log.Printf("---- HOOKS FOUND: %d - success: %d - failed: %d", len(hooks), hookSuccess, len(hooks)-hookSuccess)
+
+	for _, item := range schedules {
+		err = shuffle.SetSchedule(ctx, item)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update schedule in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set schedule %s in opensearch", item.Id)
+			scheduleSuccess += 1
+		}
+	}
+
+	log.Printf(" SCHEDULES FOUND: %d - success: %d - failed: %d", len(schedules), scheduleSuccess, len(schedules)-scheduleSuccess)
+
+	for _, item := range users {
+		err = shuffle.SetUser(ctx, &item, false)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update user in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set user %s (%s) in opensearch", item.Username, item.Id)
+			userSuccess += 1
+		}
+	}
+
+	log.Printf("---- USERS FOUND: %d - success: %d - failed: %d", len(users), userSuccess, len(users)-userSuccess)
+
+	for _, item := range workflows {
+		err = shuffle.SetWorkflow(ctx, item, item.ID)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update workflow in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set workflow %s (%s) in opensearch", item.Name, item.ID)
+			workflowSuccess += 1
+		}
+	}
+
+	log.Printf(" WORKFLOWS FOUND: %d - success: %d - failed: %d", len(workflows), workflowSuccess, len(workflows)-workflowSuccess)
+
+	for _, item := range openapiApps {
+		err = shuffle.SetOpenApiDatastore(ctx, item.ID, item)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update openapi app in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set openapi %s in opensearch", item.ID)
+			openapiSuccess += 1
+		}
+	}
+
+	log.Printf("-- OpenAPI FOUND: %d - success: %d - failed: %d", len(openapiApps), openapiSuccess, len(openapiApps)-openapiSuccess)
+
+	for _, item := range apps {
+		err = shuffle.SetWorkflowAppDatastore(ctx, item, item.ID)
+		if err != nil {
+			//log.Printf("[WARNING] Failed to update app in opensearch: %s", err)
+		} else {
+			//log.Printf("[DEBUG] Set app %s (%s) in opensearch", item.Name, item.ID)
+			appSuccess += 1
+		}
+	}
+
+	log.Printf("----- APPS FOUND: %d - success: %d - failed: %d", len(apps), appSuccess, len(apps)-appSuccess)
+
+	// Handle users
 	// 1. Get users
 	// 2. Get organizations
-	// 3. Get files
 	// 4. Get workflows
 	// 5. Get apps
 	// 6. Get workflowappauth
@@ -5368,14 +5508,14 @@ func migrateDatabase(resp http.ResponseWriter, request *http.Request) {
 	// 10. Get hooks
 	// 11. Get openapi3
 	// 12. Get schedules
-	// 13. Get sessions
-	// 14. Get workflowqueue
 
 	//log.Printf("[INFO] Successfully published workflow %s (%s) TO CLOUD", workflow.Name, workflow.ID)
+	log.Printf("\n\n[DEBUG] Successfully updated ran migration from Datastore to Opensearch!")
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
+	log.Printf("[DEBUG] Panicing to force-restart Shuffle post-migration. Stop Shuffle and change database. Docs: https://shuffler.io/docs/configuration#database_migration")
+	os.Exit(0)
 }
-*/
 
 func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) {
 	cors := shuffle.HandleCors(resp, request)
@@ -5517,17 +5657,7 @@ func makeWorkflowPublic(resp http.ResponseWriter, request *http.Request) {
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
 }
 
-func initHandlers() {
+func getEsConfig() *elasticsearch.Client {
-	var err error
-	ctx := context.Background()
-
-	log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection")
-	//requestCache = cache.New(5*time.Minute, 10*time.Minute)
-	dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy()))
-	if err != nil {
-		log.Fatalf("[DEBUG] Database client error during init: %s", err)
-	}
-
 	esUrl := os.Getenv("SHUFFLE_OPENSEARCH_URL")
 	if len(esUrl) == 0 {
 		esUrl = "http://shuffle-opensearch:9200"
@@ -5544,7 +5674,7 @@ func initHandlers() {
 
 	//config.Transport.TLSClientConfig
 	transport := http.DefaultTransport.(*http.Transport).Clone()
-	transport.MaxIdleConnsPerHost = 1000
+	transport.MaxIdleConnsPerHost = 100
 	transport.ResponseHeaderTimeout = time.Second * 10
 	transport.Proxy = nil
 
@@ -5600,6 +5730,21 @@ func initHandlers() {
 		log.Fatalf("[DEBUG] Database client for ELASTICSEARCH error during init (fatal): %s", err)
 	}
 
+	return es
+}
+
+func initHandlers() {
+	var err error
+	ctx := context.Background()
+
+	log.Printf("[DEBUG] Starting Shuffle backend - initializing database connection")
+	//requestCache = cache.New(5*time.Minute, 10*time.Minute)
+	dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy()))
+	if err != nil {
+		log.Fatalf("[DEBUG] Database client error during init: %s", err)
+	}
+
+	es := getEsConfig()
 	elasticConfig := "elasticsearch"
 	if strings.ToLower(os.Getenv("SHUFFLE_ELASTIC")) == "false" {
 		elasticConfig = ""
@@ -5749,7 +5894,7 @@ func initHandlers() {
 
 	// Docker orborus specific - downloads an image
 	r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS")
-	//r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS")
+	r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS")
 
 	// Important for email, IDS etc. Create this by:
 	// PS: For cloud, this has to use cloud storage.

backend/go-app/walkoff.go

@@ -1117,7 +1117,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
 
 	user, err := shuffle.HandleApiAuthentication(resp, request)
 	if err != nil {
-		log.Printf("Api authentication failed in deleting workflow: %s", err)
+		log.Printf("[WARNING] Api authentication failed in delete workflow: %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -1201,6 +1201,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
 	//memcache.Delete(ctx, memcacheName)
 	//memcacheName = fmt.Sprintf("%s_workflows", user.Username)
 	//memcache.Delete(ctx, memcacheName)
+	//cacheKey := fmt.Sprintf("%s_workflows", user.Id)
 	cacheKey := fmt.Sprintf("%s_workflows", user.Id)
 	shuffle.DeleteCache(ctx, cacheKey)
 
@@ -1258,7 +1259,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
 	if workflow.ID == "" || workflow.ID != id {
 		tmpworkflow, err := shuffle.GetWorkflow(ctx, id)
 		if err != nil {
-			log.Printf("Failed getting the workflow locally (execution cleanup): %s", err)
+			log.Printf("[WARNING] Failed getting the workflow locally (execution setup): %s", err)
 			return shuffle.WorkflowExecution{}, "Failed getting workflow", err
 		}
 
@@ -2438,7 +2439,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) {
 	ctx := context.Background()
 	workflow, err := shuffle.GetWorkflow(ctx, fileId)
 	if err != nil {
-		log.Printf("Failed getting the workflow locally (schedule workflow): %s", err)
+		log.Printf("[WARNING] Failed getting the workflow locally (schedule workflow): %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -3623,7 +3624,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 
 				// Fixes (appends) authentication parameters if they're required
 				if workflowapp.Authentication.Required {
-					log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name)
+					//log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name)
 					// FIXME:
 					// Might require reflection into the python code to append the fields as well
 					for index, action := range workflowapp.Actions {
@@ -3859,7 +3860,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
 
 	// Fixes (appends) authentication parameters if they're required
 	if workflowapp.Authentication.Required {
-		log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name)
+		//log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name)
 		// FIXME:
 		// Might require reflection into the python code to append the fields as well
 		for index, action := range workflowapp.Actions {

backend/tests/migrate_db.sh

@@ -1,2 +1,2 @@
-curl -XPOST -v localhost:5001/api/v1/migrate_database -H 'Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4'
+curl -XPOST -v localhost:5001/api/v1/migrate_database -H 'Authorization: Bearer 0184d7be-33c1-4391-bf9c-dfb8508a4ea2'
 

docker-compose.yml

@@ -35,8 +35,8 @@ services:
       - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps
       - SHUFFLE_FILE_LOCATION=/shuffle-files
     restart: unless-stopped
-    depends_on:
+    #depends_on:
-      - opensearch 
+      #- opensearch 
       #- database
   orborus:
     #build: ./functions/onprem/orborus
@@ -92,21 +92,20 @@ services:
     networks:
       - shuffle
     restart: unless-stopped
-  # OLD DATABASE:
+  database:
-  #database:
+    #build: ./backend/database
-  #  #build: ./backend/database
+    image: frikky/shuffle:database
-  #  image: frikky/shuffle:database
+    container_name: shuffle-database
-  #  container_name: shuffle-database
+    hostname: shuffle-database
-  #  hostname: shuffle-database
+    ports:
-  #  ports:
+      - "8000:8000"
-  #    - "8000:8000"
+    networks:
-  #  networks:
+      - shuffle
-  #    - shuffle
+    environment:
-  #  environment:
+    -  _JAVA_OPTIONS="-Xmx2g"
-  #  -  _JAVA_OPTIONS="-Xmx2g"
+    restart: unless-stopped
-  #  restart: unless-stopped
+    volumes:
-  #  volumes:
+      - ${DB_LOCATION}:/etc/shuffle
-  #    - ${DB_LOCATION}:/etc/shuffle
 
 networks:
   shuffle:

frontend/src/views/AngularWorkflow.jsx

@@ -6148,7 +6148,7 @@ const AngularWorkflow = (props) => {
 										{subworkflow.actions.map((action, index) => {
 											//console.log(action)
 											return (
-												<MenuItem disabled={getParents(selectedTrigger).find(parent => parent.id === action.id)} key={index} style={{backgroundColor: inputColor, color: "white"}} value={action}>
+												<MenuItem  key={index} style={{backgroundColor: inputColor, color: getParents(selectedTrigger).find(parent => parent.id === action.id) ? "red" : "white"}} value={action}>
 													{action.label}
 												</MenuItem>
 											)

frontend/src/views/Workflows.jsx

@@ -743,7 +743,7 @@ const Workflows = (props) => {
 
 			return response.json()
 		})
-    	.then((responseJson) => {
+    .then((responseJson) => {
 			if (!responseJson.success) {
 				alert.error(responseJson.reason)
 			}
@@ -783,6 +783,10 @@ const Workflows = (props) => {
 					if (trigger.parameters !== undefined && trigger.parameters !== null && trigger.parameters.length === 2) {
 						trigger.parameters[0].value = referenceUrl+"webhook_"+trigger.id
 						trigger.parameters[1].value = "webhook_"+trigger.id
+					} else if (trigger.parameters !== undefined && trigger.parameters !== null && trigger.parameters.length === 3) {
+						trigger.parameters[0].value = referenceUrl+"webhook_"+trigger.id
+						trigger.parameters[1].value = "webhook_"+trigger.id
+						// FIXME: Add auth here?
 					} else {
 						alert.info("Something is wrong with the webhook in the copy")
 					}
@@ -955,7 +959,9 @@ const Workflows = (props) => {
 			return response.json()
 		})
     .then((responseJson) => {
-			getAvailableWorkflows()
+			setTimeout(() => {
+				getAvailableWorkflows()
+			}, 1000)
     })
 		.catch(error => {
 			alert.error(error.toString())
@@ -983,7 +989,9 @@ const Workflows = (props) => {
 			return response.json()
 		})
 		.then((responseJson) => {
-			getAvailableWorkflows()
+			setTimeout(() => {
+				getAvailableWorkflows()
+			}, 1000)
 		})
 		.catch(error => {
 			alert.error(error.toString())
@@ -2011,7 +2019,9 @@ const Workflows = (props) => {
 		.then((response) => {
 			if (response.status === 200) {
 				alert.success("Successfully loaded workflows from "+downloadUrl)
-				getAvailableWorkflows()
+				setTimeout(() => {
+					getAvailableWorkflows()
+				}, 1000)
 			}
 
 			return response.json()

shuffle-database/README.md

@@ -1,12 +0,0 @@
-# Database folder
-This is a folder used by the Shuffle database. It has to exist with WITHOUT root permissions before starting Shuffle.
-
-All database files will recide here. IF you can't get Elasticsearch to work, this most likely has to do with permissions. To fix it, run this:
-
-```
-docker-compose down
-sudo chown 1000:1000 -R shuffle-database 
-docker-compose up -d
-```
-
-This restarts the database, and assigns 1000 (Elasticsearch process) as the owner of the database folder.