Added future image location

.env

@@ -46,3 +46,7 @@ SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.60"
 
 # Used for auto-cleanup of containers. REALLY important at scale.
 SHUFFLE_CONTAINER_AUTO_CLEANUP=false
+
+SHUFFLE_OPENSEARCH_URL="http://shuffle-opensearch:9200"
+SHUFFLE_OPENSEARCH_USERNAME=""
+SHUFFLE_OPENSEARCH_PASSWORD=""

backend/app_sdk/app_base.py

@@ -1247,7 +1247,6 @@ class AppBase:
                             basejson = basejson[value]
 
                     print("Parsed BASEJSON: %s" % basejson)
-        
                     outercnt += 1
         
             except KeyError as e:

backend/go-app/docker.go

@@ -441,7 +441,7 @@ func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	hook, err := getHook(ctx, fileId)
+	hook, err := shuffle.GetHook(ctx, fileId)
 	if err != nil {
 		log.Printf("Failed getting hook %s (stop docker): %s", fileId, err)
 		resp.WriteHeader(401)
@@ -461,8 +461,8 @@ func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) {
 
 	hook.Status = "stopped"
 	hook.Running = false
-	hook.Actions = []HookAction{}
-	err = setHook(ctx, *hook)
+	hook.Actions = []shuffle.HookAction{}
+	err = shuffle.SetHook(ctx, *hook)
 	if err != nil {
 		log.Printf("Failed setting hook: %s", err)
 		resp.WriteHeader(401)
@@ -592,7 +592,7 @@ func imageCheckBuilder(images []string) error {
 }
 
 func hookTest() {
-	var hook Hook
+	var hook shuffle.Hook
 	err := json.Unmarshal([]byte(webhook), &hook)
 	log.Println(webhook)
 	if err != nil {
@@ -601,12 +601,12 @@ func hookTest() {
 	}
 
 	ctx := context.Background()
-	err = setHook(ctx, hook)
+	err = shuffle.SetHook(ctx, hook)
 	if err != nil {
 		log.Printf("Failed setting hook: %s", err)
 	}
 
-	returnHook, err := getHook(ctx, hook.Id)
+	returnHook, err := shuffle.GetHook(ctx, hook.Id)
 	if err != nil {
 		log.Printf("Failed getting hook %s (test): %s", hook.Id, err)
 	}

backend/go-app/go.mod

@@ -2,7 +2,7 @@ module shuffle
 
 go 1.13
 
-replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
+//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
 
 //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
 
@@ -24,7 +24,7 @@ require (
 	github.com/elastic/go-elasticsearch/v7 v7.12.0 // indirect
 	github.com/elastic/go-elasticsearch/v8 v8.0.0-20210519083322-55daf7425ecb // indirect
 	github.com/frikky/kin-openapi v0.39.0
-	github.com/frikky/shuffle-shared v0.0.40
+	github.com/frikky/shuffle-shared v0.0.46
 	github.com/fsouza/go-dockerclient v1.7.2 // indirect
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-billy/v5 v5.0.0

backend/go-app/walkoff.go

@@ -533,69 +533,6 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i
 	return nil
 }
 
-func setWorkflowQueue(ctx context.Context, executionRequest shuffle.ExecutionRequest, env string) error {
-	orgKey := fmt.Sprintf("workflowqueue-%s", env)
-	key := datastore.NameKey(orgKey, executionRequest.ExecutionId, nil)
-
-	// New struct, to not add body, author etc
-	if _, err := dbclient.Put(ctx, key, &executionRequest); err != nil {
-		log.Printf("Error adding workflow queue: %s", err)
-		return err
-	}
-
-	return nil
-}
-
-//
-//func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWrapper, id string) error {
-//	key := datastore.NameKey("workflowqueue", id, nil)
-//
-//	// New struct, to not add body, author etc
-//	if _, err := dbclient.Put(ctx, key, &executionRequests); err != nil {
-//		log.Printf("Error adding workflow queue: %s", err)
-//		return err
-//	}
-//
-//	return nil
-//}
-
-func getWorkflowQueue(ctx context.Context, id string) (shuffle.ExecutionRequestWrapper, error) {
-	orgId := fmt.Sprintf("workflowqueue-%s", id)
-	q := datastore.NewQuery(orgId).Limit(10)
-	executions := []shuffle.ExecutionRequest{}
-	_, err := dbclient.GetAll(ctx, q, &executions)
-	if err != nil {
-		return shuffle.ExecutionRequestWrapper{}, err
-	}
-
-	return shuffle.ExecutionRequestWrapper{Data: executions}, nil
-
-	//key := datastore.NameKey("workflowqueue", id, nil)
-	//executions := ExecutionRequestWrapper{}
-	//if err := dbclient.Get(ctx, key, &workflows); err != nil {
-	//	return ExecutionRequestWrapper{}, err
-	//}
-
-	//return workflows, nil
-}
-
-//func setWorkflowqueuetest(id string) {
-//	data := ExecutionRequestWrapper{
-//		Data: []ExecutionRequest{
-//			ExecutionRequest{
-//				ExecutionId:   "2349bf96-51ad-68d2-5ca6-75ef8f7ee814",
-//				WorkflowId:    "8e344a2e-db51-448f-804c-eb959a32c139",
-//				Authorization: "wut",
-//			},
-//		},
-//	}
-//
-//	err := setWorkflowQueue(data, id)
-//	if err != nil {
-//		log.Printf("Fail: %s", err)
-//	}
-//}
-
 // Frequency = cronjob OR minutes between execution
 func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode, frequency, orgId string, body []byte) error {
 	var err error
@@ -704,7 +641,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
 
 	//setWorkflowqueuetest(id)
 	ctx := context.Background()
-	executionRequests, err := getWorkflowQueue(ctx, id)
+	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id)
 	if err != nil {
 		log.Printf("(1) Failed reading body for workflowqueue: %s", err)
 		resp.WriteHeader(401)
@@ -756,7 +693,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
 	if err != nil {
 		log.Printf("[ERROR] Failed deleting %d execution keys for org %s", len(ids), id)
 	} else {
-		//log.Printf("[INFO] Deleted %d keys from org %s", len(ids), parsedId)
+		log.Printf("[INFO] Deleted %d keys from org %s", len(ids), parsedId)
 	}
 
 	//var newExecutionRequests ExecutionRequestWrapper
@@ -776,7 +713,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque
 
 	// Push only the remaining to the DB (remove)
 	//if len(executionRequests.Data) != len(newExecutionRequests.Data) {
-	//	err := setWorkflowQueue(ctx, newExecutionRequests, id)
+	//	err := shuffle.SetWorkflowQueue(ctx, newExecutionRequests, id)
 	//	if err != nil {
 	//		log.Printf("Fail: %s", err)
 	//	}
@@ -803,7 +740,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	executionRequests, err := getWorkflowQueue(ctx, id)
+	executionRequests, err := shuffle.GetWorkflowQueue(ctx, id)
 	if err != nil {
 		// Skipping as this comes up over and over
 		//log.Printf("(2) Failed reading body for workflowqueue: %s", err)
@@ -816,6 +753,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
 		executionRequests.Data = []shuffle.ExecutionRequest{}
 	} else {
 		log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data))
+		log.Printf("IDS: %#v", executionRequests.Data[0].ExecutionId)
 	}
 
 	newjson, err := json.Marshal(executionRequests)
@@ -2217,7 +2155,7 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request
 			//}
 
 			//log.Printf("Execution request: %#v", executionRequest)
-			err = setWorkflowQueue(ctx, executionRequest, environment)
+			err = shuffle.SetWorkflowQueue(ctx, executionRequest, environment)
 			if err != nil {
 				log.Printf("[ERROR] Failed adding execution to db: %s", err)
 			}
@@ -3637,7 +3575,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
 	// FIXME - should have some permissions?
 	_, err := shuffle.HandleApiAuthentication(resp, request)
 	if err != nil {
-		log.Printf("Api authentication failed in load specific apps: %s", err)
+		log.Printf("[WARNING] Api authentication failed in load specific apps: %s", err)
 		resp.WriteHeader(401)
 		resp.Write([]byte(`{"success": false}`))
 		return
@@ -3707,9 +3645,9 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
 		_ = r
 
 		if tmpBody.ForceUpdate {
-			log.Printf("Running with force update!")
+			log.Printf("[INFO] Running with force update!")
 		} else {
-			log.Printf("Updating apps with updates")
+			log.Printf("[INFO] Updating apps with updates (no force)")
 		}
 
 		if tmpBody.ForceUpdate {
@@ -4230,7 +4168,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 						}
 
 						if len(appendParams) > 0 {
-							log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name)
+							//log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name)
 							workflowapp.Actions[index].Parameters = append(appendParams, workflowapp.Actions[index].Parameters...)
 						}
 
@@ -4464,7 +4402,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
 			}
 
 			if len(appendParams) > 0 {
-				log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name)
+				//log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name)
 				workflowapp.Actions[index].Parameters = append(appendParams, workflowapp.Actions[index].Parameters...)
 			}
 
@@ -4681,7 +4619,7 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	hook, err := getHook(ctx, fileId)
+	hook, err := shuffle.GetHook(ctx, fileId)
 	if err != nil {
 		log.Printf("Failed getting hook %s (stop): %s", fileId, err)
 		resp.WriteHeader(401)
@@ -4708,8 +4646,8 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) {
 
 	hook.Status = "stopped"
 	hook.Running = false
-	hook.Actions = []HookAction{}
-	err = setHook(ctx, *hook)
+	hook.Actions = []shuffle.HookAction{}
+	err = shuffle.SetHook(ctx, *hook)
 	if err != nil {
 		log.Printf("Failed setting hook: %s", err)
 		resp.WriteHeader(401)
@@ -4788,7 +4726,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) {
 	}
 
 	ctx := context.Background()
-	hook, err := getHook(ctx, fileId)
+	hook, err := shuffle.GetHook(ctx, fileId)
 	if err != nil {
 		log.Printf("Failed getting hook %s (start): %s", fileId, err)
 		resp.WriteHeader(401)
@@ -4830,7 +4768,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) {
 
 	hook.Status = "running"
 	hook.Running = true
-	err = setHook(ctx, *hook)
+	err = shuffle.SetHook(ctx, *hook)
 	if err != nil {
 		log.Printf("Failed setting hook: %s", err)
 		resp.WriteHeader(401)
@@ -4949,20 +4887,3 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId
 
 	return nil
 }
-
-// Index = Username
-func DeleteKeys(ctx context.Context, entity string, value []string) error {
-	// Non indexed User data
-	keys := []*datastore.Key{}
-	for _, item := range value {
-		keys = append(keys, datastore.NameKey(entity, item, nil))
-	}
-
-	err := dbclient.DeleteMulti(ctx, keys)
-	if err != nil {
-		log.Printf("Error deleting %s from %s: %s", value, entity, err)
-		return err
-	}
-
-	return nil
-}

docker-compose.yml

@@ -1,8 +1,8 @@
 version: '3'
 services:
   frontend:
-    #build: ./frontend
-    image: ghcr.io/frikky/shuffle-frontend:0.8.80
+    build: ./frontend
+    image: ghcr.io/frikky/shuffle-frontend:0.8.90
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -16,8 +16,8 @@ services:
     depends_on:
       - backend
   backend:
-    #build: ./backend
-    image: ghcr.io/frikky/shuffle-backend:0.8.80
+    build: ./backend
+    image: ghcr.io/frikky/shuffle-backend:0.8.90
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:
@@ -85,6 +85,33 @@ services:
     restart: unless-stopped
     volumes:
       - ${DB_LOCATION}:/etc/shuffle
+  opensearch:
+    image: opensearchproject/opensearch:latest
+    hostname: shuffle-opensearch
+    container_name: shuffle-opensearch
+    environment:
+      - cluster.name=shuffle-cluster
+      - node.name=shuffle-opensearch
+      - discovery.seed_hosts=shuffle-opensearch
+      - cluster.initial_master_nodes=shuffle-opensearch
+      - bootstrap.memory_lock=true 
+      - "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
+      - cluster.routing.allocation.disk.threshold_enabled=false
+      - opendistro_security.disabled=true
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
+        hard: 65536
+    volumes:
+      - ${DB_LOCATION}/opensearch:/usr/share/opensearch/data
+    #ports:
+    #  - 9200:9200
+    networks:
+      - shuffle
+
 networks:
   shuffle:
     driver: bridge

frontend/src/views/Apps.jsx

@@ -447,11 +447,11 @@ const Apps = (props) => {
 				</Grid>	
 
 				{data.activated && data.private_id !== undefined && data.private_id.length > 0 && data.generated ?
-				<Grid container style={{margin: "10px 10px 10px 10px", flex: "1"}} onClick={() => {downloadApp(data)}}>
-					<Tooltip title={"Download OpenAPI"} style={{marginTop: "28px", width: "100%"}} aria-label={data.name}>
-						<CloudDownloadIcon /> 
-					</Tooltip>
-				</Grid>
+					<Grid container style={{margin: "10px 10px 10px 10px", flex: "1"}} onClick={() => {downloadApp(data)}}>
+						<Tooltip title={"Download OpenAPI"} style={{marginTop: "28px", width: "100%"}} aria-label={data.name}>
+							<CloudDownloadIcon /> 
+						</Tooltip>
+					</Grid>
 				: null}
 			</Paper>
 		)

frontend/src/views/LoginPage.jsx

@@ -162,6 +162,7 @@ const LoginDialog = props => {
 
 	//var loginChange = register ? (<div><p onClick={setLoginCheck(false)}>Want to register? Click here.</p></div>) : (<div><p onClick={setLoginCheck(true)}>Go back to login? Click here.</p></div>);
 	var formtitle = register ? <div>Login</div> : <div>Register</div>
+	const imgsize = 120
 	const basedata =
 		<div style={bodyDivStyle}>
 			<Paper style={{
@@ -169,8 +170,14 @@ const LoginDialog = props => {
 				paddingRight: "30px",
 				paddingBottom: "30px",
 				paddingTop: "30px",
+				position: "relative",
 				backgroundColor: theme.palette.surfaceColor,
 			}}>
+				{/*
+				<div style={{position: "absolute", top: -imgsize/2, left: 250-imgsize/2, height: imgsize, width: imgsize, borderRadius: imgsize/2, }}>
+					<img src="https://shuffler.io/images/Shuffle_logo.png" style={{height: imgsize, width: imgsize, border: "2px solid rgba(255,255,255,0.6)", borderRadius: imgsize/2,}}/>
+				</div>
+				*/}
 				<form onSubmit={onSubmit} style={{ margin: "15px 15px 15px 15px", color: "white", }}>
 					<h2>{formtitle}</h2>
 					Username