Merge pull request #62 from frikky/1.0.0

Fixed app hotloading and minor bugs

.env

@@ -2,11 +2,14 @@
 ORG_ID=Shuffle
 ENVIRONMENT_NAME=Shuffle
 
-# Different locations etc
-APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps
+# Remote github config for first load
+APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps 
 APP_DOWNLOAD_AUTH_USERNAME=""
 APP_DOWNLOAD_AUTH_PASSWORD=""
 
+# Local location of your app directory. Can't use ~/
+APP_HOTLOAD_LOCATION=./shuffle-apps
+
 # Other configs
 BACKEND_HOSTNAME=shuffle-backend
 BACKEND_PORT=5001

backend/go-app/main.go

@@ -39,7 +39,6 @@ import (
 
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
-	"github.com/go-git/go-billy/v5/osfs"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/storage/memory"
 
@@ -5794,8 +5793,9 @@ func healthCheckHandler(resp http.ResponseWriter, request *http.Request) {
 
 // Creates osfs from folderpath with a basepath as directory base
 func createFs(basepath, pathname string) (billy.Filesystem, error) {
-	fs := osfs.New("")
+	log.Printf("base: %s, pathname: %s", basepath, pathname)
 
+	fs := memfs.New()
 	err := filepath.Walk(pathname,
 		func(path string, info os.FileInfo, err error) error {
 			if err != nil {
@@ -5806,7 +5806,9 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) {
 				return nil
 			}
 
-			fullpath := fmt.Sprintf("%s%s", basepath, path)
+			// Fix the inner path here
+			newpath := strings.ReplaceAll(path, pathname, "")
+			fullpath := fmt.Sprintf("%s%s", basepath, newpath)
 			switch mode := info.Mode(); {
 			case mode.IsDir():
 				err = fs.MkdirAll(fullpath, 0644)
@@ -5845,23 +5847,23 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) {
 }
 
 // Hotloads new apps from a folder
-// FIXME: Not finished
 func handleAppHotload(location string) error {
 	basepath := "base"
 	fs, err := createFs(basepath, location)
 	if err != nil {
-		log.Printf("Failed making files and stuff: %s", err)
+		log.Printf("Failed memfs creation - probably bad path: %s", err)
 		return err
 	} else {
-		log.Printf("Hotloading from %s finished", location)
+		log.Printf("Memfs creation from %s done", location)
 	}
 
-	dir, err := fs.ReadDir(basepath)
+	dir, err := fs.ReadDir("")
 	if err != nil {
 		log.Printf("Failed reading folder: %s", err)
 		return err
 	}
 
+	log.Printf("Reading app folder: %#v", dir)
 	err = iterateAppGithubFolders(fs, dir, "", "")
 	if err != nil {
 		log.Printf("Err: %s", err)

backend/go-app/walkoff.go

@@ -3444,6 +3444,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
 		return
 	}
 
+	log.Printf("Hotloading from %s", location)
 	err = handleAppHotload(location)
 	if err != nil {
 		resp.WriteHeader(500)
@@ -3699,13 +3700,13 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
 			dir, err := fs.ReadDir(tmpExtra)
 			if err != nil {
 				log.Printf("Failed to read dir: %s", err)
-				break
+				continue
 			}
 
 			// Go routine? Hmm, this can be super quick I guess
 			err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "")
 			if err != nil {
-				break
+				continue
 			}
 		case mode.IsRegular():
 			// Check the file
@@ -3759,28 +3760,32 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 			dir, err := fs.ReadDir(tmpExtra)
 			if err != nil {
 				log.Printf("Failed to read dir: %s", err)
-				break
+				continue
 			}
 
 			// Go routine? Hmm, this can be super quick I guess
 			err = iterateAppGithubFolders(fs, dir, tmpExtra, "")
 			if err != nil {
-				break
+				log.Printf("Error reading folder: %s", err)
+				continue
 			}
 		case mode.IsRegular():
 			// Check the file
 			filename := file.Name()
 			if filename == "Dockerfile" {
-
 				// Quick Dockerfile check
-				dockerdata, err := ioutil.ReadFile(fmt.Sprintf("%sDockerfile", extra))
-				if err != nil {
-					continue
-				}
+				//dockerfile := fmt.Sprintf("%sDockerfile", extra)
+				//log.Printf("Handling Dockerfile %s", dockerfile)
+				//dockerdata, err := ioutil.ReadFile(dockerfile)
+				//if err != nil {
+				//	log.Printf("Failed to read dockerfile")
+				//	continue
+				//}
 
-				if len(dockerdata) == 0 {
-					continue
-				}
+				//if len(dockerdata) == 0 {
+				//	log.Printf("Dockerfile is empty")
+				//	continue
+				//}
 
 				log.Printf("Handle Dockerfile in location %s", extra)
 

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   frontend:
-    #build: ./frontend
+    build: ./frontend
     image: frikky/shuffle:frontend
     container_name: shuffle-frontend
     hostname: shuffle-frontend
@@ -25,6 +25,7 @@ services:
       - shuffle
     volumes: 
       - /var/run/docker.sock:/var/run/docker.sock 
+      - ${APP_HOTLOAD_LOCATION}:/shuffle-apps     
     environment: 
       - ORG_ID=${ORG_ID}
       - DATASTORE_EMULATOR_HOST=shuffle-database:8000

frontend/src/Apps.js

@@ -20,6 +20,7 @@ import YAML from 'yaml'
 import {Link} from 'react-router-dom';
 import Breadcrumbs from '@material-ui/core/Breadcrumbs';
 
+import CachedIcon from '@material-ui/icons/Cached';
 import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
 import PublishIcon from '@material-ui/icons/Publish';
 import CloudDownload from '@material-ui/icons/CloudDownload';
@@ -348,7 +349,7 @@ const Apps = (props) => {
 					Activate App	
 				</Button></Link> : null
 
-		var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded === true)) && activateButton === null ?
+		var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true)) && activateButton === null ?
 				<Button
 					variant="outlined"
 					component="label"
@@ -536,7 +537,7 @@ const Apps = (props) => {
 				<div style={{flex: 1, marginLeft: 10, marginRight: 10}}>
 					<div style={{display: "flex"}}>
 						<div style={{flex: 1}}>
-							<h2>Available integrations</h2> 
+							<h2>All apps</h2> 
 						</div>
 						{isLoading ? <CircularProgress style={{marginTop: 13, marginRight: 15}} /> : null}
 				    <FormControlLabel
@@ -547,6 +548,19 @@ const Apps = (props) => {
 								setSearchBackend(!searchBackend)}
 							} />}
 						/>
+						<Tooltip title={"Reload apps locally"} style={{marginTop: "28px", width: "100%"}} aria-label={"Upload"}>
+							<Button
+								variant="outlined"
+								component="label"
+								color="primary"
+								style={{margin: 5, maxHeight: 50, marginTop: 10}}
+								onClick={() => {
+									hotloadApps()
+								}}
+							>
+								<CachedIcon />
+							</Button>
+						</Tooltip>
 						<Tooltip title={"Download from Github"} style={{marginTop: "28px", width: "100%"}} aria-label={"Upload"}>
 							<Button
 								variant="outlined"
@@ -667,6 +681,35 @@ const Apps = (props) => {
 		})
 	}
 
+	// Locally hotloads app from folder
+	const hotloadApps = () => {
+		alert.info("Hotloading apps from location in .env")
+		setIsLoading(true)
+		fetch(globalUrl+"/api/v1/apps/run_hotload", {
+			mode: "cors",
+			headers: {
+				'Accept': 'application/json',
+			},
+	  	credentials: "include",
+		})
+		.then((response) => {
+			setIsLoading(false)
+			if (response.status === 200) {
+				alert.success("Hotloaded apps!")
+			}
+
+			return response.json()
+		})
+		.then((responseJson) => {
+			if (responseJson.reason !== undefined && responseJson.reason.length > 0) {
+				alert.info("Hotloading: ", responseJson.reason)
+			}
+    })
+		.catch(error => {
+			alert.error(error.toString())
+		});
+	}
+
 	// Gets the URL itself (hopefully this works in most cases?
 	// Will then forward the data to an internal endpoint to validate the api
 	const validateUrl = () => {