Added workflow download from github /workflows/download_remote

backend/go-app/main.go

@@ -5883,7 +5883,7 @@ func runInit(ctx context.Context) {
 	}
 
 	log.Printf("Downloading OpenAPI data for search - EXTRA APPS")
-	apis := "https://github.com/frikky/OpenAPI-security-definitions"
+	apis := "https://github.com/frikky/security-openapis"
 
 	// THis gets memory problems hahah
 	//apis := "https://github.com/APIs-guru/openapi-directory"
@@ -5973,6 +5973,7 @@ func init() {
 	r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS")
+	r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS")
 	//r.HandleFunc("/api/v1/workflows/{key}/execute_fs", executeWorkflowFS)
 	r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS")
 	r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS")

backend/go-app/walkoff.go

@@ -3264,6 +3264,114 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation
 	return nil
 }
 
+func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
+	cors := handleCors(resp, request)
+	if cors {
+		return
+	}
+
+	// Just need to be logged in
+	// FIXME - should have some permissions?
+	user, err := handleApiAuthentication(resp, request)
+	if err != nil {
+		log.Printf("Api authentication failed in load apps: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	if user.Role != "admin" {
+		log.Printf("Wrong user (%s) when downloading from github", user.Username)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	body, err := ioutil.ReadAll(request.Body)
+	if err != nil {
+		log.Printf("Error with body read: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	// Field1 & 2 can be a lot of things..
+	type tmpStruct struct {
+		URL    string `json:"url"`
+		Field1 string `json:"field_1"`
+		Field2 string `json:"field_2"`
+	}
+	//log.Printf("Body: %s", string(body))
+
+	var tmpBody tmpStruct
+	err = json.Unmarshal(body, &tmpBody)
+	if err != nil {
+		log.Printf("Error with unmarshal tmpBody: %s", err)
+		resp.WriteHeader(401)
+		resp.Write([]byte(`{"success": false}`))
+		return
+	}
+
+	fs := memfs.New()
+
+	if strings.Contains(tmpBody.URL, "github") || strings.Contains(tmpBody.URL, "gitlab") || strings.Contains(tmpBody.URL, "bitbucket") {
+		cloneOptions := &git.CloneOptions{
+			URL: tmpBody.URL,
+		}
+
+		// FIXME: Better auth.
+		if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 {
+			cloneOptions.Auth = &http2.BasicAuth{
+
+				Username: tmpBody.Field1,
+				Password: tmpBody.Field2,
+			}
+		}
+
+		storer := memory.NewStorage()
+		r, err := git.Clone(storer, fs, cloneOptions)
+		if err != nil {
+			log.Printf("Failed loading repo into memory: %s", err)
+			resp.WriteHeader(401)
+			resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
+			return
+		}
+
+		dir, err := fs.ReadDir("/")
+		if err != nil {
+			log.Printf("FAiled reading folder: %s", err)
+		}
+		_ = r
+
+		log.Printf("Starting workflow folder iteration")
+		iterateWorkflowGithubFolders(fs, dir, "", "")
+
+	} else if strings.Contains(tmpBody.URL, "s3") {
+		//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
+
+		//sess := session.Must(session.NewSession())
+		//downloader := s3manager.NewDownloader(sess)
+
+		//// Write the contents of S3 Object to the file
+		//storer := memory.NewStorage()
+		//n, err := downloader.Download(storer, &s3.GetObjectInput{
+		//	Bucket: aws.String(myBucket),
+		//	Key:    aws.String(myString),
+		//})
+		//if err != nil {
+		//	return fmt.Errorf("failed to download file, %v", err)
+		//}
+		//fmt.Printf("file downloaded, %d bytes\n", n)
+	} else {
+		resp.WriteHeader(401)
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s is unsupported. Try e.g. github"}`, tmpBody.URL)))
+		return
+	}
+
+	resp.WriteHeader(200)
+	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
+}
+
 func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
 	cors := handleCors(resp, request)
 	if cors {
@@ -3489,6 +3597,67 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
 	return nil
 }
 
+// Onlyname is used to
+func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
+	var err error
+
+	for _, file := range dir {
+		if len(onlyname) > 0 && file.Name() != onlyname {
+			continue
+		}
+
+		// Folder?
+		switch mode := file.Mode(); {
+		case mode.IsDir():
+			tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
+			dir, err := fs.ReadDir(tmpExtra)
+			if err != nil {
+				log.Printf("Failed to read dir: %s", err)
+				break
+			}
+
+			// Go routine? Hmm, this can be super quick I guess
+			err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "")
+			if err != nil {
+				break
+			}
+		case mode.IsRegular():
+			// Check the file
+			filename := file.Name()
+			if strings.HasSuffix(filename, ".json") {
+				path := fmt.Sprintf("%s%s", extra, file.Name())
+				fileReader, err := fs.Open(path)
+				if err != nil {
+					log.Printf("Error reading file: %s", err)
+					continue
+				}
+
+				readFile, err := ioutil.ReadAll(fileReader)
+				if err != nil {
+					log.Printf("Error reading file: %s", err)
+					continue
+				}
+
+				var workflow Workflow
+				err = json.Unmarshal(readFile, &workflow)
+				if err != nil {
+					continue
+				}
+
+				ctx := context.Background()
+				err = setWorkflow(ctx, workflow, workflow.ID)
+				if err != nil {
+					log.Printf("Failed setting (download) workflow: %s", err)
+					continue
+				}
+				log.Printf("Uploaded workflow %s!", filename)
+			}
+		}
+	}
+
+	return err
+}
+
 // Onlyname is used to
 func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
 	var err error

frontend/src/Apps.js

@@ -20,7 +20,7 @@ import YAML from 'yaml'
 import {Link} from 'react-router-dom';
 import Breadcrumbs from '@material-ui/core/Breadcrumbs';
 
-import CloudUploadIcon from '@material-ui/icons/CloudUpload';
+import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
 import PublishIcon from '@material-ui/icons/Publish';
 import CloudDownload from '@material-ui/icons/CloudDownload';
 import EditIcon from '@material-ui/icons/Edit';
@@ -457,7 +457,7 @@ const Apps = (props) => {
 					<div style={{width: "100%", margin: 25}}>
 						<h2>App Creator</h2>
 						<a href="/docs/apps" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">How it works</a>
-						&nbsp;- <a href="https://github.com/frikky/OpenAPI-security-definitions" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">Security API's</a>
+						&nbsp;- <a href="https://github.com/frikky/security-openapis" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">Security API's</a>
 						&nbsp;- <a href="https://apis.guru/browse-apis/" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">OpenAPI directory</a>
 						<div/>
 						Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. Use the links above to find potential apps you're looking for using OpenAPI or make one from scratch. There's 1000+ available.
@@ -558,7 +558,7 @@ const Apps = (props) => {
 									setLoadAppsModalOpen(true)
 								}}
 							>
-								<CloudUploadIcon />
+								<CloudDownloadIcon />
 							</Button>
 						</Tooltip>
 					</div>

frontend/src/Workflows.js

@@ -14,6 +14,7 @@ import MenuItem from '@material-ui/core/MenuItem';
 import FormControlLabel from '@material-ui/core/FormControlLabel';
 import Switch from '@material-ui/core/Switch';
 
+import CircularProgress from '@material-ui/core/CircularProgress';
 import CachedIcon from '@material-ui/icons/Cached';
 import EditIcon from '@material-ui/icons/Edit';
 import MoreVertIcon from '@material-ui/icons/MoreVert';
@@ -31,6 +32,9 @@ import Dialog from '@material-ui/core/Dialog';
 import DialogTitle from '@material-ui/core/DialogTitle';
 import DialogActions from '@material-ui/core/DialogActions';
 import DialogContent from '@material-ui/core/DialogContent';
+import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
+
+const inputColor = "#383B40"
 const surfaceColor = "#27292D"
 
 const Workflows = (props) => {
@@ -51,6 +55,10 @@ const Workflows = (props) => {
 	const [, setTrackingId] = React.useState("")
 
 	const [collapseJson, setCollapseJson] = React.useState(false)
+	const [field1, setField1] = React.useState("")
+	const [field2, setField2] = React.useState("")
+	const [downloadUrl, setDownloadUrl] = React.useState("https://github.com/frikky/shuffle-workflows")
+	const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false)
 
 	const [modalOpen, setModalOpen] = React.useState(false);
 	const [newWorkflowName, setNewWorkflowName] = React.useState("");
@@ -173,6 +181,7 @@ const Workflows = (props) => {
 		.then((response) => {
 			if (response.status !== 200) {
 				console.log("Status not 200 for WORKFLOW EXECUTION :O!")
+				alert.error("Failed loading executions for current workflow")
 			}
 
 			return response.json()
@@ -758,7 +767,10 @@ const Workflows = (props) => {
 			for (var key in event.target.files) {
 				const file = event.target.files[key]
 				if (file.type !== "application/json") {
-					//alert.error("File has to contain json.")
+					if (file.type !== undefined) {
+						alert.error("File has to contain valid json")
+					}
+
 					continue
 				}
 
@@ -798,6 +810,8 @@ const Workflows = (props) => {
 		  	reader.readAsText(file)
 			}
 		}
+
+		setLoadWorkflowsModalOpen(false)
 	}
 
 	const modalView = modalOpen ? 
@@ -886,11 +900,18 @@ const Workflows = (props) => {
 					 	<Tooltip color="primary" title={"Create new workflow"} placement="top">
 							<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
 						</Tooltip>
+						{/*
 					 	<Tooltip color="primary" title={"Import workflows"} placement="top">
 							<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
 								<PublishIcon />
 							</Button> 				
 						</Tooltip>
+						*/}
+					 	<Tooltip color="primary" title={"Download workflows"} placement="top">
+							<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
+								<CloudDownloadIcon />
+							</Button> 				
+						</Tooltip>
 						<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
 					</div>
 				</div>
@@ -962,10 +983,160 @@ const Workflows = (props) => {
 			</Paper>
 		</div>
 
+
+	const importWorkflowsFromUrl = (url) => {
+		console.log("IMPORT WORKFLOWS FROM ", downloadUrl)
+
+		const parsedData = {
+			"url": url,
+		}
+
+		if (field1.length > 0) {
+			parsedData["field_1"] = field1
+		}
+
+		if (field2.length > 0) {
+			parsedData["field_2"] = field2
+		}
+
+		alert.success("Getting specific workflows from your URL.")
+		var cors = "cors"
+		fetch(globalUrl+"/api/v1/workflows/download_remote", {
+    	method: "POST",
+			mode: "cors",
+			headers: {
+				'Accept': 'application/json',
+			},
+			body: JSON.stringify(parsedData),
+	  	credentials: "include",
+		})
+		.then((response) => {
+			if (response.status === 200) {
+				response.text().then(function (text) {
+					console.log("RETURN: ", text)
+					alert.success("Loaded existing apps!")
+				})
+			}
+
+			return response.json()
+		})
+    .then((responseJson) => {
+				console.log("DATA: ", responseJson)
+				if (responseJson.reason !== undefined) {
+					alert.error("Failed loading: "+responseJson.reason)
+				} else {
+					alert.error("Failed loading")
+				}
+		})
+		.catch(error => {
+			alert.error(error.toString())
+		})
+	}
+
+	const handleGithubValidation = () => {
+		importWorkflowsFromUrl(downloadUrl)
+		setLoadWorkflowsModalOpen(false)
+	}
+
+	const workflowDownloadModalOpen = loadWorkflowsModalOpen ? 
+		<Dialog modal 
+			open={loadWorkflowsModalOpen}
+			onClose={() => {
+			}}
+			PaperProps={{
+				style: {
+					backgroundColor: surfaceColor,
+					color: "white",
+					minWidth: "800px",
+					minHeight: "320px",
+				},
+			}}
+		>
+			<DialogTitle>
+				<div style={{color: "rgba(255,255,255,0.9)"}}>
+					Load workflows from github repo
+					<div style={{float: "right"}}>
+						<Tooltip color="primary" title={"Import manually"} placement="top">
+							<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+								<PublishIcon />
+							</Button> 				
+						</Tooltip>
+					</div>
+				</div>
+			</DialogTitle>
+			<DialogContent style={{color: "rgba(255,255,255,0.65)"}}>
+				Repository (supported: github, gitlab, bitbucket)
+				<TextField
+					style={{backgroundColor: inputColor}}
+					variant="outlined"
+					margin="normal"
+					value={downloadUrl}
+					InputProps={{
+						style:{
+							color: "white",
+							height: "50px",
+							fontSize: "1em",
+						},
+					}}
+					onChange={e => setDownloadUrl(e.target.value)}
+					placeholder="https://github.com/frikky/shuffle-apps"
+					fullWidth
+					/>
+
+				<span style={{marginTop: 10}}>Authentication (optional - private repos etc):</span>
+				<div style={{display: "flex"}}>
+					<TextField
+						style={{flex: 1, backgroundColor: inputColor}}
+						variant="outlined"
+						margin="normal"
+						InputProps={{
+							style:{
+								color: "white",
+								height: "50px",
+								fontSize: "1em",
+							},
+						}}
+						onChange={e => setField1(e.target.value)}
+						type="username"
+						placeholder="Username / APIkey (optional)"
+						fullWidth
+						/>
+					<TextField
+						style={{flex: 1, backgroundColor: inputColor}}
+						variant="outlined"
+						margin="normal"
+						InputProps={{
+							style:{
+								color: "white",
+								height: "50px",
+								fontSize: "1em",
+							},
+						}}
+						onChange={e => setField2(e.target.value)}
+						type="password"
+						placeholder="Password (optional)"
+						fullWidth
+						/>
+				</div>
+			</DialogContent>
+			<DialogActions>
+				<Button style={{borderRadius: "0px"}} onClick={() => setLoadWorkflowsModalOpen(false)} color="primary">
+					Cancel
+				</Button>
+	      <Button style={{borderRadius: "0px"}} disabled={downloadUrl.length === 0 || !downloadUrl.includes("http")} onClick={() => {
+					handleGithubValidation() 
+				}} color="primary">
+	        Submit	
+	      </Button>
+			</DialogActions>
+		</Dialog>
+		: null
+
 	const loadedCheck = isLoaded && isLoggedIn && workflowDone ? 
 		<div>
 			{workflowView}
 			{modalView}
+			{workflowDownloadModalOpen}
 		</div>
 		:
 		<div>