marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added workflow download from github /workflows/download_remote

Browse Source
This commit is contained in:
frikky
2020-06-12 07:05:59 +02:00
parent c592ca649a
commit 59f3bce31a
4 changed files with 350 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/main.go
+2 -1
View File
@@ -5883,7 +5883,7 @@ func runInit(ctx context.Context) {
}
log.Printf("Downloading OpenAPI data for search - EXTRA APPS")
apis := "https://github.com/frikky/OpenAPI-security-definitions"
apis := "https://github.com/frikky/security-openapis"
// THis gets memory problems hahah
//apis := "https://github.com/APIs-guru/openapi-directory"
@@ -5973,6 +5973,7 @@ func init() {
r.HandleFunc("/api/v1/workflows", getWorkflows).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows", setNewWorkflow).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/schedules", handleGetSchedules).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/workflows/{key}/execute_fs", executeWorkflowFS)
r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/workflows/{key}/schedule", scheduleWorkflow).Methods("POST", "OPTIONS")
backend/go-app/walkoff.go
+169
View File
@@ -3264,6 +3264,114 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation
return nil
}
func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// Just need to be logged in
// FIXME - should have some permissions?
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in load apps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("Wrong user (%s) when downloading from github", user.Username)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Field1 & 2 can be a lot of things..
type tmpStruct struct {
URL string `json:"url"`
Field1 string `json:"field_1"`
Field2 string `json:"field_2"`
}
//log.Printf("Body: %s", string(body))
var tmpBody tmpStruct
err = json.Unmarshal(body, &tmpBody)
if err != nil {
log.Printf("Error with unmarshal tmpBody: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fs := memfs.New()
if strings.Contains(tmpBody.URL, "github") || strings.Contains(tmpBody.URL, "gitlab") || strings.Contains(tmpBody.URL, "bitbucket") {
cloneOptions := &git.CloneOptions{
URL: tmpBody.URL,
}
// FIXME: Better auth.
if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: tmpBody.Field1,
Password: tmpBody.Field2,
}
}
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("Failed loading repo into memory: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
dir, err := fs.ReadDir("/")
if err != nil {
log.Printf("FAiled reading folder: %s", err)
}
_ = r
log.Printf("Starting workflow folder iteration")
iterateWorkflowGithubFolders(fs, dir, "", "")
} else if strings.Contains(tmpBody.URL, "s3") {
//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
//sess := session.Must(session.NewSession())
//downloader := s3manager.NewDownloader(sess)
//// Write the contents of S3 Object to the file
//storer := memory.NewStorage()
//n, err := downloader.Download(storer, &s3.GetObjectInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
//})
//if err != nil {
// return fmt.Errorf("failed to download file, %v", err)
//}
//fmt.Printf("file downloaded, %d bytes\n", n)
} else {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s is unsupported. Try e.g. github"}`, tmpBody.URL)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -3489,6 +3597,67 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string,
return nil
}
// Onlyname is used to
func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
var err error
for _, file := range dir {
if len(onlyname) > 0 && file.Name() != onlyname {
continue
}
// Folder?
switch mode := file.Mode(); {
case mode.IsDir():
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
dir, err := fs.ReadDir(tmpExtra)
if err != nil {
log.Printf("Failed to read dir: %s", err)
break
}
// Go routine? Hmm, this can be super quick I guess
err = iterateWorkflowGithubFolders(fs, dir, tmpExtra, "")
if err != nil {
break
}
case mode.IsRegular():
// Check the file
filename := file.Name()
if strings.HasSuffix(filename, ".json") {
path := fmt.Sprintf("%s%s", extra, file.Name())
fileReader, err := fs.Open(path)
if err != nil {
log.Printf("Error reading file: %s", err)
continue
}
readFile, err := ioutil.ReadAll(fileReader)
if err != nil {
log.Printf("Error reading file: %s", err)
continue
}
var workflow Workflow
err = json.Unmarshal(readFile, &workflow)
if err != nil {
continue
}
ctx := context.Background()
err = setWorkflow(ctx, workflow, workflow.ID)
if err != nil {
log.Printf("Failed setting (download) workflow: %s", err)
continue
}
log.Printf("Uploaded workflow %s!", filename)
}
}
}
return err
}
// Onlyname is used to
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
var err error
frontend/src/Apps.js
+7 -7
View File
@@ -20,7 +20,7 @@ import YAML from 'yaml'
import {Link} from 'react-router-dom';
import Breadcrumbs from '@material-ui/core/Breadcrumbs';
import CloudUploadIcon from '@material-ui/icons/CloudUpload';
import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
import PublishIcon from '@material-ui/icons/Publish';
import CloudDownload from '@material-ui/icons/CloudDownload';
import EditIcon from '@material-ui/icons/Edit';
@@ -457,7 +457,7 @@ const Apps = (props) => {
<div style={{width: "100%", margin: 25}}>
<h2>App Creator</h2>
<a href="/docs/apps" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">How it works</a>
&nbsp;- <a href="https://github.com/frikky/OpenAPI-security-definitions" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">Security API's</a>
&nbsp;- <a href="https://github.com/frikky/security-openapis" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">Security API's</a>
&nbsp;- <a href="https://apis.guru/browse-apis/" style={{textDecoration: "none", color: "#f85a3e"}} target="_blank">OpenAPI directory</a>
<div/>
Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. Use the links above to find potential apps you're looking for using OpenAPI or make one from scratch. There's 1000+ available.
@@ -558,7 +558,7 @@ const Apps = (props) => {
setLoadAppsModalOpen(true)
}}
>
<CloudUploadIcon />
<CloudDownloadIcon />
</Button>
</Tooltip>
</div>
@@ -1023,13 +1023,13 @@ const Apps = (props) => {
</DialogContent>
<DialogActions>
{circularLoader}
<Button style={{borderRadius: "0px"}} onClick={() => setOpenApiModal(false)} color="primary">
Cancel
</Button>
<Button style={{borderRadius: "0px"}} onClick={() => setOpenApiModal(false)} color="primary">
Cancel
</Button>
<Button style={{borderRadius: "0px"}} disabled={appValidation.length === 0} onClick={() => {
redirectOpenApi()
}} color="primary">
Submit
Submit
</Button>
</DialogActions>
</FormControl>
frontend/src/Workflows.js
+172 -1
View File
@@ -14,6 +14,7 @@ import MenuItem from '@material-ui/core/MenuItem';
import FormControlLabel from '@material-ui/core/FormControlLabel';
import Switch from '@material-ui/core/Switch';
import CircularProgress from '@material-ui/core/CircularProgress';
import CachedIcon from '@material-ui/icons/Cached';
import EditIcon from '@material-ui/icons/Edit';
import MoreVertIcon from '@material-ui/icons/MoreVert';
@@ -31,6 +32,9 @@ import Dialog from '@material-ui/core/Dialog';
import DialogTitle from '@material-ui/core/DialogTitle';
import DialogActions from '@material-ui/core/DialogActions';
import DialogContent from '@material-ui/core/DialogContent';
import CloudDownloadIcon from '@material-ui/icons/CloudDownload';
const inputColor = "#383B40"
const surfaceColor = "#27292D"
const Workflows = (props) => {
@@ -51,6 +55,10 @@ const Workflows = (props) => {
const [, setTrackingId] = React.useState("")
const [collapseJson, setCollapseJson] = React.useState(false)
const [field1, setField1] = React.useState("")
const [field2, setField2] = React.useState("")
const [downloadUrl, setDownloadUrl] = React.useState("https://github.com/frikky/shuffle-workflows")
const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false)
const [modalOpen, setModalOpen] = React.useState(false);
const [newWorkflowName, setNewWorkflowName] = React.useState("");
@@ -173,6 +181,7 @@ const Workflows = (props) => {
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for WORKFLOW EXECUTION :O!")
alert.error("Failed loading executions for current workflow")
}
return response.json()
@@ -758,7 +767,10 @@ const Workflows = (props) => {
for (var key in event.target.files) {
const file = event.target.files[key]
if (file.type !== "application/json") {
//alert.error("File has to contain json.")
if (file.type !== undefined) {
alert.error("File has to contain valid json")
}
continue
}
@@ -798,6 +810,8 @@ const Workflows = (props) => {
reader.readAsText(file)
}
}
setLoadWorkflowsModalOpen(false)
}
const modalView = modalOpen ?
@@ -886,11 +900,18 @@ const Workflows = (props) => {
<Tooltip color="primary" title={"Create new workflow"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button>
</Tooltip>
{/*
<Tooltip color="primary" title={"Import workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
<PublishIcon />
</Button>
</Tooltip>
*/}
<Tooltip color="primary" title={"Download workflows"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
<CloudDownloadIcon />
</Button>
</Tooltip>
<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
</div>
</div>
@@ -962,10 +983,160 @@ const Workflows = (props) => {
</Paper>
</div>
const importWorkflowsFromUrl = (url) => {
console.log("IMPORT WORKFLOWS FROM ", downloadUrl)
const parsedData = {
"url": url,
}
if (field1.length > 0) {
parsedData["field_1"] = field1
}
if (field2.length > 0) {
parsedData["field_2"] = field2
}
alert.success("Getting specific workflows from your URL.")
var cors = "cors"
fetch(globalUrl+"/api/v1/workflows/download_remote", {
method: "POST",
mode: "cors",
headers: {
'Accept': 'application/json',
},
body: JSON.stringify(parsedData),
credentials: "include",
})
.then((response) => {
if (response.status === 200) {
response.text().then(function (text) {
console.log("RETURN: ", text)
alert.success("Loaded existing apps!")
})
}
return response.json()
})
.then((responseJson) => {
console.log("DATA: ", responseJson)
if (responseJson.reason !== undefined) {
alert.error("Failed loading: "+responseJson.reason)
} else {
alert.error("Failed loading")
}
})
.catch(error => {
alert.error(error.toString())
})
}
const handleGithubValidation = () => {
importWorkflowsFromUrl(downloadUrl)
setLoadWorkflowsModalOpen(false)
}
const workflowDownloadModalOpen = loadWorkflowsModalOpen ?
<Dialog modal
open={loadWorkflowsModalOpen}
onClose={() => {
}}
PaperProps={{
style: {
backgroundColor: surfaceColor,
color: "white",
minWidth: "800px",
minHeight: "320px",
},
}}
>
<DialogTitle>
<div style={{color: "rgba(255,255,255,0.9)"}}>
Load workflows from github repo
<div style={{float: "right"}}>
<Tooltip color="primary" title={"Import manually"} placement="top">
<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
<PublishIcon />
</Button>
</Tooltip>
</div>
</div>
</DialogTitle>
<DialogContent style={{color: "rgba(255,255,255,0.65)"}}>
Repository (supported: github, gitlab, bitbucket)
<TextField
style={{backgroundColor: inputColor}}
variant="outlined"
margin="normal"
value={downloadUrl}
InputProps={{
style:{
color: "white",
height: "50px",
fontSize: "1em",
},
}}
onChange={e => setDownloadUrl(e.target.value)}
placeholder="https://github.com/frikky/shuffle-apps"
fullWidth
/>
<span style={{marginTop: 10}}>Authentication (optional - private repos etc):</span>
<div style={{display: "flex"}}>
<TextField
style={{flex: 1, backgroundColor: inputColor}}
variant="outlined"
margin="normal"
InputProps={{
style:{
color: "white",
height: "50px",
fontSize: "1em",
},
}}
onChange={e => setField1(e.target.value)}
type="username"
placeholder="Username / APIkey (optional)"
fullWidth
/>
<TextField
style={{flex: 1, backgroundColor: inputColor}}
variant="outlined"
margin="normal"
InputProps={{
style:{
color: "white",
height: "50px",
fontSize: "1em",
},
}}
onChange={e => setField2(e.target.value)}
type="password"
placeholder="Password (optional)"
fullWidth
/>
</div>
</DialogContent>
<DialogActions>
<Button style={{borderRadius: "0px"}} onClick={() => setLoadWorkflowsModalOpen(false)} color="primary">
Cancel
</Button>
<Button style={{borderRadius: "0px"}} disabled={downloadUrl.length === 0 || !downloadUrl.includes("http")} onClick={() => {
handleGithubValidation()
}} color="primary">
Submit
</Button>
</DialogActions>
</Dialog>
: null
const loadedCheck = isLoaded && isLoggedIn && workflowDone ?
<div>
{workflowView}
{modalView}
{workflowDownloadModalOpen}
</div>
:
<div>