Fixed app download from any git(hub) repo

This commit is contained in:
frikky
2020-05-22 18:32:05 +02:00
parent 668557b52b
commit 33e8d089b0
6 changed files with 171 additions and 60 deletions
+1
View File
@@ -28,6 +28,7 @@ require (
google.golang.org/api v0.23.0
google.golang.org/appengine v1.6.6
google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31
gopkg.in/src-d/go-git.v4 v4.13.1 // indirect
gopkg.in/yaml.v2 v2.2.8
gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86
)
+14
View File
@@ -46,6 +46,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -146,6 +147,7 @@ github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
@@ -153,6 +155,7 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -160,9 +163,14 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0=
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4=
github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
@@ -178,6 +186,7 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88=
@@ -289,6 +298,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -385,6 +395,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g=
gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE=
gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8=
gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=
gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+29 -27
View File
@@ -5621,30 +5621,6 @@ func runInit(ctx context.Context) {
}
}
// Getting apps to see if we should initialize a test
workflowapps, err := getAllWorkflowApps(ctx)
if err != nil {
log.Printf("Failed getting apps: %s", err)
} else if err == nil && len(workflowapps) == 0 {
log.Printf("Apps: loading TEST")
fs := memfs.New()
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, &git.CloneOptions{
URL: "https://github.com/frikky/shuffle-apps",
})
if err != nil {
log.Printf("Failed loading repo into memory: %s", err)
}
dir, err := fs.ReadDir("")
if err != nil {
log.Printf("FAiled reading folder: %s", err)
}
_ = r
iterateAppGithubFolders(fs, dir, "", "testing")
}
// Gets schedules and starts them
schedules, err := getAllSchedules(ctx)
if err != nil {
@@ -5674,6 +5650,33 @@ func runInit(ctx context.Context) {
}
}
// Getting apps to see if we should initialize a test
workflowapps, err := getAllWorkflowApps(ctx)
if err != nil {
log.Printf("Failed getting apps: %s", err)
} else if err == nil && len(workflowapps) == 0 {
log.Printf("Apps: loading TEST")
fs := memfs.New()
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, &git.CloneOptions{
URL: "https://github.com/frikky/shuffle-apps",
})
if err != nil {
log.Printf("Failed loading repo into memory: %s", err)
}
dir, err := fs.ReadDir("")
if err != nil {
log.Printf("Failed reading folder: %s", err)
}
_ = r
//iterateAppGithubFolders(fs, dir, "", "testing")
// FIXME: Get all the apps?
iterateAppGithubFolders(fs, dir, "", "")
}
log.Printf("Finished INIT")
}
@@ -5721,9 +5724,8 @@ func init() {
r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST")
r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS")
// Apps
r.HandleFunc("/api/v1/apps/get_existing", loadExistingApps).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing/{appname}", loadExistingApps).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS")
+68 -6
View File
@@ -26,6 +26,7 @@ import (
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
newscheduler "github.com/carlescere/scheduler"
"github.com/go-git/go-git/v5/storage/memory"
@@ -3113,7 +3114,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation
return nil
}
func loadExistingApps(resp http.ResponseWriter, request *http.Request) {
func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
@@ -3129,14 +3130,53 @@ func loadExistingApps(resp http.ResponseWriter, request *http.Request) {
return
}
fs := memfs.New()
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, &git.CloneOptions{
URL: "https://github.com/frikky/shuffle-apps",
})
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Field1 & 2 can be a lot of things..
type tmpStruct struct {
URL string `json:"url"`
Field1 string `json:"field_1"`
Field2 string `json:"field_2"`
}
//log.Printf("Body: %s", string(body))
var tmpBody tmpStruct
err = json.Unmarshal(body, &tmpBody)
if err != nil {
log.Printf("Error with unmarshal tmpBody: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fs := memfs.New()
if strings.Contains(tmpBody.URL, "github") || strings.Contains(tmpBody.URL, "gitlab") || strings.Contains(tmpBody.URL, "bitbucket") {
cloneOptions := &git.CloneOptions{
URL: tmpBody.URL,
}
// FIXME: Better auth.
if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: tmpBody.Field1,
Password: tmpBody.Field2,
}
}
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("Failed loading repo into memory: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
dir, err := fs.ReadDir("/")
@@ -3146,6 +3186,28 @@ func loadExistingApps(resp http.ResponseWriter, request *http.Request) {
_ = r
iterateAppGithubFolders(fs, dir, "", "")
} else if strings.Contains(tmpBody.URL, "s3") {
//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
//sess := session.Must(session.NewSession())
//downloader := s3manager.NewDownloader(sess)
//// Write the contents of S3 Object to the file
//storer := memory.NewStorage()
//n, err := downloader.Download(storer, &s3.GetObjectInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
//})
//if err != nil {
// return fmt.Errorf("failed to download file, %v", err)
//}
//fmt.Printf("file downloaded, %d bytes\n", n)
} else {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s is unsupported"}`, tmpBody.URL)))
return
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
+51 -19
View File
@@ -30,6 +30,7 @@ const Apps = (props) => {
const { globalUrl, isLoggedIn, isLoaded } = props;
//const [workflows, setWorkflows] = React.useState([]);
const baseRepository = "https://github.com/frikky/shuffle-apps"
const alert = useAlert()
const [selectedApp, setSelectedApp] = React.useState({});
const [firstrequest, setFirstrequest] = React.useState(true)
@@ -45,6 +46,9 @@ const Apps = (props) => {
const [openApiModal, setOpenApiModal] = React.useState(false);
const [openApiModalType, setOpenApiModalType] = React.useState("");
const [openApiError, setOpenApiError] = React.useState("")
const [field1, setField1] = React.useState("")
const [field2, setField2] = React.useState("")
const { start, stop } = useInterval({
duration: 5000,
startImmediate: false,
@@ -406,10 +410,10 @@ const Apps = (props) => {
color="primary"
style={{margin: 5, maxHeight: 50, marginTop: 10}}
onClick={() => {
getExistingApps()
getSpecificApps(baseRepository)
}}
>
Load existing apps
Load apps
</Button>
<Button
variant="outlined"
@@ -417,10 +421,11 @@ const Apps = (props) => {
color="primary"
style={{margin: 5, maxHeight: 50, marginTop: 10}}
onClick={() => {
setOpenApi(baseRepository)
setLoadAppsModalOpen(true)
}}
>
Upload from github
Download from URL
</Button>
</div>
<TextField
@@ -471,28 +476,38 @@ const Apps = (props) => {
:
null
// Gets the URL itself (hopefully this works in most cases?
// Will then forward the data to an internal endpoint to validate the api
const getExistingApps = () => {
// Load data e.g. from github
const getSpecificApps = (url) => {
setValidation(true)
setIsLoading(true)
start()
alert.success("Downloading and building apps. Feel free to move around meanwhile.")
const parsedData = {
"url": url,
}
if (field1.length > 0) {
parsedData["field_1"] = field1
}
if (field2.length > 0) {
parsedData["field_2"] = field2
}
alert.success("Getting specific apps from your URL.")
var cors = "cors"
fetch(globalUrl+"/api/v1/apps/get_existing", {
method: "GET",
method: "POST",
mode: "cors",
headers: {
'Accept': 'application/json',
},
body: JSON.stringify(parsedData),
credentials: "include",
})
.then((response) => {
if (response.status !== 200) {
alert.success("Failed loading.")
} else {
if (response.status === 200) {
response.text().then(function (text) {
console.log("RETURN: ", text)
alert.success("Loaded existing apps!")
@@ -500,6 +515,16 @@ const Apps = (props) => {
}
setIsLoading(false)
stop()
return response.json()
})
.then((responseJson) => {
console.log("DATA: ", responseJson)
if (responseJson.reason !== undefined) {
alert.error("Failed loading: "+responseJson.reason)
} else {
alert.error("Failed loading")
}
})
.catch(error => {
alert.error(error.toString())
@@ -636,9 +661,8 @@ const Apps = (props) => {
const handleGithubValidation = () => {
console.log("VALIDATE")
setValidation(true)
//const circularLoader = validation ? <CircularProgress color="primary" /> : null
getSpecificApps(openApi)
setLoadAppsModalOpen(false)
}
const appsModalLoad = loadAppsModalOpen ?
@@ -647,6 +671,8 @@ const Apps = (props) => {
onClose={() => {
setOpenApi("")
setLoadAppsModalOpen(false)
setField1("")
setField2("")
}}
PaperProps={{
style: {
@@ -663,11 +689,12 @@ const Apps = (props) => {
</div>
</DialogTitle>
<DialogContent style={{color: "rgba(255,255,255,0.65)"}}>
Github repository
Repository (supported: github, gitlab, bitbucket)
<TextField
style={{backgroundColor: inputColor}}
variant="outlined"
margin="normal"
defaultValue="https://github.com/frikky/shuffle-apps"
InputProps={{
style:{
color: "white",
@@ -679,6 +706,8 @@ const Apps = (props) => {
placeholder="https://github.com/frikky/shuffle-apps"
fullWidth
/>
<span style={{marginTop: 10}}>Authentication (optional - private repos etc):</span>
<div style={{display: "flex"}}>
<TextField
style={{flex: 1, backgroundColor: inputColor}}
@@ -691,8 +720,9 @@ const Apps = (props) => {
fontSize: "1em",
},
}}
onChange={e => setOpenApi(e.target.value)}
placeholder="https://github.com/frikky/shuffle-apps"
onChange={e => setField1(e.target.value)}
type="username"
placeholder="Username / APIkey (optional)"
fullWidth
/>
<TextField
@@ -706,8 +736,9 @@ const Apps = (props) => {
fontSize: "1em",
},
}}
onChange={e => setOpenApi(e.target.value)}
placeholder="https://github.com/frikky/shuffle-apps"
onChange={e => setField2(e.target.value)}
type="password"
placeholder="Password (optional)"
fullWidth
/>
</div>
@@ -718,6 +749,7 @@ const Apps = (props) => {
Cancel
</Button>
<Button style={{borderRadius: "0px"}} disabled={openApi.length === 0 || !openApi.includes("http")} onClick={() => {
handleGithubValidation()
}} color="primary">
Submit
</Button>
+1 -1
View File
@@ -172,7 +172,7 @@ const LoginDialog = props => {
const basedata =
<div style={bodyDivStyle}>
<Paper style={boxStyle}>
<form onSubmit={onSubmit} style={{margin: "15px 15px 15px 15px"}}>
<form onSubmit={onSubmit} style={{margin: "15px 15px 15px 15px", color: "white",}}>
<h2>{formtitle}</h2>
Username
<div>