diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index d4ef8407..20a8da9f 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -28,6 +28,7 @@ require ( google.golang.org/api v0.23.0 google.golang.org/appengine v1.6.6 google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 + gopkg.in/src-d/go-git.v4 v4.13.1 // indirect gopkg.in/yaml.v2 v2.2.8 gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 2b4ecfda..6bff1e65 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -46,6 +46,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -146,6 +147,7 @@ github.com/kevinburke/ssh_config v0.0.0-20190725054713-01f96b0aa0cd/go.mod h1:CT github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -153,6 +155,7 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -160,9 +163,14 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= +github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= @@ -178,6 +186,7 @@ golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= @@ -289,6 +298,7 @@ golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBn golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -385,6 +395,10 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98= +gopkg.in/src-d/go-git-fixtures.v3 v3.5.0/go.mod h1:dLBcvytrw/TYZsNTWCnkNF2DSIlzWYqTe3rJR56Ac7g= +gopkg.in/src-d/go-git.v4 v4.13.1 h1:SRtFyV8Kxc0UP7aCHcijOMQGPxHSmMOPrzulQWolkYE= +gopkg.in/src-d/go-git.v4 v4.13.1/go.mod h1:nx5NYcxdKxq5fpltdHnPa2Exj4Sx0EclMWZQbYDu2z8= gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index e2a4e636..e20d4386 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5621,30 +5621,6 @@ func runInit(ctx context.Context) { } } - // Getting apps to see if we should initialize a test - workflowapps, err := getAllWorkflowApps(ctx) - if err != nil { - log.Printf("Failed getting apps: %s", err) - } else if err == nil && len(workflowapps) == 0 { - log.Printf("Apps: loading TEST") - fs := memfs.New() - storer := memory.NewStorage() - r, err := git.Clone(storer, fs, &git.CloneOptions{ - URL: "https://github.com/frikky/shuffle-apps", - }) - - if err != nil { - log.Printf("Failed loading repo into memory: %s", err) - } - - dir, err := fs.ReadDir("") - if err != nil { - log.Printf("FAiled reading folder: %s", err) - } - _ = r - iterateAppGithubFolders(fs, dir, "", "testing") - } - // Gets schedules and starts them schedules, err := getAllSchedules(ctx) if err != nil { @@ -5674,6 +5650,33 @@ func runInit(ctx context.Context) { } } + // Getting apps to see if we should initialize a test + workflowapps, err := getAllWorkflowApps(ctx) + if err != nil { + log.Printf("Failed getting apps: %s", err) + } else if err == nil && len(workflowapps) == 0 { + log.Printf("Apps: loading TEST") + fs := memfs.New() + storer := memory.NewStorage() + r, err := git.Clone(storer, fs, &git.CloneOptions{ + URL: "https://github.com/frikky/shuffle-apps", + }) + + if err != nil { + log.Printf("Failed loading repo into memory: %s", err) + } + + dir, err := fs.ReadDir("") + if err != nil { + log.Printf("Failed reading folder: %s", err) + } + _ = r + //iterateAppGithubFolders(fs, dir, "", "testing") + + // FIXME: Get all the apps? + iterateAppGithubFolders(fs, dir, "", "") + } + log.Printf("Finished INIT") } @@ -5721,9 +5724,8 @@ func init() { r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST") r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") - // Apps - r.HandleFunc("/api/v1/apps/get_existing", loadExistingApps).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/apps/get_existing/{appname}", loadExistingApps).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/apps/get_existing", loadSpecificApps).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}", deleteWorkflowApp).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/apps/{appId}/config", getWorkflowAppConfig).Methods("GET", "OPTIONS") diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index ac4c4653..ed5c52b4 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -26,6 +26,7 @@ import ( "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" + http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" newscheduler "github.com/carlescere/scheduler" "github.com/go-git/go-git/v5/storage/memory" @@ -3113,7 +3114,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation return nil } -func loadExistingApps(resp http.ResponseWriter, request *http.Request) { +func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return @@ -3129,22 +3130,83 @@ func loadExistingApps(resp http.ResponseWriter, request *http.Request) { return } + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Error with body read: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + // Field1 & 2 can be a lot of things.. + type tmpStruct struct { + URL string `json:"url"` + Field1 string `json:"field_1"` + Field2 string `json:"field_2"` + } + //log.Printf("Body: %s", string(body)) + + var tmpBody tmpStruct + err = json.Unmarshal(body, &tmpBody) + if err != nil { + log.Printf("Error with unmarshal tmpBody: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + fs := memfs.New() - storer := memory.NewStorage() - r, err := git.Clone(storer, fs, &git.CloneOptions{ - URL: "https://github.com/frikky/shuffle-apps", - }) - if err != nil { - log.Printf("Failed loading repo into memory: %s", err) - } + if strings.Contains(tmpBody.URL, "github") || strings.Contains(tmpBody.URL, "gitlab") || strings.Contains(tmpBody.URL, "bitbucket") { + cloneOptions := &git.CloneOptions{ + URL: tmpBody.URL, + } - dir, err := fs.ReadDir("/") - if err != nil { - log.Printf("FAiled reading folder: %s", err) + // FIXME: Better auth. + if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 { + cloneOptions.Auth = &http2.BasicAuth{ + Username: tmpBody.Field1, + Password: tmpBody.Field2, + } + } + + storer := memory.NewStorage() + r, err := git.Clone(storer, fs, cloneOptions) + if err != nil { + log.Printf("Failed loading repo into memory: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) + return + } + + dir, err := fs.ReadDir("/") + if err != nil { + log.Printf("FAiled reading folder: %s", err) + } + _ = r + iterateAppGithubFolders(fs, dir, "", "") + + } else if strings.Contains(tmpBody.URL, "s3") { + //https://docs.aws.amazon.com/sdk-for-go/api/service/s3/ + + //sess := session.Must(session.NewSession()) + //downloader := s3manager.NewDownloader(sess) + + //// Write the contents of S3 Object to the file + //storer := memory.NewStorage() + //n, err := downloader.Download(storer, &s3.GetObjectInput{ + // Bucket: aws.String(myBucket), + // Key: aws.String(myString), + //}) + //if err != nil { + // return fmt.Errorf("failed to download file, %v", err) + //} + //fmt.Printf("file downloaded, %d bytes\n", n) + } else { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s is unsupported"}`, tmpBody.URL))) + return } - _ = r - iterateAppGithubFolders(fs, dir, "", "") resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) diff --git a/frontend/src/Apps.js b/frontend/src/Apps.js index fc394cd1..aeb09d03 100644 --- a/frontend/src/Apps.js +++ b/frontend/src/Apps.js @@ -30,6 +30,7 @@ const Apps = (props) => { const { globalUrl, isLoggedIn, isLoaded } = props; //const [workflows, setWorkflows] = React.useState([]); + const baseRepository = "https://github.com/frikky/shuffle-apps" const alert = useAlert() const [selectedApp, setSelectedApp] = React.useState({}); const [firstrequest, setFirstrequest] = React.useState(true) @@ -45,6 +46,9 @@ const Apps = (props) => { const [openApiModal, setOpenApiModal] = React.useState(false); const [openApiModalType, setOpenApiModalType] = React.useState(""); const [openApiError, setOpenApiError] = React.useState("") + const [field1, setField1] = React.useState("") + const [field2, setField2] = React.useState("") + const { start, stop } = useInterval({ duration: 5000, startImmediate: false, @@ -406,10 +410,10 @@ const Apps = (props) => { color="primary" style={{margin: 5, maxHeight: 50, marginTop: 10}} onClick={() => { - getExistingApps() + getSpecificApps(baseRepository) }} > - Load existing apps + Load apps { : null - // Gets the URL itself (hopefully this works in most cases? - // Will then forward the data to an internal endpoint to validate the api - const getExistingApps = () => { + // Load data e.g. from github + const getSpecificApps = (url) => { setValidation(true) setIsLoading(true) start() - alert.success("Downloading and building apps. Feel free to move around meanwhile.") + const parsedData = { + "url": url, + } + + if (field1.length > 0) { + parsedData["field_1"] = field1 + } + + if (field2.length > 0) { + parsedData["field_2"] = field2 + } + + alert.success("Getting specific apps from your URL.") var cors = "cors" fetch(globalUrl+"/api/v1/apps/get_existing", { - method: "GET", + method: "POST", mode: "cors", headers: { 'Accept': 'application/json', }, + body: JSON.stringify(parsedData), credentials: "include", }) .then((response) => { - if (response.status !== 200) { - alert.success("Failed loading.") - } else { + if (response.status === 200) { response.text().then(function (text) { console.log("RETURN: ", text) alert.success("Loaded existing apps!") @@ -500,6 +515,16 @@ const Apps = (props) => { } setIsLoading(false) stop() + + return response.json() + }) + .then((responseJson) => { + console.log("DATA: ", responseJson) + if (responseJson.reason !== undefined) { + alert.error("Failed loading: "+responseJson.reason) + } else { + alert.error("Failed loading") + } }) .catch(error => { alert.error(error.toString()) @@ -636,9 +661,8 @@ const Apps = (props) => { const handleGithubValidation = () => { - console.log("VALIDATE") - setValidation(true) - //const circularLoader = validation ? : null + getSpecificApps(openApi) + setLoadAppsModalOpen(false) } const appsModalLoad = loadAppsModalOpen ? @@ -647,6 +671,8 @@ const Apps = (props) => { onClose={() => { setOpenApi("") setLoadAppsModalOpen(false) + setField1("") + setField2("") }} PaperProps={{ style: { @@ -663,11 +689,12 @@ const Apps = (props) => { - Github repository + Repository (supported: github, gitlab, bitbucket) { placeholder="https://github.com/frikky/shuffle-apps" fullWidth /> + + Authentication (optional - private repos etc):
{ fontSize: "1em", }, }} - onChange={e => setOpenApi(e.target.value)} - placeholder="https://github.com/frikky/shuffle-apps" + onChange={e => setField1(e.target.value)} + type="username" + placeholder="Username / APIkey (optional)" fullWidth /> { fontSize: "1em", }, }} - onChange={e => setOpenApi(e.target.value)} - placeholder="https://github.com/frikky/shuffle-apps" + onChange={e => setField2(e.target.value)} + type="password" + placeholder="Password (optional)" fullWidth />
@@ -718,6 +749,7 @@ const Apps = (props) => { Cancel diff --git a/frontend/src/LoginPage.js b/frontend/src/LoginPage.js index 040aeeac..7ff22e80 100644 --- a/frontend/src/LoginPage.js +++ b/frontend/src/LoginPage.js @@ -172,7 +172,7 @@ const LoginDialog = props => { const basedata =
-
+

{formtitle}

Username