marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Made it possible to auto-authenticate oauth2 with clientid and secret

Browse Source
This commit is contained in:
frikky
2021-08-24 01:14:32 +02:00
parent 488139c914
commit e9419eb7a8
10 changed files with 389 additions and 572 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/walkoff.go
+4 -497
View File
@@ -19,10 +19,10 @@ import (
"strings"
"time"
"github.com/docker/docker/api/types"
"github.com/docker/docker/client"
//"github.com/docker/docker/api/types"
//"github.com/docker/docker/client"
//gyaml "github.com/ghodss/yaml"
gyaml "github.com/ghodss/yaml"
"github.com/h2non/filetype"
uuid "github.com/satori/go.uuid"
@@ -2735,29 +2735,6 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) {
resp.Write(newbody)
}
// Bad check for workflowapps :)
// FIXME - use tags and struct reflection
func checkWorkflowApp(workflowApp shuffle.WorkflowApp) error {
// Validate fields
if workflowApp.Name == "" {
return errors.New("App field name doesn't exist")
}
if workflowApp.Description == "" {
return errors.New("App field description doesn't exist")
}
if workflowApp.AppVersion == "" {
return errors.New("App field app_version doesn't exist")
}
if workflowApp.ContactInfo.Name == "" {
return errors.New("App field contact_info.name doesn't exist")
}
return nil
}
func handleGetfile(resp http.ResponseWriter, request *http.Request) ([]byte, error) {
// Upload file here first
request.ParseMultipartForm(32 << 20)
@@ -3084,141 +3061,6 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// Just need to be logged in
// FIXME - should have some permissions?
_, err := shuffle.HandleApiAuthentication(resp, request)
if err != nil {
log.Printf("[WARNING] Api authentication failed in load specific apps: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
log.Printf("Error with body read: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
// Field1 & 2 can be a lot of things.
// Field1 = Username
// Field2 = Password
type tmpStruct struct {
URL string `json:"url"`
Branch string `json:"branch"`
Field1 string `json:"field_1"`
Field2 string `json:"field_2"`
ForceUpdate bool `json:"force_update"`
}
//log.Printf("Body: %s", string(body))
var tmpBody tmpStruct
err = json.Unmarshal(body, &tmpBody)
if err != nil {
log.Printf("Error with unmarshal tmpBody: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
fs := memfs.New()
if strings.Contains(tmpBody.URL, "github") || strings.Contains(tmpBody.URL, "gitlab") || strings.Contains(tmpBody.URL, "bitbucket") {
cloneOptions := &git.CloneOptions{
URL: tmpBody.URL,
}
if len(tmpBody.Branch) > 0 && tmpBody.Branch != "master" && tmpBody.Branch != "main" {
cloneOptions.ReferenceName = plumbing.ReferenceName(tmpBody.Branch)
}
// FIXME: Better auth.
if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 {
cloneOptions.Auth = &http2.BasicAuth{
Username: tmpBody.Field1,
Password: tmpBody.Field2,
}
}
storer := memory.NewStorage()
r, err := git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("Failed loading repo %s into memory (github workflows 2): %s", tmpBody.URL, err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
}
dir, err := fs.ReadDir("/")
if err != nil {
log.Printf("FAiled reading folder: %s", err)
}
_ = r
if tmpBody.ForceUpdate {
log.Printf("[INFO] Running with force update!")
} else {
log.Printf("[INFO] Updating apps with updates (no force)")
}
if tmpBody.ForceUpdate {
ctx := context.Background()
dockercli, err := client.NewEnvClient()
if err == nil {
_, err := dockercli.ImagePull(ctx, "frikky/shuffle:app_sdk", types.ImagePullOptions{})
if err != nil {
log.Printf("[WARNING] Failed to download apps with the new App SDK: %s", err)
}
} else {
log.Printf("[WARNING] Failed to download apps with the new App SDK because of docker cli: %s", err)
}
}
iterateAppGithubFolders(fs, dir, "", "", tmpBody.ForceUpdate)
} else if strings.Contains(tmpBody.URL, "s3") {
//https://docs.aws.amazon.com/sdk-for-go/api/service/s3/
//sess := session.Must(session.NewSession())
//downloader := s3manager.NewDownloader(sess)
//// Write the contents of S3 Object to the file
//storer := memory.NewStorage()
//n, err := downloader.Download(storer, &s3.GetObjectInput{
// Bucket: aws.String(myBucket),
// Key: aws.String(myString),
//})
//if err != nil {
// return fmt.Errorf("failed to download file, %v", err)
//}
//fmt.Printf("file downloaded, %d bytes\n", n)
} else {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s is unsupported"}`, tmpBody.URL)))
return
}
ctx := context.Background()
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
ctx := context.Background()
@@ -3482,341 +3324,6 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
return err
}
type buildLaterStruct struct {
Tags []string
Extra string
Id string
}
// Onlyname is used to
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) {
var err error
allapps := []shuffle.WorkflowApp{}
// These are slow apps to build with some funky mechanisms
reservedNames := []string{
"OWA",
"NLP",
"YARA",
}
// It's here to prevent getting them in every iteration
buildLaterFirst := []buildLaterStruct{}
buildLaterList := []buildLaterStruct{}
ctx := context.Background()
for _, file := range dir {
if len(onlyname) > 0 && file.Name() != onlyname {
continue
}
// Folder?
switch mode := file.Mode(); {
case mode.IsDir():
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
dir, err := fs.ReadDir(tmpExtra)
if err != nil {
log.Printf("Failed to read dir: %s", err)
continue
}
// Go routine? Hmm, this can be super quick I guess
buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
for _, item := range buildFirst {
buildLaterFirst = append(buildLaterFirst, item)
}
for _, item := range buildLast {
buildLaterList = append(buildLaterList, item)
}
if err != nil {
log.Printf("[WARNING] Error reading folder: %s", err)
//buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
if !forceUpdate {
return buildLaterFirst, buildLaterList, err
}
}
case mode.IsRegular():
// Check the file
filename := file.Name()
if filename == "Dockerfile" {
// Set up to make md5 and check if the app is new (api.yaml+src/app.py+Dockerfile)
// Check if Dockerfile, app.py or api.yaml has changed. Hash?
//log.Printf("Handle Dockerfile in location %s", extra)
// Try api.yaml and api.yml
fullPath := fmt.Sprintf("%s%s", extra, "api.yaml")
fileReader, err := fs.Open(fullPath)
if err != nil {
fullPath = fmt.Sprintf("%s%s", extra, "api.yml")
fileReader, err = fs.Open(fullPath)
if err != nil {
log.Printf("Failed finding api.yaml/yml: %s", err)
continue
}
}
//log.Printf("HANDLING DOCKER FILEREADER - SEARCH&REPLACE?")
appfileData, err := ioutil.ReadAll(fileReader)
if err != nil {
log.Printf("Failed reading %s: %s", fullPath, err)
continue
}
if len(appfileData) == 0 {
log.Printf("Failed reading %s - length is 0.", fullPath)
continue
}
// func md5sum(data []byte) string {
// Make hash
appPython := fmt.Sprintf("%s/src/app.py", extra)
appPythonReader, err := fs.Open(appPython)
if err != nil {
log.Printf("Failed to read python app %s", appPython)
continue
}
appPythonData, err := ioutil.ReadAll(appPythonReader)
if err != nil {
log.Printf("Failed reading appdata %s: %s", appPython, err)
continue
}
dockerFp := fmt.Sprintf("%s/Dockerfile", extra)
dockerfile, err := fs.Open(dockerFp)
if err != nil {
log.Printf("Failed to read dockerfil %s", appPython)
continue
}
dockerfileData, err := ioutil.ReadAll(dockerfile)
if err != nil {
log.Printf("Failed to read dockerfile")
continue
}
combined := []byte{}
combined = append(combined, appfileData...)
combined = append(combined, appPythonData...)
combined = append(combined, dockerfileData...)
md5 := md5sum(combined)
var workflowapp shuffle.WorkflowApp
err = gyaml.Unmarshal(appfileData, &workflowapp)
if err != nil {
log.Printf("[WARNING] Failed building workflowapp %s: %s", extra, err)
return buildLaterFirst, buildLaterList, errors.New(fmt.Sprintf("Failed building %s: %s", extra, err))
//continue
}
newName := workflowapp.Name
newName = strings.ReplaceAll(newName, " ", "-")
tags := []string{
fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ToLower(newName), workflowapp.AppVersion),
}
if len(allapps) == 0 {
allapps, err = shuffle.GetAllWorkflowApps(ctx, 0)
if err != nil {
log.Printf("[WARNING] Failed getting apps to verify: %s", err)
continue
}
}
// Make an option to override existing apps?
//Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps
removeApps := []string{}
skip := false
for _, app := range allapps {
if app.Name == workflowapp.Name && app.AppVersion == workflowapp.AppVersion {
// FIXME: Check if there's a new APP_SDK as well.
// Skip this check if app_sdk is new.
if app.Hash == md5 && app.Hash != "" && !forceUpdate {
skip = true
break
}
//log.Printf("Overriding app %s:%s as it exists but has different hash.", app.Name, app.AppVersion)
removeApps = append(removeApps, app.ID)
}
}
if skip && !forceUpdate {
continue
}
// Fixes (appends) authentication parameters if they're required
if workflowapp.Authentication.Required {
//log.Printf("[INFO] Checking authentication fields and appending for %s!", workflowapp.Name)
// FIXME:
// Might require reflection into the python code to append the fields as well
for index, action := range workflowapp.Actions {
if action.AuthNotRequired {
log.Printf("Skipping auth setup: %s", action.Name)
continue
}
// 1. Check if authentication params exists at all
// 2. Check if they're present in the action
// 3. Add them IF they DONT exist
// 4. Fix python code with reflection (FIXME)
appendParams := []shuffle.WorkflowAppActionParameter{}
for _, fieldname := range workflowapp.Authentication.Parameters {
found := false
for index, param := range action.Parameters {
if param.Name == fieldname.Name {
found = true
action.Parameters[index].Configuration = true
//log.Printf("Set config to true for field %s!", param.Name)
break
}
}
if !found {
appendParams = append(appendParams, shuffle.WorkflowAppActionParameter{
Name: fieldname.Name,
Description: fieldname.Description,
Example: fieldname.Example,
Required: fieldname.Required,
Configuration: true,
Schema: fieldname.Schema,
})
}
}
if len(appendParams) > 0 {
//log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name)
workflowapp.Actions[index].Parameters = append(appendParams, workflowapp.Actions[index].Parameters...)
}
}
}
err = checkWorkflowApp(workflowapp)
if err != nil {
log.Printf("[DEBUG] %s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion)
continue
}
if len(removeApps) > 0 {
for _, item := range removeApps {
log.Printf("[WARNING] Removing duplicate app: %s", item)
err = shuffle.DeleteKey(ctx, "workflowapp", item)
if err != nil {
log.Printf("[ERROR] Failed deleting duplicate %s: %s", item, err)
}
}
}
workflowapp.ID = uuid.NewV4().String()
workflowapp.IsValid = true
workflowapp.Verified = true
workflowapp.Sharing = true
workflowapp.Downloaded = true
workflowapp.Hash = md5
workflowapp.Public = true
err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID)
if err != nil {
log.Printf("[WARNING] Failed setting workflowapp in intro: %s", err)
continue
}
/*
err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "")
if err != nil {
log.Printf("Failed to increase total apps created stats: %s", err)
}
err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "")
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
*/
//log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
// ID can be used to e.g. set a build status.
buildLater := buildLaterStruct{
Tags: tags,
Extra: extra,
Id: workflowapp.ID,
}
reservedFound := false
for _, appname := range reservedNames {
if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
buildLaterList = append(buildLaterList, buildLater)
reservedFound = true
break
}
}
/// Only upload if successful and no errors
if !reservedFound {
buildLaterFirst = append(buildLaterFirst, buildLater)
} else {
log.Printf("[WARNING] Skipping build of %s to later", workflowapp.Name)
}
}
}
}
if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 {
return buildLaterFirst, buildLaterList, err
}
// This is getting silly
cacheKey := fmt.Sprintf("workflowapps-sorted-100")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-500")
shuffle.DeleteCache(ctx, cacheKey)
cacheKey = fmt.Sprintf("workflowapps-sorted-1000")
shuffle.DeleteCache(ctx, cacheKey)
//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
if len(extra) == 0 {
log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
for _, item := range buildLaterFirst {
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("[INFO] Successfully built image %s", item.Tags[0])
} else {
log.Printf("[INFO] Successfully built Docker image")
}
}
}
log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList))
for _, item := range buildLaterList {
err = buildImageMemory(fs, item.Tags, item.Extra, true)
if err != nil {
log.Printf("[INFO] Failed image build memory: %s", err)
} else {
if len(item.Tags) > 0 {
log.Printf("[INFO] Successfully built image %s", item.Tags[0])
} else {
log.Printf("[INFO] Successfully built Docker image")
}
}
}
}
return buildLaterFirst, buildLaterList, err
}
func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -3875,7 +3382,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
return
}
err = checkWorkflowApp(workflowapp)
err = shuffle.CheckWorkflowApp(workflowapp)
if err != nil {
log.Printf("%s for app %s:%s", err, workflowapp.Name, workflowapp.AppVersion)
resp.WriteHeader(401)