marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #234 from frikky/launch

Browse Source 
Shuffle v0.56 - Scalability and file fixing
This commit is contained in:
Frikky
2021-01-27 15:32:28 +01:00
committed by GitHub
parent 7899f9764d 6ad9a3bfae
commit 4fe40eae10
42 changed files with 8835 additions and 2051 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/Dockerfile
+1 -1
View File
@@ -7,7 +7,7 @@ RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
RUN pip3 install -r /requirements.txt
FROM base
backend/app_sdk/app_base.py
+609 -152
View File
File diff suppressed because it is too large Load Diff
backend/app_sdk/build.sh
+2 -1
View File
@@ -1,6 +1,6 @@
#!/bin/bash
NAME=shuffle-app_sdk
VERSION=0.8.2
VERSION=0.8.54
docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
@@ -8,6 +8,7 @@ docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.
#docker push frikky/$NAME:$VERSION
#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
#docker push ghcr.io/frikky/$NAME:$VERSION
#docker tag ghcr.io/frikky/$NAME:$VERSION frikky/shuffle:app_sdk
docker push frikky/shuffle:app_sdk
docker push ghcr.io/frikky/$NAME:$VERSION
backend/app_sdk/requirements.txt
+1 -1
View File
@@ -1,2 +1,2 @@
requests
urllib3
requests
backend/go-app/codegen.go
+90 -18
View File
@@ -244,7 +244,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
// This function generates the python code that's being used.
// This is really meta when you program it. Handling parameters is hard here.
func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string) (string, string) {
func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string, fileField string) (string, string) {
method = strings.ToLower(method)
queryString := ""
queryData := ""
@@ -365,21 +365,39 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
preparedHeaders += "}"
}
fileBalance := ""
fileAdder := ``
fileGrabber := ``
fileParameter := ``
if method == "post" && len(fileField) > 0 {
fileParameter = ", file_id"
fileGrabber = `filedata = self.get_file(file_id)`
// This indentation is confusing (but correct) ROFL
fileAdder = fmt.Sprintf(`if not filedata["success"]:
return file_id+" is not a valid File ID"
files = {"%s": (filedata["filename"], filedata["data"])}`, fileField)
fileBalance = ", files=files"
}
// Extra param for url if it's changeable
// Extra param for authentication scheme(s)
// The last weird one is the body.. Tabs & spaces sucks.
data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s):
data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s%s):
%s
url=f"%s%s"
%s
%s
%s
%s
return requests.%s(url, headers=headers%s%s%s).text
%s
%s
return requests.%s(url, headers=headers%s%s%s%s).text
`,
functionname,
authenticationParameter,
urlParameter,
fileParameter,
parameterData,
queryString,
bodyParameter,
@@ -391,18 +409,20 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet
authenticationSetup,
queryData,
bodyFormatter,
fileGrabber,
fileAdder,
method,
authenticationAddin,
bodyAddin,
verifyAddin,
fileBalance,
)
/*
if strings.Contains(functionname, "search") {
log.Println(data)
log.Printf("Queries: %s", queryString)
}
*/
if strings.Contains(functionname, "filescan") {
//log.Printf("FUNCTION: %s", data)
log.Println(data)
log.Printf("Queries: %s", queryString)
}
//log.Printf(data)
return functionname, data
@@ -446,6 +466,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
api.Sharing = false
api.Verified = false
api.Tested = false
api.Invalid = false
api.PrivateID = newmd5
api.Generated = true
api.Activated = true
@@ -638,10 +659,15 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
// This is the python code to be generated
// Could just as well be go at this point lol
pythonFunctions := []string{}
//Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"`
for actualPath, path := range swagger.Paths {
actualPath = strings.Replace(actualPath, " ", "_", -1)
//actualPath = strings.Replace(actualPath, ".", "", -1)
actualPath = strings.Replace(actualPath, "\\", "", -1)
if !api.Invalid && strings.HasPrefix(actualPath, "tmp") {
log.Printf("[WARNING] Set api %s to invalid because of path %s", swagger.Info.Title, actualPath)
api.Invalid = true
}
// FIXME: Handle everything behind questionmark (?) with dots as well.
// https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem
@@ -846,10 +872,10 @@ def run(request):
func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error {
err := setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID)
if err != nil {
log.Printf("Failed setting workflowapp: %s", err)
log.Printf("[ERROR] Failed setting workflowapp: %s", err)
return err
} else {
log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
log.Printf("[INFO] Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
}
return nil
@@ -948,6 +974,12 @@ func validateParameterName(name string) string {
}
}
newname = strings.ReplaceAll(newname, " ", "_")
newname = strings.ReplaceAll(newname, ",", "_")
newname = strings.ReplaceAll(newname, ".", "_")
newname = strings.ReplaceAll(newname, "|", "_")
newname = strings.ReplaceAll(newname, "-", "_")
return newname
}
@@ -1001,6 +1033,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
parsedName = strings.ReplaceAll(parsedName, ",", "_")
parsedName = strings.ReplaceAll(parsedName, ".", "_")
parsedName = strings.ReplaceAll(parsedName, "|", "_")
parsedName = strings.ReplaceAll(parsedName, "-", "_")
parsedName = validateParameterName(parsedName)
param.Value.Name = parsedName
path.Connect.Parameters[counter].Value.Name = parsedName
@@ -1076,7 +1109,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1211,7 +1244,7 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1344,7 +1377,7 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1478,7 +1511,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1521,6 +1554,40 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
},
})
fileField := ""
if path.Post.RequestBody != nil {
//log.Printf("DATA: %#v",
value := path.Post.RequestBody.Value
//log.Printf("VAL: %#v", value.Content)
if val, ok := value.Content["multipart/form-data"]; ok {
if val.Schema.Value != nil {
if innerval, ok := val.Schema.Value.Properties["fieldname"]; ok {
if extensionvalue, ok := innerval.Value.ExtensionProps.Extensions["value"]; ok {
fieldname := extensionvalue.(json.RawMessage)
newName := string(fmt.Sprintf("%s", string(fieldname)))
if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 {
parsedName := newName[1 : len(newName)-1]
log.Printf("Parse name: %s", parsedName)
fileField = parsedName
curParam := WorkflowAppActionParameter{
Name: "file_id",
Description: "Files to be uploaded",
Multiline: false,
Required: true,
Schema: SchemaDefinition{
Type: "string",
},
}
action.Parameters = append(action.Parameters, curParam)
}
}
}
}
}
}
headersFound := []string{}
if len(path.Post.Parameters) > 0 {
for counter, param := range path.Post.Parameters {
@@ -1610,12 +1677,17 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound, fileField)
if len(functionname) > 0 {
action.Name = functionname
}
//log.Printf("PARAMS: %d", len(action.Parameters))
//for _, param := range action.Parameters {
// log.Printf("%#v", param)
//}
return action, curCode
}
@@ -1743,7 +1815,7 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
@@ -1877,7 +1949,7 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor
action.Parameters = append(action.Parameters, optionalParam)
}
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound)
functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound, "")
if len(functionname) > 0 {
action.Name = functionname
backend/go-app/docker.go
+192 -9
View File
@@ -8,6 +8,7 @@ import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
@@ -125,12 +126,29 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri
return err
}
//log.Printf("FILENAME: %s", filename)
readFile, err := ioutil.ReadAll(fileReader)
if err != nil {
log.Printf("Not file: %s", err)
return err
}
// Fixes issues with older versions of Docker and reference formats
// Specific to Shuffle rn. Could expand.
// FIXME: Seems like the issue was with multi-stage builds
/*
if filename == "Dockerfile" {
log.Printf("Should search and replace in readfile.")
referenceCheck := "FROM frikky/shuffle:"
if strings.Contains(string(readFile), referenceCheck) {
log.Printf("SHOULD SEARCH & REPLACE!")
newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:")
readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1))
}
}
*/
//log.Printf("Filename: %s", filename)
// FIXME - might need the folder from EXTRA here
// Name has to be e.g. just "requirements.txt"
@@ -156,8 +174,23 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri
return nil
}
/*
// Fixes App SDK issues.. meh
func fixTags(tags []string) []string {
checkTag := "frikky/shuffle"
newTags := []string{}
for _, tag := range tags {
if strings.HasPrefix(tag, checkTags) {
newTags.append(newTags, fmt.Sprintf("registry.hub.docker.com/%s", tag))
}
newTags.append(tag)
}
}
*/
// Custom Docker image builder wrapper in memory
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error {
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
ctx := context.Background()
client, err := client.NewEnvClient()
if err != nil {
@@ -169,7 +202,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
tw := tar.NewWriter(buf)
defer tw.Close()
log.Printf("Setting up memory build structure for folder: %s", dockerfileFolder)
log.Printf("[INFO] Setting up memory build structure for folder: %s", dockerfileFolder)
err = getParsedTarMemory(fs, tw, dockerfileFolder, "")
if err != nil {
log.Printf("Tar issue: %s", err)
@@ -197,17 +230,56 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
}
// Build the actual image
log.Printf("[INFO] Building %s. This may take up to a few minutes.", dockerfileFolder)
imageBuildResponse, err := client.ImageBuild(
ctx,
dockerFileTarReader,
buildOptions,
)
//log.Printf("Response: %#v", imageBuildResponse.Body)
//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
defer imageBuildResponse.Body.Close()
_, newerr := io.Copy(os.Stdout, imageBuildResponse.Body)
buildBuf := new(strings.Builder)
_, newerr := io.Copy(buildBuf, imageBuildResponse.Body)
if newerr != nil {
log.Printf("Failed reading Docker build STDOUT: %s", newerr)
} else {
log.Printf("STRING: %s", buildBuf.String())
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
// Handles pulling of the same image if applicable
// This fixes some issues with older versions of Docker which can't build
// on their own ( <17.05 )
pullOptions := types.ImagePullOptions{}
downloaded := false
for _, image := range tags {
// Is this ok? Not sure. Tags shouldn't be controlled here prolly.
image = strings.ToLower(image)
newImage := fmt.Sprintf("%s/%s", registryName, image)
log.Printf("[INFO] Pulling image %s", newImage)
reader, err := client.ImagePull(ctx, newImage, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", newImage, err)
continue
}
// Attempt to retag the image to not contain registry...
//newBuf := buildBuf
downloaded = true
io.Copy(os.Stdout, reader)
log.Printf("[INFO] Successfully downloaded and built %s", newImage)
}
if !downloaded {
return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ",")))
}
//baseDockerName
}
}
if err != nil {
@@ -272,9 +344,15 @@ func buildImage(tags []string, dockerfileFolder string) error {
// Read the STDOUT from the build process
defer imageBuildResponse.Body.Close()
_, err = io.Copy(os.Stdout, imageBuildResponse.Body)
buildBuf := new(strings.Builder)
_, err = io.Copy(buildBuf, imageBuildResponse.Body)
if err != nil {
return err
} else {
if strings.Contains(buildBuf.String(), "errorDetail") {
log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n"))
return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ",")))
}
}
return nil
@@ -424,7 +502,7 @@ func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) {
ctx := context.Background()
hook, err := getHook(ctx, fileId)
if err != nil {
log.Printf("Failed getting hook: %s", err)
log.Printf("Failed getting hook %s (stop docker): %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -556,7 +634,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
ctx := context.Background()
hook, err := getHook(ctx, fileId)
if err != nil {
log.Printf("Failed getting hook: %s", err)
log.Printf("Failed getting hook %s (start docker): %s", fileId, err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
@@ -636,7 +714,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
}
// FIXME - get some real data?
log.Printf("Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath)
log.Printf("[INFO] Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true, "message": "Started webhook"}`))
return
@@ -644,7 +722,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) {
// Checks if an image exists
func imageCheckBuilder(images []string) error {
log.Printf("[FIXME] ImageNames to check: %#v", images)
//log.Printf("[FIXME] ImageNames to check: %#v", images)
return nil
ctx := context.Background()
@@ -704,10 +782,115 @@ func hookTest() {
returnHook, err := getHook(ctx, hook.Id)
if err != nil {
log.Printf("Failed getting hook: %s", err)
log.Printf("Failed getting hook %s (test): %s", hook.Id, err)
}
if len(returnHook.Id) > 0 {
log.Printf("Success! - %s", returnHook.Id)
}
}
//https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository
func getDockerImage(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// Just here to verify that the user is logged in
_, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in validate swagger: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
body, err := ioutil.ReadAll(request.Body)
if err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "Failed reading body"}`))
return
}
type requestCheck struct {
Name string `datastore:"name" json:"name" yaml:"name"`
}
//body = []byte(`swagger: "2.0"`)
//body = []byte(`swagger: '1.0'`)
//newbody := string(body)
//newbody = strings.TrimSpace(newbody)
//body = []byte(newbody)
//log.Println(string(body))
//tmpbody, err := yaml.YAMLToJSON(body)
//log.Println(err)
//log.Println(string(tmpbody))
// This has to be done in a weird way because Datastore doesn't
// support map[string]interface and similar (openapi3.Swagger)
var version requestCheck
err = json.Unmarshal(body, &version)
if err != nil {
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
return
}
log.Printf("Image to load: %s", version.Name)
//cli, err := client.NewEnvClient()
//if err != nil {
// log.Println("Unable to create docker client")
// return err
//}
dockercli, err := client.NewEnvClient()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
resp.WriteHeader(422)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed JSON marshalling: %s"}`, err)))
return
}
ctx := context.Background()
images, err := dockercli.ImageList(ctx, types.ImageListOptions{
All: true,
})
img := types.ImageSummary{}
tagFound := ""
for _, image := range images {
for _, tag := range image.RepoTags {
log.Printf("Image: %s", tag)
if strings.ToLower(tag) == strings.ToLower(version.Name) {
img = image
tagFound = tag
break
}
}
}
if len(img.ID) == 0 {
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "message": "Couldn't find image %s"}`, version.Name)))
return
}
_ = tagFound
/*
log.Printf("IMg: %#v", img)
pullOptions := types.ImagePullOptions{}
log.Printf("[INFO] Pulling image %s", image)
reader, err := dockercli.ImagePull(ctx, tag, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", image, err)
}
io.Copy(os.Stdout, r)
*/
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Downloading image %s"}`, version.Name)))
}
backend/go-app/files.go
+112 -3
View File
@@ -42,6 +42,9 @@ type File struct {
DownloadPath string `json:"download_path" datastore:"download_path"`
Md5sum string `json:"md5_sum" datastore:"md5_sum"`
Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"`
FileSize int64 `json:"filesize" datastore:"filesize"`
Duplicate bool `json:"duplicate" datastore:"duplicate"`
Subflows []string `json:"subflows" datastore:"subflows"`
}
var basepath = os.Getenv("SHUFFLE_FILE_LOCATION")
@@ -100,6 +103,51 @@ func fileExists(filename string) bool {
return !info.IsDir()
}
func handleGetFiles(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
// 1. Check user directly
// 2. Check workflow execution authorization
user, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
if user.Role != "admin" {
log.Printf("[AUTH] User isn't admin")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`)))
return
}
ctx := context.Background()
files, err := getAllFiles(ctx, user.ActiveOrg.Id)
if err != nil {
log.Printf("[ERROR] Failed to get files: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`)))
return
}
log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id)
newBody, err := json.Marshal(files)
if err != nil {
log.Printf("[ERROR] Failed marshaling files: %s", err)
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`))
return
}
resp.WriteHeader(200)
resp.Write([]byte(newBody))
}
func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -347,7 +395,7 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
return
}
log.Printf("\n\nUser is trying to download file %s\n\n", fileId)
log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId)
// 1. Check user directly
// 2. Check workflow execution authorization
@@ -417,8 +465,18 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) {
Openfile, err := os.Open(downloadPath)
defer Openfile.Close() //Close after function return
if err != nil {
file.Status = "deleted"
err = setFile(ctx, *file)
if err != nil {
log.Printf("Failed setting file to uploading")
resp.WriteHeader(500)
resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`))
return
}
//File not found, send 404
http.Error(resp, "File not found.", 404)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`))
return
}
@@ -552,6 +610,7 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) {
var buf bytes.Buffer
io.Copy(&buf, parsedFile)
contents := buf.Bytes()
file.FileSize = int64(len(contents))
md5 := md5sum(contents)
buf.Reset()
@@ -642,7 +701,8 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
// Loads of validation below
if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 {
log.Printf("[ERROR] Missing field during upload.")
log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id")
log.Printf("INPUT: %s", string(body))
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`)))
return
@@ -715,6 +775,30 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
fileId := uuid.NewV4().String()
downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId)
duplicateWorkflows := []string{}
for _, trigger := range workflow.Triggers {
if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" {
for _, parameter := range trigger.Parameters {
if parameter.Name == "workflow" && len(parameter.Value) > 0 {
found := false
for _, workflow := range duplicateWorkflows {
if workflow == parameter.Value {
found = true
break
}
}
if !found {
duplicateWorkflows = append(duplicateWorkflows, parameter.Value)
}
break
}
}
}
}
timeNow := time.Now().Unix()
newFile := File{
Id: fileId,
@@ -726,6 +810,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
OrgId: curfile.OrgId,
WorkflowId: curfile.WorkflowId,
DownloadPath: downloadPath,
Subflows: duplicateWorkflows,
}
err = setFile(ctx, newFile)
@@ -740,6 +825,7 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) {
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId)))
}
func getFile(ctx context.Context, id string) (*File, error) {
@@ -754,6 +840,9 @@ func getFile(ctx context.Context, id string) (*File, error) {
func setFile(ctx context.Context, file File) error {
// clear session_token and API_token for user
timeNow := time.Now().Unix()
file.UpdatedAt = timeNow
k := datastore.NameKey("Files", file.Id, nil)
if _, err := dbclient.Put(ctx, k, &file); err != nil {
log.Println(err)
@@ -762,3 +851,23 @@ func setFile(ctx context.Context, file File) error {
return nil
}
func getAllFiles(ctx context.Context, orgId string) ([]File, error) {
var files []File
q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100)
_, err := dbclient.GetAll(ctx, q, &files)
if err != nil {
if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") {
q = q.Limit(50)
_, err := dbclient.GetAll(ctx, q, &files)
if err != nil {
return []File{}, err
}
} else {
return []File{}, err
}
}
return files, nil
}
backend/go-app/go.mod
+1
View File
@@ -23,6 +23,7 @@ require (
github.com/gorilla/mux v1.7.4
github.com/h2non/filetype v1.0.12
github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/satori/go.uuid v1.2.0
golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
backend/go-app/go.sum
+2
View File
@@ -160,6 +160,8 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
backend/go-app/main.go
+494 -115
View File
File diff suppressed because it is too large Load Diff
backend/go-app/walkoff.go
+1277 -324
View File
File diff suppressed because it is too large Load Diff
backend/tests/execute.sh
+25 -4
View File
@@ -1,5 +1,26 @@
#!/bin/sh
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26"
curl -XPOST http://localhost:5001/api/v1/workflows/1d9d8ce2-566e-4c3f-8a37-5d6c7d2000b5/execute -d '{"execution_argument":""}' -H "Authorization: Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"
curl -XPOST http://localhost:5001/api/v1/workflows/425efd39-08e7-4390-9387-170c172775f7/execute -d '{"execution_argument":""}' -H "Authorization: Bearer db0373c6-1083-4dec-a05d-3ba73f02ccd4"