From d4d704f1700b996b31e7c1a971859ba870e7adf1 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 21 Mar 2021 19:38:16 +0100 Subject: [PATCH] MAJOR migration fixes for hybrid usage --- backend/go-app/codegen.go | 2004 ------------------------------------- backend/go-app/docker.go | 4 +- backend/go-app/files.go | 882 ---------------- backend/go-app/go.mod | 32 +- backend/go-app/go.sum | 159 +++ backend/go-app/main.go | 645 ++++-------- backend/go-app/oauth2.go | 36 +- backend/go-app/walkoff.go | 1735 ++++++++++++++++---------------- 8 files changed, 1270 insertions(+), 4227 deletions(-) delete mode 100644 backend/go-app/codegen.go delete mode 100644 backend/go-app/files.go diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go deleted file mode 100644 index 5943d4bd..00000000 --- a/backend/go-app/codegen.go +++ /dev/null @@ -1,2004 +0,0 @@ -package main - -import ( - "archive/zip" - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "os" - "strconv" - "strings" - - "cloud.google.com/go/storage" - "github.com/getkin/kin-openapi/openapi3" - //"github.com/satori/go.uuid" - "gopkg.in/yaml.v2" -) - -func copyFile(fromfile, tofile string) error { - from, err := os.Open(fromfile) - if err != nil { - return err - } - defer from.Close() - - to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666) - if err != nil { - return err - } - defer to.Close() - - _, err = io.Copy(to, from) - if err != nil { - return err - } - - return nil -} - -func formatAppfile(filedata string) (string, string) { - lines := strings.Split(filedata, "\n") - - newfile := []string{} - classname := "" - for _, line := range lines { - if strings.Contains(line, "walkoff_app_sdk") { - continue - } - - // Remap logging. CBA this right now - // This issue also persists in onprem apps because of await thingies.. :( - // FIXME - if strings.Contains(line, "console_logger") && strings.Contains(line, "await") { - continue - //line = strings.Replace(line, "console_logger", "logger", -1) - //log.Println(line) - } - - // Might not work with different import names - // Could be fucked up with spaces everywhere? Idk - if strings.Contains(line, "class") && strings.Contains(line, "(AppBase)") { - items := strings.Split(line, " ") - if len(items) > 0 && strings.Contains(items[1], "(AppBase)") { - classname = strings.Split(items[1], "(")[0] - } else { - // This could break something.. - classname = "TMP" - } - } - - if strings.Contains(line, "if __name__ ==") { - break - } - - // asyncio.run(HelloWorld.run(), debug=True) - - newfile = append(newfile, line) - } - - filedata = strings.Join(newfile, "\n") - return classname, filedata -} - -// Streams the data into a zip to be used for a cloud function -func streamZipdata(ctx context.Context, identifier, pythoncode, requirements string) (string, error) { - filename := fmt.Sprintf("generated_cloudfunctions/%s.zip", identifier) - - buf := new(bytes.Buffer) - zipWriter := zip.NewWriter(buf) - - zipFile, err := zipWriter.Create("main.py") - if err != nil { - log.Printf("Packing failed to create zip file from bucket: %v", err) - return filename, err - } - - // Have to use Fprintln otherwise it tries to parse all strings etc. - if _, err := fmt.Fprintln(zipFile, pythoncode); err != nil { - return filename, err - } - - zipFile, err = zipWriter.Create("requirements.txt") - if err != nil { - log.Printf("Packing failed to create zip file from bucket: %v", err) - return filename, err - } - if _, err := fmt.Fprintln(zipFile, requirements); err != nil { - return filename, err - } - - err = zipWriter.Close() - if err != nil { - log.Printf("Packing failed to close zip file writer from bucket: %v", err) - return filename, err - } - - return filename, nil -} - -func getAppbase() ([]byte, []byte, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - static := "../app_sdk/static_baseline.py" - appbase := "../app_sdk/app_base.py" - - staticData, err := ioutil.ReadFile(static) - if err != nil { - return []byte{}, []byte{}, err - } - - appbaseData, err := ioutil.ReadFile(appbase) - if err != nil { - return []byte{}, []byte{}, err - } - - return appbaseData, staticData, nil -} - -// Builds the structure for the new generated app in storage (copying baseline files) -func getAppbaseGCP(ctx context.Context, client *storage.Client) ([]byte, []byte, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - basePath := "generated_apps/baseline" - static, err := client.Bucket(bucketName).Object(fmt.Sprintf("%s/static_baseline.py", basePath)).NewReader(ctx) - if err != nil { - return []byte{}, []byte{}, err - } - appbase, err := client.Bucket(bucketName).Object(fmt.Sprintf("%s/app_base.py", basePath)).NewReader(ctx) - if err != nil { - return []byte{}, []byte{}, err - } - - defer static.Close() - defer appbase.Close() - - staticData, err := ioutil.ReadAll(static) - if err != nil { - return []byte{}, []byte{}, err - } - - appbaseData, err := ioutil.ReadAll(appbase) - if err != nil { - return []byte{}, []byte{}, err - } - - return appbaseData, staticData, nil -} - -func fixAppbase(appbase []byte) []string { - record := false - validLines := []string{} - for _, line := range strings.Split(string(appbase), "\n") { - if strings.Contains(line, "#STOPCOPY") { - //log.Println("Stopping copy") - break - } - - if record { - validLines = append(validLines, line) - } - - if strings.Contains(line, "#STARTCOPY") { - //log.Println("Starting copy") - record = true - } - } - - return validLines -} - -// Builds the structure for the new generated app in storage (copying baseline files) -func buildStructureGCP(ctx context.Context, client *storage.Client, swagger *openapi3.Swagger, curHash string) (string, error) { - // 1. Have baseline in bucket/generated_apps/baseline - // 2. Copy the baseline to a new folder with identifier name - - basePath := "generated_apps" - identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) - appPath := fmt.Sprintf("%s/%s", basePath, identifier) - fileNames := []string{"Dockerfile", "requirements.txt"} - for _, file := range fileNames { - src := client.Bucket(bucketName).Object(fmt.Sprintf("%s/baseline/%s", basePath, file)) - dst := client.Bucket(bucketName).Object(fmt.Sprintf("%s/%s", appPath, file)) - if _, err := dst.CopierFrom(src).Run(ctx); err != nil { - return "", err - } - } - - return appPath, nil -} - -// Builds the base structure for the app that we're making -// Returns error if anything goes wrong. This has to work if -// the python code is supposed to be generated -func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { - //log.Printf("%#v", swagger) - - // adding md5 based on input data to not overwrite earlier data. - generatedPath := "generated" - subpath := "../app_gen/openapi/" - identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) - appPath := fmt.Sprintf("%s/%s", generatedPath, identifier) - - os.MkdirAll(appPath, os.ModePerm) - os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm) - - err := copyFile(fmt.Sprintf("%sbaseline/Dockerfile", subpath), fmt.Sprintf("%s/%s", appPath, "Dockerfile")) - if err != nil { - log.Println("Failed to move Dockerfile") - return appPath, err - } - - err = copyFile(fmt.Sprintf("%sbaseline/requirements.txt", subpath), fmt.Sprintf("%s/%s", appPath, "requirements.txt")) - if err != nil { - log.Println("Failed to move requrements.txt") - return appPath, err - } - - return appPath, nil -} - -// This function generates the python code that's being used. -// This is really meta when you program it. Handling parameters is hard here. -func makePythoncode(swagger *openapi3.Swagger, name, url, method string, parameters, optionalQueries, headers []string, fileField string) (string, string) { - method = strings.ToLower(method) - queryString := "" - queryData := "" - - // FIXME - this might break - need to check if ? or & should be set as query - parameterData := "" - if len(optionalQueries) > 0 { - queryString += ", " - for index, query := range optionalQueries { - // Check if it's a part of the URL already - queryString += fmt.Sprintf("%s=\"\"", query) - if index != len(optionalQueries)-1 { - queryString += ", " - } - - /* - queryData += fmt.Sprintf(` - if %s: - url += f"&%s={%s}"`, query, query, query) - */ - queryData += fmt.Sprintf(` - if %s: - params["%s"] = %s`, query, query, query) - } - } else { - //log.Printf("No optional queries?") - } - - // api.Authentication.Parameters[0].Value = "BearerAuth" - authenticationParameter := "" - authenticationSetup := "" - authenticationAddin := "" - // Python configuration code that should work :) - if swagger.Components.SecuritySchemes != nil { - if swagger.Components.SecuritySchemes["BearerAuth"] != nil { - authenticationParameter = ", apikey" - authenticationSetup = "if apikey != \" \": headers[\"Authorization\"] = f\"Bearer {apikey}\"" - } else if swagger.Components.SecuritySchemes["BasicAuth"] != nil { - authenticationParameter = ", username_basic, password_basic" - authenticationAddin = ", auth=(username_basic, password_basic)" - } else if swagger.Components.SecuritySchemes["ApiKeyAuth"] != nil { - authenticationParameter = ", apikey" - if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "header" { - // This is a way to bypass apikeys by passing " " - authenticationSetup = fmt.Sprintf(`if apikey != " ": headers["%s"] = apikey`, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) - } else if swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.In == "query" { - // This might suck lol - key := "?" - if strings.Contains(url, "?") { - key = "&" - } - - authenticationSetup = fmt.Sprintf("if apikey != \" \": url+=f\"%s%s={apikey}\"", key, swagger.Components.SecuritySchemes["ApiKeyAuth"].Value.Name) - } - } - } - - //baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - // This is a quickfix for onpremises stuff. Does work, but should really be - // part of the authentication scheme from openapi3 - urlParameter := "" - urlInline := "" - //log.Printf("URL: %s", url) - if !strings.HasPrefix(strings.ToLower(url), "http") { - urlParameter = ", url" - urlInline = "{url}" - } - - // Specific check for SSL verification - // This is critical for onprem stuff. - //verifyParam := "" - //verifyWrapper := "" - //verifyAddin := "" - verifyParam := ", ssl_verify=False" - verifyWrapper := `if type(ssl_verify) == str: ssl_verify = False if ssl_verify.lower() == "false" or ssl_verify == "0" else True` - verifyAddin := ", verify=ssl_verify" - - if len(parameters) > 0 { - parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", ")) - } - - // FIXME - add checks for query data etc - - functionname := strings.ToLower(fmt.Sprintf("%s_%s", method, name)) - if strings.Contains(strings.ToLower(name), strings.ToLower(method)) { - functionname = strings.ToLower(name) - } - - bodyParameter := "" - bodyAddin := "" - bodyFormatter := "" - postParameters := []string{"post", "patch", "put"} - for _, item := range postParameters { - if method == item { - bodyParameter = ", body=\"\"" - bodyAddin = ", data=body" - - // FIXME: Does JSON data work? - bodyFormatter = `body = " ".join(body.strip().split()).encode("utf-8")` - } - } - - preparedHeaders := "headers={}" - if len(headers) > 0 { - preparedHeaders = "headers={" - for count, header := range headers { - headerSplit := strings.Split(header, "=") - added := false - if len(headerSplit) == 2 { - if strings.Contains(preparedHeaders, headerSplit[0]) { - continue - } - - preparedHeaders += fmt.Sprintf(`"%s": "%s"`, headerSplit[0], headerSplit[1]) - added = true - } - - if count != len(headers)-1 && added { - preparedHeaders += "," - } - } - - preparedHeaders += "}" - } - - fileBalance := "" - fileAdder := `` - fileGrabber := `` - fileParameter := `` - if method == "post" && len(fileField) > 0 { - fileParameter = ", file_id" - fileGrabber = `filedata = self.get_file(file_id)` - - // This indentation is confusing (but correct) ROFL - fileAdder = fmt.Sprintf(`if not filedata["success"]: - return file_id+" is not a valid File ID" - files = {"%s": (filedata["filename"], filedata["data"])}`, fileField) - fileBalance = ", files=files" - } - - // Extra param for url if it's changeable - // Extra param for authentication scheme(s) - // The last weird one is the body.. Tabs & spaces sucks. - data := fmt.Sprintf(` async def %s(self%s%s%s%s%s%s%s): - params={} - %s - url=f"%s%s" - %s - %s - %s - %s - %s - %s - ret = requests.%s(url, headers=headers, params=params%s%s%s%s) - try: - return ret.json() - except json.decoder.JSONDecodeError: - return ret.text - `, - functionname, - authenticationParameter, - urlParameter, - fileParameter, - parameterData, - queryString, - bodyParameter, - verifyParam, - preparedHeaders, - urlInline, - url, - verifyWrapper, - authenticationSetup, - queryData, - bodyFormatter, - fileGrabber, - fileAdder, - method, - authenticationAddin, - bodyAddin, - verifyAddin, - fileBalance, - ) - - // Use lowercase when checking - - if strings.Contains(functionname, "attachment") { - //log.Printf("FUNCTION: %s", data) - //log.Println(data) - //log.Printf("Queries: %s", queryString) - } - - return functionname, data -} - -func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger, WorkflowApp, []string, error) { - api := WorkflowApp{} - //log.Printf("%#v", swagger.Info) - - if len(swagger.Info.Title) == 0 { - return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.") - } - - if len(swagger.Servers) == 0 { - //return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'") - //return swagger, WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'") - swagger.Servers = openapi3.Servers{ - &openapi3.Server{ - URL: "https://hostname.com", - }, - } - } - - api.Name = swagger.Info.Title - api.Description = swagger.Info.Description - - // FIXME: Versioning issue? - api.ID = newmd5 - //uuid.NewV4().String() - - api.IsValid = true - api.Link = swagger.Servers[0].URL // host does not exist lol - if strings.HasSuffix(api.Link, "/") { - api.Link = api.Link[:len(api.Link)-1] - } - - api.AppVersion = "1.0.0" - api.Environment = "Shuffle" - api.SmallImage = "" - api.LargeImage = "" - api.Sharing = false - api.Verified = false - api.Tested = false - api.Invalid = false - api.PrivateID = newmd5 - api.Generated = true - api.Activated = true - // Setting up security schemes - extraParameters := []WorkflowAppActionParameter{} - - if val, ok := swagger.Info.ExtensionProps.Extensions["x-logo"]; ok { - j, err := json.Marshal(&val) - if err == nil { - if j[0] == 0x22 && j[len(j)-1] == 0x22 { - j = j[1 : len(j)-1] - } - - //log.Printf("%s", j) - api.SmallImage = string(j) - api.LargeImage = string(j) - } - } - - // Jesus what a clusterfuck. - // Handles parsing of categories from OpenApi3 custom field - if val, ok := swagger.Info.ExtensionProps.Extensions["x-categories"]; ok { - //log.Printf("Categories: %#v", val) - j, err := json.Marshal(&val) - if err == nil { - if j[0] == 0x22 && j[len(j)-1] == 0x22 { - j = j[1 : len(j)-1] - } - - parsedCategories := fmt.Sprintf(`{"categories": %s}`, string(j)) - type parsed struct { - Categories []string `json:"categories"` - } - - var parse parsed - err := json.Unmarshal([]byte(parsedCategories), &parse) - if err != nil { - log.Printf("Failed unmarshaling categories: %v", err) - } else { - api.Categories = parse.Categories - } - } - } - - if len(swagger.Tags) > 0 { - newTags := []string{} - for _, tag := range swagger.Tags { - newTags = append(newTags, tag.Name) - } - - api.Tags = newTags - } - - securitySchemes := swagger.Components.SecuritySchemes - if securitySchemes != nil { - //log.Printf("%#v", securitySchemes) - - api.Authentication = Authentication{ - Required: true, - Parameters: []AuthenticationParams{}, - } - - // Used for python code generation lol - // Not sure how this should work with oauth - if securitySchemes["BearerAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "apikey", - Value: "", - Example: "******", - Description: securitySchemes["BearerAuth"].Value.Description, - In: securitySchemes["BearerAuth"].Value.In, - Scheme: securitySchemes["BearerAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BearerAuth"].Value.Scheme, - }, - }) - - //log.Printf("HANDLE BEARER AUTH") - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "apikey", - Description: "The apikey to use", - Multiline: false, - Required: true, - Example: "The API key to use. Space = skip", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else if securitySchemes["ApiKeyAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "apikey", - Value: "", - Example: "******", - Description: securitySchemes["ApiKeyAuth"].Value.Description, - In: securitySchemes["ApiKeyAuth"].Value.In, - Scheme: securitySchemes["ApiKeyAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["ApiKeyAuth"].Value.Scheme, - }, - }) - - //log.Printf("HANDLE APIKEY AUTH") - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "apikey", - Description: "The apikey to use", - Multiline: false, - Required: true, - Example: "**********", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } else if securitySchemes["BasicAuth"] != nil { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "username_basic", - Value: "", - Example: "username", - Description: securitySchemes["BasicAuth"].Value.Description, - In: securitySchemes["BasicAuth"].Value.In, - Scheme: securitySchemes["BasicAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BasicAuth"].Value.Scheme, - }, - }) - - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "password_basic", - Value: "", - Example: "*****", - Description: securitySchemes["BasicAuth"].Value.Description, - In: securitySchemes["BasicAuth"].Value.In, - Scheme: securitySchemes["BasicAuth"].Value.Scheme, - Schema: SchemaDefinition{ - Type: securitySchemes["BasicAuth"].Value.Scheme, - }, - }) - - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "username_basic", - Description: "The username to use", - Multiline: false, - Required: true, - Example: "The username to use", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "password_basic", - Description: "The password to use", - Multiline: false, - Required: true, - Example: "***********", - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - } - - // Adds a link parameter if it's not already defined - if len(api.Link) == 0 { - api.Authentication.Parameters = append(api.Authentication.Parameters, AuthenticationParams{ - Name: "url", - Description: "The URL of the app", - Multiline: false, - Required: true, - Example: "https://shuffler.io", - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - extraParameters = append(extraParameters, WorkflowAppActionParameter{ - Name: "url", - Description: "The URL of the app", - Multiline: false, - Required: true, - Configuration: true, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - } - - // This is the python code to be generated - // Could just as well be go at this point lol - pythonFunctions := []string{} - //Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - for actualPath, path := range swagger.Paths { - actualPath = strings.Replace(actualPath, " ", "_", -1) - //actualPath = strings.Replace(actualPath, ".", "", -1) - actualPath = strings.Replace(actualPath, "\\", "", -1) - if !api.Invalid && strings.HasPrefix(actualPath, "tmp") { - log.Printf("[WARNING] Set api %s to invalid because of path %s", swagger.Info.Title, actualPath) - api.Invalid = true - } - - // FIXME: Handle everything behind questionmark (?) with dots as well. - // https://godoc.org/github.com/getkin/kin-openapi/openapi3#PathItem - if path.Get != nil { - action, curCode := handleGet(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Connect != nil { - action, curCode := handleConnect(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Head != nil { - action, curCode := handleHead(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Delete != nil { - action, curCode := handleDelete(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Post != nil { - action, curCode := handlePost(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Patch != nil { - action, curCode := handlePatch(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - if path.Put != nil { - action, curCode := handlePut(swagger, api, extraParameters, path, actualPath) - api.Actions = append(api.Actions, action) - pythonFunctions = append(pythonFunctions, curCode) - } - - // Has to be here because its used differently above. - // FIXING this is done during export instead? - //log.Printf("OLDPATH: %s", actualPath) - //if strings.Contains(actualPath, "?") { - // actualPath = strings.Split(actualPath, "?")[0] - //} - - //log.Printf("NEWPATH: %s", actualPath) - //newPaths[actualPath] = path - } - - return swagger, api, pythonFunctions, nil -} - -// FIXME - have this give a real version? -func verifyApi(api WorkflowApp) WorkflowApp { - if api.AppVersion == "" { - api.AppVersion = "1.0.0" - } - - return api -} - -func getBasePython() string { - baseString := `import requests -import asyncio -import json -import urllib3 - -from walkoff_app_sdk.app_base import AppBase - -class %s(AppBase): - """ - Autogenerated class by Shuffler - """ - - __version__ = "%s" - app_name = "%s" - - def __init__(self, redis, logger, console_logger=None): - self.verify = False - urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - super().__init__(redis, logger, console_logger) - -%s - -if __name__ == "__main__": - asyncio.run(%s.run(), debug=True) -` - return baseString -} - -func dumpPythonGCP(ctx context.Context, client *storage.Client, basePath, name, version string, pythonFunctions []string) (string, error) { - parsedCode := fmt.Sprintf(getBasePython(), name, version, name, strings.Join(pythonFunctions, "\n"), name) - - // Create bucket handle - bucket := client.Bucket(bucketName) - obj := bucket.Object(fmt.Sprintf("%s/src/app.py", basePath)) - w := obj.NewWriter(ctx) - if _, err := fmt.Fprintf(w, parsedCode); err != nil { - return "", err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return "", err - } - - return parsedCode, nil -} - -func dumpPython(basePath, name, version string, pythonFunctions []string) (string, error) { - //log.Printf("%#v", api) - //log.Printf(strings.Join(pythonFunctions, "\n")) - - parsedCode := fmt.Sprintf(getBasePython(), name, version, name, strings.Join(pythonFunctions, "\n"), name) - - err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm) - if err != nil { - return "", err - } - //fmt.Println(parsedCode) - //log.Println(string(data)) - return parsedCode, nil -} - -func dumpApiGCP(ctx context.Context, client *storage.Client, swagger *openapi3.Swagger, basePath string, api WorkflowApp) error { - //log.Printf("%#v", api) - data, err := yaml.Marshal(api) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - - // Create bucket handle - bucket := client.Bucket(bucketName) - obj := bucket.Object(fmt.Sprintf("%s/app.yaml", basePath)) - w := obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, string(data)); err != nil { - return err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return err - } - - openapidata, err := yaml.Marshal(swagger) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - obj = bucket.Object(fmt.Sprintf("%s/openapi.yaml", basePath)) - //log.Println(string(openapidata)) - w = obj.NewWriter(ctx) - if _, err := fmt.Fprintln(w, string(openapidata)); err != nil { - return err - } - // Close, just like writing a file. - if err := w.Close(); err != nil { - return err - } - - //log.Println(string(data)) - return nil -} - -func dumpApi(basePath string, api WorkflowApp) error { - //log.Printf("%#v", api) - data, err := yaml.Marshal(api) - if err != nil { - log.Printf("Error with yaml marshal: %s", err) - return err - } - - err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm) - if err != nil { - return err - } - - //log.Println(string(data)) - return nil -} - -func getRunner(classname string) string { - return fmt.Sprintf(` -# Run the actual thing after we've checked params -def run(request): - print("Started execution!") - action = request.get_json() - print(action) - print(type(action)) - authorization_key = action.get("authorization") - current_execution_id = action.get("execution_id") - - if action and "name" in action and "app_name" in action: - asyncio.run(%s.run(action), debug=True) - return f'Attempting to execute function {action["name"]} in app {action["app_name"]}' - else: - return f'Invalid action' - - `, classname) -} - -func deployAppToDatastore(ctx context.Context, workflowapp WorkflowApp) error { - err := setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) - if err != nil { - log.Printf("[ERROR] Failed setting workflowapp: %s", err) - return err - } else { - log.Printf("[INFO] Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - } - - return nil -} - -// FIXME: -// https://docs.python.org/3.2/reference/lexical_analysis.html#identifiers -// This is used to build the python functions. -func fixFunctionName(functionName, actualPath string) string { - if len(functionName) == 0 { - functionName = actualPath - } - - // REGEX THIS SHIT - // ROFL - - //log.Printf("Fixing function name for %s", functionName) - functionName = strings.Replace(functionName, ".", "", -1) - functionName = strings.Replace(functionName, ",", "", -1) - functionName = strings.Replace(functionName, ".", "", -1) - functionName = strings.Replace(functionName, "&", "", -1) - functionName = strings.Replace(functionName, "/", "", -1) - functionName = strings.Replace(functionName, "\\", "", -1) - - functionName = strings.Replace(functionName, "!", "", -1) - functionName = strings.Replace(functionName, "?", "", -1) - functionName = strings.Replace(functionName, "@", "", -1) - functionName = strings.Replace(functionName, "#", "", -1) - functionName = strings.Replace(functionName, "$", "", -1) - functionName = strings.Replace(functionName, "&", "", -1) - functionName = strings.Replace(functionName, "*", "", -1) - functionName = strings.Replace(functionName, "(", "", -1) - functionName = strings.Replace(functionName, ")", "", -1) - functionName = strings.Replace(functionName, "[", "", -1) - functionName = strings.Replace(functionName, "]", "", -1) - functionName = strings.Replace(functionName, "{", "", -1) - functionName = strings.Replace(functionName, "}", "", -1) - functionName = strings.Replace(functionName, `"`, "", -1) - functionName = strings.Replace(functionName, `'`, "", -1) - functionName = strings.Replace(functionName, `|`, "", -1) - functionName = strings.Replace(functionName, `~`, "", -1) - - functionName = strings.Replace(functionName, " ", "_", -1) - functionName = strings.Replace(functionName, "-", "_", -1) - - functionName = strings.ToLower(functionName) - - return functionName -} - -// Returns a valid param name -func validateParameterName(name string) string { - invalid := []string{"False", - "await", - "else", - "import", - "pass", - "None", - "break", - "except", - "in", - "raise", - "True", - "class", - "finally", - "is", - "return", - "and", - "continue", - "for", - "lambda", - "try", - "as", - "def", - "from", - "nonlocal", - "while", - "assert", - "del", - "global", - "not", - "with", - "async", - "elif", - "if", - "or", - "yield", - } - - newname := name - for _, item := range invalid { - if item == name { - //log.Printf("%s is NOT a valid parameter name!", item) - newname = fmt.Sprintf("%s_shuffle", item) - break - } - } - - newname = strings.ReplaceAll(newname, " ", "_") - newname = strings.ReplaceAll(newname, ",", "_") - newname = strings.ReplaceAll(newname, ".", "_") - newname = strings.ReplaceAll(newname, "|", "_") - newname = strings.ReplaceAll(newname, "-", "_") - - return newname -} - -func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Connect.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Connect.Description, - Name: fmt.Sprintf("%s %s", "Connect", path.Connect.Summary), - Label: fmt.Sprintf(path.Connect.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Connect.Parameters) > 0 { - for counter, param := range path.Connect.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = strings.ReplaceAll(parsedName, "-", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Connect.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "connect", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Get.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Get.Description, - Name: fmt.Sprintf("%s %s", "Get", path.Get.Summary), - Label: fmt.Sprintf(path.Get.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - - // FIXME - remove this when authentication is properly introduced - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Get.Parameters) > 0 { - for counter, param := range path.Get.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Get.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - // Skipping simial - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "get", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Head.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Head.Description, - Name: fmt.Sprintf("%s %s", "Head", path.Head.Summary), - Label: fmt.Sprintf(path.Head.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Head.Parameters) > 0 { - for counter, param := range path.Head.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Head.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "head", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Delete.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Delete.Description, - Name: fmt.Sprintf("%s %s", "Delete", path.Delete.Summary), - Label: fmt.Sprintf(path.Delete.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Delete.Parameters) > 0 { - for counter, param := range path.Delete.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Delete.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "delete", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - //log.Printf("PATH: %s", actualPath) - functionName := fixFunctionName(path.Post.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Post.Description, - Name: fmt.Sprintf("%s %s", "Post", path.Post.Summary), - Label: fmt.Sprintf(path.Post.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - fileField := "" - if path.Post.RequestBody != nil { - //log.Printf("DATA: %#v", - value := path.Post.RequestBody.Value - //log.Printf("VAL: %#v", value.Content) - if val, ok := value.Content["multipart/form-data"]; ok { - if val.Schema.Value != nil { - if innerval, ok := val.Schema.Value.Properties["fieldname"]; ok { - if extensionvalue, ok := innerval.Value.ExtensionProps.Extensions["value"]; ok { - fieldname := extensionvalue.(json.RawMessage) - newName := string(fmt.Sprintf("%s", string(fieldname))) - if newName[0] == 0x22 && newName[len(newName)-1] == 0x22 { - parsedName := newName[1 : len(newName)-1] - //log.Printf("[INFO] Parse name: %s", parsedName) - fileField = parsedName - - curParam := WorkflowAppActionParameter{ - Name: "file_id", - Description: "Files to be uploaded", - Multiline: false, - Required: true, - Schema: SchemaDefinition{ - Type: "string", - }, - } - - action.Parameters = append(action.Parameters, curParam) - } - } - } - } - } - } - - headersFound := []string{} - if len(path.Post.Parameters) > 0 { - for counter, param := range path.Post.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Post.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if parsedName == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "post", parameters, optionalQueries, headersFound, fileField) - - if len(functionname) > 0 { - action.Name = functionname - } - - //log.Printf("PARAMS: %d", len(action.Parameters)) - //for _, param := range action.Parameters { - // log.Printf("%#v", param) - //} - - return action, curCode -} - -func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Patch.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Patch.Description, - Name: fmt.Sprintf("%s %s", "Patch", path.Patch.Summary), - Label: fmt.Sprintf(path.Patch.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Patch.Parameters) > 0 { - for counter, param := range path.Patch.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Patch.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, curParam.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "patch", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} - -func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string) (WorkflowAppAction, string) { - // What to do with this, hmm - functionName := fixFunctionName(path.Put.Summary, actualPath) - - action := WorkflowAppAction{ - Description: path.Put.Description, - Name: fmt.Sprintf("%s %s", "Put", path.Put.Summary), - Label: fmt.Sprintf(path.Put.Summary), - NodeType: "action", - Environment: api.Environment, - Parameters: extraParameters, - } - - action.Returns.Schema.Type = "string" - baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) - - //log.Println(path.Parameters) - - // Parameters: []WorkflowAppActionParameter{}, - // FIXME - add data for POST stuff - firstQuery := true - optionalQueries := []string{} - parameters := []string{} - optionalParameters := []WorkflowAppActionParameter{} - - headersFound := []string{} - if len(path.Put.Parameters) > 0 { - for counter, param := range path.Put.Parameters { - if param.Value.Schema == nil { - continue - } else if param.Value.In == "header" { - headersFound = append(headersFound, fmt.Sprintf("%s=%s", param.Value.Name, param.Value.Example)) - continue - } - - parsedName := param.Value.Name - parsedName = strings.ReplaceAll(parsedName, " ", "_") - parsedName = strings.ReplaceAll(parsedName, ",", "_") - parsedName = strings.ReplaceAll(parsedName, ".", "_") - parsedName = strings.ReplaceAll(parsedName, "|", "_") - parsedName = validateParameterName(parsedName) - param.Value.Name = parsedName - path.Put.Parameters[counter].Value.Name = parsedName - - curParam := WorkflowAppActionParameter{ - Name: parsedName, - Description: param.Value.Description, - Multiline: false, - Required: param.Value.Required, - Schema: SchemaDefinition{ - Type: param.Value.Schema.Value.Type, - }, - } - - // FIXME: Example & Multiline - if param.Value.Example != nil { - curParam.Example = param.Value.Example.(string) - - if param.Value.Name == "body" { - curParam.Value = param.Value.Example.(string) - } - } - if val, ok := param.Value.ExtensionProps.Extensions["multiline"]; ok { - j, err := json.Marshal(&val) - if err == nil { - b, err := strconv.ParseBool(string(j)) - if err == nil { - curParam.Multiline = b - } - } - } - - if param.Value.Required { - action.Parameters = append(action.Parameters, curParam) - } else { - optionalParameters = append(optionalParameters, curParam) - } - - if param.Value.In == "path" { - parameters = append(parameters, param.Value.Name) - //baseUrl = fmt.Sprintf("%s%s", baseUrl) - } else if param.Value.In == "query" { - //log.Printf("QUERY!: %s", param.Value.Name) - if !param.Value.Required { - optionalQueries = append(optionalQueries, param.Value.Name) - continue - } - - parameters = append(parameters, param.Value.Name) - - if strings.Contains(baseUrl, fmt.Sprintf("%s={%s}", param.Value.Name, param.Value.Name)) { - continue - } - - if strings.Contains(baseUrl, fmt.Sprintf("{%s}", param.Value.Name)) { - continue - } - - if firstQuery && !strings.Contains(baseUrl, "?") { - baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } else { - baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name) - } - firstQuery = false - } - - } - } - - optionalParameters = append(optionalParameters, WorkflowAppActionParameter{ - Name: "ssl_verify", - Description: "Check if you want to verify request", - Multiline: false, - Required: false, - Example: "True", - Options: []string{ - "True", - "False", - }, - Schema: SchemaDefinition{ - Type: "string", - }, - }) - - // ensuring that they end up last in the specification - // (order is ish important for optional params) - they need to be last. - for _, optionalParam := range optionalParameters { - action.Parameters = append(action.Parameters, optionalParam) - } - - functionname, curCode := makePythoncode(swagger, functionName, baseUrl, "put", parameters, optionalQueries, headersFound, "") - - if len(functionname) > 0 { - action.Name = functionname - } - - return action, curCode -} diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 27c8e436..e11934f8 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -2,6 +2,8 @@ package main // Docker import ( + "github.com/frikky/shuffle-shared" + "archive/tar" "path/filepath" @@ -797,7 +799,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) diff --git a/backend/go-app/files.go b/backend/go-app/files.go deleted file mode 100644 index 419d834d..00000000 --- a/backend/go-app/files.go +++ /dev/null @@ -1,882 +0,0 @@ -package main - -/* - Handles files within Workflows.of Shuffle -*/ - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "log" - "net/http" - "os" - "strconv" - "strings" - "time" - - "cloud.google.com/go/datastore" - "github.com/satori/go.uuid" -) - -type File struct { - Id string `json:"id" datastore:"id"` - Type string `json:"type" datastore:"type"` - CreatedAt int64 `json:"created_at" datastore:"created_at"` - UpdatedAt int64 `json:"updated_at" datastore:"updated_at"` - MetaAccessAt int64 `json:"meta_access_at" datastore:"meta_access_at"` - DownloadAt int64 `json:"last_downloaded" datastore:"last_downloaded"` - Description string `json:"description" datastore:"description"` - ExpiresAt string `json:"expires_at" datastore:"expires_at"` - Status string `json:"status" datastore:"status"` - Filename string `json:"filename" datastore:"filename"` - URL string `json:"url" datastore:"org"` - OrgId string `json:"org_id" datastore:"org_id"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Workflows []string `json:"workflows" datastore:"workflows"` - DownloadPath string `json:"download_path" datastore:"download_path"` - Md5sum string `json:"md5_sum" datastore:"md5_sum"` - Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` - FileSize int64 `json:"filesize" datastore:"filesize"` - Duplicate bool `json:"duplicate" datastore:"duplicate"` - Subflows []string `json:"subflows" datastore:"subflows"` -} - -var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") - -func fileAuthentication(request *http.Request) (string, error) { - executionId, ok := request.URL.Query()["execution_id"] - if ok && len(executionId) > 0 { - ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId[0]) - if err != nil { - log.Printf("[ERROR] Couldn't find execution ID %s", executionId[0]) - return "", err - } - - apikey := request.Header.Get("Authorization") - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("[ERROR} Apikey doesn't start with bearer (2)") - return "", errors.New("No auth key found") - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("[ERROR] Invalid format for apikey (2)") - return "", errors.New("No space in authkey") - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if newApikey != workflowExecution.Authorization { - //log.Printf("[ERROR] Bad apikey for execution %s. %s vs %s", executionId[0], apikey, workflowExecution.Authorization) - log.Printf("[ERROR] Bad apikey for execution %s.", executionId[0]) - //%s vs %s", executionId[0], apikey, workflowExecution.Authorization) - return "", errors.New("Bad authorization key") - } - - log.Printf("[INFO] Authorization is correct for execution %s!", executionId[0]) - //%s vs %s. Setting Org", executionId, apikey, workflowExecution.Authorization) - if len(workflowExecution.ExecutionOrg) > 0 { - return workflowExecution.ExecutionOrg, nil - } else if len(workflowExecution.Workflow.ExecutingOrg.Id) > 0 { - return workflowExecution.ExecutionOrg, nil - } else { - log.Printf("[ERROR] Couldn't find org for workflow execution, but auth was correct.") - } - } - - return "", errors.New("No execution id specified") -} - -// https://golangcode.com/check-if-a-file-exists/ -func fileExists(filename string) bool { - info, err := os.Stat(filename) - if os.IsNotExist(err) { - return false - } - return !info.IsDir() -} - -func handleGetFiles(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if user.Role != "admin" { - log.Printf("[AUTH] User isn't admin") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`))) - return - } - - ctx := context.Background() - files, err := getAllFiles(ctx, user.ActiveOrg.Id) - if err != nil { - log.Printf("[ERROR] Failed to get files: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`))) - return - } - - log.Printf("[INFO] Got %d files for org %s", len(files), user.ActiveOrg.Id) - newBody, err := json.Marshal(files) - if err != nil { - log.Printf("[ERROR] Failed marshaling files: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`)) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(newBody)) -} - -func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("[INFO] Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to GET File Meta for %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should GET FILE META for %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[INFO] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - newBody, err := json.Marshal(file) - if err != nil { - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed to marshal filedata"}`)) - return - } - - log.Printf("[INFO] Successfully got file meta for %s", fileId) - resp.WriteHeader(200) - resp.Write([]byte(newBody)) -} - -func handleDeleteFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("[INFO] Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if strings.Contains(fileId, "?") { - fileId = strings.Split(fileId, "?")[0] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to delete file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file deletion: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should DELETE file %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[INFO] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("[INFO] User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if file.Status == "deleted" { - log.Printf("[INFO] File with ID %s is already deleted.", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if fileExists(file.DownloadPath) { - err = os.Remove(file.DownloadPath) - if err != nil { - log.Printf("[ERROR] Failed deleting file locally: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed deleting filein path %s"}`, file.DownloadPath))) - return - } - - log.Printf("[INFO] Deleted file %s locally. Next is database.", file.DownloadPath) - } else { - log.Printf("[ERROR] File doesn't exist. Can't delete. Should maybe delete file anyway?") - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "File in location %s doesn't exist"}`, file.DownloadPath))) - return - } - - file.Status = "deleted" - err = setFile(ctx, *file) - if err != nil { - log.Printf("[ERROR] Failed setting file to deleted") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to deleted"}`)) - return - } - - /* - //Actually delete it? - err = DeleteKey(ctx, "files", fileId) - if err != nil { - log.Printf("Failed deleting file with ID %s: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - */ - - log.Printf("[INFO] Successfully deleted file %s", fileId) - resp.WriteHeader(200) - resp.Write([]byte(`{"success": true}`)) -} - -func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - log.Printf("\n\n[INFO] User is trying to download file %s\n\n", fileId) - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("INITIAL Api authentication failed in file download: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("Bad file authentication in get: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - /* - } else { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - */ - } - - // 1. Verify if the user has access to the file: org_id and workflow - log.Printf("[INFO] Should get file %s", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("[ERROR] File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - if file.Status != "active" { - log.Printf("[ERROR] File status isn't active, but %s. Can't continue.", file.Status) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "The file isn't ready to be downloaded yet. Status required: active"}`)) - return - } - - // Fixme: More auth: org and workflow! - downloadPath := file.DownloadPath - log.Printf("[INFO] Downloadpath: %s", downloadPath) - Openfile, err := os.Open(downloadPath) - defer Openfile.Close() //Close after function return - if err != nil { - file.Status = "deleted" - err = setFile(ctx, *file) - if err != nil { - log.Printf("Failed setting file to uploading") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) - return - } - - //File not found, send 404 - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`)) - return - } - - //File is found, create and send the correct headers - //Get the Content-Type of the file - //Create a buffer to store the header of the file in - FileHeader := make([]byte, 512) - //Copy the headers into the FileHeader buffer - Openfile.Read(FileHeader) - //Get content type of file - FileContentType := http.DetectContentType(FileHeader) - - //Get the file size - FileStat, _ := Openfile.Stat() //Get info from file - FileSize := strconv.FormatInt(FileStat.Size(), 10) //Get file size as a string - - //Send the headers - resp.Header().Set("Content-Disposition", "attachment; filename="+fileId) - resp.Header().Set("Content-Type", FileContentType) - resp.Header().Set("Content-Length", FileSize) - - //Send the file - //We read 512 bytes from the file already, so we reset the offset back to 0 - Openfile.Seek(0, 0) - io.Copy(resp, Openfile) //'Copy' the file to the client - return - - //log.Printf("Should download file %s", downloadPath) -} -func handleUploadFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - var fileId string - location := strings.Split(request.URL.String(), "/") - if location[1] == "api" { - if len(location) <= 4 { - log.Printf("Path too short: %d", len(location)) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - fileId = location[4] - } - - if len(fileId) != 36 { - log.Printf("Bad format for fileId %s", fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Badly formatted fileId"}`)) - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("INITIAL Api authentication failed in file upload: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("Bad file authentication in create file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - log.Printf("[INFO] Should UPLOAD file %s if user has access", fileId) - ctx := context.Background() - file, err := getFile(ctx, fileId) - if err != nil { - log.Printf("File %s not found: %s", fileId, err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - found := false - if file.OrgId == user.ActiveOrg.Id { - found = true - } else { - for _, item := range user.Orgs { - if item == file.OrgId { - found = true - break - } - } - } - - if !found { - log.Printf("User %s doesn't have access to %s", user.Username, fileId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - log.Printf("[INFO] STATUS: %s", file.Status) - if file.Status != "created" { - log.Printf("File status isn't created. Can't upload.") - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "This file already has data."}`)) - return - } - - request.ParseMultipartForm(32 << 20) - parsedFile, _, err := request.FormFile("shuffle_file") - if err != nil { - log.Printf("[ERROR] Couldn't upload file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed uploading file"}`)) - return - } - defer parsedFile.Close() - - file.Status = "uploading" - err = setFile(ctx, *file) - if err != nil { - log.Printf("Failed setting file to uploading") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) - return - } - - // Can be used for validation files for change - var buf bytes.Buffer - io.Copy(&buf, parsedFile) - contents := buf.Bytes() - file.FileSize = int64(len(contents)) - md5 := md5sum(contents) - buf.Reset() - - sha256Sum := sha256.Sum256(contents) - //parsedFile.Reset() - - f, err := os.OpenFile(file.DownloadPath, os.O_WRONLY|os.O_CREATE, os.ModePerm) - if err != nil { - // Rolling back file - file.Status = "created" - setFile(ctx, *file) - - log.Printf("[ERROR] Failed uploading and creating file: %s", err) - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false}`)) - return - } - - defer f.Close() - parsedFile.Seek(0, io.SeekStart) - io.Copy(f, parsedFile) - - // FIXME: Set this one to 200 anyway? Can't download file then tho.. - file.Status = "active" - file.Md5sum = md5 - file.Sha256sum = fmt.Sprintf("%x", sha256Sum) - log.Printf("[INFO] MD5 for file %s (%s) is %s and SHA256 is %s", file.Filename, file.Id, file.Md5sum, file.Sha256sum) - - err = setFile(ctx, *file) - if err != nil { - log.Printf("[ERROR] Failed setting file back to active") - resp.WriteHeader(500) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file to active"}`)) - return - } - - log.Printf("[INFO] Successfully uploaded file ID %s", file.Id) - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) -} - -func handleCreateFile(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) - if cors { - return - } - - // 1. Check user directly - // 2. Check workflow execution authorization - user, err := handleApiAuthentication(resp, request) - if err != nil { - log.Printf("[INFO] INITIAL Api authentication failed in file creation: %s", err) - - orgId, err := fileAuthentication(request) - if err != nil { - log.Printf("[ERROR] Bad file authentication in create file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - - user.ActiveOrg.Id = orgId - user.Username = "Execution File API" - } - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Println("Failed reading body") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to read data"}`))) - return - } - - type FileStructure struct { - Filename string `json:"filename"` - OrgId string `json:"org_id"` - WorkflowId string `json:"workflow_id"` - } - - var curfile FileStructure - err = json.Unmarshal(body, &curfile) - if err != nil { - log.Printf("[ERROR] Failed unmarshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed to unmarshal data"}`))) - return - } - - // Loads of validation below - if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 { - log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id") - log.Printf("INPUT: %s", string(body)) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`))) - return - } - - ctx := context.Background() - if user.ActiveOrg.Id != curfile.OrgId { - log.Printf("[ERROR] User can't access org %s", curfile.OrgId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with organization"}`)) - return - } - - var workflow *Workflow - if curfile.WorkflowId == "global" { - // PS: Not a security issue. - // Files are global anyway, but the workflow_id is used to identify origin - log.Printf("[INFO] Uploading filename %s for org %s as global file.", curfile.Filename, curfile.OrgId) - } else { - // Try to get the org and workflow in case they don't exist - workflow, err = getWorkflow(ctx, curfile.WorkflowId) - if err != nil { - log.Printf("[ERROR] Workflow %s doesn't exist.", curfile.WorkflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - _, err = getOrg(ctx, curfile.OrgId) - if err != nil { - log.Printf("[ERROR] Org %s doesn't exist.", curfile.OrgId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - - if workflow.ExecutingOrg.Id != curfile.OrgId { - found := false - for _, curorg := range workflow.Org { - if curorg.Id == curfile.OrgId { - found = true - break - } - } - - if !found { - log.Printf("[ERROR] Org %s doesn't have access to %s.", curfile.OrgId, curfile.WorkflowId) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Error with workflow id or org id"}`)) - return - } - } - } - - if strings.Contains(curfile.Filename, "/") || strings.Contains(curfile.Filename, `"`) || strings.Contains(curfile.Filename, "..") || strings.Contains(curfile.Filename, "~") { - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Invalid characters in filename"}`)) - return - } - - // 1. Create the file object. - if len(basepath) == 0 { - basepath = "shuffle-files" - } - folderPath := fmt.Sprintf("%s/%s/%s", basepath, curfile.OrgId, curfile.WorkflowId) - - // Try to make the full file location - err = os.MkdirAll(folderPath, os.ModePerm) - if err != nil { - log.Printf("[ERROR] Writing issue for file location creation: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed creating upload location"}`)) - return - } - - filename := curfile.Filename - fileId := uuid.NewV4().String() - downloadPath := fmt.Sprintf("%s/%s", folderPath, fileId) - - duplicateWorkflows := []string{} - if curfile.WorkflowId != "global" { - for _, trigger := range workflow.Triggers { - if trigger.AppName == "Shuffle Workflow" && trigger.TriggerType == "SUBFLOW" { - for _, parameter := range trigger.Parameters { - if parameter.Name == "workflow" && len(parameter.Value) > 0 { - - found := false - for _, workflow := range duplicateWorkflows { - if workflow == parameter.Value { - found = true - break - } - } - - if !found { - duplicateWorkflows = append(duplicateWorkflows, parameter.Value) - } - - break - } - } - } - } - } - - timeNow := time.Now().Unix() - newFile := File{ - Id: fileId, - CreatedAt: timeNow, - UpdatedAt: timeNow, - Description: "", - Status: "created", - Filename: filename, - OrgId: curfile.OrgId, - WorkflowId: curfile.WorkflowId, - DownloadPath: downloadPath, - Subflows: duplicateWorkflows, - } - - err = setFile(ctx, newFile) - if err != nil { - log.Printf("[ERROR] Failed setting file: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false, "reason": "Failed setting file reference"}`)) - return - } else { - log.Printf("[INFO] Created file %s", newFile.DownloadPath) - } - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "id": "%s"}`, fileId))) - -} - -func getFile(ctx context.Context, id string) (*File, error) { - key := datastore.NameKey("Files", id, nil) - curFile := &File{} - if err := dbclient.Get(ctx, key, curFile); err != nil { - return &File{}, err - } - - return curFile, nil -} - -func setFile(ctx context.Context, file File) error { - // clear session_token and API_token for user - timeNow := time.Now().Unix() - file.UpdatedAt = timeNow - - k := datastore.NameKey("Files", file.Id, nil) - if _, err := dbclient.Put(ctx, k, &file); err != nil { - log.Println(err) - return err - } - - return nil -} - -func getAllFiles(ctx context.Context, orgId string) ([]File, error) { - var files []File - q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100) - - _, err := dbclient.GetAll(ctx, q, &files) - if err != nil { - if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - q = q.Limit(50) - _, err := dbclient.GetAll(ctx, q, &files) - if err != nil { - return []File{}, err - } - } else { - return []File{}, err - } - } - - return files, nil -} diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0bf0ccce..e403c8bd 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,11 +2,15 @@ module shuffle go 1.13 +replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared + +replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi + require ( - cloud.google.com/go v0.57.0 - cloud.google.com/go/datastore v1.1.0 + cloud.google.com/go v0.75.0 + cloud.google.com/go/datastore v1.4.0 cloud.google.com/go/pubsub v1.3.1 - cloud.google.com/go/storage v1.7.0 + cloud.google.com/go/storage v1.12.0 github.com/Microsoft/go-winio v0.4.14 // indirect github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 @@ -14,24 +18,26 @@ require ( github.com/docker/docker v1.13.1 github.com/docker/go-connections v0.4.0 github.com/docker/go-units v0.4.0 // indirect - github.com/getkin/kin-openapi v0.8.0 + github.com/frikky/shuffle-shared v0.0.12 // indirect + github.com/getkin/kin-openapi v0.52.0 // indirect + //github.com/getkin/kin-openapi v0.8.0 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.0.0 github.com/google/go-github/v28 v28.1.1 github.com/gorilla/handlers v1.4.2 // indirect - github.com/gorilla/mux v1.7.4 + github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.0.12 github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible github.com/satori/go.uuid v1.2.0 - golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 - golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - google.golang.org/api v0.23.0 - google.golang.org/appengine v1.6.6 - google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 - google.golang.org/grpc v1.29.1 + golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 + golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 + google.golang.org/api v0.36.0 + google.golang.org/appengine v1.6.7 + google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 + google.golang.org/grpc v1.34.1 gopkg.in/src-d/go-git.v4 v4.13.1 - gopkg.in/yaml.v2 v2.2.8 - gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/backend/go-app/go.sum b/backend/go-app/go.sum index 2d63be91..de114e19 100644 --- a/backend/go-app/go.sum +++ b/backend/go-app/go.sum @@ -12,14 +12,24 @@ cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bP cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= cloud.google.com/go v0.57.0 h1:EpMNVUorLiZIELdMZbCYX/ByTFCdoYopYAGxaGVz9ms= cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.66.0/go.mod h1:dgqGAjKCDxyhGTtC9dAREQGUJpkceNm1yt590Qno0Ko= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.75.0 h1:XgtDnVJRCPEUG21gjFiRPz4zI1Mjg16R+NYQjfmU4XY= +cloud.google.com/go v0.75.0/go.mod h1:VGuuCn7PG0dwsd5XPVm2Mm3wlh3EL55/79EKB6hlPTY= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.6.0/go.mod h1:hyFDG0qSGdHNz8Q6nDN8rYIkld0q/+5uBZaelxiDLfE= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0 h1:/May9ojXjRkPBNVrq+oWLqmWCkr4OU5uRY29bu0mRyQ= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/datastore v1.4.0 h1:CFDJm15RpYXeEblQ0TMDUrYtqmBmbAWTy536nA8JIc8= +cloud.google.com/go/datastore v1.4.0/go.mod h1:d18825/a9bICdAIJy2EkHs9joU4RlIZ1t6l8WDdbdY0= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -30,6 +40,10 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.7.0 h1:DzdLPI8Em+DEk7IzA2a10ivq3mxIEASC9GeNJ6FFt5Q= cloud.google.com/go/storage v1.7.0/go.mod h1:jGMIBwF+L/tL6WN/W5InNgYYu4HP0DvGB6rQ1mufWfs= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.12.0 h1:4y3gHptW1EHVtcPAVE0eBBlFuGqEejTTG3KdIE0lUX4= +cloud.google.com/go/storage v1.12.0/go.mod h1:fFLk2dp2oAhDz8QFKwqrjdJvxSp/W2g7nillojlL5Ho= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -49,6 +63,7 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -66,10 +81,17 @@ github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/frikky/kin-openapi v0.38.0 h1:V7ttwIJS8Vks4KL+mZVj1ZSqhIcQtgaG8akeqXEQgsE= +github.com/frikky/kin-openapi v0.38.0/go.mod h1:Fr28TtCHL4K0kIqtqui8HWxN1LG5uAh3z/tDfFyiA1s= +github.com/frikky/shuffle-shared v0.0.12 h1:+0EIfThmK47Po+LogPYZR4XjbS4Ds19WNMFu2YUSjhw= +github.com/frikky/shuffle-shared v0.0.12/go.mod h1:SEY432/xs4oBkOUnGwxiYKCZWZLRaheGInhAoW7N8ww= github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE= github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw= +github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M= +github.com/getkin/kin-openapi v0.52.0/go.mod h1:fRpo2Nw4Czgy0QnrIesRrEXs5+15N1F9mGZLP/aIomE= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= @@ -85,6 +107,10 @@ github.com/go-git/go-git/v5 v5.0.0/go.mod h1:oYD8y9kWsGINPFJoLdaScGCN6dlKg23blmC github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5 h1:lTz6Ys4CmqqCQmZPBlbQENR1/GucA2bzYTE12Pw4tFY= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -96,6 +122,7 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -108,6 +135,10 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -115,19 +146,32 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200905233945-acf8798be1f7/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -135,11 +179,14 @@ github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YAR github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.7.4 h1:VuZ8uybHlWmqV03+zRzdwKL4tUnIp1MAQtp1mIFE1bc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/h2non/filetype v1.0.12 h1:yHCsIe0y2cvbDARtJhGBTD2ecvqMSTvlIcph9En/Zao= github.com/h2non/filetype v1.0.12/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -155,6 +202,9 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e h1:hB2xlXdHp/pmPZq0y3QnmWAArdw9PqbmotexnWx/FU8= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= @@ -183,15 +233,21 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -201,6 +257,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79 h1:IaQbIIB2X/Mp/DKctl6ROxz1KyMlKp4uyvL6+kQ7C88= golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -224,6 +282,7 @@ golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRu golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -232,6 +291,9 @@ golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -242,6 +304,7 @@ golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -252,12 +315,28 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5 h1:WQ8q63x+f/zpC8Ac1s9wLElVoHhm32p6tudrU72n1QA= golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200904194848-62affa334b73/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423 h1:/hEknzWkMPCjTo7StMHRrBRa8YBbXuBWfck8680k3RE= +golang.org/x/oauth2 v0.0.0-20210113160501-8b1d76fa0423/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -266,6 +345,9 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -291,11 +373,25 @@ golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200409092240-59c9f1ba88fa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e h1:hq86ru83GdWTlfQFZGO4nZJTU4Bs2wfHl8oFHRaXsfc= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200828194041-157a740278f4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -336,10 +432,25 @@ golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWc golang.org/x/tools v0.0.0-20200409170454-77362c5149f0/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d h1:lzLdP95xJmMpwQ6LUHwrc5V7js93hTiY7gkznu0BgmY= golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200828161849-5deb26317202/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200915173823-2db8f0ff891c/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20200918232735-d647fc253266/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210114065538-d78b04bdf963/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= @@ -355,6 +466,15 @@ google.golang.org/api v0.21.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/ google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.23.0 h1:YlvGEOq2NA2my8cZ/9V8BcEO9okD48FlJcdqN0xJL3s= google.golang.org/api v0.23.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.31.0/go.mod h1:CL+9IBCa2WWU6gRuBWaKqGWLFFwbEUXkfeMkHLQWYWo= +google.golang.org/api v0.32.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0 h1:l2Nfbl2GPXdWorv+dT2XfinX2jOOw4zv1VhLstx+6rE= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -362,6 +482,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6 h1:lMO5rYAqUxkmaj76jAkRUvt5JZgFymx/+Q5Mzfivuhc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -387,6 +509,22 @@ google.golang.org/genproto v0.0.0-20200409111301-baae70f3302d/go.mod h1:55QSHmfG google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31 h1:Bz1qTn2YRWV+9OKJtxHJiQKCiXIdf+kwuKXdt9cBxyU= google.golang.org/genproto v0.0.0-20200507105951-43844f6eee31/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200914193844-75d14daec038/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200921151605-7abf4a1a14d5/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595 h1:x7nk+/4+SvuTDI4wnzQUlhvi+DTpyfncXBo3QWTFs7U= +google.golang.org/genproto v0.0.0-20210113195801-ae06605f4595/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -399,12 +537,26 @@ google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKa google.golang.org/grpc v1.28.1/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4= google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.32.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.34.1 h1:ugq+9++ZQPFzM2pKUMCIK8gj9M0pFyuUWO9Q8kwEDQw= +google.golang.org/grpc v1.34.1/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -421,8 +573,14 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86 h1:OfFoIUYv/me30yv7XlMy4F9RJw8DEm8WQ6QG1Ph4bH0= gopkg.in/yaml.v3 v3.0.0-20200506231410-2ff61e1afc86/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= @@ -430,6 +588,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3 h1:sXmLre5bzIR6ypkjXCDI3jHPssRhc8KD/Ome589sc3U= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/backend/go-app/main.go b/backend/go-app/main.go index deeb5bc8..11df5536 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bufio" "bytes" @@ -29,12 +31,11 @@ import ( "cloud.google.com/go/datastore" "cloud.google.com/go/pubsub" "cloud.google.com/go/storage" - "google.golang.org/api/option" "google.golang.org/appengine/mail" - "github.com/getkin/kin-openapi/openapi2" - "github.com/getkin/kin-openapi/openapi2conv" - "github.com/getkin/kin-openapi/openapi3" + "github.com/frikky/kin-openapi/openapi2" + "github.com/frikky/kin-openapi/openapi2conv" + "github.com/frikky/kin-openapi/openapi3" /* "github.com/frikky/kin-openapi/openapi2" "github.com/frikky/kin-openapi/openapi2conv" @@ -63,6 +64,7 @@ import ( // Web "github.com/gorilla/mux" "github.com/patrickmn/go-cache" + "google.golang.org/api/option" "google.golang.org/grpc" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" ) @@ -150,11 +152,11 @@ type UserLimits struct { } type retStruct struct { - Success bool `json:"success"` - SyncFeatures SyncFeatures `json:"sync_features"` - SessionKey string `json:"session_key"` - IntervalSeconds int64 `json:"interval_seconds"` - Reason string `json:"reason"` + Success bool `json:"success"` + SyncFeatures shuffle.SyncFeatures `json:"sync_features"` + SessionKey string `json:"session_key"` + IntervalSeconds int64 `json:"interval_seconds"` + Reason string `json:"reason"` } // Saves some data, not sure what to have here lol @@ -172,37 +174,15 @@ type UserAuthField struct { } // Not environment, but execution environment -type Environment struct { - Name string `datastore:"name"` - Type string `datastore:"type"` - Registered bool `datastore:"registered"` - Default bool `datastore:"default" json:"default"` - Archived bool `datastore:"archived" json:"archived"` - Id string `datastore:"id" json:"id"` - OrgId string `datastore:"org_id" json:"org_id"` -} - -type User struct { - Username string `datastore:"Username" json:"username"` - Password string `datastore:"password,noindex" password:"password,omitempty"` - Session string `datastore:"session,noindex" json:"session"` - Verified bool `datastore:"verified,noindex" json:"verified"` - PrivateApps []WorkflowApp `datastore:"privateapps" json:"privateapps":` - Role string `datastore:"role" json:"role"` - Roles []string `datastore:"roles" json:"roles"` - VerificationToken string `datastore:"verification_token" json:"verification_token"` - ApiKey string `datastore:"apikey" json:"apikey"` - ResetReference string `datastore:"reset_reference" json:"reset_reference"` - Executions ExecutionInfo `datastore:"executions" json:"executions"` - Limits UserLimits `datastore:"limits" json:"limits"` - Authentication []UserAuth `datastore:"authentication,noindex" json:"authentication"` - ResetTimeout int64 `datastore:"reset_timeout,noindex" json:"reset_timeout"` - Id string `datastore:"id" json:"id"` - Orgs []string `datastore:"orgs" json:"orgs"` - CreationTime int64 `datastore:"creation_time" json:"creation_time"` - ActiveOrg Org `json:"active_org" datastore:"active_org"` - Active bool `datastore:"active" json:"active"` -} +//type Environment struct { +// Name string `datastore:"name"` +// Type string `datastore:"type"` +// Registered bool `datastore:"registered"` +// Default bool `datastore:"default" json:"default"` +// Archived bool `datastore:"archived" json:"archived"` +// Id string `datastore:"id" json:"id"` +// OrgId string `datastore:"org_id" json:"org_id"` +//} // timeout maybe? idk type session struct { @@ -611,83 +591,6 @@ func checkFileExistsLocal(basepath string, filepath string) bool { return true } -func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (User, error) { - apikey := request.Header.Get("Authorization") - if len(apikey) > 0 { - if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("[WARNING] Apikey doesn't start with bearer") - return User{}, errors.New("No bearer token for authorization header") - } - - apikeyCheck := strings.Split(apikey, " ") - if len(apikeyCheck) != 2 { - log.Printf("[WARNING] Invalid format for apikey.") - return User{}, errors.New("Invalid format for apikey") - } - - // This is annoying af and is done because of maxlength lol - newApikey := apikeyCheck[1] - if len(newApikey) > 249 { - newApikey = newApikey[0:248] - } - - ctx := context.Background() - - // Make specific check for just service user? - // Get the user based on APIkey here - Userdata, err := getApikey(ctx, apikeyCheck[1]) - if err != nil { - log.Printf("Apikey %s doesn't exist: %s", apikey, err) - return User{}, err - } - - if len(Userdata.Username) > 0 { - return Userdata, nil - } else { - return Userdata, errors.New(fmt.Sprintf("[WARNING] User is invalid - no username found")) - } - } - - // One time API keys - authorizationArr, ok := request.URL.Query()["authorization"] - ctx := context.Background() - if ok { - authorization := "" - if len(authorizationArr) > 0 { - authorization = authorizationArr[0] - } - _ = authorization - } - - c, err := request.Cookie("session_token") - if err == nil { - sessionToken := c.Value - session, err := getSession(ctx, sessionToken) - if err != nil { - log.Printf("[WARNING] Session %s doesn't exist (session auth): %s", sessionToken, err) - return User{}, err - } - - // Get session first - // Should basically never happen - Userdata, err := getUser(ctx, session.Id) - if err != nil { - log.Printf("[INFO] Username %s doesn't exist (authcheck): %s", session.Username, err) - return User{}, err - } - - if Userdata.Session != sessionToken { - return User{}, errors.New("Wrong session token") - } - - // Means session exists, but - return *Userdata, nil - } - - // Key = apikey - return User{}, errors.New("Missing authentication") -} - func handleGetallSchedules(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -800,7 +703,7 @@ func deleteUser(resp http.ResponseWriter, request *http.Request) { return } - userInfo, userErr := handleApiAuthentication(resp, request) + userInfo, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -828,7 +731,7 @@ func deleteUser(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - foundUser, err := getUser(ctx, userId) + foundUser, err := shuffle.GetUser(ctx, userId) if err != nil { log.Printf("Can't find user %s (delete user): %s", userId, err) resp.WriteHeader(401) @@ -864,7 +767,7 @@ func deleteUser(resp http.ResponseWriter, request *http.Request) { foundUser.Active = true } - err = setUser(ctx, foundUser) + err = shuffle.SetUser(ctx, foundUser) if err != nil { log.Printf("Failed swapping active for user %s (%s)", foundUser.Username, foundUser.Id) resp.WriteHeader(401) @@ -920,7 +823,7 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("verification_token =", reference) - var users []User + var users []shuffle.User _, err := dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users for verification token: %s", err) @@ -941,7 +844,7 @@ func handleRegisterVerification(resp http.ResponseWriter, request *http.Request) // FIXME: Not for cloud! Userdata.Verified = true - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Failed adding verification for user %s: %s", Userdata.Username, err) resp.WriteHeader(401) @@ -962,7 +865,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { // FIXME: Overhaul the top part. // Only admin can change environments, but if there are no users, anyone can make (first) - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Can't handle set env auth"}`)) @@ -976,7 +879,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { @@ -993,7 +896,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { return } - var newEnvironments []Environment + var newEnvironments []shuffle.Environment err = json.Unmarshal(body, &newEnvironments) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -1053,7 +956,7 @@ func handleSetEnvironments(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func createNewUser(username, password, role, apikey string, org Org) error { +func createNewUser(username, password, role, apikey string, org shuffle.Org) error { // Returns false if there is an issue // Use this for register err := checkPasswordStrength(password) @@ -1070,7 +973,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { ctx := context.Background() q := datastore.NewQuery("Users").Filter("Username =", username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user for registration: %s", err) @@ -1087,7 +990,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { return err } - newUser := new(User) + newUser := new(shuffle.User) newUser.Username = username newUser.Password = string(hashedPassword) newUser.Verified = false @@ -1136,13 +1039,13 @@ func createNewUser(username, password, role, apikey string, org Org) error { newUser.Id = ID.String() newUser.VerificationToken = verifyToken.String() - err = setUser(ctx, newUser) + err = shuffle.SetUser(ctx, newUser) if err != nil { log.Printf("Error adding User %s: %s", username, err) return err } - neworg, err := getOrg(ctx, org.Id) + neworg, err := shuffle.GetOrg(ctx, org.Id) if err == nil { //neworg.Users = append(neworg.Users, *newUser) err = setOrg(ctx, *neworg, neworg.Id) @@ -1170,7 +1073,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { // FIXME: Overhaul the top part. // Only admin can CREATE users, but if there are no users, anyone can make (first) count, countErr := getUserCount() - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { if (countErr == nil && count > 0) || countErr != nil { resp.WriteHeader(401) @@ -1206,7 +1109,7 @@ func handleRegister(resp http.ResponseWriter, request *http.Request) { if user.ActiveOrg.Id == "" { log.Printf("There's no active org for the user. Checking if there's a single one to assing it to.") - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err == nil && len(orgs) == 1 { @@ -1255,7 +1158,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { Expires: time.Unix(0, 0), }) - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in handleLogout: %s", err) resp.WriteHeader(200) @@ -1264,7 +1167,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - session, err := getSession(ctx, userInfo.Session) + session, err := shuffle.GetSession(ctx, userInfo.Session) if err != nil { log.Printf("Session %#v doesn't exist: %s", session, err) resp.WriteHeader(401) @@ -1295,7 +1198,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { // Get session first // Should basically never happen - //_, err = getUser(ctx, session.Id) + //_, err = shuffle.GetUser(ctx, session.Id) //if err != nil { // log.Printf("Username %s doesn't exist (logout): %s", session.Username, err) // resp.WriteHeader(401) @@ -1309,7 +1212,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { // FIXME // Session might delete someone elses here? // No need to think about before possible scale..? - err = SetSession(ctx, userInfo, "") + err = shuffle.SetSession(ctx, userInfo, "") if err != nil { log.Printf("Error removing session for: %s", err) resp.WriteHeader(401) @@ -1326,7 +1229,7 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { } userInfo.Session = "" - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Failed updating user: %s", err) resp.WriteHeader(401) @@ -1340,35 +1243,13 @@ func handleLogout(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": false, "reason": "Successfully logged out"}`)) } -func generateApikey(ctx context.Context, userInfo User) (User, error) { - // Generate UUID - // Set uuid to apikey in backend (update) - apikey := uuid.NewV4() - userInfo.ApiKey = apikey.String() - - err := SetApikey(ctx, userInfo) - if err != nil { - log.Printf("Failed updating apikey: %s", err) - return userInfo, err - } - - // Updating user - err = setUser(ctx, &userInfo) - if err != nil { - log.Printf("Failed updating user: %s", err) - return userInfo, err - } - - return userInfo, nil -} - func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in apigen: %s", err) resp.WriteHeader(401) @@ -1409,7 +1290,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { return } - foundUser, err := getUser(ctx, t.UserId) + foundUser, err := shuffle.GetUser(ctx, t.UserId) if err != nil { log.Printf("Can't find user %s (update user): %s", t.UserId, err) resp.WriteHeader(401) @@ -1452,7 +1333,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { if len(t.Username) > 0 { q := datastore.NewQuery("Users").Filter("username =", t.Username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { resp.WriteHeader(401) @@ -1477,7 +1358,7 @@ func handleUpdateUser(resp http.ResponseWriter, request *http.Request) { foundUser.Username = t.Username } - err = setUser(ctx, foundUser) + err = shuffle.SetUser(ctx, foundUser) if err != nil { log.Printf("Error patching user %s: %s", foundUser.Username, err) resp.WriteHeader(401) @@ -1495,7 +1376,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in apigen: %s", err) resp.WriteHeader(401) @@ -1505,7 +1386,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() if request.Method == "GET" { - newUserInfo, err := generateApikey(ctx, userInfo) + newUserInfo, err := shuffle.GenerateApikey(ctx, userInfo) if err != nil { log.Printf("Failed to generate apikey for user %s: %s", userInfo.Username, err) resp.WriteHeader(401) @@ -1544,7 +1425,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { return } - foundUser, err := getUser(ctx, t.UserId) + foundUser, err := shuffle.GetUser(ctx, t.UserId) if err != nil { log.Printf("Can't find user %s (apikey gen): %s", t.UserId, err) resp.WriteHeader(401) @@ -1552,7 +1433,7 @@ func handleApiGeneration(resp http.ResponseWriter, request *http.Request) { return } - newUserInfo, err := generateApikey(ctx, *foundUser) + newUserInfo, err := shuffle.GenerateApikey(ctx, *foundUser) if err != nil { log.Printf("Failed to generate apikey for user %s: %s", foundUser.Username, err) resp.WriteHeader(401) @@ -1576,7 +1457,7 @@ func handleSettings(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in apigen: %s", err) resp.WriteHeader(401) @@ -1594,7 +1475,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in handleInfo: %s", err) resp.WriteHeader(401) @@ -1623,7 +1504,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { resp.WriteHeader(401) @@ -1688,14 +1569,14 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { // Updating user info if there's something wrong if (len(userInfo.ActiveOrg.Name) == 0 || len(userInfo.ActiveOrg.Id) == 0) && len(userInfo.Orgs) > 0 { - _, err := getOrg(ctx, userInfo.Orgs[0]) + _, err := shuffle.GetOrg(ctx, userInfo.Orgs[0]) if err != nil { - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err == nil { newStringOrgs := []string{} - newOrgs := []Org{} + newOrgs := []shuffle.Org{} for _, org := range orgs { if strings.ToLower(org.Name) == strings.ToLower(userInfo.Orgs[0]) { newOrgs = append(newOrgs, org) @@ -1707,7 +1588,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { userInfo.ActiveOrg = newOrgs[0] userInfo.Orgs = newStringOrgs - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } else { @@ -1721,10 +1602,10 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } else { // 1. Check if the org exists by ID // 2. if it does, overwrite user - userInfo.ActiveOrg = Org{ + userInfo.ActiveOrg = shuffle.Org{ Id: userInfo.Orgs[0], } - err = setUser(ctx, &userInfo) + err = shuffle.SetUser(ctx, &userInfo) if err != nil { log.Printf("Error patching User for activeOrg: %s", err) } @@ -1732,10 +1613,10 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } // FIXME: Remove this dependency by updating users' orgs when org itself is updated - org, err := getOrg(ctx, userInfo.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, userInfo.ActiveOrg.Id) if err == nil { userInfo.ActiveOrg = *org - userInfo.ActiveOrg.Users = []User{} + userInfo.ActiveOrg.Users = []shuffle.User{} } currentOrg, err := json.Marshal(userInfo.ActiveOrg) @@ -1816,7 +1697,7 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { // With user, do a search for workflows with user or user's org attached // Only giving 200 to not give any suspicion whether they're onto an actual user or not q := datastore.NewQuery("Users").Filter("reset_reference =", t.Reference) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting users: %s", err) @@ -1845,7 +1726,7 @@ func handlePasswordReset(resp http.ResponseWriter, request *http.Request) { Userdata.Password = string(hashedPassword) Userdata.ResetTimeout = 0 Userdata.ResetReference = "" - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Error adding User %s: %s", Userdata.Username, err) resp.WriteHeader(200) @@ -1884,7 +1765,7 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { return } - userInfo, err := handleApiAuthentication(resp, request) + userInfo, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -1929,11 +1810,11 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - foundUser := User{} + foundUser := shuffle.User{} if !curUserFound { log.Printf("Have to find a different user") q := datastore.NewQuery("Users").Filter("Username =", strings.ToLower(t.Username)) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user %s", t.Username) @@ -1998,7 +1879,7 @@ func handlePasswordChange(resp http.ResponseWriter, request *http.Request) { } userInfo.Password = string(hashedPassword) - err = setUser(ctx, &foundUser) + err = shuffle.SetUser(ctx, &foundUser) if err != nil { log.Printf("Error fixing password for user %s: %s", userInfo.Username, err) resp.WriteHeader(401) @@ -2087,7 +1968,7 @@ func handleGetSchedules(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2130,7 +2011,7 @@ func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2139,7 +2020,7 @@ func handleGetEnvironments(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", user.ActiveOrg.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { @@ -2181,7 +2062,7 @@ func handleGetOrg(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2190,7 +2071,7 @@ func handleGetOrg(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - org, err := getOrg(ctx, fileId) + org, err := shuffle.GetOrg(ctx, fileId) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`)) @@ -2212,7 +2093,7 @@ func handleGetOrg(resp http.ResponseWriter, request *http.Request) { return } - org.Users = []User{} + org.Users = []shuffle.User{} org.SyncConfig.Apikey = "" newjson, err := json.Marshal(org) if err != nil { @@ -2232,7 +2113,7 @@ func handleGetOrgs(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2247,7 +2128,7 @@ func handleGetOrgs(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - var orgs []Org + var orgs []shuffle.Org q := datastore.NewQuery("Organizations") _, err = dbclient.GetAll(ctx, q, &orgs) if err != nil { @@ -2287,7 +2168,7 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -2303,14 +2184,14 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) { // FIXME: Check by org. ctx := context.Background() - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "reason": "Failed getting org users"}`)) return } - newUsers := []User{} + newUsers := []shuffle.User{} for _, item := range org.Users { if len(item.Username) == 0 { continue @@ -2390,7 +2271,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("[INFO] Login Username: %s", data.Username) q := datastore.NewQuery("Users").Filter("Username =", data.Username) - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err != nil { log.Printf("Failed getting user %s", data.Username) @@ -2438,7 +2319,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { loginData = fmt.Sprintf(`{"success": true, "cookies": [{"key": "session_token", "value": "%s", "expiration": %d}]}`, Userdata.Session, expiration.Unix()) //log.Printf("SESSION LENGTH MORE THAN 0 IN LOGIN: %s", Userdata.Session) - err = SetSession(ctx, Userdata, Userdata.Session) + err = shuffle.SetSession(ctx, Userdata, Userdata.Session) if err != nil { log.Printf("Error adding session to database: %s", err) } @@ -2458,13 +2339,13 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { }) // ADD TO DATABASE - err = SetSession(ctx, Userdata, sessionToken) + err = shuffle.SetSession(ctx, Userdata, sessionToken) if err != nil { log.Printf("Error adding session to database: %s", err) } Userdata.Session = sessionToken - err = setUser(ctx, &Userdata) + err = shuffle.SetUser(ctx, &Userdata) if err != nil { log.Printf("Failed updating user when setting session: %s", err) resp.WriteHeader(500) @@ -2481,46 +2362,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(loginData)) } -func getApikey(ctx context.Context, apikey string) (User, error) { - // Query for the specifci workflowId - q := datastore.NewQuery("Users").Filter("apikey =", apikey) - var users []User - _, err := dbclient.GetAll(ctx, q, &users) - if err != nil { - log.Printf("[ERROR] Error getting users apikey (getapikey): %s", err) - return User{}, err - } - - if len(users) == 0 { - log.Printf("[WARNING] No users found for apikey %s", apikey) - return User{}, err - } - - return users[0], nil -} - -func getSession(ctx context.Context, thissession string) (*session, error) { - key := datastore.NameKey("sessions", thissession, nil) - curUser := &session{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &session{}, err - } - - return curUser, nil -} - -// ListBooks returns a list of books, ordered by title. -func getOrg(ctx context.Context, id string) (*Org, error) { - key := datastore.NameKey("Organizations", id, nil) - curOrg := &Org{} - if err := dbclient.Get(ctx, key, curOrg); err != nil { - return &Org{}, err - } - - return curOrg, nil -} - -func setOrg(ctx context.Context, org Org, id string) error { +func setOrg(ctx context.Context, org shuffle.Org, id string) error { // clear session_token and API_token for user timeNow := int64(time.Now().Unix()) if org.Created == 0 { @@ -2543,15 +2385,15 @@ func setOrg(ctx context.Context, org Org, id string) error { } // ListBooks returns a list of books, ordered by title. -func getUser(ctx context.Context, id string) (*User, error) { - key := datastore.NameKey("Users", id, nil) - curUser := &User{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &User{}, err - } - - return curUser, nil -} +//func getUser(ctx context.Context, id string) (*User, error) { +// key := datastore.NameKey("Users", id, nil) +// curUser := &User{} +// if err := dbclient.Get(ctx, key, curUser); err != nil { +// return &User{}, err +// } +// +// return curUser, nil +//} // Index = Username func DeleteKeys(ctx context.Context, entity string, value []string) error { @@ -2584,52 +2426,6 @@ func DeleteKey(ctx context.Context, entity string, value string) error { return nil } -// Index = Username -func SetApikey(ctx context.Context, Userdata User) error { - // Non indexed User data - newapiUser := new(Userapi) - newapiUser.ApiKey = Userdata.ApiKey - newapiUser.Username = Userdata.Username - key1 := datastore.NameKey("apikey", newapiUser.ApiKey, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, newapiUser); err != nil { - log.Printf("Error adding apikey: %s", err) - return err - } - - return nil -} - -// Index = Username -func SetSession(ctx context.Context, Userdata User, value string) error { - // Non indexed User data - Userdata.Session = value - key1 := datastore.NameKey("Users", Userdata.Id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key1, &Userdata); err != nil { - log.Printf("rror adding Usersession: %s", err) - return err - } - - if len(Userdata.Session) > 0 { - // Indexed session data - sessiondata := new(session) - sessiondata.Username = Userdata.Username - sessiondata.Session = Userdata.Session - sessiondata.Id = Userdata.Id - key2 := datastore.NameKey("sessions", sessiondata.Session, nil) - - if _, err := dbclient.Put(ctx, key2, sessiondata); err != nil { - log.Printf("Error adding session: %s", err) - return err - } - } - - return nil -} - func setOpenApiDatastore(ctx context.Context, id string, data ParsedOpenApi) error { k := datastore.NameKey("openapi3", id, nil) if _, err := dbclient.Put(ctx, k, &data); err != nil { @@ -2649,7 +2445,7 @@ func getOpenApiDatastore(ctx context.Context, id string) (ParsedOpenApi, error) return *api, nil } -func setEnvironment(ctx context.Context, data *Environment) error { +func setEnvironment(ctx context.Context, data *shuffle.Environment) error { // clear session_token and API_token for user k := datastore.NameKey("Environments", strings.ToLower(data.Name), nil) @@ -2663,7 +2459,7 @@ func setEnvironment(ctx context.Context, data *Environment) error { return nil } -func fixOrgUser(ctx context.Context, org *Org) *Org { +func fixOrgUser(ctx context.Context, org *shuffle.Org) *shuffle.Org { //found := false //for _, id := range user.Orgs { // if user.ActiveOrg.Id == id { @@ -2682,7 +2478,7 @@ func fixOrgUser(ctx context.Context, org *Org) *Org { // continue // } - // org, err := getOrg(ctx, orgId) + // org, err := shuffle.GetOrg(ctx, orgId) // if err != nil { // log.Printf("Error getting org %s", orgId) // continue @@ -2718,21 +2514,7 @@ func fixOrgUser(ctx context.Context, org *Org) *Org { return org } -// ListBooks returns a list of books, ordered by title. -func setUser(ctx context.Context, data *User) error { - data = fixUserOrg(ctx, data) - - // clear session_token and API_token for user - k := datastore.NameKey("Users", data.Id, nil) - if _, err := dbclient.Put(ctx, k, data); err != nil { - log.Println(err) - return err - } - - return nil -} - -func fixUserOrg(ctx context.Context, user *User) *User { +func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { found := false for _, id := range user.Orgs { if user.ActiveOrg.Id == id { @@ -2751,7 +2533,7 @@ func fixUserOrg(ctx context.Context, user *User) *User { continue } - org, err := getOrg(ctx, orgId) + org, err := shuffle.GetOrg(ctx, orgId) if err != nil { log.Printf("Error getting org %s", orgId) continue @@ -2768,10 +2550,10 @@ func fixUserOrg(ctx context.Context, user *User) *User { } if userFound { - user.PrivateApps = []WorkflowApp{} - user.Executions = ExecutionInfo{} - user.Limits = UserLimits{} - user.Authentication = []UserAuth{} + user.PrivateApps = []shuffle.WorkflowApp{} + user.Executions = shuffle.ExecutionInfo{} + user.Limits = shuffle.UserLimits{} + user.Authentication = []shuffle.UserAuth{} org.Users[orgIndex] = *user } else { @@ -2955,7 +2737,7 @@ func handleSetHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3197,15 +2979,15 @@ func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { return } -func getSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) { - key := datastore.NameKey("schedules", strings.ToLower(schedulename), nil) - curUser := &ScheduleOld{} - if err := dbclient.Get(ctx, key, curUser); err != nil { - return &ScheduleOld{}, err - } - - return curUser, nil -} +//func GetSchedule(ctx context.Context, schedulename string) (*ScheduleOld, error) { +// key := datastore.NameKey("schedules", strings.ToLower(schedulename), nil) +// curUser := &ScheduleOld{} +// if err := dbclient.Get(ctx, key, curUser); err != nil { +// return &ScheduleOld{}, err +// } +// +// return curUser, nil +//} func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) @@ -3234,7 +3016,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() // FIXME: Schedule = trigger? - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule: %s", err) resp.WriteHeader(401) @@ -3265,7 +3047,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3476,7 +3258,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { for _, item := range hook.Workflows { //log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) - workflow := Workflow{ + workflow := shuffle.Workflow{ ID: "", } @@ -3573,7 +3355,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3653,7 +3435,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { if requestdata.Environment == "cloud" { // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c log.Printf("[INFO] Should START a cloud webhook for url %s for startnode %s", currentUrl, startNode) - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -3730,7 +3512,7 @@ func sendHookResult(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3798,7 +3580,7 @@ func handleGetHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -3886,7 +3668,7 @@ func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed getting schedule: %s", err) resp.WriteHeader(401) @@ -3953,7 +3735,7 @@ func executeSchedule(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("[INFO] EXECUTING %s!", workflowId) - idConfig, err := getSchedule(ctx, workflowId) + idConfig, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Error getting schedule: %s", err) resp.WriteHeader(401) @@ -4064,7 +3846,7 @@ func uploadWorkflowResult(resp http.ResponseWriter, request *http.Request) { // FIXME - validate ID as well ctx := context.Background() - schedule, err := getSchedule(ctx, workflowId) + schedule, err := shuffle.GetSchedule(ctx, workflowId) if err != nil { log.Printf("Failed setting schedule %s: %s", workflowId, err) resp.WriteHeader(401) @@ -4496,7 +4278,7 @@ func handleGetallHooks(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -4566,7 +4348,7 @@ func findAvailablePorts(startRange int64, endRange int64) string { } func handleSendalert(resp http.ResponseWriter, request *http.Request) { - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in sendalert: %s", err) resp.WriteHeader(401) @@ -4903,7 +4685,7 @@ func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) { return } - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -4953,7 +4735,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -4981,7 +4763,7 @@ func getOpenapi(resp http.ResponseWriter, request *http.Request) { // FIXME - FIX AUTH WITH APP ctx := context.Background() - //_, err = getApp(ctx, id) + //_, err = shuffle.GetApp(ctx, id) //if err == nil { // log.Println("You're supposed to be able to continue now.") //} @@ -5014,7 +4796,7 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -5201,7 +4983,7 @@ func validateSwagger(resp http.ResponseWriter, request *http.Request) { } // Just here to verify that the user is logged in - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in validate swagger: %s", err) resp.WriteHeader(401) @@ -5394,7 +5176,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } log.Printf("[INFO] SETTING APP TO LIVE!!!") - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in verify swagger: %s", err) resp.WriteHeader(401) @@ -5431,7 +5213,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { if test.Editing { // Quick verification test ctx := context.Background() - app, err := getApp(ctx, test.Id) + app, err := shuffle.GetApp(ctx, test.Id) if err != nil { log.Printf("Error getting app when editing: %s", app.Name) resp.WriteHeader(401) @@ -5476,7 +5258,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { swagger.Info.Title = strings.Replace(swagger.Info.Title, " ", "_", -1) } - basePath, err := buildStructure(swagger, newmd5) + basePath, err := shuffle.BuildStructure(swagger, newmd5) if err != nil { log.Printf("Failed to build base structure: %s", err) resp.WriteHeader(500) @@ -5485,7 +5267,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } //log.Printf("Should generate yaml") - swagger, api, pythonfunctions, err := generateYaml(swagger, newmd5) + swagger, api, pythonfunctions, err := shuffle.GenerateYaml(swagger, newmd5) if err != nil { log.Printf("Failed building and generating yaml: %s", err) resp.WriteHeader(500) @@ -5495,7 +5277,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // FIXME: CHECK IF SAME NAME AS NORMAL APP // Can't overwrite existing normal app - workflowApps, err := getAllWorkflowApps(ctx, 500) + workflowApps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting all workflow apps from database to verify: %s", err) resp.WriteHeader(401) @@ -5515,7 +5297,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { api.Owner = user.Id - err = dumpApi(basePath, api) + err = shuffle.DumpApi(basePath, api) if err != nil { log.Printf("Failed dumping yaml: %s", err) resp.WriteHeader(500) @@ -5526,7 +5308,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, newmd5) classname := strings.Replace(identifier, " ", "", -1) classname = strings.Replace(classname, "-", "", -1) - parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) + parsedCode, err := shuffle.DumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) if err != nil { log.Printf("Failed dumping python: %s", err) resp.WriteHeader(500) @@ -5546,7 +5328,8 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // 5. Upload as cloud function // 1. Upload the API to datastore - err = deployAppToDatastore(ctx, api) + err = shuffle.DeployAppToDatastore(ctx, api) + //func DeployAppToDatastore(ctx context.Context, workflowapp WorkflowApp, bucketName string) error { if err != nil { log.Printf("Failed adding app to db: %s", err) resp.WriteHeader(500) @@ -5555,7 +5338,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } // 2. Get all the required code - appbase, staticBaseline, err := getAppbase() + appbase, staticBaseline, err := shuffle.GetAppbase() if err != nil { log.Printf("Failed getting appbase: %s", err) resp.WriteHeader(500) @@ -5564,17 +5347,17 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { } // Have to do some quick checks of the python code (: - _, parsedCode = formatAppfile(parsedCode) + _, parsedCode = shuffle.FormatAppfile(parsedCode) - fixedAppbase := fixAppbase(appbase) - runner := getRunner(classname) + fixedAppbase := shuffle.FixAppbase(appbase) + runner := shuffle.GetRunnerOnprem(classname) // 2. Put it together stitched := string(staticBaseline) + strings.Join(fixedAppbase, "\n") + parsedCode + string(runner) //log.Println(stitched) // 3. Zip and stream it directly in the directory - _, err = streamZipdata(ctx, identifier, stitched, "requests\nurllib3") + _, err = shuffle.StreamZipdata(ctx, identifier, stitched, "requests\nurllib3", "") if err != nil { log.Printf("[ERROR] Zipfile error: %s", err) resp.WriteHeader(500) @@ -5643,7 +5426,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { user.PrivateApps[foundNumber] = api } - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("[ERROR] Failed adding verification for user %s: %s", user.Username, err) resp.WriteHeader(500) @@ -5804,7 +5587,7 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio ctx := context.Background() // 1. Get the workflow // 2. Execute it with the data - workflow, err := getWorkflow(ctx, workflowId) + workflow, err := shuffle.GetWorkflow(ctx, workflowId) if err != nil { return err } @@ -5813,12 +5596,12 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio _ = workflow parsedArgument := executionArgument - newExec := ExecutionRequest{ + newExec := shuffle.ExecutionRequest{ ExecutionSource: executionSource, ExecutionArgument: parsedArgument, } - var execution ExecutionRequest + var execution shuffle.ExecutionRequest err = json.Unmarshal([]byte(parsedArgument), &execution) if err == nil { //log.Printf("[INFO] FOUND EXEC %#v", execution) @@ -5853,7 +5636,7 @@ func handleCloudExecutionOnprem(workflowId, startNode, executionSource, executio Body: ioutil.NopCloser(bytes.NewReader(b)), } - _, _, err = handleExecution(workflowId, Workflow{}, newRequest) + _, _, err = handleExecution(workflowId, shuffle.Workflow{}, newRequest) return err } @@ -5974,7 +5757,7 @@ func handleCloudJob(job CloudSyncJob) error { return err } - _, _, err = handleExecution(job.PrimaryItemId, Workflow{}, newRequest) + _, _, err = handleExecution(job.PrimaryItemId, shuffle.Workflow{}, newRequest) if err != nil { log.Printf("Failed continuing workflow from cloud user_input: %s", err) return err @@ -5999,7 +5782,7 @@ func handleCloudJob(job CloudSyncJob) error { } */ - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, result := range workflowExecution.Results { if result.Action.AppName == "User Input" && result.Result == "Waiting for user feedback based on configuration" { result.Status = "ABORTED" @@ -6026,7 +5809,7 @@ func handleCloudJob(job CloudSyncJob) error { } // Handles jobs from remote (cloud) -func remoteOrgJobController(org Org, body []byte) error { +func remoteOrgJobController(org shuffle.Org, body []byte) error { type retStruct struct { Success bool `json:"success"` Reason string `json:"reason"` @@ -6051,7 +5834,7 @@ func remoteOrgJobController(org Org, body []byte) error { log.Printf("[WARNING] STOPPING ORG SCHEDULE for: %s", org.Id) value.Lock() - org, err := getOrg(ctx, org.Id) + org, err := shuffle.GetOrg(ctx, org.Id) if err != nil { log.Printf("[WARNING] Failed finding org %s: %s", org.Id, err) return err @@ -6091,7 +5874,7 @@ func remoteOrgJobController(org Org, body []byte) error { return nil } -func remoteOrgJobHandler(org Org, interval int) error { +func remoteOrgJobHandler(org shuffle.Org, interval int) error { client := &http.Client{} syncUrl := fmt.Sprintf("%s/api/v1/cloud/sync", syncUrl) req, err := http.NewRequest( @@ -6179,7 +5962,7 @@ func runInit(ctx context.Context) { setUsers := false orgQuery := datastore.NewQuery("Organizations") - var activeOrgs []Org + var activeOrgs []shuffle.Org _, err = dbclient.GetAll(ctx, orgQuery, &activeOrgs) if err != nil { log.Printf("Error getting organizations!") @@ -6194,11 +5977,11 @@ func runInit(ctx context.Context) { log.Printf(`No orgs. Setting org "default"`) orgSetupName := "default" orgId := uuid.NewV4().String() - newOrg := Org{ + newOrg := shuffle.Org{ Name: orgSetupName, Id: orgId, Org: orgSetupName, - Users: []User{}, + Users: []shuffle.User{}, Roles: []string{"admin", "user"}, CloudSync: false, } @@ -6220,15 +6003,15 @@ func runInit(ctx context.Context) { activeOrg := activeOrgs[0] q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err = dbclient.GetAll(ctx, q, &users) if err == nil { setOrgBool := false for _, user := range users { - newUser := User{ + newUser := shuffle.User{ Username: user.Username, Id: user.Id, - ActiveOrg: Org{ + ActiveOrg: shuffle.Org{ Id: activeOrg.Id, }, Orgs: []string{activeOrg.Id}, @@ -6272,13 +6055,13 @@ func runInit(ctx context.Context) { // Fix active users etc q := datastore.NewQuery("Users").Filter("active =", true) - var activeusers []User + var activeusers []shuffle.User _, err = dbclient.GetAll(ctx, q, &activeusers) if err != nil { log.Printf("Error getting users during init: %s", err) } else { q := datastore.NewQuery("Users") - var users []User + var users []shuffle.User _, err := dbclient.GetAll(ctx, q, &users) if len(activeusers) == 0 && len(users) > 0 { @@ -6298,13 +6081,13 @@ func runInit(ctx context.Context) { if len(user.Orgs) == 0 { defaultName := "default" user.Orgs = []string{defaultName} - user.ActiveOrg = Org{ + user.ActiveOrg = shuffle.Org{ Name: defaultName, Role: "user", } } - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("Failed to reset user") } else { @@ -6325,7 +6108,7 @@ func runInit(ctx context.Context) { } else { apikey := os.Getenv("SHUFFLE_DEFAULT_APIKEY") - tmpOrg := Org{ + tmpOrg := shuffle.Org{ Name: "default", } err = createNewUser(username, password, "admin", apikey, tmpOrg) @@ -6348,7 +6131,7 @@ func runInit(ctx context.Context) { for _, user := range users { if user.ActiveOrg.Id == "" && len(user.Username) > 0 { user.ActiveOrg = activeOrgs[0] - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("Failed updating user %s with org", user.Username) } else { @@ -6365,7 +6148,7 @@ func runInit(ctx context.Context) { count, err := getEnvironmentCount() if count == 0 && err == nil && len(activeOrgs) == 1 { log.Printf("Setting up environment with org %s", activeOrgs[0].Id) - item := Environment{ + item := shuffle.Environment{ Name: "Shuffle", Type: "onprem", OrgId: activeOrgs[0].Id, @@ -6378,7 +6161,7 @@ func runInit(ctx context.Context) { } } else if len(activeOrgs) == 1 { log.Printf("Setting up all environments with org %s", activeOrgs[0].Id) - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments") _, err = dbclient.GetAll(ctx, q, &environments) if err == nil { @@ -6399,7 +6182,7 @@ func runInit(ctx context.Context) { // Fixing workflows to have real activeorg IDs if len(activeOrgs) == 1 { q := datastore.NewQuery("workflow").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Error getting workflows in runinit: %s", err) @@ -6467,7 +6250,7 @@ func runInit(ctx context.Context) { } */ - var allworkflowapps []AppAuthenticationStorage + var allworkflowapps []shuffle.AppAuthenticationStorage q = datastore.NewQuery("workflowappauth") _, err = dbclient.GetAll(ctx, q, &allworkflowapps) if err == nil { @@ -6479,7 +6262,7 @@ func runInit(ctx context.Context) { //log.Printf("Should update auth for %#v!", item) item.OrgId = activeOrgs[0].Id - err = setWorkflowAppAuthDatastore(ctx, item, item.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, item, item.Id) if err != nil { log.Printf("Failed adding AUTH to org %s", activeOrgs[0].Id) } @@ -6567,7 +6350,7 @@ func runInit(ctx context.Context) { Body: ioutil.NopCloser(strings.NewReader(schedule.WrappedArgument)), } - _, _, err := handleExecution(schedule.WorkflowId, Workflow{}, request) + _, _, err := handleExecution(schedule.WorkflowId, shuffle.Workflow{}, request) if err != nil { log.Printf("Failed to execute %s: %s", schedule.WorkflowId, err) } @@ -6593,18 +6376,18 @@ func runInit(ctx context.Context) { // Getting apps to see if we should initialize a test log.Printf("Getting remote workflow apps") - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (runInit): %s", err) } else if err == nil && len(workflowapps) > 0 { - //getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { - var allworkflowapps []WorkflowApp + //shuffle.GetAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { + var allworkflowapps []shuffle.WorkflowApp q := datastore.NewQuery("workflowapp") _, err := dbclient.GetAll(ctx, q, &allworkflowapps) if err == nil { for _, workflowapp := range allworkflowapps { if workflowapp.Edited == 0 { - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err == nil { log.Printf("Updating time for workflowapp %s:%s", workflowapp.Name, workflowapp.AppVersion) } @@ -6693,7 +6476,7 @@ func runInit(ctx context.Context) { if len(workflowLocation) > 0 { log.Printf("Downloading WORKFLOWS from %s if no workflows - EXTRA workflows", workflowLocation) q := datastore.NewQuery("workflow").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Error getting workflows: %s", err) @@ -6722,11 +6505,11 @@ func runInit(ctx context.Context) { log.Printf("[INFO] Finished INIT") } -func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { +func handleVerifyCloudsync(orgId string) (shuffle.SyncFeatures, error) { ctx := context.Background() - org, err := getOrg(ctx, orgId) + org, err := shuffle.GetOrg(ctx, orgId) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } //r.HandleFunc("/api/v1/getorgs", handleGetOrgs).Methods("GET", "OPTIONS") @@ -6742,26 +6525,26 @@ func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, org.SyncConfig.Apikey)) newresp, err := client.Do(req) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } respBody, err := ioutil.ReadAll(newresp.Body) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } responseData := retStruct{} err = json.Unmarshal(respBody, &responseData) if err != nil { - return SyncFeatures{}, err + return shuffle.SyncFeatures{}, err } if newresp.StatusCode != 200 { - return SyncFeatures{}, errors.New(fmt.Sprintf("Got status code %d when getting org remotely. Expected 200. Contact support.", newresp.StatusCode)) + return shuffle.SyncFeatures{}, errors.New(fmt.Sprintf("Got status code %d when getting org remotely. Expected 200. Contact support.", newresp.StatusCode)) } if !responseData.Success { - return SyncFeatures{}, errors.New(responseData.Reason) + return shuffle.SyncFeatures{}, errors.New(responseData.Reason) } return responseData.SyncFeatures, nil @@ -6769,7 +6552,7 @@ func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { // Actually stops syncing with cloud for an org. // Disables potential schedules, removes environments, breaks workflows etc. -func handleStopCloudSync(syncUrl string, org Org) error { +func handleStopCloudSync(syncUrl string, org shuffle.Org) error { if len(org.SyncConfig.Apikey) == 0 { return errors.New(fmt.Sprintf("Couldn't find any sync key to disable org %s", org.Id)) } @@ -6814,8 +6597,8 @@ func handleStopCloudSync(syncUrl string, org Org) error { ctx := context.Background() org.CloudSync = false - org.SyncFeatures = SyncFeatures{} - org.SyncConfig = SyncConfig{} + org.SyncFeatures = shuffle.SyncFeatures{} + org.SyncConfig = shuffle.SyncConfig{} err = setOrg(ctx, org, org.Id) if err != nil { @@ -6824,7 +6607,7 @@ func handleStopCloudSync(syncUrl string, org Org) error { return errors.New(newerror) } - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err != nil { @@ -6922,7 +6705,7 @@ func handleKeyValueCheck(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() - org, err := getOrg(ctx, tmpData.OrgId) + org, err := shuffle.GetOrg(ctx, tmpData.OrgId) if err != nil { log.Printf("[INFO] Organization doesn't exist: %s", err) resp.WriteHeader(401) @@ -7103,7 +6886,7 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in cloud setup: %s", err) resp.WriteHeader(401) @@ -7126,11 +6909,11 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { } type ReturnData struct { - Image string `json:"image" datastore:"image"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - OrgId string `json:"org_id" datastore:"org_id"` - Defaults Defaults `json:"defaults" datastore:"defaults"` + Image string `json:"image" datastore:"image"` + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description"` + OrgId string `json:"org_id" datastore:"org_id"` + Defaults shuffle.Defaults `json:"defaults" datastore:"defaults"` } var tmpData ReturnData @@ -7163,7 +6946,7 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - org, err := getOrg(ctx, tmpData.OrgId) + org, err := shuffle.GetOrg(ctx, tmpData.OrgId) if err != nil { log.Printf("Organization doesn't exist: %s", err) resp.WriteHeader(401) @@ -7228,7 +7011,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in cloud setup: %s", err) resp.WriteHeader(401) @@ -7251,9 +7034,9 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } type ReturnData struct { - Apikey string `datastore:"apikey"` - Organization Org `datastore:"organization"` - Disable bool `datastore:"disable"` + Apikey string `datastore:"apikey"` + Organization shuffle.Org `datastore:"organization"` + Disable bool `datastore:"disable"` } var tmpData ReturnData @@ -7266,7 +7049,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - org, err := getOrg(ctx, tmpData.Organization.Id) + org, err := shuffle.GetOrg(ctx, tmpData.Organization.Id) if err != nil { log.Printf("Organization doesn't exist: %s", err) resp.WriteHeader(401) @@ -7411,7 +7194,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { org.CloudSync = true org.SyncFeatures = responseData.SyncFeatures - org.SyncConfig = SyncConfig{ + org.SyncConfig = shuffle.SyncConfig{ Apikey: responseData.SessionKey, Interval: responseData.IntervalSeconds, } @@ -7440,7 +7223,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // 1. Find environment // 2. If cloud env found, enable it (un-archive) // 3. If it doesn't create it - var environments []Environment + var environments []shuffle.Environment q := datastore.NewQuery("Environments").Filter("org_id =", org.Id) _, err = dbclient.GetAll(ctx, q, &environments) if err == nil { @@ -7465,7 +7248,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { if !found { log.Printf("Env for cloud not found. Should add it!") - newEnv := Environment{ + newEnv := shuffle.Environment{ Name: "Cloud", Type: "cloud", Archived: false, @@ -7509,7 +7292,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // return // } // -// user, err := handleApiAuthentication(resp, request) +// user, err := shuffle.HandleApiAuthentication(resp, request) // if err != nil { // log.Printf("Api authentication failed in cloud setup: %s", err) // resp.WriteHeader(401) @@ -7584,7 +7367,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { // return // } // -// org, err := getOrg(ctx, tmpData.OrgId) +// org, err := shuffle.GetOrg(ctx, tmpData.OrgId) // if err != nil { // log.Printf("Organization doesn't exist: %s", err) // resp.WriteHeader(401) @@ -7684,12 +7467,18 @@ func initHandlers() { ctx := context.Background() log.Printf("Starting Shuffle backend - initializing database connection") - // option.WithoutAuthentication - + requestCache = cache.New(5*time.Minute, 10*time.Minute) dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) if err != nil { panic(fmt.Sprintf("DBclient error during init: %s", err)) } + + //dbclient, err := shuffle.GetDatastoreClient(ctx, gceProject) + //if err != nil { + // panic(fmt.Sprintf("Error setting datastore connector: %s", err)) + //} + + _ = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true) log.Printf("Finished Shuffle database init") go runInit(ctx) @@ -7815,12 +7604,12 @@ func initHandlers() { // PS: For cloud, this has to use cloud storage. // https://developer.box.com/reference/get-files-id-content/ // 1. Creating the "get file" option. Make it possible to run this in the frontend. - r.HandleFunc("/api/v1/files/{fileId}/content", handleGetFileContent).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/files/create", handleCreateFile).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") - r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") - r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/content", shuffle.HandleGetFileContent).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/create", shuffle.HandleCreateFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}/upload", shuffle.HandleUploadFile).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleGetFileMeta).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/files/{fileId}", shuffle.HandleDeleteFile).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/files", shuffle.HandleGetFiles).Methods("GET", "OPTIONS") // Trigger hmm r.HandleFunc("/api/v1/triggers/outlook/register", handleNewOutlookRegister).Methods("GET", "OPTIONS") diff --git a/backend/go-app/oauth2.go b/backend/go-app/oauth2.go index b00f6c7a..3c03ea13 100644 --- a/backend/go-app/oauth2.go +++ b/backend/go-app/oauth2.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -423,32 +425,32 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { } // Should also update the user - Userdata, err := getUser(ctx, senderUser) + Userdata, err := shuffle.GetUser(ctx, senderUser) if err != nil { log.Printf("[INFO] Username %s doesn't exist (oauth2): %s", trigger.Username, err) resp.WriteHeader(401) return } - Userdata.Authentication = append(Userdata.Authentication, UserAuth{ + Userdata.Authentication = append(Userdata.Authentication, shuffle.UserAuth{ Name: "Outlook", Description: "oauth2", Workflows: []string{trigger.WorkflowId}, Username: trigger.Username, - Fields: []UserAuthField{ - UserAuthField{ + Fields: []shuffle.UserAuthField{ + shuffle.UserAuthField{ Key: "trigger_id", Value: trigger.Id, }, - UserAuthField{ + shuffle.UserAuthField{ Key: "username", Value: trigger.Username, }, - UserAuthField{ + shuffle.UserAuthField{ Key: "code", Value: code, }, - UserAuthField{ + shuffle.UserAuthField{ Key: "type", Value: trigger.Type, }, @@ -456,7 +458,7 @@ func handleNewOutlookRegister(resp http.ResponseWriter, request *http.Request) { }) // Set apikey for the user if they don't have one - err = setUser(ctx, Userdata) + err = shuffle.SetUser(ctx, Userdata) if err != nil { log.Printf("Failed setting user data for %s: %s", Userdata.Username, err) resp.WriteHeader(401) @@ -636,7 +638,7 @@ func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -714,7 +716,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) + workflow, err := shuffle.GetWorkflow(ctx, workflowId) if err != nil { log.Printf("Failed getting the workflow locally (outlook sub): %s", err) resp.WriteHeader(401) @@ -722,7 +724,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in outlook deploy: %s", err) resp.WriteHeader(401) @@ -808,7 +810,7 @@ func createOutlookSub(resp http.ResponseWriter, request *http.Request) { // 10 * 5 = 50 seconds. That's waaay too much :( if runningEnvironment != "cloud" { - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -1147,7 +1149,7 @@ func handleOutlookCallback(resp http.ResponseWriter, request *http.Request) { Body: ioutil.NopCloser(bytes.NewReader(b)), } - workflow := Workflow{ + workflow := shuffle.Workflow{ ID: "", } @@ -1196,7 +1198,7 @@ func removeOutlookSubscription(outlookClient *http.Client, subscriptionId string // Remove AUTH // Remove function // Remove subscription -func handleOutlookSubRemoval(ctx context.Context, user User, workflowId, triggerId string) error { +func handleOutlookSubRemoval(ctx context.Context, user shuffle.User, workflowId, triggerId string) error { // 1. Get the auth for trigger // 2. Stop the subscription // 3. Remove the function @@ -1209,7 +1211,7 @@ func handleOutlookSubRemoval(ctx context.Context, user User, workflowId, trigger if runningEnvironment != "cloud" { log.Printf("[INFO] SHOULD STOP OUTLOOK SUB ONPREM SYNC WITH CLOUD for workflow ID %s", workflowId) - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("[INFO] Failed finding org %s during outlook removal: %s", org.Id, err) return err @@ -1289,7 +1291,7 @@ func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, workflowId) + workflow, err := shuffle.GetWorkflow(ctx, workflowId) if err != nil { log.Printf("Failed getting the workflow locally (delete outlook): %s", err) resp.WriteHeader(401) @@ -1297,7 +1299,7 @@ func handleDeleteOutlookSub(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in outlook deploy: %s", err) resp.WriteHeader(401) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 8af8055e..10b96f19 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1,6 +1,8 @@ package main import ( + "github.com/frikky/shuffle-shared" + "bytes" "context" "encoding/json" @@ -29,7 +31,7 @@ import ( schedulerpb "google.golang.org/genproto/googleapis/cloud/scheduler/v1" newscheduler "github.com/carlescere/scheduler" - "github.com/getkin/kin-openapi/openapi3" + "github.com/frikky/kin-openapi/openapi3" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/memfs" "github.com/go-git/go-git/v5" @@ -63,435 +65,416 @@ var scheduledOrgs = map[string]*newscheduler.Job{} // }, //} -type ExecutionRequest struct { - ExecutionId string `json:"execution_id,omitempty"` - ExecutionArgument string `json:"execution_argument,omitempty"` - ExecutionSource string `json:"execution_source,omitempty"` - WorkflowId string `json:"workflow_id,omitempty"` - Environments []string `json:"environments,omitempty"` - Authorization string `json:"authorization,omitempty"` - Status string `json:"status,omitempty"` - Start string `json:"start,omitempty"` - Type string `json:"type,omitempty"` -} - -type SyncFeatures struct { - Webhook SyncData `json:"webhook" datastore:"webhook"` - Schedules SyncData `json:"schedules" datastore:"schedules"` - UserInput SyncData `json:"user_input" datastore:"user_input"` - SendMail SyncData `json:"send_mail" datastore:"send_mail"` - SendSms SyncData `json:"send_sms" datastore:"send_sms"` - Updates SyncData `json:"updates" datastore:"updates"` - Notifications SyncData `json:"notifications" datastore:"notifications"` - EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` - AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` - WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` - Apps SyncData `json:"apps" datastore:"apps"` - Workflows SyncData `json:"workflows" datastore:"workflows"` - Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` - Authentication SyncData `json:"authentication" datastore:"authentication"` - Schedule SyncData `json:"schedule" datastore:"schedule"` -} - -type SyncData struct { - Active bool `json:"active" datastore:"active"` - Type string `json:"type,omitempty" datastore:"type"` - Name string `json:"name,omitempty" datastore:"name"` - Description string `json:"description,omitempty" datastore:"description"` - Limit int64 `json:"limit,omitempty" datastore:"limit"` - StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` - EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` - DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` -} - -type SyncConfig struct { - Interval int64 `json:"interval" datastore:"interval"` - Apikey string `json:"api_key" datastore:"api_key"` -} - -// Role is just used for feedback for a user -type Org struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Image string `json:"image" datastore:"image,noindex"` - Id string `json:"id" datastore:"id"` - Org string `json:"org" datastore:"org"` - Users []User `json:"users" datastore:"users"` - Role string `json:"role" datastore:"role"` - Roles []string `json:"roles" datastore:"roles"` - CloudSync bool `json:"cloud_sync" datastore:"CloudSync"` - SyncConfig SyncConfig `json:"sync_config" datastore:"sync_config"` - SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` - Subscriptions []PaymentSubscription `json:"subscriptions" datastore:"subscriptions"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - Defaults Defaults `json:"defaults" datastore:"defaults"` -} - -type PaymentSubscription struct { - Active bool `json:"active" datastore:"active"` - Startdate int64 `json:"startdate" datastore:"startdate"` - CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` - Enddate int64 `json:"enddate" datastore:"enddate"` - Name string `json:"name" datastore:"name"` - Recurrence string `json:"recurrence" datastore:"recurrence"` - Reference string `json:"reference" datastore:"reference"` - Level string `json:"level" datastore:"level"` - Amount string `json:"amount" datastore:"amount"` - Currency string `json:"currency" datastore:"currency"` -} - -type Defaults struct { - AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` - AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` - WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` - WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` -} - -type AppAuthenticationStorage struct { - Active bool `json:"active" datastore:"active"` - Label string `json:"label" datastore:"label"` - Id string `json:"id" datastore:"id"` - App WorkflowApp `json:"app" datastore:"app,noindex"` - Fields []AuthenticationStore `json:"fields" datastore:"fields"` - Usage []AuthenticationUsage `json:"usage" datastore:"usage"` - WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` - NodeCount int64 `json:"node_count" datastore:"node_count"` - OrgId string `json:"org_id" datastore:"org_id"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - Defined bool `json:"defined" datastore:"defined"` -} - -type AuthenticationUsage struct { - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - Nodes []string `json:"nodes" datastore:"nodes"` -} - -// An app inside Shuffle -// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation -type WorkflowApp struct { - Name string `json:"name" yaml:"name" required:true datastore:"name"` - IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` - ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` - Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` - AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` - SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` - Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` - Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` - Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` - Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` - Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` - Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` - Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` - Owner string `json:"owner" datastore:"owner" yaml:"owner"` - Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps - PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` - Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - ContactInfo struct { - Name string `json:"name" datastore:"name" yaml:"name"` - Url string `json:"url" datastore:"url" yaml:"url"` - } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` - ReferenceInfo struct { - DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` - GithubUrl string `json:"github_url" datastore:"github_url"` - } - FolderMount struct { - FolderMount bool `json:"folder_mount" datastore:"folder_mount"` - SourceFolder string `json:"source_folder" datastore:"source_folder"` - DestinationFolder string `json:"destination_folder" datastore:"destination_folder"` - } - Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` - Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` -} - -type WorkflowAppActionParameter struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example,noindex" yaml:"example"` - Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Options []string `json:"options" datastore:"options" yaml:"options"` - ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` - Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` - Required bool `json:"required" datastore:"required" yaml:"required"` - Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` - ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` - UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` -} - -type Valuereplace struct { - Key string `json:"key" datastore:"key" yaml:"key"` - Value string `json:"value" datastore:"value" yaml:"value"` -} - -type SchemaDefinition struct { - Type string `json:"type" datastore:"type"` -} - -type WorkflowAppAction struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Name string `json:"name" datastore:"name"` - Label string `json:"label" datastore:"label"` - NodeType string `json:"node_type" datastore:"node_type"` - Environment string `json:"environment" datastore:"environment"` - Sharing bool `json:"sharing" datastore:"sharing"` - PrivateID string `json:"private_id" datastore:"private_id"` - AppID string `json:"app_id" datastore:"app_id"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` - Tested bool `json:"tested" datastore:"tested" yaml:"tested"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` - ExecutionVariable struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variable" datastore:"execution_variables"` - Returns struct { - Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` - Example string `json:"example" datastore:"example,noindex" yaml:"example"` - ID string `json:"id" datastore:"id" yaml:"id,omitempty"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - } `json:"returns" datastore:"returns"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example,noindex" datastore:"example" yaml:"example"` - AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` -} +//type ExecutionRequest struct { +// ExecutionId string `json:"execution_id,omitempty"` +// ExecutionArgument string `json:"execution_argument,omitempty"` +// ExecutionSource string `json:"execution_source,omitempty"` +// WorkflowId string `json:"workflow_id,omitempty"` +// Environments []string `json:"environments,omitempty"` +// Authorization string `json:"authorization,omitempty"` +// Status string `json:"status,omitempty"` +// Start string `json:"start,omitempty"` +// Type string `json:"type,omitempty"` +//} +// +//type SyncFeatures struct { +// Webhook SyncData `json:"webhook" datastore:"webhook"` +// Schedules SyncData `json:"schedules" datastore:"schedules"` +// UserInput SyncData `json:"user_input" datastore:"user_input"` +// SendMail SyncData `json:"send_mail" datastore:"send_mail"` +// SendSms SyncData `json:"send_sms" datastore:"send_sms"` +// Updates SyncData `json:"updates" datastore:"updates"` +// Notifications SyncData `json:"notifications" datastore:"notifications"` +// EmailTrigger SyncData `json:"email_trigger" datastore:"email_trigger"` +// AppExecutions SyncData `json:"app_executions" datastore:"app_executions"` +// WorkflowExecutions SyncData `json:"workflow_executions" datastore:"workflow_executions"` +// Apps SyncData `json:"apps" datastore:"apps"` +// Workflows SyncData `json:"workflows" datastore:"workflows"` +// Autocomplete SyncData `json:"autocomplete" datastore:"autocomplete"` +// Authentication SyncData `json:"authentication" datastore:"authentication"` +// Schedule SyncData `json:"schedule" datastore:"schedule"` +//} +// +//type SyncData struct { +// Active bool `json:"active" datastore:"active"` +// Type string `json:"type,omitempty" datastore:"type"` +// Name string `json:"name,omitempty" datastore:"name"` +// Description string `json:"description,omitempty" datastore:"description"` +// Limit int64 `json:"limit,omitempty" datastore:"limit"` +// StartDate int64 `json:"start_date,omitempty" datastore:"start_date"` +// EndDate int64 `json:"end_date,omitempty" datastore:"end_date"` +// DataCollection int64 `json:"data_collection,omitempty" datastore:"data_collection"` +//} +// +//type SyncConfig struct { +// Interval int64 `json:"interval" datastore:"interval"` +// Apikey string `json:"api_key" datastore:"api_key"` +//} +// +//type PaymentSubscription struct { +// Active bool `json:"active" datastore:"active"` +// Startdate int64 `json:"startdate" datastore:"startdate"` +// CancellationDate int64 `json:"cancellationdate" datastore:"cancellationdate"` +// Enddate int64 `json:"enddate" datastore:"enddate"` +// Name string `json:"name" datastore:"name"` +// Recurrence string `json:"recurrence" datastore:"recurrence"` +// Reference string `json:"reference" datastore:"reference"` +// Level string `json:"level" datastore:"level"` +// Amount string `json:"amount" datastore:"amount"` +// Currency string `json:"currency" datastore:"currency"` +//} +// +//type Defaults struct { +// AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` +// AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` +// WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` +// WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` +//} +// +//type AppAuthenticationStorage struct { +// Active bool `json:"active" datastore:"active"` +// Label string `json:"label" datastore:"label"` +// Id string `json:"id" datastore:"id"` +// App WorkflowApp `json:"app" datastore:"app,noindex"` +// Fields []AuthenticationStore `json:"fields" datastore:"fields"` +// Usage []AuthenticationUsage `json:"usage" datastore:"usage"` +// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"` +// NodeCount int64 `json:"node_count" datastore:"node_count"` +// OrgId string `json:"org_id" datastore:"org_id"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// Defined bool `json:"defined" datastore:"defined"` +//} +// +//type AuthenticationUsage struct { +// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` +// Nodes []string `json:"nodes" datastore:"nodes"` +//} +// +//// An app inside Shuffle +//// Source string `json:"source" datastore:"soure" yaml:"source"` - downloadlocation +//type WorkflowApp struct { +// Name string `json:"name" yaml:"name" required:true datastore:"name"` +// IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"` +// ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"` +// Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"` +// AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"` +// SharingConfig string `json:"sharing_config" yaml:"sharing_config" datastore:"sharing_config"` +// Generated bool `json:"generated" yaml:"generated" required:false datastore:"generated"` +// Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` +// Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` +// Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` +// Invalid bool `json:"invalid" yaml:"invalid" required:false datastore:"invalid"` +// Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` +// Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` +// Owner string `json:"owner" datastore:"owner" yaml:"owner"` +// Hash string `json:"hash" datastore:"hash" yaml:"hash"` // api.yaml+dockerfile+src/app.py for apps +// PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` +// Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` +// Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` +// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` +// ContactInfo struct { +// Name string `json:"name" datastore:"name" yaml:"name"` +// Url string `json:"url" datastore:"url" yaml:"url"` +// } `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false` +// ReferenceInfo struct { +// DocumentationUrl string `json:"documentation_url" datastore:"documentation_url"` +// GithubUrl string `json:"github_url" datastore:"github_url"` +// } +// FolderMount struct { +// FolderMount bool `json:"folder_mount" datastore:"folder_mount"` +// SourceFolder string `json:"source_folder" datastore:"source_folder"` +// DestinationFolder string `json:"destination_folder" datastore:"destination_folder"` +// } +// Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` +// Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` +// Tags []string `json:"tags" yaml:"tags" required:false datastore:"activated"` +// Categories []string `json:"categories" yaml:"categories" required:false datastore:"categories"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` +//} +// +//type WorkflowAppActionParameter struct { +// Description string `json:"description" datastore:"description,noindex" yaml:"description"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Name string `json:"name" datastore:"name" yaml:"name"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// Value string `json:"value" datastore:"value,noindex" yaml:"value,omitempty"` +// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` +// Options []string `json:"options" datastore:"options" yaml:"options"` +// ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"` +// Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"` +// Required bool `json:"required" datastore:"required" yaml:"required"` +// Configuration bool `json:"configuration" datastore:"configuration" yaml:"configuration"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// SkipMulticheck bool `json:"skip_multicheck" datastore:"skip_multicheck" yaml:"skip_multicheck"` +// ValueReplace []Valuereplace `json:"value_replace" datastore:"value_replace,noindex" yaml:"value_replace,omitempty"` +// UniqueToggled bool `json:"unique_toggled" datastore:"unique_toggled" yaml:"unique_toggled"` +//} +// +//type Valuereplace struct { +// Key string `json:"key" datastore:"key" yaml:"key"` +// Value string `json:"value" datastore:"value" yaml:"value"` +//} +// +//type SchemaDefinition struct { +// Type string `json:"type" datastore:"type"` +//} +// +//type WorkflowAppAction struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Name string `json:"name" datastore:"name"` +// Label string `json:"label" datastore:"label"` +// NodeType string `json:"node_type" datastore:"node_type"` +// Environment string `json:"environment" datastore:"environment"` +// Sharing bool `json:"sharing" datastore:"sharing"` +// PrivateID string `json:"private_id" datastore:"private_id"` +// AppID string `json:"app_id" datastore:"app_id"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Authentication []AuthenticationStore `json:"authentication" datastore:"authentication,noindex" yaml:"authentication,omitempty"` +// Tested bool `json:"tested" datastore:"tested" yaml:"tested"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"` +// ExecutionVariable struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variable" datastore:"execution_variables"` +// Returns struct { +// Description string `json:"description" datastore:"returns" yaml:"description,omitempty"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// ID string `json:"id" datastore:"id" yaml:"id,omitempty"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// } `json:"returns" datastore:"returns"` +// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` +// Example string `json:"example,noindex" datastore:"example" yaml:"example"` +// AuthNotRequired bool `json:"auth_not_required" datastore:"auth_not_required" yaml:"auth_not_required"` +//} // FIXME: Generate a callback authentication ID? // FIXME: Add org check .. -type WorkflowExecution struct { - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Start string `json:"start" datastore:"start"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` - ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` - ExecutionOrg string `json:"execution_org" datastore:"execution_org"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - LastNode string `json:"last_node" datastore:"last_node"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - ProjectId string `json:"project_id" datastore:"project_id"` - Locations []string `json:"locations" datastore:"locations"` - Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` - OrgId string `json:"org_id" datastore:"org_id"` -} +//type WorkflowExecution struct { +// Type string `json:"type" datastore:"type"` +// Status string `json:"status" datastore:"status"` +// Start string `json:"start" datastore:"start"` +// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` +// ExecutionId string `json:"execution_id" datastore:"execution_id"` +// ExecutionSource string `json:"execution_source" datastore:"execution_source"` +// ExecutionParent string `json:"execution_parent" datastore:"execution_parent"` +// ExecutionOrg string `json:"execution_org" datastore:"execution_org"` +// WorkflowId string `json:"workflow_id" datastore:"workflow_id"` +// LastNode string `json:"last_node" datastore:"last_node"` +// Authorization string `json:"authorization" datastore:"authorization"` +// Result string `json:"result" datastore:"result,noindex"` +// StartedAt int64 `json:"started_at" datastore:"started_at"` +// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` +// ProjectId string `json:"project_id" datastore:"project_id"` +// Locations []string `json:"locations" datastore:"locations"` +// Workflow Workflow `json:"workflow" datastore:"workflow,noindex"` +// Results []ActionResult `json:"results" datastore:"results,noindex"` +// ExecutionVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variables,omitempty" datastore:"execution_variables,omitempty"` +// OrgId string `json:"org_id" datastore:"org_id"` +//} // This is for the nodes in a workflow, NOT the app action itself. -type Action struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppID string `json:"app_id" datastore:"app_id"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` - Sharing bool `json:"sharing,omitempty" datastore:"sharing"` - PrivateID string `json:"private_id,omitempty" datastore:"private_id"` - Label string `json:"label,omitempty" datastore:"label"` - SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment,omitempty" datastore:"environment"` - Name string `json:"name" datastore:"name"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - ExecutionVariable struct { - Description string `json:"description,omitempty" datastore:"description,noindex"` - ID string `json:"id,omitempty" datastore:"id"` - Name string `json:"name,omitempty" datastore:"name"` - Value string `json:"value,omitempty" datastore:"value,noindex"` - } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` - Position struct { - X float64 `json:"x,omitempty" datastore:"x"` - Y float64 `json:"y,omitempty" datastore:"y"` - } `json:"position,omitempty"` - Priority int `json:"priority,omitempty" datastore:"priority"` - AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` - Example string `json:"example,omitempty" datastore:"example,noindex"` - AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` - Category string `json:"category" datastore:"category"` -} +//type Action struct { +// AppName string `json:"app_name" datastore:"app_name"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// AppID string `json:"app_id" datastore:"app_id"` +// Errors []string `json:"errors" datastore:"errors"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// IsStartNode bool `json:"isStartNode,omitempty" datastore:"isStartNode"` +// Sharing bool `json:"sharing,omitempty" datastore:"sharing"` +// PrivateID string `json:"private_id,omitempty" datastore:"private_id"` +// Label string `json:"label,omitempty" datastore:"label"` +// SmallImage string `json:"small_image,omitempty" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image,omitempty" datastore:"large_image,noindex" yaml:"large_image" required:false` +// Environment string `json:"environment,omitempty" datastore:"environment"` +// Name string `json:"name" datastore:"name"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` +// ExecutionVariable struct { +// Description string `json:"description,omitempty" datastore:"description,noindex"` +// ID string `json:"id,omitempty" datastore:"id"` +// Name string `json:"name,omitempty" datastore:"name"` +// Value string `json:"value,omitempty" datastore:"value,noindex"` +// } `json:"execution_variable,omitempty" datastore:"execution_variable,omitempty"` +// Position struct { +// X float64 `json:"x,omitempty" datastore:"x"` +// Y float64 `json:"y,omitempty" datastore:"y"` +// } `json:"position,omitempty"` +// Priority int `json:"priority,omitempty" datastore:"priority"` +// AuthenticationId string `json:"authentication_id" datastore:"authentication_id"` +// Example string `json:"example,omitempty" datastore:"example,noindex"` +// AuthNotRequired bool `json:"auth_not_required,omitempty" datastore:"auth_not_required" yaml:"auth_not_required"` +// Category string `json:"category" datastore:"category"` +//} +// +//// Added environment for location to execute +//type Trigger struct { +// AppName string `json:"app_name" datastore:"app_name"` +// Description string `json:"description" datastore:"description,noindex"` +// LongDescription string `json:"long_description" datastore:"long_description"` +// Status string `json:"status" datastore:"status"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// Errors []string `json:"errors" datastore:"errors"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` +// Label string `json:"label" datastore:"label"` +// SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` +// LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` +// Environment string `json:"environment" datastore:"environment"` +// TriggerType string `json:"trigger_type" datastore:"trigger_type"` +// Name string `json:"name" datastore:"name"` +// Tags []string `json:"tags" datastore:"tags" yaml:"tags"` +// Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` +// Position struct { +// X float64 `json:"x" datastore:"x"` +// Y float64 `json:"y" datastore:"y"` +// } `json:"position"` +// Priority int `json:"priority" datastore:"priority"` +//} +// +//type Branch struct { +// DestinationID string `json:"destination_id" datastore:"destination_id"` +// ID string `json:"id" datastore:"id"` +// SourceID string `json:"source_id" datastore:"source_id"` +// Label string `json:"label" datastore:"label"` +// HasError bool `json:"has_errors" datastore: "has_errors"` +// Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` +//} +// +//// Same format for a lot of stuff +//type Condition struct { +// Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` +// Source WorkflowAppActionParameter `json:"source" datastore:"source"` +// Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` +//} +// +//type Schedule struct { +// Name string `json:"name" datastore:"name"` +// Frequency string `json:"frequency" datastore:"frequency"` +// ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` +// Id string `json:"id" datastore:"id"` +// OrgId string `json:"org_id" datastore:"org_id"` +// Environment string `json:"environment" datastore:"environment"` +//} -// Added environment for location to execute -type Trigger struct { - AppName string `json:"app_name" datastore:"app_name"` - Description string `json:"description" datastore:"description,noindex"` - LongDescription string `json:"long_description" datastore:"long_description"` - Status string `json:"status" datastore:"status"` - AppVersion string `json:"app_version" datastore:"app_version"` - Errors []string `json:"errors" datastore:"errors"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - IsStartNode bool `json:"isStartNode" datastore:"isStartNode"` - Label string `json:"label" datastore:"label"` - SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` - LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` - Environment string `json:"environment" datastore:"environment"` - TriggerType string `json:"trigger_type" datastore:"trigger_type"` - Name string `json:"name" datastore:"name"` - Tags []string `json:"tags" datastore:"tags" yaml:"tags"` - Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters,noindex"` - Position struct { - X float64 `json:"x" datastore:"x"` - Y float64 `json:"y" datastore:"y"` - } `json:"position"` - Priority int `json:"priority" datastore:"priority"` -} +//type Workflow struct { +// Actions []Action `json:"actions" datastore:"actions,noindex"` +// Branches []Branch `json:"branches" datastore:"branches,noindex"` +// Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` +// Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` +// Configuration struct { +// ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` +// StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` +// } `json:"configuration,omitempty" datastore:"configuration"` +// Created int64 `json:"created" datastore:"created"` +// Edited int64 `json:"edited" datastore:"edited"` +// LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` +// Errors []string `json:"errors,omitempty" datastore:"errors"` +// Tags []string `json:"tags,omitempty" datastore:"tags"` +// ID string `json:"id" datastore:"id"` +// IsValid bool `json:"is_valid" datastore:"is_valid"` +// Name string `json:"name" datastore:"name"` +// Description string `json:"description" datastore:"description,noindex"` +// Start string `json:"start" datastore:"start"` +// Owner string `json:"owner" datastore:"owner"` +// Sharing string `json:"sharing" datastore:"sharing"` +// Org []Org `json:"org,omitempty" datastore:"org"` +// ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` +// OrgId string `json:"org_id,omitempty" datastore:"org_id"` +// WorkflowVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"workflow_variables" datastore:"workflow_variables"` +// ExecutionVariables []struct { +// Description string `json:"description" datastore:"description,noindex"` +// ID string `json:"id" datastore:"id"` +// Name string `json:"name" datastore:"name"` +// Value string `json:"value" datastore:"value,noindex"` +// } `json:"execution_variables,omitempty" datastore:"execution_variables"` +// ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` +// PreviouslySaved bool `json:"previously_saved" datastore:"first_save"` +// Categories Categories `json:"categories" datastore:"categories"` +// ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` +//} -type Branch struct { - DestinationID string `json:"destination_id" datastore:"destination_id"` - ID string `json:"id" datastore:"id"` - SourceID string `json:"source_id" datastore:"source_id"` - Label string `json:"label" datastore:"label"` - HasError bool `json:"has_errors" datastore: "has_errors"` - Conditions []Condition `json:"conditions" datastore: "conditions,noindex"` -} +//type Category struct { +// Name string `json:"name" datastore:"name"` +// Description string `json:"description" datastore:"description"` +// Count int64 `json:"count" datastore:"count"` +//} +// +//type Categories struct { +// SIEM Category `json:"siem" datastore:"siem"` +// Communication Category `json:"communication" datastore:"communication"` +// Assets Category `json:"assets" datastore:"assets"` +// Cases Category `json:"cases" datastore:"cases"` +// Network Category `json:"network" datastore:"network"` +// Intel Category `json:"intel" datastore:"intel"` +// EDR Category `json:"edr" datastore:"edr"` +// Other Category `json:"other" datastore:"other"` +//} -// Same format for a lot of stuff -type Condition struct { - Condition WorkflowAppActionParameter `json:"condition" datastore:"condition"` - Source WorkflowAppActionParameter `json:"source" datastore:"source"` - Destination WorkflowAppActionParameter `json:"destination" datastore:"destination"` -} - -type Schedule struct { - Name string `json:"name" datastore:"name"` - Frequency string `json:"frequency" datastore:"frequency"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - Id string `json:"id" datastore:"id"` - OrgId string `json:"org_id" datastore:"org_id"` - Environment string `json:"environment" datastore:"environment"` -} - -type Workflow struct { - Actions []Action `json:"actions" datastore:"actions,noindex"` - Branches []Branch `json:"branches" datastore:"branches,noindex"` - Triggers []Trigger `json:"triggers" datastore:"triggers,noindex"` - Schedules []Schedule `json:"schedules" datastore:"schedules,noindex"` - Configuration struct { - ExitOnError bool `json:"exit_on_error" datastore:"exit_on_error"` - StartFromTop bool `json:"start_from_top" datastore:"start_from_top"` - } `json:"configuration,omitempty" datastore:"configuration"` - Created int64 `json:"created" datastore:"created"` - Edited int64 `json:"edited" datastore:"edited"` - LastRuntime int64 `json:"last_runtime" datastore:"last_runtime"` - Errors []string `json:"errors,omitempty" datastore:"errors"` - Tags []string `json:"tags,omitempty" datastore:"tags"` - ID string `json:"id" datastore:"id"` - IsValid bool `json:"is_valid" datastore:"is_valid"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description,noindex"` - Start string `json:"start" datastore:"start"` - Owner string `json:"owner" datastore:"owner"` - Sharing string `json:"sharing" datastore:"sharing"` - Org []Org `json:"org,omitempty" datastore:"org"` - ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"` - OrgId string `json:"org_id,omitempty" datastore:"org_id"` - WorkflowVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"workflow_variables" datastore:"workflow_variables"` - ExecutionVariables []struct { - Description string `json:"description" datastore:"description,noindex"` - ID string `json:"id" datastore:"id"` - Name string `json:"name" datastore:"name"` - Value string `json:"value" datastore:"value,noindex"` - } `json:"execution_variables,omitempty" datastore:"execution_variables"` - ExecutionEnvironment string `json:"execution_environment" datastore:"execution_environment"` - PreviouslySaved bool `json:"previously_saved" datastore:"first_save"` - Categories Categories `json:"categories" datastore:"categories"` - ExampleArgument string `json:"example_argument" datastore:"example_argument,noindex"` -} - -type Category struct { - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - Count int64 `json:"count" datastore:"count"` -} - -type Categories struct { - SIEM Category `json:"siem" datastore:"siem"` - Communication Category `json:"communication" datastore:"communication"` - Assets Category `json:"assets" datastore:"assets"` - Cases Category `json:"cases" datastore:"cases"` - Network Category `json:"network" datastore:"network"` - Intel Category `json:"intel" datastore:"intel"` - EDR Category `json:"edr" datastore:"edr"` - Other Category `json:"other" datastore:"other"` -} - -type ActionResult struct { - Action Action `json:"action" datastore:"action,noindex"` - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Authorization string `json:"authorization" datastore:"authorization"` - Result string `json:"result" datastore:"result,noindex"` - StartedAt int64 `json:"started_at" datastore:"started_at"` - CompletedAt int64 `json:"completed_at" datastore:"completed_at"` - Status string `json:"status" datastore:"status"` -} - -type Authentication struct { - Required bool `json:"required" datastore:"required" yaml:"required" ` - Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` -} - -type AuthenticationParams struct { - Description string `json:"description" datastore:"description,noindex" yaml:"description"` - ID string `json:"id" datastore:"id" yaml:"id"` - Name string `json:"name" datastore:"name" yaml:"name"` - Example string `json:"example" datastore:"example,noindex" yaml:"example"` - Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` - Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` - Required bool `json:"required" datastore:"required" yaml:"required"` - In string `json:"in" datastore:"in" yaml:"in"` - Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` - Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated -} - -type AuthenticationStore struct { - Key string `json:"key" datastore:"key"` - Value string `json:"value" datastore:"value,noindex"` -} - -type ExecutionRequestWrapper struct { - Data []ExecutionRequest `json:"data"` -} - -type AppExecutionExample struct { - AppName string `json:"app_name" datastore:"app_name"` - AppVersion string `json:"app_version" datastore:"app_version"` - AppAction string `json:"app_action" datastore:"app_action"` - AppId string `json:"app_id" datastore:"app_id"` - ExampleId string `json:"example_id" datastore:"example_id"` - SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` - FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` -} +//type ActionResult struct { +// Action Action `json:"action" datastore:"action,noindex"` +// ExecutionId string `json:"execution_id" datastore:"execution_id"` +// Authorization string `json:"authorization" datastore:"authorization"` +// Result string `json:"result" datastore:"result,noindex"` +// StartedAt int64 `json:"started_at" datastore:"started_at"` +// CompletedAt int64 `json:"completed_at" datastore:"completed_at"` +// Status string `json:"status" datastore:"status"` +//} +// +//type Authentication struct { +// Required bool `json:"required" datastore:"required" yaml:"required" ` +// Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"` +//} +// +//type AuthenticationParams struct { +// Description string `json:"description" datastore:"description,noindex" yaml:"description"` +// ID string `json:"id" datastore:"id" yaml:"id"` +// Name string `json:"name" datastore:"name" yaml:"name"` +// Example string `json:"example" datastore:"example,noindex" yaml:"example"` +// Value string `json:"value,omitempty" datastore:"value,noindex" yaml:"value"` +// Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"` +// Required bool `json:"required" datastore:"required" yaml:"required"` +// In string `json:"in" datastore:"in" yaml:"in"` +// Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"` +// Scheme string `json:"scheme" datastore:"scheme" yaml:"scheme"` // Deprecated +//} +// +//type AuthenticationStore struct { +// Key string `json:"key" datastore:"key"` +// Value string `json:"value" datastore:"value,noindex"` +//} +// +//type ExecutionRequestWrapper struct { +// Data []ExecutionRequest `json:"data"` +//} +// +//type AppExecutionExample struct { +// AppName string `json:"app_name" datastore:"app_name"` +// AppVersion string `json:"app_version" datastore:"app_version"` +// AppAction string `json:"app_action" datastore:"app_action"` +// AppId string `json:"app_id" datastore:"app_id"` +// ExampleId string `json:"example_id" datastore:"example_id"` +// SuccessExamples []string `json:"success_examples" datastore:"success_examples,noindex"` +// FailureExamples []string `json:"failure_examples" datastore:"failure_examples,noindex"` +//} // This might be... a bit off, but that's fine :) // This might also be stupid, as we want timelines and such @@ -551,7 +534,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i return nil } -func setWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, env string) error { +func setWorkflowQueue(ctx context.Context, executionRequest shuffle.ExecutionRequest, env string) error { orgKey := fmt.Sprintf("workflowqueue-%s", env) key := datastore.NameKey(orgKey, executionRequest.ExecutionId, nil) @@ -577,16 +560,16 @@ func setWorkflowQueue(ctx context.Context, executionRequest ExecutionRequest, en // return nil //} -func getWorkflowQueue(ctx context.Context, id string) (ExecutionRequestWrapper, error) { +func getWorkflowQueue(ctx context.Context, id string) (shuffle.ExecutionRequestWrapper, error) { orgId := fmt.Sprintf("workflowqueue-%s", id) q := datastore.NewQuery(orgId).Limit(10) - executions := []ExecutionRequest{} + executions := []shuffle.ExecutionRequest{} _, err := dbclient.GetAll(ctx, q, &executions) if err != nil { - return ExecutionRequestWrapper{}, err + return shuffle.ExecutionRequestWrapper{}, err } - return ExecutionRequestWrapper{Data: executions}, nil + return shuffle.ExecutionRequestWrapper{Data: executions}, nil //key := datastore.NameKey("workflowqueue", id, nil) //executions := ExecutionRequestWrapper{} @@ -660,7 +643,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode Body: ioutil.NopCloser(strings.NewReader(bodyWrapper)), } - _, _, err := handleExecution(workflowId, Workflow{ExecutingOrg: Org{Id: orgId}}, request) + _, _, err := handleExecution(workflowId, shuffle.Workflow{ExecutingOrg: shuffle.Org{Id: orgId}}, request) if err != nil { log.Printf("Failed to execute %s: %s", workflowId, err) } @@ -747,7 +730,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque // Getting from the request //log.Println(string(body)) - var removeExecutionRequests ExecutionRequestWrapper + var removeExecutionRequests shuffle.ExecutionRequestWrapper err = json.Unmarshal(body, &removeExecutionRequests) if err != nil { log.Printf("Failed executionrequest in queue unmarshaling: %s", err) @@ -831,7 +814,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } if len(executionRequests.Data) == 0 { - executionRequests.Data = []ExecutionRequest{} + executionRequests.Data = []shuffle.ExecutionRequest{} } else { log.Printf("[INFO] Executionrequests (%s): %d", id, len(executionRequests.Data)) } @@ -861,7 +844,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("Failed ActionResult unmarshaling: %s", err) @@ -871,7 +854,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { //log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) @@ -901,7 +884,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { // Finds the child nodes of a node in execution and returns them // Used if e.g. a node in a branch is exited, and all children have to be stopped -func findChildNodes(workflowExecution WorkflowExecution, nodeId string) []string { +func findChildNodes(workflowExecution shuffle.WorkflowExecution, nodeId string) []string { //log.Printf("\nNODE TO FIX: %s\n\n", nodeId) allChildren := []string{nodeId} @@ -955,14 +938,14 @@ func validateNewWorkerExecution(body []byte) error { //} ctx := context.Background() - var execution WorkflowExecution + var execution shuffle.WorkflowExecution err := json.Unmarshal(body, &execution) if err != nil { log.Printf("[WARNING] Failed execution unmarshaling: %s", err) return err } - baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId) + baseExecution, err := shuffle.GetWorkflowExecution(ctx, execution.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err) return err @@ -1044,7 +1027,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { //log.Printf("[WARNING] Handling other execution variant: %s", err) } - var actionResult ActionResult + var actionResult shuffle.ActionResult err = json.Unmarshal(body, &actionResult) if err != nil { log.Printf("Failed ActionResult unmarshaling: %s", err) @@ -1060,7 +1043,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { // IF FAIL: Set executionstatus: abort or cancel ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) @@ -1099,7 +1082,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { if actionResult.Status == "WAITING" && actionResult.Action.AppName == "User Input" { log.Printf("SHOULD WAIT A BIT AND RUN CLOUD STUFF WITH USER INPUT! WAITING!") - var trigger Trigger + var trigger shuffle.Trigger err = json.Unmarshal([]byte(actionResult.Result), &trigger) if err != nil { log.Printf("Failed unmarshaling actionresult for user input: %s", err) @@ -1152,9 +1135,9 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { } // Will make sure transactions are always ran for an execution. This is recursive if it fails. Allowed to fail up to 5 times -func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult ActionResult, resp http.ResponseWriter) { +func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workflowExecutionId string, actionResult shuffle.ActionResult, resp http.ResponseWriter) { // Should start a tx for the execution here - workflowExecution, err := getWorkflowExecution(ctx, workflowExecutionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, workflowExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution cache: %s", err) resp.WriteHeader(401) @@ -1192,7 +1175,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { dbSave = true - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} childNodes := []string{} if workflowExecution.Workflow.Configuration.ExitOnError { log.Printf("[WARNING] Actionresult is %s for node %s in %s. Should set workflowExecution and exit all running functions", actionResult.Status, actionResult.Action.ID, workflowExecution.ExecutionId) @@ -1214,7 +1197,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // 1. Find the action itself // 2. Create an actionresult - curAction := Action{ID: ""} + curAction := shuffle.Action{ID: ""} for _, action := range workflowExecution.Workflow.Actions { if action.ID == nodeId { curAction = action @@ -1258,7 +1241,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if !skipNodeAdd { - newAction := Action{ + newAction := shuffle.Action{ AppName: curAction.AppName, AppVersion: curAction.AppVersion, Label: curAction.Label, @@ -1266,7 +1249,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl ID: curAction.ID, } - newResult := ActionResult{ + newResult := shuffle.ActionResult{ Action: newAction, ExecutionId: actionResult.ExecutionId, Authorization: actionResult.Authorization, @@ -1474,7 +1457,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } // FIXME - why isn't this how it works otherwise, wtf? - //workflow, err := getWorkflow(workflowExecution.Workflow.ID) + //workflow, err := shuffle.GetWorkflow(workflowExecution.Workflow.ID) //newActions := []Action{} //for _, action := range workflowExecution.Workflow.Actions { // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) @@ -1489,7 +1472,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Result string `json:"result" datastore:"result,noindex"` // Arbitrary reduction size maxSize := 500000 - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} for _, item := range workflowExecution.Results { if len(item.Result) > maxSize { item.Result = "[ERROR] Result too large to handle (https://github.com/frikky/shuffle/issues/171)" @@ -1506,7 +1489,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Handled using cachhing, so actually pretty fast cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) if len(parsedValue.Results) > 0 && len(parsedValue.Results) != resultLength { setExecution = false if attempts > 5 { @@ -1587,10 +1570,10 @@ func JSONCheck(str string) bool { return json.Unmarshal([]byte(str), &jsonStr) == nil } -func handleExecutionStatistics(execution WorkflowExecution) { +func handleExecutionStatistics(execution shuffle.WorkflowExecution) { // FIXME: CLEAN UP THE JSON THAT'S SAVED. // https://github.com/frikky/Shuffle/issues/172 - appResults := []AppExecutionExample{} + appResults := []shuffle.AppExecutionExample{} for _, result := range execution.Results { resultCheck := JSONCheck(result.Result) if !resultCheck { @@ -1625,7 +1608,7 @@ func handleExecutionStatistics(execution WorkflowExecution) { } else { // CREATE SuccessExamples or FailureExamples - executionExample := AppExecutionExample{ + executionExample := shuffle.AppExecutionExample{ AppName: result.Action.AppName, AppVersion: result.Action.AppVersion, AppAction: result.Action.Name, @@ -1672,7 +1655,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getworkflows: %s", err) resp.WriteHeader(401) @@ -1703,7 +1686,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { q = q.Order("-edited") - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { @@ -1765,7 +1748,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -1781,7 +1764,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { return } - var workflow Workflow + var workflow shuffle.Workflow err = json.Unmarshal(body, &workflow) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -1793,7 +1776,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.ID = uuid.NewV4().String() workflow.Owner = user.Id workflow.Sharing = "private" - user.ActiveOrg.Users = []User{} + user.ActiveOrg.Users = []shuffle.User{} workflow.ExecutingOrg = user.ActiveOrg workflow.OrgId = user.ActiveOrg.Id //log.Printf("TRIGGERS: %d", len(workflow.Triggers)) @@ -1805,19 +1788,19 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { //} if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} } - newActions := []Action{} + newActions := []shuffle.Action{} for _, action := range workflow.Actions { if action.Environment == "" { //action.Environment = baseEnvironment @@ -1833,11 +1816,11 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { //log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") // Adds the Testing app if it's a new workflow - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err == nil { // FIXME: Add real env envName := "Shuffle" - environments, err := getEnvironments(ctx, user.ActiveOrg.Id) + environments, err := shuffle.GetEnvironments(ctx, user.ActiveOrg.Id) if err == nil { for _, env := range environments { if env.Default { @@ -1851,11 +1834,11 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { if item.Name == "Testing" && item.AppVersion == "1.0.0" { nodeId := "40447f30-fa44-4a4f-a133-4ee710368737" workflow.Start = nodeId - newActions = append(newActions, Action{ + newActions = append(newActions, shuffle.Action{ Label: "Start node", Name: "hello_world", Environment: envName, - Parameters: []WorkflowAppActionParameter{}, + Parameters: []shuffle.WorkflowAppActionParameter{}, Position: struct { X float64 "json:\"x,omitempty\" datastore:\"x\"" Y float64 "json:\"y,omitempty\" datastore:\"y\"" @@ -1882,7 +1865,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME: Check if they require authentication and if they exist locally //log.Printf("\n\nSHOULD VALIDATE AUTHENTICATION") //AuthenticationId string `json:"authentication_id,omitempty" datastore:"authentication_id"` - //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + //allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) //if err == nil { // log.Printf("AUTH: %#v", allAuths) // for _, action := range newActions { @@ -1891,7 +1874,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { //} } - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} for _, item := range workflow.Actions { oldId := item.ID sourceIndexes := []int{} @@ -1918,7 +1901,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, item) } - newTriggers := []Trigger{} + newTriggers := []shuffle.Trigger{} for _, item := range workflow.Triggers { oldId := item.ID sourceIndexes := []int{} @@ -1946,7 +1929,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { newTriggers = append(newTriggers, item) } - newSchedules := []Schedule{} + newSchedules := []shuffle.Schedule{} for _, item := range workflow.Schedules { item.Id = uuid.NewV4().String() newSchedules = append(newSchedules, item) @@ -1991,7 +1974,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in deleting workflow: %s", err) resp.WriteHeader(401) @@ -2019,7 +2002,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (delete workflow): %s", err) resp.WriteHeader(401) @@ -2084,7 +2067,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } // Adds app auth tracking -func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add bool) error { +func updateAppAuth(auth shuffle.AppAuthenticationStorage, workflowId, nodeId string, add bool) error { workflowFound := false workflowIndex := 0 nodeFound := false @@ -2108,7 +2091,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add updateAuth := false if !workflowFound && add { log.Printf("[INFO] Adding workflow things to auth!") - usageItem := AuthenticationUsage{ + usageItem := shuffle.AuthenticationUsage{ WorkflowId: workflowId, Nodes: []string{nodeId}, } @@ -2127,7 +2110,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add if updateAuth { log.Printf("[INFO] Updating auth!") ctx := context.Background() - err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id) + err := shuffle.SetWorkflowAppAuthDatastore(ctx, auth, auth.Id) if err != nil { log.Printf("Failed setting up app auth %s: %s", auth.Id, err) return err @@ -2138,7 +2121,7 @@ func updateAppAuth(auth AppAuthenticationStorage, workflowId, nodeId string, add } // Identifies what a category defined really is -func handleCategoryIncrease(categories Categories, action Action, workflowapps []WorkflowApp) Categories { +func handleCategoryIncrease(categories shuffle.Categories, action shuffle.Action, workflowapps []shuffle.WorkflowApp) shuffle.Categories { if action.Category == "" { appName := action.AppName for _, app := range workflowapps { @@ -2191,7 +2174,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } //log.Println("Start") - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -2222,7 +2205,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Here to check access rights ctx := context.Background() - tmpworkflow, err := getWorkflow(ctx, fileId) + tmpworkflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (save workflow): %s", err) resp.WriteHeader(401) @@ -2247,7 +2230,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { return } - var workflow Workflow + var workflow shuffle.Workflow err = json.Unmarshal([]byte(body), &workflow) //log.Printf(string(body)) if err != nil { @@ -2275,17 +2258,17 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if len(workflow.ExecutingOrg.Id) == 0 { log.Printf("[INFO] Setting executing org for workflow") - user.ActiveOrg.Users = []User{} + user.ActiveOrg.Users = []shuffle.User{} workflow.ExecutingOrg = user.ActiveOrg } // FIXME - this shouldn't be necessary with proper API checks - newActions := []Action{} + newActions := []shuffle.Action{} allNodes := []string{} - workflow.Categories = Categories{} + workflow.Categories = shuffle.Categories{} //log.Printf("PRE APPS") - workflowapps, apperr := getAllWorkflowApps(ctx, 500) + workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 500) //log.Printf("Action: %#v", action.Authentication) for _, action := range workflow.Actions { @@ -2322,14 +2305,14 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = append(newActions, action) } - newTriggers := []Trigger{} + newTriggers := []shuffle.Trigger{} for _, trigger := range workflow.Triggers { log.Printf("[INFO] Trigger %s: %s", trigger.TriggerType, trigger.Status) // Check if it's actually running // FIXME: Do this for other triggers too if trigger.TriggerType == "SCHEDULE" && trigger.Status != "uninitialized" { - schedule, err := getSchedule(ctx, trigger.ID) + schedule, err := shuffle.GetSchedule(ctx, trigger.ID) if err != nil { trigger.Status = "stopped" } else if schedule.Id == "" { @@ -2344,7 +2327,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if len(user.ApiKey) > 0 { apikey = user.ApiKey } else { - user, err = generateApikey(ctx, user) + user, err = shuffle.GenerateApikey(ctx, user) if err != nil { workflow.IsValid = false workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} @@ -2448,13 +2431,13 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.Triggers = newTriggers if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -2500,7 +2483,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if len(foundNodes) != len(allNodes) || len(workflow.Actions) <= 0 { // This shit takes a few seconds lol if !workflow.IsValid { - oldworkflow, err := getWorkflow(ctx, fileId) + oldworkflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Workflow %s doesn't exist - oldworkflow.", fileId) if workflow.PreviouslySaved { @@ -2542,7 +2525,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Have to do it like this to add the user's apps //log.Println("Apps set starting") //log.Printf("EXIT ON ERROR: %#v", workflow.Configuration.ExitOnError) - //workflowapps, apperr := getAllWorkflowApps(ctx, 500) + //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 500) // Started getting the single apps, but if it's weird, this is faster // 1. Check workflow.Start @@ -2567,7 +2550,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) if workflow.PreviouslySaved { @@ -2579,8 +2562,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Check every app action and param to see whether they exist //log.Printf("PRE ACTIONS 2") - allAuths, autherr := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - newActions = []Action{} + allAuths, autherr := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + newActions = []shuffle.Action{} for _, action := range workflow.Actions { reservedApps := []string{ "0ca8887e-b4af-4e3e-887c-87e9d3bc3d3e", @@ -2629,7 +2612,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if builtin { newActions = append(newActions, action) } else { - curapp := WorkflowApp{} + curapp := shuffle.WorkflowApp{} // FIXME - can this work with ONLY AppID? for _, app := range workflowapps { if app.ID == action.AppID { @@ -2659,7 +2642,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { //return } else { // Check tosee if the appaction is valid - curappaction := WorkflowAppAction{} + curappaction := shuffle.WorkflowAppAction{} for _, curAction := range curapp.Actions { if action.Name == curAction.Name { curappaction = curAction @@ -2685,7 +2668,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME - check all parameters to see if they're valid // Includes checking required fields - selectedAuth := AppAuthenticationStorage{} + selectedAuth := shuffle.AppAuthenticationStorage{} if len(action.AuthenticationId) > 0 && autherr == nil { for _, auth := range allAuths { if auth.Id == action.AuthenticationId { @@ -2695,7 +2678,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } - newParams := []WorkflowAppActionParameter{} + newParams := []shuffle.WorkflowAppActionParameter{} for _, param := range curappaction.Parameters { paramFound := false @@ -2765,7 +2748,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { if autherr == nil && len(workflowapps) > 0 && apperr == nil { //log.Printf("Setting actions") - actionFixing := []Action{} + actionFixing := []shuffle.Action{} appsAdded := []string{} for _, action := range newActions { setAuthentication := false @@ -2813,7 +2796,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME: Only o this IF there isn't another one for the app already if !authSet { //log.Printf("Validate if the app NEEDS auth or not") - outerapp := WorkflowApp{} + outerapp := shuffle.WorkflowApp{} for _, app := range workflowapps { if app.Name == action.AppName { outerapp = app @@ -2839,21 +2822,21 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // FIXME: Add app auth if !found { timeNow := int64(time.Now().Unix()) - authFields := []AuthenticationStore{} + authFields := []shuffle.AuthenticationStore{} for _, param := range outerapp.Authentication.Parameters { - authFields = append(authFields, AuthenticationStore{ + authFields = append(authFields, shuffle.AuthenticationStore{ Key: param.Name, Value: "", }) } - appAuth := AppAuthenticationStorage{ + appAuth := shuffle.AppAuthenticationStorage{ Active: true, Label: fmt.Sprintf("default_%s", outerapp.Name), Id: uuid.NewV4().String(), App: outerapp, Fields: authFields, - Usage: []AuthenticationUsage{}, + Usage: []shuffle.AuthenticationUsage{}, WorkflowCount: 0, NodeCount: 0, OrgId: user.ActiveOrg.Id, @@ -2861,7 +2844,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Edited: timeNow, } - err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) if err != nil { log.Printf("Failed setting appauth for with name %s", appAuth.Label) } else { @@ -2876,7 +2859,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { //outerapp.Authentication.Required // Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"` - //workflowapps, apperr := getAllWorkflowApps(ctx, 100) + //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 100) } } @@ -2886,8 +2869,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = actionFixing } else { log.Printf("FirstSave error: %s - %s", err, apperr) - //workflowapps, apperr := getAllWorkflowApps(ctx, 100) - //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + //workflowapps, apperr := shuffle.GetAllWorkflowApps(ctx, 100) + //allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) } workflow.PreviouslySaved = true @@ -3022,7 +3005,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflowExecution, err := getWorkflowExecution(ctx, executionId) + workflowExecution, err := shuffle.GetWorkflowExecution(ctx, executionId) if err != nil { log.Printf("[ERROR] Failed getting execution (abort) %s: %s", executionId, err) resp.WriteHeader(401) @@ -3041,7 +3024,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { if workflowExecution.Authorization != parsedKey { // FIXME: Check the execution if this fails. - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in abort workflow: %s", err) resp.WriteHeader(401) @@ -3074,7 +3057,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Running shutdown of %s", workflowExecution.ExecutionId) lastResult := "" - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} // type ActionResult struct { for _, result := range workflowExecution.Results { if result.Status == "EXECUTING" { @@ -3116,7 +3099,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } if len(workflowExecution.Results) == 0 || addResult { - newaction := Action{ + newaction := shuffle.Action{ ID: workflowExecution.Start, } @@ -3127,7 +3110,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } } - workflowExecution.Results = append(workflowExecution.Results, ActionResult{ + workflowExecution.Results = append(workflowExecution.Results, shuffle.ActionResult{ Action: newaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3144,7 +3127,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { if nodeok { nodeId := node[0] log.Printf("[INFO] Found abort node %s", nodeId) - newaction := Action{ + newaction := shuffle.Action{ ID: nodeId, } @@ -3155,7 +3138,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { } } - workflowExecution.Results = append(workflowExecution.Results, ActionResult{ + workflowExecution.Results = append(workflowExecution.Results, shuffle.ActionResult{ Action: newaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3201,7 +3184,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("[INFO] Api authentication failed in cleanup executions: %s", err) resp.WriteHeader(401) @@ -3221,7 +3204,7 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { timestamp := int64(time.Now().AddDate(0, -2, 0).Unix()) log.Println(timestamp) q := datastore.NewQuery("workflowexecution").Filter("started_at <", timestamp) - var workflowExecutions []WorkflowExecution + var workflowExecutions []shuffle.WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { log.Printf("Error getting workflowexec (cleanup): %s", err) @@ -3234,13 +3217,13 @@ func cleanupExecutions(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(`{"success": true}`)) } -func handleExecution(id string, workflow Workflow, request *http.Request) (WorkflowExecution, string, error) { +func handleExecution(id string, workflow shuffle.Workflow, request *http.Request) (shuffle.WorkflowExecution, string, error) { ctx := context.Background() if workflow.ID == "" || workflow.ID != id { - tmpworkflow, err := getWorkflow(ctx, id) + tmpworkflow, err := shuffle.GetWorkflow(ctx, id) if err != nil { log.Printf("Failed getting the workflow locally (execution cleanup): %s", err) - return WorkflowExecution{}, "Failed getting workflow", err + return shuffle.WorkflowExecution{}, "Failed getting workflow", err } workflow = *tmpworkflow @@ -3248,13 +3231,13 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(workflow.ExecutingOrg.Id) == 0 { log.Printf("[INFO] Stopped execution because there is no executing org for workflow %s", workflow.ID) - return WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow has no executing org defined"), errors.New("Workflow has no executing org defined") } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } else { - newactions := []Action{} + newactions := []shuffle.Action{} for _, action := range workflow.Actions { action.LargeImage = "" action.SmallImage = "" @@ -3266,12 +3249,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } else { - newtriggers := []Trigger{} + newtriggers := []shuffle.Trigger{} for _, trigger := range workflow.Triggers { trigger.LargeImage = "" trigger.SmallImage = "" @@ -3288,21 +3271,21 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !workflow.IsValid { log.Printf("[ERROR] Stopped execution as workflow %s is not valid.", workflow.ID) - return WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") + return shuffle.WorkflowExecution{}, fmt.Sprintf(`workflow %s is invalid`, workflow.ID), errors.New("Failed getting workflow") } workflowBytes, err := json.Marshal(workflow) if err != nil { log.Printf("Failed workflow unmarshal in execution: %s", err) - return WorkflowExecution{}, "", err + return shuffle.WorkflowExecution{}, "", err } //log.Println(workflow) - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution err = json.Unmarshal(workflowBytes, &workflowExecution.Workflow) if err != nil { log.Printf("Failed execution unmarshaling: %s", err) - return WorkflowExecution{}, "Failed unmarshal during execution", err + return shuffle.WorkflowExecution{}, "Failed unmarshal during execution", err } makeNew := true @@ -3311,7 +3294,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf body, err := ioutil.ReadAll(request.Body) if err != nil { log.Printf("[ERROR] Failed request POST read: %s", err) - return WorkflowExecution{}, "Failed getting body", err + return shuffle.WorkflowExecution{}, "Failed getting body", err } // This one doesn't really matter. @@ -3353,11 +3336,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf log.Printf("Body: %s", string(body)) } - var execution ExecutionRequest + var execution shuffle.ExecutionRequest err = json.Unmarshal(body, &execution) if err != nil { log.Printf("[WARNING] Failed execution POST unmarshaling - continuing anyway: %s", err) - //return WorkflowExecution{}, "", err + //return shuffle.WorkflowExecution{}, "", err } if execution.Start == "" && len(body) > 0 { @@ -3388,12 +3371,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { log.Printf("[ERROR] ACTION %s WAS NOT FOUND!", workflow.Start) - return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", workflow.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", workflow.Start)) } } else if len(execution.Start) > 0 { log.Printf("[ERROR] START ACTION %s IS WRONG ID LENGTH %d!", execution.Start, len(execution.Start)) - return WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Startnode %s was not found in actions", execution.Start), errors.New(fmt.Sprintf("Startnode %s was not found in actions", execution.Start)) } if len(execution.ExecutionId) == 36 { @@ -3415,18 +3398,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf log.Printf("Should update reference and return, no need for further execution!") // Get the reference execution - oldExecution, err := getWorkflowExecution(ctx, referenceId[0]) + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) if err != nil { log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err } if oldExecution.Workflow.ID != id { log.Println("Wrong workflowid!") - return WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Bad ID %s", referenceId), errors.New("Bad ID") } - newResults := []ActionResult{} + newResults := []shuffle.ActionResult{} //log.Printf("%#v", oldExecution.Results) for _, result := range oldExecution.Results { log.Printf("%s - %s", result.Action.ID, start[0]) @@ -3453,20 +3436,20 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf err = setWorkflowExecution(ctx, *oldExecution, true) if err != nil { log.Printf("Error saving workflow execution actionresult setting: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed setting workflowexecution actionresult in execution: %s", err), err } - return WorkflowExecution{}, "", nil + return shuffle.WorkflowExecution{}, "", nil } } if referenceok { log.Printf("Handling an old execution continuation!") // Will use the old name, but still continue with NEW ID - oldExecution, err := getWorkflowExecution(ctx, referenceId[0]) + oldExecution, err := shuffle.GetWorkflowExecution(ctx, referenceId[0]) if err != nil { log.Printf("Failed getting execution (execution) %s: %s", referenceId[0], err) - return WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed getting execution ID %s because it doesn't exist.", referenceId[0]), err } workflowExecution = *oldExecution @@ -3492,7 +3475,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // FIXME - regex uuid, and check if already exists? if len(workflowExecution.ExecutionId) != 36 { log.Printf("Invalid uuid: %s", workflowExecution.ExecutionId) - return WorkflowExecution{}, "Invalid uuid", err + return shuffle.WorkflowExecution{}, "Invalid uuid", err } // FIXME - find owner of workflow @@ -3553,10 +3536,10 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf topic := "workflows" startFound := false // FIXME - remove this? - newActions := []Action{} - defaultResults := []ActionResult{} + newActions := []shuffle.Action{} + defaultResults := []shuffle.ActionResult{} - allAuths := []AppAuthenticationStorage{} + allAuths := []shuffle.AppAuthenticationStorage{} for _, action := range workflowExecution.Workflow.Actions { //action.LargeImage = "" if action.ID == workflowExecution.Start { @@ -3565,20 +3548,20 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //log.Println(action.Environment) if action.Environment == "" { - return WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") + return shuffle.WorkflowExecution{}, fmt.Sprintf("Environment is not defined for %s", action.Name), errors.New("Environment not defined!") } // FIXME: Authentication parameters if len(action.AuthenticationId) > 0 { if len(allAuths) == 0 { - allAuths, err = getAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) + allAuths, err = shuffle.GetAllWorkflowAppAuth(ctx, workflow.ExecutingOrg.Id) if err != nil { log.Printf("Api authentication failed in get all app auth: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err + return shuffle.WorkflowExecution{}, fmt.Sprintf("Api authentication failed in get all app auth: %s", err), err } } - curAuth := AppAuthenticationStorage{Id: ""} + curAuth := shuffle.AppAuthenticationStorage{Id: ""} for _, auth := range allAuths { if auth.Id == action.AuthenticationId { curAuth = auth @@ -3587,11 +3570,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } if len(curAuth.Id) == 0 { - return WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId), errors.New(fmt.Sprintf("Auth ID %s doesn't exist", action.AuthenticationId)) } // Rebuild params with the right data. This is to prevent issues on the frontend - newParams := []WorkflowAppActionParameter{} + newParams := []shuffle.WorkflowAppActionParameter{} for _, param := range action.Parameters { for _, authparam := range curAuth.Fields { @@ -3620,7 +3603,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // as it's not a childnode of the startnode // This is a configuration item for the workflow itself. if len(workflowExecution.Results) > 0 { - defaultResults = []ActionResult{} + defaultResults = []shuffle.ActionResult{} for _, result := range workflowExecution.Results { if result.Status == "WAITING" { result.Status = "FINISHED" @@ -3644,7 +3627,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } //log.Printf("[WARNING] Set %s to SKIPPED as it's NOT a childnode of the startnode.", action.ID) - curaction := Action{ + curaction := shuffle.Action{ AppName: action.AppName, AppVersion: action.AppVersion, Label: action.Label, @@ -3653,7 +3636,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } //action //curaction.Parameters = [] - defaultResults = append(defaultResults, ActionResult{ + defaultResults = append(defaultResults, shuffle.ActionResult{ Action: curaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3687,7 +3670,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { //log.Printf("SHOULD SET TRIGGER %s TO BE SKIPPED", trigger.ID) - curaction := Action{ + curaction := shuffle.Action{ AppName: "shuffle-subflow", AppVersion: trigger.AppVersion, Label: trigger.Label, @@ -3695,7 +3678,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf ID: trigger.ID, } - defaultResults = append(defaultResults, ActionResult{ + defaultResults = append(defaultResults, shuffle.ActionResult{ Action: curaction, ExecutionId: workflowExecution.ExecutionId, Authorization: workflowExecution.Authorization, @@ -3713,7 +3696,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !startFound { log.Printf("[ERROR] Startnode %s doesn't exist!!", workflowExecution.Start) - return WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow action %s doesn't exist in workflow", workflowExecution.Start), errors.New(fmt.Sprintf(`Workflow start node "%s" doesn't exist. Exiting!`, workflowExecution.Start)) } // Verification for execution environments @@ -3726,14 +3709,14 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf workflowExecution.ExecutionOrg = workflow.ExecutingOrg.Id } - var allEnvs []Environment + var allEnvs []shuffle.Environment if len(workflowExecution.ExecutionOrg) > 0 { //log.Printf("[INFO] Executing ORG: %s", workflowExecution.ExecutionOrg) - allEnvironments, err := getEnvironments(ctx, workflowExecution.ExecutionOrg) + allEnvironments, err := shuffle.GetEnvironments(ctx, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed finding environments: %s", err) - return WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) + return shuffle.WorkflowExecution{}, fmt.Sprintf("Workflow environments not found for this org"), errors.New(fmt.Sprintf("Workflow environments not found for this org")) } for _, curenv := range allEnvironments { @@ -3745,12 +3728,12 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } else { log.Printf("[ERROR] No org identified for execution of %s. Returning", workflowExecution.Workflow.ID) - return WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") + return shuffle.WorkflowExecution{}, "No org identified for execution", errors.New("No org identified for execution") } if len(allEnvs) == 0 { log.Printf("[ERROR] No active environments found for org: %s", workflowExecution.ExecutionOrg) - return WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No active env found for org %s", workflowExecution.ExecutionOrg)) } // Check if the actions are children of the startnode? @@ -3769,7 +3752,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf onpremExecution = true } else { log.Printf("[ERROR] No handler for environment type %s", env.Type) - return WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) + return shuffle.WorkflowExecution{}, "No active environments found", errors.New(fmt.Sprintf("No handler for environment type %s", env.Type)) } break } @@ -3777,7 +3760,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !found { log.Printf("[ERROR] Couldn't find environment %s. Maybe it's inactive?", action.Environment) - return WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) + return shuffle.WorkflowExecution{}, "Couldn't find the environment", errors.New(fmt.Sprintf("Couldn't find env %s in org %s", action.Environment, workflowExecution.ExecutionOrg)) } found = false @@ -3802,7 +3785,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf err = imageCheckBuilder(imageNames) if err != nil { log.Printf("[ERROR] Failed building the required images from %#v: %s", imageNames, err) - return WorkflowExecution{}, "Failed building missing Docker images", err + return shuffle.WorkflowExecution{}, "Failed building missing Docker images", err } //b, err := json.Marshal(workflowExecution) @@ -3812,17 +3795,17 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} //} - workflowExecution.Workflow.ExecutingOrg = Org{ + workflowExecution.Workflow.ExecutingOrg = shuffle.Org{ Id: workflowExecution.Workflow.ExecutingOrg.Id, } - workflowExecution.Workflow.Org = []Org{ + workflowExecution.Workflow.Org = []shuffle.Org{ workflowExecution.Workflow.ExecutingOrg, } //Org []Org `json:"org,omitempty" datastore:"org"` err = setWorkflowExecution(ctx, workflowExecution, true) if err != nil { log.Printf("Error saving workflow execution for updates %s: %s", topic, err) - return WorkflowExecution{}, "Failed getting workflowexecution", err + return shuffle.WorkflowExecution{}, "Failed getting workflowexecution", err } // Adds queue for onprem execution @@ -3833,7 +3816,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf for _, environment := range environments { log.Printf("[INFO] Execution: %s should execute onprem with execution environment \"%s\". Workflow: %s", workflowExecution.ExecutionId, environment, workflowExecution.Workflow.ID) - executionRequest := ExecutionRequest{ + executionRequest := shuffle.ExecutionRequest{ ExecutionId: workflowExecution.ExecutionId, WorkflowId: workflowExecution.Workflow.ID, Authorization: workflowExecution.Authorization, @@ -3863,7 +3846,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if !featuresList.Workflows.Active || err != nil { log.Printf("Error: %s", err) log.Printf("[ERROR] Cloud not implemented yet. May need to work on app checking and such") - return WorkflowExecution{}, "Cloud not implemented yet", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet", errors.New("Cloud not implemented yet") } // What it needs to know: @@ -3873,11 +3856,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf //cloudExecuteAction(workflowExecution.ExecutionId, workflowExecution.Workflow.Actions[0], workflowExecution.ExecutionOrg, workflowExecution.Workflow.ID) cloudExecuteAction(workflowExecution) - return WorkflowExecution{}, "Cloud not implemented yet (1)", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet (1)", errors.New("Cloud not implemented yet") } else { // If it's here, it should be controlled by Worker. // If worker, should this backend be a proxy? I think so. - return WorkflowExecution{}, "Cloud not implemented yet (2)", errors.New("Cloud not implemented yet") + return shuffle.WorkflowExecution{}, "Cloud not implemented yet (2)", errors.New("Cloud not implemented yet") } } @@ -3890,21 +3873,21 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } // This updates stuff locally from remote executions -func cloudExecuteAction(execution WorkflowExecution) error { +func cloudExecuteAction(execution shuffle.WorkflowExecution) error { ctx := context.Background() - org, err := getOrg(ctx, execution.ExecutionOrg) + org, err := shuffle.GetOrg(ctx, execution.ExecutionOrg) if err != nil { return err } type ExecutionStruct struct { - ExecutionId string `json:"execution_id" datastore:"execution_id"` - Action Action `json:"action" datastore:"action"` - Authorization string `json:"authorization" datastore:"authorization"` - Results []ActionResult `json:"results" datastore:"results,noindex"` - ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` - WorkflowId string `json:"workflow_id" datastore:"workflow_id"` - ExecutionSource string `json:"execution_source" datastore:"execution_source"` + ExecutionId string `json:"execution_id" datastore:"execution_id"` + Action shuffle.Action `json:"action" datastore:"action"` + Authorization string `json:"authorization" datastore:"authorization"` + Results []shuffle.ActionResult `json:"results" datastore:"results,noindex"` + ExecutionArgument string `json:"execution_argument" datastore:"execution_argument,noindex"` + WorkflowId string `json:"workflow_id" datastore:"workflow_id"` + ExecutionSource string `json:"execution_source" datastore:"execution_source"` } data := ExecutionStruct{ @@ -3966,7 +3949,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("[INFO] Api authentication failed in execute workflow: %s", err) resp.WriteHeader(401) @@ -3995,7 +3978,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { //memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId) ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (execute workflow): %s", err) resp.WriteHeader(401) @@ -4014,7 +3997,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { log.Printf("[INFO] Starting execution of %s!", fileId) - user.ActiveOrg.Users = []User{} + user.ActiveOrg.Users = []shuffle.User{} workflow.ExecutingOrg = user.ActiveOrg workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) @@ -4034,7 +4017,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4070,7 +4053,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("[WARNING] Failed getting the workflow locally (stop schedule): %s", err) resp.WriteHeader(401) @@ -4087,7 +4070,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { return } - schedule, err := getSchedule(ctx, scheduleId) + schedule, err := shuffle.GetSchedule(ctx, scheduleId) if err != nil { log.Printf("[WARNING] Failed finding schedule %s", scheduleId) resp.WriteHeader(401) @@ -4100,7 +4083,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { if schedule.Environment == "cloud" { log.Printf("[INFO] Should STOP a cloud schedule for workflow %s with schedule ID %s", fileId, scheduleId) // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -4163,7 +4146,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4199,7 +4182,7 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (stop schedule GCP): %s", err) resp.WriteHeader(401) @@ -4217,13 +4200,13 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4292,7 +4275,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in schedule workflow: %s", err) resp.WriteHeader(401) @@ -4320,7 +4303,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow locally (schedule workflow): %s", err) resp.WriteHeader(401) @@ -4338,13 +4321,13 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4358,7 +4341,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { return } - var schedule Schedule + var schedule shuffle.Schedule err = json.Unmarshal(body, &schedule) if err != nil { log.Printf("Failed schedule POST unmarshaling: %s", err) @@ -4422,7 +4405,7 @@ func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { if schedule.Environment == "cloud" { log.Printf("[INFO] Should START a cloud schedule for workflow %s with schedule ID %s", workflow.ID, schedule.Id) // https://shuffler.io/v1/hooks/webhook_80184973-3e82-4852-842e-0290f7f34d7c - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -4523,7 +4506,7 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -4569,7 +4552,7 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { // return //} - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Workflow %s doesn't exist.", fileId) resp.WriteHeader(401) @@ -4588,13 +4571,13 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(workflow.Actions) == 0 { - workflow.Actions = []Action{} + workflow.Actions = []shuffle.Action{} } if len(workflow.Branches) == 0 { - workflow.Branches = []Branch{} + workflow.Branches = []shuffle.Branch{} } if len(workflow.Triggers) == 0 { - workflow.Triggers = []Trigger{} + workflow.Triggers = []shuffle.Trigger{} } if len(workflow.Errors) == 0 { workflow.Errors = []string{} @@ -4643,7 +4626,7 @@ func getSpecificWorkflow(resp http.ResponseWriter, request *http.Request) { resp.Write(body) } -func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecution, dbSave bool) error { +func setWorkflowExecution(ctx context.Context, workflowExecution shuffle.WorkflowExecution, dbSave bool) error { //log.Printf("\n\n\nRESULT: %s\n\n\n", workflowExecution.Status) if len(workflowExecution.ExecutionId) == 0 { log.Printf("Workflowexeciton executionId can't be empty.") @@ -4667,11 +4650,11 @@ func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti return nil } -func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, error) { - workflowExecution := &WorkflowExecution{} +func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExecution, error) { + workflowExecution := &shuffle.WorkflowExecution{} cacheKey := fmt.Sprintf("workflowexecution-%s", id) if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*WorkflowExecution) + parsedValue := value.(*shuffle.WorkflowExecution) //log.Printf("Found execution for id %s with %d results", parsedValue.ExecutionId, len(parsedValue.Results)) return parsedValue, nil @@ -4688,58 +4671,46 @@ func getWorkflowExecution(ctx context.Context, id string) (*WorkflowExecution, e key := datastore.NameKey("workflowexecution", strings.ToLower(id), nil) if err := dbclient.Get(ctx, key, workflowExecution); err != nil { - return &WorkflowExecution{}, err + return &shuffle.WorkflowExecution{}, err } return workflowExecution, nil } -func getApp(ctx context.Context, id string) (*WorkflowApp, error) { - key := datastore.NameKey("workflowapp", strings.ToLower(id), nil) - workflowApp := &WorkflowApp{} - if err := dbclient.Get(ctx, key, workflowApp); err != nil { - return &WorkflowApp{}, err +//func shuffle.GetApp(ctx context.Context, id string) (*WorkflowApp, error) { +// key := datastore.NameKey("workflowapp", strings.ToLower(id), nil) +// workflowApp := &WorkflowApp{} +// if err := dbclient.Get(ctx, key, workflowApp); err != nil { +// return &WorkflowApp{}, err +// +// } +// +// return workflowApp, nil +//} +// +//func shuffle.GetWorkflow(ctx context.Context, id string) (*shuffle.Workflow, error) { +// key := datastore.NameKey("workflow", strings.ToLower(id), nil) +// workflow := &Workflow{} +// if err := dbclient.Get(ctx, key, workflow); err != nil { +// return &Workflow{}, err +// } +// +// return workflow, nil +//} +// +//func shuffle.GetEnvironments(ctx context.Context, orgId string) ([]Environment, error) { +// var environments []Environment +// q := datastore.NewQuery("Environments").Filter("org_id =", orgId) +// +// _, err := dbclient.GetAll(ctx, q, &environments) +// if err != nil { +// return []Environment{}, err +// } +// +// return environments, nil +//} - } - - return workflowApp, nil -} - -func getWorkflow(ctx context.Context, id string) (*Workflow, error) { - key := datastore.NameKey("workflow", strings.ToLower(id), nil) - workflow := &Workflow{} - if err := dbclient.Get(ctx, key, workflow); err != nil { - return &Workflow{}, err - } - - return workflow, nil -} - -func getEnvironments(ctx context.Context, orgId string) ([]Environment, error) { - var environments []Environment - q := datastore.NewQuery("Environments").Filter("org_id =", orgId) - - _, err := dbclient.GetAll(ctx, q, &environments) - if err != nil { - return []Environment{}, err - } - - return environments, nil -} - -func getAllWorkflows(ctx context.Context, orgId string) ([]Workflow, error) { - var allworkflows []Workflow - q := datastore.NewQuery("workflow").Filter("org_id = ", orgId) - - _, err := dbclient.GetAll(ctx, q, &allworkflows) - if err != nil { - return []Workflow{}, err - } - - return allworkflows, nil -} - -func setExampleresult(ctx context.Context, result AppExecutionExample) error { +func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) error { // FIXME: Reintroduce this for stats //key := datastore.NameKey("example_result", result.ExampleId, nil) @@ -4754,7 +4725,7 @@ func setExampleresult(ctx context.Context, result AppExecutionExample) error { // Hmm, so I guess this should use uuid :( // Consistency PLX -func setWorkflow(ctx context.Context, workflow Workflow, id string, optionalEditedSecondsOffset ...int) error { +func setWorkflow(ctx context.Context, workflow shuffle.Workflow, id string, optionalEditedSecondsOffset ...int) error { workflow.Edited = int64(time.Now().Unix()) if len(optionalEditedSecondsOffset) > 0 { workflow.Edited += int64(optionalEditedSecondsOffset[0]) @@ -4777,7 +4748,7 @@ func deleteAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -4831,7 +4802,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in edit workflow: %s", userErr) resp.WriteHeader(401) @@ -4854,7 +4825,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("ID: %s", fileId) - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId) if err != nil { log.Printf("Error getting app (delete) %s: %s", fileId, err) resp.WriteHeader(401) @@ -4879,7 +4850,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // FIXME: Make workflows track themself INSIDE apps, or with a reference q := datastore.NewQuery("workflow").Filter("org_id = ", user.ActiveOrg.Id).Limit(30) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Failed getting related workflows for the app: %s", err) @@ -4893,7 +4864,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { for _, workflow := range workflows { found := false - newActions := []Action{} + newActions := []shuffle.Action{} for _, action := range workflow.Actions { if action.AppName == app.Name && action.AppVersion == app.AppVersion { found = true @@ -4944,7 +4915,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Not really deleting it, just removing from user cache if private { log.Printf("[INFO] Deleting private app") - var privateApps []WorkflowApp + var privateApps []shuffle.WorkflowApp for _, item := range user.PrivateApps { if item.ID == fileId { continue @@ -4954,7 +4925,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { } user.PrivateApps = privateApps - err = setUser(ctx, &user) + err = shuffle.SetUser(ctx, &user) if err != nil { log.Printf("[ERROR] Failed removing %s app for user %s: %s", app.Name, user.Username, err) resp.WriteHeader(401) @@ -5006,7 +4977,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { fileId = location[4] } - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId) if err != nil { log.Printf("[WARNING] Error getting app %s (app config): %s", fileId, err) resp.WriteHeader(401) @@ -5051,7 +5022,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("[WARNING] Api authentication failed in get app: %s", userErr) resp.WriteHeader(401) @@ -5099,7 +5070,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5155,7 +5126,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - auth, err := getWorkflowAppAuthDatastore(ctx, fileId) + auth, err := shuffle.GetWorkflowAppAuthDatastore(ctx, fileId) if err != nil { log.Printf("Authget error: %s", err) resp.WriteHeader(401) @@ -5174,7 +5145,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { q := datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) q = q.Order("-edited").Limit(35) - var workflows []Workflow + var workflows []shuffle.Workflow _, err = dbclient.GetAll(ctx, q, &workflows) if err != nil { log.Printf("Getall error in auth update: %s", err) @@ -5186,11 +5157,11 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { // FIXME: Add function to remove auth from other auth's actionCnt := 0 workflowCnt := 0 - authenticationUsage := []AuthenticationUsage{} + authenticationUsage := []shuffle.AuthenticationUsage{} for _, workflow := range workflows { - newActions := []Action{} + newActions := []shuffle.Action{} edited := false - usage := AuthenticationUsage{ + usage := shuffle.AuthenticationUsage{ WorkflowId: workflow.ID, Nodes: []string{}, } @@ -5229,7 +5200,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { auth.Usage = authenticationUsage auth.Defined = true - err = setWorkflowAppAuthDatastore(ctx, *auth, auth.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, *auth, auth.Id) if err != nil { log.Printf("Failed setting appauth: %s", err) resp.WriteHeader(401) @@ -5254,7 +5225,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5270,7 +5241,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - var appAuth AppAuthenticationStorage + var appAuth shuffle.AppAuthenticationStorage err = json.Unmarshal(body, &appAuth) if err != nil { log.Printf("Failed unmarshaling (appauth): %s", err) @@ -5283,7 +5254,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { if len(appAuth.Id) == 0 { appAuth.Id = uuid.NewV4().String() } else { - auth, err := getWorkflowAppAuthDatastore(ctx, appAuth.Id) + auth, err := shuffle.GetWorkflowAppAuthDatastore(ctx, appAuth.Id) if err == nil { // OrgId string `json:"org_id" datastore:"org_id"` if auth.OrgId != user.ActiveOrg.Id { @@ -5331,10 +5302,10 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { } // FIXME: Doens't validate Org - app, err := getApp(ctx, appAuth.App.ID) + app, err := shuffle.GetApp(ctx, appAuth.App.ID) if err != nil { log.Printf("[WARNING] Failed finding app %s while setting auth. Finding it by looping apps.", appAuth.App.ID) - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { resp.WriteHeader(409) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) @@ -5379,7 +5350,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { //appAuth.LargeImage = "" appAuth.OrgId = user.ActiveOrg.Id appAuth.Defined = true - err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) + err = shuffle.SetWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) if err != nil { log.Printf("Failed setting up app auth %s: %s", appAuth.Id, err) resp.WriteHeader(409) @@ -5397,7 +5368,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5413,7 +5384,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { // return //} ctx := context.Background() - allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) + allAuths, err := shuffle.GetAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Api authentication failed in get all app auth: %s", err) resp.WriteHeader(401) @@ -5428,7 +5399,7 @@ func getAppAuthentication(resp http.ResponseWriter, request *http.Request) { } // Cleanup for frontend usage. User shouldn't be able to get the data. - newAuth := []AppAuthenticationStorage{} + newAuth := []shuffle.AppAuthenticationStorage{} for _, auth := range allAuths { newAuthField := auth for index, _ := range auth.Fields { @@ -5509,7 +5480,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Api authentication failed in get all apps: %s", userErr) resp.WriteHeader(401) @@ -5530,7 +5501,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - app, err := getApp(ctx, fileId) + app, err := shuffle.GetApp(ctx, fileId) if err != nil { log.Printf("Error getting app (update app): %s", fileId) resp.WriteHeader(401) @@ -5575,7 +5546,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { app.SharingConfig = tmpfields.SharingConfig } - err = setWorkflowAppDatastore(ctx, *app, app.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, *app, app.ID) if err != nil { log.Printf("Failed patching workflowapp: %s", err) resp.WriteHeader(401) @@ -5606,7 +5577,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() // Just need to be logged in // FIXME - need to be logged in? - user, userErr := handleApiAuthentication(resp, request) + user, userErr := shuffle.HandleApiAuthentication(resp, request) if userErr != nil { log.Printf("Continuing with apps even without auth") //log.Printf("Api authentication failed in get all apps: %s", userErr) @@ -5643,7 +5614,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // return //} - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps (getworkflowapps): %s", err) resp.WriteHeader(401) @@ -5756,7 +5727,7 @@ func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { // Bad check for workflowapps :) // FIXME - use tags and struct reflection -func checkWorkflowApp(workflowApp WorkflowApp) error { +func checkWorkflowApp(workflowApp shuffle.WorkflowApp) error { // Validate fields if workflowApp.Name == "" { return errors.New("App field name doesn't exist") @@ -5804,7 +5775,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -5836,7 +5807,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Error: Failed getting workflowapps: %s", err) resp.WriteHeader(401) @@ -5844,7 +5815,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { return } - returnValues := []WorkflowApp{} + returnValues := []shuffle.WorkflowApp{} search := strings.ToLower(tmpBody.Search) for _, app := range workflowapps { if !app.Activated && app.Generated { @@ -5879,7 +5850,7 @@ func validateAppInput(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -6138,7 +6109,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in load apps: %s", err) resp.WriteHeader(401) @@ -6205,7 +6176,7 @@ func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in app hotload: %s", err) resp.WriteHeader(401) @@ -6252,7 +6223,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in load specific apps: %s", err) resp.WriteHeader(401) @@ -6378,7 +6349,7 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { ctx := context.Background() - workflowapps, err := getAllWorkflowApps(ctx, 500) + workflowapps, err := shuffle.GetAllWorkflowApps(ctx, 500) appCounter := 0 if err != nil { log.Printf("Failed to get existing generated apps") @@ -6465,7 +6436,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } //log.Printf("Should generate yaml") - swagger, api, _, err := generateYaml(swagger, parsedOpenApi.ID) + swagger, api, _, err := shuffle.GenerateYaml(swagger, parsedOpenApi.ID) if err != nil { log.Printf("Failed building and generating yaml in loop (%s): %s", filename, err) continue @@ -6490,7 +6461,7 @@ func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, } if !found { - err = setWorkflowAppDatastore(ctx, api, api.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, api, api.ID) if err != nil { log.Printf("Failed setting workflowapp in loop: %s", err) continue @@ -6582,7 +6553,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra continue } - var workflow Workflow + var workflow shuffle.Workflow err = json.Unmarshal(readFile, &workflow) if err != nil { continue @@ -6595,11 +6566,11 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra workflow.ID = uuid.NewV4().String() workflow.OrgId = orgId - workflow.ExecutingOrg = Org{ + workflow.ExecutingOrg = shuffle.Org{ Id: orgId, } - workflow.Org = append(workflow.Org, Org{ + workflow.Org = append(workflow.Org, shuffle.Org{ Id: orgId, }) workflow.IsValid = false @@ -6646,7 +6617,7 @@ type buildLaterStruct struct { func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) { var err error - allapps := []WorkflowApp{} + allapps := []shuffle.WorkflowApp{} // These are slow apps to build with some funky mechanisms reservedNames := []string{ @@ -6757,7 +6728,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin combined = append(combined, dockerfileData...) md5 := md5sum(combined) - var workflowapp WorkflowApp + var workflowapp shuffle.WorkflowApp err = gyaml.Unmarshal(appfileData, &workflowapp) if err != nil { log.Printf("Failed unmarshaling workflowapp %s: %s", fullPath, err) @@ -6772,7 +6743,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if len(allapps) == 0 { - allapps, err = getAllWorkflowApps(ctx, 500) + allapps, err = shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) continue @@ -6816,7 +6787,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin // 2. Check if they're present in the action // 3. Add them IF they DONT exist // 4. Fix python code with reflection (FIXME) - appendParams := []WorkflowAppActionParameter{} + appendParams := []shuffle.WorkflowAppActionParameter{} for _, fieldname := range workflowapp.Authentication.Parameters { found := false for index, param := range action.Parameters { @@ -6830,7 +6801,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if !found { - appendParams = append(appendParams, WorkflowAppActionParameter{ + appendParams = append(appendParams, shuffle.WorkflowAppActionParameter{ Name: fieldname.Name, Description: fieldname.Description, Example: fieldname.Example, @@ -6872,7 +6843,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin workflowapp.Downloaded = true workflowapp.Hash = md5 - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) continue @@ -6972,7 +6943,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { // Just need to be logged in // FIXME - should have some permissions? - _, err := handleApiAuthentication(resp, request) + _, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new app: %s", err) resp.WriteHeader(401) @@ -6988,7 +6959,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - var workflowapp WorkflowApp + var workflowapp shuffle.WorkflowApp err = json.Unmarshal(body, &workflowapp) if err != nil { log.Printf("Failed unmarshaling: %s", err) @@ -6998,7 +6969,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - allapps, err := getAllWorkflowApps(ctx, 500) + allapps, err := shuffle.GetAllWorkflowApps(ctx, 500) if err != nil { log.Printf("Failed getting apps to verify: %s", err) resp.WriteHeader(401) @@ -7039,7 +7010,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { workflowapp.Generated = false workflowapp.Activated = true - err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) + err = shuffle.SetWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { log.Printf("Failed setting workflowapp: %s", err) resp.WriteHeader(401) @@ -7065,7 +7036,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in getting specific workflow: %s", err) resp.WriteHeader(401) @@ -7093,7 +7064,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } ctx := context.Background() - workflow, err := getWorkflow(ctx, fileId) + workflow, err := shuffle.GetWorkflow(ctx, fileId) if err != nil { log.Printf("Failed getting the workflow %s locally (get executions): %s", fileId, err) resp.WriteHeader(401) @@ -7112,7 +7083,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { // Query for the specifci workflowId maxAmount := 30 q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(maxAmount) - var workflowExecutions []WorkflowExecution + var workflowExecutions []shuffle.WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { @@ -7133,7 +7104,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { it := dbclient.Run(ctx, q) //_, err = it.Next(&app) for { - var workflowExecution WorkflowExecution + var workflowExecution shuffle.WorkflowExecution _, err := it.Next(&workflowExecution) if err != nil { break @@ -7227,169 +7198,169 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) { } //FIXME: Add cursor -func getAllWorkflowApps(ctx context.Context, maxLen int) ([]WorkflowApp, error) { - var apps []WorkflowApp - query := datastore.NewQuery("workflowapp").Order("-edited").Limit(10) - //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) +//func shuffle.GetAllWorkflowApps(ctx context.Context, maxLen int) ([]shuffle.WorkflowApp, error) { +// var apps []WorkflowApp +// query := datastore.NewQuery("workflowapp").Order("-edited").Limit(10) +// //query := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) +// +// cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) +// if value, found := requestCache.Get(cacheKey); found { +// parsedValue := value.(*[]WorkflowApp) +// log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) +// return *parsedValue, nil +// } +// +// cursorStr := "" +// +// // NOT BEING UPDATED +// // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? +// // Seems like only actions are wrong. Could get the app individually. +// // Guessing it's a memory issue. +// //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` +// //errors.New(nil) +// var err error +// for { +// it := dbclient.Run(ctx, query) +// //_, err = it.Next(&app) +// for { +// var app WorkflowApp +// _, err := it.Next(&app) +// if err != nil { +// break +// } +// +// if app.Name == "Shuffle Subflow" { +// continue +// } +// +// found := false +// //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) +// for _, innerapp := range apps { +// if innerapp.Name == app.Name { +// found = true +// break +// } +// } +// +// if !found { +// apps = append(apps, app) +// } +// } +// +// if err != iterator.Done { +// //log.Printf("[INFO] Failed fetching results: %v", err) +// //break +// } +// +// // Get the cursor for the next page of results. +// nextCursor, err := it.Cursor() +// if err != nil { +// log.Printf("Cursorerror: %s", err) +// break +// } else { +// //log.Printf("NEXTCURSOR: %s", nextCursor) +// nextStr := fmt.Sprintf("%s", nextCursor) +// if cursorStr == nextStr { +// break +// } +// +// cursorStr = nextStr +// query = query.Start(nextCursor) +// //cursorStr = nextCursor +// //break +// } +// +// if len(apps) > maxLen { +// break +// } +// } +// +// if len(apps) > 20 { +// log.Printf("[INFO] Setting %d apps in cache", len(apps)) +// requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) +// } +// +// //var allworkflowapps []WorkflowApp +// //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) +// //if err != nil { +// // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { +// // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") +// // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) +// // //q := q.Limit(25) +// // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) +// // if err != nil { +// // return []WorkflowApp{}, err +// // } +// // } else { +// // return []WorkflowApp{}, err +// // } +// //} +// +// return apps, nil +//} - cacheKey := fmt.Sprintf("workflowapps-sorted-%d", maxLen) - if value, found := requestCache.Get(cacheKey); found { - parsedValue := value.(*[]WorkflowApp) - log.Printf("[INFO] Returning %d apps from cache", len(*parsedValue)) - return *parsedValue, nil - } - - cursorStr := "" - - // NOT BEING UPDATED - // FIXME: Update the app with the correct actions. HOW DOES THIS WORK?? - // Seems like only actions are wrong. Could get the app individually. - // Guessing it's a memory issue. - //Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions,noindex"` - //errors.New(nil) - var err error - for { - it := dbclient.Run(ctx, query) - //_, err = it.Next(&app) - for { - var app WorkflowApp - _, err := it.Next(&app) - if err != nil { - break - } - - if app.Name == "Shuffle Subflow" { - continue - } - - found := false - //log.Printf("ACTIONS: %d - %s", len(app.Actions), app.Name) - for _, innerapp := range apps { - if innerapp.Name == app.Name { - found = true - break - } - } - - if !found { - apps = append(apps, app) - } - } - - if err != iterator.Done { - //log.Printf("[INFO] Failed fetching results: %v", err) - //break - } - - // Get the cursor for the next page of results. - nextCursor, err := it.Cursor() - if err != nil { - log.Printf("Cursorerror: %s", err) - break - } else { - //log.Printf("NEXTCURSOR: %s", nextCursor) - nextStr := fmt.Sprintf("%s", nextCursor) - if cursorStr == nextStr { - break - } - - cursorStr = nextStr - query = query.Start(nextCursor) - //cursorStr = nextCursor - //break - } - - if len(apps) > maxLen { - break - } - } - - if len(apps) > 20 { - log.Printf("[INFO] Setting %d apps in cache", len(apps)) - requestCache.Set(cacheKey, &apps, cache.DefaultExpiration) - } - - //var allworkflowapps []WorkflowApp - //_, err := dbclient.GetAll(ctx, query, &allworkflowapps) - //if err != nil { - // if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - // //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") - // query = datastore.NewQuery("workflowapp").Order("-edited").Limit(25) - // //q := q.Limit(25) - // _, err := dbclient.GetAll(ctx, query, &allworkflowapps) - // if err != nil { - // return []WorkflowApp{}, err - // } - // } else { - // return []WorkflowApp{}, err - // } - //} - - return apps, nil -} - -func getAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]AppAuthenticationStorage, error) { - var allworkflowapps []AppAuthenticationStorage - q := datastore.NewQuery("workflowappauth").Filter("org_id = ", OrgId) - - _, err := dbclient.GetAll(ctx, q, &allworkflowapps) - if err != nil { - return []AppAuthenticationStorage{}, err - } - - return allworkflowapps, nil -} - -func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { - - key := datastore.NameKey("workflowappauth", id, nil) - appAuth := &AppAuthenticationStorage{} - // New struct, to not add body, author etc - if err := dbclient.Get(ctx, key, appAuth); err != nil { - return &AppAuthenticationStorage{}, err - } - - return appAuth, nil -} - -func setWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { - timeNow := int64(time.Now().Unix()) - if workflowappauth.Created == 0 { - workflowappauth.Created = timeNow - } - - workflowappauth.Edited = timeNow - - key := datastore.NameKey("workflowappauth", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { - log.Printf("Error adding workflow app auth: %s", err) - return err - } - - return nil -} - -// Hmm, so I guess this should use uuid :( -// Consistency PLX -func setWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { - timeNow := int64(time.Now().Unix()) - if workflowapp.Created == 0 { - workflowapp.Created = timeNow - } - - workflowapp.Edited = timeNow - key := datastore.NameKey("workflowapp", id, nil) - - // New struct, to not add body, author etc - if _, err := dbclient.Put(ctx, key, &workflowapp); err != nil { - log.Printf("Error adding workflow app: %s", err) - return err - } - - return nil -} +//func shuffle.GetAllWorkflowAppAuth(ctx context.Context, OrgId string) ([]shuffle.AppAuthenticationStorage, error) { +// var allworkflowapps []AppAuthenticationStorage +// q := datastore.NewQuery("workflowappauth").Filter("org_id = ", OrgId) +// +// _, err := dbclient.GetAll(ctx, q, &allworkflowapps) +// if err != nil { +// return []AppAuthenticationStorage{}, err +// } +// +// return allworkflowapps, nil +//} +// +//func getWorkflowAppAuthDatastore(ctx context.Context, id string) (*AppAuthenticationStorage, error) { +// +// key := datastore.NameKey("workflowappauth", id, nil) +// appAuth := &AppAuthenticationStorage{} +// // New struct, to not add body, author etc +// if err := dbclient.Get(ctx, key, appAuth); err != nil { +// return &AppAuthenticationStorage{}, err +// } +// +// return appAuth, nil +//} +// +//func shuffle.SetWorkflowAppAuthDatastore(ctx context.Context, workflowappauth AppAuthenticationStorage, id string) error { +// timeNow := int64(time.Now().Unix()) +// if workflowappauth.Created == 0 { +// workflowappauth.Created = timeNow +// } +// +// workflowappauth.Edited = timeNow +// +// key := datastore.NameKey("workflowappauth", id, nil) +// +// // New struct, to not add body, author etc +// if _, err := dbclient.Put(ctx, key, &workflowappauth); err != nil { +// log.Printf("Error adding workflow app auth: %s", err) +// return err +// } +// +// return nil +//} +// +//// Hmm, so I guess this should use uuid :( +//// Consistency PLX +//func SetWorkflowAppDatastore(ctx context.Context, workflowapp WorkflowApp, id string) error { +// timeNow := int64(time.Now().Unix()) +// if workflowapp.Created == 0 { +// workflowapp.Created = timeNow +// } +// +// workflowapp.Edited = timeNow +// key := datastore.NameKey("workflowapp", id, nil) +// +// // New struct, to not add body, author etc +// if _, err := dbclient.Put(ctx, key, &workflowapp); err != nil { +// log.Printf("Error adding workflow app: %s", err) +// return err +// } +// +// return nil +//} // Starts a new webhook func handleStopHook(resp http.ResponseWriter, request *http.Request) { @@ -7398,7 +7369,7 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7481,7 +7452,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7543,7 +7514,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { log.Printf("Hook: %#v", hook) if hook.Environment == "cloud" { log.Printf("[INFO] Should STOP cloud webhook https://shuffler.io/api/v1/hooks/webhook_%s", hook.Id) - org, err := getOrg(ctx, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) if err != nil { log.Printf("Failed finding org %s: %s", org.Id, err) return @@ -7622,7 +7593,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) { return } - user, err := handleApiAuthentication(resp, request) + user, err := shuffle.HandleApiAuthentication(resp, request) if err != nil { log.Printf("Api authentication failed in set new workflowhandler: %s", err) resp.WriteHeader(401) @@ -7730,7 +7701,7 @@ func removeOutlookTriggerFunction(ctx context.Context, triggerId string) error { return nil } -func handleUserInput(trigger Trigger, organizationId string, workflowId string, referenceExecution string) error { +func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId string, referenceExecution string) error { // E.g. check email sms := "" email := "" @@ -7768,7 +7739,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, FifthItem: referenceExecution, } - org, err := getOrg(ctx, organizationId) + org, err := shuffle.GetOrg(ctx, organizationId) if err != nil { log.Printf("Failed email send to cloud (1): %s", err) return err @@ -7794,7 +7765,7 @@ func handleUserInput(trigger Trigger, organizationId string, workflowId string, FifthItem: referenceExecution, } - org, err := getOrg(ctx, organizationId) + org, err := shuffle.GetOrg(ctx, organizationId) if err != nil { log.Printf("Failed sms send to cloud (3): %s", err) return err