From 4efaf4c75423d700480b40832f0c77ca4b2f0d4e Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 May 2020 20:06:56 +0200 Subject: [PATCH] Reimplemented app_sdk for code gen --- backend/go-app/main.go | 151 +++++++++++++++++++++- backend/go-app/walkoff.go | 108 ++++++++++++++++ frontend/src/App.js | 6 +- functions/onprem/app_sdk/Dockerfile | 16 --- functions/onprem/app_sdk/LICENSE | 21 +++ functions/onprem/app_sdk/README.md | 12 +- functions/onprem/app_sdk/build.sh | 4 - functions/onprem/app_sdk/requirements.txt | 2 - functions/triggers/msteams/hook.go | 1 + 9 files changed, 285 insertions(+), 36 deletions(-) delete mode 100644 functions/onprem/app_sdk/Dockerfile create mode 100644 functions/onprem/app_sdk/LICENSE delete mode 100644 functions/onprem/app_sdk/build.sh delete mode 100644 functions/onprem/app_sdk/requirements.txt diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3b413141..294c8bb7 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5148,6 +5148,135 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { resp.Write(urlbody) } +func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { + type versionCheck struct { + Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"` + SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"` + OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"` + } + + //body = []byte(`swagger: "2.0"`) + //body = []byte(`swagger: '1.0'`) + //newbody := string(body) + //newbody = strings.TrimSpace(newbody) + //body = []byte(newbody) + //log.Println(string(body)) + //tmpbody, err := yaml.YAMLToJSON(body) + //log.Println(err) + //log.Println(string(tmpbody)) + + // This has to be done in a weird way because Datastore doesn't + // support map[string]interface and similar (openapi3.Swagger) + var version versionCheck + + parsed := ParsedOpenApi{} + swaggerdata := []byte{} + idstring := "" + + isJson := false + err := json.Unmarshal(body, &version) + if err != nil { + //log.Printf("Json err: %s", err) + err = yaml.Unmarshal(body, &version) + if err != nil { + log.Printf("Yaml error: %s", err) + } else { + log.Printf("Successfully parsed YAML!") + } + } else { + isJson = true + log.Printf("Successfully parsed JSON!") + } + + if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 { + version.Swagger = version.SwaggerVersion + } + + if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { + log.Println("Handling v3 API") + swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + if err != nil { + return ParsedOpenApi{}, err + } + + hasher := md5.New() + hasher.Write(body) + idstring = hex.EncodeToString(hasher.Sum(nil)) + + log.Printf("Swagger v3 validation success with ID %s!", idstring) + log.Printf("Paths: %d", len(swagger.Paths)) + + if !isJson { + log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring) + } + + log.Printf("Body: %s", string(swaggerdata)) + + //return nil + //return ParsedOpenApi{}, err + } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { + // Convert + log.Println("Handling v2 API") + var swagger openapi2.Swagger + //log.Println(string(body)) + err = json.Unmarshal(body, &swagger) + if err != nil { + //log.Printf("Json error? %s", err) + err = gyaml.Unmarshal(body, &swagger) + if err != nil { + log.Printf("Yaml error: %s", err) + return ParsedOpenApi{}, err + } else { + log.Printf("Valid yaml!") + } + + } + + swaggerv3, err := openapi2conv.ToV3Swagger(&swagger) + if err != nil { + log.Printf("Failed converting from openapi2 to 3: %s", err) + return ParsedOpenApi{}, err + } + + swaggerdata, err = json.Marshal(swaggerv3) + if err != nil { + log.Printf("Failed unmarshaling v3 data: %s", err) + return ParsedOpenApi{}, err + } + + hasher := md5.New() + hasher.Write(swaggerdata) + idstring = hex.EncodeToString(hasher.Sum(nil)) + if !isJson { + log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring) + } + log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring) + + ctx := context.Background() + err = setOpenApiDatastore(ctx, idstring, parsed) + if err != nil { + log.Printf("Failed uploading openapi2 to datastore: %s", err) + return ParsedOpenApi{}, err + } + + } + + log.Printf("Down here?") + if len(swaggerdata) > 0 { + body = swaggerdata + } + + // Parsing it to swagger 3 + parsed = ParsedOpenApi{ + ID: idstring, + Body: string(body), + Success: true, + } + + return parsed, err +} + +// FIXME: Migrate this to use handleSwaggerValidation() func validateSwagger(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5656,7 +5785,7 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed getting apps: %s", err) } else if err == nil && len(workflowapps) == 0 { - log.Printf("Apps: loading TEST") + log.Printf("Downloading default workflow apps") fs := memfs.New() storer := memory.NewStorage() @@ -5696,6 +5825,26 @@ func runInit(ctx context.Context) { iterateAppGithubFolders(fs, dir, "", "") } + log.Printf("Downloading OpenAPI data for search - EXTRA APPS") + apis := "https://github.com/frikky/OpenAPI-security-definitions" + fs := memfs.New() + storer := memory.NewStorage() + cloneOptions := &git.CloneOptions{ + URL: apis, + } + _, err = git.Clone(storer, fs, cloneOptions) + if err != nil { + log.Printf("Failed loading repo %s into memory: %s", err) + } else { + dir, err := fs.ReadDir("") + if err != nil { + log.Printf("Failed reading folder: %s", err) + } + + iterateOpenApiGithub(fs, dir, "", "") + log.Printf("Finished downloading extra API samples") + } + log.Printf("Finished INIT") } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 9c039655..7234c6b7 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -29,6 +29,7 @@ import ( http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" newscheduler "github.com/carlescere/scheduler" + "github.com/getkin/kin-openapi/openapi3" "github.com/go-git/go-git/v5/storage/memory" //"github.com/gorilla/websocket" //"google.golang.org/appengine" @@ -3257,6 +3258,113 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { + for _, file := range dir { + if len(onlyname) > 0 && file.Name() != onlyname { + continue + } + + // Folder? + switch mode := file.Mode(); { + case mode.IsDir(): + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + dir, err := fs.ReadDir(tmpExtra) + if err != nil { + log.Printf("Failed to read dir: %s", err) + break + } + + // Go routine? Hmm, this can be super quick I guess + err = iterateOpenApiGithub(fs, dir, tmpExtra, "") + if err != nil { + break + } + case mode.IsRegular(): + // Check the file + filename := file.Name() + if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") { + //log.Printf("Found file: %s", filename) + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + + fileReader, err := fs.Open(tmpExtra) + if err != nil { + continue + } + + readFile, err := ioutil.ReadAll(fileReader) + if err != nil { + log.Printf("Filereader error yaml: %s", err) + continue + } + + // 1. This parses OpenAPI v2 to v3 etc, for use. + parsedOpenApi, err := handleSwaggerValidation(readFile) + if err != nil { + log.Printf("Validation error: %s", err) + continue + } + + log.Printf("%#v", parsedOpenApi) + + // 2. With parsedOpenApi.ID: + //http://localhost:3000/apps/new?id=06b1376f77b0563a3b1747a3a1253e88 + + // 3. Load this as a "standby" app + // FIXME: This should be a function ROFL + //log.Printf("%s", string(readFile)) + swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData([]byte(parsedOpenApi.Body)) + if err != nil { + log.Printf("Swagger validation error in loop: %s", err) + continue + } + + if strings.Contains(swagger.Info.Title, " ") { + strings.Replace(swagger.Info.Title, " ", "", -1) + } + + basePath, err := buildStructure(swagger, parsedOpenApi.ID) + if err != nil { + log.Printf("Failed to build base structure in loop: %s", err) + continue + } + + log.Printf("Should generate yaml") + api, pythonfunctions, err := generateYaml(swagger, parsedOpenApi.ID) + if err != nil { + log.Printf("Failed building and generating yaml in loop: %s", err) + continue + } + + // FIXME: Configure user? + api.Owner = "" + err = dumpApi(basePath, api) + if err != nil { + log.Printf("Failed dumping yaml in loop: %s", err) + continue + } + + // FIXME: Should this continue on activation? + + identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, parsedOpenApi.ID) + classname := strings.Replace(identifier, " ", "", -1) + classname = strings.Replace(classname, "-", "", -1) + parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions) + if err != nil { + log.Printf("Failed dumping python in loop: %s", err) + continue + } + + identifier = strings.Replace(identifier, " ", "-", -1) + identifier = strings.Replace(identifier, "_", "-", -1) + + return nil + } + } + } + + return nil +} + // Onlyname is used to func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { var err error diff --git a/frontend/src/App.js b/frontend/src/App.js index a492c501..63c4f860 100644 --- a/frontend/src/App.js +++ b/frontend/src/App.js @@ -38,12 +38,10 @@ import AlertTemplate from "react-alert-template-basic"; import { positions, Provider } from "react-alert"; // Testing - localhost -//const globalUrl = "http://192.168.3.6:5001" -//console.log("HOST: ", process.env) - +const globalUrl = "http://192.168.3.6:5001" // Production - backend proxy forwarding in nginx -const globalUrl = window.location.origin +//const globalUrl = window.location.origin const surfaceColor = "#27292D" const inputColor = "#383B40" diff --git a/functions/onprem/app_sdk/Dockerfile b/functions/onprem/app_sdk/Dockerfile deleted file mode 100644 index 44294a5b..00000000 --- a/functions/onprem/app_sdk/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM python:3.7-alpine as base - -FROM base as builder -RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev - -RUN mkdir /install -WORKDIR /install - -COPY requirements.txt /requirements.txt -RUN pip install --prefix="/install" -r /requirements.txt - -FROM base - -COPY --from=builder /install /usr/local -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/functions/onprem/app_sdk/LICENSE b/functions/onprem/app_sdk/LICENSE new file mode 100644 index 00000000..ce11f6f3 --- /dev/null +++ b/functions/onprem/app_sdk/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Frikkylikeme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/functions/onprem/app_sdk/README.md b/functions/onprem/app_sdk/README.md index a68aa39d..873d45d3 100644 --- a/functions/onprem/app_sdk/README.md +++ b/functions/onprem/app_sdk/README.md @@ -1,10 +1,4 @@ -# app_sdk -This is the SDK used for apps to behave like they should. -To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. +# App_sdk +App_sdk development is continued here: https://github.com/frikky/Shuffle-apps/tree/master/app_sdk -## If you want to update apps.. PS: downloads from docker hub do overrides.. :) -1. Write your code & check if runtime works -2. Build app_base image -3. docker rm $(docker ps -aq) # Remove all stopped containers -4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) -5. Rebuild the Docker image (click load in GUI?) +It's under MIT license, NOT AGPLv3. diff --git a/functions/onprem/app_sdk/build.sh b/functions/onprem/app_sdk/build.sh deleted file mode 100644 index c93c83fc..00000000 --- a/functions/onprem/app_sdk/build.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -docker rmi frikky/shuffle:app_sdk -docker build . -t frikky/shuffle:app_sdk --no-cache -docker push frikky/shuffle:app_sdk diff --git a/functions/onprem/app_sdk/requirements.txt b/functions/onprem/app_sdk/requirements.txt deleted file mode 100644 index 804abb1b..00000000 --- a/functions/onprem/app_sdk/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -requests -urllib3 diff --git a/functions/triggers/msteams/hook.go b/functions/triggers/msteams/hook.go index fae0e594..5a3a0b0d 100644 --- a/functions/triggers/msteams/hook.go +++ b/functions/triggers/msteams/hook.go @@ -168,6 +168,7 @@ func sendRequest(token OauthToken, message TeamsHook) error { return nil } +// If you're finding this: its from a test project :) func get_accesstoken() (OauthToken, error) { client_id := "9a2a2a63-c63c-4487-baf0-4ff3f4873a7f" client_secret := ":3]D6oFimiXbuV20xH?Dzu@LR*6IFVbq"