Reimplemented app_sdk for code gen
This commit is contained in:
frikky
+150
-1
@@ -5148,6 +5148,135 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) {
|
||||
resp.Write(urlbody)
|
||||
}
|
||||
|
||||
func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) {
|
||||
type versionCheck struct {
|
||||
Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"`
|
||||
SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"`
|
||||
OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"`
|
||||
}
|
||||
|
||||
//body = []byte(`swagger: "2.0"`)
|
||||
//body = []byte(`swagger: '1.0'`)
|
||||
//newbody := string(body)
|
||||
//newbody = strings.TrimSpace(newbody)
|
||||
//body = []byte(newbody)
|
||||
//log.Println(string(body))
|
||||
//tmpbody, err := yaml.YAMLToJSON(body)
|
||||
//log.Println(err)
|
||||
//log.Println(string(tmpbody))
|
||||
|
||||
// This has to be done in a weird way because Datastore doesn't
|
||||
// support map[string]interface and similar (openapi3.Swagger)
|
||||
var version versionCheck
|
||||
|
||||
parsed := ParsedOpenApi{}
|
||||
swaggerdata := []byte{}
|
||||
idstring := ""
|
||||
|
||||
isJson := false
|
||||
err := json.Unmarshal(body, &version)
|
||||
if err != nil {
|
||||
//log.Printf("Json err: %s", err)
|
||||
err = yaml.Unmarshal(body, &version)
|
||||
if err != nil {
|
||||
log.Printf("Yaml error: %s", err)
|
||||
} else {
|
||||
log.Printf("Successfully parsed YAML!")
|
||||
}
|
||||
} else {
|
||||
isJson = true
|
||||
log.Printf("Successfully parsed JSON!")
|
||||
}
|
||||
|
||||
if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 {
|
||||
version.Swagger = version.SwaggerVersion
|
||||
}
|
||||
|
||||
if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") {
|
||||
log.Println("Handling v3 API")
|
||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body)
|
||||
if err != nil {
|
||||
return ParsedOpenApi{}, err
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
hasher.Write(body)
|
||||
idstring = hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
log.Printf("Swagger v3 validation success with ID %s!", idstring)
|
||||
log.Printf("Paths: %d", len(swagger.Paths))
|
||||
|
||||
if !isJson {
|
||||
log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring)
|
||||
}
|
||||
|
||||
log.Printf("Body: %s", string(swaggerdata))
|
||||
|
||||
//return nil
|
||||
//return ParsedOpenApi{}, err
|
||||
} else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") {
|
||||
// Convert
|
||||
log.Println("Handling v2 API")
|
||||
var swagger openapi2.Swagger
|
||||
//log.Println(string(body))
|
||||
err = json.Unmarshal(body, &swagger)
|
||||
if err != nil {
|
||||
//log.Printf("Json error? %s", err)
|
||||
err = gyaml.Unmarshal(body, &swagger)
|
||||
if err != nil {
|
||||
log.Printf("Yaml error: %s", err)
|
||||
return ParsedOpenApi{}, err
|
||||
} else {
|
||||
log.Printf("Valid yaml!")
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
swaggerv3, err := openapi2conv.ToV3Swagger(&swagger)
|
||||
if err != nil {
|
||||
log.Printf("Failed converting from openapi2 to 3: %s", err)
|
||||
return ParsedOpenApi{}, err
|
||||
}
|
||||
|
||||
swaggerdata, err = json.Marshal(swaggerv3)
|
||||
if err != nil {
|
||||
log.Printf("Failed unmarshaling v3 data: %s", err)
|
||||
return ParsedOpenApi{}, err
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
hasher.Write(swaggerdata)
|
||||
idstring = hex.EncodeToString(hasher.Sum(nil))
|
||||
if !isJson {
|
||||
log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring)
|
||||
}
|
||||
log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring)
|
||||
|
||||
ctx := context.Background()
|
||||
err = setOpenApiDatastore(ctx, idstring, parsed)
|
||||
if err != nil {
|
||||
log.Printf("Failed uploading openapi2 to datastore: %s", err)
|
||||
return ParsedOpenApi{}, err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
log.Printf("Down here?")
|
||||
if len(swaggerdata) > 0 {
|
||||
body = swaggerdata
|
||||
}
|
||||
|
||||
// Parsing it to swagger 3
|
||||
parsed = ParsedOpenApi{
|
||||
ID: idstring,
|
||||
Body: string(body),
|
||||
Success: true,
|
||||
}
|
||||
|
||||
return parsed, err
|
||||
}
|
||||
|
||||
// FIXME: Migrate this to use handleSwaggerValidation()
|
||||
func validateSwagger(resp http.ResponseWriter, request *http.Request) {
|
||||
cors := handleCors(resp, request)
|
||||
if cors {
|
||||
@@ -5656,7 +5785,7 @@ func runInit(ctx context.Context) {
|
||||
if err != nil {
|
||||
log.Printf("Failed getting apps: %s", err)
|
||||
} else if err == nil && len(workflowapps) == 0 {
|
||||
log.Printf("Apps: loading TEST")
|
||||
log.Printf("Downloading default workflow apps")
|
||||
fs := memfs.New()
|
||||
storer := memory.NewStorage()
|
||||
|
||||
@@ -5696,6 +5825,26 @@ func runInit(ctx context.Context) {
|
||||
iterateAppGithubFolders(fs, dir, "", "")
|
||||
}
|
||||
|
||||
log.Printf("Downloading OpenAPI data for search - EXTRA APPS")
|
||||
apis := "https://github.com/frikky/OpenAPI-security-definitions"
|
||||
fs := memfs.New()
|
||||
storer := memory.NewStorage()
|
||||
cloneOptions := &git.CloneOptions{
|
||||
URL: apis,
|
||||
}
|
||||
_, err = git.Clone(storer, fs, cloneOptions)
|
||||
if err != nil {
|
||||
log.Printf("Failed loading repo %s into memory: %s", err)
|
||||
} else {
|
||||
dir, err := fs.ReadDir("")
|
||||
if err != nil {
|
||||
log.Printf("Failed reading folder: %s", err)
|
||||
}
|
||||
|
||||
iterateOpenApiGithub(fs, dir, "", "")
|
||||
log.Printf("Finished downloading extra API samples")
|
||||
}
|
||||
|
||||
log.Printf("Finished INIT")
|
||||
}
|
||||
|
||||
|
||||
@@ -29,6 +29,7 @@ import (
|
||||
http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
|
||||
|
||||
newscheduler "github.com/carlescere/scheduler"
|
||||
"github.com/getkin/kin-openapi/openapi3"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
//"github.com/gorilla/websocket"
|
||||
//"google.golang.org/appengine"
|
||||
@@ -3257,6 +3258,113 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
|
||||
}
|
||||
|
||||
func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
|
||||
for _, file := range dir {
|
||||
if len(onlyname) > 0 && file.Name() != onlyname {
|
||||
continue
|
||||
}
|
||||
|
||||
// Folder?
|
||||
switch mode := file.Mode(); {
|
||||
case mode.IsDir():
|
||||
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
|
||||
dir, err := fs.ReadDir(tmpExtra)
|
||||
if err != nil {
|
||||
log.Printf("Failed to read dir: %s", err)
|
||||
break
|
||||
}
|
||||
|
||||
// Go routine? Hmm, this can be super quick I guess
|
||||
err = iterateOpenApiGithub(fs, dir, tmpExtra, "")
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
case mode.IsRegular():
|
||||
// Check the file
|
||||
filename := file.Name()
|
||||
if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") {
|
||||
//log.Printf("Found file: %s", filename)
|
||||
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
|
||||
|
||||
fileReader, err := fs.Open(tmpExtra)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
readFile, err := ioutil.ReadAll(fileReader)
|
||||
if err != nil {
|
||||
log.Printf("Filereader error yaml: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// 1. This parses OpenAPI v2 to v3 etc, for use.
|
||||
parsedOpenApi, err := handleSwaggerValidation(readFile)
|
||||
if err != nil {
|
||||
log.Printf("Validation error: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("%#v", parsedOpenApi)
|
||||
|
||||
// 2. With parsedOpenApi.ID:
|
||||
//http://localhost:3000/apps/new?id=06b1376f77b0563a3b1747a3a1253e88
|
||||
|
||||
// 3. Load this as a "standby" app
|
||||
// FIXME: This should be a function ROFL
|
||||
//log.Printf("%s", string(readFile))
|
||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData([]byte(parsedOpenApi.Body))
|
||||
if err != nil {
|
||||
log.Printf("Swagger validation error in loop: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if strings.Contains(swagger.Info.Title, " ") {
|
||||
strings.Replace(swagger.Info.Title, " ", "", -1)
|
||||
}
|
||||
|
||||
basePath, err := buildStructure(swagger, parsedOpenApi.ID)
|
||||
if err != nil {
|
||||
log.Printf("Failed to build base structure in loop: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("Should generate yaml")
|
||||
api, pythonfunctions, err := generateYaml(swagger, parsedOpenApi.ID)
|
||||
if err != nil {
|
||||
log.Printf("Failed building and generating yaml in loop: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// FIXME: Configure user?
|
||||
api.Owner = ""
|
||||
err = dumpApi(basePath, api)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping yaml in loop: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// FIXME: Should this continue on activation?
|
||||
|
||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, parsedOpenApi.ID)
|
||||
classname := strings.Replace(identifier, " ", "", -1)
|
||||
classname = strings.Replace(classname, "-", "", -1)
|
||||
parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping python in loop: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
identifier = strings.Replace(identifier, " ", "-", -1)
|
||||
identifier = strings.Replace(identifier, "_", "-", -1)
|
||||
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Onlyname is used to
|
||||
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
|
||||
var err error
|
||||
|
||||
Reference in New Issue
Block a user