marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Reimplemented app_sdk for code gen

Browse Source
This commit is contained in:
frikky
2020-05-23 20:06:56 +02:00
parent d3187b16c0
commit 4efaf4c754
9 changed files with 285 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/main.go
+150 -1
View File
@@ -5148,6 +5148,135 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) {
resp.Write(urlbody)
}
func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) {
type versionCheck struct {
Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"`
SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"`
OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"`
}
//body = []byte(`swagger: "2.0"`)
//body = []byte(`swagger: '1.0'`)
//newbody := string(body)
//newbody = strings.TrimSpace(newbody)
//body = []byte(newbody)
//log.Println(string(body))
//tmpbody, err := yaml.YAMLToJSON(body)
//log.Println(err)
//log.Println(string(tmpbody))
// This has to be done in a weird way because Datastore doesn't
// support map[string]interface and similar (openapi3.Swagger)
var version versionCheck
parsed := ParsedOpenApi{}
swaggerdata := []byte{}
idstring := ""
isJson := false
err := json.Unmarshal(body, &version)
if err != nil {
//log.Printf("Json err: %s", err)
err = yaml.Unmarshal(body, &version)
if err != nil {
log.Printf("Yaml error: %s", err)
} else {
log.Printf("Successfully parsed YAML!")
}
} else {
isJson = true
log.Printf("Successfully parsed JSON!")
}
if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 {
version.Swagger = version.SwaggerVersion
}
if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") {
log.Println("Handling v3 API")
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body)
if err != nil {
return ParsedOpenApi{}, err
}
hasher := md5.New()
hasher.Write(body)
idstring = hex.EncodeToString(hasher.Sum(nil))
log.Printf("Swagger v3 validation success with ID %s!", idstring)
log.Printf("Paths: %d", len(swagger.Paths))
if !isJson {
log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s", idstring)
}
log.Printf("Body: %s", string(swaggerdata))
//return nil
//return ParsedOpenApi{}, err
} else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") {
// Convert
log.Println("Handling v2 API")
var swagger openapi2.Swagger
//log.Println(string(body))
err = json.Unmarshal(body, &swagger)
if err != nil {
//log.Printf("Json error? %s", err)
err = gyaml.Unmarshal(body, &swagger)
if err != nil {
log.Printf("Yaml error: %s", err)
return ParsedOpenApi{}, err
} else {
log.Printf("Valid yaml!")
}
}
swaggerv3, err := openapi2conv.ToV3Swagger(&swagger)
if err != nil {
log.Printf("Failed converting from openapi2 to 3: %s", err)
return ParsedOpenApi{}, err
}
swaggerdata, err = json.Marshal(swaggerv3)
if err != nil {
log.Printf("Failed unmarshaling v3 data: %s", err)
return ParsedOpenApi{}, err
}
hasher := md5.New()
hasher.Write(swaggerdata)
idstring = hex.EncodeToString(hasher.Sum(nil))
if !isJson {
log.Printf("FIXME: NEED TO TRANSFORM FROM YAML TO JSON for %s?", idstring)
}
log.Printf("Swagger v2 -> v3 validation success with ID %s!", idstring)
ctx := context.Background()
err = setOpenApiDatastore(ctx, idstring, parsed)
if err != nil {
log.Printf("Failed uploading openapi2 to datastore: %s", err)
return ParsedOpenApi{}, err
}
}
log.Printf("Down here?")
if len(swaggerdata) > 0 {
body = swaggerdata
}
// Parsing it to swagger 3
parsed = ParsedOpenApi{
ID: idstring,
Body: string(body),
Success: true,
}
return parsed, err
}
// FIXME: Migrate this to use handleSwaggerValidation()
func validateSwagger(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -5656,7 +5785,7 @@ func runInit(ctx context.Context) {
if err != nil {
log.Printf("Failed getting apps: %s", err)
} else if err == nil && len(workflowapps) == 0 {
log.Printf("Apps: loading TEST")
log.Printf("Downloading default workflow apps")
fs := memfs.New()
storer := memory.NewStorage()
@@ -5696,6 +5825,26 @@ func runInit(ctx context.Context) {
iterateAppGithubFolders(fs, dir, "", "")
}
log.Printf("Downloading OpenAPI data for search - EXTRA APPS")
apis := "https://github.com/frikky/OpenAPI-security-definitions"
fs := memfs.New()
storer := memory.NewStorage()
cloneOptions := &git.CloneOptions{
URL: apis,
}
_, err = git.Clone(storer, fs, cloneOptions)
if err != nil {
log.Printf("Failed loading repo %s into memory: %s", err)
} else {
dir, err := fs.ReadDir("")
if err != nil {
log.Printf("Failed reading folder: %s", err)
}
iterateOpenApiGithub(fs, dir, "", "")
log.Printf("Finished downloading extra API samples")
}
log.Printf("Finished INIT")
}
backend/go-app/walkoff.go
+108
View File
@@ -29,6 +29,7 @@ import (
http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
newscheduler "github.com/carlescere/scheduler"
"github.com/getkin/kin-openapi/openapi3"
"github.com/go-git/go-git/v5/storage/memory"
//"github.com/gorilla/websocket"
//"google.golang.org/appengine"
@@ -3257,6 +3258,113 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) {
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
}
func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
for _, file := range dir {
if len(onlyname) > 0 && file.Name() != onlyname {
continue
}
// Folder?
switch mode := file.Mode(); {
case mode.IsDir():
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
dir, err := fs.ReadDir(tmpExtra)
if err != nil {
log.Printf("Failed to read dir: %s", err)
break
}
// Go routine? Hmm, this can be super quick I guess
err = iterateOpenApiGithub(fs, dir, tmpExtra, "")
if err != nil {
break
}
case mode.IsRegular():
// Check the file
filename := file.Name()
if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") {
//log.Printf("Found file: %s", filename)
tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name())
fileReader, err := fs.Open(tmpExtra)
if err != nil {
continue
}
readFile, err := ioutil.ReadAll(fileReader)
if err != nil {
log.Printf("Filereader error yaml: %s", err)
continue
}
// 1. This parses OpenAPI v2 to v3 etc, for use.
parsedOpenApi, err := handleSwaggerValidation(readFile)
if err != nil {
log.Printf("Validation error: %s", err)
continue
}
log.Printf("%#v", parsedOpenApi)
// 2. With parsedOpenApi.ID:
//http://localhost:3000/apps/new?id=06b1376f77b0563a3b1747a3a1253e88
// 3. Load this as a "standby" app
// FIXME: This should be a function ROFL
//log.Printf("%s", string(readFile))
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData([]byte(parsedOpenApi.Body))
if err != nil {
log.Printf("Swagger validation error in loop: %s", err)
continue
}
if strings.Contains(swagger.Info.Title, " ") {
strings.Replace(swagger.Info.Title, " ", "", -1)
}
basePath, err := buildStructure(swagger, parsedOpenApi.ID)
if err != nil {
log.Printf("Failed to build base structure in loop: %s", err)
continue
}
log.Printf("Should generate yaml")
api, pythonfunctions, err := generateYaml(swagger, parsedOpenApi.ID)
if err != nil {
log.Printf("Failed building and generating yaml in loop: %s", err)
continue
}
// FIXME: Configure user?
api.Owner = ""
err = dumpApi(basePath, api)
if err != nil {
log.Printf("Failed dumping yaml in loop: %s", err)
continue
}
// FIXME: Should this continue on activation?
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, parsedOpenApi.ID)
classname := strings.Replace(identifier, " ", "", -1)
classname = strings.Replace(classname, "-", "", -1)
parsedCode, err := dumpPython(basePath, classname, swagger.Info.Version, pythonfunctions)
if err != nil {
log.Printf("Failed dumping python in loop: %s", err)
continue
}
identifier = strings.Replace(identifier, " ", "-", -1)
identifier = strings.Replace(identifier, "_", "-", -1)
return nil
}
}
}
return nil
}
// Onlyname is used to
func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error {
var err error
frontend/src/App.js
+2 -4
View File
@@ -38,12 +38,10 @@ import AlertTemplate from "react-alert-template-basic";
import { positions, Provider } from "react-alert";
// Testing - localhost
//const globalUrl = "http://192.168.3.6:5001"
//console.log("HOST: ", process.env)
const globalUrl = "http://192.168.3.6:5001"
// Production - backend proxy forwarding in nginx
const globalUrl = window.location.origin
//const globalUrl = window.location.origin
const surfaceColor = "#27292D"
const inputColor = "#383B40"
functions/onprem/app_sdk/Dockerfile
-16
View File
@@ -1,16 +0,0 @@
FROM python:3.7-alpine as base
FROM base as builder
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip install --prefix="/install" -r /requirements.txt
FROM base
COPY --from=builder /install /usr/local
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
functions/onprem/app_sdk/LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2020 Frikkylikeme
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
functions/onprem/app_sdk/README.md
+3 -9
View File
@@ -1,10 +1,4 @@
# app_sdk
This is the SDK used for apps to behave like they should.
To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline.
# App_sdk
App_sdk development is continued here: https://github.com/frikky/Shuffle-apps/tree/master/app_sdk
## If you want to update apps.. PS: downloads from docker hub do overrides.. :)
1. Write your code & check if runtime works
2. Build app_base image
3. docker rm $(docker ps -aq) # Remove all stopped containers
4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
5. Rebuild the Docker image (click load in GUI?)
It's under MIT license, NOT AGPLv3.
functions/onprem/app_sdk/build.sh
-4
View File
@@ -1,4 +0,0 @@
#!/bin/bash
docker rmi frikky/shuffle:app_sdk
docker build . -t frikky/shuffle:app_sdk --no-cache
docker push frikky/shuffle:app_sdk
functions/onprem/app_sdk/requirements.txt
-2
View File
@@ -1,2 +0,0 @@
requests
urllib3
functions/triggers/msteams/hook.go
+1
View File
@@ -168,6 +168,7 @@ func sendRequest(token OauthToken, message TeamsHook) error {
return nil
}
// If you're finding this: its from a test project :)
func get_accesstoken() (OauthToken, error) {
client_id := "9a2a2a63-c63c-4487-baf0-4ff3f4873a7f"
client_secret := ":3]D6oFimiXbuV20xH?Dzu@LR*6IFVbq"