Merge branch 'main' into nightly
This commit is contained in:
Frikky
GitHub
@@ -20,19 +20,19 @@ jobs:
|
||||
include:
|
||||
- app: frontend
|
||||
path: frontend
|
||||
version: 2.0.2
|
||||
version: 2.1.0-rc2
|
||||
experimental: true
|
||||
- app: backend
|
||||
path: backend
|
||||
version: 2.0.2
|
||||
version: 2.1.0-rc2
|
||||
experimental: true
|
||||
- app: orborus
|
||||
path: functions/onprem/orborus
|
||||
version: 2.0.2
|
||||
version: 2.1.0-rc2
|
||||
experimental: true
|
||||
- app: worker
|
||||
path: functions/onprem/worker
|
||||
version: 2.0.2
|
||||
version: 2.1.0-rc2
|
||||
experimental: true
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
import json
|
||||
import yaml
|
||||
|
||||
items = []
|
||||
|
||||
openapi = {
|
||||
"openapi": "3.0.2",
|
||||
"info": {
|
||||
"title": "MISP",
|
||||
"description": "MISP API generated from the misp book: https://github.com/MISP/misp-book/blob/master/automation/README.md",
|
||||
"version": "1.0.0",
|
||||
"contact": {
|
||||
"name": "@frikkylikeme",
|
||||
"url": "https://twitter.com/frikkylikeme",
|
||||
"email": "frikky@shuffler.io"
|
||||
}
|
||||
},
|
||||
"paths": {},
|
||||
"components": {
|
||||
"schemas": {},
|
||||
"securitySchemes": {
|
||||
"ApiKeyAuth": {
|
||||
"type": "apikey",
|
||||
"in": "header",
|
||||
"name": "Authorization",
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
with open("misp.txt", "r") as tmp:
|
||||
newitem = {}
|
||||
recorditem = False
|
||||
|
||||
counter = 0
|
||||
itemsplit = tmp.read().split("\n")
|
||||
for item in itemsplit:
|
||||
counter += 1
|
||||
if item.startswith("### ") and "/" in item:
|
||||
try:
|
||||
path = item.split(" ")[2]
|
||||
method = item.split(" ")[1].lower()
|
||||
newitem = {
|
||||
"path": path,
|
||||
"method": method,
|
||||
}
|
||||
|
||||
try:
|
||||
openapi["paths"][path][method] = {}
|
||||
except KeyError:
|
||||
openapi["paths"][path] = {}
|
||||
openapi["paths"][path][method] = {}
|
||||
|
||||
except IndexError:
|
||||
newitem = {}
|
||||
continue
|
||||
|
||||
recorditem = True
|
||||
#print(newitem)
|
||||
|
||||
if not recorditem:
|
||||
continue
|
||||
|
||||
if "Description" in item:
|
||||
openapi["paths"][newitem["path"]][newitem["method"]]["description"] = itemsplit[counter+1]
|
||||
elif "URL Arguments" in item:
|
||||
parameters = []
|
||||
innercnt = 0
|
||||
|
||||
openapi["paths"][newitem["path"]][newitem["method"]]["parameters"] = []
|
||||
while True:
|
||||
curline = itemsplit[counter+1+innercnt]
|
||||
if "#" in curline:
|
||||
break
|
||||
|
||||
innercnt += 1
|
||||
if not curline:
|
||||
continue
|
||||
|
||||
print(curline)
|
||||
parameters.append({
|
||||
"description": curline.split(" ")[1],
|
||||
"in": "query",
|
||||
"name": curline.split(" ")[1],
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
})
|
||||
|
||||
openapi["paths"][newitem["path"]][newitem["method"]]["parameters"] = parameters
|
||||
elif "Output" in item:
|
||||
# FIXME
|
||||
innercnt = 0
|
||||
while True:
|
||||
curline = itemsplit[counter+1+innercnt]
|
||||
|
||||
if "#" in curline:
|
||||
break
|
||||
|
||||
innercnt += 1
|
||||
if "json" in curline:
|
||||
continue
|
||||
#print(curline)
|
||||
|
||||
print(json.dumps(openapi, indent=4))
|
||||
|
||||
|
||||
generatedfile = "generated/misp.yaml"
|
||||
with open(generatedfile, "w+") as tmp:
|
||||
tmp.write(yaml.dump(openapi))
|
||||
@@ -1,311 +0,0 @@
|
||||
import requests
|
||||
import yaml
|
||||
import json
|
||||
import os
|
||||
import io
|
||||
import base64
|
||||
from PIL import Image
|
||||
#import tkinter
|
||||
#import _tkinter
|
||||
#tkinter._test()
|
||||
|
||||
|
||||
#sudo apt-get install python-imaging-tk
|
||||
#sudo apt-get install python3-tk
|
||||
|
||||
# USAGE:
|
||||
# 1. Find the item here:
|
||||
# https://apphub.swimlane.com/swimbundles/swimlane/sw_alienvault_threatcrowd
|
||||
# 2.
|
||||
# https://jsonlint.com/
|
||||
|
||||
# META: data["meta"]. Stuff like count. May be useful :)
|
||||
|
||||
def parse_data(data):
|
||||
openapi = {
|
||||
"openapi": "3.0.2",
|
||||
"info": {
|
||||
"title": "",
|
||||
"description": "",
|
||||
"version": "1.0.0",
|
||||
"contact": {
|
||||
"name": "@frikkylikeme",
|
||||
"url": "https://twitter.com/frikkylikeme",
|
||||
"email": "frikky@shuffler.io"
|
||||
}
|
||||
},
|
||||
"paths": {},
|
||||
"components": {
|
||||
"schemas": {},
|
||||
"securitySchemes": {},
|
||||
}
|
||||
}
|
||||
|
||||
data = data["swimbundle"]
|
||||
filename = "%s.yaml" % data["product"].replace(" ", "_").lower()
|
||||
openapi["info"]["title"] = "%s %s" % (data["vendor"], data["product"])
|
||||
openapi["info"]["description"] = "Automated generation of %s" % (openapi["info"]["title"])
|
||||
# data["description"]
|
||||
|
||||
# https://swagger.io/docs/specification/authentication/
|
||||
try:
|
||||
asset = data["asset"]
|
||||
inputparams = asset["inputParameters"]
|
||||
|
||||
try:
|
||||
openapi["servers"] = [inputparams["api_url"]["example"]]
|
||||
except KeyError as e:
|
||||
#print(inputparams)
|
||||
#print("Field error: %s" % e)
|
||||
pass
|
||||
|
||||
authset = False
|
||||
try:
|
||||
tmpauth = inputparams["api_user"]
|
||||
tmpauth = inputparams["api_key"]
|
||||
|
||||
openapi["components"]["securitySchemes"] = {
|
||||
"BasicAuth": {
|
||||
"type": "http",
|
||||
"scheme": "basic"
|
||||
}
|
||||
}
|
||||
authset = True
|
||||
except KeyError as e:
|
||||
pass
|
||||
|
||||
try:
|
||||
tmpauth = inputparams["username"]
|
||||
tmpauth = inputparams["password"]
|
||||
|
||||
openapi["components"]["securitySchemes"] = {
|
||||
"BasicAuth": {
|
||||
"type": "http",
|
||||
"scheme": "basic"
|
||||
}
|
||||
}
|
||||
authset = True
|
||||
except KeyError as e:
|
||||
pass
|
||||
|
||||
#if not authset:
|
||||
# print("AUTH NOT SET: %s" % inputparams)
|
||||
|
||||
except KeyError as e:
|
||||
print("KeyError asset: %s" % e)
|
||||
|
||||
cnt = 0
|
||||
paramnames = []
|
||||
for task in data["tasks"]:
|
||||
method = "post"
|
||||
|
||||
openapi["paths"]["tmp%d" % cnt] = {}
|
||||
openapi["paths"]["tmp%d" % cnt][method] = {
|
||||
"summary": task["name"],
|
||||
"description": task["description"],
|
||||
"parameters": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Successful request",
|
||||
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
taskcategory = task["family"]
|
||||
taskname = task["name"]
|
||||
paramnames.append(taskname)
|
||||
|
||||
for key, value in task["inputParameters"].items():
|
||||
schema = "string"
|
||||
inVar = "query"
|
||||
|
||||
if value["type"] == 6:
|
||||
inVar = "body"
|
||||
|
||||
schema = "string"
|
||||
schemaset = False
|
||||
if value["type"] != 1:
|
||||
if (value["type"] == 7):
|
||||
schema = "boolean"
|
||||
schemaset = True
|
||||
|
||||
if schema == "string" and schemaset:
|
||||
print("Should change type: %d" % value["type"])
|
||||
print(task["name"])
|
||||
print(value["name"])
|
||||
|
||||
example = ""
|
||||
try:
|
||||
example = value["example"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
description = ""
|
||||
try:
|
||||
description = value["description"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
required = False
|
||||
try:
|
||||
required = value["required"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
openapi["paths"]["tmp%d" % cnt][method]["parameters"].append({
|
||||
"name": value["name"],
|
||||
"required": required,
|
||||
"example": example,
|
||||
"description": description,
|
||||
"schema": {"type": schema},
|
||||
"in": inVar
|
||||
})
|
||||
|
||||
if len(task["availableOutputVariables"]) > 0:
|
||||
openapi["paths"]["tmp%d" % cnt][method]["responses"]["200"]["content"] = {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/tmp%d" % cnt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#responses:
|
||||
# '200':
|
||||
# content:
|
||||
# application/json:
|
||||
# schema:
|
||||
# $ref: '#/components/schemas/tmp1'
|
||||
#description: Successful request
|
||||
|
||||
openapi["components"]["schemas"]["tmp%d" % cnt] = {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
}
|
||||
|
||||
for key, value in task["availableOutputVariables"].items():
|
||||
if key == "response_code":
|
||||
continue
|
||||
|
||||
openapi["components"]["schemas"]["tmp%d" % cnt]["properties"][key] = {
|
||||
"type": "string"
|
||||
}
|
||||
|
||||
cnt += 1
|
||||
|
||||
print("%s: %d" % (openapi["info"]["title"], len(paramnames)))
|
||||
|
||||
return filename, openapi
|
||||
|
||||
def dump_data(filename, openapi, category):
|
||||
generatedfile = "generated/%s/%s" % (category, filename)
|
||||
try:
|
||||
with open(generatedfile, "w+") as tmp:
|
||||
tmp.write(yaml.dump(openapi))
|
||||
except FileNotFoundError:
|
||||
try:
|
||||
os.mkdir("generated/%s" % category)
|
||||
with open(generatedfile, "w+") as tmp:
|
||||
tmp.write(yaml.dump(openapi))
|
||||
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
#https://apphub.swimlane.com/
|
||||
categories = [
|
||||
"Investigation",
|
||||
"Endpoint Security & Management",
|
||||
"Network Security & Management",
|
||||
"Communication",
|
||||
"SIEM & Log Management",
|
||||
"Governance & Risk Management",
|
||||
"Vulnerability & Patch Management",
|
||||
"Ticket Management",
|
||||
"DevOps & Application Security",
|
||||
"Identity & Access Management",
|
||||
"Infrastructure",
|
||||
"Miscellaneous",
|
||||
]
|
||||
|
||||
search_category = categories[2]
|
||||
total = 0
|
||||
for search_category in categories:
|
||||
number = 1
|
||||
innertotal = 0
|
||||
|
||||
while(True):
|
||||
url = "https://apphub.swimlane.io/api/search/swimbundles?page=%d" % number
|
||||
|
||||
json = {"fields": {"family": search_category}}
|
||||
ret = requests.post(
|
||||
url,
|
||||
json=json,
|
||||
)
|
||||
|
||||
if ret.status_code != 201:
|
||||
print("RET NOT 201: %d" % ret.status_code)
|
||||
break
|
||||
|
||||
parsed = ret.json()
|
||||
try:
|
||||
category = parsed["data"][0]["swimbundleMeta"]["family"][0]
|
||||
except KeyError:
|
||||
category = ""
|
||||
except IndexError:
|
||||
category = ""
|
||||
|
||||
if category == "":
|
||||
break
|
||||
|
||||
for data in parsed["data"]:
|
||||
try:
|
||||
filename, openapi = parse_data(data)
|
||||
except:
|
||||
try:
|
||||
print("Skipping %s %s because of an error" % (data["vendor"], data["product"]))
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
continue
|
||||
|
||||
openapi["tags"] = [
|
||||
{
|
||||
"name": category,
|
||||
}
|
||||
]
|
||||
|
||||
appid = data["swimbundleMeta"]["logo"]["id"]
|
||||
logoUrl = "https://apphub.swimlane.io/api/logos/%s" % appid
|
||||
logodata = requests.get(logoUrl)
|
||||
if logodata.status_code == 200:
|
||||
logojson = logodata.json()
|
||||
try:
|
||||
logobase64 = logojson["data"]["base64"]
|
||||
#.split(",")[1]
|
||||
|
||||
openapi["info"]["x-logo"] = logobase64
|
||||
#print(logobase64)
|
||||
#msg = base64.b64decode(logobase64)
|
||||
#with io.BytesIO(msg) as buf:
|
||||
# with Image.open(buf) as tempImg:
|
||||
# newWidth = 174 / tempImg.width # change this to what ever width you need.
|
||||
# newHeight = 174 / tempImg.height # change this to what ever height you need.
|
||||
# newSize = (int(newWidth * tempImg.width), int(newHeight * tempImg.height))
|
||||
# newImg1 = tempImg.resize(newSize)
|
||||
# lbl1.IMG = ImageTk.PhotoImage(image=newImg1)
|
||||
# lbl1.configure(image=lbl1.IMG)
|
||||
except KeyError:
|
||||
print("Failed logo parsing for %s" % appid)
|
||||
pass
|
||||
|
||||
dump_data(filename, openapi, category)
|
||||
innertotal += 1
|
||||
total += 1
|
||||
|
||||
number += 1
|
||||
|
||||
print("Created %d openapi specs from Swimlane with category %s" % (innertotal, search_category))
|
||||
|
||||
print("\nCreated %d TOTAL openapi specs from Swimlane" % (total))
|
||||
@@ -1,7 +0,0 @@
|
||||
# OpenAPI generator
|
||||
This contains test code that's been moved to shaffuru/backend/go-app/codegen.go
|
||||
|
||||
## Todo:
|
||||
1. Don't use filesystem, but rather store in GCP
|
||||
2. Add swagger 2.0 to 3.0 converter
|
||||
3. Fix body / data parsing
|
||||
@@ -1,29 +0,0 @@
|
||||
# Base our app image off of the WALKOFF App SDK image
|
||||
FROM frikky/shuffle:app_sdk as base
|
||||
|
||||
# We're going to stage away all of the bloat from the build tools so lets create a builder stage
|
||||
#FROM base as builder
|
||||
|
||||
# Install all alpine build tools needed for our pip installs
|
||||
#RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
|
||||
|
||||
# Install all of our pip packages in a single directory that we can copy to our base image later
|
||||
#RUN mkdir /install
|
||||
#WORKDIR /install
|
||||
#COPY requirements.txt /requirements.txt
|
||||
#
|
||||
## Switch back to our base image and copy in all of our built packages and source code
|
||||
#FROM base
|
||||
#COPY --from=builder /install /usr/local
|
||||
|
||||
|
||||
# Install any binary dependencies needed in our final image - this can be a lot of different stuff
|
||||
#RUN apk --no-cache add --update libmagic
|
||||
WORKDIR /
|
||||
COPY requirements.txt /requirements.txt
|
||||
RUN pip install --prefix="/usr/local" -r /requirements.txt
|
||||
COPY src /app
|
||||
|
||||
# Finally, lets run our app!
|
||||
WORKDIR /app
|
||||
CMD python app.py --log-level DEBUG
|
||||
@@ -1,11 +0,0 @@
|
||||
# No extra requirements needed
|
||||
requests==2.32.3
|
||||
urllib3==2.3.0
|
||||
liquidpy==0.8.2
|
||||
MarkupSafe==3.0.2
|
||||
flask[async]==3.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
PyJWT==2.10.1
|
||||
cryptography==44.0.2
|
||||
shufflepy==0.1.0
|
||||
shuffle-sdk==0.0.25
|
||||
@@ -1,387 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/getkin/kin-openapi/openapi3"
|
||||
"gopkg.in/yaml.v2"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type WorkflowApp struct {
|
||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
||||
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
|
||||
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
|
||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
|
||||
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
|
||||
ContactInfo struct {
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
||||
}
|
||||
|
||||
type AuthenticationParams struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"s`
|
||||
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
}
|
||||
|
||||
type Authentication struct {
|
||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
||||
}
|
||||
|
||||
type AuthenticationStore struct {
|
||||
Key string `json:"key" datastore:"key"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
}
|
||||
|
||||
type WorkflowAppActionParameter struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
||||
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
|
||||
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
}
|
||||
|
||||
type SchemaDefinition struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
}
|
||||
|
||||
type WorkflowAppAction struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
NodeType string `json:"node_type" datastore:"node_type"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Returns struct {
|
||||
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
} `json:"returns" datastore:"returns"`
|
||||
}
|
||||
|
||||
func copyFile(fromfile, tofile string) error {
|
||||
from, err := os.Open(fromfile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer from.Close()
|
||||
|
||||
to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer to.Close()
|
||||
|
||||
_, err = io.Copy(to, from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Builds the base structure for the app that we're making
|
||||
// Returns error if anything goes wrong. This has to work if
|
||||
// the python code is supposed to be generated
|
||||
func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
|
||||
//log.Printf("%#v", swagger)
|
||||
|
||||
// adding md5 based on input data to not overwrite earlier data.
|
||||
generatedPath := "generated"
|
||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
||||
appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
|
||||
|
||||
os.MkdirAll(appPath, os.ModePerm)
|
||||
os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm)
|
||||
|
||||
err := copyFile("baseline/Dockerfile", fmt.Sprintf("%s/%s", appPath, "Dockerfile"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move Dockerfile")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
err = copyFile("baseline/requirements.txt", fmt.Sprintf("%s/%s", appPath, "requirements.txt"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move requrements.txt")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
return appPath, nil
|
||||
}
|
||||
|
||||
func makePythoncode(name, url, method string, parameters, optionalQueries []string) string {
|
||||
method = strings.ToLower(method)
|
||||
queryString := ""
|
||||
queryData := ""
|
||||
|
||||
// FIXME - this might break - need to check if ? or & should be set as query
|
||||
parameterData := ""
|
||||
if len(optionalQueries) > 0 {
|
||||
queryString += ", "
|
||||
for _, query := range optionalQueries {
|
||||
queryString += fmt.Sprintf("%s=\"\"", query)
|
||||
queryData += fmt.Sprintf(`
|
||||
if %s:
|
||||
url += f"&%s={%s}"`, query, query, query)
|
||||
}
|
||||
}
|
||||
|
||||
if len(parameters) > 0 {
|
||||
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
|
||||
}
|
||||
|
||||
// FIXME - add checks for query data etc
|
||||
data := fmt.Sprintf(` async def %s_%s(self%s%s):
|
||||
url=f"%s"
|
||||
%s
|
||||
return requests.%s(url).text
|
||||
`, name, method, parameterData, queryString, url, queryData, method)
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
|
||||
api := WorkflowApp{}
|
||||
log.Printf("%#v", swagger.Info)
|
||||
|
||||
if len(swagger.Info.Title) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.")
|
||||
}
|
||||
|
||||
if len(swagger.Servers) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'")
|
||||
}
|
||||
|
||||
api.Name = swagger.Info.Title
|
||||
api.Description = swagger.Info.Description
|
||||
api.IsValid = true
|
||||
api.Link = swagger.Servers[0].URL // host does not exist lol
|
||||
api.AppVersion = "1.0.0"
|
||||
api.Environment = "cloud"
|
||||
api.ID = ""
|
||||
api.SmallImage = ""
|
||||
api.LargeImage = ""
|
||||
|
||||
// This is the python code to be generated
|
||||
// Could just as well be go at this point lol
|
||||
pythonFunctions := []string{}
|
||||
|
||||
for actualPath, path := range swagger.Paths {
|
||||
//log.Printf("%#v", path)
|
||||
//log.Printf("%#v", actualPath)
|
||||
// Find the path name and add it to makeCode() param
|
||||
|
||||
firstQuery := true
|
||||
if path.Get != nil {
|
||||
// What to do with this, hmm
|
||||
functionName := strings.ReplaceAll(path.Get.Summary, " ", "_")
|
||||
functionName = strings.ToLower(functionName)
|
||||
|
||||
action := WorkflowAppAction{
|
||||
Description: path.Get.Description,
|
||||
Name: path.Get.Summary,
|
||||
NodeType: "action",
|
||||
Environment: api.Environment,
|
||||
Parameters: []WorkflowAppActionParameter{},
|
||||
}
|
||||
|
||||
action.Returns.Schema.Type = "string"
|
||||
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
|
||||
|
||||
//log.Println(path.Parameters)
|
||||
|
||||
// Parameters: []WorkflowAppActionParameter{},
|
||||
// FIXME - add data for POST stuff
|
||||
firstQuery = true
|
||||
optionalQueries := []string{}
|
||||
parameters := []string{}
|
||||
optionalParameters := []WorkflowAppActionParameter{}
|
||||
if len(path.Get.Parameters) > 0 {
|
||||
for _, param := range path.Get.Parameters {
|
||||
curParam := WorkflowAppActionParameter{
|
||||
Name: param.Value.Name,
|
||||
Description: param.Value.Description,
|
||||
Multiline: false,
|
||||
Required: param.Value.Required,
|
||||
Schema: SchemaDefinition{
|
||||
Type: param.Value.Schema.Value.Type,
|
||||
},
|
||||
}
|
||||
|
||||
if param.Value.Required {
|
||||
action.Parameters = append(action.Parameters, curParam)
|
||||
} else {
|
||||
optionalParameters = append(optionalParameters, curParam)
|
||||
}
|
||||
|
||||
if param.Value.In == "path" {
|
||||
log.Printf("PATH!: %s", param.Value.Name)
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
|
||||
} else if param.Value.In == "query" {
|
||||
log.Printf("QUERY!: %s", param.Value.Name)
|
||||
if !param.Value.Required {
|
||||
optionalQueries = append(optionalQueries, param.Value.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
|
||||
if firstQuery {
|
||||
baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
} else {
|
||||
baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// ensuring that they end up last in the specification
|
||||
// (order is ish important for optional params) - they need to be last.
|
||||
for _, optionalParam := range optionalParameters {
|
||||
action.Parameters = append(action.Parameters, optionalParam)
|
||||
}
|
||||
|
||||
curCode := makePythoncode(functionName, baseUrl, "get", parameters, optionalQueries)
|
||||
pythonFunctions = append(pythonFunctions, curCode)
|
||||
|
||||
api.Actions = append(api.Actions, action)
|
||||
}
|
||||
}
|
||||
|
||||
return api, pythonFunctions, nil
|
||||
}
|
||||
|
||||
func verifyApi(api WorkflowApp) WorkflowApp {
|
||||
if api.AppVersion == "" {
|
||||
api.AppVersion = "1.0.0"
|
||||
}
|
||||
|
||||
return api
|
||||
}
|
||||
|
||||
func dumpPython(basePath, name, version string, pythonFunctions []string) error {
|
||||
//log.Printf("%#v", api)
|
||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
parsedCode := fmt.Sprintf(`import requests
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from walkoff_app_sdk.app_base import AppBase
|
||||
|
||||
class %s(AppBase):
|
||||
"""
|
||||
Autogenerated class by Shuffler
|
||||
"""
|
||||
|
||||
__version__ = "%s"
|
||||
app_name = "%s"
|
||||
|
||||
def __init__(self, redis, logger, console_logger=None):
|
||||
self.verify = False
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
super().__init__(redis, logger, console_logger)
|
||||
|
||||
%s
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CarbonBlack.run(), debug=True)
|
||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(parsedCode)
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpApi(basePath string, api WorkflowApp) error {
|
||||
//log.Printf("%#v", api)
|
||||
data, err := yaml.Marshal(api)
|
||||
if err != nil {
|
||||
log.Printf("Error with yaml marshal: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
data := []byte(`{"swagger":"3.0","info":{"title":"hi","description":"you","version":"1.0"},"servers":[{"url":"https://shuffler.io/api/v1"}],"host":"shuffler.io","basePath":"/api/v1","schemes":["https:"],"paths":{"/workflows":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflows","description":"Get workflows","parameters":[]}},"/workflows/{id}":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflow","description":"Get workflow","parameters":[{"in":"query","name":"forgetme","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"query","name":"anotherone","description":"Generated by shuffler.io OpenAPI","required":false,"schema":{"type":"string"}},{"in":"query","name":"hi","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"path","name":"id","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}}]}}},"securityDefinitions":{}}`)
|
||||
|
||||
hasher := md5.New()
|
||||
hasher.Write(data)
|
||||
newmd5 := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(data)
|
||||
if err != nil {
|
||||
log.Printf("Swagger validation error: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
if strings.Contains(swagger.Info.Title, " ") {
|
||||
strings.ReplaceAll(swagger.Info.Title, " ", "")
|
||||
}
|
||||
|
||||
basePath, err := buildStructure(swagger, newmd5)
|
||||
if err != nil {
|
||||
log.Printf("Failed to build base structure: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
api, pythonfunctions, err := generateYaml(swagger)
|
||||
if err != nil {
|
||||
log.Printf("Failed building and generating yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpApi(basePath, api)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpPython(basePath, swagger.Info.Title, swagger.Info.Version, pythonfunctions)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping python: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
@@ -1,499 +0,0 @@
|
||||
package main
|
||||
|
||||
/*
|
||||
Code used to generate apps from OpenAPI JSON data
|
||||
Any function ending with GCP doesn't use local fileIO, but rather
|
||||
google cloud storage, and also has a normal filesystem version of the
|
||||
same code (doesn't required client as first argument).
|
||||
|
||||
This code is used in the backend to generate apps on the fly for users.
|
||||
All new code is appended to backend/go-app/codegen.go
|
||||
*/
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/getkin/kin-openapi/openapi3"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var bucketName = "shuffler.appspot.com"
|
||||
|
||||
type WorkflowApp struct {
|
||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
||||
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
|
||||
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
|
||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
|
||||
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
|
||||
ContactInfo struct {
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
||||
}
|
||||
|
||||
type AuthenticationParams struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"s`
|
||||
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
}
|
||||
|
||||
type Authentication struct {
|
||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
||||
}
|
||||
|
||||
type AuthenticationStore struct {
|
||||
Key string `json:"key" datastore:"key"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
}
|
||||
|
||||
type WorkflowAppActionParameter struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
||||
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
|
||||
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
}
|
||||
|
||||
type SchemaDefinition struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
}
|
||||
|
||||
type WorkflowAppAction struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
NodeType string `json:"node_type" datastore:"node_type"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Returns struct {
|
||||
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
} `json:"returns" datastore:"returns"`
|
||||
}
|
||||
|
||||
func copyFile(fromfile, tofile string) error {
|
||||
from, err := os.Open(fromfile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer from.Close()
|
||||
|
||||
to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer to.Close()
|
||||
|
||||
_, err = io.Copy(to, from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildStructureGCP(client *storage.Client, swagger *openapi3.Swagger, curHash string) (string, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
// 1. Have baseline in bucket/generated_apps/baseline
|
||||
// 2. Copy the baseline to a new folder with identifier name
|
||||
|
||||
basePath := "generated_apps"
|
||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
||||
appPath := fmt.Sprintf("%s/%s", basePath, identifier)
|
||||
fileNames := []string{"Dockerfile", "requirements.txt"}
|
||||
for _, file := range fileNames {
|
||||
src := client.Bucket(bucketName).Object(fmt.Sprintf("%s/baseline/%s", basePath, file))
|
||||
dst := client.Bucket(bucketName).Object(fmt.Sprintf("%s/%s", appPath, file))
|
||||
if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return appPath, nil
|
||||
}
|
||||
|
||||
// Builds the base structure for the app that we're making
|
||||
// Returns error if anything goes wrong. This has to work if
|
||||
// the python code is supposed to be generated
|
||||
func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
|
||||
//log.Printf("%#v", swagger)
|
||||
|
||||
// adding md5 based on input data to not overwrite earlier data.
|
||||
generatedPath := "generated"
|
||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
||||
appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
|
||||
|
||||
os.MkdirAll(appPath, os.ModePerm)
|
||||
os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm)
|
||||
|
||||
err := copyFile("baseline/Dockerfile", fmt.Sprintf("%s/%s", appPath, "Dockerfile"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move Dockerfile")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
err = copyFile("baseline/requirements.txt", fmt.Sprintf("%s/%s", appPath, "requirements.txt"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move requrements.txt")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
return appPath, nil
|
||||
}
|
||||
|
||||
func makePythoncode(name, url, method string, parameters, optionalQueries []string) string {
|
||||
method = strings.ToLower(method)
|
||||
queryString := ""
|
||||
queryData := ""
|
||||
|
||||
// FIXME - this might break - need to check if ? or & should be set as query
|
||||
parameterData := ""
|
||||
if len(optionalQueries) > 0 {
|
||||
queryString += ", "
|
||||
for _, query := range optionalQueries {
|
||||
queryString += fmt.Sprintf("%s=\"\"", query)
|
||||
queryData += fmt.Sprintf(`
|
||||
if %s:
|
||||
url += f"&%s={%s}"`, query, query, query)
|
||||
}
|
||||
}
|
||||
|
||||
if len(parameters) > 0 {
|
||||
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
|
||||
}
|
||||
|
||||
// FIXME - add checks for query data etc
|
||||
data := fmt.Sprintf(` async def %s_%s(self%s%s):
|
||||
url=f"%s"
|
||||
%s
|
||||
return requests.%s(url).text
|
||||
`, name, method, parameterData, queryString, url, queryData, method)
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
|
||||
api := WorkflowApp{}
|
||||
log.Printf("%#v", swagger.Info)
|
||||
|
||||
if len(swagger.Info.Title) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.")
|
||||
}
|
||||
|
||||
if len(swagger.Servers) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'")
|
||||
}
|
||||
|
||||
api.Name = swagger.Info.Title
|
||||
api.Description = swagger.Info.Description
|
||||
api.IsValid = true
|
||||
api.Link = swagger.Servers[0].URL // host does not exist lol
|
||||
api.AppVersion = "1.0.0"
|
||||
api.Environment = "cloud"
|
||||
api.ID = ""
|
||||
api.SmallImage = ""
|
||||
api.LargeImage = ""
|
||||
|
||||
// This is the python code to be generated
|
||||
// Could just as well be go at this point lol
|
||||
pythonFunctions := []string{}
|
||||
|
||||
for actualPath, path := range swagger.Paths {
|
||||
//log.Printf("%#v", path)
|
||||
//log.Printf("%#v", actualPath)
|
||||
// Find the path name and add it to makeCode() param
|
||||
|
||||
firstQuery := true
|
||||
if path.Get != nil {
|
||||
// What to do with this, hmm
|
||||
functionName := strings.ReplaceAll(path.Get.Summary, " ", "_")
|
||||
functionName = strings.ToLower(functionName)
|
||||
|
||||
action := WorkflowAppAction{
|
||||
Description: path.Get.Description,
|
||||
Name: path.Get.Summary,
|
||||
NodeType: "action",
|
||||
Environment: api.Environment,
|
||||
Parameters: []WorkflowAppActionParameter{},
|
||||
}
|
||||
|
||||
action.Returns.Schema.Type = "string"
|
||||
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
|
||||
|
||||
//log.Println(path.Parameters)
|
||||
|
||||
// Parameters: []WorkflowAppActionParameter{},
|
||||
// FIXME - add data for POST stuff
|
||||
firstQuery = true
|
||||
optionalQueries := []string{}
|
||||
parameters := []string{}
|
||||
optionalParameters := []WorkflowAppActionParameter{}
|
||||
if len(path.Get.Parameters) > 0 {
|
||||
for _, param := range path.Get.Parameters {
|
||||
curParam := WorkflowAppActionParameter{
|
||||
Name: param.Value.Name,
|
||||
Description: param.Value.Description,
|
||||
Multiline: false,
|
||||
Required: param.Value.Required,
|
||||
Schema: SchemaDefinition{
|
||||
Type: param.Value.Schema.Value.Type,
|
||||
},
|
||||
}
|
||||
|
||||
if param.Value.Required {
|
||||
action.Parameters = append(action.Parameters, curParam)
|
||||
} else {
|
||||
optionalParameters = append(optionalParameters, curParam)
|
||||
}
|
||||
|
||||
if param.Value.In == "path" {
|
||||
log.Printf("PATH!: %s", param.Value.Name)
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
|
||||
} else if param.Value.In == "query" {
|
||||
log.Printf("QUERY!: %s", param.Value.Name)
|
||||
if !param.Value.Required {
|
||||
optionalQueries = append(optionalQueries, param.Value.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
|
||||
if firstQuery {
|
||||
baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
} else {
|
||||
baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// ensuring that they end up last in the specification
|
||||
// (order is ish important for optional params) - they need to be last.
|
||||
for _, optionalParam := range optionalParameters {
|
||||
action.Parameters = append(action.Parameters, optionalParam)
|
||||
}
|
||||
|
||||
curCode := makePythoncode(functionName, baseUrl, "get", parameters, optionalQueries)
|
||||
pythonFunctions = append(pythonFunctions, curCode)
|
||||
|
||||
api.Actions = append(api.Actions, action)
|
||||
}
|
||||
}
|
||||
|
||||
return api, pythonFunctions, nil
|
||||
}
|
||||
|
||||
func verifyApi(api WorkflowApp) WorkflowApp {
|
||||
if api.AppVersion == "" {
|
||||
api.AppVersion = "1.0.0"
|
||||
}
|
||||
|
||||
return api
|
||||
}
|
||||
|
||||
func dumpPythonGCP(client *storage.Client, basePath, name, version string, pythonFunctions []string) error {
|
||||
//log.Printf("%#v", api)
|
||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
parsedCode := fmt.Sprintf(`import requests
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from walkoff_app_sdk.app_base import AppBase
|
||||
|
||||
class %s(AppBase):
|
||||
"""
|
||||
Autogenerated class by Shuffler
|
||||
"""
|
||||
|
||||
__version__ = "%s"
|
||||
app_name = "%s"
|
||||
|
||||
def __init__(self, redis, logger, console_logger=None):
|
||||
self.verify = False
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
super().__init__(redis, logger, console_logger)
|
||||
|
||||
%s
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CarbonBlack.run(), debug=True)
|
||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
// Create bucket handle
|
||||
ctx := context.Background()
|
||||
bucket := client.Bucket(bucketName)
|
||||
obj := bucket.Object(fmt.Sprintf("%s/src/app.py", basePath))
|
||||
w := obj.NewWriter(ctx)
|
||||
if _, err := fmt.Fprintf(w, parsedCode); err != nil {
|
||||
return err
|
||||
}
|
||||
// Close, just like writing a file.
|
||||
if err := w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpPython(basePath, name, version string, pythonFunctions []string) error {
|
||||
//log.Printf("%#v", api)
|
||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
parsedCode := fmt.Sprintf(`import requests
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from walkoff_app_sdk.app_base import AppBase
|
||||
|
||||
class %s(AppBase):
|
||||
"""
|
||||
Autogenerated class by Shuffler
|
||||
"""
|
||||
|
||||
__version__ = "%s"
|
||||
app_name = "%s"
|
||||
|
||||
def __init__(self, redis, logger, console_logger=None):
|
||||
self.verify = False
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
super().__init__(redis, logger, console_logger)
|
||||
|
||||
%s
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CarbonBlack.run(), debug=True)
|
||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(parsedCode)
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpApiGCP(client *storage.Client, basePath string, api WorkflowApp) error {
|
||||
//log.Printf("%#v", api)
|
||||
data, err := yaml.Marshal(api)
|
||||
if err != nil {
|
||||
log.Printf("Error with yaml marshal: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Create bucket handle
|
||||
ctx := context.Background()
|
||||
bucket := client.Bucket(bucketName)
|
||||
obj := bucket.Object(fmt.Sprintf("%s/app.yaml", basePath))
|
||||
w := obj.NewWriter(ctx)
|
||||
if _, err := fmt.Fprintf(w, string(data)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Close, just like writing a file.
|
||||
if err := w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpApi(basePath string, api WorkflowApp) error {
|
||||
//log.Printf("%#v", api)
|
||||
data, err := yaml.Marshal(api)
|
||||
if err != nil {
|
||||
log.Printf("Error with yaml marshal: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
data := []byte(`{"swagger":"3.0","info":{"title":"hi","description":"you","version":"1.0"},"servers":[{"url":"https://shuffler.io/api/v1"}],"host":"shuffler.io","basePath":"/api/v1","schemes":["https:"],"paths":{"/workflows":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflows","description":"Get workflows","parameters":[]}},"/workflows/{id}":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflow","description":"Get workflow","parameters":[{"in":"query","name":"forgetme","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"query","name":"anotherone","description":"Generated by shuffler.io OpenAPI","required":false,"schema":{"type":"string"}},{"in":"query","name":"hi","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"path","name":"id","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}}]}}},"securityDefinitions":{}}`)
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create client: %v", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
hasher.Write(data)
|
||||
newmd5 := hex.EncodeToString(hasher.Sum(nil))
|
||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(data)
|
||||
if err != nil {
|
||||
log.Printf("Swagger validation error: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
if strings.Contains(swagger.Info.Title, " ") {
|
||||
strings.ReplaceAll(swagger.Info.Title, " ", "")
|
||||
}
|
||||
|
||||
basePath, err := buildStructureGCP(client, swagger, newmd5)
|
||||
if err != nil {
|
||||
log.Printf("Failed to build base structure: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
api, pythonfunctions, err := generateYaml(swagger)
|
||||
if err != nil {
|
||||
log.Printf("Failed building and generating yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpApiGCP(client, basePath, api)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpPythonGCP(client, basePath, swagger.Info.Title, swagger.Info.Version, pythonfunctions)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping python: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
@@ -353,6 +353,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
|
||||
}
|
||||
|
||||
if len(environment) > 0 {
|
||||
|
||||
env, err := shuffle.GetEnvironment(ctx, environment, foundId)
|
||||
if err != nil {
|
||||
log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err)
|
||||
|
||||
+5
-5
@@ -1,6 +1,6 @@
|
||||
services:
|
||||
frontend:
|
||||
image: ghcr.io/shuffle/shuffle-frontend:nightly
|
||||
image: ghcr.io/shuffle/shuffle-frontend:latest
|
||||
container_name: shuffle-frontend
|
||||
hostname: shuffle-frontend
|
||||
ports:
|
||||
@@ -14,7 +14,7 @@ services:
|
||||
depends_on:
|
||||
- backend
|
||||
backend:
|
||||
image: ghcr.io/shuffle/shuffle-backend:nightly
|
||||
image: ghcr.io/shuffle/shuffle-backend:latest
|
||||
container_name: shuffle-backend
|
||||
hostname: ${BACKEND_HOSTNAME}
|
||||
# Here for debugging:
|
||||
@@ -33,7 +33,7 @@ services:
|
||||
- SHUFFLE_FILE_LOCATION=/shuffle-files
|
||||
restart: unless-stopped
|
||||
orborus:
|
||||
image: ghcr.io/shuffle/shuffle-orborus:nightly
|
||||
image: ghcr.io/shuffle/shuffle-orborus:latest
|
||||
container_name: shuffle-orborus
|
||||
hostname: shuffle-orborus
|
||||
networks:
|
||||
@@ -55,7 +55,7 @@ services:
|
||||
- SHUFFLE_STATS_DISABLED=true
|
||||
- SHUFFLE_LOGS_DISABLED=true
|
||||
- SHUFFLE_SWARM_CONFIG=run
|
||||
- SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:nightly
|
||||
- SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest
|
||||
env_file: .env
|
||||
restart: unless-stopped
|
||||
security_opt:
|
||||
@@ -65,7 +65,7 @@ services:
|
||||
hostname: shuffle-opensearch
|
||||
container_name: shuffle-opensearch
|
||||
environment:
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms2048m -Xmx2048m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms4096m -Xmx4096m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
|
||||
- bootstrap.memory_lock=true
|
||||
- DISABLE_PERFORMANCE_ANALYZER_AGENT_CLI=true
|
||||
- cluster.initial_master_nodes=shuffle-opensearch
|
||||
|
||||
+1
-1
@@ -27,7 +27,7 @@ COPY ./*.json /usr/src/app/
|
||||
RUN npm run build --loglevel verbose 2>&1
|
||||
|
||||
# Production environment
|
||||
FROM nginx:1.26.0
|
||||
FROM nginx:1.29.0
|
||||
|
||||
RUN mkdir -p /usr/share/nginx/html/build
|
||||
RUN mkdir -p /usr/share/nginx/html/css
|
||||
|
||||
@@ -33,7 +33,6 @@
|
||||
"cytoscape-node-html-label": "^1.2.2",
|
||||
"cytoscape-panzoom": "^2.5.3",
|
||||
"cytoscape-undo-redo": "^1.3.3",
|
||||
"d3": "~4.10.0",
|
||||
"dayjs": "^1.11.10",
|
||||
"dotenv": "^6.1.0",
|
||||
"downshift": "^3.4.8",
|
||||
@@ -51,8 +50,6 @@
|
||||
"json-bigint": "^1.0.0",
|
||||
"match-sorter": "^6.3.1",
|
||||
"md5-file": "^4.0.0",
|
||||
"mdbreact": "^4.21.1",
|
||||
"moment": "~2.29.4",
|
||||
"mui-chips-input": "^2.1.3",
|
||||
"mui-nested-menu": "^3.2.1",
|
||||
"react": "^18.3.1",
|
||||
@@ -60,7 +57,6 @@
|
||||
"react-alice-carousel": "^2.6.4",
|
||||
"react-avatar-editor": "^11.1.0",
|
||||
"react-beforeunload": "^2.2.1",
|
||||
"react-chartjs-2": "^2.11.2",
|
||||
"react-cookie": "^4.0.1",
|
||||
"react-cytoscapejs": "^2.0.0",
|
||||
"react-device-detect": "^2.2.3",
|
||||
@@ -91,11 +87,10 @@
|
||||
"shellwords": "^1.0.1",
|
||||
"simplebar": "^4.2.3",
|
||||
"styled-components": "^4.4.1",
|
||||
"sync-fetch": "^0.3.0",
|
||||
"terser-webpack-plugin": "^4.2.3",
|
||||
"yaml": "^1.10.0",
|
||||
"yamljs": "^0.3.0",
|
||||
"zone.js": "~0.8.26"
|
||||
"zone.js": "^0.15.1"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "HTTPS=false&&PORT=3000 GENERATE_SOURCEMAP=false react-scripts --openssl-legacy-provider start",
|
||||
|
||||
@@ -14,7 +14,7 @@ RUN go mod tidy
|
||||
#RUN CGO_ENABLED=1 GOOS=linux go build -a -installsuffix cgo -o orborus.
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o /app/orborus .
|
||||
|
||||
FROM alpine:3.21.2
|
||||
FROM alpine:3.22.1
|
||||
|
||||
RUN apk add --no-cache bash tzdata
|
||||
#COPY --from=builder /app/orborus orborus
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
FROM golang:1.22 as builder
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
|
||||
COPY orborus.go /app/orborus.go
|
||||
#COPY go.mod /app/go.mod
|
||||
#COPY go.sum /app/go.sum
|
||||
RUN go mod init orborus
|
||||
|
||||
RUN go get github.com/docker/docker/api/types
|
||||
RUN go get github.com/docker/docker/api/types/container
|
||||
#RUN go get github.com/docker/docker/client
|
||||
RUN go get github.com/mackerelio/go-osstat/cpu
|
||||
RUN go get github.com/mackerelio/go-osstat/memory
|
||||
RUN go get github.com/satori/go.uuid
|
||||
RUN go get github.com/shuffle/shuffle-shared
|
||||
RUN go get github.com/shirou/gopsutil
|
||||
# RUN go get k8s.io/client-go
|
||||
# RUN go get k8s.io/apimachinery
|
||||
|
||||
RUN go get
|
||||
RUN go mod tidy
|
||||
RUN go build
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
|
||||
|
||||
FROM alpine:3.15.0
|
||||
RUN apk add --no-cache bash tzdata
|
||||
COPY --from=builder /app/ /
|
||||
|
||||
ENV ENVIRONMENT_NAME=Shuffle
|
||||
ENV BASE_URL=http://shuffle-backend:5001
|
||||
ENV DOCKER_API_VERSION=1.40
|
||||
ENV SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
|
||||
|
||||
CMD ["./orborus"]
|
||||
@@ -28,7 +28,7 @@ RUN go mod tidy
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
|
||||
|
||||
## ALPINE IMAGE
|
||||
FROM alpine:3.21.2
|
||||
FROM alpine:3.22.1
|
||||
|
||||
ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
|
||||
ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
|
||||
|
||||
Reference in New Issue
Block a user