Removed all irrelevant files
This commit is contained in:
Frikky
@@ -1,109 +0,0 @@
|
||||
import json
|
||||
import yaml
|
||||
|
||||
items = []
|
||||
|
||||
openapi = {
|
||||
"openapi": "3.0.2",
|
||||
"info": {
|
||||
"title": "MISP",
|
||||
"description": "MISP API generated from the misp book: https://github.com/MISP/misp-book/blob/master/automation/README.md",
|
||||
"version": "1.0.0",
|
||||
"contact": {
|
||||
"name": "@frikkylikeme",
|
||||
"url": "https://twitter.com/frikkylikeme",
|
||||
"email": "frikky@shuffler.io"
|
||||
}
|
||||
},
|
||||
"paths": {},
|
||||
"components": {
|
||||
"schemas": {},
|
||||
"securitySchemes": {
|
||||
"ApiKeyAuth": {
|
||||
"type": "apikey",
|
||||
"in": "header",
|
||||
"name": "Authorization",
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
with open("misp.txt", "r") as tmp:
|
||||
newitem = {}
|
||||
recorditem = False
|
||||
|
||||
counter = 0
|
||||
itemsplit = tmp.read().split("\n")
|
||||
for item in itemsplit:
|
||||
counter += 1
|
||||
if item.startswith("### ") and "/" in item:
|
||||
try:
|
||||
path = item.split(" ")[2]
|
||||
method = item.split(" ")[1].lower()
|
||||
newitem = {
|
||||
"path": path,
|
||||
"method": method,
|
||||
}
|
||||
|
||||
try:
|
||||
openapi["paths"][path][method] = {}
|
||||
except KeyError:
|
||||
openapi["paths"][path] = {}
|
||||
openapi["paths"][path][method] = {}
|
||||
|
||||
except IndexError:
|
||||
newitem = {}
|
||||
continue
|
||||
|
||||
recorditem = True
|
||||
#print(newitem)
|
||||
|
||||
if not recorditem:
|
||||
continue
|
||||
|
||||
if "Description" in item:
|
||||
openapi["paths"][newitem["path"]][newitem["method"]]["description"] = itemsplit[counter+1]
|
||||
elif "URL Arguments" in item:
|
||||
parameters = []
|
||||
innercnt = 0
|
||||
|
||||
openapi["paths"][newitem["path"]][newitem["method"]]["parameters"] = []
|
||||
while True:
|
||||
curline = itemsplit[counter+1+innercnt]
|
||||
if "#" in curline:
|
||||
break
|
||||
|
||||
innercnt += 1
|
||||
if not curline:
|
||||
continue
|
||||
|
||||
print(curline)
|
||||
parameters.append({
|
||||
"description": curline.split(" ")[1],
|
||||
"in": "query",
|
||||
"name": curline.split(" ")[1],
|
||||
"required": True,
|
||||
"schema": {"type": "string"},
|
||||
})
|
||||
|
||||
openapi["paths"][newitem["path"]][newitem["method"]]["parameters"] = parameters
|
||||
elif "Output" in item:
|
||||
# FIXME
|
||||
innercnt = 0
|
||||
while True:
|
||||
curline = itemsplit[counter+1+innercnt]
|
||||
|
||||
if "#" in curline:
|
||||
break
|
||||
|
||||
innercnt += 1
|
||||
if "json" in curline:
|
||||
continue
|
||||
#print(curline)
|
||||
|
||||
print(json.dumps(openapi, indent=4))
|
||||
|
||||
|
||||
generatedfile = "generated/misp.yaml"
|
||||
with open(generatedfile, "w+") as tmp:
|
||||
tmp.write(yaml.dump(openapi))
|
||||
@@ -1,311 +0,0 @@
|
||||
import requests
|
||||
import yaml
|
||||
import json
|
||||
import os
|
||||
import io
|
||||
import base64
|
||||
from PIL import Image
|
||||
#import tkinter
|
||||
#import _tkinter
|
||||
#tkinter._test()
|
||||
|
||||
|
||||
#sudo apt-get install python-imaging-tk
|
||||
#sudo apt-get install python3-tk
|
||||
|
||||
# USAGE:
|
||||
# 1. Find the item here:
|
||||
# https://apphub.swimlane.com/swimbundles/swimlane/sw_alienvault_threatcrowd
|
||||
# 2.
|
||||
# https://jsonlint.com/
|
||||
|
||||
# META: data["meta"]. Stuff like count. May be useful :)
|
||||
|
||||
def parse_data(data):
|
||||
openapi = {
|
||||
"openapi": "3.0.2",
|
||||
"info": {
|
||||
"title": "",
|
||||
"description": "",
|
||||
"version": "1.0.0",
|
||||
"contact": {
|
||||
"name": "@frikkylikeme",
|
||||
"url": "https://twitter.com/frikkylikeme",
|
||||
"email": "frikky@shuffler.io"
|
||||
}
|
||||
},
|
||||
"paths": {},
|
||||
"components": {
|
||||
"schemas": {},
|
||||
"securitySchemes": {},
|
||||
}
|
||||
}
|
||||
|
||||
data = data["swimbundle"]
|
||||
filename = "%s.yaml" % data["product"].replace(" ", "_").lower()
|
||||
openapi["info"]["title"] = "%s %s" % (data["vendor"], data["product"])
|
||||
openapi["info"]["description"] = "Automated generation of %s" % (openapi["info"]["title"])
|
||||
# data["description"]
|
||||
|
||||
# https://swagger.io/docs/specification/authentication/
|
||||
try:
|
||||
asset = data["asset"]
|
||||
inputparams = asset["inputParameters"]
|
||||
|
||||
try:
|
||||
openapi["servers"] = [inputparams["api_url"]["example"]]
|
||||
except KeyError as e:
|
||||
#print(inputparams)
|
||||
#print("Field error: %s" % e)
|
||||
pass
|
||||
|
||||
authset = False
|
||||
try:
|
||||
tmpauth = inputparams["api_user"]
|
||||
tmpauth = inputparams["api_key"]
|
||||
|
||||
openapi["components"]["securitySchemes"] = {
|
||||
"BasicAuth": {
|
||||
"type": "http",
|
||||
"scheme": "basic"
|
||||
}
|
||||
}
|
||||
authset = True
|
||||
except KeyError as e:
|
||||
pass
|
||||
|
||||
try:
|
||||
tmpauth = inputparams["username"]
|
||||
tmpauth = inputparams["password"]
|
||||
|
||||
openapi["components"]["securitySchemes"] = {
|
||||
"BasicAuth": {
|
||||
"type": "http",
|
||||
"scheme": "basic"
|
||||
}
|
||||
}
|
||||
authset = True
|
||||
except KeyError as e:
|
||||
pass
|
||||
|
||||
#if not authset:
|
||||
# print("AUTH NOT SET: %s" % inputparams)
|
||||
|
||||
except KeyError as e:
|
||||
print("KeyError asset: %s" % e)
|
||||
|
||||
cnt = 0
|
||||
paramnames = []
|
||||
for task in data["tasks"]:
|
||||
method = "post"
|
||||
|
||||
openapi["paths"]["tmp%d" % cnt] = {}
|
||||
openapi["paths"]["tmp%d" % cnt][method] = {
|
||||
"summary": task["name"],
|
||||
"description": task["description"],
|
||||
"parameters": [],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "Successful request",
|
||||
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
taskcategory = task["family"]
|
||||
taskname = task["name"]
|
||||
paramnames.append(taskname)
|
||||
|
||||
for key, value in task["inputParameters"].items():
|
||||
schema = "string"
|
||||
inVar = "query"
|
||||
|
||||
if value["type"] == 6:
|
||||
inVar = "body"
|
||||
|
||||
schema = "string"
|
||||
schemaset = False
|
||||
if value["type"] != 1:
|
||||
if (value["type"] == 7):
|
||||
schema = "boolean"
|
||||
schemaset = True
|
||||
|
||||
if schema == "string" and schemaset:
|
||||
print("Should change type: %d" % value["type"])
|
||||
print(task["name"])
|
||||
print(value["name"])
|
||||
|
||||
example = ""
|
||||
try:
|
||||
example = value["example"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
description = ""
|
||||
try:
|
||||
description = value["description"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
required = False
|
||||
try:
|
||||
required = value["required"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
openapi["paths"]["tmp%d" % cnt][method]["parameters"].append({
|
||||
"name": value["name"],
|
||||
"required": required,
|
||||
"example": example,
|
||||
"description": description,
|
||||
"schema": {"type": schema},
|
||||
"in": inVar
|
||||
})
|
||||
|
||||
if len(task["availableOutputVariables"]) > 0:
|
||||
openapi["paths"]["tmp%d" % cnt][method]["responses"]["200"]["content"] = {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/tmp%d" % cnt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#responses:
|
||||
# '200':
|
||||
# content:
|
||||
# application/json:
|
||||
# schema:
|
||||
# $ref: '#/components/schemas/tmp1'
|
||||
#description: Successful request
|
||||
|
||||
openapi["components"]["schemas"]["tmp%d" % cnt] = {
|
||||
"type": "object",
|
||||
"properties": {},
|
||||
}
|
||||
|
||||
for key, value in task["availableOutputVariables"].items():
|
||||
if key == "response_code":
|
||||
continue
|
||||
|
||||
openapi["components"]["schemas"]["tmp%d" % cnt]["properties"][key] = {
|
||||
"type": "string"
|
||||
}
|
||||
|
||||
cnt += 1
|
||||
|
||||
print("%s: %d" % (openapi["info"]["title"], len(paramnames)))
|
||||
|
||||
return filename, openapi
|
||||
|
||||
def dump_data(filename, openapi, category):
|
||||
generatedfile = "generated/%s/%s" % (category, filename)
|
||||
try:
|
||||
with open(generatedfile, "w+") as tmp:
|
||||
tmp.write(yaml.dump(openapi))
|
||||
except FileNotFoundError:
|
||||
try:
|
||||
os.mkdir("generated/%s" % category)
|
||||
with open(generatedfile, "w+") as tmp:
|
||||
tmp.write(yaml.dump(openapi))
|
||||
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
if __name__ == "__main__":
|
||||
#https://apphub.swimlane.com/
|
||||
categories = [
|
||||
"Investigation",
|
||||
"Endpoint Security & Management",
|
||||
"Network Security & Management",
|
||||
"Communication",
|
||||
"SIEM & Log Management",
|
||||
"Governance & Risk Management",
|
||||
"Vulnerability & Patch Management",
|
||||
"Ticket Management",
|
||||
"DevOps & Application Security",
|
||||
"Identity & Access Management",
|
||||
"Infrastructure",
|
||||
"Miscellaneous",
|
||||
]
|
||||
|
||||
search_category = categories[2]
|
||||
total = 0
|
||||
for search_category in categories:
|
||||
number = 1
|
||||
innertotal = 0
|
||||
|
||||
while(True):
|
||||
url = "https://apphub.swimlane.io/api/search/swimbundles?page=%d" % number
|
||||
|
||||
json = {"fields": {"family": search_category}}
|
||||
ret = requests.post(
|
||||
url,
|
||||
json=json,
|
||||
)
|
||||
|
||||
if ret.status_code != 201:
|
||||
print("RET NOT 201: %d" % ret.status_code)
|
||||
break
|
||||
|
||||
parsed = ret.json()
|
||||
try:
|
||||
category = parsed["data"][0]["swimbundleMeta"]["family"][0]
|
||||
except KeyError:
|
||||
category = ""
|
||||
except IndexError:
|
||||
category = ""
|
||||
|
||||
if category == "":
|
||||
break
|
||||
|
||||
for data in parsed["data"]:
|
||||
try:
|
||||
filename, openapi = parse_data(data)
|
||||
except:
|
||||
try:
|
||||
print("Skipping %s %s because of an error" % (data["vendor"], data["product"]))
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
continue
|
||||
|
||||
openapi["tags"] = [
|
||||
{
|
||||
"name": category,
|
||||
}
|
||||
]
|
||||
|
||||
appid = data["swimbundleMeta"]["logo"]["id"]
|
||||
logoUrl = "https://apphub.swimlane.io/api/logos/%s" % appid
|
||||
logodata = requests.get(logoUrl)
|
||||
if logodata.status_code == 200:
|
||||
logojson = logodata.json()
|
||||
try:
|
||||
logobase64 = logojson["data"]["base64"]
|
||||
#.split(",")[1]
|
||||
|
||||
openapi["info"]["x-logo"] = logobase64
|
||||
#print(logobase64)
|
||||
#msg = base64.b64decode(logobase64)
|
||||
#with io.BytesIO(msg) as buf:
|
||||
# with Image.open(buf) as tempImg:
|
||||
# newWidth = 174 / tempImg.width # change this to what ever width you need.
|
||||
# newHeight = 174 / tempImg.height # change this to what ever height you need.
|
||||
# newSize = (int(newWidth * tempImg.width), int(newHeight * tempImg.height))
|
||||
# newImg1 = tempImg.resize(newSize)
|
||||
# lbl1.IMG = ImageTk.PhotoImage(image=newImg1)
|
||||
# lbl1.configure(image=lbl1.IMG)
|
||||
except KeyError:
|
||||
print("Failed logo parsing for %s" % appid)
|
||||
pass
|
||||
|
||||
dump_data(filename, openapi, category)
|
||||
innertotal += 1
|
||||
total += 1
|
||||
|
||||
number += 1
|
||||
|
||||
print("Created %d openapi specs from Swimlane with category %s" % (innertotal, search_category))
|
||||
|
||||
print("\nCreated %d TOTAL openapi specs from Swimlane" % (total))
|
||||
@@ -1,7 +0,0 @@
|
||||
# OpenAPI generator
|
||||
This contains test code that's been moved to shaffuru/backend/go-app/codegen.go
|
||||
|
||||
## Todo:
|
||||
1. Don't use filesystem, but rather store in GCP
|
||||
2. Add swagger 2.0 to 3.0 converter
|
||||
3. Fix body / data parsing
|
||||
@@ -1,29 +0,0 @@
|
||||
# Base our app image off of the WALKOFF App SDK image
|
||||
FROM frikky/shuffle:app_sdk as base
|
||||
|
||||
# We're going to stage away all of the bloat from the build tools so lets create a builder stage
|
||||
#FROM base as builder
|
||||
|
||||
# Install all alpine build tools needed for our pip installs
|
||||
#RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
|
||||
|
||||
# Install all of our pip packages in a single directory that we can copy to our base image later
|
||||
#RUN mkdir /install
|
||||
#WORKDIR /install
|
||||
#COPY requirements.txt /requirements.txt
|
||||
#
|
||||
## Switch back to our base image and copy in all of our built packages and source code
|
||||
#FROM base
|
||||
#COPY --from=builder /install /usr/local
|
||||
|
||||
|
||||
# Install any binary dependencies needed in our final image - this can be a lot of different stuff
|
||||
#RUN apk --no-cache add --update libmagic
|
||||
WORKDIR /
|
||||
COPY requirements.txt /requirements.txt
|
||||
RUN pip install --prefix="/usr/local" -r /requirements.txt
|
||||
COPY src /app
|
||||
|
||||
# Finally, lets run our app!
|
||||
WORKDIR /app
|
||||
CMD python app.py --log-level DEBUG
|
||||
@@ -1,11 +0,0 @@
|
||||
# No extra requirements needed
|
||||
requests==2.32.3
|
||||
urllib3==2.3.0
|
||||
liquidpy==0.8.2
|
||||
MarkupSafe==3.0.2
|
||||
flask[async]==3.1.0
|
||||
python-dateutil==2.9.0.post0
|
||||
PyJWT==2.10.1
|
||||
cryptography==44.0.2
|
||||
shufflepy==0.1.0
|
||||
shuffle-sdk==0.0.25
|
||||
@@ -1,387 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/getkin/kin-openapi/openapi3"
|
||||
"gopkg.in/yaml.v2"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type WorkflowApp struct {
|
||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
||||
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
|
||||
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
|
||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
|
||||
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
|
||||
ContactInfo struct {
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
||||
}
|
||||
|
||||
type AuthenticationParams struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"s`
|
||||
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
}
|
||||
|
||||
type Authentication struct {
|
||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
||||
}
|
||||
|
||||
type AuthenticationStore struct {
|
||||
Key string `json:"key" datastore:"key"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
}
|
||||
|
||||
type WorkflowAppActionParameter struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
||||
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
|
||||
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
}
|
||||
|
||||
type SchemaDefinition struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
}
|
||||
|
||||
type WorkflowAppAction struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
NodeType string `json:"node_type" datastore:"node_type"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Returns struct {
|
||||
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
} `json:"returns" datastore:"returns"`
|
||||
}
|
||||
|
||||
func copyFile(fromfile, tofile string) error {
|
||||
from, err := os.Open(fromfile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer from.Close()
|
||||
|
||||
to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer to.Close()
|
||||
|
||||
_, err = io.Copy(to, from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Builds the base structure for the app that we're making
|
||||
// Returns error if anything goes wrong. This has to work if
|
||||
// the python code is supposed to be generated
|
||||
func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
|
||||
//log.Printf("%#v", swagger)
|
||||
|
||||
// adding md5 based on input data to not overwrite earlier data.
|
||||
generatedPath := "generated"
|
||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
||||
appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
|
||||
|
||||
os.MkdirAll(appPath, os.ModePerm)
|
||||
os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm)
|
||||
|
||||
err := copyFile("baseline/Dockerfile", fmt.Sprintf("%s/%s", appPath, "Dockerfile"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move Dockerfile")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
err = copyFile("baseline/requirements.txt", fmt.Sprintf("%s/%s", appPath, "requirements.txt"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move requrements.txt")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
return appPath, nil
|
||||
}
|
||||
|
||||
func makePythoncode(name, url, method string, parameters, optionalQueries []string) string {
|
||||
method = strings.ToLower(method)
|
||||
queryString := ""
|
||||
queryData := ""
|
||||
|
||||
// FIXME - this might break - need to check if ? or & should be set as query
|
||||
parameterData := ""
|
||||
if len(optionalQueries) > 0 {
|
||||
queryString += ", "
|
||||
for _, query := range optionalQueries {
|
||||
queryString += fmt.Sprintf("%s=\"\"", query)
|
||||
queryData += fmt.Sprintf(`
|
||||
if %s:
|
||||
url += f"&%s={%s}"`, query, query, query)
|
||||
}
|
||||
}
|
||||
|
||||
if len(parameters) > 0 {
|
||||
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
|
||||
}
|
||||
|
||||
// FIXME - add checks for query data etc
|
||||
data := fmt.Sprintf(` async def %s_%s(self%s%s):
|
||||
url=f"%s"
|
||||
%s
|
||||
return requests.%s(url).text
|
||||
`, name, method, parameterData, queryString, url, queryData, method)
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
|
||||
api := WorkflowApp{}
|
||||
log.Printf("%#v", swagger.Info)
|
||||
|
||||
if len(swagger.Info.Title) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.")
|
||||
}
|
||||
|
||||
if len(swagger.Servers) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'")
|
||||
}
|
||||
|
||||
api.Name = swagger.Info.Title
|
||||
api.Description = swagger.Info.Description
|
||||
api.IsValid = true
|
||||
api.Link = swagger.Servers[0].URL // host does not exist lol
|
||||
api.AppVersion = "1.0.0"
|
||||
api.Environment = "cloud"
|
||||
api.ID = ""
|
||||
api.SmallImage = ""
|
||||
api.LargeImage = ""
|
||||
|
||||
// This is the python code to be generated
|
||||
// Could just as well be go at this point lol
|
||||
pythonFunctions := []string{}
|
||||
|
||||
for actualPath, path := range swagger.Paths {
|
||||
//log.Printf("%#v", path)
|
||||
//log.Printf("%#v", actualPath)
|
||||
// Find the path name and add it to makeCode() param
|
||||
|
||||
firstQuery := true
|
||||
if path.Get != nil {
|
||||
// What to do with this, hmm
|
||||
functionName := strings.ReplaceAll(path.Get.Summary, " ", "_")
|
||||
functionName = strings.ToLower(functionName)
|
||||
|
||||
action := WorkflowAppAction{
|
||||
Description: path.Get.Description,
|
||||
Name: path.Get.Summary,
|
||||
NodeType: "action",
|
||||
Environment: api.Environment,
|
||||
Parameters: []WorkflowAppActionParameter{},
|
||||
}
|
||||
|
||||
action.Returns.Schema.Type = "string"
|
||||
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
|
||||
|
||||
//log.Println(path.Parameters)
|
||||
|
||||
// Parameters: []WorkflowAppActionParameter{},
|
||||
// FIXME - add data for POST stuff
|
||||
firstQuery = true
|
||||
optionalQueries := []string{}
|
||||
parameters := []string{}
|
||||
optionalParameters := []WorkflowAppActionParameter{}
|
||||
if len(path.Get.Parameters) > 0 {
|
||||
for _, param := range path.Get.Parameters {
|
||||
curParam := WorkflowAppActionParameter{
|
||||
Name: param.Value.Name,
|
||||
Description: param.Value.Description,
|
||||
Multiline: false,
|
||||
Required: param.Value.Required,
|
||||
Schema: SchemaDefinition{
|
||||
Type: param.Value.Schema.Value.Type,
|
||||
},
|
||||
}
|
||||
|
||||
if param.Value.Required {
|
||||
action.Parameters = append(action.Parameters, curParam)
|
||||
} else {
|
||||
optionalParameters = append(optionalParameters, curParam)
|
||||
}
|
||||
|
||||
if param.Value.In == "path" {
|
||||
log.Printf("PATH!: %s", param.Value.Name)
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
|
||||
} else if param.Value.In == "query" {
|
||||
log.Printf("QUERY!: %s", param.Value.Name)
|
||||
if !param.Value.Required {
|
||||
optionalQueries = append(optionalQueries, param.Value.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
|
||||
if firstQuery {
|
||||
baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
} else {
|
||||
baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// ensuring that they end up last in the specification
|
||||
// (order is ish important for optional params) - they need to be last.
|
||||
for _, optionalParam := range optionalParameters {
|
||||
action.Parameters = append(action.Parameters, optionalParam)
|
||||
}
|
||||
|
||||
curCode := makePythoncode(functionName, baseUrl, "get", parameters, optionalQueries)
|
||||
pythonFunctions = append(pythonFunctions, curCode)
|
||||
|
||||
api.Actions = append(api.Actions, action)
|
||||
}
|
||||
}
|
||||
|
||||
return api, pythonFunctions, nil
|
||||
}
|
||||
|
||||
func verifyApi(api WorkflowApp) WorkflowApp {
|
||||
if api.AppVersion == "" {
|
||||
api.AppVersion = "1.0.0"
|
||||
}
|
||||
|
||||
return api
|
||||
}
|
||||
|
||||
func dumpPython(basePath, name, version string, pythonFunctions []string) error {
|
||||
//log.Printf("%#v", api)
|
||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
parsedCode := fmt.Sprintf(`import requests
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from walkoff_app_sdk.app_base import AppBase
|
||||
|
||||
class %s(AppBase):
|
||||
"""
|
||||
Autogenerated class by Shuffler
|
||||
"""
|
||||
|
||||
__version__ = "%s"
|
||||
app_name = "%s"
|
||||
|
||||
def __init__(self, redis, logger, console_logger=None):
|
||||
self.verify = False
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
super().__init__(redis, logger, console_logger)
|
||||
|
||||
%s
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CarbonBlack.run(), debug=True)
|
||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(parsedCode)
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpApi(basePath string, api WorkflowApp) error {
|
||||
//log.Printf("%#v", api)
|
||||
data, err := yaml.Marshal(api)
|
||||
if err != nil {
|
||||
log.Printf("Error with yaml marshal: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
data := []byte(`{"swagger":"3.0","info":{"title":"hi","description":"you","version":"1.0"},"servers":[{"url":"https://shuffler.io/api/v1"}],"host":"shuffler.io","basePath":"/api/v1","schemes":["https:"],"paths":{"/workflows":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflows","description":"Get workflows","parameters":[]}},"/workflows/{id}":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflow","description":"Get workflow","parameters":[{"in":"query","name":"forgetme","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"query","name":"anotherone","description":"Generated by shuffler.io OpenAPI","required":false,"schema":{"type":"string"}},{"in":"query","name":"hi","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"path","name":"id","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}}]}}},"securityDefinitions":{}}`)
|
||||
|
||||
hasher := md5.New()
|
||||
hasher.Write(data)
|
||||
newmd5 := hex.EncodeToString(hasher.Sum(nil))
|
||||
|
||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(data)
|
||||
if err != nil {
|
||||
log.Printf("Swagger validation error: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
if strings.Contains(swagger.Info.Title, " ") {
|
||||
strings.ReplaceAll(swagger.Info.Title, " ", "")
|
||||
}
|
||||
|
||||
basePath, err := buildStructure(swagger, newmd5)
|
||||
if err != nil {
|
||||
log.Printf("Failed to build base structure: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
api, pythonfunctions, err := generateYaml(swagger)
|
||||
if err != nil {
|
||||
log.Printf("Failed building and generating yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpApi(basePath, api)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpPython(basePath, swagger.Info.Title, swagger.Info.Version, pythonfunctions)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping python: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
@@ -1,499 +0,0 @@
|
||||
package main
|
||||
|
||||
/*
|
||||
Code used to generate apps from OpenAPI JSON data
|
||||
Any function ending with GCP doesn't use local fileIO, but rather
|
||||
google cloud storage, and also has a normal filesystem version of the
|
||||
same code (doesn't required client as first argument).
|
||||
|
||||
This code is used in the backend to generate apps on the fly for users.
|
||||
All new code is appended to backend/go-app/codegen.go
|
||||
*/
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/md5"
|
||||
"encoding/hex"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/getkin/kin-openapi/openapi3"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var bucketName = "shuffler.appspot.com"
|
||||
|
||||
type WorkflowApp struct {
|
||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
||||
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
|
||||
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
|
||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
|
||||
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
|
||||
ContactInfo struct {
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
||||
}
|
||||
|
||||
type AuthenticationParams struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"s`
|
||||
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
}
|
||||
|
||||
type Authentication struct {
|
||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
||||
}
|
||||
|
||||
type AuthenticationStore struct {
|
||||
Key string `json:"key" datastore:"key"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
}
|
||||
|
||||
type WorkflowAppActionParameter struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
||||
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
|
||||
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
}
|
||||
|
||||
type SchemaDefinition struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
}
|
||||
|
||||
type WorkflowAppAction struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
NodeType string `json:"node_type" datastore:"node_type"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Returns struct {
|
||||
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
||||
} `json:"returns" datastore:"returns"`
|
||||
}
|
||||
|
||||
func copyFile(fromfile, tofile string) error {
|
||||
from, err := os.Open(fromfile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer from.Close()
|
||||
|
||||
to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer to.Close()
|
||||
|
||||
_, err = io.Copy(to, from)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func buildStructureGCP(client *storage.Client, swagger *openapi3.Swagger, curHash string) (string, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
// 1. Have baseline in bucket/generated_apps/baseline
|
||||
// 2. Copy the baseline to a new folder with identifier name
|
||||
|
||||
basePath := "generated_apps"
|
||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
||||
appPath := fmt.Sprintf("%s/%s", basePath, identifier)
|
||||
fileNames := []string{"Dockerfile", "requirements.txt"}
|
||||
for _, file := range fileNames {
|
||||
src := client.Bucket(bucketName).Object(fmt.Sprintf("%s/baseline/%s", basePath, file))
|
||||
dst := client.Bucket(bucketName).Object(fmt.Sprintf("%s/%s", appPath, file))
|
||||
if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
return appPath, nil
|
||||
}
|
||||
|
||||
// Builds the base structure for the app that we're making
|
||||
// Returns error if anything goes wrong. This has to work if
|
||||
// the python code is supposed to be generated
|
||||
func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
|
||||
//log.Printf("%#v", swagger)
|
||||
|
||||
// adding md5 based on input data to not overwrite earlier data.
|
||||
generatedPath := "generated"
|
||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
||||
appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
|
||||
|
||||
os.MkdirAll(appPath, os.ModePerm)
|
||||
os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm)
|
||||
|
||||
err := copyFile("baseline/Dockerfile", fmt.Sprintf("%s/%s", appPath, "Dockerfile"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move Dockerfile")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
err = copyFile("baseline/requirements.txt", fmt.Sprintf("%s/%s", appPath, "requirements.txt"))
|
||||
if err != nil {
|
||||
log.Println("Failed to move requrements.txt")
|
||||
return appPath, err
|
||||
}
|
||||
|
||||
return appPath, nil
|
||||
}
|
||||
|
||||
func makePythoncode(name, url, method string, parameters, optionalQueries []string) string {
|
||||
method = strings.ToLower(method)
|
||||
queryString := ""
|
||||
queryData := ""
|
||||
|
||||
// FIXME - this might break - need to check if ? or & should be set as query
|
||||
parameterData := ""
|
||||
if len(optionalQueries) > 0 {
|
||||
queryString += ", "
|
||||
for _, query := range optionalQueries {
|
||||
queryString += fmt.Sprintf("%s=\"\"", query)
|
||||
queryData += fmt.Sprintf(`
|
||||
if %s:
|
||||
url += f"&%s={%s}"`, query, query, query)
|
||||
}
|
||||
}
|
||||
|
||||
if len(parameters) > 0 {
|
||||
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
|
||||
}
|
||||
|
||||
// FIXME - add checks for query data etc
|
||||
data := fmt.Sprintf(` async def %s_%s(self%s%s):
|
||||
url=f"%s"
|
||||
%s
|
||||
return requests.%s(url).text
|
||||
`, name, method, parameterData, queryString, url, queryData, method)
|
||||
|
||||
return data
|
||||
}
|
||||
|
||||
func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
|
||||
api := WorkflowApp{}
|
||||
log.Printf("%#v", swagger.Info)
|
||||
|
||||
if len(swagger.Info.Title) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.")
|
||||
}
|
||||
|
||||
if len(swagger.Servers) == 0 {
|
||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'")
|
||||
}
|
||||
|
||||
api.Name = swagger.Info.Title
|
||||
api.Description = swagger.Info.Description
|
||||
api.IsValid = true
|
||||
api.Link = swagger.Servers[0].URL // host does not exist lol
|
||||
api.AppVersion = "1.0.0"
|
||||
api.Environment = "cloud"
|
||||
api.ID = ""
|
||||
api.SmallImage = ""
|
||||
api.LargeImage = ""
|
||||
|
||||
// This is the python code to be generated
|
||||
// Could just as well be go at this point lol
|
||||
pythonFunctions := []string{}
|
||||
|
||||
for actualPath, path := range swagger.Paths {
|
||||
//log.Printf("%#v", path)
|
||||
//log.Printf("%#v", actualPath)
|
||||
// Find the path name and add it to makeCode() param
|
||||
|
||||
firstQuery := true
|
||||
if path.Get != nil {
|
||||
// What to do with this, hmm
|
||||
functionName := strings.ReplaceAll(path.Get.Summary, " ", "_")
|
||||
functionName = strings.ToLower(functionName)
|
||||
|
||||
action := WorkflowAppAction{
|
||||
Description: path.Get.Description,
|
||||
Name: path.Get.Summary,
|
||||
NodeType: "action",
|
||||
Environment: api.Environment,
|
||||
Parameters: []WorkflowAppActionParameter{},
|
||||
}
|
||||
|
||||
action.Returns.Schema.Type = "string"
|
||||
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
|
||||
|
||||
//log.Println(path.Parameters)
|
||||
|
||||
// Parameters: []WorkflowAppActionParameter{},
|
||||
// FIXME - add data for POST stuff
|
||||
firstQuery = true
|
||||
optionalQueries := []string{}
|
||||
parameters := []string{}
|
||||
optionalParameters := []WorkflowAppActionParameter{}
|
||||
if len(path.Get.Parameters) > 0 {
|
||||
for _, param := range path.Get.Parameters {
|
||||
curParam := WorkflowAppActionParameter{
|
||||
Name: param.Value.Name,
|
||||
Description: param.Value.Description,
|
||||
Multiline: false,
|
||||
Required: param.Value.Required,
|
||||
Schema: SchemaDefinition{
|
||||
Type: param.Value.Schema.Value.Type,
|
||||
},
|
||||
}
|
||||
|
||||
if param.Value.Required {
|
||||
action.Parameters = append(action.Parameters, curParam)
|
||||
} else {
|
||||
optionalParameters = append(optionalParameters, curParam)
|
||||
}
|
||||
|
||||
if param.Value.In == "path" {
|
||||
log.Printf("PATH!: %s", param.Value.Name)
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
|
||||
} else if param.Value.In == "query" {
|
||||
log.Printf("QUERY!: %s", param.Value.Name)
|
||||
if !param.Value.Required {
|
||||
optionalQueries = append(optionalQueries, param.Value.Name)
|
||||
continue
|
||||
}
|
||||
|
||||
parameters = append(parameters, param.Value.Name)
|
||||
|
||||
if firstQuery {
|
||||
baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
} else {
|
||||
baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
||||
firstQuery = false
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
// ensuring that they end up last in the specification
|
||||
// (order is ish important for optional params) - they need to be last.
|
||||
for _, optionalParam := range optionalParameters {
|
||||
action.Parameters = append(action.Parameters, optionalParam)
|
||||
}
|
||||
|
||||
curCode := makePythoncode(functionName, baseUrl, "get", parameters, optionalQueries)
|
||||
pythonFunctions = append(pythonFunctions, curCode)
|
||||
|
||||
api.Actions = append(api.Actions, action)
|
||||
}
|
||||
}
|
||||
|
||||
return api, pythonFunctions, nil
|
||||
}
|
||||
|
||||
func verifyApi(api WorkflowApp) WorkflowApp {
|
||||
if api.AppVersion == "" {
|
||||
api.AppVersion = "1.0.0"
|
||||
}
|
||||
|
||||
return api
|
||||
}
|
||||
|
||||
func dumpPythonGCP(client *storage.Client, basePath, name, version string, pythonFunctions []string) error {
|
||||
//log.Printf("%#v", api)
|
||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
parsedCode := fmt.Sprintf(`import requests
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from walkoff_app_sdk.app_base import AppBase
|
||||
|
||||
class %s(AppBase):
|
||||
"""
|
||||
Autogenerated class by Shuffler
|
||||
"""
|
||||
|
||||
__version__ = "%s"
|
||||
app_name = "%s"
|
||||
|
||||
def __init__(self, redis, logger, console_logger=None):
|
||||
self.verify = False
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
super().__init__(redis, logger, console_logger)
|
||||
|
||||
%s
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CarbonBlack.run(), debug=True)
|
||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
// Create bucket handle
|
||||
ctx := context.Background()
|
||||
bucket := client.Bucket(bucketName)
|
||||
obj := bucket.Object(fmt.Sprintf("%s/src/app.py", basePath))
|
||||
w := obj.NewWriter(ctx)
|
||||
if _, err := fmt.Fprintf(w, parsedCode); err != nil {
|
||||
return err
|
||||
}
|
||||
// Close, just like writing a file.
|
||||
if err := w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpPython(basePath, name, version string, pythonFunctions []string) error {
|
||||
//log.Printf("%#v", api)
|
||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
parsedCode := fmt.Sprintf(`import requests
|
||||
import asyncio
|
||||
import json
|
||||
|
||||
from walkoff_app_sdk.app_base import AppBase
|
||||
|
||||
class %s(AppBase):
|
||||
"""
|
||||
Autogenerated class by Shuffler
|
||||
"""
|
||||
|
||||
__version__ = "%s"
|
||||
app_name = "%s"
|
||||
|
||||
def __init__(self, redis, logger, console_logger=None):
|
||||
self.verify = False
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
super().__init__(redis, logger, console_logger)
|
||||
|
||||
%s
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(CarbonBlack.run(), debug=True)
|
||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
||||
|
||||
err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(parsedCode)
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpApiGCP(client *storage.Client, basePath string, api WorkflowApp) error {
|
||||
//log.Printf("%#v", api)
|
||||
data, err := yaml.Marshal(api)
|
||||
if err != nil {
|
||||
log.Printf("Error with yaml marshal: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Create bucket handle
|
||||
ctx := context.Background()
|
||||
bucket := client.Bucket(bucketName)
|
||||
obj := bucket.Object(fmt.Sprintf("%s/app.yaml", basePath))
|
||||
w := obj.NewWriter(ctx)
|
||||
if _, err := fmt.Fprintf(w, string(data)); err != nil {
|
||||
return err
|
||||
}
|
||||
// Close, just like writing a file.
|
||||
if err := w.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func dumpApi(basePath string, api WorkflowApp) error {
|
||||
//log.Printf("%#v", api)
|
||||
data, err := yaml.Marshal(api)
|
||||
if err != nil {
|
||||
log.Printf("Error with yaml marshal: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
//log.Println(string(data))
|
||||
return nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
data := []byte(`{"swagger":"3.0","info":{"title":"hi","description":"you","version":"1.0"},"servers":[{"url":"https://shuffler.io/api/v1"}],"host":"shuffler.io","basePath":"/api/v1","schemes":["https:"],"paths":{"/workflows":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflows","description":"Get workflows","parameters":[]}},"/workflows/{id}":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflow","description":"Get workflow","parameters":[{"in":"query","name":"forgetme","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"query","name":"anotherone","description":"Generated by shuffler.io OpenAPI","required":false,"schema":{"type":"string"}},{"in":"query","name":"hi","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"path","name":"id","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}}]}}},"securityDefinitions":{}}`)
|
||||
|
||||
ctx := context.Background()
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create client: %v", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
hasher := md5.New()
|
||||
hasher.Write(data)
|
||||
newmd5 := hex.EncodeToString(hasher.Sum(nil))
|
||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(data)
|
||||
if err != nil {
|
||||
log.Printf("Swagger validation error: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
if strings.Contains(swagger.Info.Title, " ") {
|
||||
strings.ReplaceAll(swagger.Info.Title, " ", "")
|
||||
}
|
||||
|
||||
basePath, err := buildStructureGCP(client, swagger, newmd5)
|
||||
if err != nil {
|
||||
log.Printf("Failed to build base structure: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
api, pythonfunctions, err := generateYaml(swagger)
|
||||
if err != nil {
|
||||
log.Printf("Failed building and generating yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpApiGCP(client, basePath, api)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping yaml: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = dumpPythonGCP(client, basePath, swagger.Info.Title, swagger.Info.Version, pythonfunctions)
|
||||
if err != nil {
|
||||
log.Printf("Failed dumping python: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
@@ -1,2 +0,0 @@
|
||||
cortexutils
|
||||
requests
|
||||
@@ -1,43 +0,0 @@
|
||||
{
|
||||
"name": "Shuffle",
|
||||
"version": "1.0",
|
||||
"author": "@frikkylikeme",
|
||||
"url": "https://github.com/frikky/shuffle",
|
||||
"license": "AGPL-V3",
|
||||
"description": "Execute a workflow in Shuffle",
|
||||
"dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"],
|
||||
"command": "Shuffle/shuffle.py",
|
||||
"baseConfig": "Shuffle",
|
||||
"configurationItems": [
|
||||
{
|
||||
"name": "url",
|
||||
"description": "The URL to your shuffle instance",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true,
|
||||
"defaultValue": "https://shuffler.io"
|
||||
},
|
||||
{
|
||||
"name": "api_key",
|
||||
"description": "The API key to your Shuffle user",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"name": "verifyssl",
|
||||
"description": "Verify SSL certificate",
|
||||
"type": "boolean",
|
||||
"multi": false,
|
||||
"required": true,
|
||||
"defaultValue": true
|
||||
},
|
||||
{
|
||||
"name": "workflow_id",
|
||||
"description": "The ID of the workflow to execute",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#encoding: utf-8
|
||||
|
||||
from cortexutils.responder import Responder
|
||||
import requests
|
||||
|
||||
class Shuffle(Responder):
|
||||
def __init__(self):
|
||||
Responder.__init__(self)
|
||||
self.api_key = self.get_param("config.api_key", "")
|
||||
self.url = self.get_param("config.url", "")
|
||||
self.workflow_id = self.get_param("config.workflow_id", "")
|
||||
self.verify = self.get_param('config.verifyssl', True, None)
|
||||
|
||||
def run(self):
|
||||
Responder.run(self)
|
||||
parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id)
|
||||
headers = {
|
||||
"Authorization": "Bearer %s" % self.api_key,
|
||||
"User-Agent": "Cortex-Analyzer"
|
||||
}
|
||||
requests.post(parsed_url, headers=headers,verify=self.verify)
|
||||
|
||||
self.report({'message': 'message sent'})
|
||||
|
||||
if __name__ == '__main__':
|
||||
Shuffle().run()
|
||||
@@ -1,2 +0,0 @@
|
||||
cortexutils
|
||||
requests
|
||||
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"name": "Shuffle_webhook",
|
||||
"version": "1.0",
|
||||
"author": "@azgaviperr",
|
||||
"url": "https://github.com/frikky/shuffle",
|
||||
"license": "AGPL-V3",
|
||||
"description": "Execute a webhook in Shuffle",
|
||||
"dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"],
|
||||
"command": "Shuffle_Webhook/shuffle_webhook.py",
|
||||
"baseConfig": "Shuffle_Webhook",
|
||||
"configurationItems": [
|
||||
{
|
||||
"name": "webhook_url",
|
||||
"description": "The URL to your shuffle instance",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"name": "verifyssl",
|
||||
"description": "Verify SSL certificate",
|
||||
"type": "boolean",
|
||||
"multi": false,
|
||||
"required": true,
|
||||
"defaultValue": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#encoding: utf-8
|
||||
|
||||
from cortexutils.responder import Responder
|
||||
import requests
|
||||
|
||||
class Shuffle(Responder):
|
||||
def __init__(self):
|
||||
Responder.__init__(self)
|
||||
self.api_key = self.get_param("config.api_key", "")
|
||||
self.webhook_url = self.get_param("config.webhook_url", "")
|
||||
self.webhook_id = self.get_param("config.webhook_id", "")
|
||||
self.verify = self.get_param('config.verifyssl', True, None)
|
||||
self.data = self.get_param('data')
|
||||
|
||||
def run(self):
|
||||
Responder.run(self)
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "Cortex-Analyzer"
|
||||
}
|
||||
requests.post(self.webhook_url, headers=headers,verify=self.verify, json=self.data)
|
||||
|
||||
self.report({'message': 'message sent'})
|
||||
|
||||
if __name__ == '__main__':
|
||||
Shuffle().run()
|
||||
@@ -1,23 +0,0 @@
|
||||
FROM python:3.9.4-alpine as base
|
||||
|
||||
FROM base as builder
|
||||
|
||||
RUN mkdir /install
|
||||
WORKDIR /install
|
||||
|
||||
FROM base
|
||||
RUN apk add g++
|
||||
|
||||
COPY --from=builder /install /usr/local
|
||||
COPY requirements.txt /requirements.txt
|
||||
RUN pip3 install -r /requirements.txt
|
||||
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN python3 -m pip install -r /app/requirements.txt
|
||||
|
||||
COPY sub.py /app/sub.py
|
||||
|
||||
CMD ["python3", "sub.py"]
|
||||
@@ -1,9 +0,0 @@
|
||||
version: '3'
|
||||
services:
|
||||
zmq:
|
||||
image: ghcr.io/frikky/shuffle-zmq:latest
|
||||
environment:
|
||||
- ZMQ_HOSTNAME=localhost
|
||||
- ZMQ_PORT=50000
|
||||
- ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e
|
||||
restart: unless-stopped
|
||||
@@ -1,2 +0,0 @@
|
||||
pyzmq
|
||||
requests
|
||||
@@ -1,57 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
print("Running imports")
|
||||
import sys
|
||||
import zmq
|
||||
import json
|
||||
import time
|
||||
import pprint
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
|
||||
forward_url = os.getenv("ZMQ_FORWARD_URL", "")
|
||||
print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url)
|
||||
def handle_hook(data):
|
||||
ret = requests.post(forward_url, json=data)
|
||||
print(ret.text)
|
||||
print(ret.status_code)
|
||||
|
||||
def main():
|
||||
host = os.getenv("ZMQ_HOST", "localhost")
|
||||
port = os.getenv("ZMQ_PORT", "50000")
|
||||
|
||||
if len(forward_url) == 0:
|
||||
print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder")
|
||||
exit(0)
|
||||
|
||||
print("Starting connection setup to %s:%s" % (host, port))
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.SUB)
|
||||
socket.connect ("tcp://%s:%s" % (host, port))
|
||||
socket.setsockopt(zmq.SUBSCRIBE, b'')
|
||||
|
||||
poller = zmq.Poller()
|
||||
poller.register(socket, zmq.POLLIN)
|
||||
|
||||
print("Starting zmq check for %s:%s" % (host, port))
|
||||
while True:
|
||||
socks = dict(poller.poll(timeout=None))
|
||||
if socket in socks and socks[socket] == zmq.POLLIN:
|
||||
message = socket.recv()
|
||||
#print(message)
|
||||
topic, s, m = message.decode('utf-8').partition(" ")
|
||||
|
||||
d = json.loads(m)
|
||||
try:
|
||||
# print test if you want status (heartbeat)
|
||||
test = d["status"]
|
||||
except KeyError:
|
||||
handle_hook(d)
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("In init ")
|
||||
main()
|
||||
@@ -1,36 +0,0 @@
|
||||
FROM golang:1.22 as builder
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
|
||||
COPY orborus.go /app/orborus.go
|
||||
#COPY go.mod /app/go.mod
|
||||
#COPY go.sum /app/go.sum
|
||||
RUN go mod init orborus
|
||||
|
||||
RUN go get github.com/docker/docker/api/types
|
||||
RUN go get github.com/docker/docker/api/types/container
|
||||
#RUN go get github.com/docker/docker/client
|
||||
RUN go get github.com/mackerelio/go-osstat/cpu
|
||||
RUN go get github.com/mackerelio/go-osstat/memory
|
||||
RUN go get github.com/satori/go.uuid
|
||||
RUN go get github.com/shuffle/shuffle-shared
|
||||
RUN go get github.com/shirou/gopsutil
|
||||
# RUN go get k8s.io/client-go
|
||||
# RUN go get k8s.io/apimachinery
|
||||
|
||||
RUN go get
|
||||
RUN go mod tidy
|
||||
RUN go build
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
|
||||
|
||||
FROM alpine:3.15.0
|
||||
RUN apk add --no-cache bash tzdata
|
||||
COPY --from=builder /app/ /
|
||||
|
||||
ENV ENVIRONMENT_NAME=Shuffle
|
||||
ENV BASE_URL=http://shuffle-backend:5001
|
||||
ENV DOCKER_API_VERSION=1.40
|
||||
ENV SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
|
||||
|
||||
CMD ["./orborus"]
|
||||
Reference in New Issue
Block a user