Removed all irrelevant files
This commit is contained in:
Frikky
@@ -1,109 +0,0 @@
|
|||||||
import json
|
|
||||||
import yaml
|
|
||||||
|
|
||||||
items = []
|
|
||||||
|
|
||||||
openapi = {
|
|
||||||
"openapi": "3.0.2",
|
|
||||||
"info": {
|
|
||||||
"title": "MISP",
|
|
||||||
"description": "MISP API generated from the misp book: https://github.com/MISP/misp-book/blob/master/automation/README.md",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"contact": {
|
|
||||||
"name": "@frikkylikeme",
|
|
||||||
"url": "https://twitter.com/frikkylikeme",
|
|
||||||
"email": "frikky@shuffler.io"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"paths": {},
|
|
||||||
"components": {
|
|
||||||
"schemas": {},
|
|
||||||
"securitySchemes": {
|
|
||||||
"ApiKeyAuth": {
|
|
||||||
"type": "apikey",
|
|
||||||
"in": "header",
|
|
||||||
"name": "Authorization",
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
with open("misp.txt", "r") as tmp:
|
|
||||||
newitem = {}
|
|
||||||
recorditem = False
|
|
||||||
|
|
||||||
counter = 0
|
|
||||||
itemsplit = tmp.read().split("\n")
|
|
||||||
for item in itemsplit:
|
|
||||||
counter += 1
|
|
||||||
if item.startswith("### ") and "/" in item:
|
|
||||||
try:
|
|
||||||
path = item.split(" ")[2]
|
|
||||||
method = item.split(" ")[1].lower()
|
|
||||||
newitem = {
|
|
||||||
"path": path,
|
|
||||||
"method": method,
|
|
||||||
}
|
|
||||||
|
|
||||||
try:
|
|
||||||
openapi["paths"][path][method] = {}
|
|
||||||
except KeyError:
|
|
||||||
openapi["paths"][path] = {}
|
|
||||||
openapi["paths"][path][method] = {}
|
|
||||||
|
|
||||||
except IndexError:
|
|
||||||
newitem = {}
|
|
||||||
continue
|
|
||||||
|
|
||||||
recorditem = True
|
|
||||||
#print(newitem)
|
|
||||||
|
|
||||||
if not recorditem:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if "Description" in item:
|
|
||||||
openapi["paths"][newitem["path"]][newitem["method"]]["description"] = itemsplit[counter+1]
|
|
||||||
elif "URL Arguments" in item:
|
|
||||||
parameters = []
|
|
||||||
innercnt = 0
|
|
||||||
|
|
||||||
openapi["paths"][newitem["path"]][newitem["method"]]["parameters"] = []
|
|
||||||
while True:
|
|
||||||
curline = itemsplit[counter+1+innercnt]
|
|
||||||
if "#" in curline:
|
|
||||||
break
|
|
||||||
|
|
||||||
innercnt += 1
|
|
||||||
if not curline:
|
|
||||||
continue
|
|
||||||
|
|
||||||
print(curline)
|
|
||||||
parameters.append({
|
|
||||||
"description": curline.split(" ")[1],
|
|
||||||
"in": "query",
|
|
||||||
"name": curline.split(" ")[1],
|
|
||||||
"required": True,
|
|
||||||
"schema": {"type": "string"},
|
|
||||||
})
|
|
||||||
|
|
||||||
openapi["paths"][newitem["path"]][newitem["method"]]["parameters"] = parameters
|
|
||||||
elif "Output" in item:
|
|
||||||
# FIXME
|
|
||||||
innercnt = 0
|
|
||||||
while True:
|
|
||||||
curline = itemsplit[counter+1+innercnt]
|
|
||||||
|
|
||||||
if "#" in curline:
|
|
||||||
break
|
|
||||||
|
|
||||||
innercnt += 1
|
|
||||||
if "json" in curline:
|
|
||||||
continue
|
|
||||||
#print(curline)
|
|
||||||
|
|
||||||
print(json.dumps(openapi, indent=4))
|
|
||||||
|
|
||||||
|
|
||||||
generatedfile = "generated/misp.yaml"
|
|
||||||
with open(generatedfile, "w+") as tmp:
|
|
||||||
tmp.write(yaml.dump(openapi))
|
|
||||||
@@ -1,311 +0,0 @@
|
|||||||
import requests
|
|
||||||
import yaml
|
|
||||||
import json
|
|
||||||
import os
|
|
||||||
import io
|
|
||||||
import base64
|
|
||||||
from PIL import Image
|
|
||||||
#import tkinter
|
|
||||||
#import _tkinter
|
|
||||||
#tkinter._test()
|
|
||||||
|
|
||||||
|
|
||||||
#sudo apt-get install python-imaging-tk
|
|
||||||
#sudo apt-get install python3-tk
|
|
||||||
|
|
||||||
# USAGE:
|
|
||||||
# 1. Find the item here:
|
|
||||||
# https://apphub.swimlane.com/swimbundles/swimlane/sw_alienvault_threatcrowd
|
|
||||||
# 2.
|
|
||||||
# https://jsonlint.com/
|
|
||||||
|
|
||||||
# META: data["meta"]. Stuff like count. May be useful :)
|
|
||||||
|
|
||||||
def parse_data(data):
|
|
||||||
openapi = {
|
|
||||||
"openapi": "3.0.2",
|
|
||||||
"info": {
|
|
||||||
"title": "",
|
|
||||||
"description": "",
|
|
||||||
"version": "1.0.0",
|
|
||||||
"contact": {
|
|
||||||
"name": "@frikkylikeme",
|
|
||||||
"url": "https://twitter.com/frikkylikeme",
|
|
||||||
"email": "frikky@shuffler.io"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"paths": {},
|
|
||||||
"components": {
|
|
||||||
"schemas": {},
|
|
||||||
"securitySchemes": {},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
data = data["swimbundle"]
|
|
||||||
filename = "%s.yaml" % data["product"].replace(" ", "_").lower()
|
|
||||||
openapi["info"]["title"] = "%s %s" % (data["vendor"], data["product"])
|
|
||||||
openapi["info"]["description"] = "Automated generation of %s" % (openapi["info"]["title"])
|
|
||||||
# data["description"]
|
|
||||||
|
|
||||||
# https://swagger.io/docs/specification/authentication/
|
|
||||||
try:
|
|
||||||
asset = data["asset"]
|
|
||||||
inputparams = asset["inputParameters"]
|
|
||||||
|
|
||||||
try:
|
|
||||||
openapi["servers"] = [inputparams["api_url"]["example"]]
|
|
||||||
except KeyError as e:
|
|
||||||
#print(inputparams)
|
|
||||||
#print("Field error: %s" % e)
|
|
||||||
pass
|
|
||||||
|
|
||||||
authset = False
|
|
||||||
try:
|
|
||||||
tmpauth = inputparams["api_user"]
|
|
||||||
tmpauth = inputparams["api_key"]
|
|
||||||
|
|
||||||
openapi["components"]["securitySchemes"] = {
|
|
||||||
"BasicAuth": {
|
|
||||||
"type": "http",
|
|
||||||
"scheme": "basic"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
authset = True
|
|
||||||
except KeyError as e:
|
|
||||||
pass
|
|
||||||
|
|
||||||
try:
|
|
||||||
tmpauth = inputparams["username"]
|
|
||||||
tmpauth = inputparams["password"]
|
|
||||||
|
|
||||||
openapi["components"]["securitySchemes"] = {
|
|
||||||
"BasicAuth": {
|
|
||||||
"type": "http",
|
|
||||||
"scheme": "basic"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
authset = True
|
|
||||||
except KeyError as e:
|
|
||||||
pass
|
|
||||||
|
|
||||||
#if not authset:
|
|
||||||
# print("AUTH NOT SET: %s" % inputparams)
|
|
||||||
|
|
||||||
except KeyError as e:
|
|
||||||
print("KeyError asset: %s" % e)
|
|
||||||
|
|
||||||
cnt = 0
|
|
||||||
paramnames = []
|
|
||||||
for task in data["tasks"]:
|
|
||||||
method = "post"
|
|
||||||
|
|
||||||
openapi["paths"]["tmp%d" % cnt] = {}
|
|
||||||
openapi["paths"]["tmp%d" % cnt][method] = {
|
|
||||||
"summary": task["name"],
|
|
||||||
"description": task["description"],
|
|
||||||
"parameters": [],
|
|
||||||
"responses": {
|
|
||||||
"200": {
|
|
||||||
"description": "Successful request",
|
|
||||||
|
|
||||||
}
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
taskcategory = task["family"]
|
|
||||||
taskname = task["name"]
|
|
||||||
paramnames.append(taskname)
|
|
||||||
|
|
||||||
for key, value in task["inputParameters"].items():
|
|
||||||
schema = "string"
|
|
||||||
inVar = "query"
|
|
||||||
|
|
||||||
if value["type"] == 6:
|
|
||||||
inVar = "body"
|
|
||||||
|
|
||||||
schema = "string"
|
|
||||||
schemaset = False
|
|
||||||
if value["type"] != 1:
|
|
||||||
if (value["type"] == 7):
|
|
||||||
schema = "boolean"
|
|
||||||
schemaset = True
|
|
||||||
|
|
||||||
if schema == "string" and schemaset:
|
|
||||||
print("Should change type: %d" % value["type"])
|
|
||||||
print(task["name"])
|
|
||||||
print(value["name"])
|
|
||||||
|
|
||||||
example = ""
|
|
||||||
try:
|
|
||||||
example = value["example"]
|
|
||||||
except KeyError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
description = ""
|
|
||||||
try:
|
|
||||||
description = value["description"]
|
|
||||||
except KeyError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
required = False
|
|
||||||
try:
|
|
||||||
required = value["required"]
|
|
||||||
except KeyError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
openapi["paths"]["tmp%d" % cnt][method]["parameters"].append({
|
|
||||||
"name": value["name"],
|
|
||||||
"required": required,
|
|
||||||
"example": example,
|
|
||||||
"description": description,
|
|
||||||
"schema": {"type": schema},
|
|
||||||
"in": inVar
|
|
||||||
})
|
|
||||||
|
|
||||||
if len(task["availableOutputVariables"]) > 0:
|
|
||||||
openapi["paths"]["tmp%d" % cnt][method]["responses"]["200"]["content"] = {
|
|
||||||
"application/json": {
|
|
||||||
"schema": {
|
|
||||||
"$ref": "#/components/schemas/tmp%d" % cnt
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#responses:
|
|
||||||
# '200':
|
|
||||||
# content:
|
|
||||||
# application/json:
|
|
||||||
# schema:
|
|
||||||
# $ref: '#/components/schemas/tmp1'
|
|
||||||
#description: Successful request
|
|
||||||
|
|
||||||
openapi["components"]["schemas"]["tmp%d" % cnt] = {
|
|
||||||
"type": "object",
|
|
||||||
"properties": {},
|
|
||||||
}
|
|
||||||
|
|
||||||
for key, value in task["availableOutputVariables"].items():
|
|
||||||
if key == "response_code":
|
|
||||||
continue
|
|
||||||
|
|
||||||
openapi["components"]["schemas"]["tmp%d" % cnt]["properties"][key] = {
|
|
||||||
"type": "string"
|
|
||||||
}
|
|
||||||
|
|
||||||
cnt += 1
|
|
||||||
|
|
||||||
print("%s: %d" % (openapi["info"]["title"], len(paramnames)))
|
|
||||||
|
|
||||||
return filename, openapi
|
|
||||||
|
|
||||||
def dump_data(filename, openapi, category):
|
|
||||||
generatedfile = "generated/%s/%s" % (category, filename)
|
|
||||||
try:
|
|
||||||
with open(generatedfile, "w+") as tmp:
|
|
||||||
tmp.write(yaml.dump(openapi))
|
|
||||||
except FileNotFoundError:
|
|
||||||
try:
|
|
||||||
os.mkdir("generated/%s" % category)
|
|
||||||
with open(generatedfile, "w+") as tmp:
|
|
||||||
tmp.write(yaml.dump(openapi))
|
|
||||||
|
|
||||||
except FileExistsError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
#https://apphub.swimlane.com/
|
|
||||||
categories = [
|
|
||||||
"Investigation",
|
|
||||||
"Endpoint Security & Management",
|
|
||||||
"Network Security & Management",
|
|
||||||
"Communication",
|
|
||||||
"SIEM & Log Management",
|
|
||||||
"Governance & Risk Management",
|
|
||||||
"Vulnerability & Patch Management",
|
|
||||||
"Ticket Management",
|
|
||||||
"DevOps & Application Security",
|
|
||||||
"Identity & Access Management",
|
|
||||||
"Infrastructure",
|
|
||||||
"Miscellaneous",
|
|
||||||
]
|
|
||||||
|
|
||||||
search_category = categories[2]
|
|
||||||
total = 0
|
|
||||||
for search_category in categories:
|
|
||||||
number = 1
|
|
||||||
innertotal = 0
|
|
||||||
|
|
||||||
while(True):
|
|
||||||
url = "https://apphub.swimlane.io/api/search/swimbundles?page=%d" % number
|
|
||||||
|
|
||||||
json = {"fields": {"family": search_category}}
|
|
||||||
ret = requests.post(
|
|
||||||
url,
|
|
||||||
json=json,
|
|
||||||
)
|
|
||||||
|
|
||||||
if ret.status_code != 201:
|
|
||||||
print("RET NOT 201: %d" % ret.status_code)
|
|
||||||
break
|
|
||||||
|
|
||||||
parsed = ret.json()
|
|
||||||
try:
|
|
||||||
category = parsed["data"][0]["swimbundleMeta"]["family"][0]
|
|
||||||
except KeyError:
|
|
||||||
category = ""
|
|
||||||
except IndexError:
|
|
||||||
category = ""
|
|
||||||
|
|
||||||
if category == "":
|
|
||||||
break
|
|
||||||
|
|
||||||
for data in parsed["data"]:
|
|
||||||
try:
|
|
||||||
filename, openapi = parse_data(data)
|
|
||||||
except:
|
|
||||||
try:
|
|
||||||
print("Skipping %s %s because of an error" % (data["vendor"], data["product"]))
|
|
||||||
except KeyError:
|
|
||||||
pass
|
|
||||||
|
|
||||||
continue
|
|
||||||
|
|
||||||
openapi["tags"] = [
|
|
||||||
{
|
|
||||||
"name": category,
|
|
||||||
}
|
|
||||||
]
|
|
||||||
|
|
||||||
appid = data["swimbundleMeta"]["logo"]["id"]
|
|
||||||
logoUrl = "https://apphub.swimlane.io/api/logos/%s" % appid
|
|
||||||
logodata = requests.get(logoUrl)
|
|
||||||
if logodata.status_code == 200:
|
|
||||||
logojson = logodata.json()
|
|
||||||
try:
|
|
||||||
logobase64 = logojson["data"]["base64"]
|
|
||||||
#.split(",")[1]
|
|
||||||
|
|
||||||
openapi["info"]["x-logo"] = logobase64
|
|
||||||
#print(logobase64)
|
|
||||||
#msg = base64.b64decode(logobase64)
|
|
||||||
#with io.BytesIO(msg) as buf:
|
|
||||||
# with Image.open(buf) as tempImg:
|
|
||||||
# newWidth = 174 / tempImg.width # change this to what ever width you need.
|
|
||||||
# newHeight = 174 / tempImg.height # change this to what ever height you need.
|
|
||||||
# newSize = (int(newWidth * tempImg.width), int(newHeight * tempImg.height))
|
|
||||||
# newImg1 = tempImg.resize(newSize)
|
|
||||||
# lbl1.IMG = ImageTk.PhotoImage(image=newImg1)
|
|
||||||
# lbl1.configure(image=lbl1.IMG)
|
|
||||||
except KeyError:
|
|
||||||
print("Failed logo parsing for %s" % appid)
|
|
||||||
pass
|
|
||||||
|
|
||||||
dump_data(filename, openapi, category)
|
|
||||||
innertotal += 1
|
|
||||||
total += 1
|
|
||||||
|
|
||||||
number += 1
|
|
||||||
|
|
||||||
print("Created %d openapi specs from Swimlane with category %s" % (innertotal, search_category))
|
|
||||||
|
|
||||||
print("\nCreated %d TOTAL openapi specs from Swimlane" % (total))
|
|
||||||
@@ -1,7 +0,0 @@
|
|||||||
# OpenAPI generator
|
|
||||||
This contains test code that's been moved to shaffuru/backend/go-app/codegen.go
|
|
||||||
|
|
||||||
## Todo:
|
|
||||||
1. Don't use filesystem, but rather store in GCP
|
|
||||||
2. Add swagger 2.0 to 3.0 converter
|
|
||||||
3. Fix body / data parsing
|
|
||||||
@@ -1,29 +0,0 @@
|
|||||||
# Base our app image off of the WALKOFF App SDK image
|
|
||||||
FROM frikky/shuffle:app_sdk as base
|
|
||||||
|
|
||||||
# We're going to stage away all of the bloat from the build tools so lets create a builder stage
|
|
||||||
#FROM base as builder
|
|
||||||
|
|
||||||
# Install all alpine build tools needed for our pip installs
|
|
||||||
#RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
|
|
||||||
|
|
||||||
# Install all of our pip packages in a single directory that we can copy to our base image later
|
|
||||||
#RUN mkdir /install
|
|
||||||
#WORKDIR /install
|
|
||||||
#COPY requirements.txt /requirements.txt
|
|
||||||
#
|
|
||||||
## Switch back to our base image and copy in all of our built packages and source code
|
|
||||||
#FROM base
|
|
||||||
#COPY --from=builder /install /usr/local
|
|
||||||
|
|
||||||
|
|
||||||
# Install any binary dependencies needed in our final image - this can be a lot of different stuff
|
|
||||||
#RUN apk --no-cache add --update libmagic
|
|
||||||
WORKDIR /
|
|
||||||
COPY requirements.txt /requirements.txt
|
|
||||||
RUN pip install --prefix="/usr/local" -r /requirements.txt
|
|
||||||
COPY src /app
|
|
||||||
|
|
||||||
# Finally, lets run our app!
|
|
||||||
WORKDIR /app
|
|
||||||
CMD python app.py --log-level DEBUG
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
# No extra requirements needed
|
|
||||||
requests==2.32.3
|
|
||||||
urllib3==2.3.0
|
|
||||||
liquidpy==0.8.2
|
|
||||||
MarkupSafe==3.0.2
|
|
||||||
flask[async]==3.1.0
|
|
||||||
python-dateutil==2.9.0.post0
|
|
||||||
PyJWT==2.10.1
|
|
||||||
cryptography==44.0.2
|
|
||||||
shufflepy==0.1.0
|
|
||||||
shuffle-sdk==0.0.25
|
|
||||||
@@ -1,387 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"crypto/md5"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"github.com/getkin/kin-openapi/openapi3"
|
|
||||||
"gopkg.in/yaml.v2"
|
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
)
|
|
||||||
|
|
||||||
type WorkflowApp struct {
|
|
||||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
|
||||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
|
||||||
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
|
|
||||||
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
|
|
||||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
|
||||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
|
||||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
|
||||||
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
|
|
||||||
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
|
|
||||||
ContactInfo struct {
|
|
||||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
|
||||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
|
||||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
|
||||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
|
||||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuthenticationParams struct {
|
|
||||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
|
||||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
|
||||||
Example string `json:"example" datastore:"example" yaml:"example"s`
|
|
||||||
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
|
|
||||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
|
||||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Authentication struct {
|
|
||||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
|
||||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuthenticationStore struct {
|
|
||||||
Key string `json:"key" datastore:"key"`
|
|
||||||
Value string `json:"value" datastore:"value"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type WorkflowAppActionParameter struct {
|
|
||||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
|
||||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
|
||||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
|
||||||
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
|
|
||||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
|
||||||
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
|
|
||||||
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
|
|
||||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
|
||||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SchemaDefinition struct {
|
|
||||||
Type string `json:"type" datastore:"type"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type WorkflowAppAction struct {
|
|
||||||
Description string `json:"description" datastore:"description"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
|
||||||
Name string `json:"name" datastore:"name"`
|
|
||||||
NodeType string `json:"node_type" datastore:"node_type"`
|
|
||||||
Environment string `json:"environment" datastore:"environment"`
|
|
||||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
|
|
||||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
|
||||||
Returns struct {
|
|
||||||
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
|
||||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
|
||||||
} `json:"returns" datastore:"returns"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyFile(fromfile, tofile string) error {
|
|
||||||
from, err := os.Open(fromfile)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer from.Close()
|
|
||||||
|
|
||||||
to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer to.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(to, from)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Builds the base structure for the app that we're making
|
|
||||||
// Returns error if anything goes wrong. This has to work if
|
|
||||||
// the python code is supposed to be generated
|
|
||||||
func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
|
|
||||||
//log.Printf("%#v", swagger)
|
|
||||||
|
|
||||||
// adding md5 based on input data to not overwrite earlier data.
|
|
||||||
generatedPath := "generated"
|
|
||||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
|
||||||
appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
|
|
||||||
|
|
||||||
os.MkdirAll(appPath, os.ModePerm)
|
|
||||||
os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm)
|
|
||||||
|
|
||||||
err := copyFile("baseline/Dockerfile", fmt.Sprintf("%s/%s", appPath, "Dockerfile"))
|
|
||||||
if err != nil {
|
|
||||||
log.Println("Failed to move Dockerfile")
|
|
||||||
return appPath, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = copyFile("baseline/requirements.txt", fmt.Sprintf("%s/%s", appPath, "requirements.txt"))
|
|
||||||
if err != nil {
|
|
||||||
log.Println("Failed to move requrements.txt")
|
|
||||||
return appPath, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return appPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func makePythoncode(name, url, method string, parameters, optionalQueries []string) string {
|
|
||||||
method = strings.ToLower(method)
|
|
||||||
queryString := ""
|
|
||||||
queryData := ""
|
|
||||||
|
|
||||||
// FIXME - this might break - need to check if ? or & should be set as query
|
|
||||||
parameterData := ""
|
|
||||||
if len(optionalQueries) > 0 {
|
|
||||||
queryString += ", "
|
|
||||||
for _, query := range optionalQueries {
|
|
||||||
queryString += fmt.Sprintf("%s=\"\"", query)
|
|
||||||
queryData += fmt.Sprintf(`
|
|
||||||
if %s:
|
|
||||||
url += f"&%s={%s}"`, query, query, query)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(parameters) > 0 {
|
|
||||||
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME - add checks for query data etc
|
|
||||||
data := fmt.Sprintf(` async def %s_%s(self%s%s):
|
|
||||||
url=f"%s"
|
|
||||||
%s
|
|
||||||
return requests.%s(url).text
|
|
||||||
`, name, method, parameterData, queryString, url, queryData, method)
|
|
||||||
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
|
|
||||||
api := WorkflowApp{}
|
|
||||||
log.Printf("%#v", swagger.Info)
|
|
||||||
|
|
||||||
if len(swagger.Info.Title) == 0 {
|
|
||||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(swagger.Servers) == 0 {
|
|
||||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'")
|
|
||||||
}
|
|
||||||
|
|
||||||
api.Name = swagger.Info.Title
|
|
||||||
api.Description = swagger.Info.Description
|
|
||||||
api.IsValid = true
|
|
||||||
api.Link = swagger.Servers[0].URL // host does not exist lol
|
|
||||||
api.AppVersion = "1.0.0"
|
|
||||||
api.Environment = "cloud"
|
|
||||||
api.ID = ""
|
|
||||||
api.SmallImage = ""
|
|
||||||
api.LargeImage = ""
|
|
||||||
|
|
||||||
// This is the python code to be generated
|
|
||||||
// Could just as well be go at this point lol
|
|
||||||
pythonFunctions := []string{}
|
|
||||||
|
|
||||||
for actualPath, path := range swagger.Paths {
|
|
||||||
//log.Printf("%#v", path)
|
|
||||||
//log.Printf("%#v", actualPath)
|
|
||||||
// Find the path name and add it to makeCode() param
|
|
||||||
|
|
||||||
firstQuery := true
|
|
||||||
if path.Get != nil {
|
|
||||||
// What to do with this, hmm
|
|
||||||
functionName := strings.ReplaceAll(path.Get.Summary, " ", "_")
|
|
||||||
functionName = strings.ToLower(functionName)
|
|
||||||
|
|
||||||
action := WorkflowAppAction{
|
|
||||||
Description: path.Get.Description,
|
|
||||||
Name: path.Get.Summary,
|
|
||||||
NodeType: "action",
|
|
||||||
Environment: api.Environment,
|
|
||||||
Parameters: []WorkflowAppActionParameter{},
|
|
||||||
}
|
|
||||||
|
|
||||||
action.Returns.Schema.Type = "string"
|
|
||||||
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
|
|
||||||
|
|
||||||
//log.Println(path.Parameters)
|
|
||||||
|
|
||||||
// Parameters: []WorkflowAppActionParameter{},
|
|
||||||
// FIXME - add data for POST stuff
|
|
||||||
firstQuery = true
|
|
||||||
optionalQueries := []string{}
|
|
||||||
parameters := []string{}
|
|
||||||
optionalParameters := []WorkflowAppActionParameter{}
|
|
||||||
if len(path.Get.Parameters) > 0 {
|
|
||||||
for _, param := range path.Get.Parameters {
|
|
||||||
curParam := WorkflowAppActionParameter{
|
|
||||||
Name: param.Value.Name,
|
|
||||||
Description: param.Value.Description,
|
|
||||||
Multiline: false,
|
|
||||||
Required: param.Value.Required,
|
|
||||||
Schema: SchemaDefinition{
|
|
||||||
Type: param.Value.Schema.Value.Type,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if param.Value.Required {
|
|
||||||
action.Parameters = append(action.Parameters, curParam)
|
|
||||||
} else {
|
|
||||||
optionalParameters = append(optionalParameters, curParam)
|
|
||||||
}
|
|
||||||
|
|
||||||
if param.Value.In == "path" {
|
|
||||||
log.Printf("PATH!: %s", param.Value.Name)
|
|
||||||
parameters = append(parameters, param.Value.Name)
|
|
||||||
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
|
|
||||||
} else if param.Value.In == "query" {
|
|
||||||
log.Printf("QUERY!: %s", param.Value.Name)
|
|
||||||
if !param.Value.Required {
|
|
||||||
optionalQueries = append(optionalQueries, param.Value.Name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
parameters = append(parameters, param.Value.Name)
|
|
||||||
|
|
||||||
if firstQuery {
|
|
||||||
baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
|
||||||
firstQuery = false
|
|
||||||
} else {
|
|
||||||
baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
|
||||||
firstQuery = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensuring that they end up last in the specification
|
|
||||||
// (order is ish important for optional params) - they need to be last.
|
|
||||||
for _, optionalParam := range optionalParameters {
|
|
||||||
action.Parameters = append(action.Parameters, optionalParam)
|
|
||||||
}
|
|
||||||
|
|
||||||
curCode := makePythoncode(functionName, baseUrl, "get", parameters, optionalQueries)
|
|
||||||
pythonFunctions = append(pythonFunctions, curCode)
|
|
||||||
|
|
||||||
api.Actions = append(api.Actions, action)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return api, pythonFunctions, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func verifyApi(api WorkflowApp) WorkflowApp {
|
|
||||||
if api.AppVersion == "" {
|
|
||||||
api.AppVersion = "1.0.0"
|
|
||||||
}
|
|
||||||
|
|
||||||
return api
|
|
||||||
}
|
|
||||||
|
|
||||||
func dumpPython(basePath, name, version string, pythonFunctions []string) error {
|
|
||||||
//log.Printf("%#v", api)
|
|
||||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
|
||||||
|
|
||||||
parsedCode := fmt.Sprintf(`import requests
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
|
|
||||||
from walkoff_app_sdk.app_base import AppBase
|
|
||||||
|
|
||||||
class %s(AppBase):
|
|
||||||
"""
|
|
||||||
Autogenerated class by Shuffler
|
|
||||||
"""
|
|
||||||
|
|
||||||
__version__ = "%s"
|
|
||||||
app_name = "%s"
|
|
||||||
|
|
||||||
def __init__(self, redis, logger, console_logger=None):
|
|
||||||
self.verify = False
|
|
||||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
||||||
super().__init__(redis, logger, console_logger)
|
|
||||||
|
|
||||||
%s
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(CarbonBlack.run(), debug=True)
|
|
||||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
|
||||||
|
|
||||||
err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Println(parsedCode)
|
|
||||||
|
|
||||||
//log.Println(string(data))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func dumpApi(basePath string, api WorkflowApp) error {
|
|
||||||
//log.Printf("%#v", api)
|
|
||||||
data, err := yaml.Marshal(api)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error with yaml marshal: %s", err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
//log.Println(string(data))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
data := []byte(`{"swagger":"3.0","info":{"title":"hi","description":"you","version":"1.0"},"servers":[{"url":"https://shuffler.io/api/v1"}],"host":"shuffler.io","basePath":"/api/v1","schemes":["https:"],"paths":{"/workflows":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflows","description":"Get workflows","parameters":[]}},"/workflows/{id}":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflow","description":"Get workflow","parameters":[{"in":"query","name":"forgetme","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"query","name":"anotherone","description":"Generated by shuffler.io OpenAPI","required":false,"schema":{"type":"string"}},{"in":"query","name":"hi","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"path","name":"id","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}}]}}},"securityDefinitions":{}}`)
|
|
||||||
|
|
||||||
hasher := md5.New()
|
|
||||||
hasher.Write(data)
|
|
||||||
newmd5 := hex.EncodeToString(hasher.Sum(nil))
|
|
||||||
|
|
||||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(data)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Swagger validation error: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(swagger.Info.Title, " ") {
|
|
||||||
strings.ReplaceAll(swagger.Info.Title, " ", "")
|
|
||||||
}
|
|
||||||
|
|
||||||
basePath, err := buildStructure(swagger, newmd5)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to build base structure: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
api, pythonfunctions, err := generateYaml(swagger)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed building and generating yaml: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = dumpApi(basePath, api)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed dumping yaml: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = dumpPython(basePath, swagger.Info.Title, swagger.Info.Version, pythonfunctions)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed dumping python: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,499 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
/*
|
|
||||||
Code used to generate apps from OpenAPI JSON data
|
|
||||||
Any function ending with GCP doesn't use local fileIO, but rather
|
|
||||||
google cloud storage, and also has a normal filesystem version of the
|
|
||||||
same code (doesn't required client as first argument).
|
|
||||||
|
|
||||||
This code is used in the backend to generate apps on the fly for users.
|
|
||||||
All new code is appended to backend/go-app/codegen.go
|
|
||||||
*/
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"crypto/md5"
|
|
||||||
"encoding/hex"
|
|
||||||
"errors"
|
|
||||||
"fmt"
|
|
||||||
"io"
|
|
||||||
"io/ioutil"
|
|
||||||
"log"
|
|
||||||
"os"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"cloud.google.com/go/storage"
|
|
||||||
"github.com/getkin/kin-openapi/openapi3"
|
|
||||||
"gopkg.in/yaml.v2"
|
|
||||||
)
|
|
||||||
|
|
||||||
var bucketName = "shuffler.appspot.com"
|
|
||||||
|
|
||||||
type WorkflowApp struct {
|
|
||||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
|
||||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
|
||||||
ID string `json:"id" yaml:"id,omitempty" required:false datastore:"id"`
|
|
||||||
Link string `json:"link" yaml:"link" required:false datastore:"link,noindex"`
|
|
||||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
|
||||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
|
||||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
|
||||||
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
|
|
||||||
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
|
|
||||||
ContactInfo struct {
|
|
||||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
|
||||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
|
||||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
|
||||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
|
||||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuthenticationParams struct {
|
|
||||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
|
||||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
|
||||||
Example string `json:"example" datastore:"example" yaml:"example"s`
|
|
||||||
Value string `json:"value,omitempty" datastore:"value" yaml:"value"`
|
|
||||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
|
||||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type Authentication struct {
|
|
||||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
|
||||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type AuthenticationStore struct {
|
|
||||||
Key string `json:"key" datastore:"key"`
|
|
||||||
Value string `json:"value" datastore:"value"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type WorkflowAppActionParameter struct {
|
|
||||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
|
||||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
|
||||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
|
||||||
Value string `json:"value" datastore:"value" yaml:"value,omitempty"`
|
|
||||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
|
||||||
ActionField string `json:"action_field" datastore:"action_field" yaml:"actionfield,omitempty"`
|
|
||||||
Variant string `json:"variant" datastore:"variant" yaml:"variant,omitempty"`
|
|
||||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
|
||||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type SchemaDefinition struct {
|
|
||||||
Type string `json:"type" datastore:"type"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type WorkflowAppAction struct {
|
|
||||||
Description string `json:"description" datastore:"description"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
|
||||||
Name string `json:"name" datastore:"name"`
|
|
||||||
NodeType string `json:"node_type" datastore:"node_type"`
|
|
||||||
Environment string `json:"environment" datastore:"environment"`
|
|
||||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication" yaml:"authentication,omitempty"`
|
|
||||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
|
||||||
Returns struct {
|
|
||||||
Description string `json:"description" datastore:"returns" yaml:"description,omitempty"`
|
|
||||||
ID string `json:"id" datastore:"id" yaml:"id,omitempty"`
|
|
||||||
Schema SchemaDefinition `json:"schema" datastore:"schema" yaml:"schema"`
|
|
||||||
} `json:"returns" datastore:"returns"`
|
|
||||||
}
|
|
||||||
|
|
||||||
func copyFile(fromfile, tofile string) error {
|
|
||||||
from, err := os.Open(fromfile)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer from.Close()
|
|
||||||
|
|
||||||
to, err := os.OpenFile(tofile, os.O_RDWR|os.O_CREATE, 0666)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
defer to.Close()
|
|
||||||
|
|
||||||
_, err = io.Copy(to, from)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func buildStructureGCP(client *storage.Client, swagger *openapi3.Swagger, curHash string) (string, error) {
|
|
||||||
ctx := context.Background()
|
|
||||||
|
|
||||||
// 1. Have baseline in bucket/generated_apps/baseline
|
|
||||||
// 2. Copy the baseline to a new folder with identifier name
|
|
||||||
|
|
||||||
basePath := "generated_apps"
|
|
||||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
|
||||||
appPath := fmt.Sprintf("%s/%s", basePath, identifier)
|
|
||||||
fileNames := []string{"Dockerfile", "requirements.txt"}
|
|
||||||
for _, file := range fileNames {
|
|
||||||
src := client.Bucket(bucketName).Object(fmt.Sprintf("%s/baseline/%s", basePath, file))
|
|
||||||
dst := client.Bucket(bucketName).Object(fmt.Sprintf("%s/%s", appPath, file))
|
|
||||||
if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return appPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Builds the base structure for the app that we're making
|
|
||||||
// Returns error if anything goes wrong. This has to work if
|
|
||||||
// the python code is supposed to be generated
|
|
||||||
func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) {
|
|
||||||
//log.Printf("%#v", swagger)
|
|
||||||
|
|
||||||
// adding md5 based on input data to not overwrite earlier data.
|
|
||||||
generatedPath := "generated"
|
|
||||||
identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash)
|
|
||||||
appPath := fmt.Sprintf("%s/%s", generatedPath, identifier)
|
|
||||||
|
|
||||||
os.MkdirAll(appPath, os.ModePerm)
|
|
||||||
os.Mkdir(fmt.Sprintf("%s/src", appPath), os.ModePerm)
|
|
||||||
|
|
||||||
err := copyFile("baseline/Dockerfile", fmt.Sprintf("%s/%s", appPath, "Dockerfile"))
|
|
||||||
if err != nil {
|
|
||||||
log.Println("Failed to move Dockerfile")
|
|
||||||
return appPath, err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = copyFile("baseline/requirements.txt", fmt.Sprintf("%s/%s", appPath, "requirements.txt"))
|
|
||||||
if err != nil {
|
|
||||||
log.Println("Failed to move requrements.txt")
|
|
||||||
return appPath, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return appPath, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func makePythoncode(name, url, method string, parameters, optionalQueries []string) string {
|
|
||||||
method = strings.ToLower(method)
|
|
||||||
queryString := ""
|
|
||||||
queryData := ""
|
|
||||||
|
|
||||||
// FIXME - this might break - need to check if ? or & should be set as query
|
|
||||||
parameterData := ""
|
|
||||||
if len(optionalQueries) > 0 {
|
|
||||||
queryString += ", "
|
|
||||||
for _, query := range optionalQueries {
|
|
||||||
queryString += fmt.Sprintf("%s=\"\"", query)
|
|
||||||
queryData += fmt.Sprintf(`
|
|
||||||
if %s:
|
|
||||||
url += f"&%s={%s}"`, query, query, query)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(parameters) > 0 {
|
|
||||||
parameterData = fmt.Sprintf(", %s", strings.Join(parameters, ", "))
|
|
||||||
}
|
|
||||||
|
|
||||||
// FIXME - add checks for query data etc
|
|
||||||
data := fmt.Sprintf(` async def %s_%s(self%s%s):
|
|
||||||
url=f"%s"
|
|
||||||
%s
|
|
||||||
return requests.%s(url).text
|
|
||||||
`, name, method, parameterData, queryString, url, queryData, method)
|
|
||||||
|
|
||||||
return data
|
|
||||||
}
|
|
||||||
|
|
||||||
func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
|
|
||||||
api := WorkflowApp{}
|
|
||||||
log.Printf("%#v", swagger.Info)
|
|
||||||
|
|
||||||
if len(swagger.Info.Title) == 0 {
|
|
||||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Info.Title can't be empty.")
|
|
||||||
}
|
|
||||||
|
|
||||||
if len(swagger.Servers) == 0 {
|
|
||||||
return WorkflowApp{}, []string{}, errors.New("Swagger.Servers can't be empty. Add 'servers':[{'url':'hostname.com'}'")
|
|
||||||
}
|
|
||||||
|
|
||||||
api.Name = swagger.Info.Title
|
|
||||||
api.Description = swagger.Info.Description
|
|
||||||
api.IsValid = true
|
|
||||||
api.Link = swagger.Servers[0].URL // host does not exist lol
|
|
||||||
api.AppVersion = "1.0.0"
|
|
||||||
api.Environment = "cloud"
|
|
||||||
api.ID = ""
|
|
||||||
api.SmallImage = ""
|
|
||||||
api.LargeImage = ""
|
|
||||||
|
|
||||||
// This is the python code to be generated
|
|
||||||
// Could just as well be go at this point lol
|
|
||||||
pythonFunctions := []string{}
|
|
||||||
|
|
||||||
for actualPath, path := range swagger.Paths {
|
|
||||||
//log.Printf("%#v", path)
|
|
||||||
//log.Printf("%#v", actualPath)
|
|
||||||
// Find the path name and add it to makeCode() param
|
|
||||||
|
|
||||||
firstQuery := true
|
|
||||||
if path.Get != nil {
|
|
||||||
// What to do with this, hmm
|
|
||||||
functionName := strings.ReplaceAll(path.Get.Summary, " ", "_")
|
|
||||||
functionName = strings.ToLower(functionName)
|
|
||||||
|
|
||||||
action := WorkflowAppAction{
|
|
||||||
Description: path.Get.Description,
|
|
||||||
Name: path.Get.Summary,
|
|
||||||
NodeType: "action",
|
|
||||||
Environment: api.Environment,
|
|
||||||
Parameters: []WorkflowAppActionParameter{},
|
|
||||||
}
|
|
||||||
|
|
||||||
action.Returns.Schema.Type = "string"
|
|
||||||
baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath)
|
|
||||||
|
|
||||||
//log.Println(path.Parameters)
|
|
||||||
|
|
||||||
// Parameters: []WorkflowAppActionParameter{},
|
|
||||||
// FIXME - add data for POST stuff
|
|
||||||
firstQuery = true
|
|
||||||
optionalQueries := []string{}
|
|
||||||
parameters := []string{}
|
|
||||||
optionalParameters := []WorkflowAppActionParameter{}
|
|
||||||
if len(path.Get.Parameters) > 0 {
|
|
||||||
for _, param := range path.Get.Parameters {
|
|
||||||
curParam := WorkflowAppActionParameter{
|
|
||||||
Name: param.Value.Name,
|
|
||||||
Description: param.Value.Description,
|
|
||||||
Multiline: false,
|
|
||||||
Required: param.Value.Required,
|
|
||||||
Schema: SchemaDefinition{
|
|
||||||
Type: param.Value.Schema.Value.Type,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
if param.Value.Required {
|
|
||||||
action.Parameters = append(action.Parameters, curParam)
|
|
||||||
} else {
|
|
||||||
optionalParameters = append(optionalParameters, curParam)
|
|
||||||
}
|
|
||||||
|
|
||||||
if param.Value.In == "path" {
|
|
||||||
log.Printf("PATH!: %s", param.Value.Name)
|
|
||||||
parameters = append(parameters, param.Value.Name)
|
|
||||||
//baseUrl = fmt.Sprintf("%s%s", baseUrl)
|
|
||||||
} else if param.Value.In == "query" {
|
|
||||||
log.Printf("QUERY!: %s", param.Value.Name)
|
|
||||||
if !param.Value.Required {
|
|
||||||
optionalQueries = append(optionalQueries, param.Value.Name)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
parameters = append(parameters, param.Value.Name)
|
|
||||||
|
|
||||||
if firstQuery {
|
|
||||||
baseUrl = fmt.Sprintf("%s?%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
|
||||||
firstQuery = false
|
|
||||||
} else {
|
|
||||||
baseUrl = fmt.Sprintf("%s&%s={%s}", baseUrl, param.Value.Name, param.Value.Name)
|
|
||||||
firstQuery = false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// ensuring that they end up last in the specification
|
|
||||||
// (order is ish important for optional params) - they need to be last.
|
|
||||||
for _, optionalParam := range optionalParameters {
|
|
||||||
action.Parameters = append(action.Parameters, optionalParam)
|
|
||||||
}
|
|
||||||
|
|
||||||
curCode := makePythoncode(functionName, baseUrl, "get", parameters, optionalQueries)
|
|
||||||
pythonFunctions = append(pythonFunctions, curCode)
|
|
||||||
|
|
||||||
api.Actions = append(api.Actions, action)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return api, pythonFunctions, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func verifyApi(api WorkflowApp) WorkflowApp {
|
|
||||||
if api.AppVersion == "" {
|
|
||||||
api.AppVersion = "1.0.0"
|
|
||||||
}
|
|
||||||
|
|
||||||
return api
|
|
||||||
}
|
|
||||||
|
|
||||||
func dumpPythonGCP(client *storage.Client, basePath, name, version string, pythonFunctions []string) error {
|
|
||||||
//log.Printf("%#v", api)
|
|
||||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
|
||||||
|
|
||||||
parsedCode := fmt.Sprintf(`import requests
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
|
|
||||||
from walkoff_app_sdk.app_base import AppBase
|
|
||||||
|
|
||||||
class %s(AppBase):
|
|
||||||
"""
|
|
||||||
Autogenerated class by Shuffler
|
|
||||||
"""
|
|
||||||
|
|
||||||
__version__ = "%s"
|
|
||||||
app_name = "%s"
|
|
||||||
|
|
||||||
def __init__(self, redis, logger, console_logger=None):
|
|
||||||
self.verify = False
|
|
||||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
||||||
super().__init__(redis, logger, console_logger)
|
|
||||||
|
|
||||||
%s
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(CarbonBlack.run(), debug=True)
|
|
||||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
|
||||||
|
|
||||||
// Create bucket handle
|
|
||||||
ctx := context.Background()
|
|
||||||
bucket := client.Bucket(bucketName)
|
|
||||||
obj := bucket.Object(fmt.Sprintf("%s/src/app.py", basePath))
|
|
||||||
w := obj.NewWriter(ctx)
|
|
||||||
if _, err := fmt.Fprintf(w, parsedCode); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Close, just like writing a file.
|
|
||||||
if err := w.Close(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func dumpPython(basePath, name, version string, pythonFunctions []string) error {
|
|
||||||
//log.Printf("%#v", api)
|
|
||||||
log.Printf(strings.Join(pythonFunctions, "\n"))
|
|
||||||
|
|
||||||
parsedCode := fmt.Sprintf(`import requests
|
|
||||||
import asyncio
|
|
||||||
import json
|
|
||||||
|
|
||||||
from walkoff_app_sdk.app_base import AppBase
|
|
||||||
|
|
||||||
class %s(AppBase):
|
|
||||||
"""
|
|
||||||
Autogenerated class by Shuffler
|
|
||||||
"""
|
|
||||||
|
|
||||||
__version__ = "%s"
|
|
||||||
app_name = "%s"
|
|
||||||
|
|
||||||
def __init__(self, redis, logger, console_logger=None):
|
|
||||||
self.verify = False
|
|
||||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
||||||
super().__init__(redis, logger, console_logger)
|
|
||||||
|
|
||||||
%s
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
asyncio.run(CarbonBlack.run(), debug=True)
|
|
||||||
`, name, version, name, strings.Join(pythonFunctions, "\n"))
|
|
||||||
|
|
||||||
err := ioutil.WriteFile(fmt.Sprintf("%s/src/app.py", basePath), []byte(parsedCode), os.ModePerm)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
fmt.Println(parsedCode)
|
|
||||||
|
|
||||||
//log.Println(string(data))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func dumpApiGCP(client *storage.Client, basePath string, api WorkflowApp) error {
|
|
||||||
//log.Printf("%#v", api)
|
|
||||||
data, err := yaml.Marshal(api)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error with yaml marshal: %s", err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Create bucket handle
|
|
||||||
ctx := context.Background()
|
|
||||||
bucket := client.Bucket(bucketName)
|
|
||||||
obj := bucket.Object(fmt.Sprintf("%s/app.yaml", basePath))
|
|
||||||
w := obj.NewWriter(ctx)
|
|
||||||
if _, err := fmt.Fprintf(w, string(data)); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
// Close, just like writing a file.
|
|
||||||
if err := w.Close(); err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
//log.Println(string(data))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func dumpApi(basePath string, api WorkflowApp) error {
|
|
||||||
//log.Printf("%#v", api)
|
|
||||||
data, err := yaml.Marshal(api)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error with yaml marshal: %s", err)
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
err = ioutil.WriteFile(fmt.Sprintf("%s/api.yaml", basePath), []byte(data), os.ModePerm)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
//log.Println(string(data))
|
|
||||||
return nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
data := []byte(`{"swagger":"3.0","info":{"title":"hi","description":"you","version":"1.0"},"servers":[{"url":"https://shuffler.io/api/v1"}],"host":"shuffler.io","basePath":"/api/v1","schemes":["https:"],"paths":{"/workflows":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflows","description":"Get workflows","parameters":[]}},"/workflows/{id}":{"get":{"responses":{"default":{"description":"default","schema":{}}},"summary":"Get workflow","description":"Get workflow","parameters":[{"in":"query","name":"forgetme","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"query","name":"anotherone","description":"Generated by shuffler.io OpenAPI","required":false,"schema":{"type":"string"}},{"in":"query","name":"hi","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}},{"in":"path","name":"id","description":"Generated by shuffler.io OpenAPI","required":true,"schema":{"type":"string"}}]}}},"securityDefinitions":{}}`)
|
|
||||||
|
|
||||||
ctx := context.Background()
|
|
||||||
client, err := storage.NewClient(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to create client: %v", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
hasher := md5.New()
|
|
||||||
hasher.Write(data)
|
|
||||||
newmd5 := hex.EncodeToString(hasher.Sum(nil))
|
|
||||||
swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(data)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Swagger validation error: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
if strings.Contains(swagger.Info.Title, " ") {
|
|
||||||
strings.ReplaceAll(swagger.Info.Title, " ", "")
|
|
||||||
}
|
|
||||||
|
|
||||||
basePath, err := buildStructureGCP(client, swagger, newmd5)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed to build base structure: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
api, pythonfunctions, err := generateYaml(swagger)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed building and generating yaml: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = dumpApiGCP(client, basePath, api)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed dumping yaml: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
|
|
||||||
err = dumpPythonGCP(client, basePath, swagger.Info.Title, swagger.Info.Version, pythonfunctions)
|
|
||||||
if err != nil {
|
|
||||||
log.Printf("Failed dumping python: %s", err)
|
|
||||||
os.Exit(3)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
cortexutils
|
|
||||||
requests
|
|
||||||
@@ -1,43 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "Shuffle",
|
|
||||||
"version": "1.0",
|
|
||||||
"author": "@frikkylikeme",
|
|
||||||
"url": "https://github.com/frikky/shuffle",
|
|
||||||
"license": "AGPL-V3",
|
|
||||||
"description": "Execute a workflow in Shuffle",
|
|
||||||
"dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"],
|
|
||||||
"command": "Shuffle/shuffle.py",
|
|
||||||
"baseConfig": "Shuffle",
|
|
||||||
"configurationItems": [
|
|
||||||
{
|
|
||||||
"name": "url",
|
|
||||||
"description": "The URL to your shuffle instance",
|
|
||||||
"type": "string",
|
|
||||||
"multi": false,
|
|
||||||
"required": true,
|
|
||||||
"defaultValue": "https://shuffler.io"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "api_key",
|
|
||||||
"description": "The API key to your Shuffle user",
|
|
||||||
"type": "string",
|
|
||||||
"multi": false,
|
|
||||||
"required": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "verifyssl",
|
|
||||||
"description": "Verify SSL certificate",
|
|
||||||
"type": "boolean",
|
|
||||||
"multi": false,
|
|
||||||
"required": true,
|
|
||||||
"defaultValue": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "workflow_id",
|
|
||||||
"description": "The ID of the workflow to execute",
|
|
||||||
"type": "string",
|
|
||||||
"multi": false,
|
|
||||||
"required": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,27 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
#encoding: utf-8
|
|
||||||
|
|
||||||
from cortexutils.responder import Responder
|
|
||||||
import requests
|
|
||||||
|
|
||||||
class Shuffle(Responder):
|
|
||||||
def __init__(self):
|
|
||||||
Responder.__init__(self)
|
|
||||||
self.api_key = self.get_param("config.api_key", "")
|
|
||||||
self.url = self.get_param("config.url", "")
|
|
||||||
self.workflow_id = self.get_param("config.workflow_id", "")
|
|
||||||
self.verify = self.get_param('config.verifyssl', True, None)
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
Responder.run(self)
|
|
||||||
parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id)
|
|
||||||
headers = {
|
|
||||||
"Authorization": "Bearer %s" % self.api_key,
|
|
||||||
"User-Agent": "Cortex-Analyzer"
|
|
||||||
}
|
|
||||||
requests.post(parsed_url, headers=headers,verify=self.verify)
|
|
||||||
|
|
||||||
self.report({'message': 'message sent'})
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
Shuffle().run()
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
cortexutils
|
|
||||||
requests
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
{
|
|
||||||
"name": "Shuffle_webhook",
|
|
||||||
"version": "1.0",
|
|
||||||
"author": "@azgaviperr",
|
|
||||||
"url": "https://github.com/frikky/shuffle",
|
|
||||||
"license": "AGPL-V3",
|
|
||||||
"description": "Execute a webhook in Shuffle",
|
|
||||||
"dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"],
|
|
||||||
"command": "Shuffle_Webhook/shuffle_webhook.py",
|
|
||||||
"baseConfig": "Shuffle_Webhook",
|
|
||||||
"configurationItems": [
|
|
||||||
{
|
|
||||||
"name": "webhook_url",
|
|
||||||
"description": "The URL to your shuffle instance",
|
|
||||||
"type": "string",
|
|
||||||
"multi": false,
|
|
||||||
"required": true
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"name": "verifyssl",
|
|
||||||
"description": "Verify SSL certificate",
|
|
||||||
"type": "boolean",
|
|
||||||
"multi": false,
|
|
||||||
"required": true,
|
|
||||||
"defaultValue": true
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
@@ -1,28 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
#encoding: utf-8
|
|
||||||
|
|
||||||
from cortexutils.responder import Responder
|
|
||||||
import requests
|
|
||||||
|
|
||||||
class Shuffle(Responder):
|
|
||||||
def __init__(self):
|
|
||||||
Responder.__init__(self)
|
|
||||||
self.api_key = self.get_param("config.api_key", "")
|
|
||||||
self.webhook_url = self.get_param("config.webhook_url", "")
|
|
||||||
self.webhook_id = self.get_param("config.webhook_id", "")
|
|
||||||
self.verify = self.get_param('config.verifyssl', True, None)
|
|
||||||
self.data = self.get_param('data')
|
|
||||||
|
|
||||||
def run(self):
|
|
||||||
Responder.run(self)
|
|
||||||
headers = {
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
"Accept": "application/json",
|
|
||||||
"User-Agent": "Cortex-Analyzer"
|
|
||||||
}
|
|
||||||
requests.post(self.webhook_url, headers=headers,verify=self.verify, json=self.data)
|
|
||||||
|
|
||||||
self.report({'message': 'message sent'})
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
Shuffle().run()
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
FROM python:3.9.4-alpine as base
|
|
||||||
|
|
||||||
FROM base as builder
|
|
||||||
|
|
||||||
RUN mkdir /install
|
|
||||||
WORKDIR /install
|
|
||||||
|
|
||||||
FROM base
|
|
||||||
RUN apk add g++
|
|
||||||
|
|
||||||
COPY --from=builder /install /usr/local
|
|
||||||
COPY requirements.txt /requirements.txt
|
|
||||||
RUN pip3 install -r /requirements.txt
|
|
||||||
|
|
||||||
|
|
||||||
RUN mkdir /app
|
|
||||||
WORKDIR /app
|
|
||||||
COPY requirements.txt /app/requirements.txt
|
|
||||||
RUN python3 -m pip install -r /app/requirements.txt
|
|
||||||
|
|
||||||
COPY sub.py /app/sub.py
|
|
||||||
|
|
||||||
CMD ["python3", "sub.py"]
|
|
||||||
@@ -1,9 +0,0 @@
|
|||||||
version: '3'
|
|
||||||
services:
|
|
||||||
zmq:
|
|
||||||
image: ghcr.io/frikky/shuffle-zmq:latest
|
|
||||||
environment:
|
|
||||||
- ZMQ_HOSTNAME=localhost
|
|
||||||
- ZMQ_PORT=50000
|
|
||||||
- ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e
|
|
||||||
restart: unless-stopped
|
|
||||||
@@ -1,2 +0,0 @@
|
|||||||
pyzmq
|
|
||||||
requests
|
|
||||||
@@ -1,57 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
print("Running imports")
|
|
||||||
import sys
|
|
||||||
import zmq
|
|
||||||
import json
|
|
||||||
import time
|
|
||||||
import pprint
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import requests
|
|
||||||
|
|
||||||
forward_url = os.getenv("ZMQ_FORWARD_URL", "")
|
|
||||||
print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url)
|
|
||||||
def handle_hook(data):
|
|
||||||
ret = requests.post(forward_url, json=data)
|
|
||||||
print(ret.text)
|
|
||||||
print(ret.status_code)
|
|
||||||
|
|
||||||
def main():
|
|
||||||
host = os.getenv("ZMQ_HOST", "localhost")
|
|
||||||
port = os.getenv("ZMQ_PORT", "50000")
|
|
||||||
|
|
||||||
if len(forward_url) == 0:
|
|
||||||
print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder")
|
|
||||||
exit(0)
|
|
||||||
|
|
||||||
print("Starting connection setup to %s:%s" % (host, port))
|
|
||||||
context = zmq.Context()
|
|
||||||
socket = context.socket(zmq.SUB)
|
|
||||||
socket.connect ("tcp://%s:%s" % (host, port))
|
|
||||||
socket.setsockopt(zmq.SUBSCRIBE, b'')
|
|
||||||
|
|
||||||
poller = zmq.Poller()
|
|
||||||
poller.register(socket, zmq.POLLIN)
|
|
||||||
|
|
||||||
print("Starting zmq check for %s:%s" % (host, port))
|
|
||||||
while True:
|
|
||||||
socks = dict(poller.poll(timeout=None))
|
|
||||||
if socket in socks and socks[socket] == zmq.POLLIN:
|
|
||||||
message = socket.recv()
|
|
||||||
#print(message)
|
|
||||||
topic, s, m = message.decode('utf-8').partition(" ")
|
|
||||||
|
|
||||||
d = json.loads(m)
|
|
||||||
try:
|
|
||||||
# print test if you want status (heartbeat)
|
|
||||||
test = d["status"]
|
|
||||||
except KeyError:
|
|
||||||
handle_hook(d)
|
|
||||||
|
|
||||||
time.sleep(1)
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
print("In init ")
|
|
||||||
main()
|
|
||||||
@@ -1,36 +0,0 @@
|
|||||||
FROM golang:1.22 as builder
|
|
||||||
|
|
||||||
RUN mkdir /app
|
|
||||||
WORKDIR /app
|
|
||||||
|
|
||||||
COPY orborus.go /app/orborus.go
|
|
||||||
#COPY go.mod /app/go.mod
|
|
||||||
#COPY go.sum /app/go.sum
|
|
||||||
RUN go mod init orborus
|
|
||||||
|
|
||||||
RUN go get github.com/docker/docker/api/types
|
|
||||||
RUN go get github.com/docker/docker/api/types/container
|
|
||||||
#RUN go get github.com/docker/docker/client
|
|
||||||
RUN go get github.com/mackerelio/go-osstat/cpu
|
|
||||||
RUN go get github.com/mackerelio/go-osstat/memory
|
|
||||||
RUN go get github.com/satori/go.uuid
|
|
||||||
RUN go get github.com/shuffle/shuffle-shared
|
|
||||||
RUN go get github.com/shirou/gopsutil
|
|
||||||
# RUN go get k8s.io/client-go
|
|
||||||
# RUN go get k8s.io/apimachinery
|
|
||||||
|
|
||||||
RUN go get
|
|
||||||
RUN go mod tidy
|
|
||||||
RUN go build
|
|
||||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
|
|
||||||
|
|
||||||
FROM alpine:3.15.0
|
|
||||||
RUN apk add --no-cache bash tzdata
|
|
||||||
COPY --from=builder /app/ /
|
|
||||||
|
|
||||||
ENV ENVIRONMENT_NAME=Shuffle
|
|
||||||
ENV BASE_URL=http://shuffle-backend:5001
|
|
||||||
ENV DOCKER_API_VERSION=1.40
|
|
||||||
ENV SHUFFLE_OPENSEARCH_URL=https://opensearch:9200
|
|
||||||
|
|
||||||
CMD ["./orborus"]
|
|
||||||
Reference in New Issue
Block a user