Bumped docker versions
This commit is contained in:
frikky
+42
-15
@@ -9,7 +9,6 @@ import requests
|
||||
import urllib.parse
|
||||
|
||||
class AppBase:
|
||||
""" The base class for Python-based apps in Shuffle, handles logging and callbacks configurations"""
|
||||
__version__ = None
|
||||
app_name = None
|
||||
|
||||
@@ -21,7 +20,7 @@ class AppBase:
|
||||
# apikey is for the user / org
|
||||
# authorization is for the specific workflow
|
||||
self.url = os.getenv("CALLBACK_URL", "https://shuffler.io")
|
||||
self.base_url = os.getenv("BASE_URL", "")
|
||||
self.base_url = os.getenv("BASE_URL", "https://shuffler.io")
|
||||
self.action = os.getenv("ACTION", "")
|
||||
self.authorization = os.getenv("AUTHORIZATION", "")
|
||||
self.current_execution_id = os.getenv("EXECUTIONID", "")
|
||||
@@ -29,7 +28,10 @@ class AppBase:
|
||||
self.result_wrapper_count = 0
|
||||
|
||||
if isinstance(self.action, str):
|
||||
self.action = json.loads(self.action)
|
||||
try:
|
||||
self.action = json.loads(self.action)
|
||||
except:
|
||||
print("[WARNING] Failed parsing action as JSON")
|
||||
|
||||
if len(self.base_url) == 0:
|
||||
self.base_url = self.url
|
||||
@@ -530,9 +532,19 @@ class AppBase:
|
||||
"status": "EXECUTING"
|
||||
}
|
||||
|
||||
try:
|
||||
print("Parameters: %d" % len(action["parameters"]))
|
||||
except KeyError:
|
||||
action["parameters"] = []
|
||||
|
||||
self.action = copy.deepcopy(action)
|
||||
self.logger.info("ACTION RESULT (start): %s", action_result)
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer %s" % self.authorization
|
||||
}
|
||||
|
||||
if len(self.action) == 0:
|
||||
print("ACTION env not defined")
|
||||
action_result["result"] = "Error in setup ENV: ACTION not defined"
|
||||
@@ -551,10 +563,6 @@ class AppBase:
|
||||
self.send_result(action_result, headers, stream_path)
|
||||
return
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer %s" % self.authorization
|
||||
}
|
||||
|
||||
# Add async logger
|
||||
# self.console_logger.handlers[0].stream.set_execution_id()
|
||||
@@ -1363,6 +1371,8 @@ class AppBase:
|
||||
actionname = action["name"]
|
||||
if " " in actionname:
|
||||
actionname.replace(" ", "_", -1)
|
||||
|
||||
|
||||
#if action.generated:
|
||||
# actionname = actionname.lower()
|
||||
|
||||
@@ -1919,21 +1929,38 @@ class AppBase:
|
||||
self.send_result(action_result, headers, stream_path)
|
||||
return
|
||||
|
||||
|
||||
#STOPCOPY
|
||||
# !!! Let the above line stay - its used for some horrible codegeneration / stitching !!! #
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
""" Connect to Redis and HTTP session, await actions """
|
||||
async def run(cls, action=""):
|
||||
logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{')
|
||||
logger = logging.getLogger(f"{cls.__name__}")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
print("Started execution!!")
|
||||
|
||||
app = cls(redis=None, logger=logger, console_logger=logger)
|
||||
print("Started execution: %s!!" % cls)
|
||||
print("Action: %s" % action)
|
||||
#if isinstance(cls, object):
|
||||
# self.action = cls
|
||||
|
||||
# Authorization for the app/function to control the workflow
|
||||
# Function will crash if its wrong, which it probably should.
|
||||
app = cls(redis=None, logger=logger, console_logger=logger)
|
||||
|
||||
if isinstance(action, object):
|
||||
app.action = action
|
||||
|
||||
try:
|
||||
app.authorization = action["authorization"]
|
||||
app.current_execution_id = action["execution_id"]
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
app.url = action["url"]
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
app.base_url = action["base_url"]
|
||||
except:
|
||||
pass
|
||||
|
||||
await app.execute_action(app.action)
|
||||
|
||||
@@ -2025,11 +2025,10 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
|
||||
return
|
||||
}
|
||||
|
||||
err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId)
|
||||
if err != nil {
|
||||
log.Printf("Failed to increase total workflows: %s", err)
|
||||
}
|
||||
|
||||
//err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId)
|
||||
//if err != nil {
|
||||
// log.Printf("Failed to increase total workflows: %s", err)
|
||||
//}
|
||||
//memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId)
|
||||
//memcache.Delete(ctx, memcacheName)
|
||||
//memcacheName = fmt.Sprintf("%s_workflows", user.Username)
|
||||
|
||||
+3
-3
@@ -2,7 +2,7 @@ version: '3'
|
||||
services:
|
||||
frontend:
|
||||
#build: ./frontend
|
||||
image: ghcr.io/frikky/shuffle-frontend:0.8.56
|
||||
image: ghcr.io/frikky/shuffle-frontend:0.8.57
|
||||
container_name: shuffle-frontend
|
||||
hostname: shuffle-frontend
|
||||
ports:
|
||||
@@ -45,7 +45,7 @@ services:
|
||||
- database
|
||||
orborus:
|
||||
#build: ./functions/onprem/orborus
|
||||
image: ghcr.io/frikky/shuffle-orborus:0.8.5
|
||||
image: ghcr.io/frikky/shuffle-orborus:0.8.56
|
||||
container_name: shuffle-orborus
|
||||
hostname: shuffle-orborus
|
||||
networks:
|
||||
@@ -54,7 +54,7 @@ services:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
environment:
|
||||
- SHUFFLE_APP_SDK_VERSION=0.8.51
|
||||
- SHUFFLE_WORKER_VERSION=0.8.54
|
||||
- SHUFFLE_WORKER_VERSION=0.8.56
|
||||
- ORG_ID=${ORG_ID}
|
||||
- ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
|
||||
- BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}
|
||||
|
||||
@@ -31,6 +31,7 @@ import { useTheme } from '@material-ui/core/styles';
|
||||
import HandlePayment from './HandlePayment'
|
||||
import OrgHeader from '../components/OrgHeader'
|
||||
|
||||
import CircularProgress from '@material-ui/core/CircularProgress';
|
||||
import EditIcon from '@material-ui/icons/Edit';
|
||||
import SelectAllIcon from '@material-ui/icons/SelectAll';
|
||||
import OpenInNewIcon from '@material-ui/icons/OpenInNew';
|
||||
@@ -1374,12 +1375,24 @@ const Admin = (props) => {
|
||||
</span>
|
||||
</div>
|
||||
{selectedOrganization.id === undefined ?
|
||||
<div style={{height: 250}}/>
|
||||
:
|
||||
<div style={{paddingTop: 250, width: 250, margin: "auto", textAlign: "center"}}>
|
||||
<CircularProgress />
|
||||
<Typography>
|
||||
Loading Organization
|
||||
</Typography>
|
||||
</div>
|
||||
:
|
||||
<div>
|
||||
{selectedOrganization.name.length > 0 ?
|
||||
<OrgHeader setSelectedOrganization={setSelectedOrganization} globalUrl={globalUrl} selectedOrganization={selectedOrganization}/>
|
||||
: null}
|
||||
:
|
||||
<div style={{paddingTop: 250, width: 250, margin: "auto", textAlign: "center"}}>
|
||||
<CircularProgress />
|
||||
<Typography>
|
||||
Loading Organization
|
||||
</Typography>
|
||||
</div>
|
||||
}
|
||||
<Divider style={{ marginTop: 20, marginBottom: 20, backgroundColor: theme.palette.inputColor }} />
|
||||
<Typography variant="h6" style={{marginBottom: "10px", color: "white"}}>Cloud syncronization</Typography>
|
||||
What does <a href="https://shuffler.io/docs/organizations#cloud_sync" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>cloud sync</a> do? Cloud syncronization is a way of getting more out of Shuffle. Shuffle will <b>ALWAYS</b> make every option open source, but features relying on other users can't be done without a collaborative approach.
|
||||
@@ -1690,7 +1703,7 @@ const Admin = (props) => {
|
||||
style={{ minWidth: 180, maxWidth: 180 }}
|
||||
/>
|
||||
</ListItem>
|
||||
{users === undefined ? null : users.map((data, index) => {
|
||||
{users === undefined || users === null ? null : users.map((data, index) => {
|
||||
var bgColor = "#27292d"
|
||||
if (index % 2 === 0) {
|
||||
bgColor = "#1f2023"
|
||||
@@ -2074,7 +2087,7 @@ const Admin = (props) => {
|
||||
primary="Actions"
|
||||
/>
|
||||
</ListItem>
|
||||
{authentication === undefined ? null : authentication.map((data, index) => {
|
||||
{authentication === undefined || authentication === null ? null : authentication.map((data, index) => {
|
||||
var bgColor = "#27292d"
|
||||
if (index % 2 === 0) {
|
||||
bgColor = "#1f2023"
|
||||
@@ -2392,7 +2405,7 @@ const Admin = (props) => {
|
||||
|
||||
const iconStyle = {marginRight: 10}
|
||||
const data =
|
||||
<div style={{width: 1366, margin: "auto", overflowX: "hidden",}}>
|
||||
<div style={{width: 1366, margin: "auto", overflowX: "hidden", marginTop: 25,}}>
|
||||
<Paper style={paperStyle}>
|
||||
<Tabs
|
||||
value={curTab}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
NAME=shuffle-orborus
|
||||
VERSION=0.8.54
|
||||
VERSION=0.8.56
|
||||
|
||||
echo "Running docker build with $NAME:$VERSION"
|
||||
#docker rmi frikky/shuffle:$NAME --force
|
||||
|
||||
@@ -1634,7 +1634,6 @@ func executionInit(workflowExecution WorkflowExecution) error {
|
||||
}
|
||||
|
||||
nextActions = append(nextActions, startAction)
|
||||
|
||||
for _, branch := range workflowExecution.Workflow.Branches {
|
||||
// Check what the parent is first. If it's trigger - skip
|
||||
sourceFound := false
|
||||
|
||||
@@ -1,703 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"archive/tar"
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/client"
|
||||
"google.golang.org/api/cloudfunctions/v1"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var gceProject = "shuffler"
|
||||
var bucketName = "shuffler.appspot.com"
|
||||
|
||||
type WorkflowAppActionParameter struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Example string `json:"example" datastore:"example"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline"`
|
||||
ActionField string `json:"action_field" datastore:"action_field"`
|
||||
Variant string `json:"variant", datastore:"variant"`
|
||||
Required bool `json:"required" datastore:"required"`
|
||||
Schema struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
} `json:"schema"`
|
||||
}
|
||||
|
||||
type Authentication struct {
|
||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
||||
}
|
||||
|
||||
type AuthenticationParams struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
||||
Value string `json:"value" datastore:"value" yaml:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
}
|
||||
|
||||
type WorkflowApp struct {
|
||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
||||
ID string `json:"id" yaml:"id" required:false datastore:"id"`
|
||||
Link string `json:"link" yaml:"link" required:false datastore:"link"`
|
||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
Sharing bool `json:"sharing" datastore:"sharing" yaml:"sharing"`
|
||||
SmallImage string `json:"small_image" datastore:"small_image" required:false yaml:"small_image"`
|
||||
LargeImage string `json:"large_image" datastore:"large_image" yaml:"large_image" requred:false`
|
||||
ContactInfo struct {
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
||||
}
|
||||
|
||||
type AuthenticationStore struct {
|
||||
Key string `json:"key" datastore:"key"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
}
|
||||
|
||||
type WorkflowAppAction struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
NodeType string `json:"node_type" datastore:"node_type"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication"`
|
||||
Returns struct {
|
||||
Description string `json:"description" datastore:"returns"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Schema struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
} `json:"schema" datastore:"schema"`
|
||||
} `json:"returns" datastore:"returns"`
|
||||
}
|
||||
|
||||
func getRunner(classname string) string {
|
||||
return fmt.Sprintf(`
|
||||
# Run the actual thing after we've checked params
|
||||
def run(request):
|
||||
action = request.get_json()
|
||||
print(action)
|
||||
print(type(action))
|
||||
authorization_key = action.get("authorization")
|
||||
current_execution_id = action.get("execution_id")
|
||||
|
||||
if action and "name" in action and "app_name" in action:
|
||||
asyncio.run(%s.run(action), debug=True)
|
||||
return f'Attempting to execute function {action["name"]} in app {action["app_name"]}'
|
||||
else:
|
||||
return f'Invalid action'
|
||||
|
||||
`, classname)
|
||||
}
|
||||
|
||||
// Could use some kind of linting system too for this, but meh
|
||||
func formatAppfile(filedata []byte) (string, []byte) {
|
||||
lines := strings.Split(string(filedata), "\n")
|
||||
|
||||
newfile := []string{}
|
||||
classname := ""
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "walkoff_app_sdk") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Remap logging. CBA this right now
|
||||
// This issue also persists in onprem apps because of await thingies.. :(
|
||||
// FIXME
|
||||
if strings.Contains(line, "console_logger") && strings.Contains(line, "await") {
|
||||
continue
|
||||
//line = strings.Replace(line, "console_logger", "logger", -1)
|
||||
//log.Println(line)
|
||||
}
|
||||
|
||||
// Might not work with different import names
|
||||
// Could be fucked up with spaces everywhere? Idk
|
||||
if strings.Contains(line, "class") && strings.Contains(line, "(AppBase)") {
|
||||
items := strings.Split(line, " ")
|
||||
if len(items) > 0 && strings.Contains(items[1], "(AppBase)") {
|
||||
classname = strings.Split(items[1], "(")[0]
|
||||
} else {
|
||||
log.Println("Something wrong :( (horrible programming right here)")
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(line, "if __name__ ==") {
|
||||
break
|
||||
}
|
||||
|
||||
// asyncio.run(HelloWorld.run(), debug=True)
|
||||
|
||||
newfile = append(newfile, line)
|
||||
}
|
||||
|
||||
filedata = []byte(strings.Join(newfile, "\n"))
|
||||
return classname, filedata
|
||||
}
|
||||
|
||||
// https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
|
||||
func Copy(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
func ZipFiles(filename string, files []string) error {
|
||||
newZipFile, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer newZipFile.Close()
|
||||
|
||||
zipWriter := zip.NewWriter(newZipFile)
|
||||
defer zipWriter.Close()
|
||||
|
||||
// Add files to zip
|
||||
for _, file := range files {
|
||||
zipfile, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zipfile.Close()
|
||||
|
||||
// Get the file information
|
||||
info, err := zipfile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Using FileInfoHeader() above only uses the basename of the file. If we want
|
||||
// to preserve the folder structure we can overwrite this with the full path.
|
||||
filesplit := strings.Split(file, "/")
|
||||
if len(filesplit) > 1 {
|
||||
header.Name = filesplit[len(filesplit)-1]
|
||||
} else {
|
||||
header.Name = file
|
||||
}
|
||||
|
||||
// Change to deflate to gain better compression
|
||||
// see http://golang.org/pkg/archive/zip/#pkg-constants
|
||||
header.Method = zip.Deflate
|
||||
|
||||
writer, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = io.Copy(writer, zipfile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getAppbase(filepath string) []string {
|
||||
appBase, err := ioutil.ReadFile(filepath)
|
||||
if err != nil {
|
||||
log.Printf("Readerror: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
record := false
|
||||
validLines := []string{}
|
||||
for _, line := range strings.Split(string(appBase), "\n") {
|
||||
if strings.Contains(line, "#STOPCOPY") {
|
||||
log.Println("Stopping copy")
|
||||
break
|
||||
}
|
||||
|
||||
if record {
|
||||
validLines = append(validLines, line)
|
||||
}
|
||||
|
||||
if strings.Contains(line, "#STARTCOPY") {
|
||||
log.Println("Starting copy")
|
||||
record = true
|
||||
}
|
||||
}
|
||||
|
||||
return validLines
|
||||
}
|
||||
|
||||
// Puts together ./static_baseline.py, onprem/app_sdk_app_base.py and the
|
||||
// appcode in a generated_app folder based on appname+version
|
||||
func stitcher(appname string, appversion string) string {
|
||||
baselinefile := "static_baseline.py"
|
||||
appfolder := "apps"
|
||||
appbasefile := "onprem/app_sdk/app_base.py"
|
||||
|
||||
baseline, err := ioutil.ReadFile(baselinefile)
|
||||
if err != nil {
|
||||
log.Printf("Readerror: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sourceappfile := fmt.Sprintf("%s/%s/%s/src/app.py", appfolder, appname, appversion)
|
||||
appfile, err := ioutil.ReadFile(sourceappfile)
|
||||
if err != nil {
|
||||
log.Printf("App readerror: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
classname, appfile := formatAppfile(appfile)
|
||||
if len(classname) == 0 {
|
||||
log.Println("Failed finding classname in file.")
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
runner := getRunner(classname)
|
||||
appBase := getAppbase(appbasefile)
|
||||
|
||||
foldername := fmt.Sprintf("generated_apps/%s_%s", appname, appversion)
|
||||
err = os.Mkdir(foldername, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Println("Failed making temporary app folder. Probably already exists. Remaking")
|
||||
os.RemoveAll(foldername)
|
||||
os.MkdirAll(foldername, os.ModePerm)
|
||||
}
|
||||
|
||||
stitched := []byte(string(baseline) + strings.Join(appBase, "\n") + string(appfile) + string(runner))
|
||||
err = ioutil.WriteFile(fmt.Sprintf("%s/main.py", foldername), stitched, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Println("Failed writing to stitched: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = Copy(fmt.Sprintf("%s/%s/%s/requirements.txt", appfolder, appname, appversion), fmt.Sprintf("%s/requirements.txt", foldername))
|
||||
if err != nil {
|
||||
log.Println("Failed writing to requirement: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
log.Printf("Successfully stitched files in %s/main.py", foldername)
|
||||
// Zip the folder
|
||||
files := []string{
|
||||
fmt.Sprintf("%s/main.py", foldername),
|
||||
fmt.Sprintf("%s/requirements.txt", foldername),
|
||||
}
|
||||
outputfile := fmt.Sprintf("%s.zip", foldername)
|
||||
|
||||
err = ZipFiles(outputfile, files)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Creates a client.
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create client: %v", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
// Create bucket handle
|
||||
bucket := client.Bucket(bucketName)
|
||||
|
||||
remotePath := fmt.Sprintf("apps/%s_%s.zip", appname, appversion)
|
||||
err = createFileFromFile(bucket, remotePath, outputfile)
|
||||
if err != nil {
|
||||
log.Printf("Failed to upload to bucket: %v", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
os.Remove(outputfile)
|
||||
return fmt.Sprintf("gs://%s/apps/%s_%s.zip", bucketName, appname, appversion)
|
||||
}
|
||||
|
||||
func createFileFromFile(bucket *storage.BucketHandle, remotePath, localPath string) error {
|
||||
ctx := context.Background()
|
||||
// [START upload_file]
|
||||
f, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
wc := bucket.Object(remotePath).NewWriter(ctx)
|
||||
if _, err = io.Copy(wc, f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := wc.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
// [END upload_file]
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deploy to google cloud function :)
|
||||
func deployFunction(appname, localization, applocation string, environmentVariables map[string]string) error {
|
||||
ctx := context.Background()
|
||||
service, err := cloudfunctions.NewService(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// ProjectsLocationsListCall
|
||||
projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service)
|
||||
location := fmt.Sprintf("projects/%s/locations/%s", gceProject, localization)
|
||||
functionName := fmt.Sprintf("%s/functions/%s", location, appname)
|
||||
|
||||
cloudFunction := &cloudfunctions.CloudFunction{
|
||||
AvailableMemoryMb: 128,
|
||||
EntryPoint: "authorization",
|
||||
EnvironmentVariables: environmentVariables,
|
||||
HttpsTrigger: &cloudfunctions.HttpsTrigger{},
|
||||
MaxInstances: 0,
|
||||
Name: functionName,
|
||||
Runtime: "python37",
|
||||
SourceArchiveUrl: applocation,
|
||||
}
|
||||
|
||||
//getCall := projectsLocationsFunctionsService.Get(fmt.Sprintf("%s/functions/function-5", location))
|
||||
//resp, err := getCall.Do()
|
||||
|
||||
createCall := projectsLocationsFunctionsService.Create(location, cloudFunction)
|
||||
_, err = createCall.Do()
|
||||
if err != nil {
|
||||
log.Println("Failed creating new function. Attempting patch, as it might exist already")
|
||||
|
||||
createCall := projectsLocationsFunctionsService.Patch(fmt.Sprintf("%s/functions/%s", location, appname), cloudFunction)
|
||||
_, err = createCall.Do()
|
||||
if err != nil {
|
||||
log.Println("Failed patching function")
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Successfully patched %s to %s", appname, localization)
|
||||
} else {
|
||||
log.Printf("Successfully deployed %s to %s", appname, localization)
|
||||
}
|
||||
|
||||
// FIXME - use response to define the HTTPS entrypoint. It's default to an easy one tho
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deployAppCloudFunc(appname string, appversion string) {
|
||||
_ = os.Mkdir("generated_apps", os.ModePerm)
|
||||
|
||||
apikey := "eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
|
||||
fullAppname := fmt.Sprintf("%s-%s", strings.Replace(appname, "_", "-", -1), strings.Replace(appversion, ".", "-", -1))
|
||||
locations := []string{"europe-west2"}
|
||||
|
||||
// Deploys the app to all locations
|
||||
bucketname := stitcher(appname, appversion)
|
||||
environmentVariables := map[string]string{
|
||||
"FUNCTION_APIKEY": apikey,
|
||||
}
|
||||
|
||||
for _, location := range locations {
|
||||
err := deployFunction(fullAppname, location, bucketname, environmentVariables)
|
||||
if err != nil {
|
||||
log.Printf("Failed to deploy: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func loadYaml(fileLocation string) (WorkflowApp, error) {
|
||||
action := WorkflowApp{}
|
||||
|
||||
yamlFile, err := ioutil.ReadFile(fileLocation)
|
||||
if err != nil {
|
||||
log.Printf("yamlFile.Get err: %s", err)
|
||||
return WorkflowApp{}, err
|
||||
}
|
||||
|
||||
//log.Printf(string(yamlFile))
|
||||
err = yaml.Unmarshal([]byte(yamlFile), &action)
|
||||
if err != nil {
|
||||
return WorkflowApp{}, err
|
||||
}
|
||||
|
||||
return action, nil
|
||||
}
|
||||
|
||||
// FIXME - deploy to backend (YAML config)
|
||||
func deployConfigToBackend(appname string, appversion string) error {
|
||||
// FIXME - no static path pls
|
||||
action, err := loadYaml(fmt.Sprintf("apps/%s/%s/api.yaml", appname, appversion))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
action.Sharing = true
|
||||
|
||||
data, err := json.Marshal(action)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
url := "http://localhost:5001/api/v1/workflows/apps"
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ")
|
||||
|
||||
ret, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Status: %s", ret.Status)
|
||||
body, err := ioutil.ReadAll(ret.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ret.StatusCode != 200 {
|
||||
return errors.New(fmt.Sprintf("Status %s. App probably already exists. Raw:\n%s", ret.Status, string(body)))
|
||||
}
|
||||
|
||||
log.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
func tarDirectory(filecontext string) (io.Reader, error) {
|
||||
|
||||
// Create a filereader
|
||||
//dockerFileReader, err := os.Open(dockerfile)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
|
||||
//// Read the actual Dockerfile
|
||||
//readDockerFile, err := ioutil.ReadAll(dockerFileReader)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
|
||||
// Make a TAR header for the file
|
||||
tarHeader := &tar.Header{
|
||||
Name: filecontext,
|
||||
Typeflag: tar.TypeDir,
|
||||
}
|
||||
|
||||
// Writes the header described for the TAR file
|
||||
buf := new(bytes.Buffer)
|
||||
tw := tar.NewWriter(buf)
|
||||
defer tw.Close()
|
||||
err := tw.WriteHeader(tarHeader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
||||
return dockerFileTarReader, nil
|
||||
}
|
||||
|
||||
func tarDir(source string, target string) (*bytes.Reader, error) {
|
||||
filename := filepath.Base(source)
|
||||
target = filepath.Join(target, fmt.Sprintf("%s.tar", filename))
|
||||
tarfile, err := os.Create(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer tarfile.Close()
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
_ = buf
|
||||
tarball := tar.NewWriter(tarfile)
|
||||
defer tarball.Close()
|
||||
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var baseDir string
|
||||
if info.IsDir() {
|
||||
baseDir = filepath.Base(source)
|
||||
}
|
||||
|
||||
_ = filepath.Walk(source,
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header, err := tar.FileInfoHeader(info, info.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if baseDir != "" {
|
||||
header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
|
||||
}
|
||||
|
||||
if err := tarball.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(tarball, file)
|
||||
return nil
|
||||
})
|
||||
|
||||
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
||||
return dockerFileTarReader, nil
|
||||
}
|
||||
|
||||
func buildImage(client *client.Client, tags []string, dockerBuildCtxDir string) error {
|
||||
dockerBuildContext, err := tarDir(dockerBuildCtxDir, ".")
|
||||
if err != nil {
|
||||
log.Printf("Error in taring the docker root folder - %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
imageBuildResponse, err := client.ImageBuild(
|
||||
context.Background(),
|
||||
dockerBuildContext,
|
||||
types.ImageBuildOptions{
|
||||
Dockerfile: "Dockerfile",
|
||||
PullParent: true,
|
||||
Remove: true,
|
||||
Tags: tags,
|
||||
NetworkMode: "host",
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read the STDOUT from the build process
|
||||
defer imageBuildResponse.Body.Close()
|
||||
_, err = io.Copy(os.Stdout, imageBuildResponse.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FIXME - deploy to dockerhub
|
||||
func deployWorker(appname, appversion string) error {
|
||||
// Get dockerfile from ./apps/appname/appversion/Dockerfile
|
||||
client, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tags := []string{fmt.Sprintf("%s-%s", appname, appversion)}
|
||||
err = buildImage(client, tags, fmt.Sprintf("./apps/%s/%s", appname, appversion))
|
||||
if err != nil {
|
||||
log.Printf("Build error: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deploys all cloud functions. Onprem thooo :(
|
||||
func deployAll() {
|
||||
allapps := []string{
|
||||
"hoxhunt",
|
||||
"secureworks",
|
||||
"servicenow",
|
||||
"lastline",
|
||||
"netcraft",
|
||||
"misp",
|
||||
"email",
|
||||
"testing",
|
||||
"http",
|
||||
"recordedfuture",
|
||||
"passivetotal",
|
||||
"carbon_black",
|
||||
"thehive",
|
||||
"cortex",
|
||||
"splunk",
|
||||
}
|
||||
|
||||
for _, appname := range allapps {
|
||||
appversion := "1.0.0"
|
||||
|
||||
err := deployConfigToBackend(appname, appversion)
|
||||
if err != nil {
|
||||
log.Printf("Failed uploading config: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
deployAppCloudFunc(appname, appversion)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
deployAll()
|
||||
return
|
||||
|
||||
appname := "testing"
|
||||
appversion := "1.0.0"
|
||||
|
||||
err := deployConfigToBackend(appname, appversion)
|
||||
if err != nil {
|
||||
log.Printf("Failed uploading config: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
deployAppCloudFunc(appname, appversion)
|
||||
|
||||
// FIXME - build and deploy to dockerhub as well :)
|
||||
// Not able to work in remote directory propely... Even tried making an actual tar and checking it rofl
|
||||
//err := deployWorker(appname, appversion)
|
||||
//if err != nil {
|
||||
// log.Printf("Failed to deploy docker worker: %s", err)
|
||||
//}
|
||||
}
|
||||
Reference in New Issue
Block a user