Initial open source commit
This commit is contained in:
frikky
@@ -0,0 +1,19 @@
|
||||
# Functions
|
||||
The point of this folder is to make GCP Cloud functions able to run default WALKOFF apps.
|
||||
|
||||
# How it works:
|
||||
* Subsequent info is based on the appname in main
|
||||
1. stitcher.go deploys the config to the app part of the website
|
||||
2. stitcher.go deploys the cloud function based on baseline.py
|
||||
3. stitcher.go SHOULD deploy the app to dockerhub for onpremise usecases
|
||||
|
||||
## How to fix an appfile (done in stitcher.go)
|
||||
* Remove walkoff_app_sdk.app_base import
|
||||
* Remove anything with __name__ == "__main"__ (runner)
|
||||
|
||||
## Stitching order:
|
||||
* Base imports
|
||||
* Authorization
|
||||
* class AppBase
|
||||
* class <APP>
|
||||
* Runner
|
||||
@@ -0,0 +1,25 @@
|
||||
FROM python:3.7-alpine as base
|
||||
|
||||
FROM base as builder
|
||||
RUN apk --no-cache add --update alpine-sdk
|
||||
|
||||
RUN mkdir /install
|
||||
WORKDIR /install
|
||||
|
||||
COPY ./worker/requirements.txt /requirements.txt
|
||||
|
||||
RUN git clone "https://github.com/aio-libs/aioredis.git"
|
||||
RUN pip install --prefix="/install" ./aioredis
|
||||
RUN pip install --prefix="/install" --no-deps asteval
|
||||
RUN pip install --prefix="/install" six
|
||||
RUN pip install --prefix="/install" -r /requirements.txt
|
||||
|
||||
FROM base
|
||||
|
||||
COPY --from=builder /install /usr/local
|
||||
COPY ./umpire/common /app/common
|
||||
COPY ./worker /app/worker
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
CMD python -m worker.worker
|
||||
@@ -0,0 +1,253 @@
|
||||
import logging
|
||||
import asyncio
|
||||
import sys
|
||||
import warnings
|
||||
from logging import StreamHandler, DEBUG, INFO, ERROR, WARNING, CRITICAL, raiseExceptions
|
||||
|
||||
|
||||
class AsyncHandler(StreamHandler):
|
||||
""" An async wrapper around logging.StreamHandler for async log streams like Redis PUB/SUB"""
|
||||
def __init__(self, stream=None, loop=None):
|
||||
"""
|
||||
Initialize the handler.
|
||||
|
||||
If stream is not specified, sys.stderr is used.
|
||||
"""
|
||||
super().__init__(stream)
|
||||
self.loop = loop
|
||||
|
||||
async def flush(self):
|
||||
"""
|
||||
Flushes the stream.
|
||||
"""
|
||||
await self.stream.flush()
|
||||
|
||||
async def emit(self, record):
|
||||
"""
|
||||
Emit a record.
|
||||
|
||||
If a formatter is specified, it is used to format the record.
|
||||
The record is then written to the stream with a trailing newline. If
|
||||
exception information is present, it is formatted using
|
||||
traceback.print_exception and appended to the stream. If the stream
|
||||
has an 'encoding' attribute, it is used to determine how to do the
|
||||
output to the stream.
|
||||
"""
|
||||
try:
|
||||
msg = self.format(record)
|
||||
await self.stream.write(msg + self.terminator)
|
||||
await self.flush()
|
||||
except Exception:
|
||||
self.handleError(record)
|
||||
|
||||
async def handle(self, record):
|
||||
"""
|
||||
Conditionally emit the specified logging record.
|
||||
|
||||
Emission depends on filters which may have been added to the handler.
|
||||
Wrap the actual emission of the record with acquisition/release of
|
||||
the I/O thread lock. Returns whether the filter passed the record for
|
||||
emission.
|
||||
"""
|
||||
rv = self.filter(record)
|
||||
if rv:
|
||||
self.acquire()
|
||||
try:
|
||||
await self.emit(record)
|
||||
finally:
|
||||
self.release()
|
||||
return rv
|
||||
|
||||
async def close(self):
|
||||
if self.stream is not None:
|
||||
await self.flush()
|
||||
await self.stream.close()
|
||||
super().close()
|
||||
|
||||
|
||||
class AsyncLogger(logging.Logger):
|
||||
""" An async wrapper around logging.Logger for async log streams like Redis PUB/SUB """
|
||||
def __init__(self, name, level=logging.ERROR, loop=asyncio.get_event_loop()):
|
||||
super().__init__(name, level=level)
|
||||
self.loop = loop
|
||||
|
||||
async def debug(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Log 'msg % args' with severity 'DEBUG'.
|
||||
|
||||
To pass exception information, use the keyword argument exc_info with
|
||||
a true value, e.g.
|
||||
|
||||
logger.debug("Houston, we have a %s", "thorny problem", exc_info=1)
|
||||
"""
|
||||
if self.isEnabledFor(DEBUG):
|
||||
await self._log(DEBUG, msg, args, **kwargs)
|
||||
|
||||
async def info(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Log 'msg % args' with severity 'INFO'.
|
||||
|
||||
To pass exception information, use the keyword argument exc_info with
|
||||
a true value, e.g.
|
||||
|
||||
logger.info("Houston, we have a %s", "interesting problem", exc_info=1)
|
||||
"""
|
||||
if self.isEnabledFor(INFO):
|
||||
await self._log(INFO, msg, args, **kwargs)
|
||||
|
||||
async def warning(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Log 'msg % args' with severity 'WARNING'.
|
||||
|
||||
To pass exception information, use the keyword argument exc_info with
|
||||
a true value, e.g.
|
||||
|
||||
logger.warning("Houston, we have a %s", "bit of a problem", exc_info=1)
|
||||
"""
|
||||
if self.isEnabledFor(WARNING):
|
||||
await self._log(WARNING, msg, args, **kwargs)
|
||||
|
||||
async def warn(self, msg, *args, **kwargs):
|
||||
warnings.warn("The 'warn' method is deprecated, use 'warning' instead", DeprecationWarning, 2)
|
||||
await self.warning(msg, *args, **kwargs)
|
||||
|
||||
async def error(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Log 'msg % args' with severity 'ERROR'.
|
||||
|
||||
To pass exception information, use the keyword argument exc_info with
|
||||
a true value, e.g.
|
||||
|
||||
logger.error("Houston, we have a %s", "major problem", exc_info=1)
|
||||
"""
|
||||
if self.isEnabledFor(ERROR):
|
||||
await self._log(ERROR, msg, args, **kwargs)
|
||||
|
||||
async def exception(self, msg, *args, exc_info=True, **kwargs):
|
||||
"""
|
||||
Convenience method for logging an ERROR with exception information.
|
||||
"""
|
||||
await self.error(msg, *args, exc_info=exc_info, **kwargs)
|
||||
|
||||
async def critical(self, msg, *args, **kwargs):
|
||||
"""
|
||||
Log 'msg % args' with severity 'CRITICAL'.
|
||||
|
||||
To pass exception information, use the keyword argument exc_info with
|
||||
a true value, e.g.
|
||||
|
||||
logger.critical("Houston, we have a %s", "major disaster", exc_info=1)
|
||||
"""
|
||||
if self.isEnabledFor(CRITICAL):
|
||||
await self._log(CRITICAL, msg, args, **kwargs)
|
||||
|
||||
fatal = critical
|
||||
|
||||
async def log(self, level, msg, *args, **kwargs):
|
||||
"""
|
||||
Log 'msg % args' with the integer severity 'level'.
|
||||
|
||||
To pass exception information, use the keyword argument exc_info with
|
||||
a true value, e.g.
|
||||
|
||||
logger.log(level, "We have a %s", "mysterious problem", exc_info=1)
|
||||
"""
|
||||
if not isinstance(level, int):
|
||||
if raiseExceptions:
|
||||
raise TypeError("level must be an integer")
|
||||
else:
|
||||
return
|
||||
if self.isEnabledFor(level):
|
||||
await self._log(level, msg, args, **kwargs)
|
||||
|
||||
async def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
|
||||
"""
|
||||
Low-level logging routine which creates a LogRecord and then calls
|
||||
all the handlers of this logger to handle the record.
|
||||
"""
|
||||
sinfo = None
|
||||
if logging._srcfile:
|
||||
try:
|
||||
fn, lno, func, sinfo = self.findCaller(stack_info)
|
||||
except ValueError:
|
||||
fn, lno, func = "(unknown file)", 0, "(unknown function)"
|
||||
else:
|
||||
fn, lno, func = "(unknown file)", 0, "(unknown function)"
|
||||
if exc_info:
|
||||
if isinstance(exc_info, BaseException):
|
||||
exc_info = (type(exc_info), exc_info, exc_info.__traceback__)
|
||||
elif not isinstance(exc_info, tuple):
|
||||
exc_info = sys.exc_info()
|
||||
record = self.makeRecord(self.name, level, fn, lno, msg, args,
|
||||
exc_info, func, extra, sinfo)
|
||||
await self.handle(record)
|
||||
|
||||
async def handle(self, record):
|
||||
"""
|
||||
Call the handlers for the specified record.
|
||||
|
||||
This method is used for unpickled records received from a socket, as
|
||||
well as those created locally. Logger-level filtering is applied.
|
||||
"""
|
||||
if (not self.disabled) and self.filter(record):
|
||||
await self.callHandlers(record)
|
||||
|
||||
async def callHandlers(self, record):
|
||||
"""
|
||||
Pass a record to all relevant handlers.
|
||||
|
||||
Loop through all handlers for this logger and its parents in the
|
||||
logger hierarchy. If no handler was found, output a one-off error
|
||||
message to sys.stderr. Stop searching up the hierarchy whenever a
|
||||
logger with the "propagate" attribute set to zero is found - that
|
||||
will be the last logger whose handlers are called.
|
||||
"""
|
||||
c = self
|
||||
found = 0
|
||||
while c:
|
||||
for hdlr in c.handlers:
|
||||
found += 1
|
||||
if record.levelno >= hdlr.level:
|
||||
await hdlr.handle(record)
|
||||
if not c.propagate:
|
||||
c = None
|
||||
else:
|
||||
c = c.parent
|
||||
if found == 0:
|
||||
if logging.lastResort:
|
||||
if record.levelno >= logging.lastResort.level:
|
||||
logging.lastResort.handle(record)
|
||||
elif logging.raiseExceptions and not self.manager.emittedNoHandlerWarning:
|
||||
sys.stderr.write("No handlers could be found for logger"
|
||||
" \"%s\"\n" % self.name)
|
||||
self.manager.emittedNoHandlerWarning = True
|
||||
|
||||
async def shutdown(self):
|
||||
"""
|
||||
Perform any cleanup actions in the logging system (e.g. flushing
|
||||
buffers).
|
||||
|
||||
Should be called at application exit.
|
||||
"""
|
||||
for handler in reversed(self.handlers):
|
||||
# errors might occur, for example, if files are locked
|
||||
# we just ignore them if raiseExceptions is not set
|
||||
try:
|
||||
if handler:
|
||||
try:
|
||||
# await handler.acquire() # do we need to lock?
|
||||
await handler.flush()
|
||||
await handler.close()
|
||||
except (OSError, ValueError):
|
||||
# Ignore errors which might be caused
|
||||
# because handlers have been closed but
|
||||
# references to them are still around at
|
||||
# application exit.
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
finally:
|
||||
# handler.release() # We need to release if we decide to lock I guess
|
||||
self.removeHandler(handler)
|
||||
except Exception: # ignore everything, as we're shutting down
|
||||
pass
|
||||
@@ -0,0 +1,73 @@
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="{asctime} - {name} - {levelname}:{message}", style='{')
|
||||
logger = logging.getLogger("WALKOFF")
|
||||
CONFIG_PATH = (Path(__file__).parent / "config.ini").resolve()
|
||||
|
||||
|
||||
def sint(value, default):
|
||||
if not isinstance(default, int):
|
||||
raise TypeError("Default value must be of integer type")
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def sfloat(value, default):
|
||||
if not isinstance(default, int):
|
||||
raise TypeError("Default value must be of float type")
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
class Config:
|
||||
# Worker options
|
||||
WORKER_TIMEOUT = os.environ.get("WORKER_TIMEOUT", "30")
|
||||
API_GATEWAY_URI = os.environ.get("API_GATEWAY_URI", "http://localhost:8001")
|
||||
WALKOFF_USERNAME = os.environ.get("WALKOFF_USERNAME", '')
|
||||
WALKOFF_PASSWORD = os.environ.get("WALKOFF_PASSWORD", '')
|
||||
|
||||
# Umpire options
|
||||
APPS_PATH = os.getenv("APPS_PATH", "../apps")
|
||||
APP_REFRESH = os.getenv("APP_REFRESH", "60")
|
||||
SWARM_NETWORK = os.getenv("SWARM_NETWORK", "walkoff_default")
|
||||
APP_PREFIX = os.getenv("APP_PREFIX", "walkoff_app")
|
||||
STACK_PREFIX = os.getenv("STACK_PREFIX", "walkoff")
|
||||
DOCKER_REGISTRY = os.getenv("DOCKER_REGISTRY", "127.0.0.1:5000")
|
||||
UMPIRE_HEARTBEAT = os.getenv("UMPIRE_HEARTBEAT", "1")
|
||||
|
||||
# Redis options
|
||||
REDIS_URI = os.getenv("REDIS_URI", "redis://192.168.239.145:6379")
|
||||
REDIS_EXECUTING_WORKFLOWS = os.getenv("REDIS_EXECUTING_WORKFLOWS", "executing-workflows")
|
||||
REDIS_PENDING_WORKFLOWS = os.getenv("REDIS_PENDING_WORKFLOWS", "pending-workflows")
|
||||
REDIS_ABORTING_WORKFLOWS = os.getenv("REDIS_ABORTING_WORKFLOWS", "aborting-workflows")
|
||||
REDIS_ACTIONS_IN_PROCESS = os.getenv("REDIS_ACTIONS_IN_PROCESS", "actions-in-process")
|
||||
REDIS_WORKFLOW_QUEUE = os.getenv("REDIS_WORKFLOW_Q", "workflow-queue")
|
||||
REDIS_WORKFLOWS_IN_PROCESS = os.getenv("REDIS_WORKFLOWS_IN_PROCESS", "workflows-in-process")
|
||||
REDIS_WORKFLOW_GROUP = os.getenv("REDIS_WORKFLOW_GROUP", "workflow-group")
|
||||
REDIS_ACTION_RESULTS_GROUP = os.getenv("REDIS_ACTION_RESULTS_GROUP", "action-results-group")
|
||||
REDIS_WORKFLOW_TRIGGERS_GROUP = os.getenv("REDIS_WORKFLOW_TRIGGERS_GROUP", "workflow-triggers-group")
|
||||
REDIS_WORKFLOW_CONTROL = os.getenv("REDIS_WORKFLOW_CONTROL", "workflow-control")
|
||||
REDIS_WORKFLOW_CONTROL_GROUP = os.getenv("REDIS_WORKFLOW_CONTROL_GROUP", "workflow-control-group")
|
||||
|
||||
# Overrides the environment variables for docker-compose and docker commands on the docker machine at 'DOCKER_HOST'
|
||||
# See: https://docs.docker.com/compose/reference/envvars/ for more information.
|
||||
# DOCKER_HOST = os.environ.get("DOCKER_HOST", "tcp://ip_of_docker_swarm_manager:2376")
|
||||
# DOCKER_HOST = os.environ.get("DOCKER_HOST", "unix:///var/run/docker.sock")
|
||||
# DOCKER_TLS_VERIFY = os.environ.get("DOCKER_TLS_VERIFY", "1")
|
||||
# DOCKER_CERT_PATH = os.environ.get("DOCKER_CERT_PATH", "/Path/to/certs/for/remote/docker/daemon")
|
||||
|
||||
def get_int(self, key, default):
|
||||
return sint(getattr(self, key), default)
|
||||
|
||||
def get_float(self, key, default):
|
||||
return sfloat(getattr(self, key), default)
|
||||
|
||||
|
||||
config = Config()
|
||||
@@ -0,0 +1,321 @@
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import json
|
||||
import copy
|
||||
import base64
|
||||
import tarfile
|
||||
from io import BytesIO
|
||||
from pathlib import Path
|
||||
from contextlib import contextmanager, asynccontextmanager
|
||||
|
||||
import aiodocker
|
||||
from aiodocker.utils import clean_map
|
||||
from aiodocker.exceptions import DockerError
|
||||
import docker
|
||||
from docker.models.services import _get_create_service_kwargs
|
||||
from docker.types.services import ServiceMode, Resources, EndpointSpec, RestartPolicy, SecretReference
|
||||
|
||||
from compose.cli.command import get_project as get_compose_project
|
||||
from compose.utils import timeparse, parse_bytes
|
||||
from compose.config.environment import Environment
|
||||
|
||||
from config import config
|
||||
from helpers import sint, sfloat
|
||||
|
||||
logger = logging.getLogger("UMPIRE")
|
||||
|
||||
|
||||
class DockerBuildError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
# TODO: Clean a lot of this up and rectify the inconsistencies between the different docker libraries
|
||||
class ServiceKwargs:
|
||||
@classmethod
|
||||
def configure(cls, image, service, secrets=None, mounts=None, **kwargs):
|
||||
self = ServiceKwargs()
|
||||
options = service.options
|
||||
deploy_opts = options.get("deploy", {})
|
||||
prefs = deploy_opts.get("placement", {}).get("preferences", {})
|
||||
|
||||
# Map compose options to service options
|
||||
self.image = image
|
||||
self.constraints = deploy_opts.get("placement", {}).get("constraints")
|
||||
self.preferences = [kv for pref in prefs for kv in pref.items()]
|
||||
self.container_labels = options.get("labels")
|
||||
|
||||
self.endpoint_spec = EndpointSpec(deploy_opts.get("endpoint_mode"),
|
||||
{p.published: p.target for p in options.get("ports", [])})
|
||||
|
||||
self.env = options.get("environment", None)
|
||||
self.hostname = options.get("hostname")
|
||||
self.isolation = options.get("isolation")
|
||||
self.labels = {k: v for k, v in (kv.split('=') for kv in deploy_opts.get("labels", []))}
|
||||
self.log_driver = options.get("logging", {}).get("driver")
|
||||
self.log_driver_options = options.get("logging", {}).get("options")
|
||||
self.mode = ServiceMode(deploy_opts.get("mode", "replicated"), deploy_opts.get("replicas", 1))
|
||||
self.networks = [config.SWARM_NETWORK] # Similar to mounts. I don't see the use case but see the issues
|
||||
|
||||
resource_opts = deploy_opts.get("resources", {})
|
||||
if resource_opts:
|
||||
# Unpack any generic_resources defined i.e. gpus and such
|
||||
reservation_opts = resource_opts.get("reservations", {})
|
||||
generic_resources = {}
|
||||
for generic_resource in reservation_opts.get("generic_resources", {}):
|
||||
discrete_resource_spec = generic_resource["discrete_resource_spec"]
|
||||
generic_resources[discrete_resource_spec["kind"]] = discrete_resource_spec["value"]
|
||||
cpu_limit = sfloat(resource_opts.get("limits", {}).get("cpus"), 0)
|
||||
cpu_reservation = sfloat(reservation_opts.get("cpus"), 0)
|
||||
nano_cpu_limit = sint(cpu_limit * 1e9, 0) if cpu_limit is not None else None
|
||||
nano_cpu_reservation = sint(cpu_reservation * 1e9, 0) if cpu_reservation is not None else None
|
||||
self.resources = Resources(cpu_limit=nano_cpu_limit,
|
||||
mem_limit=parse_bytes(resource_opts.get("limits", {}).get("memory", '')),
|
||||
cpu_reservation=nano_cpu_reservation,
|
||||
mem_reservation=parse_bytes(reservation_opts.get("memory", '')),
|
||||
generic_resources=generic_resources)
|
||||
|
||||
restart_opts = deploy_opts.get("restart_policy", {})
|
||||
if restart_opts:
|
||||
# Parse the restart policy
|
||||
delay = timeparse(restart_opts.get("delay", "0s"))
|
||||
window = timeparse(restart_opts.get("restart_opts", "0s"))
|
||||
self.restart_policy = RestartPolicy(condition=restart_opts.get("condition", ),
|
||||
delay=delay,
|
||||
max_attempts=sint(restart_opts.get("max_attempts", 0), 0),
|
||||
window=window)
|
||||
|
||||
self.secrets = secrets
|
||||
self.mounts = mounts
|
||||
|
||||
# Grab any key word arguments that may have been given
|
||||
[setattr(self, k, v) for k, v in kwargs.items() if hasattr(self, k)]
|
||||
|
||||
service_kwargs = _get_create_service_kwargs('create', copy.copy(self.__dict__))
|
||||
|
||||
# This is needed because aiodocker assumes the Env is a dictionary for some reason...
|
||||
if self.env is not None:
|
||||
service_kwargs["task_template"]["ContainerSpec"]["Env"] = self.env
|
||||
|
||||
return service_kwargs
|
||||
|
||||
|
||||
async def create_secret(client, name, data):
|
||||
data = base64.b64encode(data)
|
||||
data = data.decode("ascii")
|
||||
body = {"Data": data, "Name": name}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
resp = await client._query("secrets/create", "POST", data=json.dumps(body), headers=headers)
|
||||
return await resp.json()
|
||||
|
||||
|
||||
async def update_service(client, service_id, version, *, image=None, rollback=None, mode=None):
|
||||
if image is None and rollback is False:
|
||||
raise ValueError("You need to specify an image.")
|
||||
|
||||
inspect_service = await client.services.inspect(service_id)
|
||||
spec = inspect_service["Spec"]
|
||||
|
||||
if mode is not None:
|
||||
spec["Mode"] = mode
|
||||
|
||||
if image is not None:
|
||||
spec["TaskTemplate"]["ContainerSpec"]["Image"] = image
|
||||
|
||||
params = {"version": version}
|
||||
if rollback is True:
|
||||
params["rollback"] = "previous"
|
||||
|
||||
data = json.dumps(clean_map(spec))
|
||||
|
||||
await client._query_json(
|
||||
"services/{service_id}/update".format(service_id=service_id),
|
||||
method="POST",
|
||||
data=data,
|
||||
params=params,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
async def get_secret(client: aiodocker.Docker, secret_id):
|
||||
resp = await client._query(f"secrets/{secret_id}")
|
||||
print(resp)
|
||||
print(resp)
|
||||
return await resp.json()
|
||||
|
||||
|
||||
async def delete_secret(client: aiodocker.Docker, secret_id):
|
||||
await client._query(f"secrets/{secret_id}", "DELETE")
|
||||
|
||||
|
||||
async def get_nodes(client: aiodocker.Docker):
|
||||
resp = await client._query("nodes")
|
||||
return await resp.json()
|
||||
|
||||
|
||||
async def get_tasks(client: aiodocker.Docker, params):
|
||||
resp = await client._query("tasks" + '?' + params)
|
||||
return await resp.json()
|
||||
|
||||
|
||||
def normalize_name(name, delimiter=''):
|
||||
""" Super arbitrary naming convention for docker images/services... """
|
||||
return re.sub(r'[^-_a-z0-9]', delimiter, name.lower())
|
||||
|
||||
|
||||
def get_project(path):
|
||||
project = get_compose_project(path, environment=load_docker_env(), project_name=config.APP_PREFIX)
|
||||
project.path = path # we'll add this in to refresh the project later
|
||||
return project
|
||||
|
||||
|
||||
def load_docker_env():
|
||||
# TODO: remove this since it is likely no longer needed
|
||||
environment = os.environ
|
||||
# environment.update({key: val for key, val in config["DOCKER_ENV"].items()})
|
||||
return Environment(environment)
|
||||
|
||||
|
||||
async def get_service(docker_client, service_id):
|
||||
try:
|
||||
s = await docker_client.services.inspect(service_id)
|
||||
return {'id': s["ID"], 'version': s['Version']['Index']}
|
||||
except DockerError:
|
||||
return {}
|
||||
|
||||
|
||||
async def remove_service(docker_client, service):
|
||||
try:
|
||||
return await docker_client.services.delete(service)
|
||||
except DockerError:
|
||||
logger.error(f"Could not delete {service}.")
|
||||
return False
|
||||
|
||||
|
||||
async def get_replicas(docker_client, service):
|
||||
"""
|
||||
Gets the running and desired replica counts for the given service ID
|
||||
:param service: The docker id of the service
|
||||
:return: a dictionary giving the number of "running" and "desired" replicas
|
||||
"""
|
||||
tasks = await docker_client.tasks.list(filters={"service": [service]})
|
||||
desired = sum([t["DesiredState"] == "running" for t in tasks])
|
||||
running = sum([t["Status"]["State"] == "running" for t in tasks])
|
||||
return {"running": running, "desired": desired}
|
||||
|
||||
|
||||
async def get_containers(docker_client, service, short_ids=False):
|
||||
"""
|
||||
Gets the running containers the given service ID
|
||||
:param service: The docker id of the service
|
||||
:return: a set of the running containers
|
||||
"""
|
||||
def get_container_id(task_spec):
|
||||
return task_spec["Status"]["ContainerStatus"]["ContainerID"]
|
||||
|
||||
def get_state(task_spec):
|
||||
return task_spec["Status"]["State"]
|
||||
|
||||
def has_container(task_spec):
|
||||
return task_spec["Status"].get("ContainerStatus") is not None
|
||||
|
||||
tasks = await docker_client.tasks.list(filters={"service": [service]})
|
||||
|
||||
if short_ids:
|
||||
return set(get_container_id(t)[:12] for t in tasks if get_state(t) == "running" and has_container(t))
|
||||
return set(get_container_id(t) for t in tasks if get_state(t) == "running" and has_container(t))
|
||||
|
||||
|
||||
|
||||
async def load_secrets(docker_client, project):
|
||||
service = project.services[0]
|
||||
secret_references = []
|
||||
for service_secret in service.secrets:
|
||||
secret = service_secret["secret"]
|
||||
filename = service_secret.get("file", secret.source)
|
||||
|
||||
# Compose doesn't parse external secrets so we'll assume there is one and build if it doesn't exist
|
||||
try:
|
||||
secret_id = await get_secret(docker_client, secret.source)
|
||||
|
||||
except (AttributeError, DockerError):
|
||||
with open(filename, 'rb') as fp:
|
||||
data = fp.read()
|
||||
secret_id = (await create_secret(docker_client, name=secret.source, data=data)).get("ID")
|
||||
|
||||
if secret_id is not None:
|
||||
secret_references.append(SecretReference(secret_id=secret_id, secret_name=secret.source,
|
||||
uid=secret.uid, gid=secret.gid, mode=secret.mode))
|
||||
return secret_references
|
||||
|
||||
def connect_to_docker():
|
||||
client = docker.from_env(environment=load_docker_env())
|
||||
try:
|
||||
if client.ping():
|
||||
logger.debug(f"Connected to Docker Engine: v{client.version()['Version']}")
|
||||
return client
|
||||
except docker.errors.APIError as e:
|
||||
logger.error(f"Docker API error during connect: {e}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def connect_to_aiodocker():
|
||||
client = aiodocker.Docker()
|
||||
try:
|
||||
|
||||
if (await client._query("_ping")).status == 200:
|
||||
resp = await client._query("version")
|
||||
version = (await resp.json())["Version"]
|
||||
logger.debug(f"Connected to Docker Engine: v{version}")
|
||||
yield client
|
||||
finally:
|
||||
await client.close()
|
||||
logger.info("Docker connection closed.")
|
||||
|
||||
|
||||
@contextmanager
|
||||
def docker_context(path, dirs=None):
|
||||
"""
|
||||
Tars and compresses the given docker context in memory. Useful for sending contexts to `docker build` commands.
|
||||
:param path: str or pathlib.Path object representing the path of the context
|
||||
:param dirs: white list of directories under path to grab
|
||||
:return: an in memory tar of the context
|
||||
"""
|
||||
if not isinstance(path, Path):
|
||||
try:
|
||||
path = Path(path)
|
||||
except (ValueError, NotImplementedError):
|
||||
logger.exception(f"Error accessing path: \"{path}\"")
|
||||
return
|
||||
|
||||
fileobj = BytesIO()
|
||||
tar = tarfile.open(fileobj=fileobj, mode="w")
|
||||
|
||||
# If a list of subdirectories is listed, only grab them
|
||||
if dirs is not None:
|
||||
for d in dirs:
|
||||
tar.add(path / d, arcname=d)
|
||||
else:
|
||||
tar.add(path, arcname='')
|
||||
tar.close()
|
||||
try:
|
||||
fileobj.seek(0) # must go back to start of file after tarfile writes to it
|
||||
yield fileobj
|
||||
finally:
|
||||
fileobj.close()
|
||||
|
||||
|
||||
async def stream_docker_log(log_stream):
|
||||
async for line in log_stream:
|
||||
if "stream" in line and line["stream"].strip():
|
||||
print(line["stream"].strip())
|
||||
logger.debug(line["stream"].strip())
|
||||
elif "status" in line:
|
||||
print(line["status"].strip())
|
||||
logger.debug(line["status"].strip())
|
||||
elif "error" in line:
|
||||
print(line["error"].strip())
|
||||
logger.error(line["error"].strip())
|
||||
raise DockerBuildError
|
||||
else:
|
||||
print(line)
|
||||
@@ -0,0 +1,101 @@
|
||||
import logging
|
||||
from config import config
|
||||
|
||||
import aiohttp
|
||||
import requests
|
||||
|
||||
from message_types import(message_dumps, NodeStatusMessage, WorkflowStatusMessage,
|
||||
StatusEnum, JSONPatch, JSONPatchOps)
|
||||
|
||||
logger = logging.getLogger("WALKOFF")
|
||||
|
||||
HEX_CHARS = 'abcdefABCDEF0123456789'
|
||||
UUID_GLOB = "-".join((f"[{HEX_CHARS}]" * i for i in (8, 4, 4, 4, 12)))
|
||||
UUID_REGEX = "[a-f0-9]{8}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{4}-[a-f0-9]{12}"
|
||||
|
||||
|
||||
def sint(value, default):
|
||||
if not isinstance(default, int):
|
||||
raise TypeError("Default value must be of integer type")
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
def sfloat(value, default):
|
||||
if not isinstance(default, int):
|
||||
raise TypeError("Default value must be of float type")
|
||||
try:
|
||||
return float(value)
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
|
||||
async def get_walkoff_auth_header(session, token=None, timeout=5*60):
|
||||
url = config.API_GATEWAY_URI.rstrip('/') + '/api'
|
||||
|
||||
# TODO: make this secure and don't use default admin user
|
||||
if token is None:
|
||||
async with session.post(url + "/auth", json={"username": config.WALKOFF_USERNAME,
|
||||
"password": config.WALKOFF_PASSWORD}, timeout=timeout) as resp:
|
||||
resp_json = await resp.json()
|
||||
#token = resp_json["refresh_token"]
|
||||
token = "refresh"
|
||||
logger.debug("Successfully logged into WALKOFF")
|
||||
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
async with session.post(url + "/auth/refresh", headers=headers, timeout=timeout) as resp:
|
||||
resp_json = await resp.json()
|
||||
#access_token = resp_json["access_token"]
|
||||
access_token = "access"
|
||||
logger.debug("Successfully refreshed WALKOFF JWT")
|
||||
|
||||
return {"Authorization": f"Bearer {access_token}"}, token
|
||||
|
||||
|
||||
def make_patch(message, root, op, value_only=False, white_list=None, black_list=None):
|
||||
if white_list is None and black_list is None:
|
||||
raise ValueError("Either white_list or black_list must be provided")
|
||||
|
||||
if white_list is not None and black_list is not None:
|
||||
raise ValueError("Either white_list or black_list must be provided, not both")
|
||||
|
||||
# convert blacklist to whitelist and grab those attrs from the message
|
||||
white_list = set(message.__slots__).difference(black_list) if black_list is not None else white_list
|
||||
|
||||
if value_only and len(white_list) != 1:
|
||||
raise ValueError("value_only can only be set if a single key is in white_list")
|
||||
|
||||
if value_only:
|
||||
(key,) = white_list
|
||||
values = getattr(message, key)
|
||||
else:
|
||||
values = {k: getattr(message, k) for k in message.__slots__ if k in white_list}
|
||||
|
||||
return JSONPatch(op, path=root, value=values)
|
||||
|
||||
|
||||
def get_patches(message):
|
||||
patches = []
|
||||
if isinstance(message, NodeStatusMessage):
|
||||
root = f"/node_statuses/{message.node_id}"
|
||||
if message.status == StatusEnum.EXECUTING:
|
||||
patches.append(make_patch(message, root, JSONPatchOps.ADD, black_list={"result", "completed_at"}))
|
||||
|
||||
else:
|
||||
patches.append(make_patch(message, root, JSONPatchOps.REPLACE, black_list={}))
|
||||
|
||||
elif isinstance(message, WorkflowStatusMessage):
|
||||
if message.status == StatusEnum.EXECUTING:
|
||||
for key in [attr for attr in message.__slots__ if getattr(message, attr)]:
|
||||
patches.append(make_patch(message, f"/{key}", JSONPatchOps.REPLACE, value_only=True,
|
||||
white_list={f"{key}"}))
|
||||
|
||||
elif message.status == StatusEnum.COMPLETED or message.status == StatusEnum.ABORTED:
|
||||
patches.append(make_patch(message, f"/status", JSONPatchOps.REPLACE, value_only=True,
|
||||
white_list={"status"}))
|
||||
patches.append(make_patch(message, f"/completed_at", JSONPatchOps.REPLACE, value_only=True,
|
||||
white_list={"completed_at"}))
|
||||
|
||||
return patches
|
||||
@@ -0,0 +1,583 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import json
|
||||
import sys
|
||||
import os
|
||||
import signal
|
||||
import requests
|
||||
import os
|
||||
import time
|
||||
from collections import deque
|
||||
from inspect import getcoroutinelocals
|
||||
|
||||
from google.cloud import pubsub
|
||||
|
||||
import aiohttp
|
||||
import aioredis
|
||||
|
||||
from message_types import message_dumps, message_loads, NodeStatusMessage, WorkflowStatusMessage, StatusEnum
|
||||
from helpers import get_walkoff_auth_header
|
||||
from redis_helpers import connect_to_redis_pool, xdel, deref_stream_message
|
||||
from workflow_types import (Node, Action, Condition, Transform, Parameter, Trigger,
|
||||
ParameterVariant, Workflow, workflow_dumps, workflow_loads, ConditionException)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="{asctime} - {name} - {levelname}:{message}", style='{')
|
||||
logger = logging.getLogger("WORKER")
|
||||
# logging.getLogger("asyncio").setLevel(logging.DEBUG)
|
||||
# logger.setLevel(logging.DEBUG)
|
||||
|
||||
CONTAINER_ID = ""#os.getenv("HOSTNAME")
|
||||
APIKEY = ""#os.getenv("FUNCTION_APIKEY")
|
||||
|
||||
# FIXME
|
||||
#apiurl = "http://localhost:5001"
|
||||
apiurl = "https://shuffler.io"
|
||||
|
||||
class Worker:
|
||||
def __init__(self, workflow: Workflow = None, start_action: str = None, redis: aioredis.Redis = None,
|
||||
session: aiohttp.ClientSession = None):
|
||||
self.workflow = workflow
|
||||
self.start_action = start_action if start_action is not None else self.workflow.start
|
||||
self.results_stream = f"{workflow.execution_id}:results"
|
||||
self.parallel_accumulator = {}
|
||||
self.accumulator = {}
|
||||
self.parallel_in_process = {}
|
||||
self.in_process = {}
|
||||
self.redis = redis
|
||||
self.streams = set()
|
||||
self.scheduling_tasks = set()
|
||||
self.results_getter_task = None
|
||||
self.parallel_tasks = set()
|
||||
self.workflow_tasks = set()
|
||||
self.execution_task = None
|
||||
self.session = session
|
||||
self.token = None
|
||||
self.parent_map = {}
|
||||
self.cancelled = []
|
||||
self.results = {}
|
||||
|
||||
self.execution_id = ""
|
||||
self.workflow_id = ""
|
||||
self.id = ""
|
||||
self.locations = []
|
||||
self.project_id = ""
|
||||
self.authorization = ""
|
||||
self.start_id = start_action.id if start_action is not None else ""
|
||||
|
||||
async def cancel_subgraph(self, node):
|
||||
"""
|
||||
Cancels the task related to the current node as well as the tasks related to every child of that node.
|
||||
Also removes them from the worker's internal in_process queue.
|
||||
"""
|
||||
# dependents = self.workflow.get_dependents(node)
|
||||
cancelled_tasks = set()
|
||||
|
||||
self.cancelled.append(node.id)
|
||||
to_cancel = await self.cancel_helper(node, [node.id])
|
||||
|
||||
for task in self.scheduling_tasks:
|
||||
for _, arg in getcoroutinelocals(task._coro).items():
|
||||
if isinstance(arg, Node):
|
||||
if arg.id in to_cancel:
|
||||
self.in_process.pop(arg.id)
|
||||
self.accumulator[arg.id] = None
|
||||
self.cancelled.append(arg.id)
|
||||
task.cancel()
|
||||
cancelled_tasks.add(task)
|
||||
|
||||
await asyncio.gather(*cancelled_tasks, return_exceptions=True)
|
||||
|
||||
# This is a very specific one, that might be fucked up by an action named the same thing.
|
||||
# Its this way because of a weird translation from Triggers to Actions that didn't
|
||||
# really work very well
|
||||
def handle_user_input_node(self, node):
|
||||
print("Handle user input. Params: %d!" % len(node.parameters))
|
||||
|
||||
data = ""
|
||||
options = ""
|
||||
actiontypes = []
|
||||
for parameter in node.parameters:
|
||||
print("Param: %s" % parameter)
|
||||
if parameter.name == "alertinfo":
|
||||
data = parameter.value
|
||||
elif parameter.name == "options":
|
||||
options = parameter.value
|
||||
elif parameter.name == "type":
|
||||
actiontypes = parameter.value.split(",")
|
||||
|
||||
print("Data: ", data)
|
||||
print("Options: ", options)
|
||||
print("Types: ", actiontypes)
|
||||
|
||||
executed = False
|
||||
headers = {
|
||||
"Authorization": "Bearer %s" % APIKEY,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
for actiontype in actiontypes:
|
||||
if actiontype == "email":
|
||||
print("SEND EMAIL!")
|
||||
|
||||
#apiurl = "http://localhost:5001"
|
||||
mailurl = "%s/functions/sendmail" % apiurl
|
||||
data = {
|
||||
"targets": ["frikky@shuffler.io"],
|
||||
"body": data,
|
||||
"subject": "Shuffle alert requires input!",
|
||||
"type": "User input",
|
||||
"sender_company": "Shuffle",
|
||||
"reference_execution": self.execution_id,
|
||||
"workflow_id": self.workflow_id,
|
||||
"execution_type": options,
|
||||
"start": node.id,
|
||||
}
|
||||
|
||||
# Add it to actionResult here because of start time!
|
||||
params = self.dereference_params_pubsub(node)
|
||||
|
||||
ret = requests.post(mailurl, headers=headers, json=data)
|
||||
logger.debug("Ret: %s" % ret.text)
|
||||
logger.debug("Status: %d" % ret.status_code)
|
||||
|
||||
if ret.status_code == 200 or ret.status_code == 201:
|
||||
executed = True
|
||||
elif actiontype.lower() == "sms":
|
||||
print("Handle SMS!")
|
||||
executed = True
|
||||
|
||||
if executed:
|
||||
actionurl = "%s/api/v1/streams" % apiurl
|
||||
action = {
|
||||
"name": node.name,
|
||||
"app_name": node.app_name,
|
||||
"app_version": node.app_version,
|
||||
"label": node.label,
|
||||
"environment": node.environment,
|
||||
"id": node.id,
|
||||
}
|
||||
|
||||
action_result = {
|
||||
"action": action,
|
||||
"authorization": self.authorization,
|
||||
"execution_id": self.execution_id,
|
||||
"result": "",
|
||||
"started_at": int(time.time()),
|
||||
"status": "WAITING",
|
||||
}
|
||||
|
||||
|
||||
actionret = requests.post(actionurl, headers=headers, json=action_result)
|
||||
logger.debug("Actionret: %d", actionret.status_code)
|
||||
logger.debug("Actionret: %s", actionret.text)
|
||||
|
||||
print("SHOULD KILL THE EXECUTION (stop this branch)!")
|
||||
|
||||
def execute_workflow_pubsub(self):
|
||||
"""
|
||||
Do a simple BFS to visit and schedule each node in the workflow. We assume every node will run and thus preemptively schedule them all. We will clean up any nodes that will not run due to conditions or triggers
|
||||
"""
|
||||
visited = {self.start_action}
|
||||
queue = deque([self.start_action])
|
||||
self.scheduling_tasks = set()
|
||||
while queue:
|
||||
node = queue.pop()
|
||||
logger.debug("NODE INFO: %s, %s, %s, %s" % (node.name, node.app_name, node.app_version, node.label))
|
||||
parents = {n.id: n for n in self.workflow.predecessors(node)} if node is not self.start_action and node.id is not self.workflow.start else {}
|
||||
children = {n.id: n for n in self.workflow.successors(node)}
|
||||
|
||||
for parent_id in parents:
|
||||
if node.id not in self.parent_map.keys():
|
||||
self.parent_map[node.id] = 1
|
||||
else:
|
||||
self.parent_map[node.id] = self.parent_map[node.id] + 1
|
||||
|
||||
self.in_process[node.id] = node
|
||||
|
||||
if isinstance(node, Action):
|
||||
node.execution_id = self.workflow.execution_id # the app needs this as a key for the redis queue
|
||||
|
||||
# Custom for trigger actions
|
||||
if node.name == "User Input" and node.app_name == "User Input":
|
||||
logger.info("Handling user input!")
|
||||
|
||||
# Skipping new nodes
|
||||
if self.start_id != node.id:
|
||||
self.handle_user_input_node(node)
|
||||
break
|
||||
else:
|
||||
logger.info("Skipping user input as its start node!")
|
||||
|
||||
print("NAME: %s, ENV: %s, LABEL" % (node.name, node.environment))
|
||||
if node.environment == "cloud":
|
||||
self.scheduling_tasks.add(self.schedule_node_pubsub(node, parents, children))
|
||||
|
||||
print("EXIT NAME: %s, ENV: %s, LABEL" % (node.name, node.environment))
|
||||
for child in sorted(children.values(), reverse=True):
|
||||
if child not in visited:
|
||||
queue.appendleft(child)
|
||||
visited.add(child)
|
||||
|
||||
# Checks whether all actions are finished
|
||||
finished = self.get_action_results_pubsub()
|
||||
if finished:
|
||||
print("Got finished and will return!")
|
||||
break
|
||||
|
||||
def dereference_params_pubsub(self, action: Action):
|
||||
param_ret = []
|
||||
global_vars = {}
|
||||
|
||||
print(action.parameters)
|
||||
for param in action.parameters:
|
||||
data = {"value": param.value, "name": param.name, "action_field": param.action_field, "variant": "STATIC_VALUE"}
|
||||
|
||||
if param.variant == ParameterVariant.STATIC_VALUE:
|
||||
data["variant"] = "STATIC_VALUE"
|
||||
elif param.variant == ParameterVariant.ACTION_RESULT:
|
||||
data["variant"] = "ACTION_RESULT"
|
||||
elif param.variant == ParameterVariant.WORKFLOW_VARIABLE:
|
||||
data["variant"] = "WORKFLOW_VARIABLE"
|
||||
elif param.variant == ParameterVariant.GLOBAL:
|
||||
data["variant"] = "GLOBAL"
|
||||
else:
|
||||
logger.error(f"Unable to dereference parameter:{param} for action:{action}")
|
||||
break
|
||||
|
||||
param_ret.append(data)
|
||||
|
||||
return param_ret
|
||||
|
||||
def abort(self):
|
||||
logger.info("ABORTING %s BECAUSE OF ERROR WITH FUNCTION EXECUTION" % self.execution_id)
|
||||
url = f"{apiurl}/api/v1/workflows/{self.workflow_id}/executions/{self.execution_id}/abort"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
"Authorization": f"Bearer {APIKEY}"
|
||||
}
|
||||
|
||||
ret = requests.get(url, headers=headers, timeout=5)
|
||||
logger.info("Aborted with status: %d and text:\n%s" % (ret.status_code, ret.text))
|
||||
sys.exit(0)
|
||||
|
||||
def schedule_node_pubsub(self, node, parents, children):
|
||||
""" Waits until all dependencies of an action are met and then schedules the action """
|
||||
logger.info(f"Scheduling node {node.id} ({node.name})...")
|
||||
|
||||
logger.info(self.accumulator)
|
||||
while not all(parent.id in self.accumulator for parent in parents.values()):
|
||||
time.sleep(1)
|
||||
#await asyncio.sleep(0)
|
||||
|
||||
logger.info(f"Node {node.id} ({node.name}) ready to execute.")
|
||||
|
||||
# node has more than one parent, check if both parent nodes have been cancelled
|
||||
if len(parents) > 1:
|
||||
count = 0
|
||||
for parent in parents:
|
||||
if parent in self.cancelled:
|
||||
count = count + 1
|
||||
|
||||
if count == self.parent_map[node.id]:
|
||||
self.cancel_subgraph(node)
|
||||
|
||||
print(type(node))
|
||||
|
||||
if isinstance(node, Action):
|
||||
print("NODE: %s" % node)
|
||||
params = self.dereference_params_pubsub(node)
|
||||
print("PARAMS: %s" % params)
|
||||
|
||||
# Added authorization to send to function
|
||||
message = {
|
||||
"parameters": params,
|
||||
"execution_id": self.execution_id,
|
||||
"authorization": self.authorization,
|
||||
"node_project": self.project_id,
|
||||
"name": node.name,
|
||||
"app_name": node.app_name,
|
||||
"app_version": node.app_version,
|
||||
"id": node.id,
|
||||
"label": node.name,
|
||||
}
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
"Authorization": f"Bearer {APIKEY}"
|
||||
}
|
||||
|
||||
# Uses version for production apps, but ID for private apps
|
||||
functionname = f"{node.app_name}-{node.app_version}"
|
||||
if not node.sharing:
|
||||
functionname = f"{node.app_name}-{node.private_id}"
|
||||
|
||||
print(f"Functionname (pre): {functionname}")
|
||||
|
||||
functionname = functionname.replace("_", "-")
|
||||
functionname = functionname.replace(":", "-")
|
||||
functionname = functionname.replace(".", "-")
|
||||
functionname = functionname.replace(" ", "-")
|
||||
|
||||
print(f"Functionname (post): {functionname}")
|
||||
|
||||
logger.info(self.locations)
|
||||
logger.info(self.project_id)
|
||||
for location in self.locations:
|
||||
url = f"https://{location}-{self.project_id}.cloudfunctions.net/{functionname}"
|
||||
|
||||
#print(message)
|
||||
try:
|
||||
ret = requests.post(url, headers=headers, json=message)
|
||||
|
||||
# If any error at all, just quit the entire thing (abort)
|
||||
if ret.status_code == 500 or ret.status_code == 401:
|
||||
logger.info("Status: %d. There is an error with ret when starting %s. Should cancel execution and exit. RAW: %s" % (ret.status_code, url, ret.text))
|
||||
self.abort()
|
||||
except requests.exceptions.ReadTimeout as e:
|
||||
logger.debug(e)
|
||||
logger.info("There is an error with ret (readtimeout). Should cancel execution and exit.")
|
||||
self.abort()
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
logger.debug(e)
|
||||
logger.info("There is an error with ret (connectionerror). Should cancel execution and exit.")
|
||||
self.abort()
|
||||
|
||||
#logger.debug(ret.text)
|
||||
logger.debug(ret.status_code)
|
||||
|
||||
# FIXME - only in one location, e.g. eu-west?
|
||||
break
|
||||
|
||||
group = f"{node.app_name}:{node.app_version}"
|
||||
stream = f"{node.execution_id}:{group}"
|
||||
|
||||
logger.info(f"Scheduled {node}")
|
||||
|
||||
|
||||
def get_action_results_pubsub(self):
|
||||
""" Continuously monitors the results queue until all scheduled actions have been completed """
|
||||
results_stream = f"{self.workflow.execution_id}:results"
|
||||
|
||||
# 1. Get the results for the workflowexecution. POST with authorization and ID should do the trick
|
||||
# 2. Check whether the whole thing is still executing
|
||||
# 3. Check whether the status of self.in_process is updated, if so, remove it from in progress
|
||||
# 4. Schedule the next nodes somehow
|
||||
print(len(self.in_process), len(self.parallel_in_process))
|
||||
print(self.in_process, len(self.parallel_in_process))
|
||||
|
||||
url = f"{apiurl}/api/v1/streams/results"
|
||||
#if self.project_id != "":
|
||||
# url = f"https://{self.project_id}.appspot.com/api/v1/streams/results"
|
||||
#
|
||||
|
||||
headers = {"Content-Type": "application/json"}
|
||||
|
||||
# Uses workflow specific authorization generated for priviliged access
|
||||
message = {"authorization": self.authorization, "execution_id": self.execution_id}
|
||||
|
||||
sleeptime = 2
|
||||
logger.info(url)
|
||||
|
||||
logger.info(f"Waiting {sleeptime} seconds for new updates in the nodestream...")
|
||||
while len(self.in_process) > 0 or len(self.parallel_in_process) > 0:
|
||||
# Ask for all nodes, and check every single one that's in progress
|
||||
print("Items in process: %s" % self.in_process)
|
||||
ret = requests.post(url, headers=headers, json=message)
|
||||
if ret.status_code != 200:
|
||||
logger.exception("Something went wrong getting workflow status for %s with auth %s. Raw: %s. Status: %d" % (self.execution_id, self.authorization, ret.text, ret.status_code))
|
||||
time.sleep(sleeptime)
|
||||
continue
|
||||
|
||||
# PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
|
||||
# FIXME - have this?
|
||||
if ret.json()["status"] == "FINISHED" or ret.json()["status"] == "ABORTED" or ret.json()["status"] == "FAILURE":
|
||||
print("Entire thing is done with status %s - exiting" % ret.json()["status"])
|
||||
return True
|
||||
|
||||
self.results = ret.json()
|
||||
|
||||
# FIXME - REMOVE COMMENTS
|
||||
# FIXME - This might be wrong for multiple reasons
|
||||
if self.results.get("results") == "" or self.results.get("results") == None:
|
||||
print("Couldn't find results in results - getting new")
|
||||
logger.info(self.results)
|
||||
self.results["results"] = []
|
||||
#print("IS IT DONE? - RETURNING TRUE")
|
||||
#return
|
||||
|
||||
for node_message in self.results["results"]:
|
||||
# Ensure that the received NodeStatusMessage is for an action we launched
|
||||
#print(node_message)
|
||||
#print(self.in_process)
|
||||
# FIXME - might be an issue with same kind of node with same ID here
|
||||
if node_message["action"]["id"] in self.in_process:
|
||||
if node_message["status"] == "EXECUTING":
|
||||
logger.info(f"Got EXECUTING result for: {node_message['action']['name']}-{node_message['execution_id']}")
|
||||
elif node_message["status"] == "WAITING":
|
||||
# This is just for user-inputted items
|
||||
logger.info("Should only be here the SECOND time around (after user inputted)!")
|
||||
logger.info(f"Got WAITING result for: {node_message['action']['name']}-{node_message['execution_id']}. Updating it to SUCCESS now that a user continued.")
|
||||
self.accumulator[node_message["action"]["id"]] = "SUCCESS"
|
||||
self.in_process.pop(node_message["action"]["id"], None)
|
||||
|
||||
logger.debug("start_id: %s, node.id: %s", self.start_id, node_message["action"]["id"])
|
||||
if self.start_id == node_message["action"]["id"]:
|
||||
logger.info("HANDLING USER INPUT AS START NODE - SETTING TO SUCCESS!")
|
||||
# Check if its the same, then update it to success
|
||||
headers = {
|
||||
"Authorization": "Bearer %s" % APIKEY,
|
||||
"Content-Type": "application/json",
|
||||
}
|
||||
|
||||
# Set it to successful here?
|
||||
actionurl = "%s/api/v1/streams" % apiurl
|
||||
action_result = node_message
|
||||
action_result["status"] = "SUCCESS"
|
||||
action_result["authorization"] = self.authorization
|
||||
action_result["completed_at"] = int(time.time())
|
||||
action_result["result"] = "User clicked continue!"
|
||||
actionret = requests.post(actionurl, headers=headers, json=action_result)
|
||||
elif node_message["status"] == "SKIPPED":
|
||||
# FIXME - handle SKIPPED - these are
|
||||
logger.info(f"GOT SKIPPEED result for: {node_message['action']['name']}-{node_message['execution_id']}")
|
||||
|
||||
elif node_message["status"] == "SUCCESS":
|
||||
# Adds the data to accumulator with success AND
|
||||
# removes the successful ones, which breaks the loop
|
||||
self.accumulator[node_message["action"]["id"]] = node_message["result"]
|
||||
logger.info(f"Worker received result for: {node_message['action']['name']}-{node_message['execution_id']}: {node_message['result']}")
|
||||
self.in_process.pop(node_message["action"]["id"], None)
|
||||
elif node_message["status"] == "FAILURE":
|
||||
self.accumulator[node_message["action"]["id"]] = node_message["result"]
|
||||
|
||||
# FIXME - cancel nodes
|
||||
#await self.cancel_subgraph(self.workflow.nodes[node_message.node_id]) # kill the children!
|
||||
logger.info(f"Worker received error \"{node_message['result']}\" for: {node_message['action']['name']}-"
|
||||
f"{node_message['execution_id']}")
|
||||
|
||||
else:
|
||||
logger.error(f"Unknown message status received: {node_message}")
|
||||
node_message = None
|
||||
|
||||
time.sleep(sleeptime)
|
||||
|
||||
return False
|
||||
|
||||
def abort(message, workflow_id, execution_id):
|
||||
logger.info("ABORTING %s BECAUSE OF ERROR WITH FUNCTION STARTUP" % execution_id)
|
||||
logger.info("Message: %s" % message)
|
||||
url = f"{apiurl}/api/v1/workflows/{workflow_id}/executions/{execution_id}/abort"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
"Authorization": f"Bearer {APIKEY}"
|
||||
}
|
||||
|
||||
ret = requests.get(url, headers=headers, timeout=5)
|
||||
logger.info("Aborted with status: %d and text:\n%s" % (ret.status_code, ret.text))
|
||||
sys.exit(0)
|
||||
|
||||
def run_function(message):
|
||||
messagedata = message
|
||||
|
||||
# Raise exception?
|
||||
if messagedata["type"] != "workflow":
|
||||
return f"Wrong type" % e, 500
|
||||
|
||||
# Required fields
|
||||
execution_id = messagedata["execution_id"]
|
||||
workflow_id = messagedata["workflow_id"]
|
||||
|
||||
# FIXME - add exception handler -> abort
|
||||
workflow = workflow_loads(json.dumps(messagedata["workflow"]))
|
||||
|
||||
id = messagedata["workflow"]["id"]
|
||||
locations = messagedata["locations"]
|
||||
project_id = messagedata["project_id"]
|
||||
authorization = messagedata["authorization"]
|
||||
execution_id = messagedata["execution_id"]
|
||||
workflow_id = messagedata["workflow_id"]
|
||||
|
||||
logger.info("Exec_id: %s, authorization: %s" % (execution_id, authorization))
|
||||
|
||||
if execution_id == None:
|
||||
logger.info("NO EXECUTION ID")
|
||||
abort("NO EXECUTION ID", workflow_id, execution_id)
|
||||
|
||||
if len(locations) <= 0:
|
||||
logger.info("NO LOCATIONS")
|
||||
abort("NO LOCATIONS", workflow_id, execution_id)
|
||||
if not project_id:
|
||||
logger.info("NO PROJECT_ID")
|
||||
abort("NO PROJECT_ID", workflow_id, execution_id)
|
||||
if not authorization:
|
||||
logger.info("NO AUTHORIZATION")
|
||||
abort("NO AUTHORIZATION", workflow_id, execution_id)
|
||||
if not workflow_id:
|
||||
logger.info("NO workflow_id")
|
||||
abort("NO WORKFLOW_ID", workflow_id, execution_id)
|
||||
|
||||
worker = Worker(workflow)
|
||||
worker.locations = locations
|
||||
worker.execution_id = execution_id
|
||||
worker.id = id
|
||||
worker.project_id = project_id
|
||||
worker.authorization = authorization
|
||||
worker.workflow_id = workflow_id
|
||||
|
||||
try:
|
||||
worker.start_id = messagedata["start"]
|
||||
logger.debug("Start node is %s!" % messagedata["start"])
|
||||
except KeyError:
|
||||
try:
|
||||
worker.start_id = messagedata["workflow"]["start"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
logger.info("STARTING EXECUTION TASK FOR %s" % execution_id)
|
||||
try:
|
||||
worker.execution_task = worker.execute_workflow_pubsub()
|
||||
except Exception as e:
|
||||
logger.error("Execution exception: %s" % e)
|
||||
abort(e, workflow_id, execution_id)
|
||||
|
||||
# def abort(self):
|
||||
|
||||
logger.info(worker.execution_task)
|
||||
return f"OK", 200
|
||||
|
||||
def authorization(data, context):
|
||||
logger.info("JUST STARTED")
|
||||
|
||||
# Rofl
|
||||
import base64
|
||||
data = base64.b64decode(data['data']).decode('utf-8')
|
||||
return main(data)
|
||||
|
||||
def main(data):
|
||||
import argparse
|
||||
|
||||
LOG_LEVELS = ("debug", "info", "error", "warn", "fatal", "DEBUG", "INFO", "ERROR", "WARN", "FATAL")
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--log-level", dest="log_level", choices=LOG_LEVELS, default="DEBUG")
|
||||
parser.add_argument("--debug", "-d", dest="debug", action="store_true",
|
||||
help="Enables debug level logging for the umpire as well as asyncio debug mode.")
|
||||
args = parser.parse_args()
|
||||
|
||||
logger.setLevel(args.log_level.upper())
|
||||
logger.info("STARTED")
|
||||
|
||||
if isinstance(data, str):
|
||||
data = json.loads(data)
|
||||
|
||||
return run_function(data)
|
||||
|
||||
def test():
|
||||
# Used for testing
|
||||
with open("data.json", "r") as tmp:
|
||||
print(main(tmp.read()))
|
||||
|
||||
if __name__ == "__main__":
|
||||
test()
|
||||
@@ -0,0 +1,214 @@
|
||||
import enum
|
||||
import json
|
||||
import datetime
|
||||
|
||||
|
||||
def message_dumps(obj):
|
||||
return json.dumps(obj, cls=MessageJSONEncoder)
|
||||
|
||||
|
||||
def message_loads(obj):
|
||||
return json.loads(obj, cls=MessageJSONDecoder)
|
||||
|
||||
|
||||
def message_dump(obj, fp):
|
||||
return json.dump(obj, fp, cls=MessageJSONEncoder)
|
||||
|
||||
|
||||
def message_load(obj):
|
||||
return json.load(obj, cls=MessageJSONDecoder)
|
||||
|
||||
|
||||
class MessageJSONDecoder(json.JSONDecoder):
|
||||
""" A custom decoder for decoding JSON strings to Message types. """
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
json.JSONDecoder.__init__(self, object_hook=self.object_hook, *args, **kwargs)
|
||||
|
||||
def object_hook(self, o):
|
||||
if "result" in o and "app_name" in o:
|
||||
o["status"] = StatusEnum[o["status"]]
|
||||
return NodeStatusMessage(**o)
|
||||
|
||||
elif "workflow_id" in o and "execution_id" in o:
|
||||
o["status"] = StatusEnum[o["status"]]
|
||||
return WorkflowStatusMessage(**o)
|
||||
|
||||
elif "trigger_data" in o:
|
||||
return TriggerMessage(**o)
|
||||
|
||||
else:
|
||||
return o
|
||||
|
||||
|
||||
class MessageJSONEncoder(json.JSONEncoder):
|
||||
""" A custom encoder for encoding Message types to JSON strings. """
|
||||
|
||||
def default(self, o):
|
||||
if isinstance(o, NodeStatusMessage):
|
||||
r = {"name": o.name, "node_id": o.node_id, "label": o.label, "app_name": o.app_name,
|
||||
"execution_id": o.execution_id, "result": o.result, "status": o.status,
|
||||
"started_at": o.started_at, "completed_at": o.completed_at, "combined_id": o.combined_id,
|
||||
"parameters": o.parameters}
|
||||
|
||||
try:
|
||||
json.dumps(o.result)
|
||||
except (TypeError, ValueError):
|
||||
r["result"] = f"Node returned result of type '{type(o.result)}' which is not JSON serializable."
|
||||
r["status"] = StatusEnum.FAILURE
|
||||
finally:
|
||||
return r
|
||||
|
||||
elif isinstance(o, WorkflowStatusMessage):
|
||||
return {"execution_id": o.execution_id, "workflow_id": o.workflow_id, "name": o.name, "status": o.status,
|
||||
"started_at": o.started_at, "completed_at": o.completed_at, "user": o.user}
|
||||
|
||||
elif isinstance(o, TriggerMessage):
|
||||
return {"trigger_data": o.trigger_data}
|
||||
|
||||
elif isinstance(o, JSONPatch):
|
||||
if o.op in JSONPatchOps:
|
||||
return {k: getattr(o, k, None) for k in o.__slots__ if getattr(o, k, None) is not None}
|
||||
else:
|
||||
raise ValueError("Improper JSON Patch operation")
|
||||
|
||||
elif isinstance(o, StatusEnum):
|
||||
return o.value
|
||||
|
||||
elif isinstance(o, JSONPatchOps):
|
||||
return o.value.lower()
|
||||
|
||||
elif isinstance(o, JSONPatch):
|
||||
return {k: getattr(o, k, None) for k in o.__slots__ if getattr(o, k, None) is not None}
|
||||
|
||||
elif isinstance(o, datetime.datetime):
|
||||
return str(o)
|
||||
|
||||
else:
|
||||
return o
|
||||
|
||||
|
||||
class JSONPatch:
|
||||
__slots__ = ("op", "path", "value", "from_")
|
||||
|
||||
def __init__(self, op=None, path=None, value=None, from_=None):
|
||||
self.op = op
|
||||
self.path = path
|
||||
self.value = value
|
||||
self.from_ = from_
|
||||
|
||||
|
||||
class JSONPatchOps(enum.Enum):
|
||||
TEST = "TEST"
|
||||
REMOVE = "REMOVE"
|
||||
ADD = "ADD"
|
||||
REPLACE = "REPLACE"
|
||||
MOVE = "MOVE"
|
||||
COPY = "COPY"
|
||||
|
||||
|
||||
class StatusEnum(enum.Enum):
|
||||
""" Holds statuses used for Workflow and Action status messages """
|
||||
PAUSED = "PAUSED" # not currently implemented but may be if we see a use case
|
||||
AWAITING_DATA = "AWAITING_DATA" # possibly for triggers?
|
||||
PENDING = "PENDING"
|
||||
COMPLETED = "COMPLETED"
|
||||
ABORTED = "ABORTED"
|
||||
EXECUTING = "EXECUTING"
|
||||
SUCCESS = "SUCCESS"
|
||||
FAILURE = "FAILURE"
|
||||
|
||||
|
||||
class WorkflowStatusMessage(object):
|
||||
""" Class that formats a WorkflowStatusMessage message """
|
||||
__slots__ = ("execution_id", "workflow_id", "name", "status", "started_at", "completed_at", "user")
|
||||
|
||||
def __init__(self, execution_id, workflow_id, name, started_at=None, completed_at=None, status=None, user=None):
|
||||
self.execution_id = execution_id
|
||||
self.workflow_id = workflow_id
|
||||
self.name = name
|
||||
self.status = status
|
||||
self.started_at = started_at
|
||||
self.completed_at = completed_at
|
||||
self.user = user
|
||||
|
||||
@classmethod
|
||||
def execution_pending(cls, execution_id, workflow_id, name, user=None):
|
||||
return cls(execution_id, workflow_id, name, status=StatusEnum.PENDING, user=user)
|
||||
|
||||
@classmethod
|
||||
def execution_started(cls, execution_id, workflow_id, name, user=None):
|
||||
start_time = datetime.datetime.now()
|
||||
return cls(execution_id, workflow_id, name, started_at=start_time, status=StatusEnum.EXECUTING, user=user)
|
||||
|
||||
@classmethod
|
||||
def execution_completed(cls, execution_id, workflow_id, name, user=None):
|
||||
end_time = datetime.datetime.now()
|
||||
return cls(execution_id, workflow_id, name, completed_at=end_time, status=StatusEnum.COMPLETED, user=user)
|
||||
|
||||
@classmethod
|
||||
def execution_aborted(cls, execution_id, workflow_id, name, user=None):
|
||||
end_time = datetime.datetime.now()
|
||||
return cls(execution_id, workflow_id, name, completed_at=end_time, status=StatusEnum.ABORTED, user=user)
|
||||
|
||||
|
||||
class NodeStatusMessage(object):
|
||||
""" Class that formats a NodeStatusMessage message. """
|
||||
__slots__ = ("name", "node_id", "label", "app_name", "execution_id", "parameters", "combined_id", "result",
|
||||
"status", "started_at", "completed_at")
|
||||
|
||||
def __init__(self, name, node_id, label, app_name, execution_id, combined_id=None, parameters=None, result=None,
|
||||
status=None, started_at=None, completed_at=None):
|
||||
self.name = name
|
||||
self.node_id = node_id
|
||||
self.label = label
|
||||
self.app_name = app_name
|
||||
self.execution_id = execution_id
|
||||
self.combined_id = combined_id if combined_id is not None else ':'.join((node_id, execution_id))
|
||||
|
||||
self.result = result
|
||||
self.parameters = parameters
|
||||
self.status = status
|
||||
self.started_at = started_at
|
||||
self.completed_at = completed_at
|
||||
|
||||
@classmethod
|
||||
def from_node(cls, node, execution_id, result=None, status=None, started_at=None, completed_at=None, parameters=None):
|
||||
return cls(node.name, node.id, node.label, node.app_name, execution_id, result=result,
|
||||
status=status, started_at=started_at, completed_at=completed_at, parameters=parameters)
|
||||
|
||||
@classmethod
|
||||
def pending_from_node(cls, node, execution_id, parameters=None):
|
||||
return NodeStatusMessage.from_node(node, execution_id, status=StatusEnum.PENDING, parameters=parameters)
|
||||
|
||||
@classmethod
|
||||
def executing_from_node(cls, node, execution_id, parameters=None):
|
||||
started_at = datetime.datetime.now()
|
||||
return NodeStatusMessage.from_node(node, execution_id, started_at=started_at, status=StatusEnum.EXECUTING,
|
||||
parameters=parameters)
|
||||
|
||||
@classmethod
|
||||
def success_from_node(cls, node, execution_id, result, parameters=None):
|
||||
completed_at = datetime.datetime.now()
|
||||
return NodeStatusMessage.from_node(node, execution_id, result=result, completed_at=completed_at,
|
||||
status=StatusEnum.SUCCESS, parameters=parameters)
|
||||
|
||||
@classmethod
|
||||
def failure_from_node(cls, node, execution_id, result, parameters=None):
|
||||
completed_at = datetime.datetime.now()
|
||||
return NodeStatusMessage.from_node(node, execution_id, result=result, completed_at=completed_at,
|
||||
status=StatusEnum.FAILURE, parameters=parameters)
|
||||
|
||||
@classmethod
|
||||
def aborted_from_node(cls, node, execution_id, parameters=None):
|
||||
completed_at = datetime.datetime.now()
|
||||
return NodeStatusMessage.from_node(node, execution_id, result=None, completed_at=completed_at,
|
||||
status=StatusEnum.ABORTED, parameters=parameters)
|
||||
|
||||
|
||||
class TriggerMessage(object):
|
||||
""" Class that formats a TriggerMessage. """
|
||||
__slots__ = ("trigger_data",)
|
||||
|
||||
def __init__(self, trigger_data):
|
||||
self.trigger_data = trigger_data
|
||||
@@ -0,0 +1,40 @@
|
||||
import logging
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import aioredis
|
||||
|
||||
logger = logging.getLogger("WALKOFF")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def connect_to_redis_pool(redis_uri) -> aioredis.Redis:
|
||||
# Redis client bound to pool of connections (auto-reconnecting).
|
||||
redis = await aioredis.create_redis_pool(redis_uri)
|
||||
try:
|
||||
yield redis
|
||||
finally:
|
||||
# gracefully close pool
|
||||
redis.close()
|
||||
await redis.wait_closed()
|
||||
logger.info("Redis connection pool closed.")
|
||||
|
||||
|
||||
def deref_stream_message(message):
|
||||
try:
|
||||
key, value = message[0][-1].popitem()
|
||||
stream = message[0][0]
|
||||
id = message[0][1]
|
||||
return (key, value), stream, id
|
||||
|
||||
except:
|
||||
logger.exception("Stream message formatted incorrectly.")
|
||||
|
||||
|
||||
def xlen(redis: aioredis.Redis, key):
|
||||
"""Returns the number of entries inside a stream."""
|
||||
return redis.execute(b'XLEN', key)
|
||||
|
||||
|
||||
def xdel(redis: aioredis.Redis, stream, id):
|
||||
""" Deletes id from stream. Returns the number of items deleted. """
|
||||
return redis.execute(b'XDEL', stream, id)
|
||||
@@ -0,0 +1,12 @@
|
||||
aiodns
|
||||
aiodocker
|
||||
aioredis
|
||||
aiohttp
|
||||
cchardet
|
||||
docker
|
||||
docker-compose
|
||||
pyyaml
|
||||
sqlalchemy
|
||||
asteval
|
||||
argparse
|
||||
google-cloud-pubsub
|
||||
@@ -0,0 +1,548 @@
|
||||
import uuid
|
||||
import json
|
||||
import enum
|
||||
import logging
|
||||
from operator import attrgetter, itemgetter
|
||||
from collections import namedtuple, deque
|
||||
from asteval import Interpreter, make_symbol_table
|
||||
|
||||
logger = logging.getLogger("WALKOFF")
|
||||
|
||||
|
||||
def workflow_dumps(obj):
|
||||
return json.dumps(obj, cls=WorkflowJSONEncoder)
|
||||
|
||||
|
||||
def workflow_loads(obj):
|
||||
return json.loads(obj, cls=WorkflowJSONDecoder)
|
||||
|
||||
|
||||
def workflow_dump(obj, fp):
|
||||
return json.dump(obj, fp, cls=WorkflowJSONEncoder)
|
||||
|
||||
|
||||
def workflow_load(obj, fp):
|
||||
return json.load(obj, fp, cls=WorkflowJSONDecoder)
|
||||
|
||||
|
||||
def attrs_equal(self, other):
|
||||
attr_getters = (attrgetter(attr) for attr in self.__slots__)
|
||||
return all(attr_getter(self) == attr_getter(other) for attr_getter in attr_getters)
|
||||
|
||||
|
||||
class ConditionException(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class WorkflowJSONDecoder(json.JSONDecoder):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(object_hook=self.object_hook, *args, **kwargs)
|
||||
self.nodes = {}
|
||||
self.branches = set()
|
||||
|
||||
def object_hook(self, o):
|
||||
if "x" in o and "y" in o:
|
||||
return Point(**o)
|
||||
|
||||
elif "parameters" in o and "priority" in o:
|
||||
node = Action(**o)
|
||||
self.nodes[node.id] = node
|
||||
return node
|
||||
|
||||
elif "variant" in o:
|
||||
try:
|
||||
o["variant"] = ParameterVariant[o["variant"]]
|
||||
return Parameter(**o)
|
||||
except KeyError:
|
||||
o["variant"] = "STATIC_VALUE"
|
||||
return Parameter(**o)
|
||||
|
||||
elif "source_id" in o and "destination_id" in o:
|
||||
self.branches.add(Branch(source_id=o["source_id"], destination_id=o["destination_id"], id=o["id"]))
|
||||
|
||||
elif "conditional" in o:
|
||||
node = Condition(**o)
|
||||
self.nodes[node.id] = node
|
||||
return node
|
||||
|
||||
elif "transform" in o:
|
||||
node = Transform(**o)
|
||||
self.nodes[node.id] = node
|
||||
return node
|
||||
|
||||
elif "trigger_schema" in o:
|
||||
node = Trigger(**o)
|
||||
self.nodes[node.id] = node
|
||||
return node
|
||||
|
||||
elif "description" in o and "value" in o:
|
||||
return Variable(**o)
|
||||
|
||||
elif "actions" in o and "branches" in o:
|
||||
branches = {Branch(self.nodes[b.source_id], self.nodes[b.destination_id], b.id) for b in self.branches}
|
||||
|
||||
try:
|
||||
workflow_variables = {var.id: var for var in o["workflow_variables"]}
|
||||
except:
|
||||
workflow_variables = {}
|
||||
if o["workflow_variables"] != None:
|
||||
for var in o["workflow_variables"]:
|
||||
workflow_obj = Variable(id=var["id"], name=var["name"], value=var["value"])
|
||||
workflow_variables[workflow_obj.id] = workflow_obj
|
||||
|
||||
start = self.nodes[o["start"]]
|
||||
o["branches"] = branches
|
||||
o["workflow_variables"] = workflow_variables
|
||||
o["start"] = start
|
||||
return Workflow(**o)
|
||||
|
||||
else:
|
||||
return o
|
||||
|
||||
|
||||
class WorkflowJSONEncoder(json.JSONEncoder):
|
||||
""" A custom encoder for encoding Workflow types to JSON strings.
|
||||
Note: JSON encoded strings of our custom objects are lossy...for now.
|
||||
"""
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.workflow = {}
|
||||
|
||||
def default(self, o):
|
||||
if isinstance(o, Workflow):
|
||||
# Unpack the adjacency matrix into edges
|
||||
branches = [{"source_id": src.id, "destination_id": dst.id} for src, dsts in o.edges.items()
|
||||
for dst in dsts]
|
||||
branches.sort(key=itemgetter("source_id", "destination_id"))
|
||||
actions = [action for action in o.actions]
|
||||
triggers = [trigger for trigger in o.triggers]
|
||||
workflow_variables = list(o.workflow_variables.values())
|
||||
return {"id": o.id, "execution_id": o.execution_id, "name": o.name, "start": o.start.id,
|
||||
"actions": actions, "branches": branches,
|
||||
"triggers": triggers, "workflow_variables": workflow_variables, "is_valid": o.is_valid,
|
||||
"errors": None}
|
||||
|
||||
elif isinstance(o, Action):
|
||||
position = {"x": o.position.x, "y": o.position.y}
|
||||
return {"id": o.id, "name": o.name, "app_name": o.app_name, "app_version": o.app_version,
|
||||
"label": o.label, "position": position, "parameters": o.parameters, "priority": o.priority,
|
||||
"execution_id": o.execution_id}
|
||||
|
||||
elif isinstance(o, Condition):
|
||||
position = {"x": o.position.x, "y": o.position.y}
|
||||
return {"id": o.id, "name": o.name, "app_name": o.app_name, "app_version": o.app_version,
|
||||
"label": o.label, "position": position, "conditional": o.conditional}
|
||||
|
||||
elif isinstance(o, Transform):
|
||||
position = {"x": o.position.x, "y": o.position.y}
|
||||
return {"id": o.id, "name": o.name, "app_name": o.app_name, "app_version": o.app_version,
|
||||
"label": o.label, "position": position, "transform": o.transform, "parameter": o.parameter}
|
||||
|
||||
elif isinstance(o, Trigger):
|
||||
position = {"x": o.position.x, "y": o.position.y}
|
||||
return {"id": o.id, "name": o.name, "app_name": o.app_name, "app_version": o.app_version,
|
||||
"label": o.label, "position": position, "trigger_schema": o.trigger_schema}
|
||||
|
||||
elif isinstance(o, Parameter):
|
||||
return {"name": o.name, "variant": o.variant, "value": o.value, "id": o.id}
|
||||
|
||||
elif isinstance(o, ParameterVariant):
|
||||
return o.value
|
||||
|
||||
elif isinstance(o, Variable):
|
||||
return {"description": o.description, "id": o.id, "name": o.name, "value": o.value}
|
||||
|
||||
else:
|
||||
return o
|
||||
|
||||
|
||||
Point = namedtuple("Point", ("x", "y"))
|
||||
Branch = namedtuple("Branch", ("source_id", "destination_id", "id"))
|
||||
ParentSymbol = namedtuple("ParentSymbol", "result") # used inside conditions to further mask the parent node attrs
|
||||
ChildSymbol = namedtuple("ChildSymbol", "id") # used inside conditions to further mask the child node attrs
|
||||
|
||||
|
||||
class ParameterVariant(enum.Enum):
|
||||
STATIC_VALUE = "STATIC_VALUE"
|
||||
ACTION_RESULT = "ACTION_RESULT"
|
||||
WORKFLOW_VARIABLE = "WORKFLOW_VARIABLE"
|
||||
GLOBAL = "GLOBAL"
|
||||
|
||||
|
||||
class Parameter:
|
||||
__slots__ = ("name", "value", "selection", "variant", "id", "errors", "parallelized", "description", "required", "schema", "action_field", "multiline", "example")
|
||||
|
||||
def __init__(self, name, parallelized=False, selection=[], id=None, value=None, variant=None, errors=None, description="", required=False, schema={}, action_field="", multiline=False, example=""):
|
||||
self.id = id
|
||||
self.name = name
|
||||
self.description = description
|
||||
self.required = required
|
||||
self.parallelized = parallelized
|
||||
self.selection = selection
|
||||
self.value = value
|
||||
self.variant = variant
|
||||
self.schema = schema
|
||||
self.errors = errors
|
||||
self.action_field = action_field
|
||||
self.multiline = multiline
|
||||
self.example = example
|
||||
|
||||
def __str__(self):
|
||||
return f"Parameter-{self.name}:{self.value}"
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, Parameter) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
|
||||
class Variable:
|
||||
"""
|
||||
A lightweight class representing a WALKOFF WorkflowVariable or Global
|
||||
"""
|
||||
__slots__ = ("id", "name", "value", "description")
|
||||
|
||||
def __init__(self, id, name, value, description=None):
|
||||
self.id = id
|
||||
self.name = name
|
||||
self.value = value
|
||||
self.description = description
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
|
||||
class Node:
|
||||
__slots__ = ("id", "name", "app_name", "app_version", "label", "position", "priority", "errors", "is_valid", "parameters")
|
||||
|
||||
def __init__(self, name, position: Point, label, app_name, app_version, parameters=[], id=None, errors=None, is_valid=True, environment="cloud"):
|
||||
self.id = id if id is not None else str(uuid.uuid4())
|
||||
self.is_valid = is_valid # ToDo: Is this neccessary?
|
||||
self.name = name
|
||||
self.environment = environment
|
||||
self.app_name = app_name
|
||||
self.app_version = app_version
|
||||
self.label = label
|
||||
self.parameters = parameters
|
||||
self.position = position
|
||||
self.errors = errors if errors is not None else []
|
||||
|
||||
if hasattr(self, "priority"):
|
||||
msg = f"Call super().__init__() prior to setting self.priority in Node subclass {self.__class__.__name__}"
|
||||
logger.warning(msg)
|
||||
else:
|
||||
self.priority = 3 # initialize this to mid level for non-Action node types
|
||||
|
||||
def __repr__(self):
|
||||
return f"Node-{self.id}"
|
||||
|
||||
def __str__(self):
|
||||
return f"Node-{self.label}"
|
||||
|
||||
def __gt__(self, other):
|
||||
return self.priority > other.priority
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
|
||||
class Action(Node):
|
||||
__slots__ = ("parameters", "execution_id", "parallelized", "environment", "authentication", "sharing", "private_id")
|
||||
|
||||
def __init__(self, name, position, app_name, app_version, label, priority, environment, sharing=False, private_id="", verified=False, parallelized=False, parameters=None, id=None, execution_id=None, errors=None, is_valid=None, authentication=[], app_id="", **kwargs):
|
||||
super().__init__(name, position, label, app_name, app_version, id=id, errors=errors, is_valid=is_valid)
|
||||
self.parameters = parameters if parameters is not None else list()
|
||||
self.parallelized = parallelized
|
||||
self.priority = priority
|
||||
self.execution_id = execution_id
|
||||
self.authentication = authentication
|
||||
|
||||
self.sharing = sharing
|
||||
self.private_id = private_id
|
||||
|
||||
def __str__(self):
|
||||
return f"Action: {self.label}::{self.id}"
|
||||
|
||||
def __repr__(self):
|
||||
return f"Action: {self.label}::{self.id}"
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
|
||||
class Condition(Node):
|
||||
__slots__ = ("conditional",)
|
||||
|
||||
def __init__(self, name, position: Point, app_name, app_version, label, conditional, id=None, errors=None,
|
||||
is_valid=None):
|
||||
super().__init__(name, position, label, app_name, app_version, id, errors, is_valid)
|
||||
self.conditional = conditional
|
||||
self.priority = 3 # Conditions have a fixed, mid valued priority
|
||||
|
||||
def __str__(self):
|
||||
return f"Condition: {self.label}::{self.id}"
|
||||
|
||||
def __repr__(self):
|
||||
return f"Condition: {self.label}::{self.id}"
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
@staticmethod
|
||||
def format_node_names(nodes):
|
||||
# We need to format space delimited names into underscore delimited names
|
||||
names_to_modify = {node.label for node in nodes.values() if node.label.count(' ') > 0}
|
||||
formatted_nodes = {}
|
||||
for node in nodes.values():
|
||||
formatted_name = node.label.strip().replace(' ', '_')
|
||||
|
||||
if formatted_name in names_to_modify: # we have to check for a name conflict as described above
|
||||
logger.error(f"Error processing condition. {node.label} or {formatted_name} must be renamed.")
|
||||
|
||||
formatted_nodes[formatted_name] = node
|
||||
return formatted_nodes
|
||||
|
||||
def __call__(self, parents, children, accumulator) -> str:
|
||||
parent_symbols = {k: ParentSymbol(accumulator[v.id]) for k, v in self.format_node_names(parents).items()}
|
||||
children_symbols = {k: ChildSymbol(v.id) for k, v in self.format_node_names(children).items()}
|
||||
syms = make_symbol_table(use_numpy=False, **parent_symbols, **children_symbols)
|
||||
aeval = Interpreter(usersyms=syms, no_for=True, no_while=True, no_try=True, no_functiondef=True, no_ifexp=True,
|
||||
no_listcomp=True, no_augassign=True, no_assert=True, no_delete=True, no_raise=True,
|
||||
no_print=True, use_numpy=False, builtins_readonly=True,
|
||||
readonly_symbols=children_symbols.keys())
|
||||
|
||||
aeval(self.conditional)
|
||||
child_id = getattr(aeval.symtable.get("selected_node", None), "id", None)
|
||||
|
||||
if len(aeval.error) > 0:
|
||||
raise ConditionException
|
||||
|
||||
return child_id
|
||||
|
||||
|
||||
class Trigger(Node):
|
||||
__slots__ = ("trigger_schema",)
|
||||
|
||||
def __init__(self, name, position: Point, app_name, app_version, label, trigger_schema, id=None, errors=None,
|
||||
is_valid=None):
|
||||
super().__init__(name, position, label, app_name, app_version, id, errors, is_valid)
|
||||
self.trigger_schema = trigger_schema
|
||||
|
||||
def __str__(self):
|
||||
return f"Trigger: {self.label}::{self.id}"
|
||||
|
||||
def __repr__(self):
|
||||
return f"Trigger: {self.label}::{self.id}"
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
def __call__(self, data):
|
||||
""" A trigger simply echos the data it was given """
|
||||
result = data.trigger_data
|
||||
logger.debug(f"Executed {self.name}-{self.id} with result: {result}")
|
||||
return result
|
||||
|
||||
|
||||
class Transform(Node):
|
||||
__slots__ = ("transform", "parameter")
|
||||
|
||||
def __init__(self, name, position: Point, app_name, app_version, label, transform, parameter=None, id=None,
|
||||
errors=None, is_valid=None):
|
||||
super().__init__(name, position, label, app_name, app_version, id, errors, is_valid)
|
||||
self.transform = transform.lower()
|
||||
self.parameter = parameter
|
||||
self.priority = 3 # Transforms have a fixed, mid valued priority
|
||||
|
||||
def __str__(self):
|
||||
return f"Transform: {self.label}::{self.id}"
|
||||
|
||||
def __repr__(self):
|
||||
return f"Transform: {self.label}::{self.id}"
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
def __call__(self, data):
|
||||
""" Execute an action and ship its result """
|
||||
logger.debug(f"Attempting execution of: {self.name}-{self.id}")
|
||||
transform = f"_{self.__class__.__name__}__{self.transform}"
|
||||
if hasattr(self, transform):
|
||||
if self.parameter is None:
|
||||
result = getattr(self, transform)(data=data)
|
||||
else:
|
||||
result = getattr(self, transform)(self.parameter, data=data)
|
||||
logger.debug(f"Executed {self.name}-{self.id} with result: {result}")
|
||||
return result
|
||||
else:
|
||||
logger.error(f"{self.__class__.__name__} has no method {self.transform}")
|
||||
|
||||
# TODO: add JSON to CSV parsing and vice versa.
|
||||
def __get_value_at_index(self, index, data=None):
|
||||
return data[index]
|
||||
|
||||
def __get_value_at_key(self, key, data=None):
|
||||
return data[key]
|
||||
|
||||
def __split_string_to_array(self, delimiter=' ', data=None):
|
||||
return data.split(delimiter)
|
||||
|
||||
|
||||
class DiGraph:
|
||||
__slots__ = ("nodes", "edges", "rev_adjacency")
|
||||
|
||||
def __init__(self, nodes, edges):
|
||||
self.nodes = {}
|
||||
self.add_nodes(nodes)
|
||||
self.edges = {node: set() for node in self.nodes.values()}
|
||||
self.rev_adjacency = {} # all edges inverted for quickly getting parents of a node
|
||||
self.add_edges(edges)
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
def add_edges(self, edges):
|
||||
try:
|
||||
iter(edges) # check we got an iterable
|
||||
if callable(getattr(edges, "items", None)): # check if it's a dictionary
|
||||
for src, dest in edges.items():
|
||||
if src in self.edges:
|
||||
self.edges[src].add(dest)
|
||||
else: # This edge introduces new nodes so lets add them
|
||||
self.nodes[src.id] = src
|
||||
self.nodes[dest.id] = dest
|
||||
self.edges[src] = {dest}
|
||||
if dest in self.rev_adjacency:
|
||||
self.edges[dest].add(src)
|
||||
else:
|
||||
self.edges[dest] = {src}
|
||||
else: # it's a different iterable
|
||||
for edge in edges:
|
||||
if not (isinstance(edge, Branch) or (isinstance(edge, tuple) and not len(edge) == 2)):
|
||||
raise TypeError # it must be an iterable of (src, dest) edges
|
||||
src = edge[0]
|
||||
dest = edge[1]
|
||||
if src in self.edges:
|
||||
self.edges[src].add(dest)
|
||||
else:
|
||||
self.edges[src] = {dest}
|
||||
|
||||
if dest in self.rev_adjacency:
|
||||
self.rev_adjacency[dest].add(src)
|
||||
else:
|
||||
self.rev_adjacency[dest] = {src}
|
||||
except TypeError:
|
||||
return
|
||||
|
||||
def add_edge(self, src, dest):
|
||||
self.add_edges({src, dest})
|
||||
|
||||
def add_nodes(self, nodes):
|
||||
self.nodes = {node.id: node for node in nodes}
|
||||
|
||||
def add_node(self, node):
|
||||
return self.add_nodes([node])
|
||||
|
||||
def successors(self, node):
|
||||
return self.edges[node]
|
||||
|
||||
def predecessors(self, node):
|
||||
return self.rev_adjacency[node]
|
||||
|
||||
|
||||
# TODO: Maybe look into pooling nodes/branches and sharing them across a workflow to save memory?
|
||||
class Workflow(DiGraph):
|
||||
__slots__ = ("start", "id", "id", "is_valid", "name", "execution_id", "workflow_variables",
|
||||
"triggers", "actions", "errors", "description", "tags", "owner", "org", "execution_org", "schedules", "sharing")
|
||||
|
||||
def __init__(self, name, start, actions: [Action], branches: [Branch], workflow_variables=[],
|
||||
triggers=[], id=None, execution_id=None,
|
||||
is_valid=None, errors=None, description=None, tags=None, owner={}, org={}, execution_org={}, schedules=[], sharing="private"):
|
||||
super().__init__(nodes=[*actions, *triggers], edges=branches)
|
||||
|
||||
self.start = start
|
||||
self.id = id if id is not None else str(uuid.uuid4())
|
||||
self.is_valid = is_valid if is_valid is not None else self.validate()
|
||||
self.name = name
|
||||
self.execution_id = execution_id
|
||||
self.workflow_variables = workflow_variables if workflow_variables is not None else []
|
||||
self.triggers = triggers
|
||||
self.actions = actions
|
||||
self.errors = errors if errors is not None else []
|
||||
self.description = description
|
||||
self.owner = owner
|
||||
self.org = org
|
||||
self.execution_org = execution_org
|
||||
self.schedules=schedules
|
||||
self.tags = tags if tags is not None else []
|
||||
|
||||
def __eq__(self, other):
|
||||
if isinstance(other, self.__class__) and self.__slots__ == other.__slots__:
|
||||
return attrs_equal(self, other)
|
||||
return False
|
||||
|
||||
def __hash__(self):
|
||||
return hash(id(self))
|
||||
|
||||
def validate(self):
|
||||
# TODO: add in workflow validation from old implementation
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def dereference_environment_variables(data):
|
||||
return {ev["id"]: (ev["name"], ev["value"]) for ev in data.get("environment_variables", [])}
|
||||
|
||||
def get_dependents(self, node):
|
||||
"""
|
||||
BFS to get all nodes dependent on the current node. This includes the current node.
|
||||
"""
|
||||
visited = {node}
|
||||
queue = deque([node])
|
||||
|
||||
while queue:
|
||||
node = queue.pop()
|
||||
children = set(self.successors(node))
|
||||
for child in children:
|
||||
if child not in visited:
|
||||
queue.appendleft(child)
|
||||
visited.add(child)
|
||||
|
||||
return visited
|
||||
@@ -0,0 +1,40 @@
|
||||
# ONPREM code
|
||||
* Onprem means it's supposed to be ran on a server of the customer, and not in the cloud. These are tweaked to hit the API and look for new work throughout workflows. Everything is handled by the initial main.go, which launches the others.
|
||||
|
||||
## orborus.go - Handles NEW workflows - Same as WALKOFF UMPIRE
|
||||
* Executes and controls the docker environment used by workers.
|
||||
* A worker is deployed for every execution.
|
||||
* The apps are responsible for callbacks to the backend themselves.
|
||||
* After the worker is deployed / running, the execution ID is removed from the workflowqueue API.
|
||||
|
||||
# worker/worker.go - one for each workflow requiring onprem stuff
|
||||
* Handles a workflow from start to finish as long as the action ID.
|
||||
* Starting and stopping APPS in docker.
|
||||
|
||||
# app_sdk
|
||||
* The new APP sdk based on https://github.com/nsacyber/WALKOFF/tree/1.0.0-alpha.1/app_sdk
|
||||
* Fully functional with WALKOFF apps, which means its also functional with Cloud Function apps (these are now essentially the same with a few small tweaks)
|
||||
|
||||
# Images - all valid images are located here currently
|
||||
https://hub.docker.com/r/frikky/shuffle
|
||||
|
||||
## Setup with Dockerhub
|
||||
Requred - access to: https://hub.docker.com/r/docker/frikky/shuffle/general
|
||||
Login:
|
||||
```
|
||||
docker login
|
||||
```
|
||||
|
||||
Update worker:
|
||||
```
|
||||
cd worker
|
||||
docker build . -t frikky/shuffle:worker
|
||||
docker push frikky/shuffle:worker
|
||||
```
|
||||
|
||||
Update app_sdk:
|
||||
```
|
||||
cd app_sdk
|
||||
docker build . -t frikky/shuffle:app_sdk
|
||||
docker push frikky/shuffle:app_sdk
|
||||
```
|
||||
@@ -0,0 +1,16 @@
|
||||
FROM python:3.7-alpine as base
|
||||
|
||||
FROM base as builder
|
||||
RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev
|
||||
|
||||
RUN mkdir /install
|
||||
WORKDIR /install
|
||||
|
||||
COPY requirements.txt /requirements.txt
|
||||
RUN pip install --prefix="/install" -r /requirements.txt
|
||||
|
||||
FROM base
|
||||
|
||||
COPY --from=builder /install /usr/local
|
||||
COPY __init__.py /app/walkoff_app_sdk/__init__.py
|
||||
COPY app_base.py /app/walkoff_app_sdk/app_base.py
|
||||
@@ -0,0 +1,3 @@
|
||||
# app_sdk
|
||||
This is the SDK used for apps to behave like they should.
|
||||
To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline.
|
||||
@@ -0,0 +1,437 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import json
|
||||
import logging
|
||||
import requests
|
||||
|
||||
class AppBase:
|
||||
""" The base class for Python-based apps in Shuffle, handles logging and callbacks configurations"""
|
||||
__version__ = None
|
||||
app_name = None
|
||||
|
||||
def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None):
|
||||
self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger")
|
||||
self.redis=redis
|
||||
self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger")
|
||||
|
||||
# apikey is for the user / org
|
||||
# authorization is for the specific workflow
|
||||
self.url = os.getenv("CALLBACK_URL", "https://shuffler.io")
|
||||
self.action = os.getenv("ACTION", "")
|
||||
self.apikey = os.getenv("FUNCTION_APIKEY", "")
|
||||
self.authorization = os.getenv("AUTHORIZATION", "")
|
||||
self.current_execution_id = os.getenv("EXECUTIONID", "")
|
||||
|
||||
if len(self.action) == 0:
|
||||
print("ACTION env not defined")
|
||||
sys.exit(0)
|
||||
if len(self.apikey) == 0:
|
||||
print("FUNCTION_APIKEY env not defined")
|
||||
sys.exit(0)
|
||||
if len(self.authorization) == 0:
|
||||
print("AUTHORIZATION env not defined")
|
||||
sys.exit(0)
|
||||
if len(self.current_execution_id) == 0:
|
||||
print("EXECUTIONID env not defined")
|
||||
sys.exit(0)
|
||||
|
||||
if isinstance(self.action, str):
|
||||
self.action = json.loads(self.action)
|
||||
|
||||
async def execute_action(self, action):
|
||||
# FIXME - add request for the function STARTING here. Use "results stream" or something
|
||||
# PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
|
||||
|
||||
# !!! Let this line stay - its used for some horrible codegeneration / stitching !!! #
|
||||
#STARTCOPY
|
||||
stream_path = "/api/v1/streams"
|
||||
action_result = {
|
||||
"action": action,
|
||||
"authorization": self.authorization,
|
||||
"execution_id": self.current_execution_id,
|
||||
"result": "",
|
||||
"started_at": int(time.time()),
|
||||
"status": "EXECUTING"
|
||||
}
|
||||
self.logger.info("ACTION RESULT: %s", action_result)
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": "Bearer %s" % self.apikey
|
||||
}
|
||||
|
||||
# Add async logger
|
||||
# self.console_logger.handlers[0].stream.set_execution_id()
|
||||
self.logger.info("Before initial stream result")
|
||||
try:
|
||||
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
|
||||
self.logger.info("Workflow: %d" % ret.status_code)
|
||||
if ret.status_code != 200:
|
||||
self.logger.info(ret.text)
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
print("Connectionerror: %s" % e)
|
||||
return
|
||||
self.logger.info("AFTER initial stream result")
|
||||
|
||||
# Verify whether there are any parameters with ACTION_RESULT required
|
||||
# If found, we get the full results list from backend
|
||||
|
||||
fullexecution = {}
|
||||
try:
|
||||
tmpdata = {
|
||||
"authorization": self.authorization,
|
||||
"execution_id": self.current_execution_id
|
||||
}
|
||||
|
||||
self.logger.info("Auth: %s", tmpdata)
|
||||
|
||||
self.logger.info("Before FULLEXEC stream result")
|
||||
ret = requests.post(
|
||||
"%s/api/v1/streams/results" % (self.url),
|
||||
headers=headers,
|
||||
json=tmpdata
|
||||
)
|
||||
|
||||
if ret.status_code == 200:
|
||||
fullexecution = ret.json()
|
||||
else:
|
||||
self.logger.info("Error: Data: ", ret.json())
|
||||
self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code)
|
||||
return
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
self.logger.info("Connectionerror: %s" % e)
|
||||
return
|
||||
|
||||
self.logger.info("AFTER FULLEXEC stream result")
|
||||
|
||||
def parse_params(action, fullexecution, parameter):
|
||||
jsonparsevalue = "$."
|
||||
if parameter["variant"] == "WORKFLOW_VARIABLE":
|
||||
for item in fullexecution["workflow"]["workflow_variables"]:
|
||||
if parameter["action_field"] == item["name"]:
|
||||
parameter["value"] = item["value"]
|
||||
break
|
||||
elif parameter["variant"] == "ACTION_RESULT":
|
||||
# FIXME - calculate value based on action_field and $if prominent
|
||||
# FIND THE RIGHT LABEL
|
||||
# GET THE LABEL'S RESULT
|
||||
|
||||
tmpvalue = ""
|
||||
print(parameter["action_field"])
|
||||
|
||||
if parameter["action_field"] == "Execution Argument":
|
||||
tmpvalue = fullexecution["execution_argument"]
|
||||
else:
|
||||
self.logger.info("WORKFLOW EXEC BELOW")
|
||||
self.logger.info(fullexecution)
|
||||
self.logger.info(fullexecution["results"])
|
||||
self.logger.info(fullexecution["workflow"]["actions"])
|
||||
self.logger.info("ACTIONS ABOVE")
|
||||
# redundancy..
|
||||
|
||||
tmpid = ""
|
||||
for item in fullexecution["workflow"]["actions"]:
|
||||
if item["label"] == parameter["action_field"]:
|
||||
tmpid = item["id"]
|
||||
|
||||
if not tmpid:
|
||||
self.logger.error("Value not found for that id: %s. Exiting" % parameter["action_field"])
|
||||
raise Exception("Value for %s was not found in workflow actions" % parameter["action_field"])
|
||||
|
||||
for subresult in fullexecution["results"]:
|
||||
if subresult["action"]["id"] == tmpid:
|
||||
tmpvalue = subresult["result"]
|
||||
break
|
||||
|
||||
if not tmpvalue:
|
||||
self.logger.error("Value not found for label %s. Exiting" % parameter["action_field"])
|
||||
raise Exception("Value for %s was not found" % parameter["action_field"])
|
||||
|
||||
# Override locally with JSON data
|
||||
if parameter["value"].startswith(jsonparsevalue):
|
||||
parsersplit = parameter["value"].split(".")
|
||||
|
||||
# Convert to json here
|
||||
self.logger.info("JSON HANDLING: %s" % tmpvalue)
|
||||
tmpvalue = tmpvalue.replace("\'", "\"")
|
||||
try:
|
||||
if isinstance(tmpvalue, str):
|
||||
newtmp = json.loads(tmpvalue)
|
||||
except json.decoder.JSONDecodeError as e:
|
||||
raise Exception("JSON error: %s" % e)
|
||||
|
||||
try:
|
||||
#previousvalue = parsersplit[1]
|
||||
for value in parsersplit[1:]:
|
||||
# Might need to be recursive here, because it can go
|
||||
# multiple layers ($.result.#.test.users.#.name)
|
||||
# That would give executions of:
|
||||
# 1 + result.length + users.length
|
||||
# This is also just for one param
|
||||
#if parsersplit[1:][count] == "#":
|
||||
if value == "#":
|
||||
# This means we already have an array
|
||||
# for item in newtmp:
|
||||
self.logger.info("THERE SHOULD BE A LOOP HERE")
|
||||
# This works, but it needs to be split into multiples hurr
|
||||
# Whenever there is a loop, there is a need to
|
||||
# check whether there are more loops, then do
|
||||
# recursion to all the bottom leaves
|
||||
|
||||
#paramnamevalue.append(newtmp
|
||||
newtmp = newtmp[0]
|
||||
# Choose numero uno which will then be handled by the next again
|
||||
# params[parameter["name"]].append(value.nextitem)
|
||||
else:
|
||||
newtmp = newtmp[value]
|
||||
except KeyError as e:
|
||||
return "KeyError: %s" % e, ""
|
||||
except IndexError as e:
|
||||
return "IndexError: %s" % e, ""
|
||||
|
||||
parameter["value"] = str(newtmp)
|
||||
else:
|
||||
parameter["value"] = tmpvalue
|
||||
|
||||
return "", parameter["value"]
|
||||
|
||||
def run_validation(sourcevalue, check, destinationvalue):
|
||||
self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue))
|
||||
|
||||
if check == "=" or check.lower() == "equals":
|
||||
if sourcevalue.lower() == destinationvalue.lower():
|
||||
return True
|
||||
elif check == "!=" or check.lower() == "does not equal":
|
||||
if sourcevalue.lower() != destinationvalue.lower():
|
||||
return True
|
||||
elif check.lower() == "startswith":
|
||||
if sourcevalue.lower().startswith(destinationvalue.lower()):
|
||||
return True
|
||||
elif check.lower() == "endswith":
|
||||
if sourcevalue.lower().endswith(destinationvalue.lower()):
|
||||
return True
|
||||
elif check.lower() == "contains":
|
||||
if destinationvalue.lower() in sourcevalue.lower():
|
||||
return True
|
||||
else:
|
||||
self.logger.info("Condition: can't handle %s yet. Setting to true" % check)
|
||||
|
||||
return False
|
||||
|
||||
def check_branch_conditions(action, fullexecution):
|
||||
# relevantbranches = workflow.branches where destination = action
|
||||
try:
|
||||
if fullexecution["workflow"]["branches"] == None or len(fullexecution["workflow"]["branches"]) == 0:
|
||||
return True, ""
|
||||
except KeyError:
|
||||
return True, ""
|
||||
|
||||
relevantbranches = []
|
||||
for branch in fullexecution["workflow"]["branches"]:
|
||||
if branch["destination_id"] != action["id"]:
|
||||
continue
|
||||
|
||||
self.logger.info("Relevant branch: %s" % branch)
|
||||
|
||||
# Remove anything without a condition
|
||||
try:
|
||||
if (branch["conditions"]) == 0 or branch["conditions"] == None:
|
||||
continue
|
||||
except KeyError:
|
||||
continue
|
||||
|
||||
self.logger.info("Relevant conditions: %s" % branch["conditions"])
|
||||
successful_conditions = []
|
||||
failed_conditions = []
|
||||
for condition in branch["conditions"]:
|
||||
self.logger.info("Getting condition value of %s" % condition)
|
||||
|
||||
# Parse all values first here
|
||||
sourcevalue = condition["source"]["value"]
|
||||
if condition["source"]["variant"] == "" or condition["source"]["variant"]== "STATIC_VALUE":
|
||||
condition["source"]["variant"]= "STATIC_VALUE"
|
||||
else:
|
||||
check, sourcevalue = parse_params(action, fullexecution, condition["source"])
|
||||
if check:
|
||||
return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
|
||||
|
||||
print(sourcevalue)
|
||||
destinationvalue = condition["destination"]["value"]
|
||||
if condition["destination"]["variant"]== "" or condition["destination"]["variant"]== "STATIC_VALUE":
|
||||
condition["destination"]["variant"] = "STATIC_VALUE"
|
||||
else:
|
||||
check, destinationvalue = parse_params(action, fullexecution, condition["destination"])
|
||||
if check:
|
||||
return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check)
|
||||
|
||||
available_checks = [
|
||||
"=",
|
||||
"equals",
|
||||
"!=",
|
||||
"does not equal",
|
||||
">",
|
||||
"larger than",
|
||||
"<",
|
||||
"less than",
|
||||
">=",
|
||||
"<=",
|
||||
"startswith",
|
||||
"endswith",
|
||||
"contains",
|
||||
"re",
|
||||
"matches regex",
|
||||
]
|
||||
|
||||
# FIXME - what should I do here?
|
||||
if not condition["condition"]["value"] in available_checks:
|
||||
self.logger.info("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"]))
|
||||
continue
|
||||
|
||||
#print(destinationvalue)
|
||||
if not run_validation(sourcevalue, condition["condition"]["value"], destinationvalue):
|
||||
self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue))
|
||||
return False, "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue)
|
||||
|
||||
|
||||
# Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string
|
||||
#for condition in branch.conditons:
|
||||
|
||||
return True, ""
|
||||
|
||||
# Checks whether conditions are met, otherwise set
|
||||
branchcheck, tmpresult = check_branch_conditions(action, fullexecution)
|
||||
if not branchcheck:
|
||||
self.logger.info("Failed one or more branch conditions.")
|
||||
action_result["result"] = tmpresult
|
||||
action_result["status"] = "SKIPPED"
|
||||
try:
|
||||
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
|
||||
self.logger.info("Result: %d" % ret.status_code)
|
||||
if ret.status_code != 200:
|
||||
self.logger.info(ret.text)
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
self.logger.exception(e)
|
||||
|
||||
return
|
||||
|
||||
# Replace name cus there might be issues
|
||||
# Not doing lower() as there might be user-made functions
|
||||
actionname = action["name"]
|
||||
if " " in actionname:
|
||||
actionname.replace(" ", "_", -1)
|
||||
#if action.generated:
|
||||
# actionname = actionname.lower()
|
||||
|
||||
# Runs the actual functions
|
||||
try:
|
||||
func = getattr(self, actionname, None)
|
||||
if func == None:
|
||||
self.logger.debug("Failed executing %s because func is None." % actionname)
|
||||
action_result["status"] = "FAILURE"
|
||||
action_result["result"] = "Function %s doesn't exist." % actionname
|
||||
elif callable(func):
|
||||
try:
|
||||
if len(action["parameters"]) < 1:
|
||||
result = await func()
|
||||
else:
|
||||
# Potentially parse JSON here
|
||||
# FIXME - add potential authentication as first parameter(s) here
|
||||
# params[parameter["name"]] = parameter["value"]
|
||||
#print(fullexecution["authentication"]
|
||||
# What variables are necessary here tho hmm
|
||||
|
||||
params = {}
|
||||
try:
|
||||
for item in action["authentication"]:
|
||||
print(key, value)
|
||||
params[item["key"]] = item["value"]
|
||||
except KeyError:
|
||||
pass
|
||||
#action["authentication"]
|
||||
|
||||
# calltimes is used to handle forloops in the app itself.
|
||||
# 2 kinds of loop - one in gui with one app each, and one like this,
|
||||
# which is super fast, but has a bad overview (potentially good tho)
|
||||
calltimes = 1
|
||||
result = ""
|
||||
paramiter = []
|
||||
for parameter in action["parameters"]:
|
||||
#self.logger.info(parameter)
|
||||
#print(fullexecution)
|
||||
|
||||
|
||||
check, value = parse_params(action, fullexecution, parameter)
|
||||
if check:
|
||||
raise Exception(check)
|
||||
|
||||
params[parameter["name"]] = value
|
||||
# p["value"]
|
||||
|
||||
# FIXME - this is horrible, but works for now
|
||||
#for i in range(calltimes):
|
||||
result += await func(**params)
|
||||
|
||||
action_result["status"] = "SUCCESS"
|
||||
action_result["result"] = str(result)
|
||||
if action_result["result"] == "":
|
||||
action_result["result"] = result
|
||||
|
||||
self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}")
|
||||
self.logger.debug(f"Data: %s" % action_result)
|
||||
except TypeError as e:
|
||||
action_result["status"] = "FAILURE"
|
||||
action_result["result"] = "TypeError: %s" % str(e)
|
||||
else:
|
||||
print("Not callable?")
|
||||
self.logger.error(f"App {self.__class__.__name__}.{action['name']} is not callable")
|
||||
action_result["status"] = "FAILURE"
|
||||
action_result["result"] = "Function %s is not callable." % actionname
|
||||
|
||||
except Exception as e:
|
||||
print(f"Failed to execute: {e}")
|
||||
self.logger.exception(f"Failed to execute {e}-{action['id']}")
|
||||
action_result["status"] = "FAILURE"
|
||||
action_result["result"] = "Exception: %s" % e
|
||||
|
||||
action_result["completed_at"] = int(time.time())
|
||||
|
||||
# I wonder if this actually works
|
||||
self.logger.info("Before last stream result")
|
||||
try:
|
||||
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
|
||||
self.logger.info("Result: %d" % ret.status_code)
|
||||
if ret.status_code != 200:
|
||||
self.logger.info(ret.text)
|
||||
except requests.exceptions.ConnectionError as e:
|
||||
self.logger.exception(e)
|
||||
return
|
||||
except TypeError as e:
|
||||
self.logger.exception(e)
|
||||
action_result["status"] = "FAILURE"
|
||||
action_result["result"] = "POST error: %s" % e
|
||||
self.logger.info("Before typeerror stream result")
|
||||
ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
|
||||
self.logger.info("Result: %d" % ret.status_code)
|
||||
if ret.status_code != 200:
|
||||
self.logger.info(ret.text)
|
||||
|
||||
return
|
||||
|
||||
|
||||
#STOPCOPY
|
||||
# !!! Let the above line stay - its used for some horrible codegeneration / stitching !!! #
|
||||
|
||||
@classmethod
|
||||
async def run(cls):
|
||||
""" Connect to Redis and HTTP session, await actions """
|
||||
logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{')
|
||||
logger = logging.getLogger(f"{cls.__name__}")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
app = cls(redis=None, logger=logger, console_logger=logger)
|
||||
|
||||
# Authorization for the app/function to control the workflow
|
||||
# Function will crash if its wrong, which it probably should.
|
||||
|
||||
await app.execute_action(app.action)
|
||||
@@ -0,0 +1,2 @@
|
||||
requests
|
||||
urllib3
|
||||
@@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
docker rmi frikky/shuffle:app_sdk
|
||||
docker build . -t frikky/shuffle:app_sdk
|
||||
docker push frikky/shuffle:app_sdk
|
||||
@@ -0,0 +1,14 @@
|
||||
from golang as builder
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY orborus.go /app/orborus.go
|
||||
|
||||
RUN go get github.com/docker/docker/api/types github.com/docker/docker/api/types/container github.com/docker/docker/client
|
||||
|
||||
RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o orborus .
|
||||
|
||||
from scratch
|
||||
COPY --from=builder /app/ /
|
||||
|
||||
CMD ["./orborus"]
|
||||
@@ -0,0 +1,4 @@
|
||||
docker rmi frikky/shuffle:orborus --force
|
||||
|
||||
docker build . -t frikky/shuffle:orborus
|
||||
docker push frikky/shuffle:orborus
|
||||
@@ -0,0 +1,425 @@
|
||||
package main
|
||||
|
||||
/*
|
||||
Orborus exists to listen for new workflow executions and deploy workers.
|
||||
*/
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
dockerclient "github.com/docker/docker/client"
|
||||
//network "github.com/docker/docker/api/types/network"
|
||||
//natting "github.com/docker/go-connections/nat"
|
||||
)
|
||||
|
||||
var baseUrl = os.Getenv("BASE_URL")
|
||||
var baseimagename = "frikky/shuffle"
|
||||
var dockerApiVersion = os.Getenv("DOCKER_API_VERSION")
|
||||
var environment = os.Getenv("ENVIRONMENT_NAME")
|
||||
var orgId = os.Getenv("ORG_ID")
|
||||
var workerTimeout = 600
|
||||
|
||||
type ExecutionRequestWrapper struct {
|
||||
Data []ExecutionRequest `json:"data"`
|
||||
}
|
||||
|
||||
type ExecutionRequest struct {
|
||||
ExecutionId string `json:"execution_id"`
|
||||
WorkflowId string `json:"workflow_id"`
|
||||
Authorization string `json:"authorization"`
|
||||
ExecutionArgument string `json:"execution_argument"`
|
||||
Environments []string `json:"environments"`
|
||||
Status string `json:"status"`
|
||||
}
|
||||
|
||||
// Deploys the internal worker whenever something happens
|
||||
func deployWorker(cli *dockerclient.Client, image string, identifier string, env []string) error {
|
||||
// Binds is the actual "-v" volume.
|
||||
hostConfig := &container.HostConfig{
|
||||
LogConfig: container.LogConfig{
|
||||
Type: "json-file",
|
||||
Config: map[string]string{},
|
||||
},
|
||||
Binds: []string{
|
||||
"/var/run/docker.sock:/var/run/docker.sock:rw",
|
||||
},
|
||||
}
|
||||
|
||||
// ROFL: https://docker-py.readthedocs.io/en/1.4.0/volumes/
|
||||
config := &container.Config{
|
||||
Image: image,
|
||||
Env: env,
|
||||
}
|
||||
//Volumes: map[string]struct{}{
|
||||
// "/var/run/docker.sock": {},
|
||||
//},
|
||||
|
||||
cont, err := cli.ContainerCreate(
|
||||
context.Background(),
|
||||
config,
|
||||
hostConfig,
|
||||
nil,
|
||||
identifier,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
|
||||
log.Printf("Container %s is created", cont.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func stopWorker(containername string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
cli, err := dockerclient.NewEnvClient()
|
||||
if err != nil {
|
||||
log.Println("Unable to create docker client")
|
||||
return err
|
||||
}
|
||||
|
||||
// containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
|
||||
// All: true,
|
||||
// })
|
||||
|
||||
if err := cli.ContainerStop(ctx, containername, nil); err != nil {
|
||||
log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err)
|
||||
}
|
||||
|
||||
removeOptions := types.ContainerRemoveOptions{
|
||||
RemoveVolumes: true,
|
||||
Force: true,
|
||||
}
|
||||
|
||||
if err := cli.ContainerRemove(ctx, containername, removeOptions); err != nil {
|
||||
log.Printf("Unable to remove container: %s", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func initializeImages(dockercli *dockerclient.Client) {
|
||||
ctx := context.Background()
|
||||
|
||||
// check whether theyre the same first
|
||||
images := []string{
|
||||
fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
|
||||
fmt.Sprintf("docker.io/%s:worker", baseimagename),
|
||||
}
|
||||
|
||||
pullOptions := types.ImagePullOptions{}
|
||||
for _, image := range images {
|
||||
reader, err := dockercli.ImagePull(ctx, image, pullOptions)
|
||||
if err != nil {
|
||||
log.Printf("Failed getting %s", image)
|
||||
continue
|
||||
}
|
||||
|
||||
io.Copy(os.Stdout, reader)
|
||||
log.Printf("Successfully downloaded and built %s", image)
|
||||
}
|
||||
}
|
||||
|
||||
// Initial loop etc
|
||||
func main() {
|
||||
zombiecheck()
|
||||
log.Println("Setting up execution environment")
|
||||
|
||||
//FIXME
|
||||
if baseUrl == "" {
|
||||
baseUrl = "https://shuffler.io"
|
||||
//baseUrl = "http://localhost:5001"
|
||||
}
|
||||
|
||||
if orgId == "" {
|
||||
log.Printf("Org not defined. Set variable ORG_ID based on your org")
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
log.Printf("Running towards %s with Org %s", baseUrl, orgId)
|
||||
|
||||
if environment == "" {
|
||||
environment = "onprem"
|
||||
log.Printf("Defaulting to environment name %s. Set environment variable ENVIRONMENT_NAME to change. This should be the same as in the frontend action.", environment)
|
||||
}
|
||||
|
||||
// FIXME - during init, BUILD and/or LOAD worker and app_sdk
|
||||
// Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk
|
||||
dockercli, err := dockerclient.NewEnvClient()
|
||||
if err != nil {
|
||||
fmt.Println("Unable to create docker client")
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
log.Printf("--- Setting up Docker environment. Downloading worker and App SDK! ---")
|
||||
initializeImages(dockercli)
|
||||
workerImage := fmt.Sprintf("%s:worker", baseimagename)
|
||||
|
||||
log.Printf("--- Finished configuring docker environment ---\n")
|
||||
|
||||
// FIXME - time limit
|
||||
sleepTime := 10
|
||||
client := &http.Client{}
|
||||
|
||||
fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl)
|
||||
req, err := http.NewRequest(
|
||||
"GET",
|
||||
fullUrl,
|
||||
nil,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Failed making request builder: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
zombiecounter := 0
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Org-Id", orgId)
|
||||
log.Printf("Getting data from %s", fullUrl)
|
||||
hasStarted := false
|
||||
for {
|
||||
//log.Printf("Prerequest")
|
||||
newresp, err := client.Do(req)
|
||||
//log.Printf("Postrequest")
|
||||
if err != nil {
|
||||
log.Printf("Failed making request: %s", err)
|
||||
zombiecounter += 1
|
||||
if zombiecounter*sleepTime > workerTimeout {
|
||||
zombiecheck()
|
||||
zombiecounter = 0
|
||||
}
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
// FIXME - add check for StatusCode
|
||||
if newresp.StatusCode != 200 {
|
||||
if hasStarted {
|
||||
log.Printf("Bad statuscode: %d", newresp.StatusCode)
|
||||
}
|
||||
} else {
|
||||
hasStarted = true
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(newresp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Failed reading body: %s", err)
|
||||
zombiecounter += 1
|
||||
if zombiecounter*sleepTime > workerTimeout {
|
||||
zombiecheck()
|
||||
zombiecounter = 0
|
||||
}
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
var executionRequests ExecutionRequestWrapper
|
||||
err = json.Unmarshal(body, &executionRequests)
|
||||
if err != nil {
|
||||
log.Printf("Failed executionrequest in queue unmarshaling: %s", err)
|
||||
sleepTime = 10
|
||||
zombiecounter += 1
|
||||
if zombiecounter*sleepTime > workerTimeout {
|
||||
zombiecheck()
|
||||
zombiecounter = 0
|
||||
}
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
if hasStarted && len(executionRequests.Data) > 0 {
|
||||
log.Println(string(body))
|
||||
}
|
||||
|
||||
if len(executionRequests.Data) == 0 {
|
||||
zombiecounter += 1
|
||||
if zombiecounter*sleepTime > workerTimeout {
|
||||
zombiecheck()
|
||||
zombiecounter = 0
|
||||
}
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
// New, abortable version. Should check executionid and remove everything else
|
||||
var toBeRemoved ExecutionRequestWrapper
|
||||
for _, execution := range executionRequests.Data {
|
||||
log.Println(execution.ExecutionArgument)
|
||||
if execution.Status == "ABORT" || execution.Status == "FAILED" {
|
||||
log.Printf("Executionstatus issue: ", execution.Status)
|
||||
}
|
||||
// Now, how do I execute this one?
|
||||
// FIXME - if error, check the status of the running one. If it's bad, send data back.
|
||||
containerName := fmt.Sprintf("worker-%s", execution.ExecutionId)
|
||||
env := []string{
|
||||
fmt.Sprintf("AUTHORIZATION=%s", execution.Authorization),
|
||||
fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId),
|
||||
fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion),
|
||||
fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
|
||||
fmt.Sprintf("BASE_URL=%s", baseUrl),
|
||||
}
|
||||
|
||||
err = deployWorker(dockercli, workerImage, containerName, env)
|
||||
if err != nil {
|
||||
stats, err := dockercli.ContainerInspect(context.Background(), containerName)
|
||||
if err != nil {
|
||||
log.Printf("Failed checking worker %s", execution.ExecutionId)
|
||||
continue
|
||||
}
|
||||
|
||||
containerStatus := stats.ContainerJSONBase.State.Status
|
||||
if containerStatus != "running" {
|
||||
log.Printf("Status of %s is %s. Should be running. Will reset", containerName, containerStatus)
|
||||
err = stopWorker(containerName)
|
||||
if err != nil {
|
||||
log.Printf("Failed stopping worker %s", execution.ExecutionId)
|
||||
continue
|
||||
}
|
||||
|
||||
err = deployWorker(dockercli, workerImage, containerName, env)
|
||||
if err != nil {
|
||||
log.Printf("Failed executing worker %s in state %s", execution.ExecutionId, containerStatus)
|
||||
}
|
||||
} else {
|
||||
// Should basically never hit here rofl
|
||||
log.Printf("ERROR: I HAVE NO IDEA WHAT WENT WRONG. CHECK %s", containerName)
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("%s is deployed and to being removed from queue.", execution.ExecutionId)
|
||||
zombiecounter += 1
|
||||
toBeRemoved.Data = append(toBeRemoved.Data, execution)
|
||||
}
|
||||
|
||||
// Removes handled workflows (worker is made)
|
||||
if len(toBeRemoved.Data) > 0 {
|
||||
confirmUrl := fmt.Sprintf("%s/api/v1/workflows/queue/confirm", baseUrl)
|
||||
|
||||
data, err := json.Marshal(toBeRemoved)
|
||||
if err != nil {
|
||||
log.Printf("Failed removal marshalling: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
result, err := http.NewRequest(
|
||||
"POST",
|
||||
confirmUrl,
|
||||
bytes.NewBuffer([]byte(data)),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Failed building confirm request: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
result.Header.Add("Content-Type", "application/json")
|
||||
result.Header.Add("Org-Id", orgId)
|
||||
|
||||
resultResp, err := client.Do(result)
|
||||
if err != nil {
|
||||
log.Printf("Failed making confirm request: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resultResp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Failed reading confirm body: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Println(string(body))
|
||||
|
||||
// FIXME - remove these
|
||||
//log.Println(string(body))
|
||||
//log.Println(resultResp)
|
||||
if len(toBeRemoved.Data) == len(executionRequests.Data) {
|
||||
log.Println("Should remove ALL!")
|
||||
} else {
|
||||
log.Printf("NOT IMPLEMENTED: Should remove %d workflows from backend because they're executed!", len(toBeRemoved.Data))
|
||||
}
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME - add this to remove exited workers
|
||||
// Should it check what happened to the execution? idk
|
||||
func zombiecheck() error {
|
||||
log.Println("Running zombiecheck")
|
||||
ctx := context.Background()
|
||||
|
||||
dockercli, err := dockerclient.NewEnvClient()
|
||||
if err != nil {
|
||||
log.Println("Unable to create docker client")
|
||||
return err
|
||||
}
|
||||
|
||||
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
|
||||
All: true,
|
||||
})
|
||||
|
||||
stopContainers := []string{}
|
||||
removeContainers := []string{}
|
||||
for _, container := range containers {
|
||||
for _, name := range container.Names {
|
||||
// FIXME - add name_version_uid_uid regex check as well
|
||||
if !strings.HasPrefix(name, "/worker") {
|
||||
continue
|
||||
}
|
||||
|
||||
if container.State != "running" {
|
||||
removeContainers = append(removeContainers, container.ID)
|
||||
}
|
||||
|
||||
// stopcontainer & removecontainer
|
||||
currenttime := time.Now().Unix()
|
||||
if container.State == "running" && currenttime-container.Created > int64(workerTimeout) {
|
||||
stopContainers = append(stopContainers, container.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME - add killing of apps with same execution ID too
|
||||
for _, containername := range stopContainers {
|
||||
if err := dockercli.ContainerStop(ctx, containername, nil); err != nil {
|
||||
log.Printf("Unable to stop container: %s", err)
|
||||
} else {
|
||||
log.Printf("Stopped container %s", containername)
|
||||
}
|
||||
}
|
||||
|
||||
removeOptions := types.ContainerRemoveOptions{
|
||||
RemoveVolumes: true,
|
||||
Force: true,
|
||||
}
|
||||
|
||||
for _, containername := range removeContainers {
|
||||
if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil {
|
||||
log.Printf("Unable to remove container: %s", err)
|
||||
} else {
|
||||
log.Printf("Removed container %s", containername)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
docker run \
|
||||
--env ORG_ID=$ORG_ID \
|
||||
--env BASE_URL=$BASE_URL \
|
||||
-v /var/run/docker.sock:/var/run/docker.sock \
|
||||
frikky/shuffle:orborus
|
||||
@@ -0,0 +1,21 @@
|
||||
#from golang as builder
|
||||
#
|
||||
#RUN mkdir /app
|
||||
#WORKDIR /app
|
||||
#COPY worker.go /app/worker.go
|
||||
#
|
||||
#RUN go get github.com/docker/docker/api/types
|
||||
#RUN go get github.com/docker/docker/api/types/container
|
||||
#RUN go get -u github.com/docker/docker/client
|
||||
#
|
||||
#RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
|
||||
#
|
||||
|
||||
# THis is a workaround until I get docker/docker to build in a dockerfile
|
||||
# PS: This is tricky to google.
|
||||
# Might not work on some machines.
|
||||
from scratch
|
||||
#COPY --from=builder /app/ /
|
||||
COPY worker.bin /worker.bin
|
||||
|
||||
CMD ["./worker.bin"]
|
||||
@@ -0,0 +1,15 @@
|
||||
echo "Compiling program"
|
||||
CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
|
||||
|
||||
echo "Fixing docker env"
|
||||
docker rmi frikky/shuffle:worker --force
|
||||
docker build . -t frikky/shuffle:worker
|
||||
docker push frikky/shuffle:worker
|
||||
|
||||
#docker run \
|
||||
# --env "AUTHORIZATION=ASD" \
|
||||
# --env "DOCKER_API_VERSION=1.39" \
|
||||
# --env "EXECUTIONID=ASD" \
|
||||
# --env "BASE_URI=$BASE_URI" \
|
||||
# -v /var/run/docker.sock:/var/run/docker.sock \
|
||||
# frikky/shuffle:worker
|
||||
Executable
BIN
Binary file not shown.
@@ -0,0 +1,814 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
//"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/api/types/container"
|
||||
dockerclient "github.com/docker/docker/client"
|
||||
)
|
||||
|
||||
var environment = os.Getenv("ENVIRONMENT_NAME")
|
||||
var baseUrl = os.Getenv("BASE_URL")
|
||||
var baseimagename = "frikky/shuffle"
|
||||
|
||||
type Condition struct {
|
||||
AppName string `json:"app_name"`
|
||||
AppVersion string `json:"app_version"`
|
||||
Conditional string `json:"conditional"`
|
||||
Errors []string `json:"errors"`
|
||||
ID string `json:"id"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
Label string `json:"label"`
|
||||
Name string `json:"name"`
|
||||
Position struct {
|
||||
X float64 `json:"x"`
|
||||
Y float64 `json:"y"`
|
||||
} `json:"position"`
|
||||
}
|
||||
|
||||
type User struct {
|
||||
Username string `datastore:"Username"`
|
||||
Password string `datastore:"password,noindex"`
|
||||
Session string `datastore:"session,noindex"`
|
||||
Verified bool `datastore:"verified,noindex"`
|
||||
ApiKey string `datastore:"apikey,noindex"`
|
||||
Id string `datastore:"id" json:"id"`
|
||||
Orgs string `datastore:"orgs" json:"orgs"`
|
||||
}
|
||||
|
||||
type Org struct {
|
||||
Name string `json:"name"`
|
||||
Org string `json:"org"`
|
||||
Users []User `json:"users"`
|
||||
Id string `json:"id"`
|
||||
}
|
||||
|
||||
// FIXME: Generate a callback authentication ID?
|
||||
type WorkflowExecution struct {
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
ExecutionId string `json:"execution_id"`
|
||||
ExecutionArgument string `json:"execution_argument"`
|
||||
WorkflowId string `json:"workflow_id"`
|
||||
LastNode string `json:"last_node"`
|
||||
Authorization string `json:"authorization"`
|
||||
Result string `json:"result"`
|
||||
StartedAt int64 `json:"started_at"`
|
||||
CompletedAt int64 `json:"completed_at"`
|
||||
ProjectId string `json:"project_id"`
|
||||
Locations []string `json:"locations"`
|
||||
Workflow Workflow `json:"workflow"`
|
||||
Results []ActionResult `json:"results"`
|
||||
}
|
||||
|
||||
// Added environment for location to execute
|
||||
type Action struct {
|
||||
AppName string `json:"app_name" datastore:"app_name"`
|
||||
AppVersion string `json:"app_version" datastore:"app_version"`
|
||||
Errors []string `json:"errors" datastore:"errors"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
IsValid bool `json:"is_valid" datastore:"is_valid"`
|
||||
IsStartNode bool `json:"isStartNode" datastore:"isStartNode"`
|
||||
Label string `json:"label" datastore:"label"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Position struct {
|
||||
X float64 `json:"x" datastore:"x"`
|
||||
Y float64 `json:"y" datastore:"y"`
|
||||
} `json:"position"`
|
||||
Priority int `json:"priority" datastore:"priority"`
|
||||
}
|
||||
|
||||
type Branch struct {
|
||||
DestinationID string `json:"destination_id" datastore:"destination_id"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
SourceID string `json:"source_id" datastore:"source_id"`
|
||||
HasError bool `json:"has_errors" datastore: "has_errors"`
|
||||
}
|
||||
|
||||
type Schedule struct {
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Frequency string `json:"frequency" datastore:"frequency"`
|
||||
ExecutionArgument string `json:"execution_argument" datastore:"execution_argument"`
|
||||
Id string `json:"id" datastore:"id"`
|
||||
}
|
||||
|
||||
type Trigger struct {
|
||||
AppName string `json:"app_name" datastore:"app_name"`
|
||||
Status string `json:"status" datastore:"status"`
|
||||
AppVersion string `json:"app_version" datastore:"app_version"`
|
||||
Errors []string `json:"errors" datastore:"errors"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
IsValid bool `json:"is_valid" datastore:"is_valid"`
|
||||
IsStartNode bool `json:"isStartNode" datastore:"isStartNode"`
|
||||
Label string `json:"label" datastore:"label"`
|
||||
SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"`
|
||||
LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
TriggerType string `json:"trigger_type" datastore:"trigger_type"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Position struct {
|
||||
X float64 `json:"x" datastore:"x"`
|
||||
Y float64 `json:"y" datastore:"y"`
|
||||
} `json:"position"`
|
||||
Priority int `json:"priority" datastore:"priority"`
|
||||
}
|
||||
|
||||
type Workflow struct {
|
||||
Actions []Action `json:"actions" datastore:"actions"`
|
||||
Branches []Branch `json:"branches" datastore:"branches"`
|
||||
Triggers []Trigger `json:"triggers" datastore:"triggers"`
|
||||
Schedules []Schedule `json:"schedules" datastore:"schedules"`
|
||||
Errors []string `json:"errors,omitempty" datastore:"errors"`
|
||||
Tags []string `json:"tags,omitempty" datastore:"tags"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
IsValid bool `json:"is_valid" datastore:"is_valid"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Description string `json:"description" datastore:"description"`
|
||||
Start string `json:"start" datastore:"start"`
|
||||
Owner string `json:"owner" datastore:"owner"`
|
||||
Sharing string `json:"sharing" datastore:"sharing"`
|
||||
Org []Org `json:"org,omitempty" datastore:"org"`
|
||||
ExecutingOrg Org `json:"execution_org,omitempty" datastore:"execution_org"`
|
||||
WorkflowVariables []struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
} `json:"workflow_variables" datastore:"workflow_variables"`
|
||||
}
|
||||
|
||||
type ActionResult struct {
|
||||
Action Action `json:"action" datastore:"action"`
|
||||
ExecutionId string `json:"execution_id" datastore:"execution_id"`
|
||||
Authorization string `json:"authorization" datastore:"authorization"`
|
||||
Result string `json:"result" datastore:"result"`
|
||||
StartedAt int64 `json:"started_at" datastore:"started_at"`
|
||||
CompletedAt int64 `json:"completed_at" datastore:"completed_at"`
|
||||
Status string `json:"status" datastore:"status"`
|
||||
}
|
||||
|
||||
type WorkflowApp struct {
|
||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
||||
ID string `json:"id" yaml:"id" required:false datastore:"id"`
|
||||
Link string `json:"link" yaml:"link" required:false datastore:"link"`
|
||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
ContactInfo struct {
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
||||
}
|
||||
|
||||
// Name = current field
|
||||
// action_field is the field that it's set to
|
||||
// value, if Variant = ACTION_RESULT = the second field thingy, which will be
|
||||
type WorkflowAppActionParameter struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
ActionField string `json:"action_field" datastore:"action_field"`
|
||||
Variant string `json:"variant", datastore:"variant"`
|
||||
Required bool `json:"required" datastore:"required"`
|
||||
Schema struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
} `json:"schema"`
|
||||
}
|
||||
|
||||
type WorkflowAppAction struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
NodeType string `json:"node_type" datastore:"node_type"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Returns struct {
|
||||
Description string `json:"description" datastore:"returns"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Schema struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
} `json:"schema" datastore:"schema"`
|
||||
} `json:"returns" datastore:"returns"`
|
||||
}
|
||||
|
||||
// removes every container except itself (worker)
|
||||
func shutdown(executionId string) {
|
||||
dockercli, err := dockerclient.NewEnvClient()
|
||||
if err != nil {
|
||||
log.Printf("Unable to create docker client: %s", err)
|
||||
shutdown(executionId)
|
||||
}
|
||||
|
||||
containerOptions := types.ContainerListOptions{
|
||||
All: true,
|
||||
}
|
||||
|
||||
containers, err := dockercli.ContainerList(context.Background(), containerOptions)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
_ = containers
|
||||
|
||||
for _, container := range containers {
|
||||
for _, name := range container.Names {
|
||||
if strings.Contains(name, executionId) {
|
||||
// FIXME - reinstate - not here for debugging
|
||||
//err = removeContainer(container.ID)
|
||||
//if err != nil {
|
||||
// log.Printf("Failed removing %s before shutdown.", name)
|
||||
//}
|
||||
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// FIXME: Add an API call to the backend
|
||||
workflowid := "d0496ad4-d682-4506-bbf9-f926358a4b2a"
|
||||
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowid, executionId)
|
||||
log.Printf("ShutdownURL: %s", fullUrl)
|
||||
req, err := http.NewRequest(
|
||||
"GET",
|
||||
fullUrl,
|
||||
nil,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Println("Failed building request: %s", err)
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
_, err = client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Failed abort request: %s", err)
|
||||
}
|
||||
|
||||
log.Printf("Finished shutdown.")
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
// Deploys the internal worker whenever something happens
|
||||
func deployApp(cli *dockerclient.Client, image string, identifier string, env []string) error {
|
||||
hostConfig := &container.HostConfig{
|
||||
LogConfig: container.LogConfig{
|
||||
Type: "json-file",
|
||||
Config: map[string]string{},
|
||||
},
|
||||
}
|
||||
|
||||
config := &container.Config{
|
||||
Image: image,
|
||||
Env: env,
|
||||
}
|
||||
|
||||
cont, err := cli.ContainerCreate(
|
||||
context.Background(),
|
||||
config,
|
||||
hostConfig,
|
||||
nil,
|
||||
identifier,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
|
||||
fmt.Printf("\n")
|
||||
log.Printf("Container %s is created", cont.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
func removeContainer(containername string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
cli, err := dockerclient.NewEnvClient()
|
||||
if err != nil {
|
||||
log.Printf("Unable to create docker client: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// FIXME - ucnomment
|
||||
// containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
|
||||
// All: true,
|
||||
// })
|
||||
|
||||
_ = ctx
|
||||
_ = cli
|
||||
//if err := cli.ContainerStop(ctx, containername, nil); err != nil {
|
||||
// log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err)
|
||||
//}
|
||||
|
||||
removeOptions := types.ContainerRemoveOptions{
|
||||
RemoveVolumes: true,
|
||||
Force: true,
|
||||
}
|
||||
|
||||
// FIXME - remove comments etc
|
||||
_ = removeOptions
|
||||
//if err := cli.ContainerRemove(ctx, containername, removeOptions); err != nil {
|
||||
// log.Printf("Unable to remove container: %s", err)
|
||||
//}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func handleExecution(client *http.Client, req *http.Request, workflowExecution WorkflowExecution) error {
|
||||
// if no onprem runs (shouldn't happen, but extra check), exit
|
||||
// if there are some, load the images ASAP for the app
|
||||
dockercli, err := dockerclient.NewEnvClient()
|
||||
if err != nil {
|
||||
log.Printf("Unable to create docker client: %s", err)
|
||||
shutdown(workflowExecution.ExecutionId)
|
||||
}
|
||||
|
||||
onpremApps := []string{}
|
||||
startAction := workflowExecution.Workflow.Start
|
||||
sleepTime := 5
|
||||
toExecuteOnprem := []string{}
|
||||
parents := map[string][]string{}
|
||||
children := map[string][]string{}
|
||||
|
||||
// source = parent, dest = child
|
||||
// parent can have more children, child can have more parents
|
||||
for _, branch := range workflowExecution.Workflow.Branches {
|
||||
parents[branch.DestinationID] = append(parents[branch.DestinationID], branch.SourceID)
|
||||
children[branch.SourceID] = append(children[branch.SourceID], branch.DestinationID)
|
||||
}
|
||||
|
||||
for _, action := range workflowExecution.Workflow.Actions {
|
||||
if action.Environment != environment {
|
||||
continue
|
||||
}
|
||||
|
||||
toExecuteOnprem = append(toExecuteOnprem, action.ID)
|
||||
|
||||
actionName := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion)
|
||||
found := false
|
||||
for _, app := range onpremApps {
|
||||
if actionName == app {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
if !found {
|
||||
onpremApps = append(onpremApps, actionName)
|
||||
}
|
||||
}
|
||||
|
||||
if len(onpremApps) == 0 {
|
||||
return errors.New("No apps to handle onprem")
|
||||
}
|
||||
|
||||
pullOptions := types.ImagePullOptions{}
|
||||
for _, image := range onpremApps {
|
||||
log.Printf("Image: %s", image)
|
||||
if strings.Contains(image, " ") {
|
||||
image = strings.ReplaceAll(image, " ", "-")
|
||||
}
|
||||
|
||||
reader, err := dockercli.ImagePull(context.Background(), image, pullOptions)
|
||||
if err != nil {
|
||||
log.Printf("Failed getting %s. The app is missing or some other issue", image)
|
||||
//shutdown(workflowExecution.ExecutionId)
|
||||
}
|
||||
|
||||
//io.Copy(os.Stdout, reader)
|
||||
_ = reader
|
||||
log.Printf("Successfully downloaded and built %s", image)
|
||||
}
|
||||
|
||||
// Process the parents etc. How?
|
||||
// while queue:
|
||||
// while len(self.in_process) > 0 or len(self.parallel_in_process) > 0:
|
||||
// check if its their own turn to continue
|
||||
// visited = {self.start_action}
|
||||
visited := []string{}
|
||||
nextActions := []string{}
|
||||
queueNodes := []string{}
|
||||
|
||||
for {
|
||||
//if len(queueNodes) > 0 {
|
||||
// log.Println(queueNodes)
|
||||
// nextActions = queueNodes
|
||||
//} else {
|
||||
// nextActions := []string{}
|
||||
//}
|
||||
// FIXME - this might actually work, but probably not
|
||||
//queueNodes = []string{}
|
||||
|
||||
if len(workflowExecution.Results) == 0 {
|
||||
nextActions = []string{startAction}
|
||||
} else {
|
||||
for _, item := range workflowExecution.Results {
|
||||
visited = append(visited, item.Action.ID)
|
||||
nextActions = children[item.Action.ID]
|
||||
// FIXME: check if nextActions items are finished?
|
||||
}
|
||||
}
|
||||
|
||||
if len(nextActions) == 0 {
|
||||
log.Println("No next action. Finished?")
|
||||
//shutdown(workflowExecution.ExecutionId)
|
||||
}
|
||||
|
||||
for _, node := range nextActions {
|
||||
nodeChildren := children[node]
|
||||
for _, child := range nodeChildren {
|
||||
if !arrayContains(queueNodes, child) {
|
||||
queueNodes = append(queueNodes, child)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//log.Println(queueNodes)
|
||||
|
||||
// IF NOT VISITED && IN toExecuteOnPrem
|
||||
// SKIP if it's not onprem
|
||||
// FIXME: Find next node(s)
|
||||
//for _, result := range workflowExecution.Results {
|
||||
// log.Println(result.Status)
|
||||
//}
|
||||
|
||||
for _, nextAction := range nextActions {
|
||||
action := getAction(workflowExecution, nextAction)
|
||||
// FIXME - remove this. Should always need to be valid.
|
||||
//if action.IsValid == false {
|
||||
// log.Printf("%#v", action)
|
||||
// log.Printf("Action %s (%s) isn't valid. Exiting, BUT SHOULD CALLBACK TO SET FAILURE.", action.ID, action.Name)
|
||||
// os.Exit(3)
|
||||
//}
|
||||
|
||||
// check visited and onprem
|
||||
if arrayContains(visited, nextAction) {
|
||||
log.Printf("ALREADY VISITIED: %s", nextAction)
|
||||
continue
|
||||
}
|
||||
|
||||
// Not really sure how this edgecase happens.
|
||||
|
||||
// FIXME
|
||||
// Execute, as we don't really care if env is not set? IDK
|
||||
if action.Environment != environment { //&& action.Environment != "" {
|
||||
log.Printf("Bad environment: %s", action.Environment)
|
||||
continue
|
||||
}
|
||||
|
||||
// check whether the parent is finished executing
|
||||
//log.Printf("%s has %d parents", nextAction, len(parents[nextAction]))
|
||||
|
||||
continueOuter := true
|
||||
if action.IsStartNode {
|
||||
continueOuter = false
|
||||
} else if len(parents[nextAction]) > 0 {
|
||||
// FIXME - wait for parents to finishe executing
|
||||
fixed := 0
|
||||
for _, parent := range parents[nextAction] {
|
||||
parentResult := getResult(workflowExecution, parent)
|
||||
if parentResult.Status == "FINISHED" || parentResult.Status == "SUCCESS" {
|
||||
fixed += 1
|
||||
}
|
||||
}
|
||||
|
||||
if fixed == len(parents[nextAction]) {
|
||||
continueOuter = false
|
||||
}
|
||||
} else {
|
||||
continueOuter = false
|
||||
}
|
||||
|
||||
if continueOuter {
|
||||
log.Printf("Parents of %s aren't finished: %s", nextAction, strings.Join(parents[nextAction], ", "))
|
||||
continue
|
||||
}
|
||||
|
||||
// get action status
|
||||
actionResult := getResult(workflowExecution, nextAction)
|
||||
if actionResult.Action.ID == action.ID {
|
||||
log.Printf("%s already has status %s.", action.ID, actionResult.Status)
|
||||
continue
|
||||
} else {
|
||||
log.Printf("%s:%s has no status result yet. Should execute.", action.Name, action.ID)
|
||||
}
|
||||
|
||||
appname := action.AppName
|
||||
appversion := action.AppVersion
|
||||
appname = strings.Replace(appname, ".", "-", -1)
|
||||
appversion = strings.Replace(appversion, ".", "-", -1)
|
||||
|
||||
image := fmt.Sprintf("%s:%s_%s", baseimagename, action.AppName, action.AppVersion)
|
||||
if strings.Contains(image, " ") {
|
||||
image = strings.ReplaceAll(image, " ", "-")
|
||||
}
|
||||
|
||||
identifier := fmt.Sprintf("%s_%s_%s_%s", appname, appversion, action.ID, workflowExecution.ExecutionId)
|
||||
if strings.Contains(identifier, " ") {
|
||||
identifier = strings.ReplaceAll(identifier, " ", "-")
|
||||
}
|
||||
|
||||
// FIXME - check whether it's running locally yet too
|
||||
stats, err := dockercli.ContainerInspect(context.Background(), identifier)
|
||||
if err != nil || stats.ContainerJSONBase.State.Status != "running" {
|
||||
// REMOVE
|
||||
if err == nil {
|
||||
log.Printf("Status: %s, should kill: %s", stats.ContainerJSONBase.State.Status, identifier)
|
||||
err = removeContainer(identifier)
|
||||
if err != nil {
|
||||
log.Printf("Error killing container: %s", err)
|
||||
}
|
||||
} else {
|
||||
//log.Printf("WHAT TO DO HERE?: %s", err)
|
||||
}
|
||||
} else if stats.ContainerJSONBase.State.Status == "running" {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(action.Parameters) == 0 {
|
||||
action.Parameters = []WorkflowAppActionParameter{}
|
||||
}
|
||||
|
||||
if len(action.Errors) == 0 {
|
||||
action.Errors = []string{}
|
||||
}
|
||||
|
||||
// marshal action and put it in there rofl
|
||||
log.Printf("Time to execute %s with app %s:%s, function %s, env %s with %d parameters.", action.ID, action.AppName, action.AppVersion, action.Name, action.Environment, len(action.Parameters))
|
||||
actionData, err := json.Marshal(action)
|
||||
if err != nil {
|
||||
log.Printf("Failed unmarshalling action: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
//log.Println(string(actionData))
|
||||
// FIXME - add proper FUNCTION_APIKEY from user definition
|
||||
env := []string{
|
||||
fmt.Sprintf("ACTION=%s", string(actionData)),
|
||||
fmt.Sprintf("EXECUTIONID=%s", workflowExecution.ExecutionId),
|
||||
fmt.Sprintf("FUNCTION_APIKEY=%s", "asdasd"),
|
||||
fmt.Sprintf("AUTHORIZATION=%s", workflowExecution.Authorization),
|
||||
fmt.Sprintf("CALLBACK_URL=%s", baseUrl),
|
||||
}
|
||||
|
||||
err = deployApp(dockercli, image, identifier, env)
|
||||
if err != nil {
|
||||
log.Printf("Failed deploying %s from image %s: %s", identifier, image, err)
|
||||
log.Printf("Should send status and exit the entire thing?")
|
||||
//shutdown(workflowExecution.ExecutionId)
|
||||
}
|
||||
|
||||
visited = append(visited, action.ID)
|
||||
//log.Printf("%#v", action)
|
||||
}
|
||||
|
||||
//log.Println(nextAction)
|
||||
//log.Println(startAction, children[startAction])
|
||||
|
||||
// FIXME - new request here
|
||||
// FIXME - clean up stopped (remove) containers with this execution id
|
||||
newresp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Failed making request: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(newresp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Failed reading body: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
if newresp.StatusCode != 200 {
|
||||
log.Printf("Err: %s\nStatusCode: %d", string(body), newresp.StatusCode)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &workflowExecution)
|
||||
if err != nil {
|
||||
log.Printf("Failed workflowExecution unmarshal: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" {
|
||||
log.Printf("Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId)
|
||||
shutdown(workflowExecution.ExecutionId)
|
||||
}
|
||||
|
||||
log.Printf("Status: %s, Results: %d, actions: %d", workflowExecution.Status, len(workflowExecution.Results), len(workflowExecution.Workflow.Actions))
|
||||
if workflowExecution.Status != "EXECUTING" {
|
||||
log.Printf("Exiting as worker execution has status %s!", workflowExecution.Status)
|
||||
shutdown(workflowExecution.ExecutionId)
|
||||
}
|
||||
|
||||
if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions) {
|
||||
shutdownCheck := true
|
||||
ctx := context.Background()
|
||||
for _, result := range workflowExecution.Results {
|
||||
if result.Status == "EXECUTING" {
|
||||
// Cleaning up executing stuff
|
||||
shutdownCheck = false
|
||||
// Check status
|
||||
|
||||
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
|
||||
All: true,
|
||||
})
|
||||
if err != nil {
|
||||
log.Printf("Failed listing containers: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
stopContainers := []string{}
|
||||
removeContainers := []string{}
|
||||
for _, container := range containers {
|
||||
for _, name := range container.Names {
|
||||
if !strings.Contains(name, result.Action.ID) {
|
||||
continue
|
||||
}
|
||||
|
||||
if container.State != "running" {
|
||||
removeContainers = append(removeContainers, container.ID)
|
||||
stopContainers = append(stopContainers, container.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FIXME - add killing of apps with same execution ID too
|
||||
// FIXME - stahp
|
||||
//for _, containername := range stopContainers {
|
||||
// if err := dockercli.ContainerStop(ctx, containername, nil); err != nil {
|
||||
// log.Printf("Unable to stop container: %s", err)
|
||||
// } else {
|
||||
// log.Printf("Stopped container %s", containername)
|
||||
// }
|
||||
//}
|
||||
|
||||
removeOptions := types.ContainerRemoveOptions{
|
||||
RemoveVolumes: true,
|
||||
Force: true,
|
||||
}
|
||||
|
||||
_ = removeOptions
|
||||
|
||||
// FIXME - this
|
||||
//for _, containername := range removeContainers {
|
||||
// if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil {
|
||||
// log.Printf("Unable to remove container: %s", err)
|
||||
// } else {
|
||||
// log.Printf("Removed container %s", containername)
|
||||
// }
|
||||
//}
|
||||
|
||||
// FIXME - send POST request to kill the container
|
||||
log.Printf("Should remove (POST request) stopped containers")
|
||||
//ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result)
|
||||
}
|
||||
}
|
||||
|
||||
if shutdownCheck {
|
||||
log.Println("BREAKING BECAUSE RESULTS IS SAME LENGTH AS ACTIONS. SHOULD CHECK ALL RESULTS FOR WHETHER THEY'RE DONE")
|
||||
shutdown(workflowExecution.ExecutionId)
|
||||
}
|
||||
}
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func arrayContains(visited []string, id string) bool {
|
||||
found := false
|
||||
for _, item := range visited {
|
||||
if item == id {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
|
||||
return found
|
||||
}
|
||||
|
||||
func getResult(workflowExecution WorkflowExecution, id string) ActionResult {
|
||||
for _, actionResult := range workflowExecution.Results {
|
||||
if actionResult.Action.ID == id {
|
||||
return actionResult
|
||||
}
|
||||
}
|
||||
|
||||
return ActionResult{}
|
||||
}
|
||||
|
||||
func getAction(workflowExecution WorkflowExecution, id string) Action {
|
||||
for _, action := range workflowExecution.Workflow.Actions {
|
||||
if action.ID == id {
|
||||
return action
|
||||
}
|
||||
}
|
||||
|
||||
return Action{}
|
||||
}
|
||||
|
||||
// Initial loop etc
|
||||
func main() {
|
||||
log.Printf("Setting up worker environment")
|
||||
|
||||
sleepTime := 5
|
||||
client := &http.Client{}
|
||||
authorization := os.Getenv("AUTHORIZATION")
|
||||
executionId := os.Getenv("EXECUTIONID")
|
||||
|
||||
if len(authorization) == 0 {
|
||||
log.Println("No AUTHORIZATION key set in env")
|
||||
shutdown(executionId)
|
||||
}
|
||||
|
||||
if len(executionId) == 0 {
|
||||
log.Println("No EXECUTIONID key set in env")
|
||||
shutdown(executionId)
|
||||
}
|
||||
|
||||
// FIXME - tmp
|
||||
data := fmt.Sprintf(`{"execution_id": "%s", "authorization": "%s"}`, executionId, authorization)
|
||||
fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl)
|
||||
req, err := http.NewRequest(
|
||||
"POST",
|
||||
fullUrl,
|
||||
bytes.NewBuffer([]byte(data)),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Println("Failed making request builder")
|
||||
shutdown(executionId)
|
||||
}
|
||||
|
||||
for {
|
||||
newresp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Failed request: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(newresp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Failed reading body: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
if newresp.StatusCode != 200 {
|
||||
log.Printf("Err: %s\nStatusCode: %d", string(body), newresp.StatusCode)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
var workflowExecution WorkflowExecution
|
||||
err = json.Unmarshal(body, &workflowExecution)
|
||||
if err != nil {
|
||||
log.Printf("Failed workflowExecution unmarshal: %s", err)
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
if workflowExecution.Status == "FINISHED" || workflowExecution.Status == "SUCCESS" {
|
||||
log.Printf("Workflow %s is finished. Exiting worker.", workflowExecution.ExecutionId)
|
||||
shutdown(executionId)
|
||||
}
|
||||
|
||||
if workflowExecution.Status == "EXECUTING" || workflowExecution.Status == "RUNNING" {
|
||||
//log.Printf("Status: %s", workflowExecution.Status)
|
||||
err = handleExecution(client, req, workflowExecution)
|
||||
if err != nil {
|
||||
log.Printf("Workflow %s is finished: %s", workflowExecution.ExecutionId, err)
|
||||
shutdown(executionId)
|
||||
}
|
||||
} else {
|
||||
log.Printf("Workflow %s has status %s. Exiting worker.", workflowExecution.ExecutionId, workflowExecution.Status)
|
||||
shutdown(executionId)
|
||||
}
|
||||
|
||||
//log.Println(string(body))
|
||||
time.Sleep(time.Duration(sleepTime) * time.Second)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import logging
|
||||
import requests
|
||||
|
||||
# Goal here:
|
||||
# * Make an app from WALKOFF able to run without app_base.py from WALKOFF
|
||||
# # How:
|
||||
# * Make it rely 100% on INPUT throug HTTP invocations instead of redis READS
|
||||
# # But really, how?
|
||||
# * Make a WORKER that reads the queue, and reuses a function
|
||||
|
||||
# Here to get it global
|
||||
apikey = ""
|
||||
try:
|
||||
apikey = os.environ["FUNCTION_APIKEY"]
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
# Authorize the execution
|
||||
def authorization(request):
|
||||
# This is basically my issue, but it enforces the use of an internal API key for execution
|
||||
try:
|
||||
apikey = os.environ["FUNCTION_APIKEY"]
|
||||
except KeyError:
|
||||
return f"Internal server error", 500
|
||||
|
||||
|
||||
# Check API key from ENV authentication
|
||||
authentication = request.headers.get("Authorization")
|
||||
if authentication == None: return f"Unauthorized", 401
|
||||
|
||||
apikey_split = authentication.split(" ")
|
||||
if apikey_split[0] != "Bearer" or len(apikey_split) != 2:
|
||||
return f"Apikey error", 401
|
||||
|
||||
if apikey != apikey_split[1]:
|
||||
return f"Unauthorized", 401
|
||||
|
||||
return run(request)
|
||||
|
||||
class AppBase:
|
||||
""" The base class for Python-based Walkoff applications, handles Redis and logging configurations. """
|
||||
__version__ = None
|
||||
app_name = None
|
||||
|
||||
def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None):
|
||||
self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger")
|
||||
self.redis=redis
|
||||
self.console_logger=console_logger
|
||||
self.current_execution_id = None
|
||||
self.url = "https://shuffler.io"
|
||||
self.apikey = apikey
|
||||
|
||||
@classmethod
|
||||
async def run(cls, action):
|
||||
""" Connect to Redis and HTTP session, await actions """
|
||||
logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{')
|
||||
logger = logging.getLogger(f"{cls.__name__}")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
|
||||
app = cls(redis=None, logger=logger, console_logger=logger)
|
||||
|
||||
# Authorization for the app/function to control the workflow
|
||||
# Function will crash if its wrong, which it probably should.
|
||||
|
||||
await app.execute_action(action)
|
||||
|
||||
async def execute_action(self, action):
|
||||
# FIXME - add request for the function STARTING here. Use "results stream" or something
|
||||
# PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
|
||||
|
||||
self.authorization = action["authorization"]
|
||||
self.execution_id = action["execution_id"]
|
||||
self.current_execution_id = action["execution_id"]
|
||||
@@ -0,0 +1,702 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"archive/tar"
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/docker/docker/api/types"
|
||||
"github.com/docker/docker/client"
|
||||
"google.golang.org/api/cloudfunctions/v1"
|
||||
"gopkg.in/yaml.v2"
|
||||
)
|
||||
|
||||
var gceProject = "shuffler"
|
||||
var bucketName = "shuffler.appspot.com"
|
||||
|
||||
type WorkflowAppActionParameter struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Example string `json:"example" datastore:"example"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline"`
|
||||
ActionField string `json:"action_field" datastore:"action_field"`
|
||||
Variant string `json:"variant", datastore:"variant"`
|
||||
Required bool `json:"required" datastore:"required"`
|
||||
Schema struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
} `json:"schema"`
|
||||
}
|
||||
|
||||
type Authentication struct {
|
||||
Required bool `json:"required" datastore:"required" yaml:"required" `
|
||||
Parameters []AuthenticationParams `json:"parameters" datastore:"parameters" yaml:"parameters"`
|
||||
}
|
||||
|
||||
type AuthenticationParams struct {
|
||||
Description string `json:"description" datastore:"description" yaml:"description"`
|
||||
ID string `json:"id" datastore:"id" yaml:"id"`
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Example string `json:"example" datastore:"example" yaml:"example"`
|
||||
Value string `json:"value" datastore:"value" yaml:"value"`
|
||||
Multiline bool `json:"multiline" datastore:"multiline" yaml:"multiline"`
|
||||
Required bool `json:"required" datastore:"required" yaml:"required"`
|
||||
}
|
||||
|
||||
type WorkflowApp struct {
|
||||
Name string `json:"name" yaml:"name" required:true datastore:"name"`
|
||||
IsValid bool `json:"is_valid" yaml:"is_valid" required:true datastore:"is_valid"`
|
||||
ID string `json:"id" yaml:"id" required:false datastore:"id"`
|
||||
Link string `json:"link" yaml:"link" required:false datastore:"link"`
|
||||
AppVersion string `json:"app_version" yaml:"app_version" required:true datastore:"app_version"`
|
||||
Description string `json:"description" datastore:"description" required:false yaml:"description"`
|
||||
Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"`
|
||||
Sharing bool `json:"sharing" datastore:"sharing" yaml:"sharing"`
|
||||
SmallImage string `json:"small_image" datastore:"small_image" required:false yaml:"small_image"`
|
||||
LargeImage string `json:"large_image" datastore:"large_image" yaml:"large_image" requred:false`
|
||||
ContactInfo struct {
|
||||
Name string `json:"name" datastore:"name" yaml:"name"`
|
||||
Url string `json:"url" datastore:"url" yaml:"url"`
|
||||
} `json:"contact_info" datastore:"contact_info" yaml:"contact_info" required:false`
|
||||
Actions []WorkflowAppAction `json:"actions" yaml:"actions" required:true datastore:"actions"`
|
||||
Authentication Authentication `json:"authentication" yaml:"authentication" required:false datastore:"authentication"`
|
||||
}
|
||||
|
||||
type AuthenticationStore struct {
|
||||
Key string `json:"key" datastore:"key"`
|
||||
Value string `json:"value" datastore:"value"`
|
||||
}
|
||||
|
||||
type WorkflowAppAction struct {
|
||||
Description string `json:"description" datastore:"description"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
NodeType string `json:"node_type" datastore:"node_type"`
|
||||
Environment string `json:"environment" datastore:"environment"`
|
||||
Parameters []WorkflowAppActionParameter `json:"parameters" datastore: "parameters"`
|
||||
Authentication []AuthenticationStore `json:"authentication" datastore:"authentication"`
|
||||
Returns struct {
|
||||
Description string `json:"description" datastore:"returns"`
|
||||
ID string `json:"id" datastore:"id"`
|
||||
Schema struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
} `json:"schema" datastore:"schema"`
|
||||
} `json:"returns" datastore:"returns"`
|
||||
}
|
||||
|
||||
func getRunner(classname string) string {
|
||||
return fmt.Sprintf(`
|
||||
# Run the actual thing after we've checked params
|
||||
def run(request):
|
||||
action = request.get_json()
|
||||
print(action)
|
||||
print(type(action))
|
||||
authorization_key = action.get("authorization")
|
||||
current_execution_id = action.get("execution_id")
|
||||
|
||||
if action and "name" in action and "app_name" in action:
|
||||
asyncio.run(%s.run(action), debug=True)
|
||||
return f'Attempting to execute function {action["name"]} in app {action["app_name"]}'
|
||||
else:
|
||||
return f'Invalid action'
|
||||
|
||||
`, classname)
|
||||
}
|
||||
|
||||
// Could use some kind of linting system too for this, but meh
|
||||
func formatAppfile(filedata []byte) (string, []byte) {
|
||||
lines := strings.Split(string(filedata), "\n")
|
||||
|
||||
newfile := []string{}
|
||||
classname := ""
|
||||
for _, line := range lines {
|
||||
if strings.Contains(line, "walkoff_app_sdk") {
|
||||
continue
|
||||
}
|
||||
|
||||
// Remap logging. CBA this right now
|
||||
// This issue also persists in onprem apps because of await thingies.. :(
|
||||
// FIXME
|
||||
if strings.Contains(line, "console_logger") && strings.Contains(line, "await") {
|
||||
continue
|
||||
//line = strings.Replace(line, "console_logger", "logger", -1)
|
||||
//log.Println(line)
|
||||
}
|
||||
|
||||
// Might not work with different import names
|
||||
// Could be fucked up with spaces everywhere? Idk
|
||||
if strings.Contains(line, "class") && strings.Contains(line, "(AppBase)") {
|
||||
items := strings.Split(line, " ")
|
||||
if len(items) > 0 && strings.Contains(items[1], "(AppBase)") {
|
||||
classname = strings.Split(items[1], "(")[0]
|
||||
} else {
|
||||
log.Println("Something wrong :( (horrible programming right here)")
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
|
||||
if strings.Contains(line, "if __name__ ==") {
|
||||
break
|
||||
}
|
||||
|
||||
// asyncio.run(HelloWorld.run(), debug=True)
|
||||
|
||||
newfile = append(newfile, line)
|
||||
}
|
||||
|
||||
filedata = []byte(strings.Join(newfile, "\n"))
|
||||
return classname, filedata
|
||||
}
|
||||
|
||||
// https://stackoverflow.com/questions/21060945/simple-way-to-copy-a-file-in-golang
|
||||
func Copy(src, dst string) error {
|
||||
in, err := os.Open(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
_, err = io.Copy(out, in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return out.Close()
|
||||
}
|
||||
func ZipFiles(filename string, files []string) error {
|
||||
newZipFile, err := os.Create(filename)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer newZipFile.Close()
|
||||
|
||||
zipWriter := zip.NewWriter(newZipFile)
|
||||
defer zipWriter.Close()
|
||||
|
||||
// Add files to zip
|
||||
for _, file := range files {
|
||||
zipfile, err := os.Open(file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer zipfile.Close()
|
||||
|
||||
// Get the file information
|
||||
info, err := zipfile.Stat()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header, err := zip.FileInfoHeader(info)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Using FileInfoHeader() above only uses the basename of the file. If we want
|
||||
// to preserve the folder structure we can overwrite this with the full path.
|
||||
filesplit := strings.Split(file, "/")
|
||||
if len(filesplit) > 1 {
|
||||
header.Name = filesplit[len(filesplit)-1]
|
||||
} else {
|
||||
header.Name = file
|
||||
}
|
||||
|
||||
// Change to deflate to gain better compression
|
||||
// see http://golang.org/pkg/archive/zip/#pkg-constants
|
||||
header.Method = zip.Deflate
|
||||
|
||||
writer, err := zipWriter.CreateHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err = io.Copy(writer, zipfile); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func getAppbase(filepath string) []string {
|
||||
appBase, err := ioutil.ReadFile(filepath)
|
||||
if err != nil {
|
||||
log.Printf("Readerror: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
record := false
|
||||
validLines := []string{}
|
||||
for _, line := range strings.Split(string(appBase), "\n") {
|
||||
if strings.Contains(line, "#STOPCOPY") {
|
||||
log.Println("Stopping copy")
|
||||
break
|
||||
}
|
||||
|
||||
if record {
|
||||
validLines = append(validLines, line)
|
||||
}
|
||||
|
||||
if strings.Contains(line, "#STARTCOPY") {
|
||||
log.Println("Starting copy")
|
||||
record = true
|
||||
}
|
||||
}
|
||||
|
||||
return validLines
|
||||
}
|
||||
|
||||
// Puts together ./static_baseline.py, onprem/app_sdk_app_base.py and the
|
||||
// appcode in a generated_app folder based on appname+version
|
||||
func stitcher(appname string, appversion string) string {
|
||||
baselinefile := "static_baseline.py"
|
||||
appfolder := "apps"
|
||||
appbasefile := "onprem/app_sdk/app_base.py"
|
||||
|
||||
baseline, err := ioutil.ReadFile(baselinefile)
|
||||
if err != nil {
|
||||
log.Printf("Readerror: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
sourceappfile := fmt.Sprintf("%s/%s/%s/src/app.py", appfolder, appname, appversion)
|
||||
appfile, err := ioutil.ReadFile(sourceappfile)
|
||||
if err != nil {
|
||||
log.Printf("App readerror: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
classname, appfile := formatAppfile(appfile)
|
||||
if len(classname) == 0 {
|
||||
log.Println("Failed finding classname in file.")
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
runner := getRunner(classname)
|
||||
appBase := getAppbase(appbasefile)
|
||||
|
||||
foldername := fmt.Sprintf("generated_apps/%s_%s", appname, appversion)
|
||||
err = os.Mkdir(foldername, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Println("Failed making temporary app folder. Probably already exists. Remaking")
|
||||
os.RemoveAll(foldername)
|
||||
os.MkdirAll(foldername, os.ModePerm)
|
||||
}
|
||||
|
||||
stitched := []byte(string(baseline) + strings.Join(appBase, "\n") + string(appfile) + string(runner))
|
||||
err = ioutil.WriteFile(fmt.Sprintf("%s/main.py", foldername), stitched, os.ModePerm)
|
||||
if err != nil {
|
||||
log.Println("Failed writing to stitched: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
err = Copy(fmt.Sprintf("%s/%s/%s/requirements.txt", appfolder, appname, appversion), fmt.Sprintf("%s/requirements.txt", foldername))
|
||||
if err != nil {
|
||||
log.Println("Failed writing to requirement: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
log.Printf("Successfully stitched files in %s/main.py", foldername)
|
||||
// Zip the folder
|
||||
files := []string{
|
||||
fmt.Sprintf("%s/main.py", foldername),
|
||||
fmt.Sprintf("%s/requirements.txt", foldername),
|
||||
}
|
||||
outputfile := fmt.Sprintf("%s.zip", foldername)
|
||||
|
||||
err = ZipFiles(outputfile, files)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Creates a client.
|
||||
client, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
log.Printf("Failed to create client: %v", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
// Create bucket handle
|
||||
bucket := client.Bucket(bucketName)
|
||||
|
||||
remotePath := fmt.Sprintf("apps/%s_%s.zip", appname, appversion)
|
||||
err = createFileFromFile(bucket, remotePath, outputfile)
|
||||
if err != nil {
|
||||
log.Printf("Failed to upload to bucket: %v", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
|
||||
os.Remove(outputfile)
|
||||
return fmt.Sprintf("gs://%s/apps/%s_%s.zip", bucketName, appname, appversion)
|
||||
}
|
||||
|
||||
func createFileFromFile(bucket *storage.BucketHandle, remotePath, localPath string) error {
|
||||
ctx := context.Background()
|
||||
// [START upload_file]
|
||||
f, err := os.Open(localPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
wc := bucket.Object(remotePath).NewWriter(ctx)
|
||||
if _, err = io.Copy(wc, f); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := wc.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
// [END upload_file]
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deploy to google cloud function :)
|
||||
func deployFunction(appname, localization, applocation string, environmentVariables map[string]string) error {
|
||||
ctx := context.Background()
|
||||
service, err := cloudfunctions.NewService(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// ProjectsLocationsListCall
|
||||
projectsLocationsFunctionsService := cloudfunctions.NewProjectsLocationsFunctionsService(service)
|
||||
location := fmt.Sprintf("projects/%s/locations/%s", gceProject, localization)
|
||||
functionName := fmt.Sprintf("%s/functions/%s", location, appname)
|
||||
|
||||
cloudFunction := &cloudfunctions.CloudFunction{
|
||||
AvailableMemoryMb: 128,
|
||||
EntryPoint: "authorization",
|
||||
EnvironmentVariables: environmentVariables,
|
||||
HttpsTrigger: &cloudfunctions.HttpsTrigger{},
|
||||
MaxInstances: 0,
|
||||
Name: functionName,
|
||||
Runtime: "python37",
|
||||
SourceArchiveUrl: applocation,
|
||||
}
|
||||
|
||||
//getCall := projectsLocationsFunctionsService.Get(fmt.Sprintf("%s/functions/function-5", location))
|
||||
//resp, err := getCall.Do()
|
||||
|
||||
createCall := projectsLocationsFunctionsService.Create(location, cloudFunction)
|
||||
_, err = createCall.Do()
|
||||
if err != nil {
|
||||
log.Println("Failed creating new function. Attempting patch, as it might exist already")
|
||||
|
||||
createCall := projectsLocationsFunctionsService.Patch(fmt.Sprintf("%s/functions/%s", location, appname), cloudFunction)
|
||||
_, err = createCall.Do()
|
||||
if err != nil {
|
||||
log.Println("Failed patching function")
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Successfully patched %s to %s", appname, localization)
|
||||
} else {
|
||||
log.Printf("Successfully deployed %s to %s", appname, localization)
|
||||
}
|
||||
|
||||
// FIXME - use response to define the HTTPS entrypoint. It's default to an easy one tho
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func deployAppCloudFunc(appname string, appversion string) {
|
||||
_ = os.Mkdir("generated_apps", os.ModePerm)
|
||||
|
||||
apikey := "eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ"
|
||||
fullAppname := fmt.Sprintf("%s-%s", strings.Replace(appname, "_", "-", -1), strings.Replace(appversion, ".", "-", -1))
|
||||
locations := []string{"europe-west2"}
|
||||
|
||||
// Deploys the app to all locations
|
||||
bucketname := stitcher(appname, appversion)
|
||||
environmentVariables := map[string]string{
|
||||
"FUNCTION_APIKEY": apikey,
|
||||
}
|
||||
|
||||
for _, location := range locations {
|
||||
err := deployFunction(fullAppname, location, bucketname, environmentVariables)
|
||||
if err != nil {
|
||||
log.Printf("Failed to deploy: %s", err)
|
||||
os.Exit(3)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func loadYaml(fileLocation string) (WorkflowApp, error) {
|
||||
action := WorkflowApp{}
|
||||
|
||||
yamlFile, err := ioutil.ReadFile(fileLocation)
|
||||
if err != nil {
|
||||
log.Printf("yamlFile.Get err: %s", err)
|
||||
return WorkflowApp{}, err
|
||||
}
|
||||
|
||||
//log.Printf(string(yamlFile))
|
||||
err = yaml.Unmarshal([]byte(yamlFile), &action)
|
||||
if err != nil {
|
||||
return WorkflowApp{}, err
|
||||
}
|
||||
|
||||
return action, nil
|
||||
}
|
||||
|
||||
// FIXME - deploy to backend (YAML config)
|
||||
func deployConfigToBackend(appname string, appversion string) error {
|
||||
// FIXME - no static path pls
|
||||
action, err := loadYaml(fmt.Sprintf("apps/%s/%s/api.yaml", appname, appversion))
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
return err
|
||||
}
|
||||
|
||||
action.Sharing = true
|
||||
|
||||
data, err := json.Marshal(action)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
url := "http://localhost:5001/api/v1/workflows/apps"
|
||||
client := &http.Client{}
|
||||
req, err := http.NewRequest(http.MethodPut, url, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Set("Authorization", "Bearer eyJhbGciOiJSUzI1NiIsImtpZCI6IjYwZjQwNjBlNThkNzVmZDNmNzBiZWZmODhjNzk0YTc3NTMyN2FhMzEiLCJ0eXAiOiJKV1QifQ.eyJhdWQiOiJodHRwczovL3NodWZmbGVyLmlvL2FwaS92MS93b3JrZmxvd3MvMWQ5ZDhjZTItNTY2ZS00YzNmLThhMzctNWQ2YzdkMjAwMGI1L2V4ZWN1dGUiLCJhenAiOiIxMDMwNzY3ODIwNjE0MjQ2MTg0MjIiLCJlbWFpbCI6InNjaGVkdWxlckBzaHVmZmxlLTI0MTUxNy5pYW0uZ3NlcnZpY2VhY2NvdW50LmNvbSIsImVtYWlsX3ZlcmlmaWVkIjp0cnVlLCJleHAiOjE1NjU1Mjc1NTEsImlhdCI6MTU2NTUyMzk1MSwiaXNzIjoiaHR0cHM6Ly9hY2NvdW50cy5nb29nbGUuY29tIiwic3ViIjoiMTAzMDc2NzgyMDYxNDI0NjE4NDIyIn0.r0EDq9fjhf_5CPTiltyfk_L3uYJp577Uy0yYPcCAl2nv50_z_oUtbWGBpQLL8gcj-NGd3g4E52Qur8k6hCMIQweLS6WAb1279vGffEoCNDfkWb3Oy-yJGP1kzwLvqFJqnHLkSWYXNWvSyWnEimW8Rryx_m1BXS5wcA8l4NIr83kS7fPZrTwjnwFSeGSThwk91DVARzapQb8r0GEgOUyHZ1aBXnV98mikzSUt-5xFKe9eMdD22YJAj0Ru-DxAxs5nOqghX4PMRysWjshjOMrlR1piPWxqAmewp8YKZDCQ5gXskpeAFBDoULT971Wsx_NCohnJsFqx1JfPS9ZYMTW2oQ")
|
||||
|
||||
ret, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Status: %s", ret.Status)
|
||||
body, err := ioutil.ReadAll(ret.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if ret.StatusCode != 200 {
|
||||
return errors.New(fmt.Sprintf("Status %s. App probably already exists. Raw:\n%s", ret.Status, string(body)))
|
||||
}
|
||||
|
||||
log.Println(string(body))
|
||||
return nil
|
||||
}
|
||||
|
||||
func tarDirectory(filecontext string) (io.Reader, error) {
|
||||
|
||||
// Create a filereader
|
||||
//dockerFileReader, err := os.Open(dockerfile)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
|
||||
//// Read the actual Dockerfile
|
||||
//readDockerFile, err := ioutil.ReadAll(dockerFileReader)
|
||||
//if err != nil {
|
||||
// return err
|
||||
//}
|
||||
|
||||
// Make a TAR header for the file
|
||||
tarHeader := &tar.Header{
|
||||
Name: filecontext,
|
||||
Typeflag: tar.TypeDir,
|
||||
}
|
||||
|
||||
// Writes the header described for the TAR file
|
||||
buf := new(bytes.Buffer)
|
||||
tw := tar.NewWriter(buf)
|
||||
defer tw.Close()
|
||||
err := tw.WriteHeader(tarHeader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
||||
return dockerFileTarReader, nil
|
||||
}
|
||||
|
||||
func tarDir(source string, target string) (*bytes.Reader, error) {
|
||||
filename := filepath.Base(source)
|
||||
target = filepath.Join(target, fmt.Sprintf("%s.tar", filename))
|
||||
tarfile, err := os.Create(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer tarfile.Close()
|
||||
|
||||
buf := new(bytes.Buffer)
|
||||
_ = buf
|
||||
tarball := tar.NewWriter(tarfile)
|
||||
defer tarball.Close()
|
||||
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var baseDir string
|
||||
if info.IsDir() {
|
||||
baseDir = filepath.Base(source)
|
||||
}
|
||||
|
||||
_ = filepath.Walk(source,
|
||||
func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
header, err := tar.FileInfoHeader(info, info.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if baseDir != "" {
|
||||
header.Name = filepath.Join(baseDir, strings.TrimPrefix(path, source))
|
||||
}
|
||||
|
||||
if err := tarball.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
_, err = io.Copy(tarball, file)
|
||||
return nil
|
||||
})
|
||||
|
||||
dockerFileTarReader := bytes.NewReader(buf.Bytes())
|
||||
return dockerFileTarReader, nil
|
||||
}
|
||||
|
||||
func buildImage(client *client.Client, tags []string, dockerBuildCtxDir string) error {
|
||||
dockerBuildContext, err := tarDir(dockerBuildCtxDir, ".")
|
||||
if err != nil {
|
||||
log.Printf("Error in taring the docker root folder - %s", err.Error())
|
||||
return err
|
||||
}
|
||||
|
||||
imageBuildResponse, err := client.ImageBuild(
|
||||
context.Background(),
|
||||
dockerBuildContext,
|
||||
types.ImageBuildOptions{
|
||||
Dockerfile: "Dockerfile",
|
||||
PullParent: true,
|
||||
Remove: true,
|
||||
Tags: tags,
|
||||
},
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read the STDOUT from the build process
|
||||
defer imageBuildResponse.Body.Close()
|
||||
_, err = io.Copy(os.Stdout, imageBuildResponse.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FIXME - deploy to dockerhub
|
||||
func deployWorker(appname, appversion string) error {
|
||||
// Get dockerfile from ./apps/appname/appversion/Dockerfile
|
||||
client, err := client.NewEnvClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
tags := []string{fmt.Sprintf("%s-%s", appname, appversion)}
|
||||
err = buildImage(client, tags, fmt.Sprintf("./apps/%s/%s", appname, appversion))
|
||||
if err != nil {
|
||||
log.Printf("Build error: %s", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deploys all cloud functions. Onprem thooo :(
|
||||
func deployAll() {
|
||||
allapps := []string{
|
||||
"hoxhunt",
|
||||
"secureworks",
|
||||
"servicenow",
|
||||
"lastline",
|
||||
"netcraft",
|
||||
"misp",
|
||||
"email",
|
||||
"testing",
|
||||
"http",
|
||||
"recordedfuture",
|
||||
"passivetotal",
|
||||
"carbon_black",
|
||||
"thehive",
|
||||
"cortex",
|
||||
"splunk",
|
||||
}
|
||||
|
||||
for _, appname := range allapps {
|
||||
appversion := "1.0.0"
|
||||
|
||||
err := deployConfigToBackend(appname, appversion)
|
||||
if err != nil {
|
||||
log.Printf("Failed uploading config: %s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
deployAppCloudFunc(appname, appversion)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
deployAll()
|
||||
return
|
||||
|
||||
appname := "testing"
|
||||
appversion := "1.0.0"
|
||||
|
||||
err := deployConfigToBackend(appname, appversion)
|
||||
if err != nil {
|
||||
log.Printf("Failed uploading config: %s", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
deployAppCloudFunc(appname, appversion)
|
||||
|
||||
// FIXME - build and deploy to dockerhub as well :)
|
||||
// Not able to work in remote directory propely... Even tried making an actual tar and checking it rofl
|
||||
//err := deployWorker(appname, appversion)
|
||||
//if err != nil {
|
||||
// log.Printf("Failed to deploy docker worker: %s", err)
|
||||
//}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
main.go
|
||||
*.swo
|
||||
*.swp
|
||||
@@ -0,0 +1,17 @@
|
||||
# Local testing
|
||||
1. Change hook.go package to main
|
||||
```bash
|
||||
mv ../main.go .
|
||||
go run main.go hook.go
|
||||
```
|
||||
|
||||
# Deploy local
|
||||
```bash
|
||||
gcloud functions deploy webhook --runtime go111 --entry-point Authorization --trigger-http --project shuffle-241517 --memory=128 --set-env-vars=FUNCTION_APIKEY=asdasd,CALLBACKURL=shuffler.io,HOOKID=test123
|
||||
```
|
||||
|
||||
# Build and deploy from gui
|
||||
1. rm webhook.zip
|
||||
2. zip webhook.zip hook.go
|
||||
3. Upload to bucket https://console.cloud.google.com/storage/browser/shuffle-241517.appspot.com?project=shuffle-241517
|
||||
4. Restart hook(s) (https://shuffler.io/webhooks)
|
||||
@@ -0,0 +1,415 @@
|
||||
package main
|
||||
|
||||
// APPS:
|
||||
// apps.dev.microsoft.com
|
||||
|
||||
// REMOVE ACCESS:
|
||||
// https://portal.office.com/account/#
|
||||
|
||||
// Developer:
|
||||
// https://developer.microsoft.com/en-us/graph/docs/concepts/permissions_reference
|
||||
|
||||
// Bots:
|
||||
// https://dev.botframework.com/bots
|
||||
|
||||
// Connectors
|
||||
// https://outlook.office.com/connectors/home/login/#/new
|
||||
// https://go.microsoft.com/fwlink/?linkid=857599
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Info struct {
|
||||
Url string `json:"url" datastore:"url"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Description string `json:"description" datastore:"description"`
|
||||
}
|
||||
|
||||
// Actions to be done by webhooks etc
|
||||
// Field is the actual field to use from json
|
||||
type HookAction struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Id string `json:"id" datastore:"id"`
|
||||
Field string `json:"field" datastore:"field"`
|
||||
}
|
||||
|
||||
type Hook struct {
|
||||
Id string `json:"id" datastore:"id"`
|
||||
Info Info `json:"info" datastore:"info"`
|
||||
Actions []HookAction `json:"actions" datastore:"actions"`
|
||||
Type string `json:"type" datastore:"type"`
|
||||
Status string `json:"status" datastore:"status"`
|
||||
Running bool `json:"running" datastore:"running"`
|
||||
}
|
||||
|
||||
type TeamsHook struct {
|
||||
MembersAdded []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"membersAdded"`
|
||||
Type string `json:"type"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
LocalTimestamp string `json:"localTimestamp"`
|
||||
ID string `json:"id"`
|
||||
ChannelID string `json:"channelId"`
|
||||
ServiceURL string `json:"serviceUrl"`
|
||||
From struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"from"`
|
||||
Conversation struct {
|
||||
IsGroup bool `json:"isGroup"`
|
||||
ConversationType string `json:"conversationType"`
|
||||
ID string `json:"id"`
|
||||
TenantID string `json:"tenantId"`
|
||||
} `json:"conversation"`
|
||||
Recipient struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"recipient"`
|
||||
ChannelData struct {
|
||||
Team struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"team"`
|
||||
EventType string `json:"eventType"`
|
||||
Tenant struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tenant"`
|
||||
} `json:"channelData"`
|
||||
}
|
||||
|
||||
var hook Hook
|
||||
var baseUrl = "https://shuffler.io"
|
||||
|
||||
type OauthToken struct {
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
ExtExpiresIn int `json:"ext_expires_in"`
|
||||
AccessToken string `json:"access_token"`
|
||||
}
|
||||
|
||||
type TeamsResponse struct {
|
||||
Conversation struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"conversation"`
|
||||
From struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"from"`
|
||||
Recipient struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"recipient"`
|
||||
ReplyToId string `json:"replyToId"`
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
// This should be in a token thingy, to be controlled in workflow
|
||||
func sendRequest(token OauthToken, message TeamsHook) error {
|
||||
//POST https://smba.trafficmanager.net/apis/v3/conversations/12345/activities
|
||||
//Authorization: Bearer eyJhbGciOiJIUzI1Ni...
|
||||
//
|
||||
//(JSON-serialized Activity message goes here)
|
||||
|
||||
tmpData := TeamsResponse{}
|
||||
tmpData.Conversation.ID = message.Conversation.ID
|
||||
tmpData.From = message.Recipient
|
||||
tmpData.Recipient = message.From
|
||||
tmpData.ReplyToId = message.ID
|
||||
tmpData.Type = "message"
|
||||
tmpData.Text = "HELO"
|
||||
|
||||
data, err := json.Marshal(tmpData)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// /v3/conversations/{conversationId}/activities/{activityId}
|
||||
fullurl := fmt.Sprintf("%sv3/conversations/%s/activities", message.ServiceURL, message.Conversation.ID)
|
||||
log.Println(fullurl)
|
||||
log.Println(string(data))
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
fullurl,
|
||||
bytes.NewBuffer([]byte(data)),
|
||||
)
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", token.AccessToken))
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Status: %d", res.StatusCode)
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Println(string(body))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func get_accesstoken() (OauthToken, error) {
|
||||
client_id := "9a2a2a63-c63c-4487-baf0-4ff3f4873a7f"
|
||||
client_secret := ":3]D6oFimiXbuV20xH?Dzu@LR*6IFVbq"
|
||||
fullurl := fmt.Sprintf("https://login.microsoftonline.com/botframework.com/oauth2/v2.0/token")
|
||||
data := fmt.Sprintf("grant_type=client_credentials&client_id=%s&client_secret=%s&scope=https://api.botframework.com/.default", client_id, client_secret)
|
||||
|
||||
log.Println(data)
|
||||
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
fullurl,
|
||||
bytes.NewBuffer([]byte(data)),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return OauthToken{}, err
|
||||
}
|
||||
|
||||
client := http.Client{}
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
return OauthToken{}, err
|
||||
}
|
||||
|
||||
log.Printf("Status: %d", res.StatusCode)
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return OauthToken{}, err
|
||||
}
|
||||
|
||||
token := OauthToken{}
|
||||
err = json.Unmarshal(body, &token)
|
||||
if err != nil {
|
||||
return OauthToken{}, err
|
||||
}
|
||||
|
||||
return token, nil
|
||||
}
|
||||
|
||||
//func CheckTenantId(message TeamsHook) {
|
||||
// fullurl := fmt.Sprintf("%s/api/v1/functions/tenants/%s", baseUrl, message.Conversation.TenantID)
|
||||
// req, err := http.NewRequest(
|
||||
// http.MethodPost,
|
||||
// fullurl,
|
||||
// bytes.NewBuffer([]byte(data)),
|
||||
// )
|
||||
//
|
||||
// req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, baseApikey))
|
||||
// req.Header.Add("Content-Type", "application/json")
|
||||
// if err != nil {
|
||||
// return []string{}, err
|
||||
// }
|
||||
//
|
||||
// client := http.Client{}
|
||||
// res, err := client.Do(req)
|
||||
// if err != nil {
|
||||
// return []string{}, err
|
||||
// }
|
||||
//
|
||||
// log.Printf("Status: %d", res.StatusCode)
|
||||
// body, err := ioutil.ReadAll(res.Body)
|
||||
// if err != nil {
|
||||
// return []string{}, err
|
||||
// }
|
||||
//}
|
||||
|
||||
func Authorization(resp http.ResponseWriter, request *http.Request) {
|
||||
// FIXME - don't have this here, but before loops etc
|
||||
// How to keep it refreshed?
|
||||
token, err := get_accesstoken()
|
||||
if err != nil {
|
||||
log.Printf("Failed: %s", err)
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
log.Println("Data")
|
||||
log.Println(string(body))
|
||||
|
||||
hook := TeamsHook{}
|
||||
err = json.Unmarshal(body, &hook)
|
||||
if err != nil {
|
||||
resp.WriteHeader(200)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
}
|
||||
|
||||
// Only handle messages currently
|
||||
if hook.Type != "message" {
|
||||
resp.WriteHeader(200)
|
||||
resp.Write([]byte(`{"success": false}`))
|
||||
return
|
||||
}
|
||||
|
||||
// Find the ORG based on the above info. How?
|
||||
// MSTeams hook should have it attached somehow?
|
||||
|
||||
log.Printf(string(body))
|
||||
//log.Printf(hook.ServiceURL)
|
||||
//log.Printf(hook.ChannelID)
|
||||
//log.Printf(hook.ID)
|
||||
//log.Printf("%#v", hook.Conversation)
|
||||
|
||||
err = sendRequest(token, hook)
|
||||
if err != nil {
|
||||
log.Printf("Failed: %s", err)
|
||||
}
|
||||
|
||||
resp.WriteHeader(200)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
|
||||
}
|
||||
|
||||
func loadConfiguration(fullUrl string, apikey string) (Hook, error) {
|
||||
client := &http.Client{}
|
||||
|
||||
req, err := http.NewRequest(
|
||||
"GET",
|
||||
fullUrl,
|
||||
nil,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error making http request: %s", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Error in http request: %s", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Error reading response: %s", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &hook)
|
||||
if err != nil {
|
||||
log.Printf("Failed unmarshaling hook API", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
return hook, nil
|
||||
}
|
||||
|
||||
// GetUserDetails - Get one user's details from randomuser.me API
|
||||
func ForwardRequest(resp http.ResponseWriter, request *http.Request) error {
|
||||
callbackUrl := os.Getenv("CALLBACKURL")
|
||||
hookId := os.Getenv("HOOKID")
|
||||
apikey := os.Getenv("FUNCTION_APIKEY")
|
||||
|
||||
hook, err := loadConfiguration(
|
||||
fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId),
|
||||
apikey,
|
||||
)
|
||||
|
||||
log.Println("Done loading!")
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("%#v", hook)
|
||||
|
||||
// Find all things to execute
|
||||
workflowUrls := []string{}
|
||||
for _, item := range hook.Actions {
|
||||
if item.Type == "" {
|
||||
log.Printf("CONTINUE AAS EMPTY ITEM: %#v", item)
|
||||
continue
|
||||
}
|
||||
|
||||
if item.Type == "workflow" {
|
||||
workflowUrls = append(workflowUrls, item.Id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(workflowUrls) == 0 {
|
||||
return errors.New("No actions to do yet")
|
||||
}
|
||||
|
||||
log.Printf("Should send data to the following: %s", strings.Join(workflowUrls, ", "))
|
||||
|
||||
randomUserClient := http.Client{
|
||||
Timeout: time.Second * 3,
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare data
|
||||
type arg struct {
|
||||
ExecutionArgument string `json:"execution_argument"`
|
||||
}
|
||||
data := arg{
|
||||
ExecutionArgument: string(body),
|
||||
}
|
||||
|
||||
newjson, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Loop all executions to run
|
||||
for _, item := range workflowUrls {
|
||||
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, item)
|
||||
log.Printf("Sending data to %s", fullUrl)
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
fullUrl,
|
||||
bytes.NewBuffer(newjson),
|
||||
)
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res, err := randomUserClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Status: %d", res.StatusCode)
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf(string(body))
|
||||
}
|
||||
|
||||
//log.Println(string(newbody))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/gorilla/handlers"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
func webhook() {
|
||||
// FIXME - remove static
|
||||
port := ":8080"
|
||||
baseFilePath := "/"
|
||||
|
||||
mux := mux.NewRouter()
|
||||
mux.SkipClean(true)
|
||||
|
||||
// FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend
|
||||
mux.HandleFunc(baseFilePath, Authorization).Methods("POST")
|
||||
mux.HandleFunc("/test", Authorization).Methods("POST")
|
||||
|
||||
handlers.LoggingHandler(os.Stdout, mux)
|
||||
loggedRouter := handlers.LoggingHandler(os.Stdout, mux)
|
||||
|
||||
log.Printf("Starting on http://localhost%s", port)
|
||||
err := http.ListenAndServe(
|
||||
port,
|
||||
loggedRouter,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal("ListenAndServer: ", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func main() {
|
||||
webhook()
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"$schema": "https://developer.microsoft.com/en-us/json-schemas/teams/v1.5/MicrosoftTeams.schema.json",
|
||||
"manifestVersion": "1.5",
|
||||
"version": "1.0.0",
|
||||
"id": "9a2a2a63-c63c-4487-baf0-4ff3f4873a7f",
|
||||
"packageName": "com.example.myapp",
|
||||
"devicePermissions" : [],
|
||||
"developer": {
|
||||
"name": "@frikkylikeme",
|
||||
"websiteUrl": "https://shuffler.io/",
|
||||
"privacyUrl": "https://shuffler.io/privacy",
|
||||
"termsOfUseUrl": "https://shuffler.io/tos"
|
||||
},
|
||||
"localizationInfo": {
|
||||
"defaultLanguageTag": "en-us"
|
||||
},
|
||||
"name": {
|
||||
"short": "Shuffle",
|
||||
"full": "Shuffle"
|
||||
},
|
||||
"description": {
|
||||
"short": "Shuffle is a workflow automation platform",
|
||||
"full": "Shuffle is a workflow automation platform. Find more info at https://shuffler.io"
|
||||
},
|
||||
"icons": {
|
||||
"outline": "outline.png",
|
||||
"color": "color.png"
|
||||
},
|
||||
"accentColor": "#15202b",
|
||||
"bots": [
|
||||
{
|
||||
"botId": "9a2a2a63-c63c-4487-baf0-4ff3f4873a7f",
|
||||
"needsChannelSelector": false,
|
||||
"isNotificationOnly": false,
|
||||
"scopes": [ "team", "personal", "groupchat" ],
|
||||
"supportsFiles": false,
|
||||
"commandLists": [
|
||||
{
|
||||
"scopes": [ "team", "groupchat", "personal" ],
|
||||
"commands": [
|
||||
{
|
||||
"title": "test",
|
||||
"description": "THIS IS FOR TESTING"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
curl -XPOST http://localhost:8080 -d '{
|
||||
"membersAdded": [
|
||||
{
|
||||
"id": "28:f5d48856-5b42-41a0-8c3a-c5f944b679b0"
|
||||
}
|
||||
],
|
||||
"type": "conversationUpdate",
|
||||
"timestamp": "2017-02-23T19:38:35.312Z",
|
||||
"localTimestamp": "2017-02-23T12:38:35.312-07:00",
|
||||
"id": "f:5f85c2ad",
|
||||
"channelId": "msteams",
|
||||
"serviceUrl": "https://smba.trafficmanager.net/amer-client-ss.msg/",
|
||||
"from": {
|
||||
"id": "29:1I9Is_Sx0OIy2rQ7Xz1lcaPKlO9eqmBRTBuW6XzkFtcjqxTjPaCMij8BVMdBcL9L_RwWNJyAHFQb0TRzXgyQvA"
|
||||
},
|
||||
"conversation": {
|
||||
"isGroup": true,
|
||||
"conversationType": "channel",
|
||||
"id": "19:efa9296d959346209fea44151c742e73@thread.skype"
|
||||
},
|
||||
"recipient": {
|
||||
"id": "28:f5d48856-5b42-41a0-8c3a-c5f944b679b0",
|
||||
"name": "SongsuggesterBot"
|
||||
},
|
||||
"channelData": {
|
||||
"team": {
|
||||
"id": "19:efa9296d959346209fea44151c742e73@thread.skype"
|
||||
},
|
||||
"eventType": "teamMemberAdded",
|
||||
"tenant": {
|
||||
"id": "72f988bf-86f1-41af-91ab-2d7cd011db47"
|
||||
}
|
||||
}
|
||||
}'
|
||||
#{"type":"message","id":"4oN7bHB4dit7scwHygF1pf-h|0000000","timestamp":"2019-09-06T15:21:21.9035613Z","serviceUrl":"https://webchat.botframework.com/","channelId":"webchat","from":{"id":"4ccfb6b9-5755-426e-914d-641dd74f5e0f"},"conversation":{"id":"4oN7bHB4dit7scwHygF1pf-h"},"recipient":{"id":"Shuffle@qKw6tMx9fE8","name":"Shuffler"},"textFormat":"plain","locale":"en-US","text":"hi","entities":[{"type":"ClientCapabilities","requiresBotState":true,"supportsListening":true,"supportsTts":true}],"channelData":{"clientActivityID":"15677832808420.ishosdmdfbd"}}
|
||||
@@ -0,0 +1,57 @@
|
||||
# Outlook trigger
|
||||
Makes it possible to trigger a workflow based on an email
|
||||
|
||||
## Local testing - Same as ../webhook
|
||||
```bash
|
||||
mv ../main.go
|
||||
go run main.go hook.go
|
||||
```
|
||||
|
||||
# Deploy gcloud
|
||||
gcloud functions deploy outlooktrigger --runtime go111 --entry-point Authorization --trigger-http --project shuffler --memory=128 --set-env-vars=FUNCTION_APIKEY=asdasd,CALLBACKURL=shuffler.io,TRIGGERID=test123,WORKFLOW_ID=YOUR_WORKFLOW_ID
|
||||
|
||||
# Build and deploy
|
||||
1. Set hook.go line 1 from "package main" to "package function"
|
||||
2. zip outlooktrigger.tar hook.go
|
||||
3. Upload to bucket https://console.cloud.google.com/storage/browser/shuffler.appspot.com?project=shuffler
|
||||
4. Go to the functions https://console.cloud.google.com/functions/list?project=shuffler
|
||||
|
||||
|
||||
## How it works (from frontend to backend)
|
||||
### Choose mailfolders
|
||||
1. Use microsoft graph api to get the folders the user wants to listen to
|
||||
* Have the user write their primary email (default) or another one
|
||||
* Have it show the folders for the email with chooseable buttons somehow
|
||||
|inbox
|
||||
|-subinbox
|
||||
|--subsubinbox <-- choose e.g. this one
|
||||
|otherfolder
|
||||
|
||||
API:
|
||||
// requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/me/mailfolders")
|
||||
|
||||
### Add callback subscription
|
||||
2. Make an APIcall to ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") with callback url defined as "https://shuffler.io/api/v1/workflows/{key}/email/authorize"
|
||||
* Should this be set up whenever the user clicked start or when the workflow is created?
|
||||
* Start click ->
|
||||
1. Add another cloud function for the item
|
||||
2. When it's ready, deploy it to authorize
|
||||
3. Show it as ready
|
||||
|
||||
### Remove a subscription
|
||||
* Since everything is already generated above, one would need to
|
||||
* https://docs.microsoft.com/en-us/graph/api/subscription-delete?view=graph-rest-1.0&tabs=http
|
||||
* DELETE https://graph.microsoft.com/v1.0/subscriptions/{id}
|
||||
|
||||
|
||||
## CREATE - Fixme: LIST all current subscriptions, and stop them if they're towards the same endpoint
|
||||
* POST /api/v1/workflows/{key}/outlook
|
||||
* createOutlookSub(resp, request)
|
||||
* getOutlookSubscriptions(client) // Used to remove all existing for same endpoint
|
||||
* makeOutlookSubscription(client, folderIds, notificationUrl)
|
||||
* Add data from ^ to triggerAuth
|
||||
|
||||
## DELETE
|
||||
* DELETE /api/v1/workflows/{key}/outlook/{triggerId}
|
||||
* handleDeleteOutlookSub(resp, request)
|
||||
* handleOutlookSubRemoval(workflowId, triggerId)
|
||||
@@ -0,0 +1,222 @@
|
||||
package function
|
||||
|
||||
// Shuffle:
|
||||
// https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/Authentication/appId/e080cbf4-5dba-44b4-8643-a7c982189c16/isMSAApp//defaultBlade/Overview/servicePrincipalCreated/true
|
||||
|
||||
// Oauth playground:
|
||||
// https://oauthplay.azurewebsites.net/
|
||||
|
||||
// APPS:
|
||||
// https://apps.dev.microsoft.com
|
||||
|
||||
// REMOVE ACCESS:
|
||||
// https://portal.office.com/account/#
|
||||
|
||||
// Developer:
|
||||
// https://developer.microsoft.com/en-us/graph/docs/concepts/permissions_reference
|
||||
|
||||
// Bots:
|
||||
// https://dev.botframework.com/bots
|
||||
|
||||
// Connectors
|
||||
// https://outlook.office.com/connectors/home/login/#/new
|
||||
// https://go.microsoft.com/fwlink/?linkid=857599
|
||||
|
||||
import (
|
||||
//"encoding/json"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Info struct {
|
||||
Url string `json:"url" datastore:"url"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Description string `json:"description" datastore:"description"`
|
||||
}
|
||||
|
||||
// Actions to be done by webhooks etc
|
||||
// Field is the actual field to use from json
|
||||
type HookAction struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Id string `json:"id" datastore:"id"`
|
||||
Field string `json:"field" datastore:"field"`
|
||||
}
|
||||
|
||||
type Hook struct {
|
||||
Id string `json:"id" datastore:"id"`
|
||||
Info Info `json:"info" datastore:"info"`
|
||||
Actions []HookAction `json:"actions" datastore:"actions"`
|
||||
Type string `json:"type" datastore:"type"`
|
||||
Status string `json:"status" datastore:"status"`
|
||||
Running bool `json:"running" datastore:"running"`
|
||||
}
|
||||
|
||||
type TeamsHook struct {
|
||||
MembersAdded []struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"membersAdded"`
|
||||
Type string `json:"type"`
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
LocalTimestamp string `json:"localTimestamp"`
|
||||
ID string `json:"id"`
|
||||
ChannelID string `json:"channelId"`
|
||||
ServiceURL string `json:"serviceUrl"`
|
||||
From struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"from"`
|
||||
Conversation struct {
|
||||
IsGroup bool `json:"isGroup"`
|
||||
ConversationType string `json:"conversationType"`
|
||||
ID string `json:"id"`
|
||||
TenantID string `json:"tenantId"`
|
||||
} `json:"conversation"`
|
||||
Recipient struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"recipient"`
|
||||
ChannelData struct {
|
||||
Team struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"team"`
|
||||
EventType string `json:"eventType"`
|
||||
Tenant struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"tenant"`
|
||||
} `json:"channelData"`
|
||||
}
|
||||
|
||||
var hook Hook
|
||||
var baseUrl = "https://shuffler.io"
|
||||
|
||||
type OauthToken struct {
|
||||
TokenType string `json:"token_type"`
|
||||
ExpiresIn int `json:"expires_in"`
|
||||
ExtExpiresIn int `json:"ext_expires_in"`
|
||||
AccessToken string `json:"access_token"`
|
||||
}
|
||||
|
||||
type TeamsResponse struct {
|
||||
Conversation struct {
|
||||
ID string `json:"id"`
|
||||
} `json:"conversation"`
|
||||
From struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"from"`
|
||||
Recipient struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"recipient"`
|
||||
ReplyToId string `json:"replyToId"`
|
||||
Type string `json:"type"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type O365hook struct {
|
||||
OdataContext string `json:"@odata.context"`
|
||||
Value []struct {
|
||||
OdataType string `json:"@odata.type"`
|
||||
ID interface{} `json:"Id"`
|
||||
SubscriptionID string `json:"SubscriptionId"`
|
||||
SubscriptionExpirationDateTime time.Time `json:"SubscriptionExpirationDateTime"`
|
||||
SequenceNumber int `json:"SequenceNumber"`
|
||||
ChangeType string `json:"ChangeType"`
|
||||
Resource string `json:"Resource"`
|
||||
ResourceData struct {
|
||||
OdataType string `json:"@odata.type"`
|
||||
OdataID string `json:"@odata.id"`
|
||||
OdataEtag string `json:"@odata.etag"`
|
||||
ID string `json:"Id"`
|
||||
} `json:"ResourceData"`
|
||||
} `json:"value"`
|
||||
}
|
||||
|
||||
func Authorization(resp http.ResponseWriter, request *http.Request) {
|
||||
body, err := ioutil.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
log.Printf("Body: %s", err)
|
||||
resp.WriteHeader(403)
|
||||
return
|
||||
}
|
||||
|
||||
if len(body) > 0 {
|
||||
// In here - get the email data
|
||||
// Check who it belongs to and run those workflows
|
||||
// This should be set from run.go with the callback
|
||||
// userId: {subscriptionID: {workflowID}}
|
||||
// E.g. workflow: {auth: {userId
|
||||
// workflow: {trigger: {
|
||||
|
||||
err = forwardRequest(body)
|
||||
if err != nil {
|
||||
log.Printf("Failed unmarshal: %s", err)
|
||||
resp.WriteHeader(403)
|
||||
return
|
||||
}
|
||||
|
||||
resp.WriteHeader(200)
|
||||
resp.Write([]byte("OK"))
|
||||
return
|
||||
}
|
||||
|
||||
token := request.URL.Query().Get("validationToken")
|
||||
if len(token) == 0 {
|
||||
log.Println("Validation token is missing")
|
||||
resp.WriteHeader(403)
|
||||
return
|
||||
}
|
||||
|
||||
resp.WriteHeader(200)
|
||||
resp.Write([]byte(string(token)))
|
||||
}
|
||||
|
||||
// GetUserDetails - Get one user's details from randomuser.me API
|
||||
func forwardRequest(body []byte) error {
|
||||
callbackUrl := os.Getenv("CALLBACKURL")
|
||||
workflowId := os.Getenv("WORKFLOW_ID")
|
||||
apikey := os.Getenv("FUNCTION_APIKEY")
|
||||
|
||||
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, workflowId)
|
||||
//log.Printf("Sending data to %s", fullUrl)
|
||||
|
||||
data := fmt.Sprintf(`{"execution_argument": "%s"}`, string(body))
|
||||
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
fullUrl,
|
||||
bytes.NewBuffer([]byte(data)),
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
randomUserClient := http.Client{
|
||||
Timeout: time.Second * 5,
|
||||
}
|
||||
|
||||
res, err := randomUserClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Status: %d", res.StatusCode)
|
||||
returnbody, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("New body: %s", string(returnbody))
|
||||
|
||||
//log.Println(string(newbody))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"clientID": "70e37005-c954-4290-b573-d4b94e484336",
|
||||
"clientSecret": ".eNw/A[kQFB5zL.agvRputdEJENeJ392",
|
||||
"RedirectURL": "https://44d84ee7.ngrok.io/functions/outlook/register",
|
||||
"AuthURL": "https://login.microsoftonline.com/common/oauth2/authorize",
|
||||
"TokenURL": "https://login.microsoftonline.com/common/oauth2/token"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
-----BEGIN CERTIFICATE-----
|
||||
MIIDYDCCAkigAwIBAgIJAOvgxcclM1eyMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV
|
||||
BAYTAk5PMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX
|
||||
aWRnaXRzIFB0eSBMdGQwHhcNMTgwMzA1MTE1MTIwWhcNMjgwMzAyMTE1MTIwWjBF
|
||||
MQswCQYDVQQGEwJOTzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50
|
||||
ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
|
||||
CgKCAQEAowNqrvocJCLLcytYBfZhsqG3CkPsJ4PTicNyjuUGmlILPHlN1DE7jbDo
|
||||
KuOKImfV4AfQANnttaPksZyuKJL8XGpC7YmF5mmInUWG48SmZjvRbBi1LnCrjJKS
|
||||
ywh2lJqw4w30w2ItcpogDrYhh6+T3VyabcYngZjKSFgON3wo4I0c6aT19VVXGqnK
|
||||
y1WEejZmiChV4iwEu4vMPIzt16QpIBr8NPSkBLLRDAGWMjFnIuYDEwgjVn6XYhM9
|
||||
+NxhvY9es+qeqLQsZj2a1wcDGaw7G4iNZdltlPmlTCreRDBYBTsYCds/rJmPZ2xi
|
||||
6jgiyNZj3xHG5Knw2YIw0OwHyT9mWQIDAQABo1MwUTAdBgNVHQ4EFgQUckr5tCzg
|
||||
F1eBID7mtWTfeqyX4g0wHwYDVR0jBBgwFoAUckr5tCzgF1eBID7mtWTfeqyX4g0w
|
||||
DwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAdH1aPcWBjJHF6n/
|
||||
NgIiRSEU4mi5I6RUlPuR7dN7dcmF1Ho7quurNFzknXwks+a62oKSnkkFxxwrv/d6
|
||||
dIH5kNibVs7oRxEpA3gUmXXKUW4RPrxAp2zN37t7zs5xbpTATxfJiIMP8Rjo0sOf
|
||||
SilS22Sn0e0HxBi78t3DJEZvOQ9KSRuD1g9gOAY4lj/fni6rVJo8YCR2MyjmQoXB
|
||||
luHDF4jTqi/TkXECfqQZu0pctx3maISpB1fAuaELwPDvqbLgoC97gl6bIEiKLkJ0
|
||||
JkbGnb997K80ztFvFAKGyUtsvEDCupe/fdBPFqCruAQWI/BqVJFTdRD43dWEQANF
|
||||
S8ApvA==
|
||||
-----END CERTIFICATE-----
|
||||
@@ -0,0 +1,27 @@
|
||||
-----BEGIN RSA PRIVATE KEY-----
|
||||
MIIEowIBAAKCAQEAowNqrvocJCLLcytYBfZhsqG3CkPsJ4PTicNyjuUGmlILPHlN
|
||||
1DE7jbDoKuOKImfV4AfQANnttaPksZyuKJL8XGpC7YmF5mmInUWG48SmZjvRbBi1
|
||||
LnCrjJKSywh2lJqw4w30w2ItcpogDrYhh6+T3VyabcYngZjKSFgON3wo4I0c6aT1
|
||||
9VVXGqnKy1WEejZmiChV4iwEu4vMPIzt16QpIBr8NPSkBLLRDAGWMjFnIuYDEwgj
|
||||
Vn6XYhM9+NxhvY9es+qeqLQsZj2a1wcDGaw7G4iNZdltlPmlTCreRDBYBTsYCds/
|
||||
rJmPZ2xi6jgiyNZj3xHG5Knw2YIw0OwHyT9mWQIDAQABAoIBABJ+9L/dyQuglwz+
|
||||
QgKLLhKinq4fftAM+ReMgZcNDW69GGFIMjh9TZCKHg2fu7Cjr3S37jXqhDoz2mL8
|
||||
sBYSd2fU9rsU+4hlOQb/OIrnaSn4Z46oTwZx6kUM7HL1Bt9dnexlTPxOS3HRYwnI
|
||||
SI2oslJPi4YhEaJ2v5ztwM8y20B/E/zSW2onWz5gB8/bdxSmuJaWfHioIEoac8Gf
|
||||
BE7jiYMnx9kKeVfgkKPMBKXhAyE2lbAz7N5nDS/4HkbUMh389RBvakc4gkv/QkSW
|
||||
LNuXpbcSqJiG0FcVutjYS87a/ul3IdhAYmZDTuvRUhNsWBiY3LSY4C9NlHhiJihg
|
||||
RU1kknECgYEA1dPT6n4hy/OK2mT3ThGQQFMJWsnmcHSsvuK31+UYZOI9kNtBVxc6
|
||||
HSHkh0G53o6o2wZLr2gMXy35O5ZxSbectA1Q4MJDlYBs1MfHdOVyE4z6mmA9TLN1
|
||||
c+9pYOC0qx+6NwPdO7j2xdUMEUfzocWOeay0AzJg20BQYmKuy3Kc+tcCgYEAwyn6
|
||||
n+XS0vodfJdHhvbW/jocQlHQOBK5HklZfq2PMgpRRDaBOvHP/f07egLc2inec2sC
|
||||
yPSCEfRMMhFcU5NoBt4Unzz2Y8pbpL1L5kbM6B4IqK/5vYcbvkmBkhL3AFT6Fg/3
|
||||
3XCdygPW9Vf1nRKr2KhT9dDvB+XO2B75JmKwMk8CgYApKzGf8kz7gZZ4WfwrccI+
|
||||
QD6K1lihyjUAQ5J15Mv/kHeeDjjUVcqAlWf0irkImpr0IJAt43COWsGjsWF6efmX
|
||||
yQCLZZuxixppFVXXsd122ivd0S28OMkiWzQEzP67+83Ujc/okcIhcNVz9lB4ExtN
|
||||
Xe0CuI5haE6RwsI4tYZ33QKBgFq6ckPRcOAZ3IlmPp9Us3/+fdKq/BSFR7/3s347
|
||||
q11FBKCkghFoBxx5lCPVntxhKIQZlHLdkHZOTvnbrkNAPNUsewPIMHcVxOLiCZ3k
|
||||
/i9OfxIEtSJR5CjjPTQuUtu5pYWKKN2uE/ytKkpmeM1rt64CGv4lAmp2gGFijMs2
|
||||
h9jrAoGBANPQO6cKqtnxvst3lnljVBoftlJgeHamUac+xeYKA5Hocv5VwLXMTzzu
|
||||
09tAhQFvFwCWWrfdgtvIM6k5Sl9F5MdiO9VNflI0IVudIcm9FKorWogH02mtwxsw
|
||||
hvk5VUk3awiZ/Nu9t38ukeqCetjQEf6yupy/14ZPLndN5naSeEwo
|
||||
-----END RSA PRIVATE KEY-----
|
||||
@@ -0,0 +1,41 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/gorilla/handlers"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
func webhook() {
|
||||
// FIXME - remove static
|
||||
port := ":8080"
|
||||
baseFilePath := "/"
|
||||
|
||||
mux := mux.NewRouter()
|
||||
mux.SkipClean(true)
|
||||
|
||||
// FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend
|
||||
mux.HandleFunc(baseFilePath, Authorization).Methods("POST")
|
||||
mux.HandleFunc("/authorize", Authorization).Methods("POST")
|
||||
|
||||
handlers.LoggingHandler(os.Stdout, mux)
|
||||
loggedRouter := handlers.LoggingHandler(os.Stdout, mux)
|
||||
|
||||
log.Printf("Starting on http://localhost%s", port)
|
||||
err := http.ListenAndServe(
|
||||
port,
|
||||
loggedRouter,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal("ListenAndServer: ", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func main() {
|
||||
webhook()
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
package main
|
||||
|
||||
// This entire script should be part of the API backend
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"golang.org/x/oauth2"
|
||||
)
|
||||
|
||||
type Subscription struct {
|
||||
ChangeType string `json:"changeType"`
|
||||
NotificationURL string `json:"notificationUrl"`
|
||||
Resource string `json:"resource"`
|
||||
ExpirationDateTime string `json:"expirationDateTime"`
|
||||
ClientState string `json:"clientState"`
|
||||
}
|
||||
|
||||
// ClientState string `json:"ClientState,omitempty"`
|
||||
// OdataType string `json:"@odata.type"`
|
||||
|
||||
//odata.type - Include "@odata.type":"#Microsoft.OutlookServices.PushSubscription". The PushSubscription entity defines NotificationURL.
|
||||
//ChangeType - Specifies the types of events to monitor for that resource. See ChangeType for the supported types.
|
||||
//ClientState - Optional property that indicates that each notification should be sent with a header by the same ClientState value. This lets the listener check the legitimacy of each notification.
|
||||
//NotificationURL - Specifies where notifications should be sent to. This URL represents a web service typically implemented by the client.
|
||||
//Resource - Specifies the resource to monitor and receive notifications on. You can use the optional query parameter $filter to refine the conditions for a notification, or use $select to include specific properties in a rich notification.
|
||||
|
||||
//https://outlook.office.com/mail.read
|
||||
|
||||
type Config struct {
|
||||
ClientID string
|
||||
ClientSecret string
|
||||
RedirectUrl string
|
||||
AuthUrl string
|
||||
TokenUrl string
|
||||
}
|
||||
|
||||
func getOfficeAppInfo() (Config, error) {
|
||||
configpath := "integrations/config.json"
|
||||
|
||||
data, err := ioutil.ReadFile(configpath)
|
||||
if err != nil {
|
||||
//log.Fatal(err)
|
||||
log.Printf("Error getting hive: %s\n", err)
|
||||
}
|
||||
|
||||
config := Config{}
|
||||
err = json.Unmarshal(data, &config)
|
||||
if err != nil {
|
||||
return Config{}, err
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// This should be a popup for the user
|
||||
func get_accesstoken() (*http.Client, OauthToken, error) {
|
||||
ctx := context.Background()
|
||||
config, err := getOfficeAppInfo()
|
||||
if err != nil {
|
||||
return nil, OauthToken{}, err
|
||||
}
|
||||
|
||||
conf := &oauth2.Config{
|
||||
ClientID: config.ClientID,
|
||||
ClientSecret: config.ClientSecret,
|
||||
Scopes: []string{
|
||||
"Mail.Read",
|
||||
"User.Read",
|
||||
"https://outlook.office.com/mail.read",
|
||||
},
|
||||
RedirectURL: "https://localhost:8000",
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: config.AuthUrl,
|
||||
TokenURL: config.TokenUrl,
|
||||
},
|
||||
}
|
||||
//"Mail.Read.Shared",
|
||||
|
||||
//url := conf.AuthCodeURL("state", oauth2.SetAuthURLParam("resource", "https://outlook.office.com"))
|
||||
// ADD DATA TO STATE HERE :O
|
||||
url := conf.AuthCodeURL("workflow_id%3Dc2e0b50a-2957-427e-a97b-b989dc5a5408%26trigger_id%3D9e845679-5843-4959-a76c-a6d664e9df35%26username%3Drheyix.yt@gmail.com", oauth2.SetAuthURLParam("resource", "https://graph.microsoft.com"))
|
||||
|
||||
fmt.Printf("Visit the URL for the auth dialog: \n%v\n\n", url)
|
||||
codechannel := make(chan string)
|
||||
|
||||
// Handles the server callback, listening on port 8000
|
||||
go func() {
|
||||
port := ":8000"
|
||||
|
||||
http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) {
|
||||
tmpcode := request.URL.Query().Get("code")
|
||||
if len(tmpcode) < 100 {
|
||||
return
|
||||
} else {
|
||||
codechannel <- tmpcode
|
||||
}
|
||||
})
|
||||
|
||||
// FIX - might cause errors not being printed
|
||||
err := http.ListenAndServeTLS(port, "integrations/server.crt", "integrations/server.key", nil)
|
||||
if err != nil {
|
||||
log.Printf("%s\n", err)
|
||||
}
|
||||
}()
|
||||
|
||||
code := <-codechannel
|
||||
close(codechannel)
|
||||
|
||||
// https://stackoverflow.com/questions/52787420/multiple-resources-in-a-single-authorization-request
|
||||
// Multi resource ^
|
||||
access_token, err := conf.Exchange(ctx, code)
|
||||
//log.Printf("%#v", access_token)
|
||||
if err != nil {
|
||||
return nil, OauthToken{}, err
|
||||
}
|
||||
|
||||
//log.Printf("%#v", access_token)
|
||||
outlookClient := conf.Client(ctx, access_token)
|
||||
|
||||
oauthToken := OauthToken{
|
||||
AccessToken: access_token.AccessToken,
|
||||
TokenType: access_token.TokenType,
|
||||
RefreshToken: access_token.RefreshToken,
|
||||
Expiry: access_token.Expiry,
|
||||
}
|
||||
|
||||
return outlookClient, oauthToken, nil
|
||||
}
|
||||
|
||||
type OauthToken struct {
|
||||
AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"`
|
||||
TokenType string `json:"TokenType" datastore:"TokenType,noindex"`
|
||||
RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"`
|
||||
Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"`
|
||||
}
|
||||
|
||||
type Mailfolders struct {
|
||||
OdataContext string `json:"@odata.context"`
|
||||
OdataNextLink string `json:"@odata.nextLink"`
|
||||
Value []struct {
|
||||
ID string `json:"id"`
|
||||
DisplayName string `json:"displayName"`
|
||||
ParentFolderID string `json:"parentFolderId"`
|
||||
ChildFolderCount int `json:"childFolderCount"`
|
||||
UnreadItemCount int `json:"unreadItemCount"`
|
||||
TotalItemCount int `json:"totalItemCount"`
|
||||
} `json:"value"`
|
||||
}
|
||||
|
||||
func getFolders(client *http.Client) (Mailfolders, error) {
|
||||
requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/frikky@shuffletest.onmicrosoft.com/mailfolders")
|
||||
|
||||
ret, err := client.Get(requestUrl)
|
||||
if err != nil {
|
||||
log.Printf("FolderErr: %s", err)
|
||||
return Mailfolders{}, err
|
||||
}
|
||||
|
||||
log.Printf("Status folders: %d", ret.StatusCode)
|
||||
body, err := ioutil.ReadAll(ret.Body)
|
||||
if err != nil {
|
||||
log.Printf("Body: %s", err)
|
||||
return Mailfolders{}, err
|
||||
}
|
||||
|
||||
//log.Printf("Body: %s", string(body))
|
||||
|
||||
mailfolders := Mailfolders{}
|
||||
err = json.Unmarshal(body, &mailfolders)
|
||||
if err != nil {
|
||||
log.Printf("Unmarshal: %s", err)
|
||||
return Mailfolders{}, err
|
||||
}
|
||||
|
||||
//fmt.Printf("%#v", mailfolders)
|
||||
// FIXME - recursion for subfolders
|
||||
// Recursive struct
|
||||
// folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId)
|
||||
for _, folder := range mailfolders.Value {
|
||||
log.Println(folder.DisplayName)
|
||||
}
|
||||
|
||||
return mailfolders, nil
|
||||
}
|
||||
|
||||
// Subscribes to a mailbox based on some thingies
|
||||
func makeSubscription(client *http.Client, folderIds []string) {
|
||||
// FIXME - show the users folders from oauth and let them choose
|
||||
|
||||
fullUrl := "https://graph.microsoft.com/v1.0/subscriptions"
|
||||
//resource := fmt.Sprintf("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages")
|
||||
resource := fmt.Sprintf("me/mailfolders('inbox')/messages")
|
||||
sub := Subscription{
|
||||
ChangeType: "created",
|
||||
NotificationURL: "https://de4fc12b.ngrok.io",
|
||||
ExpirationDateTime: "2019-09-22T18:23:45.9356913Z",
|
||||
ClientState: "This is a test",
|
||||
Resource: resource,
|
||||
}
|
||||
|
||||
data, err := json.Marshal(sub)
|
||||
if err != nil {
|
||||
log.Printf("Marshal: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf(string(data))
|
||||
|
||||
req, err := http.NewRequest(
|
||||
"POST",
|
||||
fullUrl,
|
||||
bytes.NewBuffer(data),
|
||||
)
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
|
||||
res, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Client: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
log.Printf("Status: %d", res.StatusCode)
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
log.Printf("Body: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println(string(body))
|
||||
log.Println("Shoooould be set up :)")
|
||||
|
||||
}
|
||||
|
||||
func getOutlookClient(code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) {
|
||||
ctx := context.Background()
|
||||
|
||||
conf := &oauth2.Config{
|
||||
ClientID: "70e37005-c954-4290-b573-d4b94e484336",
|
||||
ClientSecret: ".eNw/A[kQFB5zL.agvRputdEJENeJ392",
|
||||
Scopes: []string{
|
||||
"Mail.Read",
|
||||
"User.Read",
|
||||
"https://outlook.office.com/mail.read",
|
||||
},
|
||||
RedirectURL: redirectUri,
|
||||
Endpoint: oauth2.Endpoint{
|
||||
AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize",
|
||||
TokenURL: "https://login.microsoftonline.com/common/oauth2/token",
|
||||
},
|
||||
}
|
||||
|
||||
if len(code) > 0 {
|
||||
access_token, err := conf.Exchange(ctx, code)
|
||||
if err != nil {
|
||||
log.Printf("Access_token issue: %s", err)
|
||||
return &http.Client{}, access_token, err
|
||||
}
|
||||
|
||||
client := conf.Client(ctx, access_token)
|
||||
return client, access_token, nil
|
||||
} else {
|
||||
// Manually recreate the oauthtoken
|
||||
access_token := &oauth2.Token{
|
||||
AccessToken: accessToken.AccessToken,
|
||||
TokenType: accessToken.TokenType,
|
||||
RefreshToken: accessToken.RefreshToken,
|
||||
Expiry: accessToken.Expiry,
|
||||
}
|
||||
|
||||
client := conf.Client(ctx, access_token)
|
||||
return client, access_token, nil
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
graphclient, oauthToken, err := get_accesstoken()
|
||||
if err != nil {
|
||||
log.Printf("Oauth setup: %s", err)
|
||||
return
|
||||
}
|
||||
|
||||
// FIXME - make this possible for alternative users (shared)
|
||||
folders, err := getFolders(graphclient)
|
||||
if err != nil {
|
||||
log.Printf("Folder get error: %s", err)
|
||||
return
|
||||
}
|
||||
_ = folders
|
||||
|
||||
folderIds := []string{"inbox"}
|
||||
//log.Println(folders)
|
||||
//log.Printf("%#v", oauthToken)
|
||||
// Use oauthToken to generate data for outlook
|
||||
outlookclient, _, err := getOutlookClient("", oauthToken, "https://localhost:8000")
|
||||
makeSubscription(outlookclient, folderIds)
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
main.go
|
||||
*.swo
|
||||
*.swp
|
||||
@@ -0,0 +1,17 @@
|
||||
# Local testing
|
||||
1. Change hook.go package to main
|
||||
```bash
|
||||
mv ../main.go .
|
||||
go run main.go hook.go
|
||||
```
|
||||
|
||||
# Deploy local
|
||||
```bash
|
||||
gcloud functions deploy webhook --runtime go111 --entry-point Authorization --trigger-http --project shuffler --memory=128 --set-env-vars=FUNCTION_APIKEY=asdasd,CALLBACKURL=shuffler.io,HOOKID=test123
|
||||
```
|
||||
|
||||
# Build and deploy from gui
|
||||
1. rm webhook.zip
|
||||
2. zip webhook.zip hook.go
|
||||
3. Upload to bucket https://console.cloud.google.com/storage/browser/shuffler.appspot.com?project=shuffler
|
||||
4. Restart hook(s) (https://shuffler.io/webhooks)
|
||||
@@ -0,0 +1,249 @@
|
||||
package function
|
||||
|
||||
// BOTS
|
||||
// https://dev.botframework.com/bots/channels?id=Shuffle
|
||||
|
||||
// APPS:
|
||||
// apps.dev.microsoft.com
|
||||
|
||||
// REMOVE ACCESS:
|
||||
// https://portal.office.com/account/#
|
||||
|
||||
// Developer:
|
||||
// https://developer.microsoft.com/en-us/graph/docs/concepts/permissions_reference
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Info struct {
|
||||
Url string `json:"url" datastore:"url"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Description string `json:"description" datastore:"description"`
|
||||
}
|
||||
|
||||
// Actions to be done by webhooks etc
|
||||
// Field is the actual field to use from json
|
||||
type HookAction struct {
|
||||
Type string `json:"type" datastore:"type"`
|
||||
Name string `json:"name" datastore:"name"`
|
||||
Id string `json:"id" datastore:"id"`
|
||||
Field string `json:"field" datastore:"field"`
|
||||
}
|
||||
|
||||
type Hook struct {
|
||||
Id string `json:"id" datastore:"id"`
|
||||
Info Info `json:"info" datastore:"info"`
|
||||
Actions []HookAction `json:"actions" datastore:"actions"`
|
||||
Type string `json:"type" datastore:"type"`
|
||||
Status string `json:"status" datastore:"status"`
|
||||
Running bool `json:"running" datastore:"running"`
|
||||
}
|
||||
|
||||
var hook Hook
|
||||
|
||||
func Authorization(resp http.ResponseWriter, request *http.Request) {
|
||||
apikey := os.Getenv("FUNCTION_APIKEY")
|
||||
callbackUrl := os.Getenv("CALLBACKURL")
|
||||
hookId := os.Getenv("HOOKID")
|
||||
if len(apikey) == 0 {
|
||||
log.Println("Env FUNCTION_APIKEY not set")
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Internal error"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
if len(callbackUrl) == 0 {
|
||||
log.Println("Env CALLBACKURL not set")
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Internal error"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
if len(hookId) == 0 {
|
||||
log.Println("Env HOOKID not set")
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Internal error"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
authorization := request.Header.Get("Authorization")
|
||||
if len(authorization) == 0 {
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Authorization header required"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(authorization, "Bearer") {
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Authorization header must start with Bearer"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
apikeyCheck := strings.Split(authorization, " ")
|
||||
if len(apikeyCheck) != 2 {
|
||||
log.Println("Length is not 2 for apikey: %s vs %s", apikeyCheck[1], apikey)
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Invalid Apikey"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
if apikeyCheck[1] != apikey {
|
||||
log.Printf("Apikeys are not equal. Failed authentication.")
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Invalid Apikey"}`)))
|
||||
return
|
||||
}
|
||||
|
||||
err := ForwardRequest(resp, request)
|
||||
if err != nil {
|
||||
log.Printf("Error: %s", err)
|
||||
resp.WriteHeader(401)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
|
||||
return
|
||||
}
|
||||
|
||||
log.Println("Success?")
|
||||
resp.WriteHeader(200)
|
||||
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
|
||||
}
|
||||
|
||||
func loadConfiguration(fullUrl string, apikey string) (Hook, error) {
|
||||
client := &http.Client{}
|
||||
|
||||
req, err := http.NewRequest(
|
||||
"GET",
|
||||
fullUrl,
|
||||
nil,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Printf("Error making http request: %s", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Error in http request: %s", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Error reading response: %s", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
err = json.Unmarshal(body, &hook)
|
||||
if err != nil {
|
||||
log.Printf("Failed unmarshaling hook API", req)
|
||||
return Hook{}, err
|
||||
}
|
||||
|
||||
return hook, nil
|
||||
}
|
||||
|
||||
// GetUserDetails - Get one user's details from randomuser.me API
|
||||
func ForwardRequest(resp http.ResponseWriter, request *http.Request) error {
|
||||
callbackUrl := os.Getenv("CALLBACKURL")
|
||||
hookId := os.Getenv("HOOKID")
|
||||
apikey := os.Getenv("FUNCTION_APIKEY")
|
||||
|
||||
hook, err := loadConfiguration(
|
||||
fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId),
|
||||
apikey,
|
||||
)
|
||||
|
||||
log.Println("Done loading!")
|
||||
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("%#v", hook)
|
||||
|
||||
// Find all things to execute
|
||||
workflowUrls := []string{}
|
||||
for _, item := range hook.Actions {
|
||||
if item.Type == "" {
|
||||
log.Printf("CONTINUE AAS EMPTY ITEM: %#v", item)
|
||||
continue
|
||||
}
|
||||
|
||||
if item.Type == "workflow" {
|
||||
workflowUrls = append(workflowUrls, item.Id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(workflowUrls) == 0 {
|
||||
return errors.New("No actions to do yet")
|
||||
}
|
||||
|
||||
log.Printf("Should send data to the following: %s", strings.Join(workflowUrls, ", "))
|
||||
|
||||
randomUserClient := http.Client{
|
||||
Timeout: time.Second * 3,
|
||||
}
|
||||
|
||||
body, err := ioutil.ReadAll(request.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Prepare data
|
||||
type arg struct {
|
||||
ExecutionArgument string `json:"execution_argument"`
|
||||
}
|
||||
data := arg{
|
||||
ExecutionArgument: string(body),
|
||||
}
|
||||
|
||||
newjson, err := json.Marshal(data)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Loop all executions to run
|
||||
for _, item := range workflowUrls {
|
||||
fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, item)
|
||||
log.Printf("Sending data to %s", fullUrl)
|
||||
req, err := http.NewRequest(
|
||||
http.MethodPost,
|
||||
fullUrl,
|
||||
bytes.NewBuffer(newjson),
|
||||
)
|
||||
|
||||
req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey))
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
res, err := randomUserClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf("Status: %d", res.StatusCode)
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Printf(string(body))
|
||||
}
|
||||
|
||||
//log.Println(string(newbody))
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/gorilla/handlers"
|
||||
"github.com/gorilla/mux"
|
||||
)
|
||||
|
||||
func webhook() {
|
||||
// FIXME - remove static
|
||||
port := ":8080"
|
||||
baseFilePath := "/"
|
||||
|
||||
mux := mux.NewRouter()
|
||||
mux.SkipClean(true)
|
||||
|
||||
// FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend
|
||||
mux.HandleFunc(baseFilePath, Authorization).Methods("POST")
|
||||
|
||||
handlers.LoggingHandler(os.Stdout, mux)
|
||||
loggedRouter := handlers.LoggingHandler(os.Stdout, mux)
|
||||
|
||||
err := http.ListenAndServe(
|
||||
port,
|
||||
loggedRouter,
|
||||
)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal("ListenAndServer: ", err)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func main() {
|
||||
webhook()
|
||||
}
|
||||
Reference in New Issue
Block a user