From d1e4f409a67129e73d93f953c63ec809465cb56d Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 29 Sep 2021 03:30:13 +0200 Subject: [PATCH] Loads of updates to extensions and their documentation --- backend/app_sdk/Dockerfile | 13 +- backend/app_sdk/app_base.py | 167 +++++++++++++----- backend/go-app/go.mod | 4 +- docker-compose.yml | 2 +- frontend/src/views/AppCreator.jsx | 8 + functions/extensions/aws-lambda/README.md | 2 - functions/extensions/aws-lambda/s3_deploy.sh | 16 -- .../extensions/aws-lambda/s3_function.py | 26 --- functions/extensions/kafka/README.md | 4 - functions/extensions/kafka/docker-compose.yml | 16 -- functions/extensions/kafka/kafka_local.py | 87 --------- functions/extensions/kafka/requirements.txt | 2 - functions/extensions/misp/Dockerfile | 23 +++ functions/extensions/misp/docker-compose.yml | 9 + functions/extensions/misp/requirements.txt | 2 + functions/extensions/misp/sub.py | 57 ++++++ 16 files changed, 232 insertions(+), 206 deletions(-) delete mode 100644 functions/extensions/aws-lambda/README.md delete mode 100644 functions/extensions/aws-lambda/s3_deploy.sh delete mode 100644 functions/extensions/aws-lambda/s3_function.py delete mode 100644 functions/extensions/kafka/README.md delete mode 100644 functions/extensions/kafka/docker-compose.yml delete mode 100644 functions/extensions/kafka/kafka_local.py delete mode 100644 functions/extensions/kafka/requirements.txt create mode 100644 functions/extensions/misp/Dockerfile create mode 100644 functions/extensions/misp/docker-compose.yml create mode 100644 functions/extensions/misp/requirements.txt create mode 100644 functions/extensions/misp/sub.py diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile index b2febec6..e50fd25a 100644 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -5,15 +5,12 @@ RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-de RUN mkdir /install WORKDIR /install +COPY requirements.txt /requirements.txt +RUN pip3 install --prefix="/install" -r /requirements.txt FROM base - -#--no-cache -RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev - COPY --from=builder /install /usr/local -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt -COPY __init__.py /app/walkoff_app_sdk/__init__.py -COPY app_base.py /app/walkoff_app_sdk/app_base.py +WORKDIR /app +COPY sub.py /app/sub.py +CMD python sub.py diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index f2a7c3f1..e9622eca 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1644,6 +1644,86 @@ class AppBase: return template + # Suboptimal cleanup script for BOdy parsing of OpenAPI + # Should have a regex which looks for the value, then goes out and cleans up the key + def recurse_cleanup_script(data): + try: + if not isinstance(data, dict): + newvalue = json.loads(data) + else: + newvalue = data + + deletekeys = [] + for key, value in newvalue.items(): + #print("%s: %s" % (key, value)) + if isinstance(value, str) and len(value) == 0: + deletekeys.append(key) + continue + + if isinstance(value, list): + try: + value = json.dumps(value) + print(value) + except: + print("Json parsing issue in recursed value") + pass + + if value == "${%s}" % key: + print("Deleting %s because key = value" % key) + deletekeys.append(key) + continue + elif "${" in value and "}" in value: + print("Deleting %s because it contains ${ and }" % key) + deletekeys.append(key) + continue + + if isinstance(value, dict): + newvalue[key] = recurse_cleanup_script(value) + + except json.decoder.JSONDecodeError as e: + print("Failed JSON replacement for OpenAPI keys (3) {e}") + + for deletekey in deletekeys: + try: + del newvalue[deletekey] + except: + pass + + try: + #print("Post delete: %s" % newvalue) + for key, value in newvalue.items(): + if isinstance(value, bool): + continue + elif isinstance(value, dict) and not bool(value): + continue + + try: + value = json.loads(value) + newvalue[key] = value + except json.decoder.JSONDecodeError as e: + #print("Inner overwrite issue for \"%s\": %s" % (key, e)) + continue + except Exception as e: + #print("General error in newvalue items loop: %s" % e) + continue + + try: + data = json.dumps(newvalue) + except json.decoder.JSONDecodeError as e: + print("[WARNING] JsonDecodeError: %s" % e) + data = newvalue + + except json.decoder.JSONDecodeError as e: + print("Failed JSON replacement for OpenAPI keys (2) {e}") + + #if isinstance(data, str): + # tmpdata = json.dumps(data) + # print(tmpdata) + # foundvalue = re.findall(".*?(${\w+})", tmpdata, re.MULTILINE) + # print("FOUND: %s", foundvalue) + + return data + # Parses parameters sent to it and returns whether it did it successfully with the values found def parse_params(action, fullexecution, parameter, self): # Skip if it starts with $? @@ -2112,60 +2192,63 @@ class AppBase: print("KeyError body OpenAPI: %s" % e) pass - + print(f"""HANDLING {action["parameters"][counter]["value"]}""") - try: - newvalue = json.loads(action["parameters"][counter]["value"]) - deletekeys = [] - for key, value in newvalue.items(): - print("%s: %s" % (key, value)) - if isinstance(value, str) and len(value) == 0: - deletekeys.append(key) - continue + action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"]) + #try: + # newvalue = json.loads(action["parameters"][counter]["value"]) + # deletekeys = [] + # for key, value in newvalue.items(): + # print("%s: %s" % (key, value)) + # if isinstance(value, str) and len(value) == 0: + # deletekeys.append(key) + # continue - if value == "${%s}" % key: - print("Deleting %s because key = value" % key) - deletekeys.append(key) - continue - - for deletekey in deletekeys: - try: - del newvalue[deletekey] - except: - pass + # if value == "${%s}" % key: + # print("Deleting %s because key = value" % key) + # deletekeys.append(key) + # continue + # + # for deletekey in deletekeys: + # try: + # del newvalue[deletekey] + # except: + # pass - #print("Post delete: %s" % newvalue) - for key, value in newvalue.items(): - if isinstance(value, bool): - continue + # #print("Post delete: %s" % newvalue) + # for key, value in newvalue.items(): + # if isinstance(value, bool): + # continue - try: - value = json.loads(value) - newvalue[key] = value - except json.decoder.JSONDecodeError as e: - print("Inner overwrite issue: %s" % e) - continue - except: - print("General error in newvalue items loop") - continue + # try: + # value = json.loads(value) + # newvalue[key] = value + # except json.decoder.JSONDecodeError as e: + # print("Inner overwrite issue: %s" % e) + # continue + # except Exception as e: + # print("General error in newvalue items loop: %s" % e) + # continue - try: - action["parameters"][counter]["value"] = json.dumps(newvalue) - except json.decoder.JSONDecodeError as e: - print("[WARNING] JsonDecodeError: %s" % e) - action["parameters"][counter]["value"] = newvalue - + # try: + # action["parameters"][counter]["value"] = json.dumps(newvalue) + # except json.decoder.JSONDecodeError as e: + # print("[WARNING] JsonDecodeError: %s" % e) + # action["parameters"][counter]["value"] = newvalue + # - except json.decoder.JSONDecodeError as e: - print("Failed JSON replacement for OpenAPI keys (2) {e}") - - break + #except json.decoder.JSONDecodeError as e: + # print("Failed JSON replacement for OpenAPI keys (2) {e}") #print(action["parameters"]) + + # This seems redundant now print("Pre parameters") for parameter in newparams: action["parameters"].append(parameter) + self.action = action + # calltimes is used to handle forloops in the app itself. # 2 kinds of loop - one in gui with one app each, and one like this, # which is super fast, but has a bad overview (potentially good tho) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 431adaf0..13cc619d 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module shuffle go 1.13 -replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared +//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi @@ -23,7 +23,7 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect github.com/frikky/kin-openapi v0.39.0 - github.com/frikky/shuffle-shared v0.0.99 + github.com/frikky/shuffle-shared v0.1.1 github.com/fsouza/go-dockerclient v1.7.2 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.0.0 diff --git a/docker-compose.yml b/docker-compose.yml index edb20333..c033b414 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index f44ce565..5b3f0be2 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1331,6 +1331,10 @@ const AppCreator = (props) => { const headersplit = header.split("= ") key = headersplit[0] value = headersplit[1] + } else if (header.length > 0 && header.includes(" =")) { + const headersplit = header.split(" =") + key = headersplit[0] + value = headersplit[1] } else if (header.length > 0 && header.includes("=")) { const headersplit = header.split("=") key = headersplit[0] @@ -1339,6 +1343,10 @@ const AppCreator = (props) => { const headersplit = header.split(": ") key = headersplit[0] value = headersplit[1] + } else if (header.length > 0 && header.includes(" :")) { + const headersplit = header.split(" :") + key = headersplit[0] + value = headersplit[1] } else if (header.length > 0 && header.includes(":")) { const headersplit = header.split(":") key = headersplit[0] diff --git a/functions/extensions/aws-lambda/README.md b/functions/extensions/aws-lambda/README.md deleted file mode 100644 index f665851a..00000000 --- a/functions/extensions/aws-lambda/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# AWS Lambda forwarder to Shuffle -This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated. diff --git a/functions/extensions/aws-lambda/s3_deploy.sh b/functions/extensions/aws-lambda/s3_deploy.sh deleted file mode 100644 index cedda90f..00000000 --- a/functions/extensions/aws-lambda/s3_deploy.sh +++ /dev/null @@ -1,16 +0,0 @@ -#GOOS=linux go build main.go -zip s3_function.zip s3_function.py - -WEBHOOK=https://shuffler.io/api/v1/hooks/webhook_eccf47b1-8f6a-49fc-b2b8-383365a22353 -ROLE=arn:aws:iam::202262580068:role/service-role/shuffle-forwarder -REGION=us-east-1 -BUCKETNAME=helo - -aws lambda create-function \ - --role $ROLE \ - --region $REGION \ - --function-name shuffler-webhook-forwarder-3 \ - --zip-file fileb://s3_function.zip \ - --runtime python3.9 \ - --environment Variables={SHUFFLE_WEBHOOK=$WEBHOOK} \ - --handler lambda_handler diff --git a/functions/extensions/aws-lambda/s3_function.py b/functions/extensions/aws-lambda/s3_function.py deleted file mode 100644 index c5e11cde..00000000 --- a/functions/extensions/aws-lambda/s3_function.py +++ /dev/null @@ -1,26 +0,0 @@ -import json -import urllib.parse -import requests -import os - -print('Loading function') - -def lambda_handler(event, context): - #print("Received event: " + json.dumps(event, indent=2)) - - # Get the object from the event and show its content type - bucket = event['Records'][0]['s3']['bucket']['name'] - - print(type(event)) - print("Getting bucket: %s" % bucket) - webhook = os.environ.get("SHUFFLE_WEBHOOK") - if not webhook: - return "No webhook environment defined: SHUFFLE_WEBHOOK" - - ret = requests.post(webhook, json=event["Records"][0]) - if ret.status_code != 200: - return "Bad status code for webhook: %d" % ret.status_code - - print("Status code: %d\nData: %s" % (ret.status_code, ret.text)) - - # response = s3.get_object(Bucket=bucket, Key=key) diff --git a/functions/extensions/kafka/README.md b/functions/extensions/kafka/README.md deleted file mode 100644 index dceb22a3..00000000 --- a/functions/extensions/kafka/README.md +++ /dev/null @@ -1,4 +0,0 @@ -``` -curl -sSL https://raw.githubusercontent.com/bitnami/bitnami-docker-kafka/master/docker-compose.yml > docker-compose.yml -docker-compose up -d -``` diff --git a/functions/extensions/kafka/docker-compose.yml b/functions/extensions/kafka/docker-compose.yml deleted file mode 100644 index 02e96798..00000000 --- a/functions/extensions/kafka/docker-compose.yml +++ /dev/null @@ -1,16 +0,0 @@ -version: '2' -services: - zookeeper: - image: wurstmeister/zookeeper - ports: - - "2181:2181" - kafka: - build: . - ports: - - "9092" - environment: - DOCKER_API_VERSION: 1.40 - KAFKA_ADVERTISED_HOST_NAME: 192.168.193.140 - KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 - volumes: - - /var/run/docker.sock:/var/run/docker.sock diff --git a/functions/extensions/kafka/kafka_local.py b/functions/extensions/kafka/kafka_local.py deleted file mode 100644 index 8ca8a6a5..00000000 --- a/functions/extensions/kafka/kafka_local.py +++ /dev/null @@ -1,87 +0,0 @@ -import json -import os -import requests -from time import sleep -from kafka import KafkaProducer, KafkaConsumer -import kafka - -shuffle_url = os.getenv("SHUFFLE_URL") -shuffle_apikey = os.getenv("SHUFFLE_APIKEY") -shuffle_workflow = os.getenv("SHUFFLE_WORKFLOW") - -headers = {"Authorization": "Bearer %s" % shuffle_apikey} -topic = "workflow_%s" % shuffle_workflow -group = "testing" -server = "localhost:9092" -def produce(): - try: - #group_id=group, - #value_serializer=lambda x: json.dumps(x).encode('utf-8') - producer = KafkaProducer( - bootstrap_servers=[server], - ) - except kafka.errors.NoBrokersAvailable as e: - print(f"Error with producer: {e}") - sleep(5) - produce() - return - except ValueError as e: - print(f"ValuError with producer: {e}") - sleep(5) - produce() - return - - - for i in range(15): - producer.send(topic, json.dumps({"some": i, "data": "luuuuul"})) - - producer.flush() - producer.close() - exit() - - #print(f"Adding data {data}") - - #try: - # ret = producer.send(topic, value=data) - # print(ret.get()) - # producer.flush() - - # #future = producer.send('foobar', b'another_message') - # #result = future.get(timeout=60) - # #print(result) - - #except kafka.errors.KafkaTimeoutError as e: - # print("Kafka error: %s" % e) - # continue - -def consume(): - print("Starting consumer") - #group_id=group, - #auto_offset_reset="earliest", - #enable_auto_commit=True, - consumer = KafkaConsumer( - topic, - bootstrap_servers=[server], - auto_offset_reset='earliest', - max_poll_records=2, - ) - #value_deserializer=lambda x: json.loads(x.decode('utf-8')) - - consumer.poll() - #consumer.seek_to_beginning() - - print(f"Getting data from topic {topic}") - for message in consumer: - #message = message.value - #message = message - print("MSG: ", message) - ret = requests.post("%s/api/v1/%s/execute" % shuffle_url, headers=headers, data=message) - print(ret.status_code) - print(ret.text) - -if __name__ == "__main__": - if topic == "workflow_None": - topic = "testing" - print("Starting producer on %s for topic %s" % (server, topic)) - produce() - #consume() diff --git a/functions/extensions/kafka/requirements.txt b/functions/extensions/kafka/requirements.txt deleted file mode 100644 index 652ae858..00000000 --- a/functions/extensions/kafka/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -kafka -kafka-python diff --git a/functions/extensions/misp/Dockerfile b/functions/extensions/misp/Dockerfile new file mode 100644 index 00000000..f2b8ae3f --- /dev/null +++ b/functions/extensions/misp/Dockerfile @@ -0,0 +1,23 @@ +FROM python:3.9.4-alpine as base + +FROM base as builder + +RUN mkdir /install +WORKDIR /install + +FROM base +RUN apk add g++ + +COPY --from=builder /install /usr/local +COPY requirements.txt /requirements.txt +RUN pip3 install -r /requirements.txt + + +RUN mkdir /app +WORKDIR /app +COPY requirements.txt /app/requirements.txt +RUN python3 -m pip install -r /app/requirements.txt + +COPY sub.py /app/sub.py + +CMD ["python3", "sub.py"] diff --git a/functions/extensions/misp/docker-compose.yml b/functions/extensions/misp/docker-compose.yml new file mode 100644 index 00000000..5b0130c0 --- /dev/null +++ b/functions/extensions/misp/docker-compose.yml @@ -0,0 +1,9 @@ +version: '3' +services: + zmq: + image: ghcr.io/frikky/shuffle-zmq:latest + environment: + - ZMQ_HOSTNAME=localhost + - ZMQ_PORT=50000 + - ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e + restart: unless-stopped diff --git a/functions/extensions/misp/requirements.txt b/functions/extensions/misp/requirements.txt new file mode 100644 index 00000000..a1cdf7dd --- /dev/null +++ b/functions/extensions/misp/requirements.txt @@ -0,0 +1,2 @@ +pyzmq +requests diff --git a/functions/extensions/misp/sub.py b/functions/extensions/misp/sub.py new file mode 100644 index 00000000..feb78c08 --- /dev/null +++ b/functions/extensions/misp/sub.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- + +print("Running imports") +import sys +import zmq +import json +import time +import pprint +import os +import sys +import requests + +forward_url = os.getenv("ZMQ_FORWARD_URL", "") +print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url) +def handle_hook(data): + ret = requests.post(forward_url, json=data) + print(ret.text) + print(ret.status_code) + +def main(): + host = os.getenv("ZMQ_HOST", "localhost") + port = os.getenv("ZMQ_PORT", "50000") + + if len(forward_url) == 0: + print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder") + exit(0) + + print("Starting connection setup to %s:%s" % (host, port)) + context = zmq.Context() + socket = context.socket(zmq.SUB) + socket.connect ("tcp://%s:%s" % (host, port)) + socket.setsockopt(zmq.SUBSCRIBE, b'') + + poller = zmq.Poller() + poller.register(socket, zmq.POLLIN) + + print("Starting zmq check for %s:%s" % (host, port)) + while True: + socks = dict(poller.poll(timeout=None)) + if socket in socks and socks[socket] == zmq.POLLIN: + message = socket.recv() + #print(message) + topic, s, m = message.decode('utf-8').partition(" ") + + d = json.loads(m) + try: + # print test if you want status (heartbeat) + test = d["status"] + except KeyError: + handle_hook(d) + + time.sleep(1) + +if __name__ == "__main__": + print("In init ") + main()