marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Loads of updates to extensions and their documentation

Browse Source
This commit is contained in:
frikky
2021-09-29 03:30:13 +02:00
parent 9d1b5a41e9
commit d1e4f409a6
16 changed files with 232 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/app_sdk/Dockerfile
+5 -8
View File
@@ -5,15 +5,12 @@ RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-de
RUN mkdir /install
WORKDIR /install
COPY requirements.txt /requirements.txt
RUN pip3 install --prefix="/install" -r /requirements.txt
FROM base
#--no-cache
RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev
COPY --from=builder /install /usr/local
COPY requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
COPY __init__.py /app/walkoff_app_sdk/__init__.py
COPY app_base.py /app/walkoff_app_sdk/app_base.py
WORKDIR /app
COPY sub.py /app/sub.py
CMD python sub.py
backend/app_sdk/app_base.py
+125 -42
View File
@@ -1644,6 +1644,86 @@ class AppBase:
return template
# Suboptimal cleanup script for BOdy parsing of OpenAPI
# Should have a regex which looks for the value, then goes out and cleans up the key
def recurse_cleanup_script(data):
try:
if not isinstance(data, dict):
newvalue = json.loads(data)
else:
newvalue = data
deletekeys = []
for key, value in newvalue.items():
#print("%s: %s" % (key, value))
if isinstance(value, str) and len(value) == 0:
deletekeys.append(key)
continue
if isinstance(value, list):
try:
value = json.dumps(value)
print(value)
except:
print("Json parsing issue in recursed value")
pass
if value == "${%s}" % key:
print("Deleting %s because key = value" % key)
deletekeys.append(key)
continue
elif "${" in value and "}" in value:
print("Deleting %s because it contains ${ and }" % key)
deletekeys.append(key)
continue
if isinstance(value, dict):
newvalue[key] = recurse_cleanup_script(value)
except json.decoder.JSONDecodeError as e:
print("Failed JSON replacement for OpenAPI keys (3) {e}")
for deletekey in deletekeys:
try:
del newvalue[deletekey]
except:
pass
try:
#print("Post delete: %s" % newvalue)
for key, value in newvalue.items():
if isinstance(value, bool):
continue
elif isinstance(value, dict) and not bool(value):
continue
try:
value = json.loads(value)
newvalue[key] = value
except json.decoder.JSONDecodeError as e:
#print("Inner overwrite issue for \"%s\": %s" % (key, e))
continue
except Exception as e:
#print("General error in newvalue items loop: %s" % e)
continue
try:
data = json.dumps(newvalue)
except json.decoder.JSONDecodeError as e:
print("[WARNING] JsonDecodeError: %s" % e)
data = newvalue
except json.decoder.JSONDecodeError as e:
print("Failed JSON replacement for OpenAPI keys (2) {e}")
#if isinstance(data, str):
# tmpdata = json.dumps(data)
# print(tmpdata)
# foundvalue = re.findall(".*?(${\w+})", tmpdata, re.MULTILINE)
# print("FOUND: %s", foundvalue)
return data
# Parses parameters sent to it and returns whether it did it successfully with the values found
def parse_params(action, fullexecution, parameter, self):
# Skip if it starts with $?
@@ -2112,60 +2192,63 @@ class AppBase:
print("KeyError body OpenAPI: %s" % e)
pass
print(f"""HANDLING {action["parameters"][counter]["value"]}""")
try:
newvalue = json.loads(action["parameters"][counter]["value"])
deletekeys = []
for key, value in newvalue.items():
print("%s: %s" % (key, value))
if isinstance(value, str) and len(value) == 0:
deletekeys.append(key)
continue
action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"])
#try:
# newvalue = json.loads(action["parameters"][counter]["value"])
# deletekeys = []
# for key, value in newvalue.items():
# print("%s: %s" % (key, value))
# if isinstance(value, str) and len(value) == 0:
# deletekeys.append(key)
# continue
if value == "${%s}" % key:
print("Deleting %s because key = value" % key)
deletekeys.append(key)
continue
for deletekey in deletekeys:
try:
del newvalue[deletekey]
except:
pass
# if value == "${%s}" % key:
# print("Deleting %s because key = value" % key)
# deletekeys.append(key)
# continue
#
# for deletekey in deletekeys:
# try:
# del newvalue[deletekey]
# except:
# pass
#print("Post delete: %s" % newvalue)
for key, value in newvalue.items():
if isinstance(value, bool):
continue
# #print("Post delete: %s" % newvalue)
# for key, value in newvalue.items():
# if isinstance(value, bool):
# continue
try:
value = json.loads(value)
newvalue[key] = value
except json.decoder.JSONDecodeError as e:
print("Inner overwrite issue: %s" % e)
continue
except:
print("General error in newvalue items loop")
continue
# try:
# value = json.loads(value)
# newvalue[key] = value
# except json.decoder.JSONDecodeError as e:
# print("Inner overwrite issue: %s" % e)
# continue
# except Exception as e:
# print("General error in newvalue items loop: %s" % e)
# continue
try:
action["parameters"][counter]["value"] = json.dumps(newvalue)
except json.decoder.JSONDecodeError as e:
print("[WARNING] JsonDecodeError: %s" % e)
action["parameters"][counter]["value"] = newvalue
# try:
# action["parameters"][counter]["value"] = json.dumps(newvalue)
# except json.decoder.JSONDecodeError as e:
# print("[WARNING] JsonDecodeError: %s" % e)
# action["parameters"][counter]["value"] = newvalue
#
except json.decoder.JSONDecodeError as e:
print("Failed JSON replacement for OpenAPI keys (2) {e}")
break
#except json.decoder.JSONDecodeError as e:
# print("Failed JSON replacement for OpenAPI keys (2) {e}")
#print(action["parameters"])
# This seems redundant now
print("Pre parameters")
for parameter in newparams:
action["parameters"].append(parameter)
self.action = action
# calltimes is used to handle forloops in the app itself.
# 2 kinds of loop - one in gui with one app each, and one like this,
# which is super fast, but has a bad overview (potentially good tho)
backend/go-app/go.mod
+2 -2
View File
@@ -2,7 +2,7 @@ module shuffle
go 1.13
replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
@@ -23,7 +23,7 @@ require (
github.com/docker/go-units v0.4.0 // indirect
github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
github.com/frikky/kin-openapi v0.39.0
github.com/frikky/shuffle-shared v0.0.99
github.com/frikky/shuffle-shared v0.1.1
github.com/fsouza/go-dockerclient v1.7.2
github.com/ghodss/yaml v1.0.0
github.com/go-git/go-billy/v5 v5.0.0
docker-compose.yml
+1 -1
View File
@@ -1,7 +1,7 @@
version: '3'
services:
frontend:
#build: ./frontend
build: ./frontend
image: ghcr.io/frikky/shuffle-frontend:nightly
container_name: shuffle-frontend
hostname: shuffle-frontend
frontend/src/views/AppCreator.jsx
+8
View File
@@ -1331,6 +1331,10 @@ const AppCreator = (props) => {
const headersplit = header.split("= ")
key = headersplit[0]
value = headersplit[1]
} else if (header.length > 0 && header.includes(" =")) {
const headersplit = header.split(" =")
key = headersplit[0]
value = headersplit[1]
} else if (header.length > 0 && header.includes("=")) {
const headersplit = header.split("=")
key = headersplit[0]
@@ -1339,6 +1343,10 @@ const AppCreator = (props) => {
const headersplit = header.split(": ")
key = headersplit[0]
value = headersplit[1]
} else if (header.length > 0 && header.includes(" :")) {
const headersplit = header.split(" :")
key = headersplit[0]
value = headersplit[1]
} else if (header.length > 0 && header.includes(":")) {
const headersplit = header.split(":")
key = headersplit[0]
functions/extensions/aws-lambda/README.md
-2
View File
@@ -1,2 +0,0 @@
# AWS Lambda forwarder to Shuffle
This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated.
functions/extensions/aws-lambda/s3_deploy.sh
-16
View File
@@ -1,16 +0,0 @@
#GOOS=linux go build main.go
zip s3_function.zip s3_function.py
WEBHOOK=https://shuffler.io/api/v1/hooks/webhook_eccf47b1-8f6a-49fc-b2b8-383365a22353
ROLE=arn:aws:iam::202262580068:role/service-role/shuffle-forwarder
REGION=us-east-1
BUCKETNAME=helo
aws lambda create-function \
--role $ROLE \
--region $REGION \
--function-name shuffler-webhook-forwarder-3 \
--zip-file fileb://s3_function.zip \
--runtime python3.9 \
--environment Variables={SHUFFLE_WEBHOOK=$WEBHOOK} \
--handler lambda_handler
functions/extensions/aws-lambda/s3_function.py
-26
View File
@@ -1,26 +0,0 @@
import json
import urllib.parse
import requests
import os
print('Loading function')
def lambda_handler(event, context):
#print("Received event: " + json.dumps(event, indent=2))
# Get the object from the event and show its content type
bucket = event['Records'][0]['s3']['bucket']['name']
print(type(event))
print("Getting bucket: %s" % bucket)
webhook = os.environ.get("SHUFFLE_WEBHOOK")
if not webhook:
return "No webhook environment defined: SHUFFLE_WEBHOOK"
ret = requests.post(webhook, json=event["Records"][0])
if ret.status_code != 200:
return "Bad status code for webhook: %d" % ret.status_code
print("Status code: %d\nData: %s" % (ret.status_code, ret.text))
# response = s3.get_object(Bucket=bucket, Key=key)
functions/extensions/kafka/README.md
-4
View File
@@ -1,4 +0,0 @@
```
curl -sSL https://raw.githubusercontent.com/bitnami/bitnami-docker-kafka/master/docker-compose.yml > docker-compose.yml
docker-compose up -d
```
functions/extensions/kafka/docker-compose.yml
-16
View File
@@ -1,16 +0,0 @@
version: '2'
services:
zookeeper:
image: wurstmeister/zookeeper
ports:
- "2181:2181"
kafka:
build: .
ports:
- "9092"
environment:
DOCKER_API_VERSION: 1.40
KAFKA_ADVERTISED_HOST_NAME: 192.168.193.140
KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
volumes:
- /var/run/docker.sock:/var/run/docker.sock
functions/extensions/kafka/kafka_local.py
-87
View File
@@ -1,87 +0,0 @@
import json
import os
import requests
from time import sleep
from kafka import KafkaProducer, KafkaConsumer
import kafka
shuffle_url = os.getenv("SHUFFLE_URL")
shuffle_apikey = os.getenv("SHUFFLE_APIKEY")
shuffle_workflow = os.getenv("SHUFFLE_WORKFLOW")
headers = {"Authorization": "Bearer %s" % shuffle_apikey}
topic = "workflow_%s" % shuffle_workflow
group = "testing"
server = "localhost:9092"
def produce():
try:
#group_id=group,
#value_serializer=lambda x: json.dumps(x).encode('utf-8')
producer = KafkaProducer(
bootstrap_servers=[server],
)
except kafka.errors.NoBrokersAvailable as e:
print(f"Error with producer: {e}")
sleep(5)
produce()
return
except ValueError as e:
print(f"ValuError with producer: {e}")
sleep(5)
produce()
return
for i in range(15):
producer.send(topic, json.dumps({"some": i, "data": "luuuuul"}))
producer.flush()
producer.close()
exit()
#print(f"Adding data {data}")
#try:
# ret = producer.send(topic, value=data)
# print(ret.get())
# producer.flush()
# #future = producer.send('foobar', b'another_message')
# #result = future.get(timeout=60)
# #print(result)
#except kafka.errors.KafkaTimeoutError as e:
# print("Kafka error: %s" % e)
# continue
def consume():
print("Starting consumer")
#group_id=group,
#auto_offset_reset="earliest",
#enable_auto_commit=True,
consumer = KafkaConsumer(
topic,
bootstrap_servers=[server],
auto_offset_reset='earliest',
max_poll_records=2,
)
#value_deserializer=lambda x: json.loads(x.decode('utf-8'))
consumer.poll()
#consumer.seek_to_beginning()
print(f"Getting data from topic {topic}")
for message in consumer:
#message = message.value
#message = message
print("MSG: ", message)
ret = requests.post("%s/api/v1/%s/execute" % shuffle_url, headers=headers, data=message)
print(ret.status_code)
print(ret.text)
if __name__ == "__main__":
if topic == "workflow_None":
topic = "testing"
print("Starting producer on %s for topic %s" % (server, topic))
produce()
#consume()
functions/extensions/kafka/requirements.txt
-2
View File
@@ -1,2 +0,0 @@
kafka
kafka-python
functions/extensions/misp/Dockerfile
+23
View File
@@ -0,0 +1,23 @@
FROM python:3.9.4-alpine as base
FROM base as builder
RUN mkdir /install
WORKDIR /install
FROM base
RUN apk add g++
COPY --from=builder /install /usr/local
COPY requirements.txt /requirements.txt
RUN pip3 install -r /requirements.txt
RUN mkdir /app
WORKDIR /app
COPY requirements.txt /app/requirements.txt
RUN python3 -m pip install -r /app/requirements.txt
COPY sub.py /app/sub.py
CMD ["python3", "sub.py"]
functions/extensions/misp/docker-compose.yml
+9
View File
@@ -0,0 +1,9 @@
version: '3'
services:
zmq:
image: ghcr.io/frikky/shuffle-zmq:latest
environment:
- ZMQ_HOSTNAME=localhost
- ZMQ_PORT=50000
- ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e
restart: unless-stopped
functions/extensions/misp/requirements.txt
+2
View File
@@ -0,0 +1,2 @@
pyzmq
requests
functions/extensions/misp/sub.py
+57
View File
@@ -0,0 +1,57 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
print("Running imports")
import sys
import zmq
import json
import time
import pprint
import os
import sys
import requests
forward_url = os.getenv("ZMQ_FORWARD_URL", "")
print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url)
def handle_hook(data):
ret = requests.post(forward_url, json=data)
print(ret.text)
print(ret.status_code)
def main():
host = os.getenv("ZMQ_HOST", "localhost")
port = os.getenv("ZMQ_PORT", "50000")
if len(forward_url) == 0:
print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder")
exit(0)
print("Starting connection setup to %s:%s" % (host, port))
context = zmq.Context()
socket = context.socket(zmq.SUB)
socket.connect ("tcp://%s:%s" % (host, port))
socket.setsockopt(zmq.SUBSCRIBE, b'')
poller = zmq.Poller()
poller.register(socket, zmq.POLLIN)
print("Starting zmq check for %s:%s" % (host, port))
while True:
socks = dict(poller.poll(timeout=None))
if socket in socks and socks[socket] == zmq.POLLIN:
message = socket.recv()
#print(message)
topic, s, m = message.decode('utf-8').partition(" ")
d = json.loads(m)
try:
# print test if you want status (heartbeat)
test = d["status"]
except KeyError:
handle_hook(d)
time.sleep(1)
if __name__ == "__main__":
print("In init ")
main()