Loads of updates to extensions and their documentation

backend/app_sdk/Dockerfile

@@ -5,15 +5,12 @@ RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-de
 
 RUN mkdir /install
 WORKDIR /install
+COPY requirements.txt /requirements.txt
+RUN pip3 install --prefix="/install" -r /requirements.txt
 
 FROM base
-
-#--no-cache 
-RUN apk update && apk add --update tzdata libmagic alpine-sdk libffi libffi-dev musl-dev openssl-dev
-
 COPY --from=builder /install /usr/local
-COPY requirements.txt /requirements.txt
-RUN pip3 install -r /requirements.txt
 
-COPY __init__.py /app/walkoff_app_sdk/__init__.py
+WORKDIR /app
-COPY app_base.py /app/walkoff_app_sdk/app_base.py
+COPY sub.py /app/sub.py
+CMD python sub.py 

backend/app_sdk/app_base.py

@@ -1644,6 +1644,86 @@ class AppBase:
 
             return template
 
+        # Suboptimal cleanup script for BOdy parsing of OpenAPI
+        # Should have a regex which looks for the value, then goes out and cleans up the key
+        def recurse_cleanup_script(data):
+            try:
+                if not isinstance(data, dict):
+                    newvalue = json.loads(data)
+                else:
+                    newvalue = data
+        
+                deletekeys = []
+                for key, value in newvalue.items():
+                    #print("%s: %s" % (key, value))
+                    if isinstance(value, str) and len(value) == 0:
+                        deletekeys.append(key)
+                        continue
+                            
+                    if isinstance(value, list):
+                        try:
+                            value = json.dumps(value)
+                            print(value)
+                        except:
+                            print("Json parsing issue in recursed value")
+                            pass
+        
+                    if value == "${%s}" % key:
+                        print("Deleting %s because key = value" % key)
+                        deletekeys.append(key)
+                        continue
+                    elif "${" in value and "}" in value:
+                        print("Deleting %s because it contains ${ and }" % key)
+                        deletekeys.append(key)
+                        continue
+        
+                    if isinstance(value, dict):
+                        newvalue[key] = recurse_cleanup_script(value)
+        
+            except json.decoder.JSONDecodeError as e:
+                print("Failed JSON replacement for OpenAPI keys (3) {e}")
+                
+            for deletekey in deletekeys:
+                try:
+                    del newvalue[deletekey]
+                except:
+                    pass
+        
+            try:
+                #print("Post delete: %s" % newvalue)
+                for key, value in newvalue.items():
+                    if isinstance(value, bool):
+                        continue
+                    elif isinstance(value, dict) and not bool(value):
+                        continue
+        
+                    try:
+                        value = json.loads(value)
+                        newvalue[key] = value
+                    except json.decoder.JSONDecodeError as e:
+                        #print("Inner overwrite issue for \"%s\": %s" % (key, e))
+                        continue
+                    except Exception as e:
+                        #print("General error in newvalue items loop: %s" % e)
+                        continue
+        
+                try:
+                    data = json.dumps(newvalue)
+                except json.decoder.JSONDecodeError as e:
+                    print("[WARNING] JsonDecodeError: %s" % e)
+                    data = newvalue
+        
+            except json.decoder.JSONDecodeError as e:
+                print("Failed JSON replacement for OpenAPI keys (2) {e}")
+        
+            #if isinstance(data, str):
+            #    tmpdata = json.dumps(data)
+            #    print(tmpdata)
+            #    foundvalue = re.findall(".*?(${\w+})", tmpdata, re.MULTILINE)
+            #    print("FOUND: %s", foundvalue)
+        
+            return data 
+
         # Parses parameters sent to it and returns whether it did it successfully with the values found
         def parse_params(action, fullexecution, parameter, self):
             # Skip if it starts with $?
@@ -2112,60 +2192,63 @@ class AppBase:
                                     print("KeyError body OpenAPI: %s" % e)
                                     pass
 
-                                
+                                 
                                 print(f"""HANDLING {action["parameters"][counter]["value"]}""")
-                                try:
+                                action["parameters"][counter]["value"] = recurse_cleanup_script(action["parameters"][counter]["value"])
-                                    newvalue = json.loads(action["parameters"][counter]["value"])
+                                #try:
-                                    deletekeys = []
+                                #    newvalue = json.loads(action["parameters"][counter]["value"])
-                                    for key, value in newvalue.items():
+                                #    deletekeys = []
-                                        print("%s: %s" % (key, value))
+                                #    for key, value in newvalue.items():
-                                        if isinstance(value, str) and len(value) == 0:
+                                #        print("%s: %s" % (key, value))
-                                            deletekeys.append(key)
+                                #        if isinstance(value, str) and len(value) == 0:
-                                            continue
+                                #            deletekeys.append(key)
+                                #            continue
 
-                                        if value == "${%s}" % key:
+                                #        if value == "${%s}" % key:
-                                            print("Deleting %s because key = value" % key)
+                                #            print("Deleting %s because key = value" % key)
-                                            deletekeys.append(key)
+                                #            deletekeys.append(key)
-                                            continue
+                                #            continue
-                                    
+                                #    
-                                    for deletekey in deletekeys:
+                                #    for deletekey in deletekeys:
-                                        try:
+                                #        try:
-                                            del newvalue[deletekey]
+                                #            del newvalue[deletekey]
-                                        except:
+                                #        except:
-                                            pass
+                                #            pass
 
-                                    #print("Post delete: %s" % newvalue)
+                                #    #print("Post delete: %s" % newvalue)
-                                    for key, value in newvalue.items():
+                                #    for key, value in newvalue.items():
-                                        if isinstance(value, bool):
+                                #        if isinstance(value, bool):
-                                            continue
+                                #            continue
 
-                                        try:
+                                #        try:
-                                            value = json.loads(value)
+                                #            value = json.loads(value)
-                                            newvalue[key] = value
+                                #            newvalue[key] = value
-                                        except json.decoder.JSONDecodeError as e:
+                                #        except json.decoder.JSONDecodeError as e:
-                                            print("Inner overwrite issue: %s" % e)
+                                #            print("Inner overwrite issue: %s" % e)
-                                            continue
+                                #            continue
-                                        except:
+                                #        except Exception as e:
-                                            print("General error in newvalue items loop")
+                                #            print("General error in newvalue items loop: %s" % e)
-                                            continue
+                                #            continue
 
-                                    try:
+                                #    try:
-                                        action["parameters"][counter]["value"] = json.dumps(newvalue)
+                                #        action["parameters"][counter]["value"] = json.dumps(newvalue)
-                                    except json.decoder.JSONDecodeError as e:
+                                #    except json.decoder.JSONDecodeError as e:
-                                        print("[WARNING] JsonDecodeError: %s" % e)
+                                #        print("[WARNING] JsonDecodeError: %s" % e)
-                                        action["parameters"][counter]["value"] = newvalue
+                                #        action["parameters"][counter]["value"] = newvalue
-                                        
+                                #        
 
-                                except json.decoder.JSONDecodeError as e:
+                                #except json.decoder.JSONDecodeError as e:
-                                    print("Failed JSON replacement for OpenAPI keys (2) {e}")
+                                #    print("Failed JSON replacement for OpenAPI keys (2) {e}")
-
-                                break
 
                         #print(action["parameters"])
+
+                        # This seems redundant now 
                         print("Pre parameters")
                         for parameter in newparams:
                             action["parameters"].append(parameter)
 
+                        self.action = action
+
                         # calltimes is used to handle forloops in the app itself.
                         # 2 kinds of loop - one in gui with one app each, and one like this,
                         # which is super fast, but has a bad overview (potentially good tho)

backend/go-app/go.mod

@@ -2,7 +2,7 @@ module shuffle
 
 go 1.13
 
-replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
+//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
 
 //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
 
@@ -23,7 +23,7 @@ require (
 	github.com/docker/go-units v0.4.0 // indirect
 	github.com/elastic/go-elasticsearch/v7 v7.13.1 // indirect
 	github.com/frikky/kin-openapi v0.39.0
-	github.com/frikky/shuffle-shared v0.0.99
+	github.com/frikky/shuffle-shared v0.1.1
 	github.com/fsouza/go-dockerclient v1.7.2
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-git/go-billy/v5 v5.0.0

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   frontend:
-    #build: ./frontend
+    build: ./frontend
     image: ghcr.io/frikky/shuffle-frontend:nightly
     container_name: shuffle-frontend
     hostname: shuffle-frontend

frontend/src/views/AppCreator.jsx

@@ -1331,6 +1331,10 @@ const AppCreator = (props) => {
 						const headersplit = header.split("= ")
 						key = headersplit[0]	
 						value = headersplit[1]	
+					} else if (header.length > 0 && header.includes(" =")) {
+						const headersplit = header.split(" =")
+						key = headersplit[0]	
+						value = headersplit[1]	
 					} else if (header.length > 0 && header.includes("=")) {
 						const headersplit = header.split("=")
 						key = headersplit[0]	
@@ -1339,6 +1343,10 @@ const AppCreator = (props) => {
 						const headersplit = header.split(": ")
 						key = headersplit[0]	
 						value = headersplit[1]	
+					} else if (header.length > 0 && header.includes(" :")) {
+						const headersplit = header.split(" :")
+						key = headersplit[0]	
+						value = headersplit[1]	
 					} else if (header.length > 0 && header.includes(":")) {
 						const headersplit = header.split(":")
 						key = headersplit[0]	

functions/extensions/aws-lambda/README.md

@@ -1,2 +0,0 @@
-# AWS Lambda forwarder to Shuffle
-This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated. 

functions/extensions/aws-lambda/s3_deploy.sh

@@ -1,16 +0,0 @@
-#GOOS=linux go build main.go
-zip s3_function.zip s3_function.py 
-
-WEBHOOK=https://shuffler.io/api/v1/hooks/webhook_eccf47b1-8f6a-49fc-b2b8-383365a22353
-ROLE=arn:aws:iam::202262580068:role/service-role/shuffle-forwarder
-REGION=us-east-1
-BUCKETNAME=helo
-
-aws lambda create-function \
-	--role $ROLE \
-	--region $REGION \
-	--function-name shuffler-webhook-forwarder-3 \
-	--zip-file fileb://s3_function.zip \
-	--runtime python3.9 \
-	--environment Variables={SHUFFLE_WEBHOOK=$WEBHOOK} \
-	--handler lambda_handler 

functions/extensions/aws-lambda/s3_function.py

@@ -1,26 +0,0 @@
-import json
-import urllib.parse
-import requests
-import os
-
-print('Loading function')
-
-def lambda_handler(event, context):
-    #print("Received event: " + json.dumps(event, indent=2))
-
-    # Get the object from the event and show its content type
-    bucket = event['Records'][0]['s3']['bucket']['name']
-    
-    print(type(event))
-    print("Getting bucket: %s" % bucket)
-    webhook = os.environ.get("SHUFFLE_WEBHOOK")
-    if not webhook:
-        return "No webhook environment defined: SHUFFLE_WEBHOOK"
-        
-    ret = requests.post(webhook, json=event["Records"][0])
-    if ret.status_code != 200:
-        return "Bad status code for webhook: %d" % ret.status_code
-        
-    print("Status code: %d\nData: %s" % (ret.status_code, ret.text))
-    
-    #    response = s3.get_object(Bucket=bucket, Key=key)

functions/extensions/kafka/README.md

@@ -1,4 +0,0 @@
-```
-curl -sSL https://raw.githubusercontent.com/bitnami/bitnami-docker-kafka/master/docker-compose.yml > docker-compose.yml
-docker-compose up -d
-```

functions/extensions/kafka/docker-compose.yml

@@ -1,16 +0,0 @@
-version: '2'
-services:
-  zookeeper:
-    image: wurstmeister/zookeeper
-    ports:
-      - "2181:2181"
-  kafka:
-    build: .
-    ports:
-      - "9092"
-    environment:
-      DOCKER_API_VERSION: 1.40
-      KAFKA_ADVERTISED_HOST_NAME: 192.168.193.140
-      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock

functions/extensions/kafka/kafka_local.py

@@ -1,87 +0,0 @@
-import json 
-import os
-import requests
-from time import sleep
-from kafka import KafkaProducer, KafkaConsumer
-import kafka
-
-shuffle_url = os.getenv("SHUFFLE_URL")
-shuffle_apikey = os.getenv("SHUFFLE_APIKEY")
-shuffle_workflow = os.getenv("SHUFFLE_WORKFLOW")
-    
-headers = {"Authorization": "Bearer %s" % shuffle_apikey}
-topic = "workflow_%s" % shuffle_workflow
-group = "testing"
-server = "localhost:9092"
-def produce():
-    try:
-        #group_id=group,
-        #value_serializer=lambda x: json.dumps(x).encode('utf-8')
-        producer = KafkaProducer(
-            bootstrap_servers=[server],
-        )
-    except kafka.errors.NoBrokersAvailable as e:
-        print(f"Error with producer: {e}")
-        sleep(5)
-        produce()
-        return
-    except ValueError as e:
-        print(f"ValuError with producer: {e}")
-        sleep(5)
-        produce()
-        return
-        
-        
-    for i in range(15):
-        producer.send(topic, json.dumps({"some": i, "data": "luuuuul"}))
-
-    producer.flush()
-    producer.close()
-    exit()
-
-        #print(f"Adding data {data}")
-
-        #try:
-        #    ret = producer.send(topic, value=data)
-        #    print(ret.get())
-        #    producer.flush()
-
-        #    #future = producer.send('foobar', b'another_message')
-        #    #result = future.get(timeout=60)
-        #    #print(result)
-
-        #except kafka.errors.KafkaTimeoutError as e:
-        #    print("Kafka error: %s" % e)
-        #    continue
-
-def consume():
-    print("Starting consumer")
-    #group_id=group,
-    #auto_offset_reset="earliest",
-    #enable_auto_commit=True,
-    consumer = KafkaConsumer(
-        topic,
-        bootstrap_servers=[server],
-        auto_offset_reset='earliest',
-        max_poll_records=2,
-    )
-    #value_deserializer=lambda x: json.loads(x.decode('utf-8'))
-
-    consumer.poll()
-    #consumer.seek_to_beginning()
-
-    print(f"Getting data from topic {topic}")
-    for message in consumer:
-        #message = message.value
-        #message = message
-        print("MSG: ", message)
-        ret = requests.post("%s/api/v1/%s/execute" % shuffle_url, headers=headers, data=message)
-        print(ret.status_code)
-        print(ret.text)
-
-if __name__ == "__main__":
-    if topic == "workflow_None":
-        topic = "testing"
-    print("Starting producer on %s for topic %s" % (server, topic))
-    produce()
-    #consume()

functions/extensions/kafka/requirements.txt

@@ -1,2 +0,0 @@
-kafka
-kafka-python

functions/extensions/misp/Dockerfile

@@ -0,0 +1,23 @@
+FROM python:3.9.4-alpine as base
+
+FROM base as builder 
+
+RUN mkdir /install
+WORKDIR /install
+
+FROM base
+RUN apk add g++
+
+COPY --from=builder /install /usr/local
+COPY requirements.txt /requirements.txt
+RUN pip3 install -r /requirements.txt
+
+
+RUN mkdir /app
+WORKDIR /app
+COPY requirements.txt /app/requirements.txt
+RUN python3 -m pip install -r /app/requirements.txt
+
+COPY sub.py /app/sub.py
+
+CMD ["python3", "sub.py"]

functions/extensions/misp/docker-compose.yml

@@ -0,0 +1,9 @@
+version: '3'
+services:
+  zmq:
+    image: ghcr.io/frikky/shuffle-zmq:latest
+    environment:
+      - ZMQ_HOSTNAME=localhost
+      - ZMQ_PORT=50000
+      - ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e
+    restart: unless-stopped

functions/extensions/misp/requirements.txt

@@ -0,0 +1,2 @@
+pyzmq
+requests

functions/extensions/misp/sub.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+print("Running imports")
+import sys
+import zmq
+import json
+import time
+import pprint
+import os
+import sys
+import requests
+
+forward_url = os.getenv("ZMQ_FORWARD_URL", "")
+print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url)
+def handle_hook(data):
+    ret = requests.post(forward_url, json=data)
+    print(ret.text)
+    print(ret.status_code)
+
+def main():
+    host = os.getenv("ZMQ_HOST", "localhost")
+    port = os.getenv("ZMQ_PORT", "50000")
+
+    if len(forward_url) == 0:
+        print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder")
+        exit(0)
+
+    print("Starting connection setup to %s:%s" % (host, port))
+    context = zmq.Context()
+    socket = context.socket(zmq.SUB)
+    socket.connect ("tcp://%s:%s" % (host, port))
+    socket.setsockopt(zmq.SUBSCRIBE, b'')
+    
+    poller = zmq.Poller()
+    poller.register(socket, zmq.POLLIN)
+    
+    print("Starting zmq check for %s:%s" % (host, port))
+    while True:
+        socks = dict(poller.poll(timeout=None))
+        if socket in socks and socks[socket] == zmq.POLLIN:
+            message = socket.recv()
+            #print(message)
+            topic, s, m = message.decode('utf-8').partition(" ")
+    
+            d = json.loads(m)
+            try:
+                # print test if you want status (heartbeat)
+                test = d["status"]
+            except KeyError:
+                handle_hook(d)
+    
+            time.sleep(1)
+
+if __name__ == "__main__":
+    print("In init ")
+    main()