Merge branch 'master' into launch

README.md

@@ -1,5 +1,5 @@
 # Shuffle 
-[Shuffle](https://shuffler.io) is an automation platform to unify your security services (SOAR). It has thousands of premade integrations and is based on open frameworks like OpenAPI and Mitre Att&ck. The workflow editor is based on a no-code thought process to empower non-developers, and the app creator makes you able to inegrate any platform in minutes.
+[Shuffle](https://shuffler.io) is an automation platform to unify your security services (SOAR). It has thousands of premade integrations and is based on open frameworks like OpenAPI and Mitre Att&ck. The workflow editor is based on a no-code thought process to empower non-developers, and the app creator makes you able to integrate any platform in minutes.
 
 [![Discord](https://img.shields.io/discord/463752820026376202.svg?label=&logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/B2CBzUm)
 
@@ -37,6 +37,9 @@ Please consider [sponsoring](https://github.com/sponsors/frikky) the project if
 * App creator for [OpenAPI](https://github.com/frikky/OpenAPI-security-definitions)
 * Easy to learn Python library for custom apps
 
+## Architecture
+![Shuffle Architecture](https://github.com/frikky/Shuffle/blob/master/frontend/src/assets/img/shuffle_architecture.png)
+
 ## Website
 https://shuffler.io
 
@@ -46,7 +49,7 @@ All modular information related to Shuffle will be under MIT (anyone can use it
 Apps & App SDK: MIT
 Shuffle backend: AGPLv3 
 
-### Project overview 
+### Repository overview 
 Below is the folder structure with a short explanation
 ```bash
 ├── README.md				# What you're reading right now

backend/Dockerfile

@@ -14,6 +14,10 @@ ADD ./go-app/go.mod /app
 # Required files for code generation
 ADD ./app_sdk/app_base.py /app_sdk
 ADD ./app_sdk/static_baseline.py /app_sdk
+ADD ./app_sdk_kali/app_base.py /app_sdk_kali
+ADD ./app_sdk_kali/static_baseline.py /app_sdk_kali
+ADD ./app_sdk_blackarch/app_base.py /app_sdk_blackarch
+ADD ./app_sdk_blackarch/static_baseline.py /app_sdk_blackarch
 ADD ./app_gen /app_gen
 
 RUN go get -v
@@ -28,6 +32,8 @@ FROM alpine:3.12
 
 COPY --from=builder /app/ /app
 COPY --from=builder /app_sdk/ /app_sdk
+COPY --from=builder /app_sdk_kali/ /app_sdk_kali
+COPY --from=builder /app_sdk_blackarch/ /app_sdk_blackarch
 COPY --from=builder /app_gen/ /app_gen
 COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
 

backend/app_sdk_blackarch/Dockerfile

@@ -0,0 +1,19 @@
+FROM peterclemenko/blackarch as base
+
+FROM base as builder
+
+RUN /bin/pacman -Syu --noconfirm
+
+RUN	/bin/pacman -Sy --noconfirm base-devel libffi musl openssl python python-pip -y
+
+RUN mkdir /install
+WORKDIR /install
+
+COPY requirements.txt /requirements.txt
+RUN pip install --prefix="/install" -r /requirements.txt
+
+FROM base
+
+COPY --from=builder /install /usr/local
+COPY __init__.py /app/walkoff_app_sdk/__init__.py
+COPY app_base.py /app/walkoff_app_sdk/app_base.py

backend/app_sdk_blackarch/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Frikkylikeme
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

backend/app_sdk_blackarch/README.md

@@ -0,0 +1,16 @@
+# app_sdk.py
+This is the SDK used for apps to behave like they should. 
+To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline.
+
+# static_baseline.py
+It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend.
+
+## If you want to update apps.. PS: downloads from docker hub do overrides.. :)
+1. Write your code & check if runtime works
+2. Build app_base image
+3. docker rm $(docker ps -aq) # Remove all stopped containers
+4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
+5. Rebuild the Docker image (click load in GUI?)
+
+# LICENSE 
+Everything in here is MIT, not AGPLv3 as indicated by the license.

backend/app_sdk_blackarch/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+NAME=app_sdk_blackarch
+VERSION=0.7.3
+
+docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
+docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
+
+#docker push frikky/$NAME:$VERSION
+#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
+#docker push ghcr.io/frikky/$NAME:$VERSION
+
+docker push frikky/shuffle:$NAME
+docker push ghcr.io/frikky/$NAME:$VERSION

backend/app_sdk_blackarch/requirements.txt

@@ -0,0 +1,2 @@
+requests
+urllib3

backend/app_sdk_blackarch/static_baseline.py

@@ -0,0 +1,76 @@
+import os
+import sys
+import time 
+import logging
+import requests
+
+# Goal here:
+# * Make an app from WALKOFF able to run without app_base.py from WALKOFF
+# # How:
+# * Make it rely 100% on INPUT throug HTTP invocations instead of redis READS 
+# # But really, how?
+# * Make a WORKER that reads the queue, and reuses a function 
+
+# Here to get it global
+apikey = ""
+try:
+    apikey = os.environ["FUNCTION_APIKEY"]
+except KeyError:
+    pass
+
+# Authorize the execution 
+def authorization(request):
+    # This is basically my issue, but it enforces the use of an internal API key for execution
+    try:
+        apikey = os.environ["FUNCTION_APIKEY"]
+    except KeyError:
+        return f"Internal server error", 500
+
+
+    # Check API key from ENV authentication
+    authentication = request.headers.get("Authorization")
+    if authentication == None: return f"Unauthorized", 401
+
+    apikey_split = authentication.split(" ")
+    if apikey_split[0] != "Bearer" or len(apikey_split) != 2:
+        return f"Apikey error", 401
+
+    if apikey != apikey_split[1]:
+        return f"Unauthorized", 401
+
+    return run(request)
+
+class AppBase:
+    """ The base class for Python-based Walkoff applications, handles Redis and logging configurations. """
+    __version__ = None
+    app_name = None
+
+    def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None):
+        self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger")
+        self.redis=redis
+        self.console_logger=console_logger 
+        self.current_execution_id = None
+        self.url = "https://shuffler.io"
+        self.apikey = apikey
+
+    @classmethod
+    async def run(cls, action):
+        """ Connect to Redis and HTTP session, await actions """
+        logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{')
+        logger = logging.getLogger(f"{cls.__name__}")
+        logger.setLevel(logging.DEBUG)
+
+        app = cls(redis=None, logger=logger, console_logger=logger)
+
+        # Authorization for the app/function to control the workflow
+        # Function will crash if its wrong, which it probably should. 
+
+        await app.execute_action(action)
+    
+    async def execute_action(self, action):
+        # FIXME - add request for the function STARTING here. Use "results stream" or something
+        # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
+
+        self.authorization = action["authorization"]
+        self.execution_id = action["execution_id"]
+        self.current_execution_id = action["execution_id"]

backend/app_sdk_kali/Dockerfile

@@ -0,0 +1,19 @@
+FROM kalilinux/kali-rolling as base
+
+FROM base as builder
+
+RUN apt-get update
+RUN apt-get dist-upgrade -y
+RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y
+
+RUN mkdir /install
+WORKDIR /install
+
+COPY requirements.txt /requirements.txt
+RUN pip install --prefix="/install" -r /requirements.txt
+
+FROM base
+
+COPY --from=builder /install /usr/local
+COPY __init__.py /app/walkoff_app_sdk/__init__.py
+COPY app_base.py /app/walkoff_app_sdk/app_base.py

backend/app_sdk_kali/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Frikkylikeme
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

backend/app_sdk_kali/README.md

@@ -0,0 +1,16 @@
+# app_sdk.py
+This is the SDK used for apps to behave like they should. 
+To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline.
+
+# static_baseline.py
+It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend.
+
+## If you want to update apps.. PS: downloads from docker hub do overrides.. :)
+1. Write your code & check if runtime works
+2. Build app_base image
+3. docker rm $(docker ps -aq) # Remove all stopped containers
+4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...)
+5. Rebuild the Docker image (click load in GUI?)
+
+# LICENSE 
+Everything in here is MIT, not AGPLv3 as indicated by the license.

backend/app_sdk_kali/build.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+NAME=app_sdk_kali
+VERSION=0.7.3
+
+docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force
+docker build . -t frikky/shuffle:$NAME -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION
+
+#docker push frikky/$NAME:$VERSION
+#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION
+#docker push ghcr.io/frikky/$NAME:$VERSION
+
+docker push frikky/shuffle:$NAME
+docker push ghcr.io/frikky/$NAME:$VERSION

backend/app_sdk_kali/requirements.txt

@@ -0,0 +1,2 @@
+requests
+urllib3

backend/app_sdk_kali/static_baseline.py

@@ -0,0 +1,76 @@
+import os
+import sys
+import time 
+import logging
+import requests
+
+# Goal here:
+# * Make an app from WALKOFF able to run without app_base.py from WALKOFF
+# # How:
+# * Make it rely 100% on INPUT throug HTTP invocations instead of redis READS 
+# # But really, how?
+# * Make a WORKER that reads the queue, and reuses a function 
+
+# Here to get it global
+apikey = ""
+try:
+    apikey = os.environ["FUNCTION_APIKEY"]
+except KeyError:
+    pass
+
+# Authorize the execution 
+def authorization(request):
+    # This is basically my issue, but it enforces the use of an internal API key for execution
+    try:
+        apikey = os.environ["FUNCTION_APIKEY"]
+    except KeyError:
+        return f"Internal server error", 500
+
+
+    # Check API key from ENV authentication
+    authentication = request.headers.get("Authorization")
+    if authentication == None: return f"Unauthorized", 401
+
+    apikey_split = authentication.split(" ")
+    if apikey_split[0] != "Bearer" or len(apikey_split) != 2:
+        return f"Apikey error", 401
+
+    if apikey != apikey_split[1]:
+        return f"Unauthorized", 401
+
+    return run(request)
+
+class AppBase:
+    """ The base class for Python-based Walkoff applications, handles Redis and logging configurations. """
+    __version__ = None
+    app_name = None
+
+    def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None):
+        self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger")
+        self.redis=redis
+        self.console_logger=console_logger 
+        self.current_execution_id = None
+        self.url = "https://shuffler.io"
+        self.apikey = apikey
+
+    @classmethod
+    async def run(cls, action):
+        """ Connect to Redis and HTTP session, await actions """
+        logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{')
+        logger = logging.getLogger(f"{cls.__name__}")
+        logger.setLevel(logging.DEBUG)
+
+        app = cls(redis=None, logger=logger, console_logger=logger)
+
+        # Authorization for the app/function to control the workflow
+        # Function will crash if its wrong, which it probably should. 
+
+        await app.execute_action(action)
+    
+    async def execute_action(self, action):
+        # FIXME - add request for the function STARTING here. Use "results stream" or something
+        # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE
+
+        self.authorization = action["authorization"]
+        self.execution_id = action["execution_id"]
+        self.current_execution_id = action["execution_id"]

functions/onprem/README.md

@@ -38,3 +38,19 @@ cd app_sdk
 docker build . -t frikky/shuffle:app_sdk
 docker push frikky/shuffle:app_sdk
 ```
+
+
+Update app_sdk_kali:
+```
+cd app_sdk_kali
+docker build . -t frikky/shuffle:app_sdk_kali
+docker push frikky/shuffle:app_sdk_kali
+```
+
+
+Update app_sdk_blackarch:
+```
+cd app_sdk_blackarch
+docker build . -t frikky/shuffle:app_sdk_blackarch
+docker push frikky/shuffle:app_sdk_blackarch
+```