From 0b3d2f929033605b5ad845e05d93ca0626ccb91c Mon Sep 17 00:00:00 2001 From: Azgaviper Date: Fri, 23 Apr 2021 22:19:09 +0200 Subject: [PATCH 1/4] Quick base change Shuffling around the .env and Readme --- .env | 8 +-- .github/install-guide.html | 126 +++++++++++++++++++++++++++++++++++++ .github/install-guide.md | 15 ++--- 3 files changed, 138 insertions(+), 11 deletions(-) create mode 100644 .github/install-guide.html diff --git a/.env b/.env index 4ffc4215..de39c311 100644 --- a/.env +++ b/.env @@ -8,15 +8,15 @@ SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME= SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD= SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH= -SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps +SHUFFLE_APP_DOWNLOAD_LOCATION=https://github.com/frikky/shuffle-apps SHUFFLE_DOWNLOAD_AUTH_USERNAME= SHUFFLE_DOWNLOAD_AUTH_PASSWORD= SHUFFLE_DOWNLOAD_AUTH_BRANCH= SHUFFLE_APP_FORCE_UPDATE=false # User config for first load. Username & PW: min length 3 -SHUFFLE_DEFAULT_USERNAME= -SHUFFLE_DEFAULT_PASSWORD= +SHUFFLE_DEFAULT_USERNAME=shuffle +SHUFFLE_DEFAULT_PASSWORD==Fr1kky1sN0t@D0g SHUFFLE_DEFAULT_APIKEY= # Local location of your app directory. Can't use ~/ @@ -33,7 +33,7 @@ OUTER_HOSTNAME=shuffle-backend DB_LOCATION=./shuffle-database # Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps. -# PS: It will skip proxy for +# PS: It will skip proxy for SHUFFLE_HTTP_PROXY= SHUFFLE_HTTPS_PROXY= SHUFFLE_PASS_WORKER_PROXY=TRUE diff --git a/.github/install-guide.html b/.github/install-guide.html new file mode 100644 index 00000000..45bf03e2 --- /dev/null +++ b/.github/install-guide.html @@ -0,0 +1,126 @@ + + + + + +

Installation guide

+

Installation of Shuffle is currently only available in docker. Looking for how to update Shuffle? Check the updating guide

+

Docker

+

The Docker setup is done with docker-compose and is a single command to get set up.

+

PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)

+
    +
  1. Make sure you have Docker and docker-compose installed.
    2. +
  2. Adapt the .env file to your convenience.
    3. +
  3. Run docker-compose.
    git clone https://github.com/frikky/Shuffle
+cd Shuffle
+docker-compose up -d
    4. +
+

When you're done, skip to the "After installation" step below.

+

Windows Docker setup

+

This step is for setting up with Docker on windows from scratch.

+
    +
  1. Make sure you have Docker and docker-compose installed. WSL2 may be required.
    2. +
  2. Go to https://github.com/frikky/shuffle/releases and download the latest .zip release (or install git)
    3. +
  3. Unzip the folder and enter it
    4. +
  4. Open the .env file and change the line with "OUTER_HOSTNAME" to contain your IP:
    OUTER_HOSTNAME=YOUR.IP.HERE
    5. +
  5. Run docker-compose
    docker-compose up -d
    6. +
+

Configurations (proxies, default users etc.)

+

https://shuffler.io/docs/configuration

+

After installation

+
    +

  1. After installation, go to http://localhost:3001/adminsetup (or your servername)

    +
    2. +

  2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password.

    +
    3. +
  3. Check out https://shuffler.io/docs/configuration as it has a lot of useful information to get started
    4. +
+

Admin account setup

+

Useful info

+
    +
  • Check out getting started
    • +
  • The server is available on http://localhost:3001 (or your servername)
    • +
  • Further configurations can be done in docker-compose.yml and .env.
    • +
  • Default database location is /etc/shuffle
    • +
+

Execution problems

+

If you have problems with your first execution (hello world), you might need to set the correct Docker API version. Here's how:

+
    +
  1. Find your API version by running "docker version" +` +$ docker version
    2. +
+

Client: + Version: 17.09.1-ce + API version: 1.32 # <-- this one + Go version: go1.8.3 + Git commit: 19e2cf6 + Built: Thu Dec 7 22:24:16 2017 + OS/Arch: linux/amd64

+

Server: + Version: 17.09.1-ce + API version: 1.32 (minimum version 1.12) + Go version: go1.8.3 + Git commit: 19e2cf6 + Built: Thu Dec 7 22:22:56 2017 + OS/Arch: linux/amd64 + Experimental: false

+

+2. Open docker-compose.yml and change the line with "DOCKER_API_VERSION" to your version.
+3. Restart docker-compose

docker-compose down +docker-compose up

+

+Related issue: #47
+
+# Local development installation
+Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes.
+
+**PS: You have to stop the Backend Docker container to get this one working**
+**PPS: Use the "Launch" branch when developing to get it set up easier**
+
+## Frontend - ReactJS /w cytoscape
+http://localhost:3000 - Requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend.
+```bash
+cd frontend
+npm i
+npm start

Backend - Golang

+

http://localhost:5001 - REST API - requires >=go1.13

+
export DATASTORE_EMULATOR_HOST=0.0.0.0:8000
+cd backend/go-app
+go run *.go
+

WINDOWS USERS: You'll have to to add the "export" part as an environment variable.

+

Database - Datastore

+

Based on Google datastore

+
docker run -p 8000:8000 google/cloud-sdk gcloud beta emulators datastore start --project=shuffle --host-port 0.0.0.0:8000 --no-store-on-disk

Orborus

+

Execution of Workflows: +PS: This requires some specific environment variables

+
cd functions/onprem/orborus
+go run orborus.go

Environments (modify for Windows):

+
export ORG_ID=Shuffle
+export ENVIRONMENT_NAME=Shuffle
+export BASE_URL=http://YOUR-IP:5001
+export DOCKER_API_VERSION=1.40

WINDOWS USERS: You'll have to to add the "export" part as an environment variable.

+

AND THAT's it - hopefully it worked. If it didn't please email frikky@shuffler.io

+
+ +
+ \ No newline at end of file diff --git a/.github/install-guide.md b/.github/install-guide.md index d1c21169..34a1f6f2 100644 --- a/.github/install-guide.md +++ b/.github/install-guide.md @@ -7,7 +7,8 @@ The Docker setup is done with docker-compose and is a single command to get set **PS: if you're setting up Shuffle on Windows, go to the next step (Windows Docker setup)** 1. Make sure you have Docker and [docker-compose](https://docs.docker.com/compose/install/) installed. -2. Run docker-compose. +2. Adapt the .env file to your convenience. +3. Run docker-compose. ``` git clone https://github.com/frikky/Shuffle cd Shuffle @@ -16,7 +17,7 @@ docker-compose up -d When you're done, skip to the "After installation" step below. -## Windows Docker setup +## Windows Docker setup This step is for setting up with Docker on windows from scratch. 1. Make sure you have [Docker](https://docs.docker.com/docker-for-windows/install/) and [docker-compose](https://docs.docker.com/compose/install/) installed. WSL2 may be required. @@ -34,7 +35,7 @@ docker-compose up -d ### Configurations (proxies, default users etc.) https://shuffler.io/docs/configuration -### After installation +### After installation 1. After installation, go to http://localhost:3001/adminsetup (or your servername) 2. Now set up your admin account (username & password). Shuffle doesn't have a default username and password. @@ -82,14 +83,14 @@ docker-compose up Related issue: #47 -# Local development installation +# Local development installation Local development is pretty straight forward with **ReactJS** and **Golang**. This part is intended to help you run the code for development purposes. **PS: You have to stop the Backend Docker container to get this one working** **PPS: Use the "Launch" branch when developing to get it set up easier** ## Frontend - ReactJS /w cytoscape -http://localhost:3000 - Requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend. +http://localhost:3000 - Requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend. ```bash cd frontend npm i @@ -97,7 +98,7 @@ npm start ``` ## Backend - Golang -http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) +http://localhost:5001 - REST API - requires [>=go1.13](https://golang.org/dl/) ```bash export DATASTORE_EMULATOR_HOST=0.0.0.0:8000 cd backend/go-app @@ -114,7 +115,7 @@ docker run -p 8000:8000 google/cloud-sdk gcloud beta emulators datastore start - ## Orborus Execution of Workflows: -PS: This requires some specific environment variables +PS: This requires some specific environment variables ``` cd functions/onprem/orborus go run orborus.go From 0952670663e0bb213c5420c850118c6885af8ed7 Mon Sep 17 00:00:00 2001 From: azgaviperr Date: Tue, 4 May 2021 22:44:57 +0200 Subject: [PATCH 2/4] Fixed the responder and improvment --- .../Shuffle/requirements.txt | 2 ++ .../cortex-responders/Shuffle/shuffle.json | 10 +++++- .../cortex-responders/Shuffle/shuffle.py | 34 +++++++++---------- 3 files changed, 27 insertions(+), 19 deletions(-) create mode 100644 functions/extensions/cortex-responders/Shuffle/requirements.txt diff --git a/functions/extensions/cortex-responders/Shuffle/requirements.txt b/functions/extensions/cortex-responders/Shuffle/requirements.txt new file mode 100644 index 00000000..6aabc3cf --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle/requirements.txt @@ -0,0 +1,2 @@ +cortexutils +requests diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.json b/functions/extensions/cortex-responders/Shuffle/shuffle.json index ef2610dd..db755b49 100644 --- a/functions/extensions/cortex-responders/Shuffle/shuffle.json +++ b/functions/extensions/cortex-responders/Shuffle/shuffle.json @@ -5,7 +5,7 @@ "url": "https://github.com/frikky/shuffle", "license": "AGPL-V3", "description": "Execute a workflow in Shuffle", - "dataTypeList": ["thehive:case", "thehive:alert"], + "dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"], "command": "Shuffle/shuffle.py", "baseConfig": "Shuffle", "configurationItems": [ @@ -24,6 +24,14 @@ "multi": false, "required": true }, + { + "name": "verifyssl", + "description": "Verify SSL certificate", + "type": "boolean", + "multi": false, + "required": true, + "defaultValue": true + }, { "name": "workflow_id", "description": "The ID of the workflow to execute", diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.py b/functions/extensions/cortex-responders/Shuffle/shuffle.py index 0816ca53..bc643eb3 100644 --- a/functions/extensions/cortex-responders/Shuffle/shuffle.py +++ b/functions/extensions/cortex-responders/Shuffle/shuffle.py @@ -1,28 +1,26 @@ - -#!/usr/bin/env python -# encoding: utf-8 +#!/usr/bin/env python3 +#encoding: utf-8 from cortexutils.responder import Responder import requests class Shuffle(Responder): - def __init__(self): - Responder.__init__(self) - self.api_key = self.get_param("config.api_key", "") - self.url = self.get_param("config.url", "") - self.workflow_id = self.get_param("config.workflow_id", "") + def __init__(self): + Responder.__init__(self) + self.api_key = self.get_param("config.api_key", "") + self.url = self.get_param("config.url", "") + self.workflow_id = self.get_param("config.workflow_id", "") + self.verify = self.get_param('config.verifyssl', True, None) - def run(self): - Responder.run(self) + def run(self): + Responder.run(self) + parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id) + headers = { + "Authorization": "Bearer %s" % self.api_key + } + requests.post(parsed_url, headers=headers,verify=self.verify) - parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id) - headers = { - "Authorization": "Bearer %s" % self.api_key - } - requests.post(parsed_url, headers=headers) - - self.report({'message': 'message sent'}) + self.report({'message': 'message sent'}) if __name__ == '__main__': Shuffle().run() - From e83017121ad305fb0f17960bd5444f453cf43045 Mon Sep 17 00:00:00 2001 From: azgaviperr Date: Tue, 4 May 2021 23:50:01 +0200 Subject: [PATCH 3/4] Quick POC This is a POC of usage of shuffle webhook as entry for a responder for thehive. I am going to add extra layer to it later but this may help others to create some quick responders. I see for example an use in things like EDR not yet supported by cortex but included in Shuffle. --- .../Shuffle_Webhook/requirements.txt | 2 ++ .../Shuffle_Webhook/shuffle_webhook.json | 35 +++++++++++++++++++ .../Shuffle_Webhook/shuffle_webhook.py | 29 +++++++++++++++ 3 files changed, 66 insertions(+) create mode 100644 functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt create mode 100644 functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json create mode 100644 functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt b/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt new file mode 100644 index 00000000..6aabc3cf --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt @@ -0,0 +1,2 @@ +cortexutils +requests diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json new file mode 100644 index 00000000..b77f1026 --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json @@ -0,0 +1,35 @@ +{ + "name": "Shuffle_webhook", + "version": "1.0", + "author": "@azgaviperr", + "url": "https://github.com/frikky/shuffle", + "license": "AGPL-V3", + "description": "Execute a webhook in Shuffle", + "dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"], + "command": "Shuffle_Webhook/shuffle_webhook.py", + "baseConfig": "Shuffle_Webhook", + "configurationItems": [ + { + "name": "webhook_url", + "description": "The URL to your shuffle instance", + "type": "string", + "multi": false, + "required": true + }, + { + "name": "api_key", + "description": "The API key to your Shuffle user", + "type": "string", + "multi": false, + "required": true + }, + { + "name": "verifyssl", + "description": "Verify SSL certificate", + "type": "boolean", + "multi": false, + "required": true, + "defaultValue": true + } + ] +} diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py new file mode 100644 index 00000000..f7f254ff --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +#encoding: utf-8 + +from cortexutils.responder import Responder +import requests + +class Shuffle(Responder): + def __init__(self): + Responder.__init__(self) + self.api_key = self.get_param("config.api_key", "") + self.webhook_url = self.get_param("config.webhook_url", "") + self.webhook_id = self.get_param("config.webhook_id", "") + self.verify = self.get_param('config.verifyssl', True, None) + self.data = self.get_param('data') + + def run(self): + Responder.run(self) + headers = { + "Authorization": "Bearer %s" % self.api_key, + "Content-Type": "application/json", + "Accept": "application/json", + "User-Agent": "Cortex-Analyzer" + } + requests.post(self.webhook_url, headers=headers,verify=self.verify, json=self.data) + + self.report({'message': 'message sent'}) + +if __name__ == '__main__': + Shuffle().run() From 2ed910a2aca6c5e29de5f7c24c32a153770862e9 Mon Sep 17 00:00:00 2001 From: azgaviperr Date: Fri, 7 May 2021 08:22:25 +0200 Subject: [PATCH 4/4] Removed api code for webhook Taking off API configuration on the webhook oriented responder --- functions/extensions/cortex-responders/Shuffle/shuffle.py | 3 ++- .../cortex-responders/Shuffle_Webhook/shuffle_webhook.json | 7 ------- .../cortex-responders/Shuffle_Webhook/shuffle_webhook.py | 1 - 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.py b/functions/extensions/cortex-responders/Shuffle/shuffle.py index bc643eb3..db343230 100644 --- a/functions/extensions/cortex-responders/Shuffle/shuffle.py +++ b/functions/extensions/cortex-responders/Shuffle/shuffle.py @@ -16,7 +16,8 @@ class Shuffle(Responder): Responder.run(self) parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id) headers = { - "Authorization": "Bearer %s" % self.api_key + "Authorization": "Bearer %s" % self.api_key, + "User-Agent": "Cortex-Analyzer" } requests.post(parsed_url, headers=headers,verify=self.verify) diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json index b77f1026..a6d3e9d8 100644 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json +++ b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json @@ -17,13 +17,6 @@ "required": true }, { - "name": "api_key", - "description": "The API key to your Shuffle user", - "type": "string", - "multi": false, - "required": true - }, - { "name": "verifyssl", "description": "Verify SSL certificate", "type": "boolean", diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py index f7f254ff..daed501c 100644 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py +++ b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py @@ -16,7 +16,6 @@ class Shuffle(Responder): def run(self): Responder.run(self) headers = { - "Authorization": "Bearer %s" % self.api_key, "Content-Type": "application/json", "Accept": "application/json", "User-Agent": "Cortex-Analyzer"