Removed more old files
This commit is contained in:
Frikky
@@ -1,2 +0,0 @@
|
||||
# AWS Lambda forwarder to Shuffle
|
||||
This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated.
|
||||
@@ -1,22 +0,0 @@
|
||||
import json
|
||||
import urllib.parse
|
||||
import urllib3
|
||||
import os
|
||||
|
||||
print('Loading function')
|
||||
|
||||
def lambda_handler(event, context):
|
||||
# Get the object from the event and show its content type
|
||||
bucket = event['Records'][0]['s3']['bucket']['name']
|
||||
|
||||
webhook = os.environ.get("SHUFFLE_WEBHOOK")
|
||||
if not webhook:
|
||||
return "No webhook environment defined: SHUFFLE_WEBHOOK"
|
||||
|
||||
http = urllib3.PoolManager()
|
||||
ret = http.request('POST', webhook, body=json.dumps(event["Records"][0]).encode("utf-8"))
|
||||
if ret.status != 200:
|
||||
return "Bad status code for webhook: %d" % ret.status
|
||||
|
||||
print("Status code: %d\nData: %s" % (ret.status, ret.data))
|
||||
return "Successfully started with data %s" % ret.data
|
||||
@@ -1,2 +0,0 @@
|
||||
cortexutils
|
||||
requests
|
||||
@@ -1,43 +0,0 @@
|
||||
{
|
||||
"name": "Shuffle",
|
||||
"version": "1.0",
|
||||
"author": "@frikkylikeme",
|
||||
"url": "https://github.com/frikky/shuffle",
|
||||
"license": "AGPL-V3",
|
||||
"description": "Execute a workflow in Shuffle",
|
||||
"dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"],
|
||||
"command": "Shuffle/shuffle.py",
|
||||
"baseConfig": "Shuffle",
|
||||
"configurationItems": [
|
||||
{
|
||||
"name": "url",
|
||||
"description": "The URL to your shuffle instance",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true,
|
||||
"defaultValue": "https://shuffler.io"
|
||||
},
|
||||
{
|
||||
"name": "api_key",
|
||||
"description": "The API key to your Shuffle user",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"name": "verifyssl",
|
||||
"description": "Verify SSL certificate",
|
||||
"type": "boolean",
|
||||
"multi": false,
|
||||
"required": true,
|
||||
"defaultValue": true
|
||||
},
|
||||
{
|
||||
"name": "workflow_id",
|
||||
"description": "The ID of the workflow to execute",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#encoding: utf-8
|
||||
|
||||
from cortexutils.responder import Responder
|
||||
import requests
|
||||
|
||||
class Shuffle(Responder):
|
||||
def __init__(self):
|
||||
Responder.__init__(self)
|
||||
self.api_key = self.get_param("config.api_key", "")
|
||||
self.url = self.get_param("config.url", "")
|
||||
self.workflow_id = self.get_param("config.workflow_id", "")
|
||||
self.verify = self.get_param('config.verifyssl', True, None)
|
||||
|
||||
def run(self):
|
||||
Responder.run(self)
|
||||
parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id)
|
||||
headers = {
|
||||
"Authorization": "Bearer %s" % self.api_key,
|
||||
"User-Agent": "Cortex-Analyzer"
|
||||
}
|
||||
requests.post(parsed_url, headers=headers,verify=self.verify)
|
||||
|
||||
self.report({'message': 'message sent'})
|
||||
|
||||
if __name__ == '__main__':
|
||||
Shuffle().run()
|
||||
@@ -1,2 +0,0 @@
|
||||
cortexutils
|
||||
requests
|
||||
@@ -1,28 +0,0 @@
|
||||
{
|
||||
"name": "Shuffle_webhook",
|
||||
"version": "1.0",
|
||||
"author": "@azgaviperr",
|
||||
"url": "https://github.com/frikky/shuffle",
|
||||
"license": "AGPL-V3",
|
||||
"description": "Execute a webhook in Shuffle",
|
||||
"dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"],
|
||||
"command": "Shuffle_Webhook/shuffle_webhook.py",
|
||||
"baseConfig": "Shuffle_Webhook",
|
||||
"configurationItems": [
|
||||
{
|
||||
"name": "webhook_url",
|
||||
"description": "The URL to your shuffle instance",
|
||||
"type": "string",
|
||||
"multi": false,
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"name": "verifyssl",
|
||||
"description": "Verify SSL certificate",
|
||||
"type": "boolean",
|
||||
"multi": false,
|
||||
"required": true,
|
||||
"defaultValue": true
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
#encoding: utf-8
|
||||
|
||||
from cortexutils.responder import Responder
|
||||
import requests
|
||||
|
||||
class Shuffle(Responder):
|
||||
def __init__(self):
|
||||
Responder.__init__(self)
|
||||
self.api_key = self.get_param("config.api_key", "")
|
||||
self.webhook_url = self.get_param("config.webhook_url", "")
|
||||
self.webhook_id = self.get_param("config.webhook_id", "")
|
||||
self.verify = self.get_param('config.verifyssl', True, None)
|
||||
self.data = self.get_param('data')
|
||||
|
||||
def run(self):
|
||||
Responder.run(self)
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
"User-Agent": "Cortex-Analyzer"
|
||||
}
|
||||
requests.post(self.webhook_url, headers=headers,verify=self.verify, json=self.data)
|
||||
|
||||
self.report({'message': 'message sent'})
|
||||
|
||||
if __name__ == '__main__':
|
||||
Shuffle().run()
|
||||
@@ -1,19 +0,0 @@
|
||||
#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ELASTICSEARCH_USERNAME=frikky -e ELASTICSEARCH_PASSWORD=likeme -e xpack.security.enabled=true docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1
|
||||
#
|
||||
#
|
||||
#
|
||||
#echo "\nWaiting for 1.5 minute, then adding data"
|
||||
#sleep 90
|
||||
#echo "\nSlept 90 seconds: ADDING DATA"
|
||||
#curl -XPOST http://localhost:9200/_security/user/frikky -H "Content-Type: application/json" -d '{"enabled": true, "email": "frikky@shuffler.io"}'
|
||||
|
||||
#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D", "hits": 0}'
|
||||
#echo
|
||||
#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A", "hits": 0}'
|
||||
#
|
||||
#echo
|
||||
#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D"}'
|
||||
#echo
|
||||
#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A"}'
|
||||
#echo
|
||||
@@ -1,23 +0,0 @@
|
||||
# Patterns to ignore when building packages.
|
||||
# This supports shell glob matching, relative path matching, and
|
||||
# negation (prefixed with !). Only one pattern per line.
|
||||
.DS_Store
|
||||
# Common VCS dirs
|
||||
.git/
|
||||
.gitignore
|
||||
.bzr/
|
||||
.bzrignore
|
||||
.hg/
|
||||
.hgignore
|
||||
.svn/
|
||||
# Common backup files
|
||||
*.swp
|
||||
*.bak
|
||||
*.tmp
|
||||
*.orig
|
||||
*~
|
||||
# Various IDEs
|
||||
.project
|
||||
.idea/
|
||||
*.tmproj
|
||||
.vscode/
|
||||
@@ -1,24 +0,0 @@
|
||||
apiVersion: v2
|
||||
name: shuffle
|
||||
description: A Helm chart for Kubernetes
|
||||
|
||||
# A chart can be either an 'application' or a 'library' chart.
|
||||
#
|
||||
# Application charts are a collection of templates that can be packaged into versioned archives
|
||||
# to be deployed.
|
||||
#
|
||||
# Library charts provide useful utilities or functions for the chart developer. They're included as
|
||||
# a dependency of application charts to inject those utilities and functions into the rendering
|
||||
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
|
||||
type: application
|
||||
|
||||
# This is the chart version. This version number should be incremented each time you make changes
|
||||
# to the chart and its templates, including the app version.
|
||||
# Versions are expected to follow Semantic Versioning (https://semver.org/)
|
||||
version: 0.2.0
|
||||
|
||||
# This is the version number of the application being deployed. This version number should be
|
||||
# incremented each time you make changes to the application. Versions are not expected to
|
||||
# follow Semantic Versioning. They should reflect the version the application is using.
|
||||
# It is recommended to use it with quotes.
|
||||
appVersion: "2.0.0"
|
||||
@@ -1,22 +0,0 @@
|
||||
1. Get the application URL by running these commands:
|
||||
{{- if .Values.ingress.enabled }}
|
||||
{{- range $host := .Values.ingress.hosts }}
|
||||
{{- range .paths }}
|
||||
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- else if contains "NodePort" .Values.service.type }}
|
||||
export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "shuffle.fullname" . }})
|
||||
export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
|
||||
echo http://$NODE_IP:$NODE_PORT
|
||||
{{- else if contains "LoadBalancer" .Values.service.type }}
|
||||
NOTE: It may take a few minutes for the LoadBalancer IP to be available.
|
||||
You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "shuffle.fullname" . }}'
|
||||
export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "shuffle.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
|
||||
echo http://$SERVICE_IP:{{ .Values.service.port }}
|
||||
{{- else if contains "ClusterIP" .Values.service.type }}
|
||||
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "shuffle.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
|
||||
export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
|
||||
echo "Visit http://127.0.0.1:8080 to use your application"
|
||||
kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
|
||||
{{- end }}
|
||||
@@ -1,62 +0,0 @@
|
||||
{{/*
|
||||
Expand the name of the chart.
|
||||
*/}}
|
||||
{{- define "shuffle.name" -}}
|
||||
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create a default fully qualified app name.
|
||||
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
|
||||
If release name contains chart name it will be used as a full name.
|
||||
*/}}
|
||||
{{- define "shuffle.fullname" -}}
|
||||
{{- if .Values.fullnameOverride }}
|
||||
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- $name := default .Chart.Name .Values.nameOverride }}
|
||||
{{- if contains $name .Release.Name }}
|
||||
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
|
||||
{{- else }}
|
||||
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create chart name and version as used by the chart label.
|
||||
*/}}
|
||||
{{- define "shuffle.chart" -}}
|
||||
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Common labels
|
||||
*/}}
|
||||
{{- define "shuffle.labels" -}}
|
||||
helm.sh/chart: {{ include "shuffle.chart" . }}
|
||||
{{ include "shuffle.selectorLabels" . }}
|
||||
{{- if .Chart.AppVersion }}
|
||||
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
|
||||
{{- end }}
|
||||
app.kubernetes.io/managed-by: {{ .Release.Service }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Selector labels
|
||||
*/}}
|
||||
{{- define "shuffle.selectorLabels" -}}
|
||||
app.kubernetes.io/name: {{ include "shuffle.name" . }}
|
||||
app.kubernetes.io/instance: {{ .Release.Name }}
|
||||
{{- end }}
|
||||
|
||||
{{/*
|
||||
Create the name of the service account to use
|
||||
*/}}
|
||||
{{- define "shuffle.serviceAccountName" -}}
|
||||
{{- if .Values.serviceAccount.create }}
|
||||
{{- default (include "shuffle.fullname" .) .Values.serviceAccount.name }}
|
||||
{{- else }}
|
||||
{{- default "default" .Values.serviceAccount.name }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,253 +0,0 @@
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ .Values.name }}frontend
|
||||
namespace: {{ .Values.namespace | quote }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
service: shuffle
|
||||
app: shuffle-frontend
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
service: shuffle
|
||||
app: shuffle-frontend
|
||||
spec:
|
||||
containers:
|
||||
- name: shuffle-frontend
|
||||
image: ghcr.io/frikky/shuffle-frontend:nightly
|
||||
imagePullPolicy: {{ .Values.image.pullPolicy | quote }}
|
||||
env:
|
||||
- name: BACKEND_HOSTNAME
|
||||
value: backend-service
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
ports:
|
||||
- name: http
|
||||
containerPort: 80
|
||||
hostPort: 3001
|
||||
- name: https
|
||||
containerPort: 443
|
||||
hostname: shuffle-frontend
|
||||
restartPolicy: Always
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ .Values.name }}backend
|
||||
namespace: {{ .Values.namespace | quote }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
service: shuffle
|
||||
app: shuffle-backend
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
service: shuffle
|
||||
app: shuffle-backend
|
||||
spec:
|
||||
containers:
|
||||
- name: shuffle-backend
|
||||
image: ghcr.io/frikky/shuffle-backend:nightly
|
||||
env:
|
||||
- name: BACKEND_HOSTNAME
|
||||
value: "backend-service"
|
||||
- name: BACKEND_PORT
|
||||
value: "5001"
|
||||
- name: ENVIRONMENT_NAME
|
||||
value: "Shuffle"
|
||||
- name: HTTPS_PROXY
|
||||
- name: HTTP_PROXY
|
||||
- name: ORG_ID
|
||||
value: "Shuffle"
|
||||
- name: OUTER_HOSTNAME
|
||||
value: "backend-service"
|
||||
- name: SHUFFLE_APP_FORCE_UPDATE
|
||||
value: "false"
|
||||
- name: SHUFFLE_APP_HOTLOAD_FOLDER
|
||||
value: "/shuffle-apps"
|
||||
- name: SHUFFLE_APP_HOTLOAD_LOCATION
|
||||
value: "/shuffle-apps"
|
||||
- name: SHUFFLE_CONTAINER_AUTO_CLEANUP
|
||||
value: "false"
|
||||
- name: SHUFFLE_DEFAULT_APIKEY
|
||||
- name: SHUFFLE_DEFAULT_PASSWORD
|
||||
- name: SHUFFLE_DEFAULT_USERNAME
|
||||
- name: SHUFFLE_DOWNLOAD_AUTH_BRANCH
|
||||
- name: SHUFFLE_DOWNLOAD_AUTH_PASSWORD
|
||||
- name: SHUFFLE_DOWNLOAD_AUTH_USERNAME
|
||||
- name: SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH
|
||||
- name: SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION
|
||||
- name: SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD
|
||||
- name: SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME
|
||||
- name: SHUFFLE_ELASTIC
|
||||
value: "true"
|
||||
- name: SHUFFLE_OPENSEARCH_APIKEY
|
||||
- name: SHUFFLE_OPENSEARCH_CERTIFICATE_FILE
|
||||
- name: SHUFFLE_OPENSEARCH_CLOUDID
|
||||
- name: SHUFFLE_OPENSEARCH_PASSWORD
|
||||
- name: SHUFFLE_OPENSEARCH_PROXY
|
||||
- name: SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY
|
||||
value: "true"
|
||||
- name: SHUFFLE_OPENSEARCH_URL
|
||||
value: http://opensearch-service:9200
|
||||
- name: SHUFFLE_OPENSEARCH_USERNAME
|
||||
value: ""
|
||||
- name: SHUFFLE_PASS_APP_PROXY
|
||||
value: "FALSE"
|
||||
- name: SHUFFLE_PASS_WORKER_PROXY
|
||||
value: "FALSE"
|
||||
volumeMounts:
|
||||
- mountPath: /var/run/docker.sock
|
||||
name: docker-sock
|
||||
- mountPath: /shuffle-apps
|
||||
name: shuffle-app-hotload-location
|
||||
- mountPath: /shuffle-files
|
||||
name: shuffle-file-location
|
||||
hostname: shuffle-backend
|
||||
volumes:
|
||||
- name: docker-sock
|
||||
hostPath:
|
||||
path: /var/run/docker.sock
|
||||
- name: shuffle-app-hotload-location
|
||||
hostPath:
|
||||
path: /data/kubernetes/shuffle-apps
|
||||
type: DirectoryOrCreate
|
||||
- name: shuffle-file-location
|
||||
hostPath:
|
||||
path: /data/kubernetes/shuffle-files
|
||||
type: DirectoryOrCreate
|
||||
|
||||
---
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: {{ .Values.name }}orborus
|
||||
namespace: {{ .Values.namespace | quote }}
|
||||
spec:
|
||||
replicas: 1
|
||||
selector:
|
||||
matchLabels:
|
||||
service: shuffle
|
||||
app: shuffle-orborus
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
service: shuffle
|
||||
app: shuffle-orborus
|
||||
spec:
|
||||
containers:
|
||||
- name: shuffle-orborus
|
||||
image: ghcr.io/frikky/shuffle-orborus:nightly
|
||||
env:
|
||||
- name: RUNNING_MODE
|
||||
value: kubernetes
|
||||
- name: BASE_URL
|
||||
value: http://backend-service:5001
|
||||
- name: CLEANUP
|
||||
value: "false"
|
||||
- name: DOCKER_API_VERSION
|
||||
value: "1.40"
|
||||
- name: ENVIRONMENT_NAME
|
||||
value: Shuffle
|
||||
- name: HTTPS_PROXY
|
||||
- name: HTTP_PROXY
|
||||
- name: ORG_ID
|
||||
value: Shuffle
|
||||
- name: SHUFFLE_APP_SDK_VERSION
|
||||
value: 0.8.97
|
||||
- name: SHUFFLE_BASE_IMAGE_NAME
|
||||
value: frikky
|
||||
- name: SHUFFLE_BASE_IMAGE_REGISTRY
|
||||
value: ghcr.io
|
||||
- name: SHUFFLE_BASE_IMAGE_TAG_SUFFIX
|
||||
value: "-0.8.80"
|
||||
- name: SHUFFLE_ORBORUS_EXECUTION_TIMEOUT
|
||||
value: "600"
|
||||
- name: SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY
|
||||
value: "50"
|
||||
- name: SHUFFLE_PASS_WORKER_PROXY
|
||||
value: "TRUE"
|
||||
- name: SHUFFLE_WORKER_VERSION
|
||||
value: nightly
|
||||
- name: TZ
|
||||
value: Asia/Shanghai
|
||||
volumeMounts:
|
||||
- mountPath: /var/run/docker.sock
|
||||
name: docker-sock
|
||||
hostname: shuffle-orborus
|
||||
volumes:
|
||||
- name: docker-sock
|
||||
hostPath:
|
||||
path: /var/run/docker.sock
|
||||
#---
|
||||
#apiVersion: apps/v1
|
||||
#kind: Deployment
|
||||
#metadata:
|
||||
# name: {{ .Values.name }}opensearch
|
||||
# namespace: {{ .Values.namespace | quote }}
|
||||
#spec:
|
||||
# replicas: 1
|
||||
# selector:
|
||||
# matchLabels:
|
||||
# service: shuffle
|
||||
# app: shuffle-opensearch
|
||||
# template:
|
||||
# metadata:
|
||||
# labels:
|
||||
# service: shuffle
|
||||
# app: shuffle-opensearch
|
||||
# spec:
|
||||
# nodeSelector:
|
||||
# node.bdlab-venus.com/opensearch: available
|
||||
# initContainers:
|
||||
# - name: permissions-fix
|
||||
# image: frikky/busybox
|
||||
# #volumeMounts:
|
||||
# # - name: opensearch-claim0
|
||||
# # mountPath: /usr/share/elasticsearch/data
|
||||
# command: [ 'chown' ]
|
||||
# args: [ '1000:1000', '/usr/share/elasticsearch/data' ]
|
||||
# containers:
|
||||
# - name: shuffle-opensearch
|
||||
# image: opensearchproject/opensearch:1.0.1
|
||||
# env:
|
||||
# - name: TZ
|
||||
# value: Asia/Shanghai
|
||||
# - name: bootstrap.memory_lock
|
||||
# value: "false"
|
||||
# - name: OPENSEARCH_JAVA_OPTS
|
||||
# value: "-Xms1024m -Xmx1024m"
|
||||
# - name: opendistro_security.disabled
|
||||
# value: "true"
|
||||
# - name: cluster.routing.allocation.disk.threshold_enabled
|
||||
# value: "false"
|
||||
# - name: cluster.name
|
||||
# value: shuffle-cluster
|
||||
# - name: node.name
|
||||
# value: opensearch-service
|
||||
# - name: discovery.seed_hosts
|
||||
# value: opensearch-service
|
||||
# - name: cluster.initial_master_nodes
|
||||
# value: opensearch-service
|
||||
# volumeMounts:
|
||||
# - mountPath: /usr/share/opensearch/data
|
||||
# name: opensearch-claim0
|
||||
#volumes:
|
||||
# - name: opensearch-claim0
|
||||
# persistentVolumeClaim:
|
||||
# claimName: opensearch-claim0
|
||||
# volumeMounts:
|
||||
# - mountPath: /usr/share/opensearch/data
|
||||
# readOnly: true
|
||||
# name: db-location
|
||||
# volumes:
|
||||
# - name: db-location
|
||||
# hostPath:
|
||||
# path: /data/kubernetes/shuffle-opensearch
|
||||
# type: DirectoryOrCreate
|
||||
@@ -1,28 +0,0 @@
|
||||
{{- if .Values.autoscaling.enabled }}
|
||||
apiVersion: autoscaling/v2beta1
|
||||
kind: HorizontalPodAutoscaler
|
||||
metadata:
|
||||
name: {{ include "shuffle.fullname" . }}
|
||||
labels:
|
||||
{{- include "shuffle.labels" . | nindent 4 }}
|
||||
spec:
|
||||
scaleTargetRef:
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
name: {{ include "shuffle.fullname" . }}
|
||||
minReplicas: {{ .Values.autoscaling.minReplicas }}
|
||||
maxReplicas: {{ .Values.autoscaling.maxReplicas }}
|
||||
metrics:
|
||||
{{- if .Values.autoscaling.targetCPUUtilizationPercentage }}
|
||||
- type: Resource
|
||||
resource:
|
||||
name: cpu
|
||||
targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }}
|
||||
{{- end }}
|
||||
{{- if .Values.autoscaling.targetMemoryUtilizationPercentage }}
|
||||
- type: Resource
|
||||
resource:
|
||||
name: memory
|
||||
targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,61 +0,0 @@
|
||||
{{- if .Values.ingress.enabled -}}
|
||||
{{- $fullName := include "shuffle.fullname" . -}}
|
||||
{{- $svcPort := .Values.service.port -}}
|
||||
{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }}
|
||||
{{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }}
|
||||
{{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}}
|
||||
apiVersion: networking.k8s.io/v1
|
||||
{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}}
|
||||
apiVersion: networking.k8s.io/v1beta1
|
||||
{{- else -}}
|
||||
apiVersion: extensions/v1beta1
|
||||
{{- end }}
|
||||
kind: Ingress
|
||||
metadata:
|
||||
name: {{ $fullName }}
|
||||
labels:
|
||||
{{- include "shuffle.labels" . | nindent 4 }}
|
||||
{{- with .Values.ingress.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
spec:
|
||||
{{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }}
|
||||
ingressClassName: {{ .Values.ingress.className }}
|
||||
{{- end }}
|
||||
{{- if .Values.ingress.tls }}
|
||||
tls:
|
||||
{{- range .Values.ingress.tls }}
|
||||
- hosts:
|
||||
{{- range .hosts }}
|
||||
- {{ . | quote }}
|
||||
{{- end }}
|
||||
secretName: {{ .secretName }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
rules:
|
||||
{{- range .Values.ingress.hosts }}
|
||||
- host: {{ .host | quote }}
|
||||
http:
|
||||
paths:
|
||||
{{- range .paths }}
|
||||
- path: {{ .path }}
|
||||
{{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }}
|
||||
pathType: {{ .pathType }}
|
||||
{{- end }}
|
||||
backend:
|
||||
{{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
|
||||
service:
|
||||
name: {{ $fullName }}
|
||||
port:
|
||||
number: {{ $svcPort }}
|
||||
{{- else }}
|
||||
serviceName: {{ $fullName }}
|
||||
servicePort: {{ $svcPort }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,24 +0,0 @@
|
||||
#apiVersion: v1
|
||||
#kind: PersistentVolume
|
||||
#metadata:
|
||||
# name: opensearch-claim0
|
||||
# labels:
|
||||
# app: opensearch-claim0
|
||||
#spec:
|
||||
# capacity:
|
||||
# storage: "10G"
|
||||
# volumeMode: Filesystem
|
||||
# persistentVolumeReclaimPolicy: Retain
|
||||
# storageClassName: local-storage
|
||||
# accessModes:
|
||||
# - "ReadWriteOnce"
|
||||
# local:
|
||||
# path: "/data/kubernetes/shuffle-opensearch"
|
||||
# nodeAffinity:
|
||||
# required:
|
||||
# nodeSelectorTerms:
|
||||
# - matchExpressions:
|
||||
# - key: node.dollar.com/opensearch
|
||||
# operator: In
|
||||
# values:
|
||||
# - available
|
||||
@@ -1,17 +0,0 @@
|
||||
#apiVersion: v1
|
||||
#kind: PersistentVolumeClaim
|
||||
#metadata:
|
||||
# name: opensearch-claim0
|
||||
# namespace: {{ .Values.namespace }}
|
||||
# labels:
|
||||
# app: opensearch-claim0
|
||||
#spec:
|
||||
# selector:
|
||||
# matchLabels:
|
||||
# app: opensearch-claim0
|
||||
# accessModes:
|
||||
# - ReadWriteOnce
|
||||
# storageClassName: local-storage
|
||||
# resources:
|
||||
# requests:
|
||||
# storage: 5Gi
|
||||
@@ -1,52 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: backend-service
|
||||
namespace: {{ .Values.namespace | quote }}
|
||||
spec:
|
||||
ports:
|
||||
- name: "5001"
|
||||
port: 5001
|
||||
targetPort: 5001
|
||||
selector:
|
||||
app: shuffle-backend
|
||||
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: frontend-service
|
||||
namespace: {{ .Values.namespace | quote }}
|
||||
spec:
|
||||
type: NodePort
|
||||
externalTrafficPolicy: Local
|
||||
ports:
|
||||
- name: "3001"
|
||||
port: 3001
|
||||
nodePort: 3001
|
||||
targetPort: 80
|
||||
- name: "3443"
|
||||
port: 3443
|
||||
nodePort: 3443
|
||||
targetPort: 443
|
||||
protocol: TCP
|
||||
selector:
|
||||
app: shuffle-frontend
|
||||
|
||||
#---
|
||||
#apiVersion: v1
|
||||
#kind: Service
|
||||
#metadata:
|
||||
# name: opensearch-service
|
||||
# namespace: {{ .Values.namespace | quote }}
|
||||
#spec:
|
||||
# type: NodePort
|
||||
# externalTrafficPolicy: Local
|
||||
# ports:
|
||||
# - name: "9200"
|
||||
# port: 9200
|
||||
# targetPort: 9200
|
||||
# nodePort: 9200
|
||||
# protocol: TCP
|
||||
# selector:
|
||||
# app: shuffle-opensearch
|
||||
@@ -1,12 +0,0 @@
|
||||
{{- if .Values.serviceAccount.create -}}
|
||||
apiVersion: v1
|
||||
kind: ServiceAccount
|
||||
metadata:
|
||||
name: {{ include "shuffle.serviceAccountName" . }}
|
||||
labels:
|
||||
{{- include "shuffle.labels" . | nindent 4 }}
|
||||
{{- with .Values.serviceAccount.annotations }}
|
||||
annotations:
|
||||
{{- toYaml . | nindent 4 }}
|
||||
{{- end }}
|
||||
{{- end }}
|
||||
@@ -1,15 +0,0 @@
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: "{{ include "shuffle.fullname" . }}-test-connection"
|
||||
labels:
|
||||
{{- include "shuffle.labels" . | nindent 4 }}
|
||||
annotations:
|
||||
"helm.sh/hook": test
|
||||
spec:
|
||||
containers:
|
||||
- name: wget
|
||||
image: busybox
|
||||
command: ['wget']
|
||||
args: ['{{ include "shuffle.fullname" . }}:{{ .Values.service.port }}']
|
||||
restartPolicy: Never
|
||||
@@ -1,82 +0,0 @@
|
||||
# Default values for shuffle.
|
||||
# This is a YAML-formatted file.
|
||||
# Declare variables to be passed into your templates.
|
||||
|
||||
replicaCount: 1
|
||||
|
||||
image:
|
||||
repository: nginx
|
||||
pullPolicy: IfNotPresent
|
||||
# Overrides the image tag whose default is the chart appVersion.
|
||||
tag: ""
|
||||
|
||||
imagePullSecrets: []
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
serviceAccount:
|
||||
# Specifies whether a service account should be created
|
||||
create: true
|
||||
# Annotations to add to the service account
|
||||
annotations: {}
|
||||
# The name of the service account to use.
|
||||
# If not set and create is true, a name is generated using the fullname template
|
||||
name: ""
|
||||
|
||||
podAnnotations: {}
|
||||
|
||||
podSecurityContext: {}
|
||||
# fsGroup: 2000
|
||||
|
||||
securityContext: {}
|
||||
# capabilities:
|
||||
# drop:
|
||||
# - ALL
|
||||
# readOnlyRootFilesystem: true
|
||||
# runAsNonRoot: true
|
||||
# runAsUser: 1000
|
||||
|
||||
service:
|
||||
type: ClusterIP
|
||||
port: 80
|
||||
|
||||
ingress:
|
||||
enabled: false
|
||||
className: ""
|
||||
annotations: {}
|
||||
# kubernetes.io/ingress.class: nginx
|
||||
# kubernetes.io/tls-acme: "true"
|
||||
hosts:
|
||||
- host: chart-example.local
|
||||
paths:
|
||||
- path: /
|
||||
pathType: ImplementationSpecific
|
||||
tls: []
|
||||
# - secretName: chart-example-tls
|
||||
# hosts:
|
||||
# - chart-example.local
|
||||
|
||||
resources: {}
|
||||
# We usually recommend not to specify default resources and to leave this as a conscious
|
||||
# choice for the user. This also increases chances charts run on environments with little
|
||||
# resources, such as Minikube. If you do want to specify resources, uncomment the following
|
||||
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
|
||||
# limits:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
# requests:
|
||||
# cpu: 100m
|
||||
# memory: 128Mi
|
||||
|
||||
autoscaling:
|
||||
enabled: false
|
||||
minReplicas: 1
|
||||
maxReplicas: 100
|
||||
targetCPUUtilizationPercentage: 80
|
||||
# targetMemoryUtilizationPercentage: 80
|
||||
|
||||
nodeSelector: {}
|
||||
|
||||
tolerations: []
|
||||
|
||||
affinity: {}
|
||||
@@ -1,23 +0,0 @@
|
||||
FROM python:3.9.4-alpine as base
|
||||
|
||||
FROM base as builder
|
||||
|
||||
RUN mkdir /install
|
||||
WORKDIR /install
|
||||
|
||||
FROM base
|
||||
RUN apk add g++
|
||||
|
||||
COPY --from=builder /install /usr/local
|
||||
COPY requirements.txt /requirements.txt
|
||||
RUN pip3 install -r /requirements.txt
|
||||
|
||||
|
||||
RUN mkdir /app
|
||||
WORKDIR /app
|
||||
COPY requirements.txt /app/requirements.txt
|
||||
RUN python3 -m pip install -r /app/requirements.txt
|
||||
|
||||
COPY sub.py /app/sub.py
|
||||
|
||||
CMD ["python3", "sub.py"]
|
||||
@@ -1,9 +0,0 @@
|
||||
version: '3'
|
||||
services:
|
||||
zmq:
|
||||
image: ghcr.io/frikky/shuffle-zmq:latest
|
||||
environment:
|
||||
- ZMQ_HOSTNAME=localhost
|
||||
- ZMQ_PORT=50000
|
||||
- ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e
|
||||
restart: unless-stopped
|
||||
@@ -1,2 +0,0 @@
|
||||
pyzmq
|
||||
requests
|
||||
@@ -1,57 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
print("Running imports")
|
||||
import sys
|
||||
import zmq
|
||||
import json
|
||||
import time
|
||||
import pprint
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
|
||||
forward_url = os.getenv("ZMQ_FORWARD_URL", "")
|
||||
print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url)
|
||||
def handle_hook(data):
|
||||
ret = requests.post(forward_url, json=data)
|
||||
print(ret.text)
|
||||
print(ret.status_code)
|
||||
|
||||
def main():
|
||||
host = os.getenv("ZMQ_HOST", "localhost")
|
||||
port = os.getenv("ZMQ_PORT", "50000")
|
||||
|
||||
if len(forward_url) == 0:
|
||||
print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder")
|
||||
exit(0)
|
||||
|
||||
print("Starting connection setup to %s:%s" % (host, port))
|
||||
context = zmq.Context()
|
||||
socket = context.socket(zmq.SUB)
|
||||
socket.connect ("tcp://%s:%s" % (host, port))
|
||||
socket.setsockopt(zmq.SUBSCRIBE, b'')
|
||||
|
||||
poller = zmq.Poller()
|
||||
poller.register(socket, zmq.POLLIN)
|
||||
|
||||
print("Starting zmq check for %s:%s" % (host, port))
|
||||
while True:
|
||||
socks = dict(poller.poll(timeout=None))
|
||||
if socket in socks and socks[socket] == zmq.POLLIN:
|
||||
message = socket.recv()
|
||||
#print(message)
|
||||
topic, s, m = message.decode('utf-8').partition(" ")
|
||||
|
||||
d = json.loads(m)
|
||||
try:
|
||||
# print test if you want status (heartbeat)
|
||||
test = d["status"]
|
||||
except KeyError:
|
||||
handle_hook(d)
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print("In init ")
|
||||
main()
|
||||
@@ -1,11 +0,0 @@
|
||||
curl -XPUT -u admin:admin https://localhost:9200/_cluster/settings -H "Content-Type:application/json" -k -d \
|
||||
'{
|
||||
"transient": {
|
||||
"cluster.routing.allocation.disk.threshold_enabled": false
|
||||
}
|
||||
}'
|
||||
|
||||
curl -XPUT -u admin:admin https://localhost:9200/_all/_settings -H "Content-Type: application/json" -k -d \
|
||||
'{
|
||||
"index.blocks.read_only_allow_delete": null
|
||||
}'
|
||||
@@ -1 +0,0 @@
|
||||
shuffle-database/nodes
|
||||
@@ -1,131 +0,0 @@
|
||||
version: '3.4'
|
||||
services:
|
||||
backend:
|
||||
image: ghcr.io/frikky/shuffle-backend:nightly
|
||||
#hostname: shuffle-backend
|
||||
environment:
|
||||
BACKEND_HOSTNAME: backend
|
||||
OUTER_HOSTNAME: backend
|
||||
BACKEND_PORT: '5001'
|
||||
HTTPS_PROXY: ''
|
||||
HTTP_PROXY: ''
|
||||
SHUFFLE_APP_DOWNLOAD_LOCATION: https://github.com/frikky/shuffle-apps
|
||||
SHUFFLE_APP_FORCE_UPDATE: 'false'
|
||||
SHUFFLE_APP_HOTLOAD_FOLDER: /shuffle-apps
|
||||
SHUFFLE_APP_HOTLOAD_LOCATION: ./shuffle-apps
|
||||
DATASTORE_EMULATOR_HOST: "shuffle-database:8000"
|
||||
DOCKER_API_VERSION: '1.40'
|
||||
SHUFFLE_BASE_IMAGE_NAME: frikky
|
||||
SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io
|
||||
SHUFFLE_BASE_IMAGE_TAG_SUFFIX: '-0.9.30'
|
||||
SHUFFLE_CONTAINER_AUTO_CLEANUP: 'true'
|
||||
SHUFFLE_DEFAULT_APIKEY: ''
|
||||
SHUFFLE_FILE_LOCATION: /shuffle-files
|
||||
SHUFFLE_OPENSEARCH_APIKEY: ''
|
||||
SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: ''
|
||||
SHUFFLE_OPENSEARCH_CLOUDID: ''
|
||||
SHUFFLE_OPENSEARCH_PROXY: ''
|
||||
SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: 'true'
|
||||
SHUFFLE_OPENSEARCH_URL: http://opensearch:9200
|
||||
SHUFFLE_PASS_APP_PROXY: 'FALSE'
|
||||
SHUFFLE_PASS_WORKER_PROXY: 'TRUE'
|
||||
SHUFFLE_ELASTIC: 'true'
|
||||
#SHUFFLE_ENCRYPTION_MODIFIER:
|
||||
ports:
|
||||
- "5001:5001"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
- ./shuffle-apps:/shuffle-apps
|
||||
- ./shuffle-files:/shuffle-files
|
||||
networks:
|
||||
- shuffle_prod
|
||||
#- reverseproxy
|
||||
depends_on:
|
||||
- opensearch
|
||||
logging:
|
||||
driver: json-file
|
||||
frontend:
|
||||
image: ghcr.io/frikky/shuffle-frontend:nightly
|
||||
healthcheck:
|
||||
test: curl -fs http://localhost:80 || exit 1
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
ports:
|
||||
- "3001:80"
|
||||
- "3443:443"
|
||||
networks:
|
||||
- shuffle_prod
|
||||
#- reverseproxy
|
||||
environment:
|
||||
- "BACKEND_HOSTNAME=backend"
|
||||
depends_on:
|
||||
- backend
|
||||
deploy:
|
||||
update_config:
|
||||
order: start-first
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:1.1.0
|
||||
healthcheck:
|
||||
test: curl -fs http://localhost:9200/_cat/health || exit 1
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
environment:
|
||||
- bootstrap.memory_lock=false
|
||||
- "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM
|
||||
- plugins.security.disabled=true
|
||||
- cluster.routing.allocation.disk.threshold_enabled=false
|
||||
- cluster.name=shuffle-cluster
|
||||
- node.name=opensearch
|
||||
- discovery.seed_hosts=opensearch
|
||||
- cluster.initial_master_nodes=opensearch
|
||||
- node.store.allow_mmap=false
|
||||
volumes:
|
||||
- ./shuffle-database:/usr/share/opensearch/data:rw
|
||||
networks:
|
||||
- shuffle_prod
|
||||
#- reverseproxy
|
||||
logging:
|
||||
driver: json-file
|
||||
|
||||
orborus:
|
||||
image: ghcr.io/frikky/shuffle-orborus:nightly
|
||||
#hostname: shuffle-orborus
|
||||
environment:
|
||||
#SHUFFLE_WORKER_VERSION: nightly
|
||||
SHUFFLE_APP_SDK_VERSION: 0.8.97
|
||||
SHUFFLE_WORKER_VERSION: nightly
|
||||
BASE_URL: http://backend:5001
|
||||
#BASE_URL: http://192.168.86.37:5001
|
||||
CLEANUP: 'true'
|
||||
DOCKER_API_VERSION: '1.40'
|
||||
ENVIRONMENT_NAME: Shuffle
|
||||
HTTPS_PROXY: ''
|
||||
HTTP_PROXY: ''
|
||||
ORG_ID: Shuffle
|
||||
SHUFFLE_BASE_IMAGE_NAME: frikky
|
||||
SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io
|
||||
SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80
|
||||
SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50'
|
||||
SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800'
|
||||
SHUFFLE_PASS_APP_PROXY: 'FALSE'
|
||||
SHUFFLE_PASS_WORKER_PROXY: 'TRUE'
|
||||
SHUFFLE_SCALE_REPLICAS: 5
|
||||
SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod
|
||||
SHUFFLE_SWARM_CONFIG: "run"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
networks:
|
||||
- shuffle_prod
|
||||
#- reverseproxy
|
||||
logging:
|
||||
driver: json-file
|
||||
|
||||
networks:
|
||||
shuffle_prod:
|
||||
driver: overlay
|
||||
external: true
|
||||
#reverseproxy:
|
||||
# driver: overlay
|
||||
# #external: true
|
||||
@@ -1 +0,0 @@
|
||||
docker network create -d overlay shuffle_prod
|
||||
@@ -1,39 +0,0 @@
|
||||
version: '3.4'
|
||||
services:
|
||||
orborus:
|
||||
image: ghcr.io/frikky/shuffle-orborus:nightly
|
||||
#hostname: shuffle-orborus
|
||||
environment:
|
||||
#SHUFFLE_WORKER_VERSION: nightly
|
||||
SHUFFLE_APP_SDK_VERSION: 0.8.97
|
||||
SHUFFLE_WORKER_VERSION: nightly
|
||||
BASE_URL: http://<BACKEND>:5001
|
||||
#BASE_URL: http://192.168.86.37:5001
|
||||
CLEANUP: 'true'
|
||||
DOCKER_API_VERSION: '1.40'
|
||||
ENVIRONMENT_NAME: Shuffle
|
||||
HTTPS_PROXY: ''
|
||||
HTTP_PROXY: ''
|
||||
ORG_ID: Shuffle
|
||||
SHUFFLE_BASE_IMAGE_NAME: frikky
|
||||
SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io
|
||||
SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80
|
||||
SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50'
|
||||
SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800'
|
||||
SHUFFLE_PASS_APP_PROXY: 'FALSE'
|
||||
SHUFFLE_PASS_WORKER_PROXY: 'TRUE'
|
||||
SHUFFLE_SCALE_REPLICAS: 5
|
||||
SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod
|
||||
SHUFFLE_SWARM_CONFIG: "run"
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
networks:
|
||||
- shuffle_prod
|
||||
#- reverseproxy
|
||||
logging:
|
||||
driver: json-file
|
||||
|
||||
networks:
|
||||
shuffle_prod:
|
||||
driver: overlay
|
||||
external: true
|
||||
@@ -1,4 +0,0 @@
|
||||
docker swarm init
|
||||
chown 1000:1000 -R shuffle-database/
|
||||
docker network create -d overlay shuffle_prod
|
||||
docker stack deploy --compose-file=docker-compose.yml shuffle_swarm
|
||||
@@ -1,4 +0,0 @@
|
||||
docker swarm init
|
||||
chown 1000:1000 -R shuffle-database/
|
||||
docker network create -d overlay shuffle_prod
|
||||
docker stack deploy --compose-file=orborus.yml shuffle_orborus
|
||||
@@ -1,3 +0,0 @@
|
||||
docker stack rm shuffle_swarm
|
||||
|
||||
#
|
||||
@@ -1,36 +0,0 @@
|
||||
#!/bin/sh
|
||||
# Created by Shuffle, AS. <frikky@shuffler.io>.
|
||||
|
||||
WPYTHON_BIN="framework/python/bin/python3"
|
||||
|
||||
SCRIPT_PATH_NAME="$0"
|
||||
|
||||
DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)"
|
||||
SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})"
|
||||
|
||||
case ${DIR_NAME} in
|
||||
*/active-response/bin | */wodles*)
|
||||
if [ -z "${WAZUH_PATH}" ]; then
|
||||
WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)"
|
||||
fi
|
||||
|
||||
PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py"
|
||||
;;
|
||||
*/bin)
|
||||
if [ -z "${WAZUH_PATH}" ]; then
|
||||
WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)"
|
||||
fi
|
||||
|
||||
PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py"
|
||||
;;
|
||||
*/integrations)
|
||||
if [ -z "${WAZUH_PATH}" ]; then
|
||||
WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)"
|
||||
fi
|
||||
|
||||
PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py"
|
||||
;;
|
||||
esac
|
||||
|
||||
|
||||
${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@"
|
||||
@@ -1,206 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
# Created by Shuffle, AS. <frikky@shuffler.io>.
|
||||
# Based on the Slack integration using Webhooks
|
||||
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
try:
|
||||
import requests
|
||||
from requests.auth import HTTPBasicAuth
|
||||
except Exception as e:
|
||||
print("No module 'requests' found. Install: pip install requests")
|
||||
sys.exit(1)
|
||||
|
||||
# ADD THIS TO ossec.conf configuration:
|
||||
# <integration>
|
||||
# <name>custom-shuffle</name>
|
||||
# <hook_url>http://<IP>:3001/api/v1/hooks/<HOOK_ID></hook_url>
|
||||
# <level>3</level>
|
||||
# <alert_format>json</alert_format>
|
||||
# </integration>
|
||||
|
||||
# Global vars
|
||||
debug_enabled = False
|
||||
pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
|
||||
json_alert = {}
|
||||
now = time.strftime("%a %b %d %H:%M:%S %Z %Y")
|
||||
|
||||
# Set paths
|
||||
log_file = '{0}/logs/integrations.log'.format(pwd)
|
||||
|
||||
try:
|
||||
with open("/tmp/shuffle_start.txt", "w+") as tmp:
|
||||
tmp.write("Script started")
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
def main(args):
|
||||
debug("# Starting")
|
||||
|
||||
# Read args
|
||||
alert_file_location = args[1]
|
||||
webhook = args[3]
|
||||
|
||||
debug("# Webhook")
|
||||
debug(webhook)
|
||||
|
||||
debug("# File location")
|
||||
debug(alert_file_location)
|
||||
|
||||
# Load alert. Parse JSON object.
|
||||
try:
|
||||
with open(alert_file_location) as alert_file:
|
||||
json_alert = json.load(alert_file)
|
||||
except:
|
||||
debug("# Alert file %s doesn't exist" % alert_file_location)
|
||||
|
||||
debug("# Processing alert")
|
||||
try:
|
||||
debug(json_alert)
|
||||
except Exception as e:
|
||||
debug("Failed getting json_alert %s" % e)
|
||||
sys.exit(1)
|
||||
|
||||
debug("# Generating message")
|
||||
msg = generate_msg(json_alert)
|
||||
if isinstance(msg, str):
|
||||
if len(msg) == 0:
|
||||
return
|
||||
debug(msg)
|
||||
|
||||
debug("# Sending message")
|
||||
|
||||
try:
|
||||
with open("/tmp/shuffle_end.txt", "w+") as tmp:
|
||||
tmp.write("Script done pre-msg sending")
|
||||
except:
|
||||
pass
|
||||
|
||||
|
||||
send_msg(msg, webhook)
|
||||
|
||||
|
||||
def debug(msg):
|
||||
if debug_enabled:
|
||||
msg = "{0}: {1}\n".format(now, msg)
|
||||
print(msg)
|
||||
f = open(log_file, "a")
|
||||
f.write(msg)
|
||||
f.close()
|
||||
|
||||
# Skips container kills to stop self-recursion
|
||||
def filter_msg(alert):
|
||||
# These are things that recursively happen because Shuffle starts Docker containers
|
||||
skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928", "5710"]
|
||||
if alert["rule"]["id"] in skip:
|
||||
return False
|
||||
|
||||
#try:
|
||||
# if "docker" in alert["rule"]["description"].lower() and "
|
||||
#msg['text'] = alert.get('full_log')
|
||||
#except:
|
||||
# pass
|
||||
#msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A"
|
||||
|
||||
return True
|
||||
|
||||
def generate_msg(alert):
|
||||
if not filter_msg(alert):
|
||||
print("Skipping rule %s" % alert["rule"]["id"])
|
||||
return ""
|
||||
|
||||
level = alert['rule']['level']
|
||||
|
||||
if (level <= 4):
|
||||
severity = 1
|
||||
elif (level >= 5 and level <= 7):
|
||||
severity = 2
|
||||
else:
|
||||
severity = 3
|
||||
|
||||
msg = {}
|
||||
msg['severity'] = severity
|
||||
msg['pretext'] = "WAZUH Alert"
|
||||
msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A"
|
||||
msg['text'] = alert.get('full_log')
|
||||
msg['rule_id'] = alert["rule"]["id"]
|
||||
msg['timestamp'] = alert["timestamp"]
|
||||
msg['id'] = alert['id']
|
||||
msg["all_fields"] = alert
|
||||
|
||||
#msg['fields'] = []
|
||||
# msg['fields'].append({
|
||||
# "title": "Agent",
|
||||
# "value": "({0}) - {1}".format(
|
||||
# alert['agent']['id'],
|
||||
# alert['agent']['name']
|
||||
# ),
|
||||
# })
|
||||
#if 'agentless' in alert:
|
||||
# msg['fields'].append({
|
||||
# "title": "Agentless Host",
|
||||
# "value": alert['agentless']['host'],
|
||||
# })
|
||||
|
||||
#msg['fields'].append({"title": "Location", "value": alert['location']})
|
||||
#msg['fields'].append({
|
||||
# "title": "Rule ID",
|
||||
# "value": "{0} _(Level {1})_".format(alert['rule']['id'], level),
|
||||
#})
|
||||
|
||||
#attach = {'attachments': [msg]}
|
||||
|
||||
return json.dumps(msg)
|
||||
|
||||
|
||||
def send_msg(msg, url):
|
||||
debug("# In send msg")
|
||||
headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'}
|
||||
res = requests.post(url, data=msg, headers=headers, verify=False)
|
||||
debug("# After send msg: %s" % res)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
# Read arguments
|
||||
bad_arguments = False
|
||||
if len(sys.argv) >= 4:
|
||||
msg = '{0} {1} {2} {3} {4}'.format(
|
||||
now,
|
||||
sys.argv[1],
|
||||
sys.argv[2],
|
||||
sys.argv[3],
|
||||
sys.argv[4] if len(sys.argv) > 4 else '',
|
||||
)
|
||||
#debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug')
|
||||
debug_enabled = True
|
||||
else:
|
||||
msg = '{0} Wrong arguments'.format(now)
|
||||
bad_arguments = True
|
||||
|
||||
# Logging the call
|
||||
try:
|
||||
f = open(log_file, 'a')
|
||||
except:
|
||||
f = open(log_file, 'w+')
|
||||
f.write("")
|
||||
f.close()
|
||||
|
||||
f = open(log_file, 'a')
|
||||
f.write(msg + '\n')
|
||||
f.close()
|
||||
|
||||
if bad_arguments:
|
||||
debug("# Exiting: Bad arguments. Inputted: %s" % sys.argv)
|
||||
sys.exit(1)
|
||||
|
||||
# Main function
|
||||
main(sys.argv)
|
||||
|
||||
except Exception as e:
|
||||
debug(str(e))
|
||||
raise
|
||||
@@ -1,6 +0,0 @@
|
||||
<integration>
|
||||
<name>custom-shuffle</name>
|
||||
<level>9</level>
|
||||
<hook_url>http://<IP>:<PORT>/api/v1/hooks/webhook_hookid</hook_url>
|
||||
<alert_format>json</alert_format>
|
||||
</integration>
|
||||
@@ -1,16 +0,0 @@
|
||||
# Mindmap exporter
|
||||
Shuffle has a mindmap for Workflow use-cases. These can be changed and exported, with the most important piece being that they're explorable and editable. This has and will come in handy for us as we build it into the product.
|
||||
|
||||
https://www.mindmeister.com/map/2172644474
|
||||
|
||||
## Editing the Mindmap
|
||||
There are a few categories. To edit them, click the small plus next to the branch you want to change.
|
||||
|
||||
## Exporting the Mindmap
|
||||
Click "Export as RTF" in the top left corner of the URL. Download it there.
|
||||
|
||||
## Generating the Shuffle-comaptible mindmap
|
||||
1. Move the rtf file here
|
||||
2. Rename it categories.rtf
|
||||
3. Run the read_categories.py file (python3 read_categories.py)
|
||||
4. You now have a file called categories.json locally with all the categories in JSON format, ready to be used in graphs.
|
||||
@@ -1,260 +0,0 @@
|
||||
[
|
||||
{
|
||||
"name": "1. Collect & Distribute",
|
||||
"color": "#c51152",
|
||||
"list": [
|
||||
{
|
||||
"name": "2-way Ticket synchronization",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Email management",
|
||||
"items": {
|
||||
"name": "Release a quarantined message",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "EDR to ticket",
|
||||
"items": {
|
||||
"name": "Get host information",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "SIEM to ticket",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "ChatOps",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Threat Intel received",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Domain investigation with LetsEncrypt",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Botnet tracker",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Get running containers",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Assign tickets",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Firewall alerts",
|
||||
"items": {
|
||||
"name": "URL filtering",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "IDS/IPS alerts",
|
||||
"items": {
|
||||
"name": "Manage policies",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Deduplicate information",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Correlate information",
|
||||
"items": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "2. Enrich",
|
||||
"color": "#f4c20d",
|
||||
"list": [
|
||||
{
|
||||
"name": "Internal Enrichment",
|
||||
"items": {
|
||||
"name": "...",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "External historical Enrichment",
|
||||
"items": {
|
||||
"name": "...",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Realtime",
|
||||
"items": {
|
||||
"name": "Analyze screenshots",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Ticketing webhook verification",
|
||||
"items": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "3. Detect",
|
||||
"color": "#3cba54",
|
||||
"list": [
|
||||
{
|
||||
"name": "Search SIEM (Sigma)",
|
||||
"items": {
|
||||
"name": "Endpoint",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Search EDR (OSQuery)",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Search emails (Phish)",
|
||||
"items": {
|
||||
"name": "Check headers and IOCs",
|
||||
"items": {}
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "Search IOCs (ioc-finder)",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Search files (Yara)",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Correlate tickets",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Honeypot access",
|
||||
"items": {
|
||||
"name": "...",
|
||||
"items": {}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "4. Respond",
|
||||
"color": "#4a148c",
|
||||
"list": [
|
||||
{
|
||||
"name": "Eradicate malware",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Quarantine host(s)",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Trigger scans",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Update indicators (FW, EDR, SIEM...)",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Autoblock activity when threat intel is received",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Lock/Delete/Reset account",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Lock vault",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Increase authentication",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Get policies from assets",
|
||||
"items": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "5. Verify",
|
||||
"color": "#4885ed",
|
||||
"list": [
|
||||
{
|
||||
"name": "Discover vulnerabilities",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Discover assets",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Ensure policies are followed",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Find Inactive users",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Ensure access rights match HR systems",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Ensure onboarding is followed",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Third party apps in SaaS",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Devices used for your cloud account",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Too much access in GCP/Azure/AWS/ other clouds",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Certificate validation",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Monitor new DNS entries for domain with passive DNS",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Monitor and track password dumps",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Monitor for mentions of domain on darknet sites",
|
||||
"items": {}
|
||||
},
|
||||
{
|
||||
"name": "Reporting",
|
||||
"items": {
|
||||
"name": "Monthly reports",
|
||||
"items": {
|
||||
"name": "...",
|
||||
"items": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,555 +0,0 @@
|
||||
{\rtf1\ansi\deff0\deflang2057\plain\fs24\fet1
|
||||
{\fonttbl
|
||||
{\f0\froman Arial;}
|
||||
}
|
||||
{\info
|
||||
{\createim\yr2022\mo2\dy20\hr1\min15}
|
||||
}
|
||||
|
||||
\paperw11907\paperh16840\margl1800\margr1800\margt1440\margb1440
|
||||
\slmult0\ltrpar\li0
|
||||
{\b\fs28
|
||||
Shuffle categories
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li200
|
||||
{\fs24
|
||||
1. Collect & Distribute
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
2-way Ticket synchronization
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Email management
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Attachments
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Manage senders
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Manage URLs
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Encode & Decode URLs
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Release a quarantined message
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
EDR to ticket
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Fetch incidents & events
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Quarantine files
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Quarantine host (respond)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Get host information
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
SIEM to ticket
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
ChatOps
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Threat Intel received
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Domain investigation with LetsEncrypt
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Botnet tracker
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Get running containers
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Assign tickets
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Firewall alerts
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Block/accept policies
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Add addresses and ports to groups
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Support custom URL categories
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Fetch logs for specific address
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
URL filtering
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
IDS/IPS alerts
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Get/Fetch alerts
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Receive alerts real-time
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Get PCAP files
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Get network logs
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Manage policies
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Deduplicate information
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Correlate information
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li200
|
||||
{\fs24
|
||||
3. Detect
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Search SIEM (Sigma)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Network
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Endpoint
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Search EDR (OSQuery)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Search emails (Phish)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Check malware
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Check targeted
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Check headers and IOCs
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Search IOCs (ioc-finder)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Search files (Yara)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Correlate tickets
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Honeypot access
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
S3 Honeypot
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
SSH Honeypot
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
FTP honeypot
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Network honeypot
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
...
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li200
|
||||
{\fs24
|
||||
rich
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li200
|
||||
{\fs24
|
||||
5. Verify
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Discover vulnerabilities
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Discover assets
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Ensure policies are followed
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Find Inactive users
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Ensure access rights match HR systems
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Ensure onboarding is followed
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Third party apps in SaaS
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Devices used for your cloud account
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Too much access in GCP/Azure/AWS/ other clouds
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Certificate validation
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Monitor new DNS entries for domain with passive DNS
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Monitor and track password dumps
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Monitor for mentions of domain on darknet sites
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Reporting
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Automation time saved
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Automation money saved
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Incident response report
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Department cost
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Monthly reports
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li800
|
||||
{\fs24
|
||||
EDR alerts
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li800
|
||||
{\fs24
|
||||
SIEM alerts
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li800
|
||||
{\fs24
|
||||
Emails quarantined
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li800
|
||||
{\fs24
|
||||
...
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li200
|
||||
{\fs24
|
||||
4. Respond
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Eradicate malware
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Quarantine host(s)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Trigger scans
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Update indicators (FW, EDR, SIEM...)
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Autoblock activity when threat intel is received
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Lock/Delete/Reset account
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Lock vault
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Increase authentication
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Get policies from assets
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li200
|
||||
{\fs24
|
||||
2. Enrich
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Internal Enrichment
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Users
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Hostnames
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
IPs
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Departments
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Role
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Software
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
...
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
External historical Enrichment
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
IPs
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
URLs
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Hashes
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Files
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
...
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Realtime
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
File detonation
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
URL detonation
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
PCAP analysis
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li600
|
||||
{\fs24
|
||||
Analyze screenshots
|
||||
}
|
||||
\par\pard\plain
|
||||
\slmult0\ltrpar\li400
|
||||
{\fs24
|
||||
Ticketing webhook verification
|
||||
}
|
||||
\par\pard\plain
|
||||
}
|
||||
@@ -1,66 +0,0 @@
|
||||
data = ""
|
||||
with open("categories.rtf", "r") as tmp:
|
||||
data = tmp.read()
|
||||
|
||||
fixed_json = []
|
||||
linearity = 0
|
||||
heading = ""
|
||||
subheading = ""
|
||||
subsubheading = ""
|
||||
|
||||
cnt = -1
|
||||
subcnt = -1
|
||||
|
||||
colors = ["#c51152", "#3cba54", "#4885ed", "#4a148c", "#f4c20d"]
|
||||
for line in data.split("\n"):
|
||||
if line == "rich":
|
||||
continue
|
||||
|
||||
if "li" in line:
|
||||
lisplit = line.split("\\")
|
||||
try:
|
||||
linearity = int(lisplit[-1][2])
|
||||
except:
|
||||
pass
|
||||
|
||||
#print("Linearity: %s" % linearity)
|
||||
|
||||
if line.startswith("{") or line.startswith("}"):
|
||||
continue
|
||||
|
||||
if line.startswith("\\"):
|
||||
continue
|
||||
|
||||
if linearity == 0:
|
||||
continue
|
||||
|
||||
if linearity == 2:
|
||||
#if cnt >= 0:
|
||||
# for key, value in fixed_json[cnt].items():
|
||||
# print(key, value)
|
||||
|
||||
|
||||
cnt += 1
|
||||
subcnt = -1
|
||||
fixed_json.append({"name": line, "color": colors[cnt], "list": []})
|
||||
heading = line
|
||||
elif linearity == 4:
|
||||
subheading = line
|
||||
fixed_json[cnt]["list"].append({"name": line, "items": {}})
|
||||
subcnt += 1
|
||||
elif linearity == 6:
|
||||
fixed_json[cnt]["list"][subcnt]["items"] = {"name": line, "items": {}}
|
||||
elif linearity == 8:
|
||||
fixed_json[cnt]["list"][subcnt]["items"]["items"] = {"name": line, "items": {}}
|
||||
else:
|
||||
print("No handler for %s" % line)
|
||||
|
||||
#print(line)
|
||||
#print(data)
|
||||
import json
|
||||
filename = "categories.json"
|
||||
fixed_json.sort(key=lambda x: x["name"])
|
||||
with open(filename, "w+") as tmp:
|
||||
tmp.write(json.dumps(fixed_json, indent=4))
|
||||
|
||||
print("Wrote to file %s" % filename)
|
||||
Reference in New Issue
Block a user