diff --git a/functions/extensions/aws-s3-lambda/README.md b/functions/extensions/aws-s3-lambda/README.md deleted file mode 100644 index f665851a..00000000 --- a/functions/extensions/aws-s3-lambda/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# AWS Lambda forwarder to Shuffle -This function is made to forward S3 notifications to Shuffle to run a workflow when an object is made or updated. diff --git a/functions/extensions/aws-s3-lambda/s3_function.py b/functions/extensions/aws-s3-lambda/s3_function.py deleted file mode 100644 index d83d6154..00000000 --- a/functions/extensions/aws-s3-lambda/s3_function.py +++ /dev/null @@ -1,22 +0,0 @@ -import json -import urllib.parse -import urllib3 -import os - -print('Loading function') - -def lambda_handler(event, context): - # Get the object from the event and show its content type - bucket = event['Records'][0]['s3']['bucket']['name'] - - webhook = os.environ.get("SHUFFLE_WEBHOOK") - if not webhook: - return "No webhook environment defined: SHUFFLE_WEBHOOK" - - http = urllib3.PoolManager() - ret = http.request('POST', webhook, body=json.dumps(event["Records"][0]).encode("utf-8")) - if ret.status != 200: - return "Bad status code for webhook: %d" % ret.status - - print("Status code: %d\nData: %s" % (ret.status, ret.data)) - return "Successfully started with data %s" % ret.data diff --git a/functions/extensions/cortex-responders/Shuffle/requirements.txt b/functions/extensions/cortex-responders/Shuffle/requirements.txt deleted file mode 100755 index 6aabc3cf..00000000 --- a/functions/extensions/cortex-responders/Shuffle/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -cortexutils -requests diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.json b/functions/extensions/cortex-responders/Shuffle/shuffle.json deleted file mode 100755 index db755b49..00000000 --- a/functions/extensions/cortex-responders/Shuffle/shuffle.json +++ /dev/null @@ -1,43 +0,0 @@ -{ - "name": "Shuffle", - "version": "1.0", - "author": "@frikkylikeme", - "url": "https://github.com/frikky/shuffle", - "license": "AGPL-V3", - "description": "Execute a workflow in Shuffle", - "dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"], - "command": "Shuffle/shuffle.py", - "baseConfig": "Shuffle", - "configurationItems": [ - { - "name": "url", - "description": "The URL to your shuffle instance", - "type": "string", - "multi": false, - "required": true, - "defaultValue": "https://shuffler.io" - }, - { - "name": "api_key", - "description": "The API key to your Shuffle user", - "type": "string", - "multi": false, - "required": true - }, - { - "name": "verifyssl", - "description": "Verify SSL certificate", - "type": "boolean", - "multi": false, - "required": true, - "defaultValue": true - }, - { - "name": "workflow_id", - "description": "The ID of the workflow to execute", - "type": "string", - "multi": false, - "required": true - } - ] -} diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.py b/functions/extensions/cortex-responders/Shuffle/shuffle.py deleted file mode 100755 index db343230..00000000 --- a/functions/extensions/cortex-responders/Shuffle/shuffle.py +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env python3 -#encoding: utf-8 - -from cortexutils.responder import Responder -import requests - -class Shuffle(Responder): - def __init__(self): - Responder.__init__(self) - self.api_key = self.get_param("config.api_key", "") - self.url = self.get_param("config.url", "") - self.workflow_id = self.get_param("config.workflow_id", "") - self.verify = self.get_param('config.verifyssl', True, None) - - def run(self): - Responder.run(self) - parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id) - headers = { - "Authorization": "Bearer %s" % self.api_key, - "User-Agent": "Cortex-Analyzer" - } - requests.post(parsed_url, headers=headers,verify=self.verify) - - self.report({'message': 'message sent'}) - -if __name__ == '__main__': - Shuffle().run() diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt b/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt deleted file mode 100755 index 6aabc3cf..00000000 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -cortexutils -requests diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json deleted file mode 100755 index a6d3e9d8..00000000 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.json +++ /dev/null @@ -1,28 +0,0 @@ -{ - "name": "Shuffle_webhook", - "version": "1.0", - "author": "@azgaviperr", - "url": "https://github.com/frikky/shuffle", - "license": "AGPL-V3", - "description": "Execute a webhook in Shuffle", - "dataTypeList": ["thehive:case", "thehive:alert", "thehive:case_artifact"], - "command": "Shuffle_Webhook/shuffle_webhook.py", - "baseConfig": "Shuffle_Webhook", - "configurationItems": [ - { - "name": "webhook_url", - "description": "The URL to your shuffle instance", - "type": "string", - "multi": false, - "required": true - }, - { - "name": "verifyssl", - "description": "Verify SSL certificate", - "type": "boolean", - "multi": false, - "required": true, - "defaultValue": true - } - ] -} diff --git a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py b/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py deleted file mode 100755 index daed501c..00000000 --- a/functions/extensions/cortex-responders/Shuffle_Webhook/shuffle_webhook.py +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env python3 -#encoding: utf-8 - -from cortexutils.responder import Responder -import requests - -class Shuffle(Responder): - def __init__(self): - Responder.__init__(self) - self.api_key = self.get_param("config.api_key", "") - self.webhook_url = self.get_param("config.webhook_url", "") - self.webhook_id = self.get_param("config.webhook_id", "") - self.verify = self.get_param('config.verifyssl', True, None) - self.data = self.get_param('data') - - def run(self): - Responder.run(self) - headers = { - "Content-Type": "application/json", - "Accept": "application/json", - "User-Agent": "Cortex-Analyzer" - } - requests.post(self.webhook_url, headers=headers,verify=self.verify, json=self.data) - - self.report({'message': 'message sent'}) - -if __name__ == '__main__': - Shuffle().run() diff --git a/functions/extensions/elasticsearch/run.sh b/functions/extensions/elasticsearch/run.sh deleted file mode 100755 index 666b1d95..00000000 --- a/functions/extensions/elasticsearch/run.sh +++ /dev/null @@ -1,19 +0,0 @@ -#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -e ELASTICSEARCH_USERNAME=frikky -e ELASTICSEARCH_PASSWORD=likeme -e xpack.security.enabled=true docker.elastic.co/elasticsearch/elasticsearch:7.12.1 -#docker run -d -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.12.1 -# -# -# -#echo "\nWaiting for 1.5 minute, then adding data" -#sleep 90 -#echo "\nSlept 90 seconds: ADDING DATA" -#curl -XPOST http://localhost:9200/_security/user/frikky -H "Content-Type: application/json" -d '{"enabled": true, "email": "frikky@shuffler.io"}' - -#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D", "hits": 0}' -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A", "hits": 0}' -# -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "122.14.137.67", "dst": "103.35.191.16", "message": "alert", "md5": "CAEF973033E593C625FB2AA34F7026DC", "sha256": "DB1AEC5222075800EDA75D7205267569679B424E5C58A28102417F46D3B5790D"}' -#echo -#curl -XPOST -u frikky:likeme http://localhost:9200/samples2/_doc -H "Content-Type: application/json" -d '{"src": "134.119.219.71", "dst": "103.35.191.41", "message": "alert", "md5": "9498FF82A64FF445398C8426ED63EA5B", "sha256": "8B2E701E91101955C73865589A4C72999AEABC11043F712E05FDB1C17C4AB19A"}' -#echo diff --git a/functions/extensions/k8s/shuffle/.helmignore b/functions/extensions/k8s/shuffle/.helmignore deleted file mode 100755 index 0e8a0eb3..00000000 --- a/functions/extensions/k8s/shuffle/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/functions/extensions/k8s/shuffle/Chart.yaml b/functions/extensions/k8s/shuffle/Chart.yaml deleted file mode 100755 index eb53a0b5..00000000 --- a/functions/extensions/k8s/shuffle/Chart.yaml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: v2 -name: shuffle -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.0 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "2.0.0" diff --git a/functions/extensions/k8s/shuffle/templates/NOTES.txt b/functions/extensions/k8s/shuffle/templates/NOTES.txt deleted file mode 100755 index eaba4460..00000000 --- a/functions/extensions/k8s/shuffle/templates/NOTES.txt +++ /dev/null @@ -1,22 +0,0 @@ -1. Get the application URL by running these commands: -{{- if .Values.ingress.enabled }} -{{- range $host := .Values.ingress.hosts }} - {{- range .paths }} - http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} - {{- end }} -{{- end }} -{{- else if contains "NodePort" .Values.service.type }} - export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "shuffle.fullname" . }}) - export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") - echo http://$NODE_IP:$NODE_PORT -{{- else if contains "LoadBalancer" .Values.service.type }} - NOTE: It may take a few minutes for the LoadBalancer IP to be available. - You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "shuffle.fullname" . }}' - export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "shuffle.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") - echo http://$SERVICE_IP:{{ .Values.service.port }} -{{- else if contains "ClusterIP" .Values.service.type }} - export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "shuffle.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") - export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") - echo "Visit http://127.0.0.1:8080 to use your application" - kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/_helpers.tpl b/functions/extensions/k8s/shuffle/templates/_helpers.tpl deleted file mode 100755 index 66efef07..00000000 --- a/functions/extensions/k8s/shuffle/templates/_helpers.tpl +++ /dev/null @@ -1,62 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "shuffle.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "shuffle.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 63 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "shuffle.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "shuffle.labels" -}} -helm.sh/chart: {{ include "shuffle.chart" . }} -{{ include "shuffle.selectorLabels" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "shuffle.selectorLabels" -}} -app.kubernetes.io/name: {{ include "shuffle.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "shuffle.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "shuffle.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/deployment.yaml b/functions/extensions/k8s/shuffle/templates/deployment.yaml deleted file mode 100755 index f2050d82..00000000 --- a/functions/extensions/k8s/shuffle/templates/deployment.yaml +++ /dev/null @@ -1,253 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}frontend - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-frontend - template: - metadata: - labels: - service: shuffle - app: shuffle-frontend - spec: - containers: - - name: shuffle-frontend - image: ghcr.io/frikky/shuffle-frontend:nightly - imagePullPolicy: {{ .Values.image.pullPolicy | quote }} - env: - - name: BACKEND_HOSTNAME - value: backend-service - - name: TZ - value: Asia/Shanghai - ports: - - name: http - containerPort: 80 - hostPort: 3001 - - name: https - containerPort: 443 - hostname: shuffle-frontend - restartPolicy: Always - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}backend - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-backend - template: - metadata: - labels: - service: shuffle - app: shuffle-backend - spec: - containers: - - name: shuffle-backend - image: ghcr.io/frikky/shuffle-backend:nightly - env: - - name: BACKEND_HOSTNAME - value: "backend-service" - - name: BACKEND_PORT - value: "5001" - - name: ENVIRONMENT_NAME - value: "Shuffle" - - name: HTTPS_PROXY - - name: HTTP_PROXY - - name: ORG_ID - value: "Shuffle" - - name: OUTER_HOSTNAME - value: "backend-service" - - name: SHUFFLE_APP_FORCE_UPDATE - value: "false" - - name: SHUFFLE_APP_HOTLOAD_FOLDER - value: "/shuffle-apps" - - name: SHUFFLE_APP_HOTLOAD_LOCATION - value: "/shuffle-apps" - - name: SHUFFLE_CONTAINER_AUTO_CLEANUP - value: "false" - - name: SHUFFLE_DEFAULT_APIKEY - - name: SHUFFLE_DEFAULT_PASSWORD - - name: SHUFFLE_DEFAULT_USERNAME - - name: SHUFFLE_DOWNLOAD_AUTH_BRANCH - - name: SHUFFLE_DOWNLOAD_AUTH_PASSWORD - - name: SHUFFLE_DOWNLOAD_AUTH_USERNAME - - name: SHUFFLE_DOWNLOAD_WORKFLOW_BRANCH - - name: SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION - - name: SHUFFLE_DOWNLOAD_WORKFLOW_PASSWORD - - name: SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME - - name: SHUFFLE_ELASTIC - value: "true" - - name: SHUFFLE_OPENSEARCH_APIKEY - - name: SHUFFLE_OPENSEARCH_CERTIFICATE_FILE - - name: SHUFFLE_OPENSEARCH_CLOUDID - - name: SHUFFLE_OPENSEARCH_PASSWORD - - name: SHUFFLE_OPENSEARCH_PROXY - - name: SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY - value: "true" - - name: SHUFFLE_OPENSEARCH_URL - value: http://opensearch-service:9200 - - name: SHUFFLE_OPENSEARCH_USERNAME - value: "" - - name: SHUFFLE_PASS_APP_PROXY - value: "FALSE" - - name: SHUFFLE_PASS_WORKER_PROXY - value: "FALSE" - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - - mountPath: /shuffle-apps - name: shuffle-app-hotload-location - - mountPath: /shuffle-files - name: shuffle-file-location - hostname: shuffle-backend - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock - - name: shuffle-app-hotload-location - hostPath: - path: /data/kubernetes/shuffle-apps - type: DirectoryOrCreate - - name: shuffle-file-location - hostPath: - path: /data/kubernetes/shuffle-files - type: DirectoryOrCreate - ---- -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ .Values.name }}orborus - namespace: {{ .Values.namespace | quote }} -spec: - replicas: 1 - selector: - matchLabels: - service: shuffle - app: shuffle-orborus - template: - metadata: - labels: - service: shuffle - app: shuffle-orborus - spec: - containers: - - name: shuffle-orborus - image: ghcr.io/frikky/shuffle-orborus:nightly - env: - - name: RUNNING_MODE - value: kubernetes - - name: BASE_URL - value: http://backend-service:5001 - - name: CLEANUP - value: "false" - - name: DOCKER_API_VERSION - value: "1.40" - - name: ENVIRONMENT_NAME - value: Shuffle - - name: HTTPS_PROXY - - name: HTTP_PROXY - - name: ORG_ID - value: Shuffle - - name: SHUFFLE_APP_SDK_VERSION - value: 0.8.97 - - name: SHUFFLE_BASE_IMAGE_NAME - value: frikky - - name: SHUFFLE_BASE_IMAGE_REGISTRY - value: ghcr.io - - name: SHUFFLE_BASE_IMAGE_TAG_SUFFIX - value: "-0.8.80" - - name: SHUFFLE_ORBORUS_EXECUTION_TIMEOUT - value: "600" - - name: SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY - value: "50" - - name: SHUFFLE_PASS_WORKER_PROXY - value: "TRUE" - - name: SHUFFLE_WORKER_VERSION - value: nightly - - name: TZ - value: Asia/Shanghai - volumeMounts: - - mountPath: /var/run/docker.sock - name: docker-sock - hostname: shuffle-orborus - volumes: - - name: docker-sock - hostPath: - path: /var/run/docker.sock -#--- -#apiVersion: apps/v1 -#kind: Deployment -#metadata: -# name: {{ .Values.name }}opensearch -# namespace: {{ .Values.namespace | quote }} -#spec: -# replicas: 1 -# selector: -# matchLabels: -# service: shuffle -# app: shuffle-opensearch -# template: -# metadata: -# labels: -# service: shuffle -# app: shuffle-opensearch -# spec: -# nodeSelector: -# node.bdlab-venus.com/opensearch: available -# initContainers: -# - name: permissions-fix -# image: frikky/busybox -# #volumeMounts: -# # - name: opensearch-claim0 -# # mountPath: /usr/share/elasticsearch/data -# command: [ 'chown' ] -# args: [ '1000:1000', '/usr/share/elasticsearch/data' ] -# containers: -# - name: shuffle-opensearch -# image: opensearchproject/opensearch:1.0.1 -# env: -# - name: TZ -# value: Asia/Shanghai -# - name: bootstrap.memory_lock -# value: "false" -# - name: OPENSEARCH_JAVA_OPTS -# value: "-Xms1024m -Xmx1024m" -# - name: opendistro_security.disabled -# value: "true" -# - name: cluster.routing.allocation.disk.threshold_enabled -# value: "false" -# - name: cluster.name -# value: shuffle-cluster -# - name: node.name -# value: opensearch-service -# - name: discovery.seed_hosts -# value: opensearch-service -# - name: cluster.initial_master_nodes -# value: opensearch-service - # volumeMounts: - # - mountPath: /usr/share/opensearch/data - # name: opensearch-claim0 - #volumes: - # - name: opensearch-claim0 - # persistentVolumeClaim: - # claimName: opensearch-claim0 -# volumeMounts: -# - mountPath: /usr/share/opensearch/data -# readOnly: true -# name: db-location -# volumes: -# - name: db-location -# hostPath: -# path: /data/kubernetes/shuffle-opensearch -# type: DirectoryOrCreate diff --git a/functions/extensions/k8s/shuffle/templates/hpa.yaml b/functions/extensions/k8s/shuffle/templates/hpa.yaml deleted file mode 100755 index 49f84ca2..00000000 --- a/functions/extensions/k8s/shuffle/templates/hpa.yaml +++ /dev/null @@ -1,28 +0,0 @@ -{{- if .Values.autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "shuffle.fullname" . }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "shuffle.fullname" . }} - minReplicas: {{ .Values.autoscaling.minReplicas }} - maxReplicas: {{ .Values.autoscaling.maxReplicas }} - metrics: - {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/ingress.yaml b/functions/extensions/k8s/shuffle/templates/ingress.yaml deleted file mode 100755 index e35d2ae0..00000000 --- a/functions/extensions/k8s/shuffle/templates/ingress.yaml +++ /dev/null @@ -1,61 +0,0 @@ -{{- if .Values.ingress.enabled -}} -{{- $fullName := include "shuffle.fullname" . -}} -{{- $svcPort := .Values.service.port -}} -{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} - {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} - {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} - {{- end }} -{{- end }} -{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1 -{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} -apiVersion: networking.k8s.io/v1beta1 -{{- else -}} -apiVersion: extensions/v1beta1 -{{- end }} -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} - ingressClassName: {{ .Values.ingress.className }} - {{- end }} - {{- if .Values.ingress.tls }} - tls: - {{- range .Values.ingress.tls }} - - hosts: - {{- range .hosts }} - - {{ . | quote }} - {{- end }} - secretName: {{ .secretName }} - {{- end }} - {{- end }} - rules: - {{- range .Values.ingress.hosts }} - - host: {{ .host | quote }} - http: - paths: - {{- range .paths }} - - path: {{ .path }} - {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} - pathType: {{ .pathType }} - {{- end }} - backend: - {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} - service: - name: {{ $fullName }} - port: - number: {{ $svcPort }} - {{- else }} - serviceName: {{ $fullName }} - servicePort: {{ $svcPort }} - {{- end }} - {{- end }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml b/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml deleted file mode 100755 index 934c68e2..00000000 --- a/functions/extensions/k8s/shuffle/templates/persistent-volume.yaml +++ /dev/null @@ -1,24 +0,0 @@ -#apiVersion: v1 -#kind: PersistentVolume -#metadata: -# name: opensearch-claim0 -# labels: -# app: opensearch-claim0 -#spec: -# capacity: -# storage: "10G" -# volumeMode: Filesystem -# persistentVolumeReclaimPolicy: Retain -# storageClassName: local-storage -# accessModes: -# - "ReadWriteOnce" -# local: -# path: "/data/kubernetes/shuffle-opensearch" -# nodeAffinity: -# required: -# nodeSelectorTerms: -# - matchExpressions: -# - key: node.dollar.com/opensearch -# operator: In -# values: -# - available diff --git a/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml b/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml deleted file mode 100755 index 7d1736a4..00000000 --- a/functions/extensions/k8s/shuffle/templates/persistentvolumeclaim.yaml +++ /dev/null @@ -1,17 +0,0 @@ -#apiVersion: v1 -#kind: PersistentVolumeClaim -#metadata: -# name: opensearch-claim0 -# namespace: {{ .Values.namespace }} -# labels: -# app: opensearch-claim0 -#spec: -# selector: -# matchLabels: -# app: opensearch-claim0 -# accessModes: -# - ReadWriteOnce -# storageClassName: local-storage -# resources: -# requests: -# storage: 5Gi diff --git a/functions/extensions/k8s/shuffle/templates/service.yaml b/functions/extensions/k8s/shuffle/templates/service.yaml deleted file mode 100755 index 8fc78dfa..00000000 --- a/functions/extensions/k8s/shuffle/templates/service.yaml +++ /dev/null @@ -1,52 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: backend-service - namespace: {{ .Values.namespace | quote }} -spec: - ports: - - name: "5001" - port: 5001 - targetPort: 5001 - selector: - app: shuffle-backend - ---- -apiVersion: v1 -kind: Service -metadata: - name: frontend-service - namespace: {{ .Values.namespace | quote }} -spec: - type: NodePort - externalTrafficPolicy: Local - ports: - - name: "3001" - port: 3001 - nodePort: 3001 - targetPort: 80 - - name: "3443" - port: 3443 - nodePort: 3443 - targetPort: 443 - protocol: TCP - selector: - app: shuffle-frontend - -#--- -#apiVersion: v1 -#kind: Service -#metadata: -# name: opensearch-service -# namespace: {{ .Values.namespace | quote }} -#spec: -# type: NodePort -# externalTrafficPolicy: Local -# ports: -# - name: "9200" -# port: 9200 -# targetPort: 9200 -# nodePort: 9200 -# protocol: TCP -# selector: -# app: shuffle-opensearch diff --git a/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml b/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml deleted file mode 100755 index 11e9bdb7..00000000 --- a/functions/extensions/k8s/shuffle/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "shuffle.serviceAccountName" . }} - labels: - {{- include "shuffle.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml b/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml deleted file mode 100755 index e68f3142..00000000 --- a/functions/extensions/k8s/shuffle/templates/tests/test-connection.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: "{{ include "shuffle.fullname" . }}-test-connection" - labels: - {{- include "shuffle.labels" . | nindent 4 }} - annotations: - "helm.sh/hook": test -spec: - containers: - - name: wget - image: busybox - command: ['wget'] - args: ['{{ include "shuffle.fullname" . }}:{{ .Values.service.port }}'] - restartPolicy: Never diff --git a/functions/extensions/k8s/shuffle/values.yaml b/functions/extensions/k8s/shuffle/values.yaml deleted file mode 100755 index 07e935c6..00000000 --- a/functions/extensions/k8s/shuffle/values.yaml +++ /dev/null @@ -1,82 +0,0 @@ -# Default values for shuffle. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: 1 - -image: - repository: nginx - pullPolicy: IfNotPresent - # Overrides the image tag whose default is the chart appVersion. - tag: "" - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: {} - -podSecurityContext: {} - # fsGroup: 2000 - -securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -service: - type: ClusterIP - port: 80 - -ingress: - enabled: false - className: "" - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - hosts: - - host: chart-example.local - paths: - - path: / - pathType: ImplementationSpecific - tls: [] - # - secretName: chart-example-tls - # hosts: - # - chart-example.local - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} diff --git a/functions/extensions/misp/Dockerfile b/functions/extensions/misp/Dockerfile deleted file mode 100755 index f2b8ae3f..00000000 --- a/functions/extensions/misp/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM python:3.9.4-alpine as base - -FROM base as builder - -RUN mkdir /install -WORKDIR /install - -FROM base -RUN apk add g++ - -COPY --from=builder /install /usr/local -COPY requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt - - -RUN mkdir /app -WORKDIR /app -COPY requirements.txt /app/requirements.txt -RUN python3 -m pip install -r /app/requirements.txt - -COPY sub.py /app/sub.py - -CMD ["python3", "sub.py"] diff --git a/functions/extensions/misp/docker-compose.yml b/functions/extensions/misp/docker-compose.yml deleted file mode 100755 index 5b0130c0..00000000 --- a/functions/extensions/misp/docker-compose.yml +++ /dev/null @@ -1,9 +0,0 @@ -version: '3' -services: - zmq: - image: ghcr.io/frikky/shuffle-zmq:latest - environment: - - ZMQ_HOSTNAME=localhost - - ZMQ_PORT=50000 - - ZMQ_FORWARD_URL=https://shuffler.io/api/v1/hooks/webhook_e09bea36-9976-1421-82bc-b8764ca83c1e - restart: unless-stopped diff --git a/functions/extensions/misp/requirements.txt b/functions/extensions/misp/requirements.txt deleted file mode 100755 index a1cdf7dd..00000000 --- a/functions/extensions/misp/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -pyzmq -requests diff --git a/functions/extensions/misp/sub.py b/functions/extensions/misp/sub.py deleted file mode 100755 index feb78c08..00000000 --- a/functions/extensions/misp/sub.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -print("Running imports") -import sys -import zmq -import json -import time -import pprint -import os -import sys -import requests - -forward_url = os.getenv("ZMQ_FORWARD_URL", "") -print("Checking forward url (ZMQ_FORWARD_URL): %s" % forward_url) -def handle_hook(data): - ret = requests.post(forward_url, json=data) - print(ret.text) - print(ret.status_code) - -def main(): - host = os.getenv("ZMQ_HOST", "localhost") - port = os.getenv("ZMQ_PORT", "50000") - - if len(forward_url) == 0: - print("Failed to start - define ZMQ_FORWARD_URL for webhook forwarder") - exit(0) - - print("Starting connection setup to %s:%s" % (host, port)) - context = zmq.Context() - socket = context.socket(zmq.SUB) - socket.connect ("tcp://%s:%s" % (host, port)) - socket.setsockopt(zmq.SUBSCRIBE, b'') - - poller = zmq.Poller() - poller.register(socket, zmq.POLLIN) - - print("Starting zmq check for %s:%s" % (host, port)) - while True: - socks = dict(poller.poll(timeout=None)) - if socket in socks and socks[socket] == zmq.POLLIN: - message = socket.recv() - #print(message) - topic, s, m = message.decode('utf-8').partition(" ") - - d = json.loads(m) - try: - # print test if you want status (heartbeat) - test = d["status"] - except KeyError: - handle_hook(d) - - time.sleep(1) - -if __name__ == "__main__": - print("In init ") - main() diff --git a/functions/extensions/scripts/disable_disk_check.sh b/functions/extensions/scripts/disable_disk_check.sh deleted file mode 100755 index cadda899..00000000 --- a/functions/extensions/scripts/disable_disk_check.sh +++ /dev/null @@ -1,11 +0,0 @@ -curl -XPUT -u admin:admin https://localhost:9200/_cluster/settings -H "Content-Type:application/json" -k -d \ -'{ - "transient": { - "cluster.routing.allocation.disk.threshold_enabled": false - } -}' - -curl -XPUT -u admin:admin https://localhost:9200/_all/_settings -H "Content-Type: application/json" -k -d \ -'{ - "index.blocks.read_only_allow_delete": null -}' diff --git a/functions/extensions/swarm/.gitignore b/functions/extensions/swarm/.gitignore deleted file mode 100644 index 9ce4a537..00000000 --- a/functions/extensions/swarm/.gitignore +++ /dev/null @@ -1 +0,0 @@ -shuffle-database/nodes diff --git a/functions/extensions/swarm/docker-compose.yml b/functions/extensions/swarm/docker-compose.yml deleted file mode 100644 index 57ea9b49..00000000 --- a/functions/extensions/swarm/docker-compose.yml +++ /dev/null @@ -1,131 +0,0 @@ -version: '3.4' -services: - backend: - image: ghcr.io/frikky/shuffle-backend:nightly - #hostname: shuffle-backend - environment: - BACKEND_HOSTNAME: backend - OUTER_HOSTNAME: backend - BACKEND_PORT: '5001' - HTTPS_PROXY: '' - HTTP_PROXY: '' - SHUFFLE_APP_DOWNLOAD_LOCATION: https://github.com/frikky/shuffle-apps - SHUFFLE_APP_FORCE_UPDATE: 'false' - SHUFFLE_APP_HOTLOAD_FOLDER: /shuffle-apps - SHUFFLE_APP_HOTLOAD_LOCATION: ./shuffle-apps - DATASTORE_EMULATOR_HOST: "shuffle-database:8000" - DOCKER_API_VERSION: '1.40' - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: '-0.9.30' - SHUFFLE_CONTAINER_AUTO_CLEANUP: 'true' - SHUFFLE_DEFAULT_APIKEY: '' - SHUFFLE_FILE_LOCATION: /shuffle-files - SHUFFLE_OPENSEARCH_APIKEY: '' - SHUFFLE_OPENSEARCH_CERTIFICATE_FILE: '' - SHUFFLE_OPENSEARCH_CLOUDID: '' - SHUFFLE_OPENSEARCH_PROXY: '' - SHUFFLE_OPENSEARCH_SKIPSSL_VERIFY: 'true' - SHUFFLE_OPENSEARCH_URL: http://opensearch:9200 - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_ELASTIC: 'true' - #SHUFFLE_ENCRYPTION_MODIFIER: - ports: - - "5001:5001" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./shuffle-apps:/shuffle-apps - - ./shuffle-files:/shuffle-files - networks: - - shuffle_prod - #- reverseproxy - depends_on: - - opensearch - logging: - driver: json-file - frontend: - image: ghcr.io/frikky/shuffle-frontend:nightly - healthcheck: - test: curl -fs http://localhost:80 || exit 1 - interval: 30s - timeout: 5s - retries: 3 - ports: - - "3001:80" - - "3443:443" - networks: - - shuffle_prod - #- reverseproxy - environment: - - "BACKEND_HOSTNAME=backend" - depends_on: - - backend - deploy: - update_config: - order: start-first - opensearch: - image: opensearchproject/opensearch:1.1.0 - healthcheck: - test: curl -fs http://localhost:9200/_cat/health || exit 1 - interval: 30s - timeout: 5s - retries: 3 - environment: - - bootstrap.memory_lock=false - - "OPENSEARCH_JAVA_OPTS=-Xms1024m -Xmx1024m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM - - plugins.security.disabled=true - - cluster.routing.allocation.disk.threshold_enabled=false - - cluster.name=shuffle-cluster - - node.name=opensearch - - discovery.seed_hosts=opensearch - - cluster.initial_master_nodes=opensearch - - node.store.allow_mmap=false - volumes: - - ./shuffle-database:/usr/share/opensearch/data:rw - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - #hostname: shuffle-orborus - environment: - #SHUFFLE_WORKER_VERSION: nightly - SHUFFLE_APP_SDK_VERSION: 0.8.97 - SHUFFLE_WORKER_VERSION: nightly - BASE_URL: http://backend:5001 - #BASE_URL: http://192.168.86.37:5001 - CLEANUP: 'true' - DOCKER_API_VERSION: '1.40' - ENVIRONMENT_NAME: Shuffle - HTTPS_PROXY: '' - HTTP_PROXY: '' - ORG_ID: Shuffle - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80 - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50' - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800' - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_SCALE_REPLICAS: 5 - SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod - SHUFFLE_SWARM_CONFIG: "run" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - -networks: - shuffle_prod: - driver: overlay - external: true - #reverseproxy: - # driver: overlay - # #external: true diff --git a/functions/extensions/swarm/network.sh b/functions/extensions/swarm/network.sh deleted file mode 100644 index f3a323ed..00000000 --- a/functions/extensions/swarm/network.sh +++ /dev/null @@ -1 +0,0 @@ -docker network create -d overlay shuffle_prod diff --git a/functions/extensions/swarm/orborus.yml b/functions/extensions/swarm/orborus.yml deleted file mode 100644 index fb90b10f..00000000 --- a/functions/extensions/swarm/orborus.yml +++ /dev/null @@ -1,39 +0,0 @@ -version: '3.4' -services: - orborus: - image: ghcr.io/frikky/shuffle-orborus:nightly - #hostname: shuffle-orborus - environment: - #SHUFFLE_WORKER_VERSION: nightly - SHUFFLE_APP_SDK_VERSION: 0.8.97 - SHUFFLE_WORKER_VERSION: nightly - BASE_URL: http://:5001 - #BASE_URL: http://192.168.86.37:5001 - CLEANUP: 'true' - DOCKER_API_VERSION: '1.40' - ENVIRONMENT_NAME: Shuffle - HTTPS_PROXY: '' - HTTP_PROXY: '' - ORG_ID: Shuffle - SHUFFLE_BASE_IMAGE_NAME: frikky - SHUFFLE_BASE_IMAGE_REGISTRY: ghcr.io - SHUFFLE_BASE_IMAGE_TAG_SUFFIX: -0.8.80 - SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY: '50' - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT: '800' - SHUFFLE_PASS_APP_PROXY: 'FALSE' - SHUFFLE_PASS_WORKER_PROXY: 'TRUE' - SHUFFLE_SCALE_REPLICAS: 5 - SHUFFLE_SWARM_NETWORK_NAME: shuffle_prod - SHUFFLE_SWARM_CONFIG: "run" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - networks: - - shuffle_prod - #- reverseproxy - logging: - driver: json-file - -networks: - shuffle_prod: - driver: overlay - external: true diff --git a/functions/extensions/swarm/run.sh b/functions/extensions/swarm/run.sh deleted file mode 100644 index faeadab8..00000000 --- a/functions/extensions/swarm/run.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker swarm init -chown 1000:1000 -R shuffle-database/ -docker network create -d overlay shuffle_prod -docker stack deploy --compose-file=docker-compose.yml shuffle_swarm diff --git a/functions/extensions/swarm/run_orborus.sh b/functions/extensions/swarm/run_orborus.sh deleted file mode 100644 index bcbd626f..00000000 --- a/functions/extensions/swarm/run_orborus.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker swarm init -chown 1000:1000 -R shuffle-database/ -docker network create -d overlay shuffle_prod -docker stack deploy --compose-file=orborus.yml shuffle_orborus diff --git a/functions/extensions/swarm/shuffle-apps/tmp b/functions/extensions/swarm/shuffle-apps/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/shuffle-database/tmp b/functions/extensions/swarm/shuffle-database/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/shuffle-files/tmp b/functions/extensions/swarm/shuffle-files/tmp deleted file mode 100644 index e69de29b..00000000 diff --git a/functions/extensions/swarm/stop.sh b/functions/extensions/swarm/stop.sh deleted file mode 100644 index c516f348..00000000 --- a/functions/extensions/swarm/stop.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker stack rm shuffle_swarm - -# diff --git a/functions/extensions/wazuh/custom-shuffle b/functions/extensions/wazuh/custom-shuffle deleted file mode 100755 index bd540414..00000000 --- a/functions/extensions/wazuh/custom-shuffle +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh -# Created by Shuffle, AS. . - -WPYTHON_BIN="framework/python/bin/python3" - -SCRIPT_PATH_NAME="$0" - -DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)" -SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})" - -case ${DIR_NAME} in - */active-response/bin | */wodles*) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)" - fi - - PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" - ;; - */bin) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" - fi - - PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py" - ;; - */integrations) - if [ -z "${WAZUH_PATH}" ]; then - WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" - fi - - PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" - ;; -esac - - -${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@" diff --git a/functions/extensions/wazuh/custom-shuffle.py b/functions/extensions/wazuh/custom-shuffle.py deleted file mode 100755 index 8e555ea9..00000000 --- a/functions/extensions/wazuh/custom-shuffle.py +++ /dev/null @@ -1,206 +0,0 @@ -#!/usr/bin/env python3 -# Created by Shuffle, AS. . -# Based on the Slack integration using Webhooks - -import json -import sys -import time -import os - -try: - import requests - from requests.auth import HTTPBasicAuth -except Exception as e: - print("No module 'requests' found. Install: pip install requests") - sys.exit(1) - -# ADD THIS TO ossec.conf configuration: -# -# custom-shuffle -# http://:3001/api/v1/hooks/ -# 3 -# json -# - -# Global vars -debug_enabled = False -pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) -json_alert = {} -now = time.strftime("%a %b %d %H:%M:%S %Z %Y") - -# Set paths -log_file = '{0}/logs/integrations.log'.format(pwd) - -try: - with open("/tmp/shuffle_start.txt", "w+") as tmp: - tmp.write("Script started") -except: - pass - - -def main(args): - debug("# Starting") - - # Read args - alert_file_location = args[1] - webhook = args[3] - - debug("# Webhook") - debug(webhook) - - debug("# File location") - debug(alert_file_location) - - # Load alert. Parse JSON object. - try: - with open(alert_file_location) as alert_file: - json_alert = json.load(alert_file) - except: - debug("# Alert file %s doesn't exist" % alert_file_location) - - debug("# Processing alert") - try: - debug(json_alert) - except Exception as e: - debug("Failed getting json_alert %s" % e) - sys.exit(1) - - debug("# Generating message") - msg = generate_msg(json_alert) - if isinstance(msg, str): - if len(msg) == 0: - return - debug(msg) - - debug("# Sending message") - - try: - with open("/tmp/shuffle_end.txt", "w+") as tmp: - tmp.write("Script done pre-msg sending") - except: - pass - - - send_msg(msg, webhook) - - -def debug(msg): - if debug_enabled: - msg = "{0}: {1}\n".format(now, msg) - print(msg) - f = open(log_file, "a") - f.write(msg) - f.close() - -# Skips container kills to stop self-recursion -def filter_msg(alert): - # These are things that recursively happen because Shuffle starts Docker containers - skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928", "5710"] - if alert["rule"]["id"] in skip: - return False - - #try: - # if "docker" in alert["rule"]["description"].lower() and " - #msg['text'] = alert.get('full_log') - #except: - # pass - #msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" - - return True - -def generate_msg(alert): - if not filter_msg(alert): - print("Skipping rule %s" % alert["rule"]["id"]) - return "" - - level = alert['rule']['level'] - - if (level <= 4): - severity = 1 - elif (level >= 5 and level <= 7): - severity = 2 - else: - severity = 3 - - msg = {} - msg['severity'] = severity - msg['pretext'] = "WAZUH Alert" - msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" - msg['text'] = alert.get('full_log') - msg['rule_id'] = alert["rule"]["id"] - msg['timestamp'] = alert["timestamp"] - msg['id'] = alert['id'] - msg["all_fields"] = alert - - #msg['fields'] = [] - # msg['fields'].append({ - # "title": "Agent", - # "value": "({0}) - {1}".format( - # alert['agent']['id'], - # alert['agent']['name'] - # ), - # }) - #if 'agentless' in alert: - # msg['fields'].append({ - # "title": "Agentless Host", - # "value": alert['agentless']['host'], - # }) - - #msg['fields'].append({"title": "Location", "value": alert['location']}) - #msg['fields'].append({ - # "title": "Rule ID", - # "value": "{0} _(Level {1})_".format(alert['rule']['id'], level), - #}) - - #attach = {'attachments': [msg]} - - return json.dumps(msg) - - -def send_msg(msg, url): - debug("# In send msg") - headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} - res = requests.post(url, data=msg, headers=headers, verify=False) - debug("# After send msg: %s" % res) - - -if __name__ == "__main__": - try: - # Read arguments - bad_arguments = False - if len(sys.argv) >= 4: - msg = '{0} {1} {2} {3} {4}'.format( - now, - sys.argv[1], - sys.argv[2], - sys.argv[3], - sys.argv[4] if len(sys.argv) > 4 else '', - ) - #debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug') - debug_enabled = True - else: - msg = '{0} Wrong arguments'.format(now) - bad_arguments = True - - # Logging the call - try: - f = open(log_file, 'a') - except: - f = open(log_file, 'w+') - f.write("") - f.close() - - f = open(log_file, 'a') - f.write(msg + '\n') - f.close() - - if bad_arguments: - debug("# Exiting: Bad arguments. Inputted: %s" % sys.argv) - sys.exit(1) - - # Main function - main(sys.argv) - - except Exception as e: - debug(str(e)) - raise diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf deleted file mode 100755 index e87bc53d..00000000 --- a/functions/extensions/wazuh/ossec.conf +++ /dev/null @@ -1,6 +0,0 @@ - - custom-shuffle - 9 - http://:/api/v1/hooks/webhook_hookid - json - diff --git a/functions/usecases/README.md b/functions/usecases/README.md deleted file mode 100644 index 553034c2..00000000 --- a/functions/usecases/README.md +++ /dev/null @@ -1,16 +0,0 @@ -# Mindmap exporter -Shuffle has a mindmap for Workflow use-cases. These can be changed and exported, with the most important piece being that they're explorable and editable. This has and will come in handy for us as we build it into the product. - -https://www.mindmeister.com/map/2172644474 - -## Editing the Mindmap -There are a few categories. To edit them, click the small plus next to the branch you want to change. - -## Exporting the Mindmap -Click "Export as RTF" in the top left corner of the URL. Download it there. - -## Generating the Shuffle-comaptible mindmap -1. Move the rtf file here -2. Rename it categories.rtf -3. Run the read_categories.py file (python3 read_categories.py) -4. You now have a file called categories.json locally with all the categories in JSON format, ready to be used in graphs. diff --git a/functions/usecases/categories.json b/functions/usecases/categories.json deleted file mode 100644 index e438a380..00000000 --- a/functions/usecases/categories.json +++ /dev/null @@ -1,260 +0,0 @@ -[ - { - "name": "1. Collect & Distribute", - "color": "#c51152", - "list": [ - { - "name": "2-way Ticket synchronization", - "items": {} - }, - { - "name": "Email management", - "items": { - "name": "Release a quarantined message", - "items": {} - } - }, - { - "name": "EDR to ticket", - "items": { - "name": "Get host information", - "items": {} - } - }, - { - "name": "SIEM to ticket", - "items": {} - }, - { - "name": "ChatOps", - "items": {} - }, - { - "name": "Threat Intel received", - "items": {} - }, - { - "name": "Domain investigation with LetsEncrypt", - "items": {} - }, - { - "name": "Botnet tracker", - "items": {} - }, - { - "name": "Get running containers", - "items": {} - }, - { - "name": "Assign tickets", - "items": {} - }, - { - "name": "Firewall alerts", - "items": { - "name": "URL filtering", - "items": {} - } - }, - { - "name": "IDS/IPS alerts", - "items": { - "name": "Manage policies", - "items": {} - } - }, - { - "name": "Deduplicate information", - "items": {} - }, - { - "name": "Correlate information", - "items": {} - } - ] - }, - { - "name": "2. Enrich", - "color": "#f4c20d", - "list": [ - { - "name": "Internal Enrichment", - "items": { - "name": "...", - "items": {} - } - }, - { - "name": "External historical Enrichment", - "items": { - "name": "...", - "items": {} - } - }, - { - "name": "Realtime", - "items": { - "name": "Analyze screenshots", - "items": {} - } - }, - { - "name": "Ticketing webhook verification", - "items": {} - } - ] - }, - { - "name": "3. Detect", - "color": "#3cba54", - "list": [ - { - "name": "Search SIEM (Sigma)", - "items": { - "name": "Endpoint", - "items": {} - } - }, - { - "name": "Search EDR (OSQuery)", - "items": {} - }, - { - "name": "Search emails (Phish)", - "items": { - "name": "Check headers and IOCs", - "items": {} - } - }, - { - "name": "Search IOCs (ioc-finder)", - "items": {} - }, - { - "name": "Search files (Yara)", - "items": {} - }, - { - "name": "Correlate tickets", - "items": {} - }, - { - "name": "Honeypot access", - "items": { - "name": "...", - "items": {} - } - } - ] - }, - { - "name": "4. Respond", - "color": "#4a148c", - "list": [ - { - "name": "Eradicate malware", - "items": {} - }, - { - "name": "Quarantine host(s)", - "items": {} - }, - { - "name": "Trigger scans", - "items": {} - }, - { - "name": "Update indicators (FW, EDR, SIEM...)", - "items": {} - }, - { - "name": "Autoblock activity when threat intel is received", - "items": {} - }, - { - "name": "Lock/Delete/Reset account", - "items": {} - }, - { - "name": "Lock vault", - "items": {} - }, - { - "name": "Increase authentication", - "items": {} - }, - { - "name": "Get policies from assets", - "items": {} - } - ] - }, - { - "name": "5. Verify", - "color": "#4885ed", - "list": [ - { - "name": "Discover vulnerabilities", - "items": {} - }, - { - "name": "Discover assets", - "items": {} - }, - { - "name": "Ensure policies are followed", - "items": {} - }, - { - "name": "Find Inactive users", - "items": {} - }, - { - "name": "Ensure access rights match HR systems", - "items": {} - }, - { - "name": "Ensure onboarding is followed", - "items": {} - }, - { - "name": "Third party apps in SaaS", - "items": {} - }, - { - "name": "Devices used for your cloud account", - "items": {} - }, - { - "name": "Too much access in GCP/Azure/AWS/ other clouds", - "items": {} - }, - { - "name": "Certificate validation", - "items": {} - }, - { - "name": "Monitor new DNS entries for domain with passive DNS", - "items": {} - }, - { - "name": "Monitor and track password dumps", - "items": {} - }, - { - "name": "Monitor for mentions of domain on darknet sites", - "items": {} - }, - { - "name": "Reporting", - "items": { - "name": "Monthly reports", - "items": { - "name": "...", - "items": {} - } - } - } - ] - } -] \ No newline at end of file diff --git a/functions/usecases/categories.rtf b/functions/usecases/categories.rtf deleted file mode 100644 index e19cf26a..00000000 --- a/functions/usecases/categories.rtf +++ /dev/null @@ -1,555 +0,0 @@ -{\rtf1\ansi\deff0\deflang2057\plain\fs24\fet1 -{\fonttbl -{\f0\froman Arial;} -} -{\info -{\createim\yr2022\mo2\dy20\hr1\min15} -} - -\paperw11907\paperh16840\margl1800\margr1800\margt1440\margb1440 -\slmult0\ltrpar\li0 -{\b\fs28 -Shuffle categories -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -1. Collect & Distribute -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -2-way Ticket synchronization -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Email management -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Attachments -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage senders -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Encode & Decode URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Release a quarantined message -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -EDR to ticket -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Fetch incidents & events -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Quarantine files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Quarantine host (respond) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get host information -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -SIEM to ticket -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -ChatOps -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Threat Intel received -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Domain investigation with LetsEncrypt -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Botnet tracker -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Get running containers -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Assign tickets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Firewall alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Block/accept policies -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Add addresses and ports to groups -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Support custom URL categories -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Fetch logs for specific address -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URL filtering -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -IDS/IPS alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get/Fetch alerts -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Receive alerts real-time -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get PCAP files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Get network logs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Manage policies -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Deduplicate information -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Correlate information -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -3. Detect -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search SIEM (Sigma) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Network -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Endpoint -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search EDR (OSQuery) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search emails (Phish) -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check malware -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check targeted -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Check headers and IOCs -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search IOCs (ioc-finder) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Search files (Yara) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Correlate tickets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Honeypot access -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -S3 Honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -SSH Honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -FTP honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Network honeypot -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -rich -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -5. Verify -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Discover vulnerabilities -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Discover assets -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure policies are followed -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Find Inactive users -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure access rights match HR systems -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ensure onboarding is followed -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Third party apps in SaaS -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Devices used for your cloud account -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Too much access in GCP/Azure/AWS/ other clouds -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Certificate validation -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor new DNS entries for domain with passive DNS -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor and track password dumps -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Monitor for mentions of domain on darknet sites -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Reporting -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Automation time saved -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Automation money saved -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Incident response report -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Department cost -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Monthly reports -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -EDR alerts -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -SIEM alerts -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -Emails quarantined -} -\par\pard\plain -\slmult0\ltrpar\li800 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -4. Respond -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Eradicate malware -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Quarantine host(s) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Trigger scans -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Update indicators (FW, EDR, SIEM...) -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Autoblock activity when threat intel is received -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Lock/Delete/Reset account -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Lock vault -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Increase authentication -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Get policies from assets -} -\par\pard\plain -\slmult0\ltrpar\li200 -{\fs24 -2. Enrich -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Internal Enrichment -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Users -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Hostnames -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -IPs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Departments -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Role -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Software -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -External historical Enrichment -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -IPs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URLs -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Hashes -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Files -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -... -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Realtime -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -File detonation -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -URL detonation -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -PCAP analysis -} -\par\pard\plain -\slmult0\ltrpar\li600 -{\fs24 -Analyze screenshots -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 -Ticketing webhook verification -} -\par\pard\plain -} \ No newline at end of file diff --git a/functions/usecases/read_categories.py b/functions/usecases/read_categories.py deleted file mode 100644 index 65df43c9..00000000 --- a/functions/usecases/read_categories.py +++ /dev/null @@ -1,66 +0,0 @@ -data = "" -with open("categories.rtf", "r") as tmp: - data = tmp.read() - -fixed_json = [] -linearity = 0 -heading = "" -subheading = "" -subsubheading = "" - -cnt = -1 -subcnt = -1 - -colors = ["#c51152", "#3cba54", "#4885ed", "#4a148c", "#f4c20d"] -for line in data.split("\n"): - if line == "rich": - continue - - if "li" in line: - lisplit = line.split("\\") - try: - linearity = int(lisplit[-1][2]) - except: - pass - - #print("Linearity: %s" % linearity) - - if line.startswith("{") or line.startswith("}"): - continue - - if line.startswith("\\"): - continue - - if linearity == 0: - continue - - if linearity == 2: - #if cnt >= 0: - # for key, value in fixed_json[cnt].items(): - # print(key, value) - - - cnt += 1 - subcnt = -1 - fixed_json.append({"name": line, "color": colors[cnt], "list": []}) - heading = line - elif linearity == 4: - subheading = line - fixed_json[cnt]["list"].append({"name": line, "items": {}}) - subcnt += 1 - elif linearity == 6: - fixed_json[cnt]["list"][subcnt]["items"] = {"name": line, "items": {}} - elif linearity == 8: - fixed_json[cnt]["list"][subcnt]["items"]["items"] = {"name": line, "items": {}} - else: - print("No handler for %s" % line) - -#print(line) -#print(data) -import json -filename = "categories.json" -fixed_json.sort(key=lambda x: x["name"]) -with open(filename, "w+") as tmp: - tmp.write(json.dumps(fixed_json, indent=4)) - -print("Wrote to file %s" % filename)