diff --git a/.gitignore b/.gitignore index 0aa56849..5bebe49f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,9 @@ backend/go-app/generated* functions/generated_apps *.zip +*openapi-parsers/generated/* +*openapi-parsers/other/* + backend/onprem/app_sdk/apps *test.py diff --git a/README.md b/README.md index 9353cc96..8f1b8a4b 100644 --- a/README.md +++ b/README.md @@ -36,15 +36,16 @@ Documentation can be found on https://shuffler.io/docs/about or in your own inst Open an issue on Github, or [join the gitter chat](https://gitter.im/Shuffle-SOAR/community). For other / private requests: [frikky@shuffler.io](mailto:frikky@shuffler.io) ### Setup - Local development -Frontend - requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend - edit frontend/src/App.yaml (line 46~) from window.location.origin to http://YOUR IP:5001 +Frontend - requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend - edit frontend/src/App.yaml (line 44~) from window.location.origin to http://YOUR IP:5001 ```bash cd frontend npm i npm start ``` -Backend - API calls - requires [>=go1.13](https://golang.org/dl/) and [gcloud](https://cloud.google.com/sdk/install) +Backend - API calls - requires [>=go1.13](https://golang.org/dl/) ```bash +export DATASTORE_EMULATOR_HOST=0.0.0.0:8000 cd backend/go-app go build go run *.go diff --git a/backend/Dockerfile b/backend/Dockerfile index 36e3c0f2..14d9137c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -2,6 +2,7 @@ from golang as builder # Add files RUN mkdir /app +RUN mkdir /app_sdk WORKDIR /app ADD ./go-app/main.go /app ADD ./go-app/walkoff.go /app @@ -10,17 +11,26 @@ ADD ./go-app/codegen.go /app ADD ./go-app/go.mod /app +# Required files for code generation +ADD ./app_sdk/app_base.py /app_sdk +ADD ./app_sdk/static_baseline.py /app_sdk +ADD ./app_gen /app_gen + RUN go get -v RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webapp . -# Certificate build +# Certificate build - gets required certs FROM alpine:latest as certs RUN apk --update add ca-certificates from scratch -COPY --from=builder /app/ / + +COPY --from=builder /app/ /app +COPY --from=builder /app_sdk/ /app_sdk +COPY --from=builder /app_gen/ /app_gen COPY --from=certs /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +WORKDIR /app EXPOSE 5001 CMD ["./webapp"] diff --git a/backend/README.md b/backend/README.md index 8cea3c80..4a3e0599 100644 --- a/backend/README.md +++ b/backend/README.md @@ -1,61 +1,18 @@ -# Backend setup -1 Go to https://console.cloud.google.com/apis/credentials?project=shuffle-241517&folder&organizationId and get credentials -2. Move the file to current folder (or make step 3 be your download folder or w/e) -3. export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/Shuffle-2a19ff64af66.json +# Backend +This folder has all parts necessary for the backend to run locally and in Docker -# Backend run testserver (appengine) -1. Set up gcloud locally -```bash -dev_appserver.py go-app/ --port=5001 --host=0.0.0.0 --enable_host_checking=false -``` +## Structure +* go-app: The backend. Modify these to edit the backend API. +* database: The datastore database. +* app_sdk: The app_sdk for apps. MIT licensed. +* app_gen: Code used when generating docker images. MIT licensed +* tests: A bunch of cronscripts. There are no real, good tests yet -# Backend deploy -* I created a simple script that moves the data into your GOPATH and deploys for you. This will require more tests in the future. +## Development +Shuffle's backend is written in Go, with apps being python (for now). More about local development can be seen in the main README. -# OpenAPI spec checks -Paths: -* /path/{variablename}?queryvar= <-- variable -* ^variablename needs to be part of parameters too. -* ^queryvar needs to be part of parameters too. +Running the backend: ``` -parameters: -- name: variablename - in: path - description: Blah blah - required: true/false - schema: - type: string - enum: [a, b, c] # <-- not necessary, but could be great -- name: queryvar - in: query - description: blah blah - required: true/false - schema: - type: string - enum: [a, b, c] -``` -* requestBody? Not in GET, DELETE & HEAD. Can consume JSON, XML, form data, plai ntext & others. Can use markdown for the description. -* Do I care about the response? Maybe :o - -``` -requestBody: - description: Optional kind of description - required: false/true - content: - application/json: - schema: - type: object - additionalProperties: true - properties: - name: - type: string - fav_number: - type: integer - required: - - name - - email - encoding: - color: - style: form - explode: false +cd go-app +go run *.go ``` diff --git a/backend/app_gen/LICENSE b/backend/app_gen/LICENSE new file mode 100644 index 00000000..ce11f6f3 --- /dev/null +++ b/backend/app_gen/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Frikkylikeme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/app_gen/openapi-parsers/generated/carbon_black_response.yaml b/backend/app_gen/openapi-parsers/generated/carbon_black_response.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/carbon_black_response.yaml rename to backend/app_gen/openapi-parsers/generated/carbon_black_response.yaml diff --git a/app_gen/openapi-parsers/generated/cyberreason.yaml b/backend/app_gen/openapi-parsers/generated/cyberreason.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/cyberreason.yaml rename to backend/app_gen/openapi-parsers/generated/cyberreason.yaml diff --git a/app_gen/openapi-parsers/generated/recorded_future.yaml b/backend/app_gen/openapi-parsers/generated/recorded_future.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/recorded_future.yaml rename to backend/app_gen/openapi-parsers/generated/recorded_future.yaml diff --git a/app_gen/openapi-parsers/generated/shodan.yaml b/backend/app_gen/openapi-parsers/generated/shodan.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/shodan.yaml rename to backend/app_gen/openapi-parsers/generated/shodan.yaml diff --git a/app_gen/openapi-parsers/generated/tenable_tenable.io.yaml b/backend/app_gen/openapi-parsers/generated/tenable_tenable.io.yaml similarity index 100% rename from app_gen/openapi-parsers/generated/tenable_tenable.io.yaml rename to backend/app_gen/openapi-parsers/generated/tenable_tenable.io.yaml diff --git a/app_gen/openapi-parsers/misp.py b/backend/app_gen/openapi-parsers/misp.py similarity index 100% rename from app_gen/openapi-parsers/misp.py rename to backend/app_gen/openapi-parsers/misp.py diff --git a/app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json b/backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v1.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json b/backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Container-Security-v2.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Downloads-API.json b/backend/app_gen/openapi-parsers/other/TIO-API-Downloads-API.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Downloads-API.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Downloads-API.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json b/backend/app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Tenable-Platform.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json b/backend/app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Vulnerability-Management.json diff --git a/app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json b/backend/app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json similarity index 100% rename from app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json rename to backend/app_gen/openapi-parsers/other/TIO-API-Web-Application-Scanning.json diff --git a/app_gen/openapi-parsers/swimlane.py b/backend/app_gen/openapi-parsers/swimlane.py similarity index 100% rename from app_gen/openapi-parsers/swimlane.py rename to backend/app_gen/openapi-parsers/swimlane.py diff --git a/app_gen/openapi/README.md b/backend/app_gen/openapi/README.md similarity index 100% rename from app_gen/openapi/README.md rename to backend/app_gen/openapi/README.md diff --git a/app_gen/openapi/baseline/Dockerfile b/backend/app_gen/openapi/baseline/Dockerfile similarity index 100% rename from app_gen/openapi/baseline/Dockerfile rename to backend/app_gen/openapi/baseline/Dockerfile diff --git a/app_gen/openapi/baseline/requirements.txt b/backend/app_gen/openapi/baseline/requirements.txt similarity index 100% rename from app_gen/openapi/baseline/requirements.txt rename to backend/app_gen/openapi/baseline/requirements.txt diff --git a/app_gen/openapi/test.go b/backend/app_gen/openapi/test.go similarity index 100% rename from app_gen/openapi/test.go rename to backend/app_gen/openapi/test.go diff --git a/app_gen/openapi/testGCP.go b/backend/app_gen/openapi/testGCP.go similarity index 100% rename from app_gen/openapi/testGCP.go rename to backend/app_gen/openapi/testGCP.go diff --git a/app_gen/python-lib/README.md b/backend/app_gen/python-lib/README.md similarity index 100% rename from app_gen/python-lib/README.md rename to backend/app_gen/python-lib/README.md diff --git a/app_gen/python-lib/baseline/Dockerfile b/backend/app_gen/python-lib/baseline/Dockerfile similarity index 100% rename from app_gen/python-lib/baseline/Dockerfile rename to backend/app_gen/python-lib/baseline/Dockerfile diff --git a/app_gen/python-lib/baseline/docker-compose.yml b/backend/app_gen/python-lib/baseline/docker-compose.yml similarity index 100% rename from app_gen/python-lib/baseline/docker-compose.yml rename to backend/app_gen/python-lib/baseline/docker-compose.yml diff --git a/app_gen/python-lib/baseline/env.txt b/backend/app_gen/python-lib/baseline/env.txt similarity index 100% rename from app_gen/python-lib/baseline/env.txt rename to backend/app_gen/python-lib/baseline/env.txt diff --git a/app_gen/python-lib/baseline/requirements.txt b/backend/app_gen/python-lib/baseline/requirements.txt similarity index 100% rename from app_gen/python-lib/baseline/requirements.txt rename to backend/app_gen/python-lib/baseline/requirements.txt diff --git a/app_gen/python-lib/generator.py b/backend/app_gen/python-lib/generator.py similarity index 100% rename from app_gen/python-lib/generator.py rename to backend/app_gen/python-lib/generator.py diff --git a/app_gen/python-lib/requirements.txt b/backend/app_gen/python-lib/requirements.txt similarity index 100% rename from app_gen/python-lib/requirements.txt rename to backend/app_gen/python-lib/requirements.txt diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile new file mode 100644 index 00000000..44294a5b --- /dev/null +++ b/backend/app_sdk/Dockerfile @@ -0,0 +1,16 @@ +FROM python:3.7-alpine as base + +FROM base as builder +RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev + +RUN mkdir /install +WORKDIR /install + +COPY requirements.txt /requirements.txt +RUN pip install --prefix="/install" -r /requirements.txt + +FROM base + +COPY --from=builder /install /usr/local +COPY __init__.py /app/walkoff_app_sdk/__init__.py +COPY app_base.py /app/walkoff_app_sdk/app_base.py diff --git a/backend/app_sdk/LICENSE b/backend/app_sdk/LICENSE new file mode 100644 index 00000000..ce11f6f3 --- /dev/null +++ b/backend/app_sdk/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Frikkylikeme + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md new file mode 100644 index 00000000..754392ff --- /dev/null +++ b/backend/app_sdk/README.md @@ -0,0 +1,16 @@ +# app_sdk.py +This is the SDK used for apps to behave like they should. +To change it in the backend, upload it to Buckets/shuffler.appspot.com/generated_apps/baseline. + +# static_baseline.py +It's used for python code generation and should be under MIT. Has to be located here because it's used by the backend. + +## If you want to update apps.. PS: downloads from docker hub do overrides.. :) +1. Write your code & check if runtime works +2. Build app_base image +3. docker rm $(docker ps -aq) # Remove all stopped containers +4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) +5. Rebuild the Docker image (click load in GUI?) + +# LICENSE +Everything in here is MIT, not AGPLv3 as indicated by the license. diff --git a/backend/app_sdk/__init__.py b/backend/app_sdk/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py new file mode 100644 index 00000000..d15792a0 --- /dev/null +++ b/backend/app_sdk/app_base.py @@ -0,0 +1,597 @@ +import os +import sys +import re +import time +import json +import logging +import requests + +class AppBase: + """ The base class for Python-based apps in Shuffle, handles logging and callbacks configurations""" + __version__ = None + app_name = None + + def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): + self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") + self.redis=redis + self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") + + # apikey is for the user / org + # authorization is for the specific workflow + self.url = os.getenv("CALLBACK_URL", "https://shuffler.io") + self.action = os.getenv("ACTION", "") + self.apikey = os.getenv("FUNCTION_APIKEY", "") + self.authorization = os.getenv("AUTHORIZATION", "") + self.current_execution_id = os.getenv("EXECUTIONID", "") + + if len(self.action) == 0: + print("ACTION env not defined") + sys.exit(0) + if len(self.apikey) == 0: + print("FUNCTION_APIKEY env not defined") + sys.exit(0) + if len(self.authorization) == 0: + print("AUTHORIZATION env not defined") + sys.exit(0) + if len(self.current_execution_id) == 0: + print("EXECUTIONID env not defined") + sys.exit(0) + + if isinstance(self.action, str): + self.action = json.loads(self.action) + + async def execute_action(self, action): + # FIXME - add request for the function STARTING here. Use "results stream" or something + # PAUSED, AWAITING_DATA, PENDING, COMPLETED, ABORTED, EXECUTING, SUCCESS, FAILURE + + # !!! Let this line stay - its used for some horrible codegeneration / stitching !!! # + #STARTCOPY + stream_path = "/api/v1/streams" + action_result = { + "action": action, + "authorization": self.authorization, + "execution_id": self.current_execution_id, + "result": "", + "started_at": int(time.time()), + "status": "EXECUTING" + } + self.logger.info("ACTION RESULT: %s", action_result) + + headers = { + "Content-Type": "application/json", + "Authorization": "Bearer %s" % self.apikey + } + + # Add async logger + # self.console_logger.handlers[0].stream.set_execution_id() + #self.logger.info("Before initial stream result") + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Workflow: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + print("Connectionerror: %s" % e) + return + #self.logger.info("AFTER initial stream result") + self.logger.info("THIS IS THE NEW UPDATE") + + # Verify whether there are any parameters with ACTION_RESULT required + # If found, we get the full results list from backend + + fullexecution = {} + try: + tmpdata = { + "authorization": self.authorization, + "execution_id": self.current_execution_id + } + + self.logger.info("Before FULLEXEC stream result") + ret = requests.post( + "%s/api/v1/streams/results" % (self.url), + headers=headers, + json=tmpdata + ) + + if ret.status_code == 200: + fullexecution = ret.json() + else: + self.logger.info("Error: Data: ", ret.json()) + self.logger.info("Error with status code for results. Crashing because ACTION_RESULTS or WORKFLOW_VARIABLE can't be handled. Status: %d" % ret.status_code) + return + except requests.exceptions.ConnectionError as e: + self.logger.info("Connectionerror: %s" % e) + return + + self.logger.info("AFTER FULLEXEC stream result") + + # Takes a workflow execution as argument + # Returns a string if the result is single, or a list if it's a list + def get_json_value(execution_data, input_data): + parsersplit = input_data.split(".") + actionname = parsersplit[0][1:].replace(" ", "_", -1) + print(f"Actionname: {actionname}") + + # 1. Find the action + baseresult = "" + try: + if actionname.lower() == "exec": + baseresult = execution_data["execution_argument"] + else: + for result in execution_data["results"]: + resultlabel = result["action"]["label"].replace(" ", "_", -1).lower() + if resultlabel.lower() == actionname.lower(): + baseresult = result["result"] + break + + except KeyError as error: + print(f"Error: {error}") + + print(f"After first trycatch") + + # 2. Find the JSON data + if len(baseresult) == 0: + return "" + + if len(parsersplit) == 1: + return baseresult + + baseresult = baseresult.replace("\'", "\"") + basejson = {} + try: + basejson = json.loads(baseresult) + except json.decoder.JSONDecodeError as e: + return baseresult + + try: + cnt = 0 + for value in parsersplit[1:]: + cnt += 1 + + if value == "#": + # FIXME - not recursive - should go deeper if there are more # + print("HANDLE RECURSIVE LOOP ") + returnlist = [] + for innervalue in basejson: + #print("Value: %s" % value[parsersplit[cnt+1]]) + returnlist.append(innervalue[parsersplit[cnt+1]]) + + # Example format: ${[]}$ + return "${%s%s}$" % (parsersplit[cnt+1], json.dumps(returnlist)) + + else: + if isinstance(basejson[value], str): + print(f"LOADING STRING '%s' AS JSON" % basejson[value]) + try: + basejson = json.loads(basejson[value]) + except json.decoder.JSONDecodeError as e: + print("RETURNING BECAUSE '%s' IS A NORMAL STRING" % basejson[value]) + return basejson[value] + else: + basejson = basejson[value] + + except KeyError as e: + return "KeyError: %s" % e + except IndexError as e: + return "IndexError: %s" % e + + return basejson + + def parse_params(action, fullexecution, parameter): + jsonparsevalue = "$." + match = ".*([$]{1}([a-zA-Z0-9()# _-]+\.?){1,})" + + # Regex to find all the things + if parameter["variant"] == "STATIC_VALUE": + data = parameter["value"] + self.logger.debug(f"\n\nHandle static data with JSON: {data}\n\n") + + actualitem = re.findall(match, data, re.MULTILINE) + self.logger.info("PARSED: %s" % actualitem) + if len(actualitem) > 0: + for replace in actualitem: + try: + to_be_replaced = replace[0] + except IndexError: + continue + + value = get_json_value(fullexecution, to_be_replaced) + if isinstance(value, str): + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + elif isinstance(value, dict): + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + else: + print("Unknown type %s" % type(value)) + try: + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + except json.decoder.JSONDecodeError as e: + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + + + if parameter["variant"] == "WORKFLOW_VARIABLE": + for item in fullexecution["workflow"]["workflow_variables"]: + if parameter["action_field"] == item["name"]: + parameter["value"] = item["value"] + break + + elif parameter["variant"] == "ACTION_RESULT": + # FIXME - calculate value based on action_field and $if prominent + # FIND THE RIGHT LABEL + # GET THE LABEL'S RESULT + + tmpvalue = "" + self.logger.info("ACTION FIELD: %s" % parameter["action_field"]) + + #"$%s%s" % + fullname = "$" + + if parameter["action_field"] == "Execution Argument": + tmpvalue = fullexecution["execution_argument"] + fullname += "exec" + else: + fullname += parameter["action_field"] + + if parameter["value"].startswith(jsonparsevalue): + fullname += parameter["value"][2:] + else: + fullname = "$%s" % parameter["action_field"] + + self.logger.info("Fullname: %s" % fullname) + actualitem = re.findall(match, fullname, re.MULTILINE) + self.logger.info("PARSED: %s" % actualitem) + if len(actualitem) > 0: + for replace in actualitem: + try: + to_be_replaced = replace[0] + except IndexError: + print("Nothing to replace?: " % e) + continue + + # This will never be a loop aka multi argument + parameter["value"] = to_be_replaced + + value = get_json_value(fullexecution, to_be_replaced) + if isinstance(value, str): + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + elif isinstance(value, dict): + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + else: + print("Unknown type %s" % type(value)) + try: + parameter["value"] = parameter["value"].replace(to_be_replaced, json.dumps(value)) + except json.decoder.JSONDecodeError as e: + parameter["value"] = parameter["value"].replace(to_be_replaced, value) + + return "", parameter["value"] + + def run_validation(sourcevalue, check, destinationvalue): + self.logger.info("Checking %s %s %s" % (sourcevalue, check, destinationvalue)) + + if check == "=" or check.lower() == "equals": + if sourcevalue.lower() == destinationvalue.lower(): + return True + elif check == "!=" or check.lower() == "does not equal": + if sourcevalue.lower() != destinationvalue.lower(): + return True + elif check.lower() == "startswith": + if sourcevalue.lower().startswith(destinationvalue.lower()): + return True + elif check.lower() == "endswith": + if sourcevalue.lower().endswith(destinationvalue.lower()): + return True + elif check.lower() == "contains": + if destinationvalue.lower() in sourcevalue.lower(): + return True + else: + self.logger.info("Condition: can't handle %s yet. Setting to true" % check) + + return False + + def check_branch_conditions(action, fullexecution): + # relevantbranches = workflow.branches where destination = action + try: + if fullexecution["workflow"]["branches"] == None or len(fullexecution["workflow"]["branches"]) == 0: + return True, "" + except KeyError: + return True, "" + + relevantbranches = [] + for branch in fullexecution["workflow"]["branches"]: + if branch["destination_id"] != action["id"]: + continue + + self.logger.info("Relevant branch: %s" % branch) + + # Remove anything without a condition + try: + if (branch["conditions"]) == 0 or branch["conditions"] == None: + continue + except KeyError: + continue + + self.logger.info("Relevant conditions: %s" % branch["conditions"]) + successful_conditions = [] + failed_conditions = [] + for condition in branch["conditions"]: + self.logger.info("Getting condition value of %s" % condition) + + # Parse all values first here + sourcevalue = condition["source"]["value"] + if condition["source"]["variant"] == "" or condition["source"]["variant"]== "STATIC_VALUE": + condition["source"]["variant"]= "STATIC_VALUE" + else: + check, sourcevalue = parse_params(action, fullexecution, condition["source"]) + if check: + return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + + print(sourcevalue) + destinationvalue = condition["destination"]["value"] + + if condition["destination"]["variant"]== "" or condition["destination"]["variant"]== "STATIC_VALUE": + condition["destination"]["variant"] = "STATIC_VALUE" + else: + check, destinationvalue = parse_params(action, fullexecution, condition["destination"]) + if check: + return False, "Failed condition: %s %s %s because %s" % (sourcevalue, condition["condition"]["value"], destinationvalue, check) + + available_checks = [ + "=", + "equals", + "!=", + "does not equal", + ">", + "larger than", + "<", + "less than", + ">=", + "<=", + "startswith", + "endswith", + "contains", + "re", + "matches regex", + ] + + # FIXME - what should I do here? + if not condition["condition"]["value"] in available_checks: + self.logger.info("Skipping %s %s %s because %s is invalid." % (sourcevalue, condition["condition"]["value"], destinationvalue, condition["condition"]["value"])) + continue + + #print(destinationvalue) + if not run_validation(sourcevalue, condition["condition"]["value"], destinationvalue): + self.logger.info("Failed condition check for %s %s %s." % (sourcevalue, condition["condition"]["value"], destinationvalue)) + return False, "Failed condition: %s %s %s" % (sourcevalue, condition["condition"]["value"], destinationvalue) + + + # Make a general parser here, at least to get param["name"] = param["value"] in maparameter[string]string + #for condition in branch.conditons: + + return True, "" + + # Checks whether conditions are met, otherwise set + branchcheck, tmpresult = check_branch_conditions(action, fullexecution) + if not branchcheck: + self.logger.info("Failed one or more branch conditions.") + action_result["result"] = tmpresult + action_result["status"] = "SKIPPED" + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + self.logger.exception(e) + + return + + # Replace name cus there might be issues + # Not doing lower() as there might be user-made functions + actionname = action["name"] + if " " in actionname: + actionname.replace(" ", "_", -1) + #if action.generated: + # actionname = actionname.lower() + + # Runs the actual functions + try: + func = getattr(self, actionname, None) + if func == None: + self.logger.debug("Failed executing %s because func is None." % actionname) + action_result["status"] = "FAILURE" + action_result["result"] = "Function %s doesn't exist." % actionname + elif callable(func): + try: + if len(action["parameters"]) < 1: + result = await func() + else: + # Potentially parse JSON here + # FIXME - add potential authentication as first parameter(s) here + # params[parameter["name"]] = parameter["value"] + #print(fullexecution["authentication"] + # What variables are necessary here tho hmm + + params = {} + try: + for item in action["authentication"]: + print("AUTH: ", key, value) + params[item["key"]] = item["value"] + except KeyError: + print("No authentication specified!") + pass + #action["authentication"] + + # calltimes is used to handle forloops in the app itself. + # 2 kinds of loop - one in gui with one app each, and one like this, + # which is super fast, but has a bad overview (potentially good tho) + calltimes = 1 + result = "" + + all_executions = [] + + # Multi_parameter has the data for each. variable + minlength = 0 + multi_parameters = json.loads(json.dumps(params)) + multiexecution = False + for parameter in action["parameters"]: + check, value = parse_params(action, fullexecution, parameter) + if check: + raise Exception(check) + + # Custom format for ${name[0,1,2,...]}$ + submatch = "([${]{2}([0-9a-zA-Z_-]+)(\[.*\])[}$]{2})" + actualitem = re.findall(submatch, value, re.MULTILINE) + if len(actualitem) > 0: + multiexecution = True + + # This is here to handle for loops within variables.. kindof + # 1. Find the length of the longest array + # 2. Build an array with the base values based on parameter["value"] + # 3. Get the n'th value of the generated list from values + # 4. Execute all n answers + replacements = {} + for replace in actualitem: + try: + to_be_replaced = replace[0] + actualitem = replace[2] + except IndexError: + continue + + itemlist = json.loads(actualitem) + if len(itemlist) > minlength: + minlength = len(itemlist) + + replacements[to_be_replaced] = actualitem + + # This is a result array for JUST this value.. + # What if there are more? + resultarray = [] + for i in range(0, minlength): + tmpitem = json.loads(json.dumps(parameter["value"])) + for key, value in replacements.items(): + replacement = json.loads(value)[i] + tmpitem = tmpitem.replace(key, replacement, -1) + + resultarray.append(tmpitem) + + # With this parameter ready, add it to... a greater list of parameters. Rofl + multi_parameters[parameter["name"]] = resultarray + else: + print("Hello, in here?: %s" % value) + params[parameter["name"]] = value + multi_parameters[parameter["name"]] = value + + # FIXME - this is horrible, but works for now + #for i in range(calltimes): + if not multiexecution: + print("Params: %s" % params) + print("RUNNING NORMAL EXECUTION") + result += await func(**params) + else: + print("MULTI EXECUTION: ", multi_parameters) + # 1. Use number of executions based on longest array + # 2. Find the right value from the parsed multi_params + + results = [] + json_object = False + for i in range(0, minlength): + # To be able to use the results as a list: + baseparams = json.loads(json.dumps(multi_parameters)) + + try: + for key, value in baseparams.items(): + if isinstance(value, list): + baseparams[key] = value[i] + except IndexError as e: + print("IndexError: %s" % e) + baseparams[key] = "IndexError: %s" % e + except KeyError as e: + print("KeyError: %s" % e) + baseparams[key] = "KeyError: %s" % e + + #print("Running with params %s" % baseparams) + ret = await func(**baseparams) + print("Inner ret: %s" % ret) + + try: + results.append(json.loads(ret)) + json_object = True + except json.decoder.JSONDecodeError as e: + results.append(ret) + + # Dump the result as a string of a list + print("RESULTS: %s" % results) + if isinstance(results, list): + print("JSON OBJECT? ", json_object) + if json_object: + result = json.dumps(results) + else: + result = "[\""+"\", \"".join(results)+"\"]" + else: + print("Normal result?") + result = results + + print("RESULT: %s" % result) + + action_result["status"] = "SUCCESS" + action_result["result"] = str(result) + if action_result["result"] == "": + action_result["result"] = result + + self.logger.debug(f"Executed {action['label']}-{action['id']} with result: {result}") + self.logger.debug(f"Data: %s" % action_result) + except TypeError as e: + action_result["status"] = "FAILURE" + action_result["result"] = "TypeError: %s" % str(e) + else: + print("Not callable?") + self.logger.error(f"App {self.__class__.__name__}.{action['name']} is not callable") + action_result["status"] = "FAILURE" + action_result["result"] = "Function %s is not callable." % actionname + + except Exception as e: + print(f"Failed to execute: {e}") + self.logger.exception(f"Failed to execute {e}-{action['id']}") + action_result["status"] = "FAILURE" + action_result["result"] = "Exception: %s" % e + + action_result["completed_at"] = int(time.time()) + + # I wonder if this actually works + self.logger.info("Before last stream result") + try: + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + except requests.exceptions.ConnectionError as e: + self.logger.exception(e) + return + except TypeError as e: + self.logger.exception(e) + action_result["status"] = "FAILURE" + action_result["result"] = "POST error: %s" % e + self.logger.info("Before typeerror stream result") + ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + self.logger.info("Result: %d" % ret.status_code) + if ret.status_code != 200: + self.logger.info(ret.text) + + return + + + #STOPCOPY + # !!! Let the above line stay - its used for some horrible codegeneration / stitching !!! # + + @classmethod + async def run(cls): + """ Connect to Redis and HTTP session, await actions """ + logging.basicConfig(format="{asctime} - {name} - {levelname}:{message}", style='{') + logger = logging.getLogger(f"{cls.__name__}") + logger.setLevel(logging.DEBUG) + + app = cls(redis=None, logger=logger, console_logger=logger) + + # Authorization for the app/function to control the workflow + # Function will crash if its wrong, which it probably should. + + await app.execute_action(app.action) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh new file mode 100644 index 00000000..c93c83fc --- /dev/null +++ b/backend/app_sdk/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash +docker rmi frikky/shuffle:app_sdk +docker build . -t frikky/shuffle:app_sdk --no-cache +docker push frikky/shuffle:app_sdk diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt new file mode 100644 index 00000000..804abb1b --- /dev/null +++ b/backend/app_sdk/requirements.txt @@ -0,0 +1,2 @@ +requests +urllib3 diff --git a/functions/static_baseline.py b/backend/app_sdk/static_baseline.py similarity index 100% rename from functions/static_baseline.py rename to backend/app_sdk/static_baseline.py diff --git a/backend/build.sh b/backend/build.sh new file mode 100644 index 00000000..bb1f4a28 --- /dev/null +++ b/backend/build.sh @@ -0,0 +1,14 @@ +#!/bin/sh +docker stop shuffle-backend +docker rm shuffle-backend +docker rmi frikky/shuffle:backend + +docker build . -t frikky/shuffle:backend +docker push frikky/shuffle:backend + +echo "Starting server" +#docker run -it \ +# -p 5001:5001 \ +# -v /var/run/docker.sock:/var/run/docker.sock \ +# --env DATASTORE_EMULATOR_HOST=192.168.3.6:8000 \ +# frikky/shuffle:backend diff --git a/backend/deploy-backend.sh b/backend/deploy-backend.sh deleted file mode 100644 index bb51e6de..00000000 --- a/backend/deploy-backend.sh +++ /dev/null @@ -1,8 +0,0 @@ -# Deploys to backend -echo "Deploying to appengine." -mkdir -p $GOPATH/src/github.com/frikky/shuffle -cp -r go-app/* $GOPATH/src/github.com/frikky/shuffle -cd $GOPATH/src/github.com/frikky/shuffle -go build -go test -gcloud app deploy $GOPATH/src/github.com/frikky/shuffle/app.yaml diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index e737c426..abe03fa8 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -122,8 +122,8 @@ func streamZipdata(ctx context.Context, identifier, pythoncode, requirements str func getAppbase() ([]byte, []byte, error) { // 1. Have baseline in bucket/generated_apps/baseline // 2. Copy the baseline to a new folder with identifier name - static := "../../functions/static_baseline.py" - appbase := "../../functions/onprem/app_sdk/app_base.py" + static := "../app_sdk/static_baseline.py" + appbase := "../app_sdk/app_base.py" staticData, err := ioutil.ReadFile(static) if err != nil { @@ -218,7 +218,7 @@ func buildStructure(swagger *openapi3.Swagger, curHash string) (string, error) { // adding md5 based on input data to not overwrite earlier data. generatedPath := "generated" - subpath := "../../app_gen/openapi/" + subpath := "../app_gen/openapi/" identifier := fmt.Sprintf("%s-%s", swagger.Info.Title, curHash) appPath := fmt.Sprintf("%s/%s", generatedPath, identifier) @@ -361,12 +361,13 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri api.Tested = false api.PrivateID = newmd5 api.Generated = true + api.Activated = true // Setting up security schemes extraParameters := []WorkflowAppActionParameter{} securitySchemes := swagger.Components.SecuritySchemes if securitySchemes != nil { - log.Printf("%#v", securitySchemes) + //log.Printf("%#v", securitySchemes) api.Authentication = Authentication{ Required: true, @@ -386,7 +387,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri api.Authentication.Parameters[0].Name = securitySchemes["BearerAuth"].Value.Name api.Authentication.Parameters[0].In = securitySchemes["BearerAuth"].Value.In api.Authentication.Parameters[0].Scheme = securitySchemes["BearerAuth"].Value.Scheme - log.Printf("HANDLE BEARER AUTH") + //log.Printf("HANDLE BEARER AUTH") extraParameters = append(extraParameters, WorkflowAppActionParameter{ Name: "apikey", Description: "The apikey to use", @@ -402,7 +403,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (WorkflowApp, []stri api.Authentication.Parameters[0].Name = securitySchemes["ApiKeyAuth"].Value.Name api.Authentication.Parameters[0].In = securitySchemes["ApiKeyAuth"].Value.In api.Authentication.Parameters[0].Scheme = securitySchemes["ApiKeyAuth"].Value.Scheme - log.Printf("HANDLE APIKEY AUTH") + //log.Printf("HANDLE APIKEY AUTH") extraParameters = append(extraParameters, WorkflowAppActionParameter{ Name: "apikey", Description: "The apikey to use", @@ -704,6 +705,9 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ optionalParameters := []WorkflowAppActionParameter{} if len(path.Connect.Parameters) > 0 { for _, param := range path.Connect.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -773,8 +777,6 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor Parameters: extraParameters, } - log.Printf("FUNCTION: %#v", action) - action.Returns.Schema.Type = "string" baseUrl := fmt.Sprintf("%s%s", api.Link, actualPath) @@ -791,6 +793,11 @@ func handleGet(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor optionalParameters := []WorkflowAppActionParameter{} if len(path.Get.Parameters) > 0 { for _, param := range path.Get.Parameters { + //log.Printf("TYPE: %#v", param.Value.Schema) + if param.Value.Schema == nil { + continue + } + curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -873,6 +880,9 @@ func handleHead(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo optionalParameters := []WorkflowAppActionParameter{} if len(path.Head.Parameters) > 0 { for _, param := range path.Head.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -955,6 +965,9 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] optionalParameters := []WorkflowAppActionParameter{} if len(path.Delete.Parameters) > 0 { for _, param := range path.Delete.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1013,7 +1026,7 @@ func handleDelete(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [] func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []WorkflowAppActionParameter, path *openapi3.PathItem, actualPath string, firstQuery bool) (WorkflowAppAction, string) { // What to do with this, hmm - log.Printf("PATH: %s", actualPath) + //log.Printf("PATH: %s", actualPath) functionName := fixFunctionName(path.Post.Summary, actualPath) action := WorkflowAppAction{ @@ -1049,6 +1062,9 @@ func handlePost(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wo } if len(path.Post.Parameters) > 0 { for _, param := range path.Post.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1142,6 +1158,9 @@ func handlePatch(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []W } if len(path.Patch.Parameters) > 0 { for _, param := range path.Patch.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, @@ -1235,6 +1254,9 @@ func handlePut(swagger *openapi3.Swagger, api WorkflowApp, extraParameters []Wor } if len(path.Put.Parameters) > 0 { for _, param := range path.Put.Parameters { + if param.Value.Schema == nil { + continue + } curParam := WorkflowAppActionParameter{ Name: param.Value.Name, Description: param.Value.Description, diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 3b413141..dc73b5c4 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5148,6 +5148,121 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { resp.Write(urlbody) } +func handleSwaggerValidation(body []byte) (ParsedOpenApi, error) { + type versionCheck struct { + Swagger string `datastore:"swagger" json:"swagger" yaml:"swagger"` + SwaggerVersion string `datastore:"swaggerVersion" json:"swaggerVersion" yaml:"swaggerVersion"` + OpenAPI string `datastore:"openapi" json:"openapi" yaml:"openapi"` + } + + //body = []byte(`swagger: "2.0"`) + //body = []byte(`swagger: '1.0'`) + //newbody := string(body) + //newbody = strings.TrimSpace(newbody) + //body = []byte(newbody) + //log.Println(string(body)) + //tmpbody, err := yaml.YAMLToJSON(body) + //log.Println(err) + //log.Println(string(tmpbody)) + + // This has to be done in a weird way because Datastore doesn't + // support map[string]interface and similar (openapi3.Swagger) + var version versionCheck + + parsed := ParsedOpenApi{} + swaggerdata := []byte{} + idstring := "" + + isJson := false + err := json.Unmarshal(body, &version) + if err != nil { + //log.Printf("Json err: %s", err) + err = yaml.Unmarshal(body, &version) + if err != nil { + log.Printf("Yaml error: %s", err) + } else { + //log.Printf("Successfully parsed YAML!") + } + } else { + isJson = true + log.Printf("Successfully parsed JSON!") + } + + if len(version.SwaggerVersion) > 0 && len(version.Swagger) == 0 { + version.Swagger = version.SwaggerVersion + } + + if strings.HasPrefix(version.Swagger, "3.") || strings.HasPrefix(version.OpenAPI, "3.") { + //log.Println("Handling v3 API") + swaggerv3, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData(body) + if err != nil { + return ParsedOpenApi{}, err + } + + swaggerdata, err = json.Marshal(swaggerv3) + if err != nil { + log.Printf("Failed unmarshaling v3 data: %s", err) + return ParsedOpenApi{}, err + } + + hasher := md5.New() + hasher.Write(swaggerdata) + idstring = hex.EncodeToString(hasher.Sum(nil)) + + } else { //strings.HasPrefix(version.Swagger, "2.") || strings.HasPrefix(version.OpenAPI, "2.") { + // Convert + //log.Println("Handling v2 API") + var swagger openapi2.Swagger + //log.Println(string(body)) + err = json.Unmarshal(body, &swagger) + if err != nil { + //log.Printf("Json error? %s", err) + err = gyaml.Unmarshal(body, &swagger) + if err != nil { + log.Printf("Yaml error: %s", err) + return ParsedOpenApi{}, err + } else { + //log.Printf("Valid yaml!") + } + + } + + swaggerv3, err := openapi2conv.ToV3Swagger(&swagger) + if err != nil { + log.Printf("Failed converting from openapi2 to 3: %s", err) + return ParsedOpenApi{}, err + } + + swaggerdata, err = json.Marshal(swaggerv3) + if err != nil { + log.Printf("Failed unmarshaling v3 data: %s", err) + return ParsedOpenApi{}, err + } + + hasher := md5.New() + hasher.Write(swaggerdata) + idstring = hex.EncodeToString(hasher.Sum(nil)) + } + + if len(swaggerdata) > 0 { + body = swaggerdata + } + + // Overwrite with new json data + _ = isJson + body = swaggerdata + + // Parsing it to swagger 3 + parsed = ParsedOpenApi{ + ID: idstring, + Body: string(body), + Success: true, + } + + return parsed, err +} + +// FIXME: Migrate this to use handleSwaggerValidation() func validateSwagger(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -5418,7 +5533,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("Should generate yaml") + //log.Printf("Should generate yaml") api, pythonfunctions, err := generateYaml(swagger, newmd5) if err != nil { log.Printf("Failed building and generating yaml: %s", err) @@ -5569,14 +5684,6 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { return } - log.Println(len(user.PrivateApps)) - c, err := request.Cookie("session_token") - if err == nil { - log.Printf("Should've deleted cache for %s with token %s", user.Username, c.Value) - //err = memcache.Delete(request.Context(), c.Value) - //err = memcache.Delete(request.Context(), user.ApiKey) - } - parsed := ParsedOpenApi{ ID: api.ID, Body: string(body), @@ -5656,7 +5763,7 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed getting apps: %s", err) } else if err == nil && len(workflowapps) == 0 { - log.Printf("Apps: loading TEST") + log.Printf("Downloading default workflow apps") fs := memfs.New() storer := memory.NewStorage() @@ -5696,6 +5803,30 @@ func runInit(ctx context.Context) { iterateAppGithubFolders(fs, dir, "", "") } + log.Printf("Downloading OpenAPI data for search - EXTRA APPS") + apis := "https://github.com/frikky/OpenAPI-security-definitions" + + // THis gets memory problems hahah + //apis := "https://github.com/APIs-guru/openapi-directory" + fs := memfs.New() + storer := memory.NewStorage() + cloneOptions := &git.CloneOptions{ + URL: apis, + } + _, err = git.Clone(storer, fs, cloneOptions) + if err != nil { + log.Printf("Failed loading repo %s into memory: %s", err) + } else { + log.Printf("Finished git clone. Looking for updates to the repo.") + dir, err := fs.ReadDir("") + if err != nil { + log.Printf("Failed reading folder: %s", err) + } + + iterateOpenApiGithub(fs, dir, "", "") + log.Printf("Finished downloading extra API samples") + } + log.Printf("Finished INIT") } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 9c039655..07ae2c18 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -29,6 +29,7 @@ import ( http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" newscheduler "github.com/carlescere/scheduler" + "github.com/getkin/kin-openapi/openapi3" "github.com/go-git/go-git/v5/storage/memory" //"github.com/gorilla/websocket" //"google.golang.org/appengine" @@ -85,10 +86,11 @@ type WorkflowApp struct { Downloaded bool `json:"downloaded" yaml:"downloaded" required:false datastore:"downloaded"` Sharing bool `json:"sharing" yaml:"sharing" required:false datastore:"sharing"` Verified bool `json:"verified" yaml:"verified" required:false datastore:"verified"` + Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` Tested bool `json:"tested" yaml:"tested" required:false datastore:"tested"` Owner string `json:"owner" datastore:"owner" yaml:"owner"` PrivateID string `json:"private_id" yaml:"private_id" required:false datastore:"private_id"` - Description string `json:"description" datastore:"description" required:false yaml:"description"` + Description string `json:"description" datastore:"description,noindex" required:false yaml:"description"` Environment string `json:"environment" datastore:"environment" required:true yaml:"environment"` SmallImage string `json:"small_image" datastore:"small_image,noindex" required:false yaml:"small_image"` LargeImage string `json:"large_image" datastore:"large_image,noindex" yaml:"large_image" required:false` @@ -636,10 +638,6 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { return } - //for _, action := range workflowExecution.Workflow.Actions { - // log.Printf("Name: %s, Env: %s", action.Name, action.Environment) - //} - newjson, err := json.Marshal(workflowExecution) if err != nil { resp.WriteHeader(401) @@ -2702,7 +2700,6 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { } location := strings.Split(request.URL.String(), "/") - log.Printf("%#v", location) var fileId string if location[1] == "api" { if len(location) <= 4 { @@ -2730,14 +2727,22 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { return } + log.Printf("Getting app %s", fileId) parsedApi, err := getOpenApiDatastore(ctx, fileId) if err != nil { + log.Printf("OpenApi doesn't exist for: %s - err: %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return } - parsedApi.Success = true + //log.Printf("Parsed API: %#v", parsedApi) + if len(parsedApi.ID) > 0 { + parsedApi.Success = true + } else { + parsedApi.Success = false + } + data, err := json.Marshal(parsedApi) if err != nil { resp.WriteHeader(422) @@ -2917,6 +2922,7 @@ func handleGetfile(resp http.ResponseWriter, request *http.Request) ([]byte, err return buf.Bytes(), nil } +// Basically a search for apps that aren't activated yet func getSpecificApps(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -2956,10 +2962,40 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { // FIXME - continue the search here with github repos etc. // Caching might be smart :D - log.Printf("Body: %s", string(body)) + ctx := context.Background() + workflowapps, err := getAllWorkflowApps(ctx) + if err != nil { + log.Printf("Error: Failed getting workflowapps: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + returnValues := []WorkflowApp{} + search := strings.ToLower(tmpBody.Search) + for _, app := range workflowapps { + if !app.Activated && app.Generated { + // This might be heavy with A LOT + // Not too worried with todays tech tbh.. + appName := strings.ToLower(app.Name) + appDesc := strings.ToLower(app.Description) + if strings.Contains(appName, search) || strings.Contains(appDesc, search) { + //log.Printf("Name: %s, Generated: %s, Activated: %s", app.Name, strconv.FormatBool(app.Generated), strconv.FormatBool(app.Activated)) + returnValues = append(returnValues, app) + } + } + } + + newbody, err := json.Marshal(returnValues) + if err != nil { + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed unpacking workflow executions"}`))) + return + } + + returnData := fmt.Sprintf(`{"success": true, "reason": %s}`, string(newbody)) resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) + resp.Write([]byte(returnData)) } func validateAppInput(resp http.ResponseWriter, request *http.Request) { @@ -3257,6 +3293,133 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) } +func iterateOpenApiGithub(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { + + ctx := context.Background() + workflowapps, err := getAllWorkflowApps(ctx) + appCounter := 0 + if err != nil { + log.Printf("Failed to get existing generated apps") + } + for _, file := range dir { + if len(onlyname) > 0 && file.Name() != onlyname { + continue + } + + // Folder? + switch mode := file.Mode(); { + case mode.IsDir(): + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + dir, err := fs.ReadDir(tmpExtra) + if err != nil { + log.Printf("Failed to read dir: %s", err) + break + } + + // Go routine? Hmm, this can be super quick I guess + err = iterateOpenApiGithub(fs, dir, tmpExtra, "") + if err != nil { + break + } + case mode.IsRegular(): + // Check the file + filename := file.Name() + if strings.Contains(filename, "yaml") || strings.Contains(filename, "yml") { + appCounter += 1 + //log.Printf("File: %s", filename) + //log.Printf("Found file: %s", filename) + tmpExtra := fmt.Sprintf("%s%s/", extra, file.Name()) + + fileReader, err := fs.Open(tmpExtra) + if err != nil { + continue + } + + readFile, err := ioutil.ReadAll(fileReader) + if err != nil { + log.Printf("Filereader error yaml: %s", err) + continue + } + + // 1. This parses OpenAPI v2 to v3 etc, for use. + parsedOpenApi, err := handleSwaggerValidation(readFile) + if err != nil { + log.Printf("Validation error: %s", err) + continue + } + + // 2. With parsedOpenApi.ID: + //http://localhost:3000/apps/new?id=06b1376f77b0563a3b1747a3a1253e88 + + // 3. Load this as a "standby" app + // FIXME: This should be a function ROFL + //log.Printf("%s", string(readFile)) + swagger, err := openapi3.NewSwaggerLoader().LoadSwaggerFromData([]byte(parsedOpenApi.Body)) + if err != nil { + log.Printf("Swagger validation error in loop (%s): %s", filename, err) + continue + } + + if strings.Contains(swagger.Info.Title, " ") { + strings.Replace(swagger.Info.Title, " ", "", -1) + } + + //log.Printf("Should generate yaml") + api, _, err := generateYaml(swagger, parsedOpenApi.ID) + if err != nil { + log.Printf("Failed building and generating yaml in loop (%s): %s", filename, err) + continue + } + + // FIXME: Configure user? + api.Owner = "" + api.ID = parsedOpenApi.ID + api.IsValid = true + api.Generated = true + api.Activated = false + + found := false + for _, app := range workflowapps { + if app.ID == api.ID { + found = true + break + } else if app.Name == api.Name && app.AppVersion == api.AppVersion { + found = true + break + } + } + + if !found { + err = setWorkflowAppDatastore(ctx, api, api.ID) + if err != nil { + log.Printf("Failed setting workflowapp in loop: %s", err) + continue + } else { + log.Printf("Added %s:%s to the database from OpenAPI repo", api.Name, api.AppVersion) + + // Set OpenAPI datastore + err = setOpenApiDatastore(ctx, parsedOpenApi.ID, parsedOpenApi) + if err != nil { + log.Printf("Failed uploading openapi to datastore in loop: %s", err) + continue + } + } + } else { + //log.Printf("Skipped upload of %s (%s)", api.Name, api.ID) + } + + //return nil + } + } + } + + if appCounter > 0 { + log.Printf("Preloaded %d OpenApi apps in %s!", appCounter, extra) + } + + return nil +} + // Onlyname is used to func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string) error { var err error @@ -3475,6 +3638,7 @@ func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { workflowapp.ID = uuid.NewV4().String() workflowapp.IsValid = true workflowapp.Generated = false + workflowapp.Activated = true err = setWorkflowAppDatastore(ctx, workflowapp, workflowapp.ID) if err != nil { @@ -3543,7 +3707,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { } // Query for the specifci workflowId - q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Limit(50) + q := datastore.NewQuery("workflowexecution").Filter("workflow_id =", fileId).Order("-started_at").Limit(50) var workflowExecutions []WorkflowExecution _, err = dbclient.GetAll(ctx, q, &workflowExecutions) if err != nil { diff --git a/backend/run.sh b/backend/run.sh deleted file mode 100644 index edfe195c..00000000 --- a/backend/run.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -# docker stop nginx -# docker rm nginx -# docker rmi nginx -# -# echo "Running build for website" -# sudo npm run build -# docker build . -t nginx - -echo "Starting server" -docker run -it \ - -p 5001:5001 \ - -v /var/run/docker.sock:/var/run/docker.sock \ - --env DATASTORE_EMULATOR_HOST=192.168.3.6:8000 \ - frikky/shuffle:backend diff --git a/backend/webhook/Dockerfile b/backend/webhook/Dockerfile deleted file mode 100644 index 4b709eb5..00000000 --- a/backend/webhook/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -# Build environment -# production environment -from golang as builder - -RUN go get github.com/gorilla/handlers -RUN go get github.com/gorilla/mux - -WORKDIR /app -COPY webhook.go /app/webhook.go -RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o webhook . - -from scratch -COPY --from=builder /app/ / - -CMD ["./webhook"] diff --git a/backend/webhook/README.md b/backend/webhook/README.md deleted file mode 100644 index 8b87bffb..00000000 --- a/backend/webhook/README.md +++ /dev/null @@ -1,7 +0,0 @@ -# Steps to deploy to Google cloud function -1. -```bash -zip webhook.zip * -``` -2. Go to google cloud bucket and upload the zip -3. Go to worker for webhook and upload diff --git a/backend/webhook/functionhook.go b/backend/webhook/functionhook.go deleted file mode 100644 index 2df691b0..00000000 --- a/backend/webhook/functionhook.go +++ /dev/null @@ -1,44 +0,0 @@ -package function - -import ( - "encoding/json" - "io/ioutil" - "log" - "net/http" - "time" -) - -// GetUserDetails - Get one user's details from randomuser.me API -func GetUserDetails(w http.ResponseWriter, r *http.Request) { - randomUserClient := http.Client{ - Timeout: time.Second * 3, - } - - req, err := http.NewRequest(http.MethodGet, "https://randomuser.me/api/", nil) - if err != nil { - log.Fatal(err) - return - } - - res, err2 := randomUserClient.Do(req) - if err2 != nil { - log.Fatal(err2) - return - } - - body, err3 := ioutil.ReadAll(res.Body) - if err3 != nil { - log.Fatal(err3) - } - - var o map[string]interface{} - json.Unmarshal([]byte(body), &o) - - results := o["results"].([]interface{}) - result := results[0].(map[string]interface{}) - - result["generator"] = "google-cloud-function" - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(result) -} diff --git a/backend/webhook/gcp_run.sh b/backend/webhook/gcp_run.sh deleted file mode 100644 index 605777b5..00000000 --- a/backend/webhook/gcp_run.sh +++ /dev/null @@ -1,4 +0,0 @@ -docker build . -t gcr.io/shuffle-241517/webhook -docker push gcr.io/shuffle-241517/webhook - -gcloud beta run deploy webhook --image gcr.io/shuffle-241517/webhook diff --git a/backend/webhook/run.sh b/backend/webhook/run.sh deleted file mode 100644 index c9914de7..00000000 --- a/backend/webhook/run.sh +++ /dev/null @@ -1,16 +0,0 @@ -docker stop webhook -docker rm webhook -docker rmi webhook - -docker build . -t webhook -docker run -d \ - -e "HOOKPORT=5001" \ - -e "URIPATH=/webhook" \ - -e "CALLBACKURL=http://192.168.3.6:5001/api/v1/hooks/d6ef8912e8bd37776e654cbc14c2629c/result" \ - -p 6000:6000 \ - --name webhook \ - -h webhook \ - --restart always \ - webhook - -docker logs -f webhook diff --git a/backend/webhook/webhook.go b/backend/webhook/webhook.go deleted file mode 100644 index bd256d78..00000000 --- a/backend/webhook/webhook.go +++ /dev/null @@ -1,260 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - - "github.com/gorilla/handlers" - "github.com/gorilla/mux" -) - -type Info struct { - Url string `json:"url" datastore:"url"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` -} - -// Actions to be done by webhooks etc -// Field is the actual field to use from json -type HookAction struct { - Type string `json:"type" datastore:"type"` - Name string `json:"name" datastore:"name"` - Id string `json:"id" datastore:"id"` - Field string `json:"field" datastore:"field"` -} - -type Hook struct { - Id string `json:"id" datastore:"id"` - Info Info `json:"info" datastore:"info"` - Transforms struct{} `json:"transforms" datastore:"transforms"` - Actions []HookAction `json:"actions" datastore:"actions"` - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Running bool `json:"running" datastore:"running"` -} - -var hook Hook - -func handleWorkflowAction(request *http.Request, action HookAction) error { - //log.Printf("WORKFLOW!: %#v", action) - log.Printf("Should execute workflow %s", action.Id) - - callbackUrl := os.Getenv("CALLBACKURL") - apikey := os.Getenv("APIKEY") - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, action.Id) - - // ret = requests.post(fullurl, headers=headers, json=data) - //if ret.status_code != 202: - // print(ret.text) - // print(ret.status_code) - // print("Exiting workflows - run queue") - // exit() - - body, err := ioutil.ReadAll(request.Body) - if err != nil { - return err - } - - // Execute a workflow - client := &http.Client{} - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer(body), - ) - - if err != nil { - log.Printf("Error making http request: %s", req) - return err - } - - req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey)) - req.Header.Add("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - log.Printf("Error in http request: %s", req) - } - - log.Printf("%#v", resp) - return nil -} - -// FIXME - refresh hook information once in a while. Compare timestamps or something -func callback(resp http.ResponseWriter, request *http.Request) { - //apikey = os.Getenv("APIKEY") - //hookId = os.Getenv("HOOKID") - - handledWorkflowIds := []string{} - for _, item := range hook.Actions { - if item.Type == "" { - log.Printf("CONTINUE AAS EMPTY ITEM: %#v", item) - continue - } - - if item.Type == "workflow" { - found := false - for _, workflowId := range handledWorkflowIds { - if item.Id == workflowId { - found = true - break - } - } - - if found { - continue - } - - handledWorkflowIds = append(handledWorkflowIds, item.Id) - err := handleWorkflowAction(request, item) - if err != nil { - log.Printf("Error in workflow exec: %s", err) - } - } - } - - // FIXME - send the webhookdata to a logging service? Idk - //body, err := ioutil.ReadAll(request.Body) - //if err != nil { - // log.Println("Failed reading body") - // resp.WriteHeader(401) - // resp.Write([]byte(fmt.Sprintf(`{"success": false}`))) - // return - //} - - //callback, err := http.Post(callbackUrl, "application/json", bytes.NewBuffer(body)) - //if err != nil { - // log.Printf("Failed sending callback to %s", callbackUrl) - //} - - resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true}`))) - return -} - -func loadConfiguration(fullUrl string, apikey string) error { - client := &http.Client{} - - req, err := http.NewRequest( - "GET", - fullUrl, - nil, - ) - - if err != nil { - log.Printf("Error making http request: %s", req) - return err - } - - req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey)) - req.Header.Add("Content-Type", "application/json") - resp, err := client.Do(req) - if err != nil { - log.Printf("Error in http request: %s", req) - return err - } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.Printf("Error reading response: %s", req) - return err - } - - err = json.Unmarshal(body, &hook) - if err != nil { - log.Printf("Failed unmarshaling hook API", req) - return err - } - - log.Printf("%#v", hook) - log.Println(hook.Actions) - return nil -} - -func webhook() { - // FIXME - remove static - ip := "0.0.0.0" - - // Basic webserver stuff - baseFilePath := os.Getenv("URIPATH") - basePort := os.Getenv("HOOKPORT") - callbackUrl := os.Getenv("CALLBACKURL") - apikey := os.Getenv("APIKEY") - hookId := os.Getenv("HOOKID") - - if len(baseFilePath) == 0 { - log.Println("Env URIPATH not set") - os.Exit(3) - } - - if len(basePort) == 0 { - log.Println("Env HOOKPORT not set") - os.Exit(3) - } - - if len(callbackUrl) == 0 { - log.Println("Env CALLBACKURL not set") - os.Exit(3) - } - - if len(apikey) == 0 { - log.Println("Env APIKEY not set") - os.Exit(3) - } - - if len(hookId) == 0 { - log.Println("Env HOOKID not set") - os.Exit(3) - } - - log.Println("Loading hook configuration") - err := loadConfiguration( - fmt.Sprintf("%s/api/v1/hooks/%s", callbackUrl, hookId), - apikey, - ) - - if err != nil { - log.Fatalf("Error loading config: %s", err) - } - - // Optional - // if len(callbackOpts) == 0 { - // log.Println("Env CALLBACKOPTS not set") - // os.Exit(3) - // } - - port := fmt.Sprintf(":%s", basePort) - log.Printf("Starting webhook on %s%s with path %s", ip, port, baseFilePath) - - // Routing - mux := mux.NewRouter() - mux.SkipClean(true) - - // FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend - mux.HandleFunc(baseFilePath, callback).Methods("POST") - - handlers.LoggingHandler(os.Stdout, mux) - loggedRouter := handlers.LoggingHandler(os.Stdout, mux) - - err = http.ListenAndServe( - port, - loggedRouter, - ) - - if err != nil { - log.Fatal("ListenAndServer: ", err) - } -} - -func F(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Write([]byte(r.Header.Get("X-Forwarded-For"))) -} - -func main() { - webhook() -} diff --git a/frontend/run.sh b/frontend/run.sh index c0253440..819c0116 100755 --- a/frontend/run.sh +++ b/frontend/run.sh @@ -1,17 +1,17 @@ #!/bin/sh -docker stop frikky/shuffle:frontend -docker rm frikky/shuffle:frontend +docker stop shuffle-frontend +docker rm shuffle-frontend docker rmi frikky/shuffle:frontend echo "Running build for website" -sudo npm run build +#sudo npm run build docker build . -t frikky/shuffle:frontend echo "Starting server" # Rerun build locally for it to update :) -docker run -it \ - -p 3001:80 \ - -p 3002:443 \ - -v $(pwd)/build:/usr/share/nginx/html:ro \ - --rm \ - nginx +#docker run -it \ +# -p 3001:80 \ +# -p 3002:443 \ +# -v $(pwd)/build:/usr/share/nginx/html:ro \ +# --rm \ +# nginx diff --git a/frontend/src/AngularWorkflow.js b/frontend/src/AngularWorkflow.js index 5ad736b9..bc9d99fe 100644 --- a/frontend/src/AngularWorkflow.js +++ b/frontend/src/AngularWorkflow.js @@ -3,7 +3,9 @@ import { useInterval } from 'react-powerhooks'; import uuid from "uuid"; +import {Link} from 'react-router-dom'; import TextField from '@material-ui/core/TextField'; +import Drawer from '@material-ui/core/Drawer'; import Button from '@material-ui/core/Button'; import Paper from '@material-ui/core/Paper'; import Grid from '@material-ui/core/Grid'; @@ -23,7 +25,12 @@ import Input from '@material-ui/core/Input'; import FormGroup from '@material-ui/core/FormGroup'; import FormControlLabel from '@material-ui/core/FormControlLabel'; import Checkbox from '@material-ui/core/Checkbox'; +import Breadcrumbs from '@material-ui/core/Breadcrumbs'; +import CircularProgress from '@material-ui/core/CircularProgress'; +import ReactJson from 'react-json-view' +import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; +import PolymerIcon from '@material-ui/icons/Polymer'; import CreateIcon from '@material-ui/icons/Create'; import PlayArrowIcon from '@material-ui/icons/PlayArrow'; import AspectRatioIcon from '@material-ui/icons/AspectRatio'; @@ -111,6 +118,7 @@ const AngularWorkflow = (props) => { const [workflowDone, setWorkflowDone] = React.useState(false) const [localFirstrequest, setLocalFirstrequest] = React.useState(true) const [requiresAuthentication, setRequiresAuthentication] = React.useState(true) + const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [variableAnchorEl, setVariableAnchorEl] = React.useState(null) @@ -148,6 +156,9 @@ const AngularWorkflow = (props) => { const [, setExecutingNodes] = React.useState([]) const [executionRunning, setExecutionRunning] = React.useState(false) + const [executionModalOpen, setExecutionModalOpen] = React.useState(false) + const [executionModalView, setExecutionModalView] = React.useState(0) + const [executionData, setExecutionData] = React.useState({}) const [lastSaved, setLastSaved] = React.useState(true) @@ -269,6 +280,14 @@ const AngularWorkflow = (props) => { //console.log(responseJson) // Loop nodes and find results // Update on every interval? idk + + if (JSON.stringify(responseJson) !== JSON.stringify(executionData)) { + // FIXME: If another is selected, don't edit.. + // Doesn't work because this is some async garbage + if (executionData.execution_id === undefined || responseJson.execution_id === executionData.execution_id) { + setExecutionData(responseJson) + } + } if (responseJson.execution_id !== executionRequest.execution_id) { cy.elements().removeClass('success-highlight failure-highlight executing-highlight') return @@ -326,7 +345,8 @@ const AngularWorkflow = (props) => { if (!visited.includes(item.action.label)) { if (executionRunning) { - alert.show("Success for "+item.action.label+" with result "+item.result) + alert.show("Success in node "+item.action.label) + //+" with result "+item.result) visited.push(item.action.label) setVisited(visited) } @@ -640,6 +660,9 @@ const AngularWorkflow = (props) => { "authorization": responseJson.authorization, }) setExecutingNodes([workflow.start]) + setExecutionData({}) + setExecutionModalOpen(true) + setExecutionModalView(1) start() }) .catch(error => { @@ -792,7 +815,8 @@ const AngularWorkflow = (props) => { //setSelectedTriggerIndex(-1) //setTriggerFolders([]) - //setLocalFirstrequest(true) + // Can be used for right side view + setRightSideBarOpen(false) console.timeEnd("UNSELECT") } @@ -1440,8 +1464,20 @@ const AngularWorkflow = (props) => { } const handleVariablesHoverOut = () => { - setVariablesHoverColor(hoverOutColor) - } + setVariablesHoverColor(hoverOutColor) + } + + const paperVariableStyle = { + minHeight: "50px", + maxHeight: "50px", + minWidth: "100%", + maxWidth: "100%", + marginTop: "5px", + color: "white", + backgroundColor: surfaceColor, + cursor: "pointer", + display: "flex", + } const VariablesView = () => { const [open, setOpen] = React.useState(false); @@ -1461,19 +1497,7 @@ const AngularWorkflow = (props) => { ) - } - - const paperVariableStyle = { - minHeight: "50px", - maxHeight: "50px", - minWidth: "100%", - maxWidth: "100%", - marginTop: "5px", - color: "white", - backgroundColor: surfaceColor, - cursor: "pointer", - display: "flex", - } + } const menuClick = (event) => { setOpen(!open) @@ -2520,10 +2544,10 @@ const AngularWorkflow = (props) => { const appApiView = Object.getOwnPropertyNames(selectedAction).length > 0 && Object.getOwnPropertyNames(selectedApp).length > 0 ?
-
+

{selectedAction.app_name}

- What are apps? + What are apps?
: - + + + +
) @@ -4281,6 +4325,7 @@ const AngularWorkflow = (props) => { const RightSideBar = () => { setLastSaved(false) if (Object.getOwnPropertyNames(selectedAction).length > 0 && Object.getOwnPropertyNames(selectedApp).length > 0) { + setRightSideBarOpen(true) //console.time('ACTIONSTART'); return(
@@ -4289,6 +4334,7 @@ const AngularWorkflow = (props) => { ) } else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (selectedTrigger.trigger_type === "SCHEDULE") { + setRightSideBarOpen(true) console.log("SCHEDULE") return(
@@ -4296,6 +4342,7 @@ const AngularWorkflow = (props) => {
) } else if (selectedTrigger.trigger_type === "WEBHOOK") { + setRightSideBarOpen(true) console.log("WEBHOOK") return(
@@ -4303,6 +4350,7 @@ const AngularWorkflow = (props) => {
) } else if (selectedTrigger.trigger_type === "EMAIL") { + setRightSideBarOpen(true) console.log("EMAIL") return(
@@ -4310,6 +4358,7 @@ const AngularWorkflow = (props) => {
) } else if (selectedTrigger.trigger_type === "USERINPUT") { + setRightSideBarOpen(true) console.log("USER INPUT SIDEBAR") return(
@@ -4323,6 +4372,7 @@ const AngularWorkflow = (props) => { return null } } else if (Object.getOwnPropertyNames(selectedEdge).length > 0) { + setRightSideBarOpen(true) return(
@@ -4356,6 +4406,165 @@ const AngularWorkflow = (props) => {
+ const executionPaperStyle = { + minWidth: "95%", + maxWidth: "95%", + marginTop: "5px", + color: "white", + marginBottom: 10, + padding: 5, + backgroundColor: surfaceColor, + cursor: "pointer", + display: "flex", + minHeight: 40, + maxHeight: 40, + } + + const executionModal = + setExecutionModalOpen(false)} PaperProps={{style: {minWidth: 375, maxWidth: 375, backgroundColor: "#1F2023", color: "white", fontSize: 18}}}> + {executionModalView === 0 ? +
+ +

+ + All Executions +

+ + + + {workflowExecutions.length > 0 ? +
+ {workflowExecutions.map(data => { + const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" ? "red" : "orange" + const timeElapsed = data.completed_at-data.started_at + const resultsLength = data.results !== undefined && data.results !== null ? data.results.length : 0 + + const timestamp = new Date(data.started_at*1000).toISOString().split('.')[0].split("T").join(" ") + + return ( + {}} onMouseOut={() => {}} onClick={() => { + setExecutionModalView(1) + setExecutionData(data) + }}> +
+
+
+ {timestamp} +
+ +
+ {resultsLength}/{data.workflow.actions.length} +
+ +
+ + + + + ) + return + })} +
+ : +
+ There are no executions yet +
+ } +
+ : +
+ +

{setExecutionModalView(0)}}> + + Other Executions +

+ + +

Executing Workflow

+ {executionData.execution_argument !== undefined && executionData.execution_argument.length > 0 ? +
+

Execution Argument:

{executionText} +
+ : null } + {executionData.status !== undefined && executionData.status.length > 0 ? +
+ Status: {executionData.status} +
+ : null + } + {executionData.started_at !== undefined ? +
+ Started: {new Date(executionData.started_at*1000).toISOString()} +
+ : null + } + {executionData.execution_id !== undefined && executionData.execution_id.length > 0 ? +
+ ID: {executionData.execution_id} +
+ : null + } + +
+ Actions +
+ {executionData.status !== undefined && executionData.status !== "ABORTED" && executionData.status !== "FINISHED" ? : null} +
+
+ {executionData.results === undefined || executionData.results === null || executionData.results.length === 0 && executionData.status === "EXECUTING" ? + + : + executionData.results.map(data => { + var showResult = data.result.trim() + showResult.split(" None").join(" \"None\"") + //showResult = replaceAll(showResult, " None", " \"None\""); + var jsonvalid = true + try { + JSON.parse(showResult) + } catch (e) { + jsonvalid = false + } + + const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) + const imgsize = 50 + const actionimg = curapp === null ? + null : + {data.action.app_name} + + return ( +
+
+ {actionimg} + {data.action.label} +
+
Status {data.status}
+ {jsonvalid ? + : +
+ Result  + {data.result} +
+ } +
+ ) + }) + } +
+ } + + const newView = isLoggedIn ?
@@ -4373,17 +4582,17 @@ const AngularWorkflow = (props) => { }} />
+ {executionModal} - {debugView}
:
TMP FOR NOT LOGGED IN
- + const variablesModal = variablesModalOpen ? { /> - - : null + var activateButton = selectedApp.generated && !selectedApp.activated ? + + : null - var deleteButton = (selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true) ? + var deleteButton = ((selectedApp.private_id !== undefined && selectedApp.private_id.length > 0 && selectedApp.generated) || (selectedApp.downloaded != undefined && selectedApp.downloaded == true)) && activateButton === null ? : null + var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? + {selectedApp.title} + : + {selectedApp.title} //fetch(globalUrl+"/api/v1/get_openapi/"+urlParams.get("id"), { var baseInfo = newAppname.length > 0 ?
-

{newAppname}

-

{description}

+
+
+ {imageline} +
+
+

{newAppname}

+

{description}

+
+
+ {activateButton} + {editButton} + {deleteButton} - {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} -

ID: {selectedApp.id}

- {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null} + {selectedApp.link.length > 0 ?

URL: {selectedApp.link}

: null} +

ID: {selectedApp.id}

+ {selectedApp.privateId !== undefined && selectedApp.privateId.length > 0 ?

PrivateID: {selectedApp.privateId}

: null}
Actions @@ -356,7 +384,7 @@ const Apps = (props) => {
- {selectedAction.parameters !== undefined ? + {selectedAction.parameters !== undefined && selectedAction.parameters !== null ?
Arguments {selectedAction.parameters.map(data => { @@ -376,19 +404,17 @@ const Apps = (props) => { })}
: null} - - {editButton} - {deleteButton}
: null return( -
+

App Creator

- Security API's + How it works +  - Security API's  - OpenAPI directory
Apps interact with eachother in workflows. They are created with the app creator, using OpenAPI specification or manually in python. Use the links above to find potential apps you're looking for using OpenAPI or make one from scratch. There's 1000+ available. @@ -431,7 +457,8 @@ const Apps = (props) => { const searchfield = event.target.value.toLowerCase() const newapps = apps.filter(data => data.name.toLowerCase().includes(searchfield) || data.description.toLowerCase().includes(searchfield)) - if (newapps.length === 0 && !appSearchLoading) { + if ((newapps.length === 0 || searchBackend) && !appSearchLoading) { + setAppSearchLoading(true) runAppSearch(searchfield) } else { @@ -445,7 +472,7 @@ const Apps = (props) => {

Upload

- +
@@ -454,19 +481,11 @@ const Apps = (props) => {

Available integrations

{isLoading ? : null} - {/* - - */} + Search OpenAPI
+ control={ {setSearchBackend(!searchBackend)}} />} + />
{ :

- Try a broader search term. E.g. "http" or "TheHive" + Try a broader search term, e.g. "http", "alert", "ticket" etc.

@@ -651,7 +670,12 @@ const Apps = (props) => { return response.json() }) .then((responseJson) => { - console.log(responseJson) + //console.log(responseJson) + if (responseJson.success) { + if (responseJson.reason !== undefined && responseJson.reason.length > 0) { + setFilteredApps(responseJson.reason) + } + } }) .catch(error => { alert.error(error.toString()) diff --git a/frontend/src/Workflows.js b/frontend/src/Workflows.js index 64acae04..de69950a 100644 --- a/frontend/src/Workflows.js +++ b/frontend/src/Workflows.js @@ -32,7 +32,7 @@ import DialogContent from '@material-ui/core/DialogContent'; const surfaceColor = "#27292D" const Workflows = (props) => { - const { globalUrl, isLoggedIn, isLoaded, } = props; + const { globalUrl, isLoggedIn, isLoaded, } = props; document.title = "Shuffle - Workflows" const alert = useAlert() @@ -158,7 +158,7 @@ const Workflows = (props) => { const getWorkflowExecution = (id) => { fetch(globalUrl+"/api/v1/workflows/"+id+"/executions", { - method: 'GET', + method: 'GET', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -186,7 +186,7 @@ const Workflows = (props) => { const abortExecution = (workflowid, executionid) => { alert.success("Aborting execution") fetch(globalUrl+"/api/v1/workflows/"+workflowid+"/executions/"+executionid+"/abort", { - method: 'GET', + method: 'GET', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -210,7 +210,7 @@ const Workflows = (props) => { alert.show("Executing workflow "+id) setTrackingId(id) fetch(globalUrl+"/api/v1/workflows/"+id+"/execute", { - method: 'GET', + method: 'GET', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', @@ -290,7 +290,7 @@ const Workflows = (props) => { const deleteWorkflow = (id) => { alert.success("Deleted workflow "+id) fetch(globalUrl+"/api/v1/workflows/"+id, { - method: 'DELETE', + method: 'DELETE', headers: { 'Content-Type': 'application/json', 'Accept': 'application/json', diff --git a/functions/README.md b/functions/README.md index 76682e26..b953b6c0 100644 --- a/functions/README.md +++ b/functions/README.md @@ -17,3 +17,6 @@ The point of this folder is to make GCP Cloud functions able to run default WALK * class AppBase * class * Runner + +## Update may 2020: +Moved static_baseline to ./onprem/app_sdk because of license. diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 3110e77a..4dd3ad47 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,4 +1,10 @@ -docker rmi frikky/shuffle:orborus --force +NAME=orborus +VERSION=0.1.0 -docker build . -t frikky/shuffle:orborus -docker push frikky/shuffle:orborus +echo "Running docker build with $NAME:$VERSION" +#docker rmi frikky/shuffle:$NAME --force +docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION + +docker push frikky/$NAME:$VERSION +docker push frikky/shuffle:$NAME +#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 6996fc5a..cbd49b83 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -146,6 +146,9 @@ func initializeImages(dockercli *dockerclient.Client) { ctx := context.Background() // check whether theyre the same first + //version := "0.1.0" + // fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/orborus:%s", version), + // fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/worker:%s", version), images := []string{ fmt.Sprintf("docker.io/%s:app_sdk", baseimagename), fmt.Sprintf("docker.io/%s:worker", baseimagename), @@ -155,7 +158,7 @@ func initializeImages(dockercli *dockerclient.Client) { for _, image := range images { reader, err := dockercli.ImagePull(ctx, image, pullOptions) if err != nil { - log.Printf("Failed getting %s", image) + log.Printf("Failed getting %s: %s", image, err) continue } @@ -197,6 +200,10 @@ func main() { log.Printf("--- Setting up Docker environment. Downloading worker and App SDK! ---") initializeImages(dockercli) + + //workerName := "worker" + //workerVersion := "0.1.0" + //workerImage := fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/%s:%s", workerName, workerVersion) workerImage := fmt.Sprintf("%s:worker", baseimagename) log.Printf("--- Finished configuring docker environment ---\n") diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh new file mode 100644 index 00000000..171308aa --- /dev/null +++ b/functions/onprem/worker/build.sh @@ -0,0 +1,11 @@ +NAME=worker +VERSION=0.1.0 + +echo "Running docker build with $NAME:$VERSION" +CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . +docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION + +# Push both for now.. +docker push frikky/$NAME:$VERSION +docker push frikky/shuffle:$NAME +#docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION diff --git a/functions/onprem/worker/run b/functions/onprem/worker/run index 3f1f1f0a..526009ee 100644 --- a/functions/onprem/worker/run +++ b/functions/onprem/worker/run @@ -1,15 +1,15 @@ echo "Compiling program" -CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . +#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . echo "Fixing docker env" -docker rmi frikky/shuffle:worker --force -docker build . -t frikky/shuffle:worker -docker push frikky/shuffle:worker +#docker rmi frikky/shuffle:worker --force +#docker build . -t frikky/shuffle:worker +#docker push frikky/shuffle:worker -#docker run \ -# --env "AUTHORIZATION=ASD" \ -# --env "DOCKER_API_VERSION=1.39" \ -# --env "EXECUTIONID=ASD" \ -# --env "BASE_URI=$BASE_URI" \ -# -v /var/run/docker.sock:/var/run/docker.sock \ -# frikky/shuffle:worker +docker run \ + --env "AUTHORIZATION=ASD" \ + --env "DOCKER_API_VERSION=1.39" \ + --env "EXECUTIONID=ASD" \ + --env "BASE_URI=$BASE_URI" \ + -v /var/run/docker.sock:/var/run/docker.sock \ + frikky/shuffle:worker diff --git a/functions/triggers/msteams/hook.go b/functions/triggers/msteams/hook.go index fae0e594..5a3a0b0d 100644 --- a/functions/triggers/msteams/hook.go +++ b/functions/triggers/msteams/hook.go @@ -168,6 +168,7 @@ func sendRequest(token OauthToken, message TeamsHook) error { return nil } +// If you're finding this: its from a test project :) func get_accesstoken() (OauthToken, error) { client_id := "9a2a2a63-c63c-4487-baf0-4ff3f4873a7f" client_secret := ":3]D6oFimiXbuV20xH?Dzu@LR*6IFVbq" diff --git a/functions/triggers/outlook/README.md b/functions/triggers/outlook/README.md deleted file mode 100644 index c1c5db5d..00000000 --- a/functions/triggers/outlook/README.md +++ /dev/null @@ -1,57 +0,0 @@ -# Outlook trigger -Makes it possible to trigger a workflow based on an email - -## Local testing - Same as ../webhook -```bash -mv ../main.go -go run main.go hook.go -``` - -# Deploy gcloud -gcloud functions deploy outlooktrigger --runtime go111 --entry-point Authorization --trigger-http --project shuffler --memory=128 --set-env-vars=FUNCTION_APIKEY=asdasd,CALLBACKURL=shuffler.io,TRIGGERID=test123,WORKFLOW_ID=YOUR_WORKFLOW_ID - -# Build and deploy -1. Set hook.go line 1 from "package main" to "package function" -2. zip outlooktrigger.tar hook.go -3. Upload to bucket https://console.cloud.google.com/storage/browser/shuffler.appspot.com?project=shuffler -4. Go to the functions https://console.cloud.google.com/functions/list?project=shuffler - - -## How it works (from frontend to backend) -### Choose mailfolders -1. Use microsoft graph api to get the folders the user wants to listen to -* Have the user write their primary email (default) or another one -* Have it show the folders for the email with chooseable buttons somehow -|inbox -|-subinbox -|--subsubinbox <-- choose e.g. this one -|otherfolder - -API: -// requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/me/mailfolders") - -### Add callback subscription -2. Make an APIcall to ("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") with callback url defined as "https://shuffler.io/api/v1/workflows/{key}/email/authorize" -* Should this be set up whenever the user clicked start or when the workflow is created? -* Start click -> -1. Add another cloud function for the item -2. When it's ready, deploy it to authorize -3. Show it as ready - -### Remove a subscription -* Since everything is already generated above, one would need to -* https://docs.microsoft.com/en-us/graph/api/subscription-delete?view=graph-rest-1.0&tabs=http -* DELETE https://graph.microsoft.com/v1.0/subscriptions/{id} - - -## CREATE - Fixme: LIST all current subscriptions, and stop them if they're towards the same endpoint -* POST /api/v1/workflows/{key}/outlook -* createOutlookSub(resp, request) -* getOutlookSubscriptions(client) // Used to remove all existing for same endpoint -* makeOutlookSubscription(client, folderIds, notificationUrl) -* Add data from ^ to triggerAuth - -## DELETE -* DELETE /api/v1/workflows/{key}/outlook/{triggerId} -* handleDeleteOutlookSub(resp, request) -* handleOutlookSubRemoval(workflowId, triggerId) diff --git a/functions/triggers/outlook/hook.go b/functions/triggers/outlook/hook.go deleted file mode 100644 index 9762c2c2..00000000 --- a/functions/triggers/outlook/hook.go +++ /dev/null @@ -1,222 +0,0 @@ -package function - -// Shuffle: -// https://portal.azure.com/#blade/Microsoft_AAD_RegisteredApps/ApplicationMenuBlade/Authentication/appId/e080cbf4-5dba-44b4-8643-a7c982189c16/isMSAApp//defaultBlade/Overview/servicePrincipalCreated/true - -// Oauth playground: -// https://oauthplay.azurewebsites.net/ - -// APPS: -// https://apps.dev.microsoft.com - -// REMOVE ACCESS: -// https://portal.office.com/account/# - -// Developer: -// https://developer.microsoft.com/en-us/graph/docs/concepts/permissions_reference - -// Bots: -// https://dev.botframework.com/bots - -// Connectors -// https://outlook.office.com/connectors/home/login/#/new -// https://go.microsoft.com/fwlink/?linkid=857599 - -import ( - //"encoding/json" - "bytes" - "fmt" - "io/ioutil" - "log" - "net/http" - "os" - "time" -) - -type Info struct { - Url string `json:"url" datastore:"url"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` -} - -// Actions to be done by webhooks etc -// Field is the actual field to use from json -type HookAction struct { - Type string `json:"type" datastore:"type"` - Name string `json:"name" datastore:"name"` - Id string `json:"id" datastore:"id"` - Field string `json:"field" datastore:"field"` -} - -type Hook struct { - Id string `json:"id" datastore:"id"` - Info Info `json:"info" datastore:"info"` - Actions []HookAction `json:"actions" datastore:"actions"` - Type string `json:"type" datastore:"type"` - Status string `json:"status" datastore:"status"` - Running bool `json:"running" datastore:"running"` -} - -type TeamsHook struct { - MembersAdded []struct { - ID string `json:"id"` - } `json:"membersAdded"` - Type string `json:"type"` - Timestamp time.Time `json:"timestamp"` - LocalTimestamp string `json:"localTimestamp"` - ID string `json:"id"` - ChannelID string `json:"channelId"` - ServiceURL string `json:"serviceUrl"` - From struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"from"` - Conversation struct { - IsGroup bool `json:"isGroup"` - ConversationType string `json:"conversationType"` - ID string `json:"id"` - TenantID string `json:"tenantId"` - } `json:"conversation"` - Recipient struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"recipient"` - ChannelData struct { - Team struct { - ID string `json:"id"` - } `json:"team"` - EventType string `json:"eventType"` - Tenant struct { - ID string `json:"id"` - } `json:"tenant"` - } `json:"channelData"` -} - -var hook Hook -var baseUrl = "https://shuffler.io" - -type OauthToken struct { - TokenType string `json:"token_type"` - ExpiresIn int `json:"expires_in"` - ExtExpiresIn int `json:"ext_expires_in"` - AccessToken string `json:"access_token"` -} - -type TeamsResponse struct { - Conversation struct { - ID string `json:"id"` - } `json:"conversation"` - From struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"from"` - Recipient struct { - ID string `json:"id"` - Name string `json:"name"` - } `json:"recipient"` - ReplyToId string `json:"replyToId"` - Type string `json:"type"` - Text string `json:"text"` -} - -type O365hook struct { - OdataContext string `json:"@odata.context"` - Value []struct { - OdataType string `json:"@odata.type"` - ID interface{} `json:"Id"` - SubscriptionID string `json:"SubscriptionId"` - SubscriptionExpirationDateTime time.Time `json:"SubscriptionExpirationDateTime"` - SequenceNumber int `json:"SequenceNumber"` - ChangeType string `json:"ChangeType"` - Resource string `json:"Resource"` - ResourceData struct { - OdataType string `json:"@odata.type"` - OdataID string `json:"@odata.id"` - OdataEtag string `json:"@odata.etag"` - ID string `json:"Id"` - } `json:"ResourceData"` - } `json:"value"` -} - -func Authorization(resp http.ResponseWriter, request *http.Request) { - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Body: %s", err) - resp.WriteHeader(403) - return - } - - if len(body) > 0 { - // In here - get the email data - // Check who it belongs to and run those workflows - // This should be set from run.go with the callback - // userId: {subscriptionID: {workflowID}} - // E.g. workflow: {auth: {userId - // workflow: {trigger: { - - err = forwardRequest(body) - if err != nil { - log.Printf("Failed unmarshal: %s", err) - resp.WriteHeader(403) - return - } - - resp.WriteHeader(200) - resp.Write([]byte("OK")) - return - } - - token := request.URL.Query().Get("validationToken") - if len(token) == 0 { - log.Println("Validation token is missing") - resp.WriteHeader(403) - return - } - - resp.WriteHeader(200) - resp.Write([]byte(string(token))) -} - -// GetUserDetails - Get one user's details from randomuser.me API -func forwardRequest(body []byte) error { - callbackUrl := os.Getenv("CALLBACKURL") - workflowId := os.Getenv("WORKFLOW_ID") - apikey := os.Getenv("FUNCTION_APIKEY") - - fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/execute", callbackUrl, workflowId) - //log.Printf("Sending data to %s", fullUrl) - - data := fmt.Sprintf(`{"execution_argument": "%s"}`, string(body)) - - req, err := http.NewRequest( - http.MethodPost, - fullUrl, - bytes.NewBuffer([]byte(data)), - ) - - if err != nil { - return err - } - - req.Header.Add("Authorization", fmt.Sprintf(`Bearer %s`, apikey)) - req.Header.Add("Content-Type", "application/json") - randomUserClient := http.Client{ - Timeout: time.Second * 5, - } - - res, err := randomUserClient.Do(req) - if err != nil { - return err - } - - log.Printf("Status: %d", res.StatusCode) - returnbody, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - - log.Printf("New body: %s", string(returnbody)) - - //log.Println(string(newbody)) - return nil -} diff --git a/functions/triggers/outlook/integrations/config.json b/functions/triggers/outlook/integrations/config.json deleted file mode 100644 index 8d98dba3..00000000 --- a/functions/triggers/outlook/integrations/config.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "clientID": "70e37005-c954-4290-b573-d4b94e484336", - "clientSecret": ".eNw/A[kQFB5zL.agvRputdEJENeJ392", - "RedirectURL": "https://44d84ee7.ngrok.io/functions/outlook/register", - "AuthURL": "https://login.microsoftonline.com/common/oauth2/authorize", - "TokenURL": "https://login.microsoftonline.com/common/oauth2/token" -} diff --git a/functions/triggers/outlook/integrations/server.crt b/functions/triggers/outlook/integrations/server.crt deleted file mode 100644 index 92244399..00000000 --- a/functions/triggers/outlook/integrations/server.crt +++ /dev/null @@ -1,21 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDYDCCAkigAwIBAgIJAOvgxcclM1eyMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV -BAYTAk5PMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX -aWRnaXRzIFB0eSBMdGQwHhcNMTgwMzA1MTE1MTIwWhcNMjgwMzAyMTE1MTIwWjBF -MQswCQYDVQQGEwJOTzETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 -ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB -CgKCAQEAowNqrvocJCLLcytYBfZhsqG3CkPsJ4PTicNyjuUGmlILPHlN1DE7jbDo -KuOKImfV4AfQANnttaPksZyuKJL8XGpC7YmF5mmInUWG48SmZjvRbBi1LnCrjJKS -ywh2lJqw4w30w2ItcpogDrYhh6+T3VyabcYngZjKSFgON3wo4I0c6aT19VVXGqnK -y1WEejZmiChV4iwEu4vMPIzt16QpIBr8NPSkBLLRDAGWMjFnIuYDEwgjVn6XYhM9 -+NxhvY9es+qeqLQsZj2a1wcDGaw7G4iNZdltlPmlTCreRDBYBTsYCds/rJmPZ2xi -6jgiyNZj3xHG5Knw2YIw0OwHyT9mWQIDAQABo1MwUTAdBgNVHQ4EFgQUckr5tCzg -F1eBID7mtWTfeqyX4g0wHwYDVR0jBBgwFoAUckr5tCzgF1eBID7mtWTfeqyX4g0w -DwYDVR0TAQH/BAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAAdH1aPcWBjJHF6n/ -NgIiRSEU4mi5I6RUlPuR7dN7dcmF1Ho7quurNFzknXwks+a62oKSnkkFxxwrv/d6 -dIH5kNibVs7oRxEpA3gUmXXKUW4RPrxAp2zN37t7zs5xbpTATxfJiIMP8Rjo0sOf -SilS22Sn0e0HxBi78t3DJEZvOQ9KSRuD1g9gOAY4lj/fni6rVJo8YCR2MyjmQoXB -luHDF4jTqi/TkXECfqQZu0pctx3maISpB1fAuaELwPDvqbLgoC97gl6bIEiKLkJ0 -JkbGnb997K80ztFvFAKGyUtsvEDCupe/fdBPFqCruAQWI/BqVJFTdRD43dWEQANF -S8ApvA== ------END CERTIFICATE----- diff --git a/functions/triggers/outlook/integrations/server.key b/functions/triggers/outlook/integrations/server.key deleted file mode 100644 index 87c60e08..00000000 --- a/functions/triggers/outlook/integrations/server.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAowNqrvocJCLLcytYBfZhsqG3CkPsJ4PTicNyjuUGmlILPHlN -1DE7jbDoKuOKImfV4AfQANnttaPksZyuKJL8XGpC7YmF5mmInUWG48SmZjvRbBi1 -LnCrjJKSywh2lJqw4w30w2ItcpogDrYhh6+T3VyabcYngZjKSFgON3wo4I0c6aT1 -9VVXGqnKy1WEejZmiChV4iwEu4vMPIzt16QpIBr8NPSkBLLRDAGWMjFnIuYDEwgj -Vn6XYhM9+NxhvY9es+qeqLQsZj2a1wcDGaw7G4iNZdltlPmlTCreRDBYBTsYCds/ -rJmPZ2xi6jgiyNZj3xHG5Knw2YIw0OwHyT9mWQIDAQABAoIBABJ+9L/dyQuglwz+ -QgKLLhKinq4fftAM+ReMgZcNDW69GGFIMjh9TZCKHg2fu7Cjr3S37jXqhDoz2mL8 -sBYSd2fU9rsU+4hlOQb/OIrnaSn4Z46oTwZx6kUM7HL1Bt9dnexlTPxOS3HRYwnI -SI2oslJPi4YhEaJ2v5ztwM8y20B/E/zSW2onWz5gB8/bdxSmuJaWfHioIEoac8Gf -BE7jiYMnx9kKeVfgkKPMBKXhAyE2lbAz7N5nDS/4HkbUMh389RBvakc4gkv/QkSW -LNuXpbcSqJiG0FcVutjYS87a/ul3IdhAYmZDTuvRUhNsWBiY3LSY4C9NlHhiJihg -RU1kknECgYEA1dPT6n4hy/OK2mT3ThGQQFMJWsnmcHSsvuK31+UYZOI9kNtBVxc6 -HSHkh0G53o6o2wZLr2gMXy35O5ZxSbectA1Q4MJDlYBs1MfHdOVyE4z6mmA9TLN1 -c+9pYOC0qx+6NwPdO7j2xdUMEUfzocWOeay0AzJg20BQYmKuy3Kc+tcCgYEAwyn6 -n+XS0vodfJdHhvbW/jocQlHQOBK5HklZfq2PMgpRRDaBOvHP/f07egLc2inec2sC -yPSCEfRMMhFcU5NoBt4Unzz2Y8pbpL1L5kbM6B4IqK/5vYcbvkmBkhL3AFT6Fg/3 -3XCdygPW9Vf1nRKr2KhT9dDvB+XO2B75JmKwMk8CgYApKzGf8kz7gZZ4WfwrccI+ -QD6K1lihyjUAQ5J15Mv/kHeeDjjUVcqAlWf0irkImpr0IJAt43COWsGjsWF6efmX -yQCLZZuxixppFVXXsd122ivd0S28OMkiWzQEzP67+83Ujc/okcIhcNVz9lB4ExtN -Xe0CuI5haE6RwsI4tYZ33QKBgFq6ckPRcOAZ3IlmPp9Us3/+fdKq/BSFR7/3s347 -q11FBKCkghFoBxx5lCPVntxhKIQZlHLdkHZOTvnbrkNAPNUsewPIMHcVxOLiCZ3k -/i9OfxIEtSJR5CjjPTQuUtu5pYWKKN2uE/ytKkpmeM1rt64CGv4lAmp2gGFijMs2 -h9jrAoGBANPQO6cKqtnxvst3lnljVBoftlJgeHamUac+xeYKA5Hocv5VwLXMTzzu -09tAhQFvFwCWWrfdgtvIM6k5Sl9F5MdiO9VNflI0IVudIcm9FKorWogH02mtwxsw -hvk5VUk3awiZ/Nu9t38ukeqCetjQEf6yupy/14ZPLndN5naSeEwo ------END RSA PRIVATE KEY----- diff --git a/functions/triggers/outlook/main.go b/functions/triggers/outlook/main.go deleted file mode 100644 index 12bc84b3..00000000 --- a/functions/triggers/outlook/main.go +++ /dev/null @@ -1,41 +0,0 @@ -package main - -import ( - "log" - "net/http" - "os" - - "github.com/gorilla/handlers" - "github.com/gorilla/mux" -) - -func webhook() { - // FIXME - remove static - port := ":8080" - baseFilePath := "/" - - mux := mux.NewRouter() - mux.SkipClean(true) - - // FIXME - Add path for updating the hook? Can be a specific POST requeuest from backend - mux.HandleFunc(baseFilePath, Authorization).Methods("POST") - mux.HandleFunc("/authorize", Authorization).Methods("POST") - - handlers.LoggingHandler(os.Stdout, mux) - loggedRouter := handlers.LoggingHandler(os.Stdout, mux) - - log.Printf("Starting on http://localhost%s", port) - err := http.ListenAndServe( - port, - loggedRouter, - ) - - if err != nil { - log.Fatal("ListenAndServer: ", err) - } - -} - -func main() { - webhook() -} diff --git a/functions/triggers/outlook/run.go b/functions/triggers/outlook/run.go deleted file mode 100644 index 55d3d420..00000000 --- a/functions/triggers/outlook/run.go +++ /dev/null @@ -1,304 +0,0 @@ -package main - -// This entire script should be part of the API backend - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io/ioutil" - "log" - "net/http" - "time" - - "golang.org/x/oauth2" -) - -type Subscription struct { - ChangeType string `json:"changeType"` - NotificationURL string `json:"notificationUrl"` - Resource string `json:"resource"` - ExpirationDateTime string `json:"expirationDateTime"` - ClientState string `json:"clientState"` -} - -// ClientState string `json:"ClientState,omitempty"` -// OdataType string `json:"@odata.type"` - -//odata.type - Include "@odata.type":"#Microsoft.OutlookServices.PushSubscription". The PushSubscription entity defines NotificationURL. -//ChangeType - Specifies the types of events to monitor for that resource. See ChangeType for the supported types. -//ClientState - Optional property that indicates that each notification should be sent with a header by the same ClientState value. This lets the listener check the legitimacy of each notification. -//NotificationURL - Specifies where notifications should be sent to. This URL represents a web service typically implemented by the client. -//Resource - Specifies the resource to monitor and receive notifications on. You can use the optional query parameter $filter to refine the conditions for a notification, or use $select to include specific properties in a rich notification. - -//https://outlook.office.com/mail.read - -type Config struct { - ClientID string - ClientSecret string - RedirectUrl string - AuthUrl string - TokenUrl string -} - -func getOfficeAppInfo() (Config, error) { - configpath := "integrations/config.json" - - data, err := ioutil.ReadFile(configpath) - if err != nil { - //log.Fatal(err) - log.Printf("Error getting hive: %s\n", err) - } - - config := Config{} - err = json.Unmarshal(data, &config) - if err != nil { - return Config{}, err - } - - return config, nil -} - -// This should be a popup for the user -func get_accesstoken() (*http.Client, OauthToken, error) { - ctx := context.Background() - config, err := getOfficeAppInfo() - if err != nil { - return nil, OauthToken{}, err - } - - conf := &oauth2.Config{ - ClientID: config.ClientID, - ClientSecret: config.ClientSecret, - Scopes: []string{ - "Mail.Read", - "User.Read", - "https://outlook.office.com/mail.read", - }, - RedirectURL: "https://localhost:8000", - Endpoint: oauth2.Endpoint{ - AuthURL: config.AuthUrl, - TokenURL: config.TokenUrl, - }, - } - //"Mail.Read.Shared", - - //url := conf.AuthCodeURL("state", oauth2.SetAuthURLParam("resource", "https://outlook.office.com")) - // ADD DATA TO STATE HERE :O - url := conf.AuthCodeURL("workflow_id%3Dc2e0b50a-2957-427e-a97b-b989dc5a5408%26trigger_id%3D9e845679-5843-4959-a76c-a6d664e9df35%26username%3Drheyix.yt@gmail.com", oauth2.SetAuthURLParam("resource", "https://graph.microsoft.com")) - - fmt.Printf("Visit the URL for the auth dialog: \n%v\n\n", url) - codechannel := make(chan string) - - // Handles the server callback, listening on port 8000 - go func() { - port := ":8000" - - http.HandleFunc("/", func(response http.ResponseWriter, request *http.Request) { - tmpcode := request.URL.Query().Get("code") - if len(tmpcode) < 100 { - return - } else { - codechannel <- tmpcode - } - }) - - // FIX - might cause errors not being printed - err := http.ListenAndServeTLS(port, "integrations/server.crt", "integrations/server.key", nil) - if err != nil { - log.Printf("%s\n", err) - } - }() - - code := <-codechannel - close(codechannel) - - // https://stackoverflow.com/questions/52787420/multiple-resources-in-a-single-authorization-request - // Multi resource ^ - access_token, err := conf.Exchange(ctx, code) - //log.Printf("%#v", access_token) - if err != nil { - return nil, OauthToken{}, err - } - - //log.Printf("%#v", access_token) - outlookClient := conf.Client(ctx, access_token) - - oauthToken := OauthToken{ - AccessToken: access_token.AccessToken, - TokenType: access_token.TokenType, - RefreshToken: access_token.RefreshToken, - Expiry: access_token.Expiry, - } - - return outlookClient, oauthToken, nil -} - -type OauthToken struct { - AccessToken string `json:"AccessToken" datastore:"AccessToken,noindex"` - TokenType string `json:"TokenType" datastore:"TokenType,noindex"` - RefreshToken string `json:"RefreshToken" datastore:"RefreshToken,noindex"` - Expiry time.Time `json:"Expiry" datastore:"Expiry,noindex"` -} - -type Mailfolders struct { - OdataContext string `json:"@odata.context"` - OdataNextLink string `json:"@odata.nextLink"` - Value []struct { - ID string `json:"id"` - DisplayName string `json:"displayName"` - ParentFolderID string `json:"parentFolderId"` - ChildFolderCount int `json:"childFolderCount"` - UnreadItemCount int `json:"unreadItemCount"` - TotalItemCount int `json:"totalItemCount"` - } `json:"value"` -} - -func getFolders(client *http.Client) (Mailfolders, error) { - requestUrl := fmt.Sprintf("https://graph.microsoft.com/v1.0/users/frikky@shuffletest.onmicrosoft.com/mailfolders") - - ret, err := client.Get(requestUrl) - if err != nil { - log.Printf("FolderErr: %s", err) - return Mailfolders{}, err - } - - log.Printf("Status folders: %d", ret.StatusCode) - body, err := ioutil.ReadAll(ret.Body) - if err != nil { - log.Printf("Body: %s", err) - return Mailfolders{}, err - } - - //log.Printf("Body: %s", string(body)) - - mailfolders := Mailfolders{} - err = json.Unmarshal(body, &mailfolders) - if err != nil { - log.Printf("Unmarshal: %s", err) - return Mailfolders{}, err - } - - //fmt.Printf("%#v", mailfolders) - // FIXME - recursion for subfolders - // Recursive struct - // folderEndpoint := fmt.Sprintf("%s/%s/childfolders?$top=40", requestUrl, parentId) - for _, folder := range mailfolders.Value { - log.Println(folder.DisplayName) - } - - return mailfolders, nil -} - -// Subscribes to a mailbox based on some thingies -func makeSubscription(client *http.Client, folderIds []string) { - // FIXME - show the users folders from oauth and let them choose - - fullUrl := "https://graph.microsoft.com/v1.0/subscriptions" - //resource := fmt.Sprintf("https://outlook.office.com/api/v2.0/me/mailfolders('inbox')/messages") - resource := fmt.Sprintf("me/mailfolders('inbox')/messages") - sub := Subscription{ - ChangeType: "created", - NotificationURL: "https://de4fc12b.ngrok.io", - ExpirationDateTime: "2019-09-22T18:23:45.9356913Z", - ClientState: "This is a test", - Resource: resource, - } - - data, err := json.Marshal(sub) - if err != nil { - log.Printf("Marshal: %s", err) - return - } - - log.Printf(string(data)) - - req, err := http.NewRequest( - "POST", - fullUrl, - bytes.NewBuffer(data), - ) - req.Header.Add("Content-Type", "application/json") - - res, err := client.Do(req) - if err != nil { - log.Printf("Client: %s", err) - return - } - - log.Printf("Status: %d", res.StatusCode) - body, err := ioutil.ReadAll(res.Body) - if err != nil { - log.Printf("Body: %s", err) - return - } - - fmt.Println(string(body)) - log.Println("Shoooould be set up :)") - -} - -func getOutlookClient(code string, accessToken OauthToken, redirectUri string) (*http.Client, *oauth2.Token, error) { - ctx := context.Background() - - conf := &oauth2.Config{ - ClientID: "70e37005-c954-4290-b573-d4b94e484336", - ClientSecret: ".eNw/A[kQFB5zL.agvRputdEJENeJ392", - Scopes: []string{ - "Mail.Read", - "User.Read", - "https://outlook.office.com/mail.read", - }, - RedirectURL: redirectUri, - Endpoint: oauth2.Endpoint{ - AuthURL: "https://login.microsoftonline.com/common/oauth2/authorize", - TokenURL: "https://login.microsoftonline.com/common/oauth2/token", - }, - } - - if len(code) > 0 { - access_token, err := conf.Exchange(ctx, code) - if err != nil { - log.Printf("Access_token issue: %s", err) - return &http.Client{}, access_token, err - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } else { - // Manually recreate the oauthtoken - access_token := &oauth2.Token{ - AccessToken: accessToken.AccessToken, - TokenType: accessToken.TokenType, - RefreshToken: accessToken.RefreshToken, - Expiry: accessToken.Expiry, - } - - client := conf.Client(ctx, access_token) - return client, access_token, nil - } -} - -func main() { - graphclient, oauthToken, err := get_accesstoken() - if err != nil { - log.Printf("Oauth setup: %s", err) - return - } - - // FIXME - make this possible for alternative users (shared) - folders, err := getFolders(graphclient) - if err != nil { - log.Printf("Folder get error: %s", err) - return - } - _ = folders - - folderIds := []string{"inbox"} - //log.Println(folders) - //log.Printf("%#v", oauthToken) - // Use oauthToken to generate data for outlook - outlookclient, _, err := getOutlookClient("", oauthToken, "https://localhost:8000") - makeSubscription(outlookclient, folderIds) -}