From 70edb366a9931820ec6ef78f7d24a4672d025b8f Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 11 May 2023 11:39:01 +0200 Subject: [PATCH] Minor JSON fixes for SDK --- backend/app_sdk/Dockerfile | 3 +- backend/app_sdk/app_base.py | 14 +- backend/go-app/walkoff.go | 14 +- docker-compose.yml | 114 +++---- frontend/src/components/Oauth2Auth.jsx | 157 +++++---- frontend/src/views/Admin.jsx | 15 +- frontend/src/views/AngularWorkflow.jsx | 56 ++- frontend/src/views/AppCreator.jsx | 449 +++++++++++++++++-------- frontend/src/views/Apps.jsx | 2 +- functions/onprem/orborus/orborus.go | 3 +- 10 files changed, 521 insertions(+), 306 deletions(-) diff --git a/backend/app_sdk/Dockerfile b/backend/app_sdk/Dockerfile index 25c9aa40..ac322bb9 100644 --- a/backend/app_sdk/Dockerfile +++ b/backend/app_sdk/Dockerfile @@ -1,5 +1,6 @@ #FROM python:3.9.1-alpine as base -FROM python:3.10.0-alpine as base +#FROM python:3.10.0-alpine as base +FROM python:3.11.3-alpine as base FROM base as builder RUN apk --no-cache add --update alpine-sdk libffi libffi-dev musl-dev openssl-dev tzdata coreutils diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 0eefe250..b4ab4ff7 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2279,6 +2279,13 @@ class AppBase: # Can't handle self yet (?) ret = run.render(**globals()) + + # Load output as JSON + try: + ret = json.loads(ret) + except: + pass + return ret except jinja2.exceptions.TemplateNotFound as e: self.logger.info(f"[ERROR] Liquid Template error: {e}") @@ -3064,7 +3071,6 @@ class AppBase: #self.logger.info(action["parameters"]) # This seems redundant now - self.logger.info("[DEBUG] Pre parameters") for parameter in newparams: action["parameters"].append(parameter) @@ -3086,7 +3092,6 @@ class AppBase: # Multi_parameter has the data for each. variable minlength = 0 - self.logger.info("[DEBUG] Pre-loading parameters") multi_parameters = json.loads(json.dumps(params)) multiexecution = False multi_execution_lists = [] @@ -3513,8 +3518,11 @@ class AppBase: try: del params[field] self.logger.info("[WARNING] Removed field invalid field %s" % field) - except KeyError: + except KeyError as e: + self.logger.info("[WARNING] Tried to remove field %s but it didn't exist" % field) break + else: + self.logger.info("[ERROR] Couldn't find fieldsplit in error. Raw error: %s" % errorstring) else: newres = json.dumps({ "success": False, diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 37413896..d410a3ed 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1923,7 +1923,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("Failed loading repo %s into memory (github workflows): %s", url, err) + log.Printf("[INFO] Failed loading repo %s into memory (github workflows): %s", url, err) return err } @@ -3107,8 +3107,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { var tmpBody tmpStruct err = json.Unmarshal(body, &tmpBody) if err != nil { - log.Printf("Error with unmarshal tmpBody: %s", err) - resp.WriteHeader(401) + log.Printf("[WARNING] Error with unmarshal app git clone: %s", err) + resp.WriteHeader(500) resp.Write([]byte(`{"success": false}`)) return } @@ -3135,20 +3135,20 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) { storer := memory.NewStorage() r, err := git.Clone(storer, fs, cloneOptions) if err != nil { - log.Printf("Failed loading repo %s into memory (github workflows 2): %s", tmpBody.URL, err) - resp.WriteHeader(401) + log.Printf("[WARNING] Failed loading repo %s into memory (github apps 2): %s", tmpBody.URL, err) + resp.WriteHeader(500) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err))) return } dir, err := fs.ReadDir("/") if err != nil { - log.Printf("FAiled reading folder: %s", err) + log.Printf("[WARNING] FAiled reading folder: %s", err) } _ = r if tmpBody.ForceUpdate { - log.Printf("[AUDIT] Running with force update from user %s (%s) for %s!", user.Username, user.Id, tmpBody.URL) + log.Printf("[AUDIT] Running app get with force update from user %s (%s) for %s!", user.Username, user.Id, tmpBody.URL) } else { log.Printf("[AUDIT] Updating apps with updates for user %s (%s) for %s (no force)", user.Username, user.Id, tmpBody.URL) } diff --git a/docker-compose.yml b/docker-compose.yml index b6878df7..1143f01c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,62 +1,62 @@ version: '3' services: - #frontend: - # image: ghcr.io/shuffle/shuffle-frontend:latest - # container_name: shuffle-frontend - # hostname: shuffle-frontend - # ports: - # - "${FRONTEND_PORT}:80" - # - "${FRONTEND_PORT_HTTPS}:443" - # networks: - # - shuffle - # environment: - # - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} - # restart: unless-stopped - # depends_on: - # - backend - #backend: - # image: ghcr.io/shuffle/shuffle-backend:latest - # container_name: shuffle-backend - # hostname: ${BACKEND_HOSTNAME} - # # Here for debugging: - # ports: - # - "${BACKEND_PORT}:5001" - # networks: - # - shuffle - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z - # - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z - # env_file: .env - # environment: - # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - # - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps - # - SHUFFLE_FILE_LOCATION=/shuffle-files - # restart: unless-stopped - #orborus: - # image: ghcr.io/shuffle/shuffle-orborus:latest - # container_name: shuffle-orborus - # hostname: shuffle-orborus - # networks: - # - shuffle - # volumes: - # - /var/run/docker.sock:/var/run/docker.sock - # environment: - # #- DOCKER_HOST=tcp://docker-socket-proxy:2375 - # - SHUFFLE_WORKER_VERSION=latest - # - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - # - BASE_URL=http://${OUTER_HOSTNAME}:5001 - # - DOCKER_API_VERSION=1.40 - # - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - # - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} - # - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} - # - HTTP_PROXY=${HTTP_PROXY} - # - HTTPS_PROXY=${HTTPS_PROXY} - # - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} - # - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} - # restart: unless-stopped - # security_opt: - # - seccomp:unconfined + frontend: + image: ghcr.io/shuffle/shuffle-frontend:latest + container_name: shuffle-frontend + hostname: shuffle-frontend + ports: + - "${FRONTEND_PORT}:80" + - "${FRONTEND_PORT_HTTPS}:443" + networks: + - shuffle + environment: + - BACKEND_HOSTNAME=${BACKEND_HOSTNAME} + restart: unless-stopped + depends_on: + - backend + backend: + image: ghcr.io/shuffle/shuffle-backend:latest + container_name: shuffle-backend + hostname: ${BACKEND_HOSTNAME} + # Here for debugging: + ports: + - "${BACKEND_PORT}:5001" + networks: + - shuffle + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - ${SHUFFLE_APP_HOTLOAD_LOCATION}:/shuffle-apps:z + - ${SHUFFLE_FILE_LOCATION}:/shuffle-files:z + env_file: .env + environment: + #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + - SHUFFLE_APP_HOTLOAD_FOLDER=/shuffle-apps + - SHUFFLE_FILE_LOCATION=/shuffle-files + restart: unless-stopped + orborus: + image: ghcr.io/shuffle/shuffle-orborus:latest + container_name: shuffle-orborus + hostname: shuffle-orborus + networks: + - shuffle + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + #- DOCKER_HOST=tcp://docker-socket-proxy:2375 + - SHUFFLE_WORKER_VERSION=latest + - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} + - BASE_URL=http://${OUTER_HOSTNAME}:5001 + - DOCKER_API_VERSION=1.40 + - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} + - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} + - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX} + - HTTP_PROXY=${HTTP_PROXY} + - HTTPS_PROXY=${HTTPS_PROXY} + - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY} + - SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY} + restart: unless-stopped + security_opt: + - seccomp:unconfined opensearch: image: opensearchproject/opensearch:2.5.0 hostname: shuffle-opensearch diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index 6157e270..f068c3bd 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -91,6 +91,7 @@ const AuthenticationOauth2 = (props) => { setAuthenticationModalOpen, isCloud, autoAuth, + authButtonOnly, } = props; //const [update, setUpdate] = React.useState("|") @@ -126,7 +127,7 @@ const AuthenticationOauth2 = (props) => { label: "", usage: [ { - workflow_id: workflow.id, + workflow_id: workflow !== undefined ? workflow.id : "", }, ], id: uuidv4(), @@ -145,26 +146,30 @@ const AuthenticationOauth2 = (props) => { } const startOauth2Request = (admin_consent) => { - console.log("APP: ", selectedApp) + // Admin consent also means to add refresh tokens + + + //console.log("APP: ", selectedApp) if (selectedApp.name.toLowerCase() == "outlook_graph" || selectedApp.name.toLowerCase() == "outlook_office365") { handleOauth2Request( - "efe4c3fe-84a1-4821-a84f-23a6cfe8e72d", "", "https://graph.microsoft.com", - ["Mail.ReadWrite", "Mail.Send"], + ["Mail.ReadWrite", "Mail.Send", "offline_access"], admin_consent, ); } else if (selectedApp.name.toLowerCase() == "gmail") { handleOauth2Request( - "253565968129-c0a35knic7q1pdk6i6qk9gdkvr07ci49.apps.googleusercontent.com", + "253565968129-6ke8086pkp0at16m8t95rdcsas69ngt1.apps.googleusercontent.com", "", "https://gmail.googleapis.com", ["https://www.googleapis.com/auth/gmail.modify", "https://www.googleapis.com/auth/gmail.send", "https://www.googleapis.com/auth/gmail.insert", - "https://www.googleapis.com/auth/gmail.compose"], + "https://www.googleapis.com/auth/gmail.compose", + ], admin_consent, + "select_account%20consent", ) } else if (selectedApp.name.toLowerCase() == "zoho_desk") { handleOauth2Request( @@ -174,7 +179,8 @@ const AuthenticationOauth2 = (props) => { ["Desk.tickets.READ", "Desk.tickets.UPDATE", "Desk.tickets.DELETE", - "Desk.tickets.CREATE"], + "Desk.tickets.CREATE", + "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase() == "slack") { @@ -182,7 +188,7 @@ const AuthenticationOauth2 = (props) => { "151779186901.2448678750935", "", "https://slack.com", - ["admin", "chat:write", "im:read", "im:write", "search:read", "usergroups:read", "usergroups:write"], + ["chat:write:user", "im:read", "im:write", "search:read", "usergroups:read", "usergroups:write", "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase() == "webex") { @@ -190,7 +196,7 @@ const AuthenticationOauth2 = (props) => { "Cab184f3d7271f540443c79b5b79845e3387abbbdb3db4233a87ea3a5432fb3d5", "", "https://webexapis.com", - ["spark:all"], + ["spark:all", "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("microsoft_teams")) { @@ -198,7 +204,7 @@ const AuthenticationOauth2 = (props) => { "31cb4c84-658e-43d5-ae84-22c9142e967a", "", "https://graph.microsoft.com", - ["ChannelMessage.Edit", "ChannelMessage.Read.All", "ChannelMessage.Send", "Chat.Create", "Chat.ReadWrite", "Chat.Read"], + ["ChannelMessage.Edit", "ChannelMessage.Read.All", "ChannelMessage.Send", "Chat.Create", "Chat.ReadWrite", "Chat.Read", "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("todoist")) { @@ -206,7 +212,7 @@ const AuthenticationOauth2 = (props) => { "35fa3a384040470db0c8527e90a3c2eb", "", "https://api.todoist.com", - ["task:add"], + ["task:add", "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("microsoft_sentinel")) { @@ -214,7 +220,7 @@ const AuthenticationOauth2 = (props) => { "4c16e8c4-3d34-4aa1-ac94-262ea170b7f7", "", "https://management.azure.com", - ["https://management.azure.com/user_impersonation"], + ["https://management.azure.com/user_impersonation", "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("microsoft_365_defender")) { @@ -222,7 +228,7 @@ const AuthenticationOauth2 = (props) => { "4c16e8c4-3d34-4aa1-ac94-262ea170b7f7", "", "https://graph.microsoft.com", - ["SecurityEvents.ReadWrite.All"], + ["SecurityEvents.ReadWrite.All", "offline_access"], admin_consent, ) } else if (selectedApp.name.toLowerCase().includes("google_sheets")) { @@ -232,14 +238,16 @@ const AuthenticationOauth2 = (props) => { "https://sheets.googleapis.com", ["https://www.googleapis.com/auth/spreadsheets"], admin_consent, + "consent", ) } else if (selectedApp.name.toLowerCase().includes("google_drive") || selectedApp.name.toLowerCase().includes("google_disk")) { handleOauth2Request( "253565968129-6pij4g6ojim4gpum0h9m9u3bc357qsq7.apps.googleusercontent.com", "", "https://www.googleapis.com/drive/v3", - ["https://www.googleapis.com/auth/drive"], + ["https://www.googleapis.com/auth/drive",], admin_consent, + "consent", ) } else if (selectedApp.name.toLowerCase().includes("jira_service_desk") || selectedApp.name.toLowerCase().includes("jira") || selectedApp.name.toLowerCase().includes("jira_service_management")) { handleOauth2Request( @@ -254,7 +262,7 @@ const AuthenticationOauth2 = (props) => { } - const handleOauth2Request = (client_id, client_secret, oauth_url, scopes, admin_consent) => { + const handleOauth2Request = (client_id, client_secret, oauth_url, scopes, admin_consent, prompt) => { setButtonClicked(true); //console.log("SCOPES: ", scopes); @@ -282,7 +290,8 @@ const AuthenticationOauth2 = (props) => { //console.log("AUTH: ", authenticationType) //console.log("SCOPES2: ", resources) const redirectUri = `${window.location.protocol}//${window.location.host}/set_authentication`; - var state = `workflow_id%3D${workflow.id}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`; + const workflowId = workflow !== undefined ? workflow.id : ""; + var state = `workflow_id%3D${workflowId}%26reference_action_id%3d${selectedAction.app_id}%26app_name%3d${selectedAction.app_name}%26app_id%3d${selectedAction.app_id}%26app_version%3d${selectedAction.app_version}%26authentication_url%3d${authentication_url}%26scope%3d${resources}%26client_id%3d${client_id}%26client_secret%3d${client_secret}`; if (oauth_url !== undefined && oauth_url !== null && oauth_url.length > 0) { state += `%26oauth_url%3d${oauth_url}`; console.log("ADDING OAUTH2 URL: ", state); @@ -299,13 +308,21 @@ const AuthenticationOauth2 = (props) => { } // No prompt forcing - var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=login&scope=${resources}&state=${state}&access_type=offline`; + //var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=login&scope=${resources}&state=${state}&access_type=offline`; + var defaultPrompt = "login" + if (prompt !== undefined && prompt !== null && prompt.length > 0) { + defaultPrompt = prompt + } + + var url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=${defaultPrompt}&scope=${resources}&state=${state}&access_type=offline`; if (admin_consent === true) { console.log("Running Oauth2 WITH admin consent") //url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=consent&scope=${resources}&state=${state}&access_type=offline`; url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&prompt=admin_consent&scope=${resources}&state=${state}&access_type=offline`; } + console.log("URL: ", url) + // Force new consent //const url = `${authenticationType.redirect_uri}?client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${resources}&prompt=consent&state=${state}&access_type=offline`; @@ -333,8 +350,11 @@ const AuthenticationOauth2 = (props) => { setButtonClicked(false); clearInterval(timer); //alert('"Secure Payment" window closed!'); + // - getAppAuthentication(true, true); + if (getAppAuthentication !== undefined) { + getAppAuthentication(true, true); + } } else { console.log("Not closed") } @@ -479,6 +499,59 @@ const AuthenticationOauth2 = (props) => { authenticationOption.label = selectedApp.name + " authentication"; } + + const autoAuthButton = + + + if (authButtonOnly === true) { + return autoAuthButton + } + console.log("Window: ", window.location) return (
@@ -505,52 +578,8 @@ const AuthenticationOauth2 = (props) => { {isCloud && registeredApps.includes(selectedApp.name.toLowerCase()) ? - + {autoAuthButton} + {buttonClicked ? null : diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 0b9f8bb1..b8c46739 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -695,15 +695,12 @@ const Admin = (props) => { const handleGetOrg = (orgId) => { if (orgId.length === 0) { - alert.error( - "Organization ID not defined. Please contact us on https://shuffler.io if this persists logout." - ); + alert.error("Organization ID not defined. Please contact us on https://shuffler.io if this persists logout."); return; } // Just use this one? - var baseurl = globalUrl; - const url = baseurl + "/api/v1/orgs/" + orgId; + const url = `${globalUrl}/api/v1/orgs/${orgId}` fetch(url, { method: "GET", credentials: "include", @@ -719,7 +716,11 @@ const Admin = (props) => { }) .then((responseJson) => { if (responseJson["success"] === false) { - alert.error("Failed getting your org. If this persists, please contact support."); + if (responseJson.reason !== undefined) { + alert.error(responseJson.reason); + } else { + alert.error("Failed getting your org. If this persists, please contact support."); + } } else { if ( responseJson.sync_features === undefined || @@ -728,8 +729,6 @@ const Admin = (props) => { responseJson.sync_features = {}; } - - setSelectedOrganization(responseJson) var lists = { active: { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e0f91e0e..de4f9c42 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -404,8 +404,8 @@ const AngularWorkflow = (defaultprops) => { const [variablesModalOpen, setVariablesModalOpen] = React.useState(false); const [executionVariablesModalOpen, setExecutionVariablesModalOpen] = React.useState(false); - const [authenticationModalOpen, setAuthenticationModalOpen] = - React.useState(false); + const [authenticationModalOpen, setAuthenticationModalOpen] = React.useState(false); + const [conditionsModalOpen, setConditionsModalOpen] = React.useState(false); const [authenticationType, setAuthenticationType] = React.useState(""); @@ -1635,14 +1635,14 @@ const AngularWorkflow = (defaultprops) => { }, credentials: "include", }) - .then((response) => { - if (response.status !== 200) { - console.log("Status not 200 for app auth :O!"); - } + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for app auth :O!"); + } - return response.json(); - }) - .then((responseJson) => { + return response.json(); + }) + .then((responseJson) => { if (responseJson.success) { var newauth = []; for (let authkey in responseJson.data) { @@ -2871,6 +2871,11 @@ const AngularWorkflow = (defaultprops) => { return; } + if (data.type === undefined) { + console.log("No type, automatically setting to action"); + data.type = "ACTION" + } + if (data.type === "ACTION") { setSelectedComment({}) //var curaction = JSON.parse(JSON.stringify(data)) @@ -3168,7 +3173,7 @@ const AngularWorkflow = (defaultprops) => { } else if (data.type === "COMMENT") { setSelectedComment(data); } else { - alert.error("Can't handle " + data.type); + alert.error("Can't handle node type " + data.type); return; } @@ -4181,13 +4186,34 @@ const AngularWorkflow = (defaultprops) => { //window.document.execCommand('insertText', false, text); // try { - const parsedjson = JSON.parse(clipboard); - //console.log("Parsed: ", parsedjson) + var parsedjson = JSON.parse(clipboard); + // Check if array + if (!Array.isArray(parsedjson)) { + console.log("Not array! Adding to array.") + parsedjson = [parsedjson] + } for (let jsonkey in parsedjson) { - const item = parsedjson[jsonkey]; + var item = parsedjson[jsonkey]; console.log("Adding: ", item); - item.data.id = uuidv4() + + if (item.data === undefined || item.data === null) { + console.log("Appending from here") + const newitem = { + "data": item, + "position": { + "x": 0, + "y": 0 + }, + "group": "nodes", + } + + item = newitem + item.type = "ACTION" + item.isStartNode = false + } + + item.data.id = uuidv4() cy.add({ group: item.group, @@ -5518,7 +5544,7 @@ const AngularWorkflow = (defaultprops) => { } } - alert.info("Creating schedule with name " + trigger.name); + alert.info("Creating schedule") const data = { name: trigger.name, frequency: workflow.triggers[triggerindex].parameters[0].value, diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index d274aea2..481898b3 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -281,6 +281,74 @@ export const base64_decode = (str) => { ); }; +// Loops through properties to find the actual JSON output to use +const getJsonObject = (properties) => { + // Loop inside the JSON object and get the value of each key + let jsonObject = {}; + for (let key in properties) { + const property = properties[key]; + + const subloop = false + if (property.hasOwnProperty("type")) { + if (property.type === "object" || property.type === "array") { + subloop = true + } + } + + if (subloop) { + if (property.hasOwnProperty("items") && property.items.hasOwnProperty("properties")) { + + const jsonret = getJsonObject(property.items.properties); + if (property.type === "array") { + console.log("ARRAY!!") + jsonObject[key] = [jsonret]; + } else { + jsonObject[key] = jsonret; + } + } else { + if (property.hasOwnProperty("properties")) { + const jsonret = getJsonObject(property.properties); + if (property.type === "array") { + console.log("ARRAY2!!") + jsonObject[key] = [jsonret]; + } else { + jsonObject[key] = jsonret; + } + } else { + console.log("No items or properties found: ", property); + } + } + + } else { + if (property.hasOwnProperty("example")) { + jsonObject[key] = property.example; + } else if (property.hasOwnProperty("enum") && property.enum.length > 0) { + jsonObject[key] = property.enum[0]; + } else if (property.hasOwnProperty("default")) { + jsonObject[key] = property.default; + } else if (property.hasOwnProperty("maximum")) { + jsonObject[key] = property.maximum; + } else if (property.hasOwnProperty("minimum")) { + jsonObject[key] = property.minimum; + } else if (property.hasOwnProperty("type")) { + if (property.type === "integer" || property.type === "number") { + jsonObject[key] = 0; + } else if (property.type === "boolean") { + jsonObject[key] = false; + } else if (property.type === "string") { + jsonObject[key] = ""; + } else { + console.log("Unknown type: ", property); + } + } else { + console.log("No example or enum found: ", property); + } + } + } + + return jsonObject +} + // Should be different if logged in :| const AppCreator = (defaultprops) => { const { globalUrl, isLoaded } = defaultprops; @@ -626,6 +694,7 @@ const AppCreator = (defaultprops) => { var wordlist = {}; var all_categories = []; console.log("Paths: ", data.paths) + var parentUrl = "" if (data.paths !== null && data.paths !== undefined) { for (let [path, pathvalue] of Object.entries(data.paths)) { @@ -742,7 +811,7 @@ const AppCreator = (defaultprops) => { console.log("Set content!") } } - + if (methodvalue["requestBody"]["content"] !== undefined) { // Handle content - XML or JSON // @@ -755,45 +824,56 @@ const AppCreator = (defaultprops) => { if ( methodvalue["requestBody"]["content"]["application/json"]["schema"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"] !== null ) { - console.log("Schema: ", methodvalue["requestBody"]["content"]["application/json"]["schema"]) + //console.log("Schema: ", methodvalue["requestBody"]["content"]["application/json"]["schema"]) - if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) { - var tmpobject = {}; - for (let prop of methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"]) { - tmpobject[prop] = `\$\{${prop}\}`; - } + try { + if (methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"] !== undefined) { + // Read out properties from a JSON object + const jsonObject = getJsonObject(methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"]) + console.log("JSON OBJECT: ", jsonObject) + if (jsonObject !== undefined && jsonObject !== null) { + try { + newaction["body"] = JSON.stringify(jsonObject, null, 2) + } catch (e) { + console.log("JSON object parse error: ", e) + } + } - //console.log("Data: ", data) - for (let subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) { - - - const tmpitem = - methodvalue["requestBody"]["content"][ - "application/json" - ]["schema"]["required"][subkey]; - tmpobject[tmpitem] = `\$\{${tmpitem}\}`; - } - newaction["body"] = JSON.stringify(tmpobject, null, 2); - } else if ( - methodvalue["requestBody"]["content"]["application/json"]["schema"]["$ref"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"]["$ref"] !== null) { - const retRef = handleGetRef( - methodvalue["requestBody"]["content"]["application/json"][ - "schema" - ], - data - ); - var newbody = {}; - // Can handle default, required, description and type - for (let propkey in retRef.properties) { - console.log("replace: ", propkey) + //newaction["body"] = JSON.stringify(jsonObject, null, 2); - const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); - newbody[parsedkey] = "${" + parsedkey + "}"; - } + var tmpobject = {}; + for (let prop of methodvalue["requestBody"]["content"]["application/json"]["schema"]["properties"]) { + tmpobject[prop] = `\$\{${prop}\}`; + } - newaction["body"] = JSON.stringify(newbody, null, 2); - } + //console.log("Data: ", data) + for (let subkey in methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"]) { + const tmpitem = methodvalue["requestBody"]["content"]["application/json"]["schema"]["required"][subkey]; + tmpobject[tmpitem] = `\$\{${tmpitem}\}`; + } + + newaction["body"] = JSON.stringify(tmpobject, null, 2); + + } else if ( + + methodvalue["requestBody"]["content"]["application/json"]["schema"]["$ref"] !== undefined && methodvalue["requestBody"]["content"]["application/json"]["schema"]["$ref"] !== null) { + const retRef = handleGetRef(methodvalue["requestBody"]["content"]["application/json"]["schema"], data); + + var newbody = {}; + // Can handle default, required, description and type + for (let propkey in retRef.properties) { + console.log("replace: ", propkey) + + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); + newbody[parsedkey] = "${" + parsedkey + "}"; + } + + newaction["body"] = JSON.stringify(newbody, null, 2); + } + } catch (e) { + console.log("RequestBody json error: ", e, path) + } } } else if ( methodvalue["requestBody"]["content"]["application/xml"] !== @@ -810,28 +890,32 @@ const AppCreator = (defaultprops) => { "schema" ] !== null ) { - if ( - methodvalue["requestBody"]["content"]["application/xml"][ - "schema" - ]["properties"] !== undefined - ) { - var tmpobject = {}; - for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"])) { - - tmpobject[prop] = `\$\{${prop}\}`; - } + try { + if ( + methodvalue["requestBody"]["content"]["application/xml"][ + "schema" + ]["properties"] !== undefined + ) { + var tmpobject = {}; + for (let [prop, propvalue] of Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["properties"])) { + + tmpobject[prop] = `\$\{${prop}\}`; + } - for (let [subkey,subkeyval] in Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"])) { - const tmpitem = - methodvalue["requestBody"]["content"][ - "application/xml" - ]["schema"]["required"][subkey]; - tmpobject[tmpitem] = `\$\{${tmpitem}\}`; - } + for (let [subkey,subkeyval] in Object.entries(methodvalue["requestBody"]["content"]["application/xml"]["schema"]["required"])) { + const tmpitem = + methodvalue["requestBody"]["content"][ + "application/xml" + ]["schema"]["required"][subkey]; + tmpobject[tmpitem] = `\$\{${tmpitem}\}`; + } - //console.log("OBJ XML: ", tmpobject) - //newaction["body"] = XML.stringify(tmpobject, null, 2) - } + //console.log("OBJ XML: ", tmpobject) + //newaction["body"] = XML.stringify(tmpobject, null, 2) + } + } catch (e) { + console.log("RequestBody xml error: ", e, path) + } } } else { if ( @@ -863,30 +947,34 @@ const AppCreator = (defaultprops) => { "multipart/form-data" ]["schema"] !== null ) { - if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { - const fieldname = - methodvalue["requestBody"]["content"][ - "multipart/form-data" - ]["schema"]["properties"]["fieldname"]; + try { + if (methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["type"] === "object") { + const fieldname = + methodvalue["requestBody"]["content"][ + "multipart/form-data" + ]["schema"]["properties"]["fieldname"]; - if (fieldname !== undefined) { - //console.log("FIELDNAME: ", fieldname); - newaction.file_field = fieldname["value"]; - } else { - for (const [subkey, subvalue] of Object.entries(methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"])) { - if (subkey.includes("file")) { - console.log("Found subkey field for file: ", path, method, methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]) - newaction.file_field = subkey - break + if (fieldname !== undefined) { + //console.log("FIELDNAME: ", fieldname); + newaction.file_field = fieldname["value"]; + } else { + for (const [subkey, subvalue] of Object.entries(methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"])) { + if (subkey.includes("file")) { + console.log("Found subkey field for file: ", path, method, methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]) + newaction.file_field = subkey + break + } + } + + if (newaction.file_field === undefined || newaction.file_field === null || newaction.file_field.length === 0) { + console.log("No file fieldname found: ", methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]) } } - - if (newaction.file_field === undefined || newaction.file_field === null || newaction.file_field.length === 0) { - console.log("No file fieldname found: ", methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]["properties"]) - } + } else { + console.log("No type found: ", methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]) } - } else { - console.log("No type found: ", methodvalue["requestBody"]["content"]["multipart/form-data"]["schema"]) + } catch (e) { + console.log("Multipart/form error: ", e, path) } } } else { @@ -914,73 +1002,76 @@ const AppCreator = (defaultprops) => { } } - if (schemas.length === 1) { - const parameter = handleGetRef({ $ref: schemas[0] }, data); + try { + if (schemas.length === 1) { + const parameter = handleGetRef({ $ref: schemas[0] }, data); - console.log("Reading type from parameter: ", parameter) - if (parameter.properties !== undefined && parameter["type"] === "object") { - - var newbody = {}; - for (let propkey in parameter.properties) { - console.log("propkey2: ", propkey) - const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); - if (parameter.properties[propkey].type === undefined) { - console.log( - "Skipping (4): ", - parameter.properties[propkey] - ); - continue; - } + console.log("Reading type from parameter: ", parameter) + if (parameter.properties !== undefined && parameter["type"] === "object") { + + var newbody = {}; + for (let propkey in parameter.properties) { + console.log("propkey2: ", propkey) + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); + if (parameter.properties[propkey].type === undefined) { + console.log( + "Skipping (4): ", + parameter.properties[propkey] + ); + continue; + } - if (parameter.properties[propkey].type === "string") { - if ( - parameter.properties[propkey].description !== - undefined - ) { - newbody[parsedkey] = - parameter.properties[propkey].description; - } else { - newbody[parsedkey] = ""; - } - } else if ( - parameter.properties[propkey].type.includes("int") || - parameter.properties[propkey].type.includes("uint64") - ) { - newbody[parsedkey] = 0; - } else if ( - parameter.properties[propkey].type.includes("boolean") - ) { - newbody[parsedkey] = false; - } else if ( - parameter.properties[propkey].type.includes("array") - ) { - newbody[parsedkey] = []; - } else { - console.log( - "CANT HANDLE JSON TYPE (4)", - parameter.properties[propkey].type, - parameter.properties[propkey], - path - ); - newbody[parsedkey] = []; - } - } + if (parameter.properties[propkey].type === "string") { + if ( + parameter.properties[propkey].description !== + undefined + ) { + newbody[parsedkey] = + parameter.properties[propkey].description; + } else { + newbody[parsedkey] = ""; + } + } else if ( + parameter.properties[propkey].type.includes("int") || + parameter.properties[propkey].type.includes("uint64") + ) { + newbody[parsedkey] = 0; + } else if ( + parameter.properties[propkey].type.includes("boolean") + ) { + newbody[parsedkey] = false; + } else if ( + parameter.properties[propkey].type.includes("array") + ) { + newbody[parsedkey] = []; + } else { + console.log( + "CANT HANDLE JSON TYPE (4)", + parameter.properties[propkey].type, + parameter.properties[propkey], + path + ); + newbody[parsedkey] = []; + } + } - newaction["body"] = JSON.stringify(newbody, null, 2); - } else { - console.log( - "CANT HANDLE PARAM: (4) ", - parameter.properties, - path - ); - } - } + newaction["body"] = JSON.stringify(newbody, null, 2); + } else { + console.log( + "CANT HANDLE PARAM: (4) ", + parameter.properties, + path + ); + } + } + } catch (e) { + console.log("Param Error: ", e, path) + } } } } } - // HAHAHA wtf is this. if ( methodvalue.responses !== undefined && methodvalue.responses !== null @@ -1033,7 +1124,20 @@ const AppCreator = (defaultprops) => { selectedExample["content"]["application/json"]["schema"] !== undefined && selectedExample["content"]["application/json"]["schema"] !== null ) { - console.log("JSON: ", selectedExample["content"]["application/json"]["schema"]) + //console.log("JSON Output: ", selectedExample["content"]["application/json"]["schema"]) + + if (selectedExample["content"]["application/json"]["schema"]["properties"] !== undefined && selectedExample["content"]["application/json"]["schema"]["properties"] !== null) { + const jsonObject = getJsonObject(selectedExample["content"]["application/json"]["schema"]["properties"]) + console.log("ReSP Return: ", jsonObject) + if (jsonObject !== undefined && jsonObject !== null) { + try { + newaction.example_response = JSON.stringify(jsonObject, null, 2) + } catch (e) { + console.log("JSON object output parse error: ", e) + } + } + } + if (selectedExample["content"]["application/json"]["schema"]["$ref"] !== undefined) { //console.log("REF EXAMPLE: ", selectedExample["content"]["application/json"]["schema"]) const parameter = handleGetRef( @@ -1043,11 +1147,11 @@ const AppCreator = (defaultprops) => { data ); - console.log("Reading parameter type 2", parameter) + //console.log("Reading parameter type 2", parameter) if (parameter.properties !== undefined && parameter["type"] === "object") { var newbody = {}; for (let propkey in parameter.properties) { - console.log("propkey3: ", propkey) + //console.log("propkey3: ", propkey) const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); if (parameter.properties[propkey].type === undefined) { @@ -1090,10 +1194,7 @@ const AppCreator = (defaultprops) => { //const parameter = handleGetRef(selectedExample["content"]["application/json"]["schema"], data) newbody[parsedkey] = []; } else { - console.log( - "CANT HANDLE JSON TYPE ", - parameter.properties[propkey].type, - parameter.properties[propkey] + console.log("CANT HANDLE JSON TYPE ", parameter.properties[propkey].type,parameter.properties[propkey] ); newbody[parsedkey] = []; } @@ -1422,13 +1523,14 @@ const AppCreator = (defaultprops) => { if (firstUrl.endsWith("/")) { setBaseUrl(firstUrl.slice(0, firstUrl.length - 1)); + parentUrl = firstUrl.slice(0, firstUrl.length - 1) } else { - setBaseUrl(firstUrl); + setBaseUrl(firstUrl) + parentUrl = firstUrl } } } - //console.log("SECURITYSCHEMES: ", securitySchemes) if (securitySchemes !== undefined) { // FIXME: Should add Oauth2 (Microsoft) and JWT (Wazuh) //console.log("SECURITY: ", securitySchemes) @@ -1598,6 +1700,61 @@ const AppCreator = (defaultprops) => { } } + console.log("PARent: ", parentUrl) + var prefixCheck = "/v1" + if (parentUrl.includes("/")) { + const urlsplit = parentUrl.split("/") + if (urlsplit.length > 2) { + // Skip if http:// in it too + prefixCheck = "/" + urlsplit.slice(3).join("/") + } + + console.log("Prefix: ", prefixCheck) + if (prefixCheck.length > 0 && prefixCheck !== "/" && prefixCheck.startsWith("/")) { + for (var actionKey in newActions) { + const action = newActions[actionKey] + + if (action.url !== undefined && action.url !== null && action.url.startsWith(prefixCheck)) { + newActions[actionKey].url = action.url.slice(prefixCheck.length, action.url.length) + } + + console.log("Action: ", newActions[actionKey].url) + } + } + } + + console.log("Actions: ", newActions.length, " BaseURL: ", parentUrl) + + var newActions2 = [] + // Remove with duplicate action URLs + for (var actionKey in newActions) { + const action = newActions[actionKey] + if (action.url === undefined || action.url === null) { + continue + } + + var found = false + for (var actionKey2 in newActions2) { + const action2 = newActions2[actionKey2] + if (action2.url === undefined || action2.url === null) { + continue + } + + if (action.url === action2.url) { + found = true + break + } + } + + if (!found) { + newActions2.push(action) + } else { + console.log("Skipping duplicate action: ", action.url, " . Should merge contents") + } + } + + console.log("Actions: ", newActions.length, " Actions2: ", newActions2.length) + newActions = newActions2 if (newActions.length > increaseAmount - 1) { setActionAmount(increaseAmount); } else { @@ -1606,11 +1763,7 @@ const AppCreator = (defaultprops) => { //const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" if (newActions.length > 1000 && isCloud) { - alert.error( - "Cut down actions from " + - newActions.length + - " to 999 because of limit" - ); + alert.error("Cut down actions from " + newActions.length + " to 999 because of limit"); newActions = newActions.slice(0, 999); } diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index ba103c25..44c3e594 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -2628,7 +2628,7 @@ const Apps = (props) => { hidden type="file" ref={upload} - accept="application/JSON, application/YAML, text/yaml, text/x-yaml, application/x-yaml, application/vnd.yaml" + accept="application/JSON,application/YAML,application/yaml,text/yaml,text/x-yaml,application/x-yaml,application/vnd.yaml,.yml,.yaml" multiple={false} onChange={uploadFile} /> diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 2e023349..940503b3 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -51,7 +51,7 @@ import ( // Starts jobs in bulk, so this could be increased var sleepTime = 3 -var maxConcurrency = 50 +var maxConcurrency = 10 // Timeout if something rashes var workerTimeoutEnv = os.Getenv("SHUFFLE_ORBORUS_EXECUTION_TIMEOUT") @@ -973,7 +973,6 @@ func main() { log.Printf("[INFO] Finished configuring docker environment") client := shuffle.GetExternalClient(baseUrl) - fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) req, err := http.NewRequest( "GET",