From 6ae79cc87d6a90b57f833cd052e8219820e21726 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 7 Dec 2020 16:05:30 +0100 Subject: [PATCH 01/89] #133: Added first part of example and body parser --- frontend/src/views/AngularWorkflow.jsx | 76 +++++++++++- frontend/src/views/AppCreator.jsx | 156 ++++++++++++++++++++++--- 2 files changed, 215 insertions(+), 17 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e47c7c6c..9dddd448 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -845,7 +845,7 @@ const AngularWorkflow = (props) => { return response.json() }) - .then((responseJson) => { + .then((responseJson) => { // FIXME - handle versions on left bar //handleAppVersioning(responseJson) //var tmpapps = [] @@ -2872,8 +2872,64 @@ const AngularWorkflow = (props) => { placeholder = data.example } + var disabled = false + var rows = "5" + var openApiHelperText = "This is an OpenAPI specific field" + if (selectedApp.generated && selectedApp.activated && data.name === "body") { + const regex = /\${(\w+)}/g + const found = placeholder.match(regex) + console.log("THERE MAY BE THINGS INSIDE THE BODY, NO: ", found) + if (found === null) { + //setExtraBodyFields([]) + } else { + rows = "1" + disabled = true + openApiHelperText = "OpenAPI spec: fill the following fields." + console.log("SHOULD ADD TO selectedActionParameters!: ", found, selectedActionParameters) + var changed = false + for (var specKey in found) { + const tmpitem = found[specKey] + var skip = false + for (var innerkey in selectedActionParameters) { + if (selectedActionParameters[innerkey].name === tmpitem) { + skip = true + break + } + } + + if (skip) { + console.log("SKIPPING ", tmpitem) + continue + } + + changed = true + selectedActionParameters.push({ + action_field: "", + configuration: false, + description: "Generated by OpenAPI body example", + example: "", + id: "", + multiline: false, + name: tmpitem, + options: null, + required: false, + schema: {type: "string"}, + skip_multicheck: false, + tags: null, + value: "", + variant: "STATIC_VALUE", + }) + } + + if (changed) { + setSelectedActionParameters(selectedActionParameters) + } + } + } + var datafield = { }} fullWidth multiline={multiline} - rows="5" + rows={rows} color="primary" defaultValue={data.value} type={placeholder.includes("***") ? "password" : "text"} @@ -2909,6 +2965,13 @@ const AngularWorkflow = (props) => { onChange={(event) => { changeActionParameter(event, count) }} + helperText={selectedApp.generated && selectedApp.activated && data.name === "body" ? + + {openApiHelperText} + + : + null + } onBlur={(event) => { // Super basic check //if (event.target.value.startsWith("{")) { @@ -3303,7 +3366,7 @@ const AngularWorkflow = (props) => {
}
- {data.name} + {data.name.charAt(0).toUpperCase()+data.name.substring(1)}
{selectedActionParameters[count].options !== undefined && selectedActionParameters[count].options !== null && selectedActionParameters[count].options.length > 0 ? null : @@ -3685,7 +3748,7 @@ const AngularWorkflow = (props) => { {selectedAction.description}
: null}
- +
@@ -6253,6 +6316,11 @@ const AngularWorkflow = (props) => { getAppAuthentication() setUpdate(authenticationOption.id) + /* + {selectedAction.authentication.map(data => ( + + */ + } return ( diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index f47330c9..210a433d 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -4,6 +4,7 @@ import {BrowserView, MobileView} from "react-device-detect"; import {Link} from 'react-router-dom'; import Paper from '@material-ui/core/Paper'; +import Typography from '@material-ui/core/Typography'; import FormControlLabel from '@material-ui/core/FormControlLabel'; import Button from '@material-ui/core/Button'; import Divider from '@material-ui/core/Divider'; @@ -218,6 +219,7 @@ const AppCreator = (props) => { const [actions, setActions] = useState([]) const [errorCode, setErrorCode] = useState("") const [appBuilding, setAppBuilding] = useState(false) + const [extraBodyFields, setExtraBodyFields] = useState([]) //const [actions, setActions] = useState([{ // "name": "Get workflows", @@ -253,6 +255,7 @@ const AppCreator = (props) => { "queries": [], "body": "", "errors": [], + "example_response": "", "method": actionNonBodyRequest[0], }); @@ -262,7 +265,7 @@ const AppCreator = (props) => { if (firstrequest) { setFirstrequest(false) if (window.location.pathname.includes("apps/edit")) { - setIsEditing(true) + setIsEditing(true) handleEditApp() } else { checkQuery() @@ -433,6 +436,22 @@ const AppCreator = (props) => { "paths": [], "body": "", "errors": [], + "example_response": "", + } + + // HAHAHA wtf is this. + if (methodvalue.responses !== undefined) { + if (methodvalue.responses.default !== undefined) { + if (methodvalue.responses.default.content !== undefined) { + if (methodvalue.responses.default.content["text/plain"] !== undefined) { + if (methodvalue.responses.default.content["text/plain"]["schema"] !== undefined) { + if (methodvalue.responses.default.content["text/plain"]["schema"]["example"] !== undefined) { + newaction.example_response = methodvalue.responses.default.content["text/plain"]["schema"]["example"] + } + } + } + } + } } for (var key in methodvalue.parameters) { @@ -469,6 +488,7 @@ const AppCreator = (props) => { } } + if (newaction.name === "" || newaction.name === undefined) { // Find a unique part of the string // FIXME: Looks for length between /, find the one where they differ @@ -668,7 +688,14 @@ const AppCreator = (props) => { "responses": { "default": { "description": "default", - "schema": {} + "content": { + "text/plain": { + "schema": { + "type": "string", + "example": "", + }, + }, + }, } }, "summary": item.name, @@ -679,6 +706,35 @@ const AppCreator = (props) => { //console.log("ACTION: ", item) + if (item.example_response !== undefined && item.example_response.length > 0) { + // FIXME: Shallow copy of the string + var showResult = Object.assign("", item.example_response).trim() + showResult = showResult.split(" None").join(" \"None\"") + showResult = showResult.split("\'").join("\"") + showResult = showResult.split(" False").join(" false") + showResult = showResult.split(" True").join(" true") + + var jsonvalid = true + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } + + data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.type = "string" + if (jsonvalid) { + // FIXME: Add a JSON parser here - don't run it as a string. + data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = showResult + + } else { + data.paths[item.url][item.method.toLowerCase()].responses["default"]["content"]["text/plain"].schema.example = item.example_response + + } + } + if (item.queries.length > 0) { for (var querykey in item.queries) { const queryitem = item.queries[querykey] @@ -761,7 +817,7 @@ const AppCreator = (props) => { "type": "string", }, } - + // FIXME - add application/json if JSON example? data.paths[item.url][item.method.toLowerCase()]["requestBody"] = { "description": "Generated by Shuffler.io", @@ -773,6 +829,18 @@ const AppCreator = (props) => { }, } + /* + data.paths[item.url][item.method.toLowerCase()]["requestBody"] = { + "description": "Generated by Shuffler.io", + "required": required, + "content": { + "example": { + "example": item.body, + }, + }, + } + */ + data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem) } @@ -782,7 +850,6 @@ const AppCreator = (props) => { const headersSplit = item.headers.split("\n") for (var key in headersSplit) { const header = headersSplit[key] - console.log("HEADER: ", header) var key = "" var value = "" if (header.length > 0 && header.includes("= ")) { @@ -1137,12 +1204,15 @@ const AppCreator = (props) => { setUrlPathQueries(data.queries) setUrlPath(data.url) setActionsModalOpen(true) + + if (data["body"] !== undefined && data["body"] !== null && data["body"].length > 0) { + findBodyParams(data["body"]) + } }}>
{url} - {data.name} @@ -1177,24 +1247,52 @@ const AppCreator = (props) => { const setActionField = (field, value) => { currentAction[field] = value setCurrentAction(currentAction) + //setUrlPathQueries(currentAction.queries) } + const findBodyParams = (body) => { + const regex = /\${(\w+)}/g + const found = body.match(regex) + console.log("FOUND: ", found) + if (found === null) { + setExtraBodyFields([]) + } else { + setExtraBodyFields(found) + } + } + const bodyInfo = actionBodyRequest.includes(currentActionMethod) ? -
- Body - used as example in action argument +
+ Request Body: {extraBodyFields.length > 0 ? + + Variables: {extraBodyFields.join(", ")} + + : + + {`Add variables with \$\{ variable_name }`} + + } setActionField("body", e.target.value)} + onChange={e => { + setActionField("body", e.target.value) + findBodyParams(e.target.value) + }} key={currentAction} + helperText={ + + Shows an example body to the user. ${} creates variables. + + } InputProps={{ classes: { notchedOutline: classes.notchedOutline, @@ -1205,9 +1303,40 @@ const AppCreator = (props) => { }} /> +
+
: null + const exampleResponse = +
+ Example success response + setActionField("example_response", e.target.value)} + helperText={ + Helps with autocompletion and understanding of the endpoint + } + key={currentAction} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> +
+ const addActionToView = (errors) => { currentAction.errors = errors currentAction.queries = urlPathQueries @@ -1361,7 +1490,6 @@ const AppCreator = (props) => { open={actionsModalOpen} fullWidth onClose={() => { - console.log("CLOSED?") setUrlPath("") setCurrentAction({ "name": "", @@ -1564,7 +1692,7 @@ const AppCreator = (props) => { addPathQuery() }}>New query
- Headers - static for the action + Headers: static for the action { id="standard-required" defaultValue={currentAction["headers"]} multiline - rows="5" + rows="2" onChange={e => setActionField("headers", e.target.value)} helperText={Headers that are part of the request. Default: EMPTY} InputProps={{ @@ -1588,6 +1716,8 @@ const AppCreator = (props) => { }} /> {bodyInfo} + + {exampleResponse} @@ -188,9 +205,151 @@ const OrgHeader = (props) => {
{orgSaveButton}
-
-
+
+ +
+ { + setExpanded(!expanded) + }}> + {expanded ? + + : + + } + + {expanded ? + + + + + App Download URL + + { + setAppDownloadUrl(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + App Download Branch + + { + setAppDownloadBranch(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + Workflow Download URL + + { + setWorkflowDownloadUrl(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + + + + Workflow Download Branch + + { + setWorkflowDownloadBranch(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + + + {/* + + {expanded ? + + : + + } + + */} + + : + null + } +
) } diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 30f759d9..8eeb54d6 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -134,6 +134,7 @@ const Apps = (props) => { const [field2, setField2] = React.useState("") const [cursearch, setCursearch] = React.useState("") const [sharingConfiguration, setSharingConfiguration] = React.useState("you") + const [downloadBranch, setDownloadBranch] = React.useState("master") const [isDropzone, setIsDropzone] = React.useState(false); const upload = React.useRef(null); @@ -959,6 +960,7 @@ const Apps = (props) => { const parsedData = { "url": url, + "branch": downloadBranch || 'master' } if (field1.length > 0) { @@ -988,18 +990,23 @@ const Apps = (props) => { } setIsLoading(false) stop() + setValidation(false) return response.json() }) .then((responseJson) => { - console.log("DATA: ", responseJson) - if (responseJson.reason !== undefined) { - alert.error("Failed loading: "+responseJson.reason) - } + console.log("DATA: ", responseJson) + if (responseJson.reason !== undefined) { + alert.error("Failed loading: "+responseJson.reason) + } }) .catch(error => { console.log("ERROR: ", error.toString()) alert.error(error.toString()) + + stop() + setIsLoading(false) + setValidation(false) }) } @@ -1321,6 +1328,25 @@ const Apps = (props) => { placeholder="https://github.com/frikky/shuffle-apps" fullWidth /> + Branch (default value is "master"): +
+ setDownloadBranch(e.target.value)} + placeholder="master" + fullWidth + /> +
Authentication (optional - private repos etc):
From 82d42cf78225e1e3d5c4f2e5ebcddfb993697303 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 13 Dec 2020 12:43:17 +0100 Subject: [PATCH 15/89] #170: More stable multi-loop structure --- backend/app_sdk/app_base.py | 144 ++++++++++++++++++------------------ 1 file changed, 74 insertions(+), 70 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 2e1afe7d..eeb0fbbf 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -170,18 +170,12 @@ class AppBase: print("PARAMLIST: %s" % paramlist) - #newlist[subitem[0]] - #if len(newlist) > 0: # itemlength = len(newlist[0]) # How do we get it back, ordered? #for item in cartesian: - - - - #print("Listlengths: %s" % listlengths) #paramlist = [baseparams] @@ -1563,15 +1557,27 @@ class AppBase: print("APP_SDK DONE: Starting MULTI execution (length: %d) with values %s" % (minlength, multi_parameters)) # 1. Use number of executions based on the arrays being similar # 2. Find the right value from the parsed multi_params - results = [] - json_object = False - for i in range(0, minlength): - # To be able to use the results as a list: - print("1: %s" % multi_parameters) - #baseparams = json.loads(json.dumps(multi_parameters)) - baseparams = copy.deepcopy(multi_parameters) - print("2: %s: %s" % (type(baseparams), baseparams)) + print("Running without outer loop") + json_object = False + results = await self.run_recursed_items(func, multi_parameters, {}) + if isinstance(results, dict) or isinstance(results, list): + json_object = True + + #for i in range(0, minlength): + # # To be able to use the results as a list: + # print("1: %s" % multi_parameters) + # #baseparams = json.loads(json.dumps(multi_parameters)) + # baseparams = copy.deepcopy(multi_parameters) + + # print("2: %s: %s" % (type(baseparams), baseparams)) + + # print("4") + # print("Running with params (1): %s" % baseparams) + + # results = await self.run_recursed_items(func, baseparams, {}) + # if isinstance(results, dict) or isinstance(results, list): + # json_object = True # {'call': ['GoogleSafebrowsing_2_0', 'VirusTotal_GetReport_3_0']} # 1. Check if list length is same as minlength @@ -1580,68 +1586,66 @@ class AppBase: # arraylength = 4 ["1", "2", "3", "4"] # minlength = 12 - 12/3 = 4 per item = ["1", "1", "1", "1", "2", "2", ...] - try: - firstlist = True - for key, value in baseparams.items(): - print("Itemtype: %s" % type(value)) - if isinstance(value, list): - try: - newvalue = value[i] - except IndexError: - pass + #try: + # firstlist = True + # for key, value in baseparams.items(): + # print("Itemtype: %s" % type(value)) + # if isinstance(value, list): + # try: + # newvalue = value[i] + # except IndexError: + # pass - if len(value) != minlength and len(value) > 0: - newarray = [] - print("VALUE: ", value) - additiontime = minlength/len(value) - print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) - if firstlist: - print("Running normal list (FIRST)") - for subvalue in value: - for number in range(int(additiontime)): - newarray.append(subvalue) - else: - #print("Running secondary lists") - ## 1. Set up length of array - ## 2. Put values spread out - # FIXME: This works well, except if lists are same length - newarray = [""] * minlength + # if len(value) != minlength and len(value) > 0: + # newarray = [] + # print("VALUE: ", value) + # additiontime = minlength/len(value) + # print("Bad length for value: %d - should be %d. Additiontime: %d" % (len(value), minlength, additiontime)) + # if firstlist: + # print("Running normal list (FIRST)") + # for subvalue in value: + # for number in range(int(additiontime)): + # newarray.append(subvalue) + # else: + # #print("Running secondary lists") + # ## 1. Set up length of array + # ## 2. Put values spread out + # # FIXME: This works well, except if lists are same length + # newarray = [""] * minlength - cnt = 0 - for number in range(int(additiontime)): - for subvaluerange in range(len(value)): - # newlocation = number+(additiontime*subvaluerange) - # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) - # Reverse if same length? - if int(minlength/len(value)) == len(value): - tmp = int(len(value)-subvaluerange-1) - print("NEW: %d" % tmp) - newarray[cnt] = value[tmp] - else: - newarray[cnt] = value[subvaluerange] - cnt += 1 + # cnt = 0 + # for number in range(int(additiontime)): + # for subvaluerange in range(len(value)): + # # newlocation = number+(additiontime*subvaluerange) + # # print("%d+(%d*%d) = %d. VAL: %s" % (number, additiontime, subvaluerange, newlocation, value[subvaluerange])) + # # Reverse if same length? + # if int(minlength/len(value)) == len(value): + # tmp = int(len(value)-subvaluerange-1) + # print("NEW: %d" % tmp) + # newarray[cnt] = value[tmp] + # else: + # newarray[cnt] = value[subvaluerange] + # cnt += 1 - #print("Newarray =", newarray) - newvalue = newarray[i] - firstlist = False + # #print("Newarray =", newarray) + # newvalue = newarray[i] + # firstlist = False - baseparams[key] = newvalue + # baseparams[key] = newvalue - print("3") - except IndexError as e: - print("IndexError: %s" % e) - baseparams[key] = "IndexError: %s" % e - except KeyError as e: - print("KeyError: %s" % e) - baseparams[key] = "KeyError: %s" % e + # print("3") + #except IndexError as e: + # print("IndexError: %s" % e) + # baseparams[key] = "IndexError: %s" % e + #except KeyError as e: + # print("KeyError: %s" % e) + # baseparams[key] = "KeyError: %s" % e + #print("4") + #print("Running with params (1): %s" % baseparams) - - print("4") - print("Running with params (1): %s" % baseparams) - - results = await self.run_recursed_items(func, baseparams, {}) - if isinstance(results, dict) or isinstance(results, list): - json_object = True + #results = await self.run_recursed_items(func, baseparams, {}) + #if isinstance(results, dict) or isinstance(results, list): + # json_object = True # Check the structure here. If "isloop", try to recurse? # ret, is_loop = recurse_json(innervalue, parsersplit[outercnt+1:]) From a5b818772f81dd05b29597282d8d1e49fb6a3dab Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 13 Dec 2020 12:43:48 +0100 Subject: [PATCH 16/89] #216: Added backend components for Org settings --- backend/app_sdk/build.sh | 2 +- backend/go-app/main.go | 25 ++++++++++++++----------- backend/go-app/walkoff.go | 17 ++++++++++++++++- docker-compose.yml | 2 +- frontend/src/views/AngularWorkflow.jsx | 6 +++--- functions/onprem/orborus/build.sh | 2 +- 6 files changed, 36 insertions(+), 18 deletions(-) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 2260c6dc..9d39852b 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.3 +VERSION=0.8.31 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/main.go b/backend/go-app/main.go index e855d058..d68218e9 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3435,18 +3435,18 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { return } - parsedBody := string(body) + //parsedBody := string(body) //parsedBody = strings.Replace(parsedBody, "\"", "\\\"", -1) - if len(parsedBody) > 0 { - if string(parsedBody[0]) == `"` && string(parsedBody[len(parsedBody)-1]) == "\"" { - parsedBody = parsedBody[1 : len(parsedBody)-1] - } - } + //if len(parsedBody) > 0 { + // if string(parsedBody[0]) == `"` && string(parsedBody[len(parsedBody)-1]) == "\"" { + // parsedBody = parsedBody[1 : len(parsedBody)-1] + // } + //} newBody := ExecutionStruct{ Start: hook.Start, ExecutionSource: "webhook", - ExecutionArgument: string(parsedBody), + ExecutionArgument: string(body), } b, err := json.Marshal(newBody) @@ -7611,10 +7611,11 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { } type ReturnData struct { - Image string `json:"image" datastore:"image"` - Name string `json:"name" datastore:"name"` - Description string `json:"description" datastore:"description"` - OrgId string `json:"org_id" datastore:"org_id"` + Image string `json:"image" datastore:"image"` + Name string `json:"name" datastore:"name"` + Description string `json:"description" datastore:"description"` + OrgId string `json:"org_id" datastore:"org_id"` + Defaults Defaults `json:"defaults" datastore:"defaults"` } var tmpData ReturnData @@ -7685,6 +7686,8 @@ func handleEditOrg(resp http.ResponseWriter, request *http.Request) { org.Image = tmpData.Image org.Name = tmpData.Name org.Description = tmpData.Description + org.Defaults = tmpData.Defaults + //log.Printf("Org: %#v", org) err = setOrg(ctx, *org, org.Id) if err != nil { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 95346ae6..d35d36f6 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -116,6 +116,14 @@ type Org struct { SyncFeatures SyncFeatures `json:"sync_features" datastore:"sync_features"` Created int64 `json:"created" datastore:"created"` Edited int64 `json:"edited" datastore:"edited"` + Defaults Defaults `json:"defaults" datastore:"defaults"` +} + +type Defaults struct { + AppDownloadRepo string `json:"app_download_repo" datastore:"app_download_repo"` + AppDownloadBranch string `json:"app_download_branch" datastore:"app_download_branch"` + WorkflowDownloadRepo string `json:"workflow_download_repo" datastore:"workflow_download_repo"` + WorkflowDownloadBranch string `json:"workflow_download_branch" datastore:"workflow_download_branch"` } type AppAuthenticationStorage struct { @@ -5134,9 +5142,12 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { return } - // Field1 & 2 can be a lot of things.. + // Field1 & 2 can be a lot of things. + // Field1 = Username + // Field2 = Password type tmpStruct struct { URL string `json:"url"` + Branch string `json:"branch"` Field1 string `json:"field_1"` Field2 string `json:"field_2"` ForceUpdate bool `json:"force_update"` @@ -5159,6 +5170,10 @@ func loadSpecificApps(resp http.ResponseWriter, request *http.Request) { URL: tmpBody.URL, } + if len(tmpBody.Branch) > 0 && tmpBody.Branch != "master" && tmpBody.Branch != "main" { + cloneOptions.ReferenceName = plumbing.ReferenceName(tmpBody.Branch) + } + // FIXME: Better auth. if len(tmpBody.Field1) > 0 && len(tmpBody.Field2) > 0 { cloneOptions.Auth = &http2.BasicAuth{ diff --git a/docker-compose.yml b/docker-compose.yml index 5ed77582..f5f69046 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -45,7 +45,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.0 + image: ghcr.io/frikky/shuffle-orborus:0.8.3 container_name: shuffle-orborus hostname: shuffle-orborus networks: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index ba891857..c4b7ad08 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -402,7 +402,7 @@ const AngularWorkflow = (props) => { if (!visited.includes(item.action.label)) { if (executionRunning) { - alert.show("WAITING FOR "+item.action.label+" with result "+item.result) + //alert.show("WAITING FOR "+item.action.label+" with result "+item.result) visited.push(item.action.label) setVisited(visited) } @@ -424,7 +424,7 @@ const AngularWorkflow = (props) => { if (!visited.includes(item.action.label)) { if (executionRunning) { - alert.show("Success in node "+item.action.label) + //alert.show("Success in node "+item.action.label) //+" with result "+item.result) visited.push(item.action.label) setVisited(visited) @@ -459,7 +459,7 @@ const AngularWorkflow = (props) => { currentnode.addClass('failure-highlight') if (!visited.includes(item.action.label)) { - alert.error("Success for "+item.action.label+" with result "+item.result) + alert.error("Error for "+item.action.label+" with result "+item.result) visited.push(item.action.label) setVisited(visited) } diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 08ddda2f..010efe8d 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -6,6 +6,6 @@ echo "Running docker build with $NAME:$VERSION" docker build . -t frikky/shuffle:$NAME -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION #docker push frikky/$NAME:$VERSION -#docker push frikky/shuffle:$NAME # docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION +docker push frikky/shuffle:$NAME docker push ghcr.io/frikky/$NAME:$VERSION From cd5bd6a348476ea54d97f279824682ee0c2e5032 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 14 Dec 2020 10:36:03 +0100 Subject: [PATCH 17/89] Minor changes --- backend/app_sdk/app_base.py | 22 +++++++++++++++++++--- backend/go-app/codegen.go | 7 +++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index eeb0fbbf..e8bd3ca9 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -218,10 +218,25 @@ class AppBase: param_multiplier = await self.get_param_multipliers(newparams) print("Multiplier length: %d" % len(param_multiplier)) for subparams in param_multiplier: - ret.append(await func(**subparams)) + tmp = await func(**subparams) + + print("Return from execution: %s" % ret) + if tmp == None: + ret.append("") + elif isinstance(tmp, dict): + ret.append(tmp) + elif isinstance(tmp, list): + ret.append(tmp) + else: + tmp = tmp.replace("\"", "\\\"", -1) + + try: + ret.append(json.loads(tmp)) + except json.decoder.JSONDecodeError as e: + #print("Json: %s" % e) + ret.append(tmp) print("Ret length: %d" % len(ret)) - if len(ret) == 1: ret = ret[0] @@ -1477,6 +1492,7 @@ class AppBase: print("SCHEMA ERROR IN FILE HANDLING: %s" % e) # Fix lists here + # FIXME: This doesn't really do anything anymore print("CHECKING multi execution list!") if len(multi_execution_lists) > 0: print("\n Multi execution list has more data: %d" % len(multi_execution_lists)) @@ -1558,7 +1574,7 @@ class AppBase: # 1. Use number of executions based on the arrays being similar # 2. Find the right value from the parsed multi_params - print("Running without outer loop") + print("Running WITHOUT outer loop") json_object = False results = await self.run_recursed_items(func, multi_parameters, {}) if isinstance(results, dict) or isinstance(results, list): diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 2099affb..fe2b6967 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -948,6 +948,12 @@ func validateParameterName(name string) string { } } + newname = strings.ReplaceAll(newname, " ", "_") + newname = strings.ReplaceAll(newname, ",", "_") + newname = strings.ReplaceAll(newname, ".", "_") + newname = strings.ReplaceAll(newname, "|", "_") + newname = strings.ReplaceAll(newname, "-", "_") + return newname } @@ -1001,6 +1007,7 @@ func handleConnect(swagger *openapi3.Swagger, api WorkflowApp, extraParameters [ parsedName = strings.ReplaceAll(parsedName, ",", "_") parsedName = strings.ReplaceAll(parsedName, ".", "_") parsedName = strings.ReplaceAll(parsedName, "|", "_") + parsedName = strings.ReplaceAll(parsedName, "-", "_") parsedName = validateParameterName(parsedName) param.Value.Name = parsedName path.Connect.Parameters[counter].Value.Name = parsedName From 11c41f7e70b003f58028b9be234d2a11baaa5252 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 14 Dec 2020 18:02:55 +0100 Subject: [PATCH 18/89] #170: Fixed looping recursion issue with single-item list --- backend/app_sdk/app_base.py | 23 ++++++++++++++++++++--- backend/app_sdk/build.sh | 2 +- backend/go-app/walkoff.go | 4 ++-- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index e8bd3ca9..1d5bb7ee 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -127,7 +127,10 @@ class AppBase: print("%s is not a list: " % value) print("Listlengths: %s" % listlengths) - if len(listlengths) <= 1: + if len(listlengths) == 0: + print("NO multiplier. Running a single iteration.") + paramlist.append(baseparams) + elif len(listlengths) == 1: print("NO MULTIPLIER NECESSARY. Length is %d" % len(listitems)) for item in listitems: # This loops should always be length 1 @@ -189,6 +192,17 @@ class AppBase: newparams = {} for key, value in baseparams.items(): + if isinstance(value, list) and len(value) > 0: + print("In list check") + try: + value[0] = json.loads(value[0]) + except json.decoder.JSONDecodeError as e: + print("JSON casting error: %s" % e) + except TypeError as e: + print("TypeError: %s" % e) + + print("POST list check") + if isinstance(value, list) and len(value) == 1 and isinstance(value[0], list): try: loop_wrapper[key] += 1 @@ -202,6 +216,7 @@ class AppBase: has_loop = True else: print("Key %s is NOT a list within a list: %s" % (key, value)) + newparams[key] = value results = [] @@ -216,6 +231,7 @@ class AppBase: # If here: check for multipliers within this scope. ret = [] param_multiplier = await self.get_param_multipliers(newparams) + print("Multiplier length: %d" % len(param_multiplier)) for subparams in param_multiplier: tmp = await func(**subparams) @@ -248,7 +264,7 @@ class AppBase: results.append(ret) json_object = True elif isinstance(ret, list): - results.append(ret) + results = ret json_object = True else: ret = ret.replace("\"", "\\\"", -1) @@ -1324,7 +1340,7 @@ class AppBase: print("Before first part in multiexec!") handled = False if len(actualitem[0]) > 2 and actualitem[0][1] == "SHUFFLE_NO_SPLITTER": - print("Pre replacement: %s" % actualitem[0][2]) + print("(1) Pre replacement: %s" % actualitem[0][2]) tmpitem = value replacement = actualitem[0][2] @@ -1379,6 +1395,7 @@ class AppBase: multi_execution_lists.append(json_replacement) print("MULTI finished: %s" % json_replacement) else: + print("(2) Pre replacement: %s" % actualitem) # This is here to handle for loops within variables.. kindof # 1. Find the length of the longest array # 2. Build an array with the base values based on parameter["value"] diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 9d39852b..fe988a1c 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.31 +VERSION=0.8.32 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index d35d36f6..0437b7a0 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -5910,7 +5910,7 @@ func getWorkflowExecutions(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflow, err := getWorkflow(ctx, fileId) if err != nil { - log.Printf("Failed getting the workflow locally (get executions): %s", err) + log.Printf("Failed getting the workflow %s locally (get executions): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -5987,7 +5987,7 @@ func getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { _, err := dbclient.GetAll(ctx, q, &allworkflowapps) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - q := datastore.NewQuery("workflowapp").Limit(30).Order("-edited") + q := datastore.NewQuery("workflowapp").Limit(40).Order("-edited") _, err := dbclient.GetAll(ctx, q, &allworkflowapps) if err != nil { return []WorkflowApp{}, err From 50bcf3bf3da58507dad4bcf7d66850f4be4df752 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 15 Dec 2020 16:12:53 +0100 Subject: [PATCH 19/89] #170: Added parameter looping check for special apps --- backend/app_sdk/app_base.py | 34 ++++++++++++++++++++++++++-------- frontend/src/views/Apps.jsx | 4 ++-- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 1d5bb7ee..fd78155f 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -105,21 +105,36 @@ class AppBase: listlengths = [] all_lists = [] all_list_keys = [] + + #check_value = "$Filter_list_testing.wrapper.#.tmp" + #self.action = action + + for key, value in baseparams.items(): + check_value = "" + for param in self.action["parameters"]: + if param["name"] == key: + check_value = param["value"] + break + + print("\nCHECK: %s" % check_value) if isinstance(value, list): if len(value) <= 1: if len(value) == 1: baseparams[key] = value[0] else: - if len(value) not in listlengths: - listlengths.append(len(value)) + if not check_value.endswith("#"): + print("Adding WITHOUT looping list") + else: + if len(value) not in listlengths: + listlengths.append(len(value)) - listitems.append( - { - key: len(value) - } - ) + listitems.append( + { + key: len(value) + } + ) all_list_keys.append(key) all_lists.append(baseparams[key]) @@ -132,6 +147,7 @@ class AppBase: paramlist.append(baseparams) elif len(listlengths) == 1: print("NO MULTIPLIER NECESSARY. Length is %d" % len(listitems)) + for item in listitems: # This loops should always be length 1 for key, value in item.items(): @@ -244,7 +260,7 @@ class AppBase: elif isinstance(tmp, list): ret.append(tmp) else: - tmp = tmp.replace("\"", "\\\"", -1) + #tmp = tmp.replace("\"", "\\\"", -1) try: ret.append(json.loads(tmp)) @@ -432,6 +448,8 @@ class AppBase: "started_at": int(time.time()), "status": "EXECUTING" } + + self.action = action self.logger.info("ACTION RESULT (start): %s", action_result) if len(self.action) == 0: diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 8eeb54d6..3113bc31 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -589,10 +589,10 @@ const Apps = (props) => {
{imageline}
-
+

{newAppname}

Version {selectedApp.app_version}

-

{description}

+

{description}

{activateButton} From be9fef6a851e40dbeecee729b0202c4a65edcf89 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 18 Dec 2020 03:44:35 +0100 Subject: [PATCH 20/89] #204: Added file listing and downloads for org --- backend/go-app/files.go | 82 ++++++++++- backend/go-app/main.go | 14 +- backend/go-app/walkoff.go | 31 ++-- frontend/package.json | 2 +- frontend/src/views/Admin.jsx | 277 +++++++++++++++++++++++++++++------ 5 files changed, 340 insertions(+), 66 deletions(-) diff --git a/backend/go-app/files.go b/backend/go-app/files.go index a2856e2b..b9b30a6d 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -42,6 +42,7 @@ type File struct { DownloadPath string `json:"download_path" datastore:"download_path"` Md5sum string `json:"md5_sum" datastore:"md5_sum"` Sha256sum string `json:"sha256_sum" datastore:"sha256_sum"` + FileSize int64 `json:"filesize" datastore:"filesize"` } var basepath = os.Getenv("SHUFFLE_FILE_LOCATION") @@ -100,6 +101,51 @@ func fileExists(filename string) bool { return !info.IsDir() } +func handleGetFiles(resp http.ResponseWriter, request *http.Request) { + cors := handleCors(resp, request) + if cors { + return + } + + // 1. Check user directly + // 2. Check workflow execution authorization + user, err := handleApiAuthentication(resp, request) + if err != nil { + log.Printf("[INFO] INITIAL Api authentication failed in file LIST: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + if user.Role != "admin" { + log.Printf("[AUTH] User isn't admin") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Need to be admin"}`))) + return + } + + ctx := context.Background() + files, err := getAllFiles(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[ERROR] Failed to get files: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error getting files."}`))) + return + } + + log.Printf("Got %d files for org %s", len(files), user.ActiveOrg.Id) + newBody, err := json.Marshal(files) + if err != nil { + log.Printf("[ERROR] Failed marshaling files: %s", err) + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed to marshal files"}`)) + return + } + + resp.WriteHeader(200) + resp.Write([]byte(newBody)) +} + func handleGetFileMeta(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -417,8 +463,18 @@ func handleGetFileContent(resp http.ResponseWriter, request *http.Request) { Openfile, err := os.Open(downloadPath) defer Openfile.Close() //Close after function return if err != nil { + file.Status = "deleted" + err = setFile(ctx, *file) + if err != nil { + log.Printf("Failed setting file to uploading") + resp.WriteHeader(500) + resp.Write([]byte(`{"success": false, "reason": "Failed setting file to uploading"}`)) + return + } + //File not found, send 404 - http.Error(resp, "File not found.", 404) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false, "reason": "File doesn't exist locally"}`)) return } @@ -552,6 +608,7 @@ func handleUploadFile(resp http.ResponseWriter, request *http.Request) { var buf bytes.Buffer io.Copy(&buf, parsedFile) contents := buf.Bytes() + file.FileSize = int64(len(contents)) md5 := md5sum(contents) buf.Reset() @@ -754,6 +811,9 @@ func getFile(ctx context.Context, id string) (*File, error) { func setFile(ctx context.Context, file File) error { // clear session_token and API_token for user + timeNow := time.Now().Unix() + file.UpdatedAt = timeNow + k := datastore.NameKey("Files", file.Id, nil) if _, err := dbclient.Put(ctx, k, &file); err != nil { log.Println(err) @@ -762,3 +822,23 @@ func setFile(ctx context.Context, file File) error { return nil } + +func getAllFiles(ctx context.Context, orgId string) ([]File, error) { + var files []File + q := datastore.NewQuery("Files").Filter("org_id =", orgId).Order("-updated_at").Limit(100) + + _, err := dbclient.GetAll(ctx, q, &files) + if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { + q = q.Limit(50) + _, err := dbclient.GetAll(ctx, q, &files) + if err != nil { + return []File{}, err + } + } else { + return []File{}, err + } + } + + return files, nil +} diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d68218e9..7257fa6f 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -108,6 +108,7 @@ type StatisticsItem struct { Total int64 `json:"total" datastore:"total"` Fieldname string `json:"field_name" datastore:"field_name"` Data []StatisticsData `json:"data" datastore:"data"` + OrgId string `json:"org_id" datastore:"org_id"` } // "Execution by status" @@ -1148,7 +1149,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { } } - err = increaseStatisticsField(ctx, "successful_register", username, 1) + err = increaseStatisticsField(ctx, "successful_register", username, 1, org.Id) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -3471,7 +3472,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // OrgId: activeOrgs[0].Id, workflowExecution, executionResp, err := handleExecution(item, workflow, newRequest) if err == nil { - err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -3679,7 +3680,7 @@ func handleNewHook(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", requestdata.Workflow, 1, user.ActiveOrg.Id) if err != nil { log.Printf("[INFO] Failed to increase total workflows: %s", err) } @@ -6481,12 +6482,12 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { // Backup every single one setOpenApiDatastore(ctx, api.ID, parsed) - err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1) + err = increaseStatisticsField(ctx, "total_apps_created", newmd5, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } - err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1) + err = increaseStatisticsField(ctx, "openapi_apps_created", newmd5, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } @@ -6873,7 +6874,7 @@ func remoteOrgJobHandler(org Org, interval int) error { func runInit(ctx context.Context) { // Setting stats for backend starts (failure count as well) log.Printf("Starting INIT setup") - err := increaseStatisticsField(ctx, "backend_executions", "", 1) + err := increaseStatisticsField(ctx, "backend_executions", "", 1, "") if err != nil { log.Printf("Failed increasing local stats: %s", err) } @@ -8125,6 +8126,7 @@ func initHandlers() { r.HandleFunc("/api/v1/files/{fileId}/upload", handleUploadFile).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", handleGetFileMeta).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/files/{fileId}", handleDeleteFile).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/files", handleGetFiles).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0437b7a0..2045cf82 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -440,7 +440,7 @@ type AppExecutionExample struct { // This might be... a bit off, but that's fine :) // This might also be stupid, as we want timelines and such // Anyway, these are super basic stupid stats. -func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64) error { +func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64, orgId string) error { // 1. Get current stats // 2. Increase field(s) @@ -461,6 +461,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i if strings.Contains(fmt.Sprintf("%s", err), "entity") { statisticsItem = StatisticsItem{ Total: amount, + OrgId: orgId, Fieldname: fieldname, Data: []StatisticsData{ newData, @@ -1069,7 +1070,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } newResults = append(newResults, newResult) - increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1) + increaseStatisticsField(ctx, "workflow_execution_actions_skipped", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) } } } @@ -1094,12 +1095,12 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution.Results = newResults if workflowExecution.Status == "ABORTED" { - err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase aborted execution stats: %s", err) } } else if workflowExecution.Status == "FAILURE" { - err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase failure execution stats: %s", err) } @@ -1237,7 +1238,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl workflowExecution.LastNode = actionResult.Action.ID } - err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase success execution stats: %s", err) } @@ -1525,7 +1526,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() log.Printf("Saved new workflow %s with name %s", workflow.ID, workflow.Name) - err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1) + err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows stats: %s", err) } @@ -1725,7 +1726,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { } } - err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", workflow.ID, -1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } @@ -1741,7 +1742,7 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_workflows", fileId, -1) + err = increaseStatisticsField(ctx, "total_workflows", fileId, -1, workflow.OrgId) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } @@ -2314,7 +2315,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { totalOldActions := len(tmpworkflow.Actions) totalNewActions := len(workflow.Actions) - err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions)) + err = increaseStatisticsField(ctx, "total_workflow_actions", workflow.ID, int64(totalNewActions-totalOldActions), workflow.OrgId) if err != nil { log.Printf("Failed to change total actions data: %s", err) } @@ -2486,7 +2487,7 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase aborted execution stats: %s", err) } @@ -3069,7 +3070,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } } - err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1) + err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1, workflowExecution.ExecutionOrg) if err != nil { log.Printf("Failed to increase stats execution stats: %s", err) } @@ -4130,7 +4131,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { return } - err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1) + err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -5706,12 +5707,12 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin continue } - err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1) + err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1, "") if err != nil { log.Printf("Failed to increase total apps created stats: %s", err) } - err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1) + err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1, "") if err != nil { log.Printf("Failed to increase total apps loaded stats: %s", err) } @@ -6185,7 +6186,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { } if len(hook.Workflows) > 0 { - err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1) + err = increaseStatisticsField(ctx, "total_workflow_triggers", hook.Workflows[0], -1, user.ActiveOrg.Id) if err != nil { log.Printf("Failed to increase total workflows: %s", err) } diff --git a/frontend/package.json b/frontend/package.json index a4c40596..b627526f 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.6.0", + "version": "0.8.3", "private": true, "dependencies": { "@material-ui/core": "^4.5.2", diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ad857ad6..252af7ef 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -31,6 +31,9 @@ import { useTheme } from '@material-ui/core/styles'; import HandlePayment from './HandlePayment' import OrgHeader from '../components/OrgHeader' +import OpenInNewIcon from '@material-ui/icons/OpenInNew'; +import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; +import DescriptionIcon from '@material-ui/icons/Description'; import PolymerIcon from '@material-ui/icons/Polymer'; import CheckCircleIcon from '@material-ui/icons/CheckCircle'; import CloseIcon from '@material-ui/icons/Close'; @@ -78,6 +81,7 @@ const Admin = (props) => { const [environments, setEnvironments] = React.useState([]); const [authentication, setAuthentication] = React.useState([]); const [schedules, setSchedules] = React.useState([]) + const [files, setFiles] = React.useState([]) const [selectedUser, setSelectedUser] = React.useState({}) const [newPassword, setNewPassword] = React.useState(""); const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false) @@ -550,6 +554,78 @@ const Admin = (props) => { }); } + const getFiles = () => { + fetch(globalUrl + "/api/v1/files", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return + } + + return response.json() + }) + .then((responseJson) => { + console.log(responseJson) + setFiles(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + + const downloadFile = (file) => { + fetch(globalUrl + "/api/v1/files/"+file.id+"/content", { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + 'Accept': 'application/json', + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for apps :O!") + return "" + } + + return response.text() + }) + .then((respdata) => { + if (respdata.length === 0) { + alert.error("Failed getting file") + return + } + + var blob = new Blob( [ respdata ], { + type: 'application/octet-stream' + }) + + var url = URL.createObjectURL( blob ) + var link = document.createElement( 'a' ) + link.setAttribute( 'href', url ) + link.setAttribute( 'download', `${file.filename}` ) + var event = document.createEvent( 'MouseEvents' ) + event.initMouseEvent( 'click', true, true, window, 1, 0, 0, 0, 0, false, false, false, false, 0, null) + link.dispatchEvent( event ) + + //return response.json() + }) + .then((responseJson) => { + //console.log(responseJson) + //setSchedules(responseJson) + }) + .catch(error => { + alert.error(error.toString()) + }); + } + const getSchedules = () => { fetch(globalUrl + "/api/v1/workflows/schedules", { method: 'GET', @@ -705,6 +781,48 @@ const Admin = (props) => { }); } + const setConfig = (event, newValue) => { + if (newValue === 1) { + getUsers() + } else if (newValue === 2) { + getAppAuthentication() + } else if (newValue === 3) { + getEnvironments() + } else if (newValue === 4) { + getSchedules() + } else if (newValue === 5) { + getFiles() + } else if (newValue === 6) { + getOrgs() + } + + if (newValue === 6) { + console.log("Should get apps for categories.") + } + + const views = { + 0: "organization", + 1: "users", + 2: "app_auth", + 3: "environments", + 4: "schedules", + 5: "files", + 6: "categories", + } + + //var theURL = window.location.pathname + //FIXME: Add url edits + //var theURL = window.location + //theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`) + //window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath); + + //console.log(newpath) + //window.location.pathame = newpath + + setModalUser({}) + setCurTab(newValue) + } + if (firstRequest) { setFirstRequest(false) @@ -720,13 +838,14 @@ const Admin = (props) => { "app_auth": 2, "environments": 3, "schedules": 4, - "categories": 5, + "files": 5, } if (props.match.params.key !== undefined) { const tmpitem = views[props.match.params.key] if (tmpitem !== undefined) { - setCurTab(tmpitem) + //setCurTab(tmpitem) + setConfig("", tmpitem) } } } @@ -1532,6 +1651,113 @@ const Admin = (props) => {
: null + const filesView = curTab === 5 ? +
+
+

Files

+ Files from Workflows. Learn more +
+ + + + + + + + + + + + {files === undefined || files === null ? null : files.map((file, index) => { + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } + + return ( + + + + + + + + + + + style={{minWidth: 100, maxWidth: 100, overflow: "hidden"}} + /> + + + + + { + downloadFile(file) + }}> + + + + style={{minWidth: 75, maxWidth: 75, overflow: "hidden"}} + /> + {/* + + + + */} + + ) + })} + +
+ : null + const schedulesView = curTab === 4 ?
@@ -1860,7 +2086,7 @@ const Admin = (props) => {
: null - const organizationsTab = curTab === 6 ? + const organizationsTab = curTab === 7 ?

Organizations

@@ -1941,7 +2167,7 @@ const Admin = (props) => {
: null - const hybridTab = curTab === 5 ? + const hybridTab = curTab === 6 ?

Hybrid

@@ -1983,48 +2209,11 @@ const Admin = (props) => { // primary={environment.Registered ? "true" : "false"} - const setConfig = (event, newValue) => { - if (newValue === 1) { - getUsers() - } else if (newValue === 2) { - getAppAuthentication() - } else if (newValue === 3) { - getEnvironments() - } else if (newValue === 4) { - getSchedules() - } else if (newValue === 6) { - getOrgs() - } - - if (newValue === 6) { - console.log("Should get apps for categories.") - } - - const views = { - 0: "organization", - 1: "users", - 2: "app_auth", - 3: "environments", - 4: "schedules", - 5: "categories", - } - - //var theURL = window.location.pathname - //FIXME: Add url edits - //var theURL = window.location - //theURL.replace(`/${views[curTab]}`, `/${views[newValue]}`) - //window.history.pushState({"html":response.html,"pageTitle":response.pageTitle},"", urlPath); - - //console.log(newpath) - //window.location.pathame = newpath - - setModalUser({}) - setCurTab(newValue) - } + const iconStyle = {marginRight: 10} const data = -
+
{ {isCloud ? null : App Authentication/>} {isCloud ? null : Environments/>} {isCloud ? null : Schedules />} + {isCloud ? null : Files />} {window.location.protocol == "http:" && window.location.port === "3000" ? Hybrid/> : null} {window.location.protocol == "http:" && window.location.port === "3000" ? Organizations/> : null} {window.location.protocol === "http:" && window.location.port === "3000" ? Categories/> : null} @@ -2049,6 +2239,7 @@ const Admin = (props) => { {usersView} {environmentView} {schedulesView} + {filesView} {hybridTab} {organizationsTab}
From 79784d322678acd37dd0096430bd1747ba46058e Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 18 Dec 2020 04:02:45 +0100 Subject: [PATCH 21/89] #216: App and workflow downloads can now be organization-wide controlled --- backend/go-app/main.go | 67 ++++++++++++++++++++++++++++++++ frontend/src/App.jsx | 2 +- frontend/src/views/Apps.jsx | 6 +-- frontend/src/views/Workflows.jsx | 6 +-- 4 files changed, 74 insertions(+), 7 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 7257fa6f..c00389ee 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1727,6 +1727,14 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } } + // FIXME: Remove this dependency by updating users' orgs when org itself is updated + org, err := getOrg(ctx, userInfo.ActiveOrg.Id) + if err == nil { + userInfo.ActiveOrg = *org + userInfo.ActiveOrg.Users = []User{} + } + + log.Printf("Org: %#v", userInfo.ActiveOrg.Defaults) currentOrg, err := json.Marshal(userInfo.ActiveOrg) if err != nil { currentOrg = []byte("{}") @@ -2604,6 +2612,10 @@ func setOrg(ctx context.Context, org Org, id string) error { return err } + // FIXME: Make this update every user to have the correct org data. + //org = fixOrgUser(ctx, &org) + //_ = org + return nil } @@ -2711,6 +2723,61 @@ func setEnvironment(ctx context.Context, data *Environment) error { return nil } +func fixOrgUser(ctx context.Context, org *Org) *Org { + //found := false + //for _, id := range user.Orgs { + // if user.ActiveOrg.Id == id { + // found = true + // break + // } + //} + + //if !found { + // user.Orgs = append(user.Orgs, user.ActiveOrg.Id) + //} + + //// Might be vulnerable to timing attacks. + //for _, orgId := range user.Orgs { + // if len(orgId) == 0 { + // continue + // } + + // org, err := getOrg(ctx, orgId) + // if err != nil { + // log.Printf("Error getting org %s", orgId) + // continue + // } + + // orgIndex := 0 + // userFound := false + // for index, orgUser := range org.Users { + // if orgUser.Id == user.Id { + // orgIndex = index + // userFound = true + // break + // } + // } + + // if userFound { + // user.PrivateApps = []WorkflowApp{} + // user.Executions = ExecutionInfo{} + // user.Limits = UserLimits{} + // user.Authentication = []UserAuth{} + + // org.Users[orgIndex] = *user + // } else { + // org.Users = append(org.Users, *user) + // } + + // err = setOrg(ctx, *org, orgId) + // if err != nil { + // log.Printf("Failed setting org %s", orgId) + // } + //} + + return org +} + func fixUserOrg(ctx context.Context, user *User) *User { found := false for _, id := range user.Orgs { diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 5bdda5b9..0864b984 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -142,7 +142,7 @@ const App = (message, props) => { } /> } /> } /> - } /> + } /> } /> } /> { window.location.pathname = "/docs/about" }} /> diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 3113bc31..dadbe288 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -106,7 +106,7 @@ export const GetParsedPaths = (inputdata, basekey) => { const Apps = (props) => { - const { globalUrl, isLoggedIn, isLoaded } = props; + const { globalUrl, isLoggedIn, isLoaded, userdata } = props; //const [workflows, setWorkflows] = React.useState([]); const baseRepository = "https://github.com/frikky/shuffle-apps" @@ -1316,7 +1316,7 @@ const Apps = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - defaultValue="https://github.com/frikky/shuffle-apps" + defaultValue={userdata.active_org.defaults.app_download_repo !== undefined && userdata.active_org.defaults.app_download_repo.length > 0 ? userdata.active_org.defaults.app_download_repo : "https://github.com/frikky/shuffle-apps"} InputProps={{ style:{ color: "white", @@ -1334,7 +1334,7 @@ const Apps = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - value={downloadBranch} + defaultValue={userdata.active_org.defaults.app_download_branch !== undefined && userdata.active_org.defaults.app_download_branch.length > 0 ? userdata.active_org.defaults.app_download_branch : downloadBranch} InputProps={{ style:{ color: "white", diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index c0387cb0..3b0ec8bc 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -44,7 +44,7 @@ const inputColor = "#383B40" const surfaceColor = "#27292D" const Workflows = (props) => { - const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies} = props; + const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies, userdata} = props; document.title = "Shuffle - Workflows" const alert = useAlert() @@ -1348,7 +1348,7 @@ const Workflows = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - value={downloadUrl} + defaultValue={userdata.active_org.defaults.workflow_download_repo !== undefined && userdata.active_org.defaults.workflow_download_repo.length > 0 ? userdata.active_org.defaults.workflow_download_repo : downloadUrl} InputProps={{ style:{ color: "white", @@ -1367,7 +1367,7 @@ const Workflows = (props) => { style={{backgroundColor: inputColor}} variant="outlined" margin="normal" - value={downloadBranch} + defaultValue={userdata.active_org.defaults.workflow_download_branch !== undefined && userdata.active_org.defaults.workflow_download_branch.length > 0 ? userdata.active_org.defaults.workflow_download_branch : downloadBranch} InputProps={{ style:{ color: "white", From 4dd3e6be535257026430c2e1571831daf4b1d679 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 18 Dec 2020 10:26:23 +0100 Subject: [PATCH 22/89] BUG: User registered twice when created under org --- backend/go-app/main.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index c00389ee..d2229a78 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1140,7 +1140,7 @@ func createNewUser(username, password, role, apikey string, org Org) error { neworg, err := getOrg(ctx, org.Id) if err == nil { - neworg.Users = append(neworg.Users, *newUser) + //neworg.Users = append(neworg.Users, *newUser) err = setOrg(ctx, *neworg, neworg.Id) if err != nil { log.Printf("Failed updating org with user %s", newUser.Username) @@ -1734,7 +1734,6 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { userInfo.ActiveOrg.Users = []User{} } - log.Printf("Org: %#v", userInfo.ActiveOrg.Defaults) currentOrg, err := json.Marshal(userInfo.ActiveOrg) if err != nil { currentOrg = []byte("{}") @@ -2397,6 +2396,10 @@ func handleGetUsers(resp http.ResponseWriter, request *http.Request) { continue } + //for _, tmpUser := range newUsers { + // if tmpUser.Name + //} + item.Password = "" item.Session = "" item.VerificationToken = "" From b5d16d7458615a234fd5b5d574c1077269df38e8 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 19 Dec 2020 06:57:18 +0100 Subject: [PATCH 23/89] BUG: Issue with cgroups in Docker on certain servers --- .env | 2 +- docker-compose.yml | 6 +++--- frontend/src/views/Admin.jsx | 26 +++++++++++++++++++++++--- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 19 +++++++++++++++++-- 5 files changed, 45 insertions(+), 10 deletions(-) diff --git a/.env b/.env index a43c6669..5266d136 100644 --- a/.env +++ b/.env @@ -39,4 +39,4 @@ SHUFFLE_PASS_WORKER_PROXY=TRUE SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_NAME=frikky -SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.0" +SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.3" diff --git a/docker-compose.yml b/docker-compose.yml index f5f69046..d366dfce 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,8 +16,8 @@ services: depends_on: - backend backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.3 + build: ./backend + image: ghcr.io/frikky/shuffle-backend:0.8.4 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -45,7 +45,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.3 + image: ghcr.io/frikky/shuffle-orborus:0.8.31 container_name: shuffle-orborus hostname: shuffle-orborus networks: diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 252af7ef..e50a5a0c 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1582,8 +1582,13 @@ const Admin = (props) => { /> {users === undefined ? null : users.map((data, index) => { + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } + return ( - + { /> {schedules === undefined || schedules === null ? null : schedules.map((schedule, index) => { + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } + return ( - + {schedule.seconds} seconds} @@ -1938,8 +1948,13 @@ const Admin = (props) => { /> {authentication === undefined ? null : authentication.map((data, index) => { + var bgColor = "#27292d" + if (index % 2 === 0) { + bgColor = "#1f2023" + } + return ( - + style={{minWidth: 150, maxWidth: 150}} @@ -2046,6 +2061,11 @@ const Admin = (props) => { return null } + //var bgColor = "#27292d" + //if (index % 2 === 0) { + // bgColor = "#1f2023" + //} + return ( Date: Sat, 19 Dec 2020 08:06:03 +0100 Subject: [PATCH 24/89] BUG: Fixed an issue with Docker build references for older versions --- backend/go-app/docker.go | 28 ++++++++++++++++++++++++++++ backend/go-app/walkoff.go | 2 ++ frontend/src/views/AppCreator.jsx | 2 +- 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index b0398045..a95f4874 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -125,12 +125,25 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri return err } + log.Printf("FILENAME: %s", filename) readFile, err := ioutil.ReadAll(fileReader) if err != nil { log.Printf("Not file: %s", err) return err } + // Fixes issues with older versions of Docker for file format + if filename == "Dockerfile" { + log.Printf("Should search and replace in readfile.") + + referenceCheck := "FROM frikky/shuffle:" + if strings.Contains(string(readFile), referenceCheck) { + log.Printf("SHOULD SEARCH & REPLACE!") + newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:") + readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1)) + } + } + //log.Printf("Filename: %s", filename) // FIXME - might need the folder from EXTRA here // Name has to be e.g. just "requirements.txt" @@ -156,6 +169,21 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri return nil } +/* +// Fixes App SDK issues.. meh +func fixTags(tags []string) []string { + checkTag := "frikky/shuffle" + newTags := []string{} + for _, tag := range tags { + if strings.HasPrefix(tag, checkTags) { + newTags.append(newTags, fmt.Sprintf("registry.hub.docker.com/%s", tag)) + } + + newTags.append(tag) + } +} +*/ + // Custom Docker image builder wrapper in memory func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error { ctx := context.Background() diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 2045cf82..19634579 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -5541,6 +5541,8 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } } + log.Printf("HANDLING DOCKER FILEREADER - SEARCH&REPLACE?") + appfileData, err := ioutil.ReadAll(fileReader) if err != nil { log.Printf("Failed reading %s: %s", fullPath, err) diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index b8a2ee01..068d67fe 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1286,7 +1286,7 @@ const AppCreator = (props) => { : - + From 0ceea929e9634808d60b1144b779ca8fd04d592f Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 19 Dec 2020 09:56:19 +0100 Subject: [PATCH 25/89] BUG: Added remote download failover if build fails for a tag --- backend/go-app/docker.go | 53 ++++++++++++---- backend/go-app/walkoff.go | 6 +- docker-compose.yml | 4 +- frontend/src/views/AppCreator.jsx | 102 ++++++++++++++++-------------- frontend/src/views/LoginPage.jsx | 2 +- 5 files changed, 102 insertions(+), 65 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index a95f4874..540f2cbe 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -125,24 +125,28 @@ func getParsedTarMemory(fs billy.Filesystem, tw *tar.Writer, baseDir, extra stri return err } - log.Printf("FILENAME: %s", filename) + //log.Printf("FILENAME: %s", filename) readFile, err := ioutil.ReadAll(fileReader) if err != nil { log.Printf("Not file: %s", err) return err } - // Fixes issues with older versions of Docker for file format - if filename == "Dockerfile" { - log.Printf("Should search and replace in readfile.") + // Fixes issues with older versions of Docker and reference formats + // Specific to Shuffle rn. Could expand. + // FIXME: Seems like the issue was with multi-stage builds + /* + if filename == "Dockerfile" { + log.Printf("Should search and replace in readfile.") - referenceCheck := "FROM frikky/shuffle:" - if strings.Contains(string(readFile), referenceCheck) { - log.Printf("SHOULD SEARCH & REPLACE!") - newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:") - readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1)) + referenceCheck := "FROM frikky/shuffle:" + if strings.Contains(string(readFile), referenceCheck) { + log.Printf("SHOULD SEARCH & REPLACE!") + newReference := fmt.Sprintf("FROM registry.hub.docker.com/frikky/shuffle:") + readFile = []byte(strings.Replace(string(readFile), referenceCheck, newReference, -1)) + } } - } + */ //log.Printf("Filename: %s", filename) // FIXME - might need the folder from EXTRA here @@ -230,12 +234,39 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin dockerFileTarReader, buildOptions, ) + + log.Printf("Response: %#v", imageBuildResponse.Body) //log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body) defer imageBuildResponse.Body.Close() - _, newerr := io.Copy(os.Stdout, imageBuildResponse.Body) + buildBuf := new(strings.Builder) + _, newerr := io.Copy(buildBuf, imageBuildResponse.Body) if newerr != nil { log.Printf("Failed reading Docker build STDOUT: %s", newerr) + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n")) + + // Handles pulling of the same image if applicable + // This fixes some issues with older versions of Docker which can't build + // on their own ( <17.05 ) + pullOptions := types.ImagePullOptions{} + canonicalName := fmt.Sprintf("registry.hub.docker.com") + for _, image := range tags { + newImage := fmt.Sprintf("%s/%s", canonicalName, image) + log.Printf("[INFO] Pulling image %s", newImage) + reader, err := client.ImagePull(ctx, newImage, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed getting image %s: %s", newImage, err) + continue + } + + //newBuf := buildBuf + io.Copy(os.Stdout, reader) + log.Printf("[INFO] Successfully downloaded and built %s", newImage) + } + //baseDockerName + } } if err != nil { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 19634579..54365d60 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -5541,7 +5541,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } } - log.Printf("HANDLING DOCKER FILEREADER - SEARCH&REPLACE?") + //log.Printf("HANDLING DOCKER FILEREADER - SEARCH&REPLACE?") appfileData, err := ioutil.ReadAll(fileReader) if err != nil { @@ -5688,10 +5688,10 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if len(removeApps) > 0 { for _, item := range removeApps { - log.Printf("Removing duplicate: %s", item) + log.Printf("[WARNING] Removing duplicate: %s", item) err = DeleteKey(ctx, "workflowapp", item) if err != nil { - log.Printf("Failed deleting %s", item) + log.Printf("[ERROR] Failed deleting duplicate %s: %s", item, err) } } } diff --git a/docker-compose.yml b/docker-compose.yml index d366dfce..a1936a20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.3 + image: ghcr.io/frikky/shuffle-frontend:0.8.4 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.4 + image: ghcr.io/frikky/shuffle-backend:0.8.42 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 068d67fe..3171610a 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -101,7 +101,12 @@ const parseCurl = (s) => { return "" } - var args = rewrite(words.split(s)) + try { + var args = rewrite(words.split(s)) + } catch (e) { + return s + } + var out = { method: 'GET', header: {} } var state = '' @@ -1731,64 +1736,65 @@ const AppCreator = (props) => { var parsedurl = event.target.value if (parsedurl.startsWith("curl")) { const request = parseCurl(event.target.value) - console.log(request) - if (request.method.toUpperCase() !== currentAction.Method) { - setCurrentActionMethod(request.method.toUpperCase()) - setActionField("method", request.method.toUpperCase()) - } + if (request !== event.target.value) { + if (request.method.toUpperCase() !== currentAction.Method) { + setCurrentActionMethod(request.method.toUpperCase()) + setActionField("method", request.method.toUpperCase()) + } - if (request.header !== undefined && request.header !== null) { - var headers = [] - for (let [key, value] of Object.entries(request.header)) { - if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) { - continue + if (request.header !== undefined && request.header !== null) { + var headers = [] + for (let [key, value] of Object.entries(request.header)) { + if (parameterName !== undefined && key.toLowerCase() === parameterName.toLowerCase()) { + continue + } + + if (key === "Authorization" && authenticationOption === "Bearer auth") { + continue + } + + headers += key+"="+value+"\n" } - if (key === "Authorization" && authenticationOption === "Bearer auth") { - continue - } - - headers += key+"="+value+"\n" + setActionField("headers", headers) } - setActionField("headers", headers) - } + if (request.body !== undefined && request.body !== null) { + setActionField("body", request.body) + } - if (request.body !== undefined && request.body !== null) { - setActionField("body", request.body) - } - - // Parse URL - if (request.url !== undefined) { - parsedurl = request.url - } + // Parse URL + if (request.url !== undefined) { + parsedurl = request.url + } } - if (parsedurl !== undefined) { - if (parsedurl.includes("<") && parsedurl.includes(">")) { - parsedurl = parsedurl.split("<").join("{") - parsedurl = parsedurl.split(">").join("}") - } - - if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { - if (parsedurl !== undefined && parsedurl.includes(parameterName)) { - // Remove <> etc. - // - - console.log("IT HAS THE PARAM NAME!") - const newurl = new URL(encodeURI(parsedurl)) - newurl.searchParams.delete(parameterName) - parsedurl = decodeURI(newurl.href) + if (parsedurl !== undefined) { + if (parsedurl.includes("<") && parsedurl.includes(">")) { + parsedurl = parsedurl.split("<").join("{") + parsedurl = parsedurl.split(">").join("}") } - // Remove the base URL itself - if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { - parsedurl = parsedurl.replace(baseUrl, "") - } + if (parsedurl.startsWith("http") || parsedurl.startsWith("ftp")) { + if (parsedurl !== undefined && parsedurl.includes(parameterName)) { + // Remove <> etc. + // + + console.log("IT HAS THE PARAM NAME!") + const newurl = new URL(encodeURI(parsedurl)) + newurl.searchParams.delete(parameterName) + parsedurl = decodeURI(newurl.href) + } - // Check URL query && headers - setActionField("url", parsedurl) - setUrlPath(parsedurl) + // Remove the base URL itself + if (parsedurl !== undefined && baseUrl !== undefined && baseUrl.length > 0 && parsedurl.includes(baseUrl)) { + parsedurl = parsedurl.replace(baseUrl, "") + } + + // Check URL query && headers + setActionField("url", parsedurl) + setUrlPath(parsedurl) + } } } diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index ef746a6d..5f9e49f2 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -130,7 +130,7 @@ const LoginDialog = props => { if (responseJson["success"] === false) { setLoginInfo(responseJson["reason"]) } else { - setLoginInfo("Successful register :)") + setLoginInfo("Successful register!") } }), ) From e38950e285901df431bca1534c6a0a4d181c606a Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 19 Dec 2020 10:27:50 +0100 Subject: [PATCH 26/89] Added extension folder for Shuffle --- .../cortex-responders/Shuffle/shuffle.json | 35 +++++++++++++++ .../cortex-responders/Shuffle/shuffle.py | 28 ++++++++++++ functions/extensions/wazuh/ossec.conf | 7 +++ functions/extensions/wazuh/requirements.txt | 1 + functions/extensions/wazuh/shuffle.py | 44 +++++++++++++++++++ 5 files changed, 115 insertions(+) create mode 100644 functions/extensions/cortex-responders/Shuffle/shuffle.json create mode 100644 functions/extensions/cortex-responders/Shuffle/shuffle.py create mode 100644 functions/extensions/wazuh/ossec.conf create mode 100644 functions/extensions/wazuh/requirements.txt create mode 100644 functions/extensions/wazuh/shuffle.py diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.json b/functions/extensions/cortex-responders/Shuffle/shuffle.json new file mode 100644 index 00000000..ef2610dd --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle/shuffle.json @@ -0,0 +1,35 @@ +{ + "name": "Shuffle", + "version": "1.0", + "author": "@frikkylikeme", + "url": "https://github.com/frikky/shuffle", + "license": "AGPL-V3", + "description": "Execute a workflow in Shuffle", + "dataTypeList": ["thehive:case", "thehive:alert"], + "command": "Shuffle/shuffle.py", + "baseConfig": "Shuffle", + "configurationItems": [ + { + "name": "url", + "description": "The URL to your shuffle instance", + "type": "string", + "multi": false, + "required": true, + "defaultValue": "https://shuffler.io" + }, + { + "name": "api_key", + "description": "The API key to your Shuffle user", + "type": "string", + "multi": false, + "required": true + }, + { + "name": "workflow_id", + "description": "The ID of the workflow to execute", + "type": "string", + "multi": false, + "required": true + } + ] +} diff --git a/functions/extensions/cortex-responders/Shuffle/shuffle.py b/functions/extensions/cortex-responders/Shuffle/shuffle.py new file mode 100644 index 00000000..0816ca53 --- /dev/null +++ b/functions/extensions/cortex-responders/Shuffle/shuffle.py @@ -0,0 +1,28 @@ + +#!/usr/bin/env python +# encoding: utf-8 + +from cortexutils.responder import Responder +import requests + +class Shuffle(Responder): + def __init__(self): + Responder.__init__(self) + self.api_key = self.get_param("config.api_key", "") + self.url = self.get_param("config.url", "") + self.workflow_id = self.get_param("config.workflow_id", "") + + def run(self): + Responder.run(self) + + parsed_url = "%s/api/v1/workflows/%s/execute" % (self.url, self.workflow_id) + headers = { + "Authorization": "Bearer %s" % self.api_key + } + requests.post(parsed_url, headers=headers) + + self.report({'message': 'message sent'}) + +if __name__ == '__main__': + Shuffle().run() + diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf new file mode 100644 index 00000000..dfaf0394 --- /dev/null +++ b/functions/extensions/wazuh/ossec.conf @@ -0,0 +1,7 @@ + + Shuffle + http://:3001/api/v1/hooks/webhook_ + 2 + multiple_drops|authentication_failures + json + diff --git a/functions/extensions/wazuh/requirements.txt b/functions/extensions/wazuh/requirements.txt new file mode 100644 index 00000000..f2293605 --- /dev/null +++ b/functions/extensions/wazuh/requirements.txt @@ -0,0 +1 @@ +requests diff --git a/functions/extensions/wazuh/shuffle.py b/functions/extensions/wazuh/shuffle.py new file mode 100644 index 00000000..306d3715 --- /dev/null +++ b/functions/extensions/wazuh/shuffle.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python + +# Based on +# https://wazuh.com/blog/how-to-integrate-external-software-using-integrator/ + +import sys +import json +import requests +from requests.auth import HTTPBasicAuth + +# Set the project attributes +project_alias = 'TI' +issue_name ='FIM' + +# Read configuration parameters +alert_file = open(sys.argv[1]) +user = sys.argv[2].split(':')[0] +api_key = sys.argv[2].split(':')[1] +hook_url = sys.argv[3] + +# Read the alert file +alert_json = json.loads(alert_file.read()) +alert_file.close() + +# Extract issue fields +alert_level = alert_json['rule']['level'] +description = alert_json['rule']['description'] +path = alert_json['syscheck']['path'] + +# Generate request +msg_data = {} +msg_data['fields'] = {} +msg_data['fields']['project'] = {} +msg_data['fields']['project']['key'] = project_alias +msg_data['fields']['summary'] = 'FIM alert on [' + path + ']' +msg_data['fields']['description'] = '- State: ' + description + '\n- Alert level: ' + str(alert_level) +msg_data['fields']['issuetype'] = {} +msg_data['fields']['issuetype']['name'] = issue_name +headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} + +# Send the request +requests.post(hook_url, data=json.dumps(msg_data), headers=headers, auth=(user, api_key)) + +sys.exit(0) From 2248e407f76e80ac983dd6b6419fd9e10181a511 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 19 Dec 2020 14:21:39 +0100 Subject: [PATCH 27/89] BUG: Docker build issue for old version workaround --- backend/go-app/docker.go | 25 +++++++++++++--- backend/go-app/main.go | 3 +- backend/go-app/walkoff.go | 4 +-- docker-compose.yml | 8 +++--- functions/onprem/worker/worker.go | 48 ++++++++++++++++++++++++++----- 5 files changed, 70 insertions(+), 18 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 540f2cbe..3d48871c 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -8,6 +8,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" @@ -235,7 +236,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin buildOptions, ) - log.Printf("Response: %#v", imageBuildResponse.Body) + //log.Printf("Response: %#v", imageBuildResponse.Body) //log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body) defer imageBuildResponse.Body.Close() @@ -251,9 +252,12 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin // This fixes some issues with older versions of Docker which can't build // on their own ( <17.05 ) pullOptions := types.ImagePullOptions{} - canonicalName := fmt.Sprintf("registry.hub.docker.com") + downloaded := false for _, image := range tags { - newImage := fmt.Sprintf("%s/%s", canonicalName, image) + // Is this ok? Not sure. Tags shouldn't be controlled here prolly. + image = strings.ToLower(image) + + newImage := fmt.Sprintf("%s/%s", registryName, image) log.Printf("[INFO] Pulling image %s", newImage) reader, err := client.ImagePull(ctx, newImage, pullOptions) if err != nil { @@ -261,10 +265,17 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin continue } + // Attempt to retag the image to not contain registry... + //newBuf := buildBuf + downloaded = true io.Copy(os.Stdout, reader) log.Printf("[INFO] Successfully downloaded and built %s", newImage) } + + if !downloaded { + return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ","))) + } //baseDockerName } } @@ -331,9 +342,15 @@ func buildImage(tags []string, dockerfileFolder string) error { // Read the STDOUT from the build process defer imageBuildResponse.Body.Close() - _, err = io.Copy(os.Stdout, imageBuildResponse.Body) + buildBuf := new(strings.Builder) + _, err = io.Copy(buildBuf, imageBuildResponse.Body) if err != nil { return err + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), strings.Join(tags, "\n")) + return errors.New(fmt.Sprintf("Failed building %s. Check backend logs for details. Most likely means you have an old version of Docker.", strings.Join(tags, ","))) + } } return nil diff --git a/backend/go-app/main.go b/backend/go-app/main.go index d2229a78..b29e9fcf 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -72,6 +72,7 @@ var gceProject = "shuffle" var bucketName = "shuffler.appspot.com" var baseAppPath = "/home/frikky/git/shaffuru/tmp/apps" var baseDockerName = "frikky/shuffle" +var registryName = "registry.hub.docker.com" //var syncUrl = "http://192.168.102.54:5002" var syncUrl = "https://shuffler.io" @@ -6484,7 +6485,7 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) { dockerLocation := fmt.Sprintf("%s/Dockerfile", basePath) log.Printf("Dockerfile: %s", dockerLocation) - versionName := fmt.Sprintf("%s_%s", strings.ReplaceAll(api.Name, " ", "-"), api.AppVersion) + versionName := fmt.Sprintf("%s_%s", strings.ToLower(strings.ReplaceAll(api.Name, " ", "-")), api.AppVersion) dockerTags := []string{ fmt.Sprintf("%s:%s", baseDockerName, identifier), fmt.Sprintf("%s:%s", baseDockerName, versionName), diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 54365d60..448d677c 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -5599,7 +5599,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin newName = strings.ReplaceAll(newName, " ", "-") tags := []string{ - fmt.Sprintf("%s:%s_%s", baseDockerName, newName, workflowapp.AppVersion), + fmt.Sprintf("%s:%s_%s", baseDockerName, strings.ToLower(newName), workflowapp.AppVersion), } if len(allapps) == 0 { @@ -5673,7 +5673,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } if len(appendParams) > 0 { - log.Printf("Appending %d params to the START of %s", len(appendParams), action.Name) + log.Printf("[AUTH] Appending %d params to the START of %s", len(appendParams), action.Name) workflowapp.Actions[index].Parameters = append(appendParams, workflowapp.Actions[index].Parameters...) } diff --git a/docker-compose.yml b/docker-compose.yml index a1936a20..a39b58c7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,8 +16,8 @@ services: depends_on: - backend backend: - build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.42 + #build: ./backend + image: ghcr.io/frikky/shuffle-backend:0.8.43 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: @@ -53,8 +53,8 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: - - SHUFFLE_APP_SDK_VERSION=0.8.0 - - SHUFFLE_WORKER_VERSION=0.8.0 + - SHUFFLE_APP_SDK_VERSION=0.8.3 + - SHUFFLE_WORKER_VERSION=0.8.3 - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 2f9dfe55..067cdbe4 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -23,6 +23,8 @@ import ( var environment = os.Getenv("ENVIRONMENT_NAME") var baseUrl = os.Getenv("BASE_URL") var baseimagename = "frikky/shuffle" +var registryName = "registry.hub.docker.com" +var fallbackName = "shuffle-orborus" var sleepTime = 2 var containerId string @@ -34,6 +36,11 @@ func getThisContainerId() string { out, err := exec.Command("bash", "-c", cmd).Output() if err == nil { id = strings.TrimSpace(string(out)) + + log.Printf("Checking if %s is in %s", ".scope", string(out)) + if strings.Contains(string(out), ".scope") { + id = fallbackName + } } return id @@ -42,7 +49,7 @@ func getThisContainerId() string { func init() { containerId = getThisContainerId() if len(containerId) == 0 { - log.Printf("[ERROR] No container ID found.") + log.Printf("[ERROR] No container ID found. Not running containerized?") } else { log.Printf("[INFO] Found container ID: %s", containerId) } @@ -854,11 +861,17 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] ) if err != nil { - log.Printf("Container error: %s", err) + log.Printf("Container CREATE error: %s", err) + return err + } + + err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) + if err != nil { + log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + //shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) return err } - cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{}) log.Printf("[INFO] Container %s is created", cont.ID) return nil } @@ -1357,12 +1370,33 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W log.Printf("Skipping FULL_EXECUTION because size is larger than %d", maxSize) } + // Try original -> Go to lowercase err = deployApp(dockercli, image, identifier, env) if err != nil { - log.Printf("[ERROR] Failed deploying %s from image %s: %s", identifier, image, err) - if strings.Contains(err.Error(), "No such image") { - log.Printf("[ERROR] Image doesn't exist. Shutting down") - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. + // FIXME: Should try to remotely download directly if this persists. + image = fmt.Sprintf("%s:%s_%s", baseimagename, strings.ToLower(action.AppName), action.AppVersion) + if strings.Contains(image, " ") { + image = strings.ReplaceAll(image, " ", "-") + } + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + image = fmt.Sprintf("%s/%s:%s_%s", registryName, baseimagename, strings.ToLower(action.AppName), action.AppVersion) + if strings.Contains(image, " ") { + image = strings.ReplaceAll(image, " ", "-") + } + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + + log.Printf("[ERROR] Failed deploying image THRICE. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "No such image") { + //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) + log.Printf("[ERROR] Image doesn't exist. Shutting down") + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + } } } From 8a99fb60480d275562bdfc9ec3cccfd8a50a0233 Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 22 Dec 2020 11:41:11 +0100 Subject: [PATCH 28/89] FIXES: Added wazuh integration and loads more --- backend/app_sdk/app_base.py | 51 ++++-- backend/app_sdk/build.sh | 2 +- backend/go-app/docker.go | 1 + backend/go-app/main.go | 14 +- backend/go-app/walkoff.go | 6 +- docker-compose.yml | 4 +- frontend/src/views/Admin.jsx | 28 +++ frontend/src/views/AngularWorkflow.jsx | 40 ++++- frontend/src/views/Apps.jsx | 4 + frontend/src/views/LoginPage.jsx | 5 +- functions/extensions/wazuh/custom-shuffle | 36 ++++ functions/extensions/wazuh/custom-shuffle.py | 177 +++++++++++++++++++ functions/extensions/wazuh/ossec.conf | 4 +- functions/extensions/wazuh/requirements.txt | 1 - functions/extensions/wazuh/shuffle.py | 44 ----- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 7 +- 17 files changed, 333 insertions(+), 93 deletions(-) create mode 100644 functions/extensions/wazuh/custom-shuffle create mode 100644 functions/extensions/wazuh/custom-shuffle.py delete mode 100644 functions/extensions/wazuh/requirements.txt delete mode 100644 functions/extensions/wazuh/shuffle.py diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index fd78155f..e226eefd 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -606,11 +606,14 @@ class AppBase: if "len" in thistype or "length" in thistype or "lenght" in thistype: tmp = "" try: - tmpdata = data.replace("\'", "\"") tmp = json.loads(tmpdata) except: - print("Passing bug") - pass + try: + tmpdata = data.replace("\'", "\"") + tmp = json.loads(tmpdata) + except: + print("[ERROR] Parsing bug for length in app sdk") + pass if isinstance(tmp, list): return len(tmp) @@ -649,14 +652,14 @@ class AppBase: return tmp except IndexError as e: return default_error - + # Parses the INNER value and recurses until everything is done def parse_wrapper(data): try: if "(" not in data or ")" not in data: - return data + return (data, False) except TypeError: - return data + return (data, False) #print("Running %s" % data) @@ -671,7 +674,7 @@ class AppBase: break if not found: - return data + return (data, False) # Do stuff here. innervalue = parse_nested_param(data, maxDepth(data)-0) @@ -692,10 +695,11 @@ class AppBase: parsed_value = parse_type(innervalue[0], thistype.lower()) print("Parsed value from %s: %s" % (thistype, parsed_value)) - return parsed_value + return (parsed_value, True) print("DATA: %s\n" % data) - return parse_wrapper(data) + return (parse_wrapper(data)[0], True) + # Looks for parantheses to grab special cases within a string, e.g: # int(1) lower(HELLO) or length(what's the length) @@ -737,15 +741,20 @@ class AppBase: if len(newstring) > 0: newdata.append(newstring) - print("Newdata: ", newdata) parsedlist = [] non_string = False + parsed = False for item in newdata: ret = parse_wrapper(item) - if not isinstance(ret, str): + if not isinstance(ret[0], str): non_string = True - - parsedlist.append(ret) + + parsedlist.append(ret[0]) + if ret[1]: + parsed = True + + if not parsed: + return data if len(parsedlist) > 0 and not non_string: print("Returning parsed list: ", parsedlist) @@ -949,7 +958,6 @@ class AppBase: if len(parsersplit) == 1: return str(baseresult)+str(appendresult), False - baseresult = baseresult.replace("\'", "\"") baseresult = baseresult.replace(" True,", " true,") baseresult = baseresult.replace(" False", " false,") @@ -958,8 +966,12 @@ class AppBase: try: basejson = json.loads(baseresult) except json.decoder.JSONDecodeError as e: - print("Parser issue with JSON: %s" % e) - return str(baseresult)+str(appendresult), False + try: + baseresult = baseresult.replace("\'", "\"") + basejson = json.loads(baseresult) + except json.decoder.JSONDecodeError as e: + print("Parser issue with JSON: %s" % e) + return str(baseresult)+str(appendresult), False print("After fourth parser return as JSON") @@ -1365,14 +1377,17 @@ class AppBase: if replacement.startswith("\"") and replacement.endswith("\""): replacement = replacement[1:len(replacement)-1] - replacement = replacement.replace("\'", "\"", -1) print("POST replacement: %s" % replacement) json_replacement = replacement try: json_replacement = json.loads(replacement) except json.decoder.JSONDecodeError as e: - print("JSON error singular: %s" % e) + try: + replacement = replacement.replace("\'", "\"", -1) + json_replacement = json.loads(replacement) + except: + print("JSON error singular: %s" % e) if len(json_replacement) > minlength: minlength = len(json_replacement) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index fe988a1c..f96da989 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.32 +VERSION=0.8.4 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 3d48871c..03720e3b 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -230,6 +230,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin } // Build the actual image + log.Printf("Building %s. This may take up to a few minutes.", dockerfileFolder) imageBuildResponse, err := client.ImageBuild( ctx, dockerFileTarReader, diff --git a/backend/go-app/main.go b/backend/go-app/main.go index b29e9fcf..263e0872 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -2483,7 +2483,7 @@ func handleLogin(resp http.ResponseWriter, request *http.Request) { if len(users) != 1 { log.Printf(`Found multiple or no users with the same username: %s: %d`, data.Username, len(users)) resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Found %d users with the same username: %s"}`, len(users), data.Username))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Error: %d users with username %s"}`, len(users), data.Username))) return } @@ -4785,7 +4785,7 @@ func getDocList(resp http.ResponseWriter, request *http.Request) { if len(item1) == 0 { resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No docs available."`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No docs available."}`))) return } @@ -4844,7 +4844,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { location := strings.Split(request.URL.String(), "/") if len(location) != 5 { resp.WriteHeader(404) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) return } @@ -4868,7 +4868,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { ) if err != nil { - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) resp.WriteHeader(404) //setBadMemcache(ctx, docPath) return @@ -4877,7 +4877,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { newresp, err := client.Do(req) if err != nil { resp.WriteHeader(404) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad path. Use e.g. /api/v1/docs/workflows.md"}`))) //setBadMemcache(ctx, docPath) return } @@ -4885,7 +4885,7 @@ func getDocs(resp http.ResponseWriter, request *http.Request) { body, err := ioutil.ReadAll(newresp.Body) if err != nil { resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse data"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't parse data"}`))) //setBadMemcache(ctx, docPath) return } @@ -5958,7 +5958,7 @@ func echoOpenapiData(resp http.ResponseWriter, request *http.Request) { if err != nil { log.Printf("[ERROR] URLbody error: %s", err) resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"`))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Can't get data from selected uri"}`))) return } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 448d677c..d2d99d56 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -4015,7 +4015,7 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) { log.Printf("ID: %s", fileId) app, err := getApp(ctx, fileId) if err != nil { - log.Printf("Error getting app %s: %s", app.Name, err) + log.Printf("Error getting app (delete) %s: %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4169,7 +4169,7 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() app, err := getApp(ctx, fileId) if err != nil { - log.Printf("Error getting app: %s", app.Name) + log.Printf("Error getting app (app config): %s", fileId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -4438,7 +4438,7 @@ func updateWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() app, err := getApp(ctx, fileId) if err != nil { - log.Printf("Error getting app: %s (update app)", app.Name) + log.Printf("Error getting app (update app): %s", fileId) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return diff --git a/docker-compose.yml b/docker-compose.yml index a39b58c7..7a356dff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.4 + image: ghcr.io/frikky/shuffle-frontend:0.8.42 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -45,7 +45,7 @@ services: - database orborus: #build: ./functions/onprem/orborus - image: ghcr.io/frikky/shuffle-orborus:0.8.31 + image: ghcr.io/frikky/shuffle-orborus:0.8.32 container_name: shuffle-orborus hostname: shuffle-orborus networks: diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index e50a5a0c..b809cf62 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -470,6 +470,33 @@ const Admin = (props) => { }) } + const flushQueue = (name) => { + // Just use this one? + const url = globalUrl + '/api/v1/flush_queue'; + fetch(url, { + method: 'DELETE', + credentials: "include", + headers: { + 'Content-Type': 'application/json', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error(responseJson.reason) + getEnvironments() + } else { + setLoginInfo("") + setModalOpen(false) + getEnvironments() + } + }), + ) + .catch(error => { + console.log("Error when deleting: ", error) + }) + } + const deleteEnvironment = (name) => { // FIXME - add some check here ROFL alert.info("Deleting environment " + name) @@ -2094,6 +2121,7 @@ const Admin = (props) => { style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}} > + {/**/} { const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false //const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" - const triggerEnvironments = isCloud ? ["cloud"] : ["cloud", "onprem"] + const triggerEnvironments = isCloud ? ["cloud"] : ["onprem", "cloud"] const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?' useBeforeunload(() => { @@ -2663,7 +2663,6 @@ const AngularWorkflow = (props) => { foundResult.result = foundResult.result.split(" None").join(" \"None\"") foundResult.result = foundResult.result.split(" False").join(" false") foundResult.result = foundResult.result.split(" True").join(" true") - foundResult.result = foundResult.result.split("\'").join("\"") var jsonvalid = true try { @@ -2672,7 +2671,15 @@ const AngularWorkflow = (props) => { jsonvalid = false } } catch (e) { - jsonvalid = false + try { + foundResult.result = foundResult.result.split("\'").join("\"") + const tmp = String(JSON.parse(foundResult.result)) + if (!foundResult.result.includes("{") && !foundResult.result.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } } // Finds the FIRST json only @@ -5874,7 +5881,6 @@ const AngularWorkflow = (props) => { const parsedExecutionArgument = () => { var showResult = executionData.execution_argument.trim() showResult = showResult.split(" None").join(" \"None\"") - showResult = showResult.split("\'").join("\"") showResult = showResult.split(" False").join(" false") showResult = showResult.split(" True").join(" true") @@ -5885,7 +5891,16 @@ const AngularWorkflow = (props) => { jsonvalid = false } } catch (e) { - jsonvalid = false + showResult = showResult.split("\'").join("\"") + + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } } if (jsonvalid) { @@ -6070,21 +6085,28 @@ const AngularWorkflow = (props) => { // // FIXME: The latter replace doens't really work if ' is used in a string var showResult = data.result.trim() + //console.log(showResult) showResult = showResult.split(" None").join(" \"None\"") showResult = showResult.split(" False").join(" false") showResult = showResult.split(" True").join(" true") - showResult = showResult.split("\'").join("\"") var jsonvalid = true try { const tmp = String(JSON.parse(showResult)) if (!showResult.includes("{") && !showResult.includes("[")) { - //console.log("IN HERE: ", tmp) jsonvalid = false } } catch (e) { - //console.log("Error: ", e) - jsonvalid = false + showResult = showResult.split("\'").join("\"") + + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } } const curapp = apps.find(a => a.name === data.action.app_name && a.app_version === data.action.app_version) diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index dadbe288..c081f94f 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -46,6 +46,10 @@ const inputColor = "#383B40" export const GetParsedPaths = (inputdata, basekey) => { const splitkey = " > " var parsedValues = [] + if (inputdata === undefined || inputdata === null) { + return parsedValues + } + if (typeof(inputdata) !== "object") { return parsedValues } diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 5f9e49f2..b365336f 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -69,7 +69,7 @@ const LoginDialog = props => { }), ) .catch(error => { - setLoginInfo("Error in userdata: ", error) + setLoginInfo("Error logging in: ", error) }) } @@ -80,6 +80,7 @@ const LoginDialog = props => { const onSubmit = (e) => { e.preventDefault() + setLoginInfo("") // FIXME - add some check here ROFL // Just use this one? @@ -114,7 +115,7 @@ const LoginDialog = props => { }), ) .catch(error => { - setLoginInfo("Error in userdata: " + error) + setLoginInfo("Error logging in: " + error) }); } else { url = baseurl + '/api/v1/users/register'; diff --git a/functions/extensions/wazuh/custom-shuffle b/functions/extensions/wazuh/custom-shuffle new file mode 100644 index 00000000..bd540414 --- /dev/null +++ b/functions/extensions/wazuh/custom-shuffle @@ -0,0 +1,36 @@ +#!/bin/sh +# Created by Shuffle, AS. . + +WPYTHON_BIN="framework/python/bin/python3" + +SCRIPT_PATH_NAME="$0" + +DIR_NAME="$(cd $(dirname ${SCRIPT_PATH_NAME}); pwd -P)" +SCRIPT_NAME="$(basename ${SCRIPT_PATH_NAME})" + +case ${DIR_NAME} in + */active-response/bin | */wodles*) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/../..; pwd)" + fi + + PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" + ;; + */bin) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" + fi + + PYTHON_SCRIPT="${WAZUH_PATH}/framework/scripts/${SCRIPT_NAME}.py" + ;; + */integrations) + if [ -z "${WAZUH_PATH}" ]; then + WAZUH_PATH="$(cd ${DIR_NAME}/..; pwd)" + fi + + PYTHON_SCRIPT="${DIR_NAME}/${SCRIPT_NAME}.py" + ;; +esac + + +${WAZUH_PATH}/${WPYTHON_BIN} ${PYTHON_SCRIPT} "$@" diff --git a/functions/extensions/wazuh/custom-shuffle.py b/functions/extensions/wazuh/custom-shuffle.py new file mode 100644 index 00000000..06fa4c7d --- /dev/null +++ b/functions/extensions/wazuh/custom-shuffle.py @@ -0,0 +1,177 @@ +#!/usr/bin/env python +# Created by Shuffle, AS. . +# Based on the Slack integration using Webhooks + +import json +import sys +import time +import os + +try: + import requests + from requests.auth import HTTPBasicAuth +except Exception as e: + print("No module 'requests' found. Install: pip install requests") + sys.exit(1) + +# ADD THIS TO ossec.conf configuration: +# +# custom-shuffle +# http://:3001/api/v1/hooks/ +# 3 +# json +# + +# Global vars + +debug_enabled = False +pwd = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) +json_alert = {} +now = time.strftime("%a %b %d %H:%M:%S %Z %Y") + +# Set paths +log_file = '{0}/logs/integrations.log'.format(pwd) + + +def main(args): + debug("# Starting") + + # Read args + alert_file_location = args[1] + webhook = args[3] + + debug("# Webhook") + debug(webhook) + + debug("# File location") + debug(alert_file_location) + + # Load alert. Parse JSON object. + with open(alert_file_location) as alert_file: + json_alert = json.load(alert_file) + debug("# Processing alert") + debug(json_alert) + + debug("# Generating message") + msg = generate_msg(json_alert) + if isinstance(msg, str): + if len(msg) == 0: + return + debug(msg) + + debug("# Sending message") + send_msg(msg, webhook) + + +def debug(msg): + if debug_enabled: + msg = "{0}: {1}\n".format(now, msg) + print(msg) + f = open(log_file, "a") + f.write(msg) + f.close() + +# Skips container kills to stop self-recursion +def filter_msg(alert): + # These are things that recursively happen because Shuffle starts Docker containers + # Docker integration rules: https://github.com/wazuh/wazuh-ruleset/blob/ae36745db1d3f312db0392f5925c2f2b0ec009a9/rules/0560-docker_integration_rules.xml + skip = ["87924", "87900", "87901", "87902", "87903", "87904", "86001", "86002", "86003", "87932", "80710", "87929", "87928",] + if alert["rule"]["id"] in skip: + return False + + #try: + # if "docker" in alert["rule"]["description"].lower() and " + #msg['text'] = alert.get('full_log') + #except: + # pass + #msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" + + return True + +def generate_msg(alert): + if not filter_msg(alert): + print("Skipping rule %s" % alert["rule"]["id"]) + return "" + + level = alert['rule']['level'] + + if (level <= 4): + color = "good" + elif (level >= 5 and level <= 7): + color = "warning" + else: + color = "danger" + + msg = {} + msg['color'] = color + msg['pretext'] = "WAZUH Alert" + msg['title'] = alert['rule']['description'] if 'description' in alert['rule'] else "N/A" + msg['text'] = alert.get('full_log') + msg['rule_id'] = alert["rule"]["id"] + msg['timestamp'] = alert["timestamp"] + msg['id'] = alert['id'] + msg["all_fields"] = alert + + #msg['fields'] = [] + # msg['fields'].append({ + # "title": "Agent", + # "value": "({0}) - {1}".format( + # alert['agent']['id'], + # alert['agent']['name'] + # ), + # }) + #if 'agentless' in alert: + # msg['fields'].append({ + # "title": "Agentless Host", + # "value": alert['agentless']['host'], + # }) + + #msg['fields'].append({"title": "Location", "value": alert['location']}) + #msg['fields'].append({ + # "title": "Rule ID", + # "value": "{0} _(Level {1})_".format(alert['rule']['id'], level), + #}) + + #attach = {'attachments': [msg]} + + return json.dumps(msg) + + +def send_msg(msg, url): + headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} + res = requests.post(url, data=msg, headers=headers) + debug(res) + + +if __name__ == "__main__": + try: + # Read arguments + bad_arguments = False + if len(sys.argv) >= 4: + msg = '{0} {1} {2} {3} {4}'.format( + now, + sys.argv[1], + sys.argv[2], + sys.argv[3], + sys.argv[4] if len(sys.argv) > 4 else '', + ) + debug_enabled = (len(sys.argv) > 4 and sys.argv[4] == 'debug') + else: + msg = '{0} Wrong arguments'.format(now) + bad_arguments = True + + # Logging the call + f = open(log_file, 'a') + f.write(msg + '\n') + f.close() + + if bad_arguments: + debug("# Exiting: Bad arguments.") + sys.exit(1) + + # Main function + main(sys.argv) + + except Exception as e: + debug(str(e)) + raise diff --git a/functions/extensions/wazuh/ossec.conf b/functions/extensions/wazuh/ossec.conf index dfaf0394..5b55f5d3 100644 --- a/functions/extensions/wazuh/ossec.conf +++ b/functions/extensions/wazuh/ossec.conf @@ -1,7 +1,5 @@ - Shuffle + custom-shuffle http://:3001/api/v1/hooks/webhook_ - 2 - multiple_drops|authentication_failures json diff --git a/functions/extensions/wazuh/requirements.txt b/functions/extensions/wazuh/requirements.txt deleted file mode 100644 index f2293605..00000000 --- a/functions/extensions/wazuh/requirements.txt +++ /dev/null @@ -1 +0,0 @@ -requests diff --git a/functions/extensions/wazuh/shuffle.py b/functions/extensions/wazuh/shuffle.py deleted file mode 100644 index 306d3715..00000000 --- a/functions/extensions/wazuh/shuffle.py +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env python - -# Based on -# https://wazuh.com/blog/how-to-integrate-external-software-using-integrator/ - -import sys -import json -import requests -from requests.auth import HTTPBasicAuth - -# Set the project attributes -project_alias = 'TI' -issue_name ='FIM' - -# Read configuration parameters -alert_file = open(sys.argv[1]) -user = sys.argv[2].split(':')[0] -api_key = sys.argv[2].split(':')[1] -hook_url = sys.argv[3] - -# Read the alert file -alert_json = json.loads(alert_file.read()) -alert_file.close() - -# Extract issue fields -alert_level = alert_json['rule']['level'] -description = alert_json['rule']['description'] -path = alert_json['syscheck']['path'] - -# Generate request -msg_data = {} -msg_data['fields'] = {} -msg_data['fields']['project'] = {} -msg_data['fields']['project']['key'] = project_alias -msg_data['fields']['summary'] = 'FIM alert on [' + path + ']' -msg_data['fields']['description'] = '- State: ' + description + '\n- Alert level: ' + str(alert_level) -msg_data['fields']['issuetype'] = {} -msg_data['fields']['issuetype']['name'] = issue_name -headers = {'content-type': 'application/json', 'Accept-Charset': 'UTF-8'} - -# Send the request -requests.post(hook_url, data=json.dumps(msg_data), headers=headers, auth=(user, api_key)) - -sys.exit(0) diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index a580aefc..57fe63a2 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.31 +VERSION=0.8.32 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 8e5cf442..06a938ff 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -562,6 +562,7 @@ func zombiecheck(workerTimeout int) error { stopContainers := []string{} removeContainers := []string{} + log.Printf("Workertimeout: %d", int64(workerTimeout)) for _, container := range containers { // Skip random containers. Only handle things related to Shuffle. if !strings.Contains(container.Image, baseimagename) { @@ -587,10 +588,10 @@ func zombiecheck(workerTimeout int) error { continue } - log.Printf("[INFO] NAME: %s", name) + currenttime := time.Now().Unix() + log.Printf("[INFO] (%s) NAME: %s. TIME: %d", container.State, name, currenttime-container.Created) // Need to check time here too because a container can be removed the same instant as its created - currenttime := time.Now().Unix() if container.State != "running" && currenttime-container.Created > int64(workerTimeout) { removeContainers = append(removeContainers, container.ID) containerNames[container.ID] = name @@ -606,6 +607,7 @@ func zombiecheck(workerTimeout int) error { } // FIXME - add killing of apps with same execution ID too + log.Printf("[INFO] Should STOP %d containers.", len(stopContainers)) for _, containername := range stopContainers { log.Printf("[INFO] Stopping and removing container %s", containerNames[containername]) go dockercli.ContainerStop(ctx, containername, nil) @@ -617,6 +619,7 @@ func zombiecheck(workerTimeout int) error { Force: true, } + log.Printf("[INFO] Should REMOVE %d containers.", len(removeContainers)) for _, containername := range removeContainers { go dockercli.ContainerRemove(ctx, containername, removeOptions) } From c2cf104700bfb8b439922164b960cba7a294220f Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 24 Dec 2020 07:35:27 +0100 Subject: [PATCH 29/89] Feature: Added contains_any_of to conditions --- backend/app_sdk/app_base.py | 16 +++++++++ backend/go-app/walkoff.go | 14 ++++---- docker-compose.yml | 2 +- frontend/src/components/OrgHeader.js | 2 +- frontend/src/views/AngularWorkflow.jsx | 5 +++ functions/onprem/orborus/orborus.go | 2 +- functions/onprem/worker/worker.go | 45 +++++++++++++++++++++----- 7 files changed, 69 insertions(+), 17 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index e226eefd..98f62feb 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1127,6 +1127,22 @@ class AppBase: elif check.lower() == "contains": if destinationvalue.lower() in sourcevalue.lower(): return True + elif check.lower() == "contains_any_of": + newvalue = [destinationvalue.lower()] + if "," in destinationvalue: + newvalue = destinationvalue.split(",") + elif ", " in destinationvalue: + newvalue = destinationvalue.split(", ") + + for item in new_value: + if not item: + continue + + if item.trim() in sourcevalue: + print("[INFO] Found %s in %s" % (item, sourcevalue)) + return True + + return False elif check.lower() == "larger than": try: if sourcevalue.isdigit() and destinationvalue.isdigit(): diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index d2d99d56..85796a74 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -785,7 +785,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() workflowExecution, err := getWorkflowExecution(ctx, actionResult.ExecutionId) if err != nil { - log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) + //log.Printf("Failed getting execution (streamresult) %s: %s", actionResult.ExecutionId, err) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Bad authorization key or execution_id might not exist."}`))) return @@ -1004,7 +1004,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if actionResult.Status == "ABORTED" || actionResult.Status == "FAILURE" { - log.Printf("Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status) + log.Printf("[WARNING] Actionresult is %s. Should set workflowExecution and exit all running functions", actionResult.Status) newResults := []ActionResult{} childNodes := []string{} @@ -1016,7 +1016,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Finds ALL childnodes to set them to SKIPPED childNodes = findChildNodes(*workflowExecution, actionResult.Action.ID) // Remove duplicates - log.Printf("CHILD NODES: %d", len(childNodes)) + //log.Printf("CHILD NODES: %d", len(childNodes)) for _, nodeId := range childNodes { if nodeId == actionResult.Action.ID { continue @@ -1290,7 +1290,7 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if _, err = tx.Commit(); err != nil { - if attempts >= 5 { + if attempts >= 7 { log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) tx.Rollback() workflowExecution.Status = "ABORTED" @@ -1301,7 +1301,9 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } - log.Printf("[WARNING] tx.Commit %d: %v", attempts, err) + if attempts > 3 { + log.Printf("[WARNING] tx.Commit %d: %v", attempts, err) + } attempts += 1 runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) @@ -2604,7 +2606,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } // This one doesn't really matter. - log.Printf("Running POST execution with data %s", body) + log.Printf("Running POST execution with body of length %d", len(string(body))) var execution ExecutionRequest err = json.Unmarshal(body, &execution) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index 7a356dff..c575474a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.42 + image: ghcr.io/frikky/shuffle-frontend:0.8.43 container_name: shuffle-frontend hostname: shuffle-frontend ports: diff --git a/frontend/src/components/OrgHeader.js b/frontend/src/components/OrgHeader.js index 7f21cdce..da6cf847 100644 --- a/frontend/src/components/OrgHeader.js +++ b/frontend/src/components/OrgHeader.js @@ -130,7 +130,7 @@ const OrgHeader = (props) => { return (
- +
0 ? null : "1px solid #f85a3e", cursor: "pointer", backgroundColor: imageData !== undefined && imageData.length > 0 ? null : theme.palette.inputColor, maxWidth: 174, maxHeight: 174}} onClick={() => {upload.click()}}> upload = ref} onChange={editHeaderImage} /> {imageInfo} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e9f2f1ef..77451c91 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4366,6 +4366,11 @@ const AngularWorkflow = (props) => { setConditionValue(conditionValue) setVariableAnchorEl(null) }} key={"contains"}>contains + { + conditionValue.value = "contains_any_of" + setConditionValue(conditionValue) + setVariableAnchorEl(null) + }} key={"contains_any_of"}>contains { conditionValue.value = "matches regex" setConditionValue(conditionValue) diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 06a938ff..cf6a8b1a 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -106,7 +106,7 @@ func getThisContainerId() { // cgroup error. Hardcoding this. // https://github.com/moby/moby/issues/7015 - log.Printf("Checking if %s is in %s", ".scope", string(out)) + //log.Printf("Checking if %s is in %s", ".scope", string(out)) if strings.Contains(string(out), ".scope") { containerId = "shuffle-orborus" //docker-76c537e9a4b7c7233011f5d70e6b7f2d600b6413ac58a96519b8dca7a3f7117a.scope diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 067cdbe4..d0267d14 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -6,7 +6,7 @@ import ( "encoding/json" "errors" "fmt" - //"io" + "io" "io/ioutil" "log" "net/http" @@ -37,7 +37,7 @@ func getThisContainerId() string { if err == nil { id = strings.TrimSpace(string(out)) - log.Printf("Checking if %s is in %s", ".scope", string(out)) + //log.Printf("Checking if %s is in %s", ".scope", string(out)) if strings.Contains(string(out), ".scope") { id = fallbackName } @@ -1370,7 +1370,12 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W log.Printf("Skipping FULL_EXECUTION because size is larger than %d", maxSize) } - // Try original -> Go to lowercase + // Uses a few ways of getting / checking if an app is available + // 1. Try original + // 2. Go to lowercase + // 3. Add remote repo location + // 4. Actually download last repo + err = deployApp(dockercli, image, identifier, env) if err != nil { // Trying to replace with lowercase to deploy again. This seems to work with Dockerhub well. @@ -1389,13 +1394,37 @@ func handleExecution(client *http.Client, req *http.Request, workflowExecution W err = deployApp(dockercli, image, identifier, env) if err != nil { - - log.Printf("[ERROR] Failed deploying image THRICE. Aborting if the image doesn't exist") - if strings.Contains(err.Error(), "No such image") { - //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) - log.Printf("[ERROR] Image doesn't exist. Shutting down") + log.Printf("[WARNING] Failed deploying image THRICE. Attempting to download the latter as last resort.") + reader, err := dockercli.ImagePull(context.Background(), image, pullOptions) + if err != nil { + log.Printf("[ERROR] Failed getting %s. The couldn't be find locally, AND is missing.", image) shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) } + + buildBuf := new(strings.Builder) + _, err = io.Copy(buildBuf, reader) + if err != nil { + log.Printf("[ERROR] Error in IO copy: %s", err) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } else { + if strings.Contains(buildBuf.String(), "errorDetail") { + log.Printf("[ERROR] Docker build:\n%s\nERROR ABOVE: Trying to pull tags from: %s", buildBuf.String(), image) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + + log.Printf("[INFO] Successfully downloaded %s", image) + } + + err = deployApp(dockercli, image, identifier, env) + if err != nil { + + log.Printf("[ERROR] Failed deploying image for the FOURTH time. Aborting if the image doesn't exist") + if strings.Contains(err.Error(), "No such image") { + //log.Printf("[WARNING] Failed deploying %s from image %s: %s", identifier, image, err) + log.Printf("[ERROR] Image doesn't exist. Shutting down") + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + } } } } From 287840b961b0dc015cde54f6d5f4a2efcd259764 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 26 Dec 2020 17:04:40 +0100 Subject: [PATCH 30/89] BUG: Fixed resource overload when getting workflows --- backend/app_sdk/build.sh | 2 +- backend/go-app/docker.go | 6 +-- backend/go-app/main.go | 61 ++++++++++++++------------ backend/go-app/walkoff.go | 25 ++++++++--- docker-compose.yml | 2 +- frontend/src/views/AngularWorkflow.jsx | 4 +- 6 files changed, 59 insertions(+), 41 deletions(-) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index f96da989..08128a44 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.4 +VERSION=0.8.41 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 03720e3b..5775b537 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -501,7 +501,7 @@ func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (stop docker): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -633,7 +633,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (start docker): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -781,7 +781,7 @@ func hookTest() { returnHook, err := getHook(ctx, hook.Id) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (test): %s", hook.Id, err) } if len(returnHook.Id) > 0 { diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 263e0872..d6ebc6cb 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3096,7 +3096,7 @@ func handleSetHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() _, err = getHook(ctx, workflowId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (set): %s", workflowId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false, "message": "Invalid ID"}`)) return @@ -3457,7 +3457,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { //log.Printf("HookID: %s", hookId) hook, err := getHook(ctx, hookId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (callback): %s", hookId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3470,7 +3470,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { //resp.WriteHeader(200) //resp.Write([]byte(`{"success": true}`)) if hook.Status == "stopped" { - log.Printf("Not running because hook status is stopped") + log.Printf("Not running %s because hook status is stopped", hook.Id) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "The webhook isn't running. Click start to start it"}`))) return @@ -3493,20 +3493,34 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { ExecutionArgument string `json:"execution_argument"` } + body, err := ioutil.ReadAll(request.Body) + if err != nil { + log.Printf("Body data error: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + + newBody := ExecutionStruct{ + Start: hook.Start, + ExecutionSource: "webhook", + ExecutionArgument: string(body), + } + + b, err := json.Marshal(newBody) + if err != nil { + log.Printf("Failed newBody marshaling: %s", err) + resp.WriteHeader(401) + resp.Write([]byte(`{"success": false}`)) + return + } + for _, item := range hook.Workflows { log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) workflow := Workflow{ ID: "", } - body, err := ioutil.ReadAll(request.Body) - if err != nil { - log.Printf("Body data error: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - //parsedBody := string(body) //parsedBody = strings.Replace(parsedBody, "\"", "\\\"", -1) //if len(parsedBody) > 0 { @@ -3515,20 +3529,6 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { // } //} - newBody := ExecutionStruct{ - Start: hook.Start, - ExecutionSource: "webhook", - ExecutionArgument: string(body), - } - - b, err := json.Marshal(newBody) - if err != nil { - log.Printf("Failed newBody marshaling: %s", err) - resp.WriteHeader(401) - resp.Write([]byte(`{"success": false}`)) - return - } - //bodyWrapper := fmt.Sprintf(`{"start": "%s", "execution_source": "webhook", "execution_argument": "%s"}`, hook.Start, string(parsedBody)) //if len(hook.Start) == 0 { // log.Printf("No start node for hook %s - running with workflow default.", hook.Id) @@ -3798,7 +3798,7 @@ func sendHookResult(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, workflowId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (send): %s", workflowId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -3865,7 +3865,7 @@ func handleGetHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, workflowId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (get hook): %s", workflowId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -7152,7 +7152,12 @@ func runInit(ctx context.Context) { } } } else { - log.Printf("Found %d users.", len(users)) + if len(users) == 1 { + log.Printf("Found 1 user - %s.", users[0].Username) + } else { + log.Printf("Found %d users.", len(users)) + } + if len(activeOrgs) == 1 && len(users) > 0 { for _, user := range users { if user.ActiveOrg.Id == "" && len(user.Username) > 0 { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 85796a74..5dab4903 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1281,8 +1281,11 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl // Prevents timing issues //ExecutionId if _, err := tx.Put(key, workflowExecution); err != nil { - tx.Rollback() - log.Printf("[ERROR] tx.Put bug: %v", err) + log.Printf("[ERROR] tx.Put error: %v", err) + err = tx.Rollback() + if err != nil { + log.Printf("[ERROR] Rollback error (3): %s", err) + } resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Failed setting workflowexecution actionresult: %s"}`, err))) @@ -1290,9 +1293,14 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl } if _, err = tx.Commit(); err != nil { + err = tx.Rollback() + if err != nil { + log.Printf("[ERROR] Rollback error expected ? (1): %s", err) + } + if attempts >= 7 { log.Printf("[ERROR] QUITTING: tx.Commit %d: %v", attempts, err) - tx.Rollback() + workflowExecution.Status = "ABORTED" setWorkflowExecution(ctx, *workflowExecution) @@ -1308,6 +1316,11 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl attempts += 1 runWorkflowExecutionTransaction(ctx, attempts, workflowExecutionId, actionResult, resp) return + } else { + //if grpc.Code(err) == codes.Aborted { + // return nil, ErrConcurrentTransaction + //} + //t.id = nil // mark the transaction as expired } resp.WriteHeader(200) @@ -6093,7 +6106,7 @@ func handleStopHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (stop): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -6176,7 +6189,7 @@ func handleDeleteHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (delete): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return @@ -6317,7 +6330,7 @@ func handleStartHook(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() hook, err := getHook(ctx, fileId) if err != nil { - log.Printf("Failed getting hook: %s", err) + log.Printf("Failed getting hook %s (start): %s", fileId, err) resp.WriteHeader(401) resp.Write([]byte(`{"success": false}`)) return diff --git a/docker-compose.yml b/docker-compose.yml index c575474a..81934cf1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.43 + image: ghcr.io/frikky/shuffle-backend:0.8.44 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 77451c91..84220f25 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4370,7 +4370,7 @@ const AngularWorkflow = (props) => { conditionValue.value = "contains_any_of" setConditionValue(conditionValue) setVariableAnchorEl(null) - }} key={"contains_any_of"}>contains + }} key={"contains_any_of"}>contains any of { conditionValue.value = "matches regex" setConditionValue(conditionValue) @@ -5979,7 +5979,7 @@ const AngularWorkflow = (props) => { {workflowExecutions.length > 0 ?
{workflowExecutions.map(data => { - const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" ? "red" : "orange" + const statusColor = data.status === "FINISHED" ? "green" : data.status === "ABORTED" || data.status === "FAILED" ? "red" : "orange" const timeElapsed = data.completed_at-data.started_at const resultsLength = data.results !== undefined && data.results !== null ? data.results.length : 0 From 7f12df6e31575b85e19906c23302842243550525 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 26 Dec 2020 18:34:57 +0100 Subject: [PATCH 31/89] Added copy feature for JSON objects --- backend/go-app/docker.go | 6 ++-- backend/go-app/walkoff.go | 12 +++---- frontend/src/views/AngularWorkflow.jsx | 47 +++++++++++++++++++++++++- 3 files changed, 55 insertions(+), 10 deletions(-) diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 5775b537..06b9b086 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -202,7 +202,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin tw := tar.NewWriter(buf) defer tw.Close() - log.Printf("Setting up memory build structure for folder: %s", dockerfileFolder) + log.Printf("[INFO] Setting up memory build structure for folder: %s", dockerfileFolder) err = getParsedTarMemory(fs, tw, dockerfileFolder, "") if err != nil { log.Printf("Tar issue: %s", err) @@ -230,7 +230,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin } // Build the actual image - log.Printf("Building %s. This may take up to a few minutes.", dockerfileFolder) + log.Printf("[INFO] Building %s. This may take up to a few minutes.", dockerfileFolder) imageBuildResponse, err := client.ImageBuild( ctx, dockerFileTarReader, @@ -713,7 +713,7 @@ func handleStartHookDocker(resp http.ResponseWriter, request *http.Request) { } // FIXME - get some real data? - log.Printf("Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath) + log.Printf("[INFO] Successfully started %s-%s on port %s with filepath %s", image, fileId, port, filepath) resp.WriteHeader(200) resp.Write([]byte(`{"success": true, "message": "Started webhook"}`)) return diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 5dab4903..60a01b11 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -5776,23 +5776,23 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin log.Printf("Failed image build memory: %s", err) } else { if len(item.Tags) > 0 { - log.Printf("Successfully built image %s", item.Tags[0]) + log.Printf("[INFO] Successfully built image %s", item.Tags[0]) } else { - log.Printf("Successfully built Docker image") + log.Printf("[INFO] Successfully built Docker image") } } } - log.Printf("Starting build of %d skipped docker images", len(buildLaterList)) + log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList)) for _, item := range buildLaterList { err = buildImageMemory(fs, item.Tags, item.Extra) if err != nil { - log.Printf("Failed image build memory: %s", err) + log.Printf("[INFO] Failed image build memory: %s", err) } else { if len(item.Tags) > 0 { - log.Printf("Successfully built image %s", item.Tags[0]) + log.Printf("[INFO] Successfully built image %s", item.Tags[0]) } else { - log.Printf("Successfully built Docker image") + log.Printf("[INFO] Successfully built Docker image") } } } diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 84220f25..e03e4138 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -5957,8 +5957,49 @@ const AngularWorkflow = (props) => { ) } + + const HandleJsonCopy = (base, copy, base_node_name) => { + console.log("COPY: ", copy) + var newitem = JSON.parse(base) + var to_be_copied = "$"+base_node_name + for (var key in copy.namespace) { + if (copy.namespace[key].includes("Results for")) { + continue + } + + if (newitem !== undefined && newitem !== null) { + newitem = newitem[copy.namespace[key]] + if (!isNaN(copy.namespace[key])) { + to_be_copied += ".#" + } else { + to_be_copied += "."+copy.namespace[key] + } + } + } + + if (newitem !== undefined && newitem !== null) { + newitem = newitem[copy.name] + if (!isNaN(copy.name)) { + to_be_copied += ".#" + } else { + to_be_copied += "."+copy.name + } + } + + console.log(to_be_copied) + //var copyText = document.getElementById("copy_element_shuffle"); + //if (copyText !== null) { + //navigator.clipboard.writeText(to_be_copied) + //copyText.select(); + //copyText.setSelectionRange(0, 99999); /* For mobile devices */ + + ///* Copy the text inside the text field */ + //document.execCommand("copy"); + //} + } + const executionModal = - setExecutionModalOpen(false)} PaperProps={{style: {minWidth: 375, maxWidth: 375, backgroundColor: "#1F2023", color: "white", fontSize: 18}}}> + setExecutionModalOpen(false)} style={{resize: "both", overflow: "auto",}} PaperProps={{style: {resize: "both", overflow: "auto", minWidth: 400, maxWidth: 400, backgroundColor: "#1F2023", color: "white", fontSize: 18}}}> {executionModalView === 0 ?
@@ -6136,6 +6177,10 @@ const AngularWorkflow = (props) => { theme="solarized" collapsed={true} displayDataTypes={false} + onSelect={(select) => { + HandleJsonCopy(showResult, select, data.action.name) + console.log("SELECTED!: ", select) + }} name={"Results for "+data.action.label} /> : From 49ee75006c5fae727113771e41cab11de1c21b34 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 27 Dec 2020 08:30:22 +0100 Subject: [PATCH 32/89] Feature: added popuut result --- frontend/src/views/AngularWorkflow.jsx | 135 ++++++++++++++++++++++--- frontend/src/views/Workflows.jsx | 85 ++++++++-------- 2 files changed, 166 insertions(+), 54 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index e03e4138..73382661 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -19,6 +19,7 @@ import Select from '@material-ui/core/Select'; import MenuItem from '@material-ui/core/MenuItem'; import Divider from '@material-ui/core/Divider'; import Dialog from '@material-ui/core/Dialog'; +import Modal from '@material-ui/core/Modal'; import DialogActions from '@material-ui/core/DialogActions'; import DialogTitle from '@material-ui/core/DialogTitle'; import InputLabel from '@material-ui/core/InputLabel'; @@ -37,8 +38,11 @@ import Switch from '@material-ui/core/Switch'; import ReactJson from 'react-json-view' import { useBeforeunload } from 'react-beforeunload'; import NestedMenuItem from "material-ui-nested-menu-item"; +import Fade from '@material-ui/core/Fade'; import ArrowUpwardIcon from '@material-ui/icons/ArrowUpward'; +import CloseIcon from '@material-ui/icons/Close'; +import ArrowLeftIcon from '@material-ui/icons/ArrowLeft'; import CachedIcon from '@material-ui/icons/Cached'; import AddIcon from '@material-ui/icons/Add'; import DirectionsRunIcon from '@material-ui/icons/DirectionsRun'; @@ -75,6 +79,7 @@ import cxtmenu from 'cytoscape-cxtmenu'; import { w3cwebsocket as W3CWebSocket } from "websocket"; import { useAlert } from "react-alert"; +import { validateJson } from "./Workflows"; import { GetParsedPaths } from "./Apps"; const surfaceColor = "#27292D" @@ -112,6 +117,7 @@ const AngularWorkflow = (props) => { const [bodyWidth, bodyHeight] = useWindowSize(); const appBarSize = 74 + var to_be_copied = "" const [cystyle, ] = useState(cytoscapestyle) const [cy, setCy] = React.useState() @@ -141,6 +147,9 @@ const AngularWorkflow = (props) => { const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) + const [selectedResult, setSelectedResult] = React.useState({}) + const [codeModalOpen, setCodeModalOpen] = React.useState(false); + const [variableAnchorEl, setVariableAnchorEl] = React.useState(null) const [sourceValue, setSourceValue] = React.useState({}) @@ -5957,11 +5966,10 @@ const AngularWorkflow = (props) => { ) } - const HandleJsonCopy = (base, copy, base_node_name) => { console.log("COPY: ", copy) var newitem = JSON.parse(base) - var to_be_copied = "$"+base_node_name + to_be_copied = "$"+base_node_name for (var key in copy.namespace) { if (copy.namespace[key].includes("Results for")) { continue @@ -5986,16 +5994,18 @@ const AngularWorkflow = (props) => { } } - console.log(to_be_copied) - //var copyText = document.getElementById("copy_element_shuffle"); - //if (copyText !== null) { - //navigator.clipboard.writeText(to_be_copied) - //copyText.select(); - //copyText.setSelectionRange(0, 99999); /* For mobile devices */ + to_be_copied.replace(" ", "_") + var copyText = document.getElementById("copy_element_shuffle"); + if (copyText !== null) { + navigator.clipboard.writeText(to_be_copied) + copyText.select(); + copyText.setSelectionRange(0, 99999); /* For mobile devices */ - ///* Copy the text inside the text field */ - //document.execCommand("copy"); - //} + /* Copy the text inside the text field */ + document.execCommand("copy"); + } + + alert.success("Copied "+to_be_copied) } const executionModal = @@ -6124,6 +6134,7 @@ const AngularWorkflow = (props) => { if (executionData.results.length !== 1 && !showSkippedActions && (data.status === "SKIPPED" || data.status === "FAILURE")) { return null } + console.log(data) // showResult = replaceAll(showResult, " None", " \"None\"") @@ -6165,6 +6176,14 @@ const AngularWorkflow = (props) => { return (
+ { + setSelectedResult(data) + setCodeModalOpen(true) + }}> + + + + {actionimg}
{data.action.label}
@@ -6178,7 +6197,7 @@ const AngularWorkflow = (props) => { collapsed={true} displayDataTypes={false} onSelect={(select) => { - HandleJsonCopy(showResult, select, data.action.name) + HandleJsonCopy(showResult, select, data.action.label) console.log("SELECTED!: ", select) }} name={"Results for "+data.action.label} @@ -6620,6 +6639,89 @@ const AngularWorkflow = (props) => { ) } + const CodePopoutModal = (props) => { + const {codeModalOpen, setCodeModalOpen, selectedResult} = props + if (!codeModalOpen) { + return null + } + + const curapp = apps.find(a => a.name === selectedResult.action.app_name && a.app_version === selectedResult.action.app_version) + const imgsize = 50 + const statusColor = selectedResult.status === "FINISHED" || selectedResult.status === "SUCCESS" ? "green" : selectedResult.status === "ABORTED" || selectedResult.status === "FAILURE" ? "red" : "orange" + const validate = validateJson(selectedResult.result.trim()) + + return ( + + { + //setCodeModalOpen(false) + }} + PaperComponent= + BackdropProps={{ + invisible: false, + open: false, + style: { + height: 0, + width: 0, + padding: 0, + margin: 0, + background: "none", + boxShadow: "none", + backgroundColor: "transparent", + } + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: 750, + padding: 30, + maxHeight: 700, + overflow: "auto", + //backgroundColor: "transparent", + boxShadow: "none", + }, + }} + > + { + setCodeModalOpen(false) + }}> + + +
+
+ {curapp === null ? null : {selectedResult.app_name}} +
+
{selectedResult.action.label}
+
{selectedResult.action.name}
+
+
+
Status {selectedResult.status}
+ {validate.valid ? { + HandleJsonCopy(JSON.stringify(validate.result), select, selectedResult.action.label) + }} + name={"Results for "+selectedResult.action.label} + /> + : +
+ Result  + {selectedResult.result} +
+ } +
+ + + ) + } + // This whole part is redundant. Made it part of Arguments instead. const authenticationModal = authenticationModalOpen ? { {executionVariableModal} {conditionsModal} {authenticationModal} + + {/* + + */}
:
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 3b0ec8bc..6c9ead3b 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -43,6 +43,37 @@ import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; const inputColor = "#383B40" const surfaceColor = "#27292D" +export const validateJson = (showResult) => { + showResult = showResult.split(" None").join(" \"None\"") + showResult = showResult.split(" False").join(" false") + showResult = showResult.split(" True").join(" true") + + var jsonvalid = true + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + showResult = showResult.split("\'").join("\"") + + try { + const tmp = String(JSON.parse(showResult)) + if (!showResult.includes("{") && !showResult.includes("[")) { + jsonvalid = false + } + } catch (e) { + jsonvalid = false + } + } + + console.log("VALID: ", jsonvalid) + return { + "valid": jsonvalid, + "result": jsonvalid ? JSON.parse(showResult) : showResult, + } +} + const Workflows = (props) => { const { globalUrl, isLoggedIn, isLoaded, removeCookie, cookies, userdata} = props; document.title = "Shuffle - Workflows" @@ -684,22 +715,12 @@ const Workflows = (props) => { } var t = new Date(data.started_at*1000) - var jsonvalid = true var showResult = data.result.trim() - showResult = replaceAll(showResult, " None", " \"None\""); - try { - const tmp = String(JSON.parse(showResult)) - if (!tmp.includes("{") && !tmp.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } + const validate = validateJson(showResult) - //console.log("VALID: ", jsonvalid) - if (jsonvalid) { + if (validate.valid) { showResult = { /> } else { // FIXME - have everything parsed as json, either just for frontend - // or in the backend + // or in the backend? /* const newdata = {"result": data.result} showResult = { No results yet
- const resultsLength = Object.getOwnPropertyNames(selectedExecution).length > 0 && selectedExecution.results !== null ? selectedExecution.results.length : 0 + const resultsLength = Object.getOwnPropertyNames(selectedExecution).length > 0 && selectedExecution.results !== null ? selectedExecution.results.length : 0 const ExecutionDetails = () => { var starttime = new Date(selectedExecution.started_at*1000) @@ -780,23 +801,12 @@ const Workflows = (props) => { var arg = null if (selectedExecution.execution_argument !== undefined && selectedExecution.execution_argument.length > 0) { - var jsonvalid = true - var showResult = selectedExecution.execution_argument.trim() - showResult = replaceAll(showResult, " None", " \"None\""); + const validate = validateJson(showResult) - try { - const tmp = String(JSON.parse(showResult)) - if (!tmp.includes("{") && !tmp.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } - - arg = jsonvalid ? + arg = validate.valid ? { var lastresult = null if (selectedExecution.result !== undefined && selectedExecution.result.length > 0) { - var jsonvalid = true var showResult = selectedExecution.result.trim() - showResult = replaceAll(showResult, " None", " \"None\""); + const validate = validateJson(showResult) + console.log("VALID: ", validate) - try { - const tmp = JSON.parse(showResult) - if (!tmp.includes("{") && !tmp.includes("[")) { - jsonvalid = false - } - } catch (e) { - jsonvalid = false - } - - lastresult = jsonvalid ? + lastresult = validate.valid ? Date: Sun, 27 Dec 2020 08:57:06 +0100 Subject: [PATCH 33/89] Got modal with Draggable working --- frontend/src/views/AngularWorkflow.jsx | 133 ++++++++++++------------- frontend/src/views/Workflows.jsx | 1 - 2 files changed, 65 insertions(+), 69 deletions(-) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 73382661..3a3598bc 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -6005,7 +6005,7 @@ const AngularWorkflow = (props) => { document.execCommand("copy"); } - alert.success("Copied "+to_be_copied) + //alert.success("Copied "+to_be_copied) } const executionModal = @@ -6134,8 +6134,6 @@ const AngularWorkflow = (props) => { if (executionData.results.length !== 1 && !showSkippedActions && (data.status === "SKIPPED" || data.status === "FAILURE")) { return null } - console.log(data) - // showResult = replaceAll(showResult, " None", " \"None\"") // Super basic check. @@ -6651,74 +6649,73 @@ const AngularWorkflow = (props) => { const validate = validateJson(selectedResult.result.trim()) return ( - - { - //setCodeModalOpen(false) - }} - PaperComponent= - BackdropProps={{ - invisible: false, - open: false, - style: { - height: 0, - width: 0, - padding: 0, - margin: 0, - background: "none", - boxShadow: "none", - backgroundColor: "transparent", - } - }} - PaperProps={{ - style: { - backgroundColor: surfaceColor, - color: "white", - minWidth: 750, - padding: 30, - maxHeight: 700, - overflow: "auto", - //backgroundColor: "transparent", - boxShadow: "none", - }, - }} - > - { - setCodeModalOpen(false) - }}> - - -
-
- {curapp === null ? null : {selectedResult.app_name}} -
-
{selectedResult.action.label}
-
{selectedResult.action.name}
-
-
-
Status {selectedResult.status}
- {validate.valid ? { - HandleJsonCopy(JSON.stringify(validate.result), select, selectedResult.action.label) - }} - name={"Results for "+selectedResult.action.label} - /> - : + + { + //setCodeModalOpen(false) + }} + BackdropProps={{ + invisible: true, + style: { + backgroundColor: "transparent", + pointerEvents: "none", + } + }} + PaperProps={{ + style: { + pointerEvents: "auto", + backgroundColor: inputColor, + color: "white", + minWidth: 750, + padding: 30, + maxHeight: 700, + overflow: "auto", + //boxShadow: "none", + }, + }} + > + { + setCodeModalOpen(false) + }}> + + +
+
+ {curapp === null ? null : {selectedResult.app_name}}
- Result  - {selectedResult.result} +
{selectedResult.action.label}
+
{selectedResult.action.name}
- }
-
- +
Status {selectedResult.status}
+ {validate.valid ? { + HandleJsonCopy(JSON.stringify(validate.result), select, selectedResult.action.label) + }} + name={"Results for "+selectedResult.action.label} + /> + : +
+ Result  + {selectedResult.result} +
+ } +
+ + ) } diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 6c9ead3b..57379cad 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -67,7 +67,6 @@ export const validateJson = (showResult) => { } } - console.log("VALID: ", jsonvalid) return { "valid": jsonvalid, "result": jsonvalid ? JSON.parse(showResult) : showResult, From 697dcb885e6f42465279eb158aab6592cbc33b9b Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 29 Dec 2020 15:43:07 +0100 Subject: [PATCH 34/89] Reduced API-calls to backend --- backend/app_sdk/app_base.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 98f62feb..155a09d7 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -457,11 +457,13 @@ class AppBase: action_result["result"] = "Error in setup ENV: ACTION not defined" self.send_result(action_result, headers, stream_path) return + if len(self.authorization) == 0: print("AUTHORIZATION env not defined") action_result["result"] = "Error in setup ENV: AUTHORIZATION not defined" self.send_result(action_result, headers, stream_path) return + if len(self.current_execution_id) == 0: print("EXECUTIONID env not defined") action_result["result"] = "Error in setup ENV: EXECUTIONID not defined" @@ -476,17 +478,19 @@ class AppBase: # Add async logger # self.console_logger.handlers[0].stream.set_execution_id() #self.logger.info("Before initial stream result") - try: - ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) - self.logger.info("Workflow: %d" % ret.status_code) - if ret.status_code != 200: - self.logger.info(ret.text) - except requests.exceptions.ConnectionError as e: - print("Connectionerror: %s" % e) - action_result["result"] = "Bad setup during startup: %s" % e - self.send_result(action_result, headers, stream_path) - return + # FIXME: Shouldn't skip this, but it's good for minimzing API calls + #try: + # ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) + # self.logger.info("Workflow: %d" % ret.status_code) + # if ret.status_code != 200: + # self.logger.info(ret.text) + #except requests.exceptions.ConnectionError as e: + # print("Connectionerror: %s" % e) + + # action_result["result"] = "Bad setup during startup: %s" % e + # self.send_result(action_result, headers, stream_path) + # return # Verify whether there are any parameters with ACTION_RESULT required # If found, we get the full results list from backend From 5beeadc1b1151e8b79b4c4fdfadf169acc9698cf Mon Sep 17 00:00:00 2001 From: frikky Date: Tue, 29 Dec 2020 16:26:40 +0100 Subject: [PATCH 35/89] FEATURE: Added popout results in workflows --- backend/app_sdk/build.sh | 2 +- frontend/src/views/AngularWorkflow.jsx | 279 ++++++++++++++++--------- frontend/src/views/Workflows.jsx | 2 - 3 files changed, 178 insertions(+), 105 deletions(-) diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 08128a44..67c75fd7 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -1,6 +1,6 @@ #!/bin/bash NAME=shuffle-app_sdk -VERSION=0.8.41 +VERSION=0.8.42 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 3a3598bc..a2adc14d 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -155,6 +155,11 @@ const AngularWorkflow = (props) => { const [sourceValue, setSourceValue] = React.useState({}) const [destinationValue, setDestinationValue] = React.useState({}) const [conditionValue, setConditionValue] = React.useState({}) + const [dragging, setDragging] = React.useState(false) + const [dragPosition, setDragPosition] = React.useState({ + x: 0, + y: 0, + }) // Trigger stuff const [selectedTrigger, setSelectedTrigger] = React.useState({}); @@ -1267,13 +1272,13 @@ const AngularWorkflow = (props) => { // CTRL = 17 //console.log(event.keyCode) switch( event.keyCode ) { - case 27: - console.log("ESCAPE") - break; + case 27: + console.log("ESCAPE") + break; case 46: removeNode() console.log("DELETE") - break; + break; case 38: console.log("UP") break; @@ -1313,11 +1318,11 @@ const AngularWorkflow = (props) => { } break; case 70: - if (previouskey === 17) { - event.preventDefault() - cy.fit(null, 50) - } - break; + //if (previouskey === 17) { + // event.preventDefault() + // cy.fit(null, 50) + //} + //break; case 65: // As a poweruser myself, I found myself hitting this a few // too many times to just edit text. Need a better bind @@ -3676,13 +3681,42 @@ const AngularWorkflow = (props) => {

{selectedAction.app_name}

- What are actions? - {selectedAction.errors !== null && selectedAction.errors.length > 0 ? -
- Errors: {selectedAction.errors.join("\n")} -
- : null - } +
+ { + console.log("FIND EXAMPLE RESULTS FOR ", selectedAction) + if (workflowExecutions.length > 0) { + // Look for the ID + const found = false + for (var key in workflowExecutions) { + if (workflowExecutions[key].results === undefined || workflowExecutions[key].results === null) { + continue + } + + var foundResult = workflowExecutions[key].results.find(result => result.action.id === selectedAction.id) + if (foundResult === undefined || foundResult === null) { + continue + } + + setSelectedResult(foundResult) + setCodeModalOpen(true) + break + } + } + }}> + + + + + + What are actions? + {selectedAction.errors !== null && selectedAction.errors.length > 0 ? +
+ Errors: {selectedAction.errors.join("\n")} +
+ : null + } + +
+ + Conditions can't be used for loops [ .# ]. Learn more + - - + setSelectedEdge(selectedEdge) + workflow.branches[selectedEdgeIndex] = selectedEdge + setWorkflow(workflow) + }} color="primary"> + Submit + + @@ -6422,15 +6436,20 @@ const AngularWorkflow = (props) => { :
-

{ - setExecutionRunning(false) - stop() - getWorkflowExecution(props.match.params.key) - setExecutionModalView(0) + { + setExecutionRunning(false) + stop() + getWorkflowExecution(props.match.params.key) + setExecutionModalView(0) }}> - - See other Executions -

+ {}}> + + +

{ + }}> + See other Executions +

+

Executing Workflow

diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index c9312c6d..97689cac 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -341,7 +341,7 @@ const Apps = (props) => { } var description = data.description - const maxDescLen = 60 + const maxDescLen = 58 if (description.length > maxDescLen) { description = data.description.slice(0, maxDescLen)+"..." } diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 7a29fcfa..5fdb61ea 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.8.52 +VERSION=0.8.53 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 42e9d097..5a7c0285 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -49,6 +49,8 @@ var baseUrl = os.Getenv("BASE_URL") var environment = os.Getenv("ENVIRONMENT_NAME") var dockerApiVersion = os.Getenv("DOCKER_API_VERSION") var runningMode = strings.ToLower(os.Getenv("RUNNING_MODE")) +var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) +var workerIds = []string{} type ExecutionRequestWrapper struct { Data []ExecutionRequest `json:"data"` @@ -144,6 +146,10 @@ func deployWorker(image string, identifier string, env []string) { //log.Printf("[INFO] Empty self container id, continue without NetworkMode") } + if cleanupEnv == "true" { + hostConfig.AutoRemove = true + } + config := &container.Config{ Image: image, Env: env, @@ -212,6 +218,7 @@ func deployWorker(image string, identifier string, env []string) { //} } else { log.Printf("[INFO] Container %s was created under environment %s", cont.ID, environment) + //workerIds = append(workerIds, cont.ID) } return @@ -248,7 +255,7 @@ func initializeImages() { log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion) } if workerVersion == "" { - workerVersion = "0.8.52" + workerVersion = "0.8.53" log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion) } @@ -499,6 +506,7 @@ func main() { fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId), fmt.Sprintf("ENVIRONMENT_NAME=%s", environment), fmt.Sprintf("BASE_URL=%s", baseUrl), + fmt.Sprintf("CLEANUP=%s", cleanupEnv), } if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" { @@ -512,7 +520,7 @@ func main() { go deployWorker(workerImage, containerName, env) - log.Printf("[INFO] %s was deployed and to be removed from queue.", execution.ExecutionId) + log.Printf("[INFO] ExecutionID %s was deployed and to be removed from queue.", execution.ExecutionId) zombiecounter += 1 toBeRemoved.Data = append(toBeRemoved.Data, execution) } diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 29828da4..fac6304e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.8.52 +VERSION=0.8.53 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . @@ -10,5 +10,5 @@ docker build . -t frikky/shuffle:$NAME -t frikky/shuffle:$NAME_$VERSION -t docke #docker push frikky/shuffle:$NAME_$VERSION #docker push docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION #docker tag frikky/shuffle:0.8.51 ghcr.io/frikky/shuffle-worker:0.8.5 -#docker push ghcr.io/frikky/$NAME:$VERSION docker tag frikky/shuffle:$NAME ghcr.io/frikky/shuffle-worker:0.8.52 +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 49c4b583..016ec96f 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -28,6 +28,7 @@ import ( var environment = os.Getenv("ENVIRONMENT_NAME") var baseUrl = os.Getenv("BASE_URL") var appCallbackUrl = os.Getenv("BASE_URL") +var cleanupEnv = strings.ToLower(os.Getenv("CLEANUP")) var baseimagename = "frikky/shuffle" var registryName = "registry.hub.docker.com" var fallbackName = "shuffle-orborus" @@ -43,6 +44,7 @@ var children map[string][]string var visited []string var executed []string var nextActions []string +var containerIds []string var extra int var startAction string @@ -68,9 +70,9 @@ func getThisContainerId() string { func init() { containerId = getThisContainerId() if len(containerId) == 0 { - log.Printf("[ERROR] No container ID found. Not running containerized?") + log.Printf("[WARNING] No container ID found. Not running containerized? This should only show during testing") } else { - log.Printf("[INFO] Found container ID: %s", containerId) + log.Printf("[INFO] Found container ID for this worker: %s", containerId) } } @@ -770,35 +772,28 @@ type AppExecutionExample struct { // removes every container except itself (worker) func shutdown(executionId, workflowId string) { - dockercli, err := dockerclient.NewEnvClient() - if err != nil { - log.Printf("[ERROR] Unable to create docker client: %s", err) - os.Exit(3) - } + log.Printf("[INFO] Shutdown started") - containerOptions := types.ContainerListOptions{ - All: true, - } + // Might not be necessary because of cleanupEnv hostconfig autoremoval + if cleanupEnv == "true" && len(containerIds) > 0 { + ctx := context.Background() + dockercli, err := dockerclient.NewEnvClient() + if err == nil { + log.Printf("[INFO] Cleaning up %d containers", len(containerIds)) + removeOptions := types.ContainerRemoveOptions{ + RemoveVolumes: true, + Force: true, + } - containers, err := dockercli.ContainerList(context.Background(), containerOptions) - if err != nil { - panic(err) - } - _ = containers - - for _, container := range containers { - for _, name := range container.Names { - if strings.Contains(name, executionId) { - // FIXME - reinstate - not here for debugging - //err = removeContainer(container.ID) - //if err != nil { - // log.Printf("Failed removing %s before shutdown.", name) - //} - - break + for _, containername := range containerIds { + log.Printf("[INFO] Stopping and removing container %s", containername) + dockercli.ContainerStop(ctx, containername, nil) + dockercli.ContainerRemove(ctx, containername, removeOptions) + //removeContainers = append(removeContainers, containername) } } - + } else { + log.Printf("[INFO] NOT cleaning up containers. IDS: %d, CLEANUP env: %s", len(containerIds), cleanupEnv) } fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowId, executionId) @@ -844,7 +839,9 @@ func shutdown(executionId, workflowId string) { log.Printf("[INFO] Failed abort request: %s", err) } - log.Printf("[INFO] Finished shutdown.") + log.Printf("[INFO] Finished shutdown (after 15 seconds).") + // Allows everything to finish in subprocesses + time.Sleep(time.Duration(15) * time.Second) os.Exit(3) } @@ -865,6 +862,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] log.Printf("[WARNING] Empty self container id, continue without NetworkMode") } + if cleanupEnv == "true" { + hostConfig.AutoRemove = true + } + config := &container.Config{ Image: image, Env: env, @@ -892,6 +893,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] } log.Printf("[INFO] Container %s is created for %s", cont.ID, identifier) + containerIds = append(containerIds, cont.ID) return nil } @@ -1537,71 +1539,16 @@ func handleExecutionResult(workflowExecution WorkflowExecution) { // FIXME - new request here // FIXME - clean up stopped (remove) containers with this execution id - dockercli, err := dockerclient.NewEnvClient() - if err != nil { - log.Printf("Unable to create docker client: %s", err) - shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) - } if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { shutdownCheck := true - ctx := context.Background() for _, result := range workflowExecution.Results { if result.Status == "EXECUTING" { // Cleaning up executing stuff shutdownCheck = false - // Check status - containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{ - All: true, - }) - if err != nil { - log.Printf("Failed listing containers: %s", err) - continue - } - - stopContainers := []string{} - removeContainers := []string{} - for _, container := range containers { - for _, name := range container.Names { - if !strings.Contains(name, result.Action.ID) { - continue - } - - if container.State != "running" { - removeContainers = append(removeContainers, container.ID) - stopContainers = append(stopContainers, container.ID) - } - } - } - - // FIXME - add killing of apps with same execution ID too - // FIXME - stahp - //for _, containername := range stopContainers { - // if err := dockercli.ContainerStop(ctx, containername, nil); err != nil { - // log.Printf("Unable to stop container: %s", err) - // } else { - // log.Printf("Stopped container %s", containername) - // } - //} - - removeOptions := types.ContainerRemoveOptions{ - RemoveVolumes: true, - Force: true, - } - - _ = removeOptions - - // FIXME - this - //for _, containername := range removeContainers { - // if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil { - // log.Printf("Unable to remove container: %s", err) - // } else { - // log.Printf("Removed container %s", containername) - // } - //} - + // USED TO BE CONTAINER REMOVAL // FIXME - send POST request to kill the container - log.Printf("Should remove (POST request) stopped containers") + //log.Printf("Should remove (POST request) stopped containers") //ret = requests.post("%s%s" % (self.url, stream_path), headers=headers, json=action_result) } } @@ -2511,7 +2458,7 @@ func validateFinished(workflowExecution WorkflowExecution) { if err != nil { log.Printf("[ERROR] Failed reading body: %s", err) } else { - log.Printf("NEWRESP: %s", string(body)) + log.Printf("[INFO] NEWRESP (from backend): %s", string(body)) } } } @@ -2583,7 +2530,7 @@ func setWorkflowExecution(ctx context.Context, workflowExecution WorkflowExecuti } // GetLocalIP returns the non loopback local IP of the host -func GetLocalIP() string { +func getLocalIP() string { addrs, err := net.InterfaceAddrs() if err != nil { return "" @@ -2599,22 +2546,46 @@ func GetLocalIP() string { return "" } -func webserverSetup() { - hostname := GetLocalIP() +func getAvailablePort() (net.Listener, error) { + listener, err := net.Listen("tcp", ":0") + if err != nil { + log.Printf("[WARNING] Failed to assign port by default. Defaulting to 5001") + //return ":5001" + return nil, err + } - log.Printf("\nStarting webserver on port 5001 with hostname: %s\n", hostname) - log.Printf("OLD HOSTNAME: %s", appCallbackUrl) - appCallbackUrl = fmt.Sprintf("http://%s:5001", hostname) - log.Printf("NEW HOSTNAME: %s", appCallbackUrl) + return listener, nil + //return fmt.Sprintf(":%d", port) } -func runWebserver() { +func webserverSetup(workflowExecution WorkflowExecution) net.Listener { + hostname := getLocalIP() + + // FIXME: This MAY not work because of speed between first + // container being launched and port being assigned to webserver + listener, err := getAvailablePort() + if err != nil { + log.Printf("Failed to created listener: %s", err) + shutdown(workflowExecution.ExecutionId, workflowExecution.Workflow.ID) + } + port := listener.Addr().(*net.TCPAddr).Port + + log.Printf("\n\nStarting webserver on port %d with hostname: %s\n\n", port, hostname) + log.Printf("OLD HOSTNAME: %s", appCallbackUrl) + appCallbackUrl = fmt.Sprintf("http://%s:%d", hostname, port) + log.Printf("NEW HOSTNAME: %s", appCallbackUrl) + + return listener +} + +func runWebserver(listener net.Listener) { r := mux.NewRouter() r.HandleFunc("/api/v1/streams", handleWorkflowQueue).Methods("POST") r.HandleFunc("/api/v1/streams/results", handleGetStreamResults).Methods("POST", "OPTIONS") http.Handle("/", r) - log.Fatal(http.ListenAndServe(":5001", nil)) + //log.Fatal(http.ListenAndServe(port, nil)) + log.Fatal(http.Serve(listener, nil)) } // Initial loop etc @@ -2735,8 +2706,8 @@ func main() { } log.Printf("Environments: %s. 1 = webserver, 0 or >1 = default", environments) - if len(environments) == 1 { - webserverSetup() + if len(environments) == 1 { //&& len(workflowExecution.Actions)+len(workflowExecution.Triggers) > 1 { + listener := webserverSetup(workflowExecution) err := executionInit(workflowExecution) if err != nil { log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) @@ -2748,7 +2719,7 @@ func main() { handleExecutionResult(workflowExecution) }() - runWebserver() + runWebserver(listener) //log.Printf("Before wait") //wg := sync.WaitGroup{} //wg.Add(1) From 38b454bc38a5f8ccc8a8253c49022bf08bd3fab4 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 17 Jan 2021 15:38:58 +0100 Subject: [PATCH 58/89] Reduced execution bytesize --- backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 12 +++++++ docker-compose.yml | 4 +-- frontend/src/views/AngularWorkflow.jsx | 29 +++++++++++++---- frontend/src/views/Workflows.jsx | 1 + functions/extensions/aws-lambda/deploy.sh | 9 ++++++ functions/onprem/orborus/orborus.go | 39 +++++++++++++++++++++++ functions/onprem/worker/worker.go | 14 ++++---- 8 files changed, 94 insertions(+), 16 deletions(-) create mode 100644 functions/extensions/aws-lambda/deploy.sh diff --git a/backend/go-app/main.go b/backend/go-app/main.go index c3987846..ef55cead 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -3535,7 +3535,7 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) { } for _, item := range hook.Workflows { - log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) + //log.Printf("Running webhook for workflow %s with startnode %s", item, hook.Start) workflow := Workflow{ ID: "", } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0ef011a8..df426463 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1727,6 +1727,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { action.IsValid = true } + action.LargeImage = "" newActions = append(newActions, action) } @@ -2797,7 +2798,18 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(workflow.Actions) == 0 { workflow.Actions = []Action{} + } else { + newactions := []Action{} + for _, action := range workflow.Actions { + action.LargeImage = "" + action.SmallImage = "" + newactions = append(newactions, action) + log.Printf("ACTION: %#v", action) + } + + workflow.Actions = newactions } + if len(workflow.Branches) == 0 { workflow.Branches = []Branch{} } diff --git a/docker-compose.yml b/docker-compose.yml index 1846975b..2f3172f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3' services: frontend: - build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.52 + #build: ./frontend + image: ghcr.io/frikky/shuffle-frontend:0.8.51 container_name: shuffle-frontend hostname: shuffle-frontend ports: diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 2f0b127a..56279e64 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -4479,10 +4479,14 @@ const AngularWorkflow = (props) => { setDestinationValue({}) }} > + + Conditions can't be used for loops [ .# ] Learn more + Condition +
- - Conditions can't be used for loops [ .# ]. Learn more - var imageData = file.length > 0 ? file : fileBase64 diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index b809cf62..ae7ec058 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -31,6 +31,8 @@ import { useTheme } from '@material-ui/core/styles'; import HandlePayment from './HandlePayment' import OrgHeader from '../components/OrgHeader' +import EditIcon from '@material-ui/icons/Edit'; +import SelectAllIcon from '@material-ui/icons/SelectAll'; import OpenInNewIcon from '@material-ui/icons/OpenInNew'; import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; import DescriptionIcon from '@material-ui/icons/Description'; @@ -87,6 +89,7 @@ const Admin = (props) => { const [selectedUserModalOpen, setSelectedUserModalOpen] = React.useState(false) const [selectedAuthentication, setSelectedAuthentication] = React.useState({}) const [selectedAuthenticationModalOpen, setSelectedAuthenticationModalOpen] = React.useState(false) + const [authenticationFields, setAuthenticationFields] = React.useState([]) const [showArchived, setShowArchived] = React.useState(false) const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" @@ -276,9 +279,69 @@ const Admin = (props) => { }) } - - - + const saveAuthentication = (authentication) => { + const data = authentication + const url = globalUrl + '/api/v1/apps/authentication'; + + fetch(url, { + mode: 'cors', + method: 'PUT', + body: JSON.stringify(data), + credentials: 'include', + crossDomain: true, + withCredentials: true, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error("Failed changing authentication") + } else { + //alert.success("Successfully password!") + setSelectedUserModalOpen(false) + } + }), + ) + .catch(error => { + alert.error("Err: " + error.toString()) + }); + } + + const editAuthenticationConfig = (id) => { + const data = { + "id": id, + "action": "assign_everywhere", + } + const url = globalUrl + '/api/v1/apps/authentication/'+id+"/config"; + + fetch(url, { + mode: 'cors', + method: 'POST', + body: JSON.stringify(data), + credentials: 'include', + crossDomain: true, + withCredentials: true, + headers: { + 'Content-Type': 'application/json; charset=utf-8', + }, + }) + .then(response => + response.json().then(responseJson => { + if (responseJson["success"] === false) { + alert.error("Failed overwriting appauth in workflows") + } else { + alert.success("Successfully updated auth everywhere!") + setSelectedUserModalOpen(false) + getAppAuthentication() + } + }), + ) + .catch(error => { + alert.error("Err: " + error.toString()) + }); + } const onPasswordChange = () => { const data = { "username": selectedUser.username, "newpassword": newPassword } @@ -300,7 +363,7 @@ const Admin = (props) => { if (responseJson["success"] === false) { alert.error("Failed setting new password") } else { - alert.success("Successfully password!") + alert.success("Successfully updated password!") setSelectedUserModalOpen(false) } }), @@ -599,7 +662,7 @@ const Admin = (props) => { return response.json() }) .then((responseJson) => { - console.log(responseJson) + //console.log(responseJson) setFiles(responseJson) }) .catch(error => { @@ -966,8 +1029,8 @@ const Admin = (props) => { }); } - const editAuthenticationModal = - { setSelectedAuthenticationModalOpen(false) }} PaperProps={{ @@ -979,56 +1042,71 @@ const Admin = (props) => { }, }} > - Edit authentication + Edit authentication for {selectedAuthentication.app.name} ({selectedAuthentication.label}) -
- setNewPassword(e.target.value)} - /> - -
- - - + {selectedAuthentication.fields.map((data, index) => { + return ( +
+ {data.key} + { + authenticationFields[index].value = e.target.value + setAuthenticationFields(authenticationFields) + }} + /> +
+ ) + })} + + + + + : null const editUserModal = { }, }} > - Edit user +
{
: null + const updateAppAuthentication = (field) => { + setSelectedAuthenticationModalOpen(true) + setSelectedAuthentication(field) + //{selectedAuthentication.fields.map((data, index) => { + var newfields = [] + for (var key in field.fields) { + newfields.push({ + "key": field.fields[key].key, + "value": "", + }) + } + setAuthenticationFields(newfields) + } + const authenticationView = curTab === 2 ?
@@ -1952,7 +2044,7 @@ const Admin = (props) => { /> { /> { /> { /> { @@ -2009,16 +2101,28 @@ const Admin = (props) => { style={{minWidth: 200, maxWidth: 200, overflow: "hidden"}} /> - + + ) From 777d8f40c393b914a1892e5ae427012ee7e0f2a8 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 17 Jan 2021 18:32:55 +0100 Subject: [PATCH 62/89] Added previouslysaved fix --- backend/go-app/walkoff.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 5be7cad8..edc1858d 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2239,6 +2239,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(outerapp.ID) > 0 && outerapp.Authentication.Required { + // FIXME: Add app auth + //log.Printf("FOUND APP TO VALIDATE: %#v", outerapp) action.Errors = append(action.Errors, "Requires authentication") action.IsValid = false @@ -2257,7 +2259,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = actionFixing } - //workflow.PreviouslySaved = true + workflow.PreviouslySaved = true } //PreviouslySaved bool `json:"first_save" datastore:"first_save"` From 82f30a38d2e5232a74ab7ebc0a9e48b0c80c73b4 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 17 Jan 2021 19:08:14 +0100 Subject: [PATCH 63/89] #168: Added dummy auths to make it simpler to import new workflows --- backend/go-app/main.go | 6 +- backend/go-app/walkoff.go | 52 +- docker-compose.yml | 6 +- frontend/package-lock.json | 1043 ++++++++++++++++++++++++++++++++---- frontend/package.json | 8 +- 5 files changed, 983 insertions(+), 132 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index fde61383..6e304dd2 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -7536,16 +7536,16 @@ func runInit(ctx context.Context) { if err != nil { log.Printf("Failed to upload workflows from github: %s", err) } else { - log.Printf("Finished downloading workflows from github!") + log.Printf("[INFO] Finished downloading workflows from github!") } } else { - log.Printf("Skipping because there are %d workflows already", len(workflows)) + log.Printf("[INFO] Skipping because there are %d workflows already", len(workflows)) } } } - log.Printf("Finished INIT") + log.Printf("[INFO] Finished INIT") } func handleVerifyCloudsync(orgId string) (SyncFeatures, error) { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index edc1858d..0c5e2fc4 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -140,6 +140,7 @@ type AppAuthenticationStorage struct { OrgId string `json:"org_id" datastore:"org_id"` Created int64 `json:"created" datastore:"created"` Edited int64 `json:"edited" datastore:"edited"` + Defined bool `json:"defined" datastore:"defined"` } type AuthenticationUsage struct { @@ -2220,6 +2221,10 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { continue } + if !auth.Defined { + continue + } + if auth.App.Name == action.AppName { //log.Printf("FOUND AUTH FOR APP %s: %s", auth.App.Name, auth.Id) action.AuthenticationId = auth.Id @@ -2228,6 +2233,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } + // FIXME: Only o this IF there isn't another one for the app already if !authSet { //log.Printf("Validate if the app NEEDS auth or not") outerapp := WorkflowApp{} @@ -2239,9 +2245,45 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } if len(outerapp.ID) > 0 && outerapp.Authentication.Required { - // FIXME: Add app auth + found := false + for _, auth := range allAuths { + if auth.App.ID == outerapp.ID { + found = true + break + } + } + + // FIXME: Add app auth + if !found { + timeNow := int64(time.Now().Unix()) + authFields := []AuthenticationStore{} + for _, param := range outerapp.Authentication.Parameters { + authFields = append(authFields, AuthenticationStore{ + Key: param.Name, + Value: "", + }) + } + + appAuth := AppAuthenticationStorage{ + Active: true, + Label: fmt.Sprintf("default_%s", outerapp.Name), + Id: uuid.NewV4().String(), + App: outerapp, + Fields: authFields, + Usage: []AuthenticationUsage{}, + WorkflowCount: 0, + NodeCount: 0, + OrgId: user.ActiveOrg.Id, + Created: timeNow, + Edited: timeNow, + } + + err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) + if err != nil { + log.Printf("Failed setting appauth for with name %s", appAuth.Label) + } + } - //log.Printf("FOUND APP TO VALIDATE: %#v", outerapp) action.Errors = append(action.Errors, "Requires authentication") action.IsValid = false workflow.IsValid = false @@ -2259,7 +2301,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { newActions = actionFixing } - workflow.PreviouslySaved = true + //workflow.PreviouslySaved = true } //PreviouslySaved bool `json:"first_save" datastore:"first_save"` @@ -4616,7 +4658,6 @@ func getWorkflowAppConfig(resp http.ResponseWriter, request *http.Request) { resp.Write(data) } -//r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", setAuthenticationConfig).Methods("POST", "OPTIONS") func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { cors := handleCors(resp, request) if cors { @@ -4649,7 +4690,6 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { fileId = location[5] } - log.Printf("FILE: %s", fileId) body, err := ioutil.ReadAll(request.Body) if err != nil { @@ -4679,7 +4719,6 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { return } - log.Printf("body: %s", string(body)) ctx := context.Background() auth, err := getWorkflowAppAuthDatastore(ctx, fileId) if err != nil { @@ -4868,6 +4907,7 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { //appAuth.LargeImage = "" appAuth.OrgId = user.ActiveOrg.Id + appAuth.Defined = true err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) if err != nil { log.Printf("Failed setting up app auth %s: %s", appAuth.Id, err) diff --git a/docker-compose.yml b/docker-compose.yml index 92a92437..c6632614 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,8 @@ version: '3' services: frontend: - #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.51 + build: ./frontend + image: ghcr.io/frikky/shuffle-frontend:0.8.53 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.52 + image: ghcr.io/frikky/shuffle-backend:0.8.53 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/package-lock.json b/frontend/package-lock.json index bd6df28b..e5e02309 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -1,6 +1,6 @@ { "name": "shuffler", - "version": "0.6.0", + "version": "0.8.53", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -336,6 +336,7 @@ "@babel/highlight": "^7.10.1" } }, + "@babel/core": {}, "@babel/helper-function-name": { "version": "7.10.1", "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.10.1.tgz", @@ -2566,37 +2567,6 @@ "@babel/helper-plugin-utils": "^7.10.1" } }, - "@babel/plugin-transform-runtime": { - "version": "7.9.0", - "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.9.0.tgz", - "integrity": "sha512-pUu9VSf3kI1OqbWINQ7MaugnitRss1z533436waNXp+0N3ur3zfut37sXiQMxkuCF4VUjwZucen/quskCh7NHw==", - "requires": { - "@babel/helper-module-imports": "^7.8.3", - "@babel/helper-plugin-utils": "^7.8.3", - "resolve": "^1.8.1", - "semver": "^5.5.1" - }, - "dependencies": { - "@babel/helper-module-imports": { - "version": "7.10.1", - "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.10.1.tgz", - "integrity": "sha512-SFxgwYmZ3HZPyZwJRiVNLRHWuW2OgE5k2nrVs6D9Iv4PPnXVffuEHy83Sfx/l4SqF+5kyJXjAyUmrG7tNm+qVg==", - "requires": { - "@babel/types": "^7.10.1" - } - }, - "@babel/types": { - "version": "7.10.2", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.2.tgz", - "integrity": "sha512-AD3AwWBSz0AWF0AkCN9VPiWrvldXq+/e3cHa4J89vo4ymjz1XwrBFFVZmkJTsQIPNk+ZVomPSXUJqq8yyjZsng==", - "requires": { - "@babel/helper-validator-identifier": "^7.10.1", - "lodash": "^4.17.13", - "to-fast-properties": "^2.0.0" - } - } - } - }, "@babel/plugin-transform-shorthand-properties": { "version": "7.10.1", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-shorthand-properties/-/plugin-transform-shorthand-properties-7.10.1.tgz", @@ -3393,6 +3363,16 @@ "react-is": "^16.7.0" } }, + "jss": { + "dependencies": { + "warning": {} + } + }, + "jss-nested": { + "dependencies": { + "warning": {} + } + }, "popper.js": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/popper.js/-/popper.js-1.16.1.tgz", @@ -4088,11 +4068,6 @@ } } }, - "acorn-jsx": { - "version": "5.2.0", - "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.2.0.tgz", - "integrity": "sha512-HiUX/+K2YpkpJ+SzBffkM/AQ2YE03S0U1kjTLVpoJdhZMOWy8qvXVN9JdLqv2QsaQ6MPYQIuNmwD8zOiYUofLQ==" - }, "acorn-walk": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-6.2.0.tgz", @@ -4706,6 +4681,7 @@ "slash": "^2.0.0" }, "dependencies": { + "@babel/core": {}, "ansi-styles": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", @@ -4747,25 +4723,6 @@ } } }, - "babel-loader": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.1.0.tgz", - "integrity": "sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==", - "requires": { - "find-cache-dir": "^2.1.0", - "loader-utils": "^1.4.0", - "mkdirp": "^0.5.3", - "pify": "^4.0.1", - "schema-utils": "^2.6.5" - }, - "dependencies": { - "pify": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", - "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" - } - } - }, "babel-plugin-dynamic-import-node": { "version": "2.3.3", "resolved": "https://registry.npmjs.org/babel-plugin-dynamic-import-node/-/babel-plugin-dynamic-import-node-2.3.3.tgz", @@ -4971,6 +4928,202 @@ "babel-plugin-transform-react-remove-prop-types": "0.4.24" }, "dependencies": { + "@babel/code-frame": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", + "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "requires": { + "@babel/highlight": "^7.10.4" + } + }, + "@babel/core": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.9.0.tgz", + "integrity": "sha512-kWc7L0fw1xwvI0zi8OKVBuxRVefwGOrKSQMvrQ3dW+bIIavBY3/NpXmpjMy7bQnLgwgzWQZ8TlM57YHpHNHz4w==", + "requires": { + "@babel/code-frame": "^7.8.3", + "@babel/generator": "^7.9.0", + "@babel/helper-module-transforms": "^7.9.0", + "@babel/helpers": "^7.9.0", + "@babel/parser": "^7.9.0", + "@babel/template": "^7.8.6", + "@babel/traverse": "^7.9.0", + "@babel/types": "^7.9.0", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.1", + "json5": "^2.1.2", + "lodash": "^4.17.13", + "resolve": "^1.3.2", + "semver": "^5.4.1", + "source-map": "^0.5.0" + } + }, + "@babel/generator": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.12.11.tgz", + "integrity": "sha512-Ggg6WPOJtSi8yYQvLVjG8F/TlpWDlKx0OpS4Kt+xMQPs5OaGYWy+v1A+1TvxI6sAMGZpKWWoAQ1DaeQbImlItA==", + "requires": { + "@babel/types": "^7.12.11", + "jsesc": "^2.5.1", + "source-map": "^0.5.0" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-annotate-as-pure": { + "version": "7.12.10", + "resolved": "https://registry.npmjs.org/@babel/helper-annotate-as-pure/-/helper-annotate-as-pure-7.12.10.tgz", + "integrity": "sha512-XplmVbC1n+KY6jL8/fgLVXXUauDIB+lD5+GsQEh6F6GBF1dq1qy4DP4yXWzDKcoqXB3X58t61e85Fitoww4JVQ==", + "requires": { + "@babel/types": "^7.12.10" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-create-regexp-features-plugin": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-create-regexp-features-plugin/-/helper-create-regexp-features-plugin-7.12.7.tgz", + "integrity": "sha512-idnutvQPdpbduutvi3JVfEgcVIHooQnhvhx0Nk9isOINOIGYkZea1Pk2JlJRiUnMefrlvr0vkByATBY/mB4vjQ==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "regexpu-core": "^4.7.1" + } + }, + "@babel/helper-define-map": { + "version": "7.10.5", + "resolved": "https://registry.npmjs.org/@babel/helper-define-map/-/helper-define-map-7.10.5.tgz", + "integrity": "sha512-fMw4kgFB720aQFXSVaXr79pjjcW5puTCM16+rECJ/plGS+zByelE8l9nCpV1GibxTnFVmUuYG9U8wYfQHdzOEQ==", + "requires": { + "@babel/helper-function-name": "^7.10.4", + "@babel/types": "^7.10.5", + "lodash": "^4.17.19" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-function-name": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.12.11.tgz", + "integrity": "sha512-AtQKjtYNolKNi6nNNVLQ27CP6D9oFR6bq/HPYSizlzbp7uC1M59XJe8L+0uXjbIaZaUJF99ruHqVGiKXU/7ybA==", + "requires": { + "@babel/helper-get-function-arity": "^7.12.10", + "@babel/template": "^7.12.7", + "@babel/types": "^7.12.11" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-get-function-arity": { + "version": "7.12.10", + "resolved": "https://registry.npmjs.org/@babel/helper-get-function-arity/-/helper-get-function-arity-7.12.10.tgz", + "integrity": "sha512-mm0n5BPjR06wh9mPQaDdXWDoll/j5UpCAPl1x8fS71GHm7HA6Ua2V4ylG1Ju8lvcTOietbPNNPaSilKj+pj+Ag==", + "requires": { + "@babel/types": "^7.12.10" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-member-expression-to-functions": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/helper-member-expression-to-functions/-/helper-member-expression-to-functions-7.12.7.tgz", + "integrity": "sha512-DCsuPyeWxeHgh1Dus7APn7iza42i/qXqiFPWyBDdOFtvS581JQePsc1F/nD+fHrcswhLlRc2UpYS1NwERxZhHw==", + "requires": { + "@babel/types": "^7.12.7" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, "@babel/helper-module-imports": { "version": "7.10.1", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.10.1.tgz", @@ -4979,6 +5132,106 @@ "@babel/types": "^7.10.1" } }, + "@babel/helper-optimise-call-expression": { + "version": "7.12.10", + "resolved": "https://registry.npmjs.org/@babel/helper-optimise-call-expression/-/helper-optimise-call-expression-7.12.10.tgz", + "integrity": "sha512-4tpbU0SrSTjjt65UMWSrUOPZTsgvPgGG4S8QSTNHacKzpS51IVWGDj0yCwyeZND/i+LSN2g/O63jEXEWm49sYQ==", + "requires": { + "@babel/types": "^7.12.10" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-replace-supers": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-replace-supers/-/helper-replace-supers-7.12.11.tgz", + "integrity": "sha512-q+w1cqmhL7R0FNzth/PLLp2N+scXEK/L2AHbXUyydxp828F4FEa5WcVoqui9vFRiHDQErj9Zof8azP32uGVTRA==", + "requires": { + "@babel/helper-member-expression-to-functions": "^7.12.7", + "@babel/helper-optimise-call-expression": "^7.12.10", + "@babel/traverse": "^7.12.10", + "@babel/types": "^7.12.11" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/helper-split-export-declaration": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-split-export-declaration/-/helper-split-export-declaration-7.12.11.tgz", + "integrity": "sha512-LsIVN8j48gHgwzfocYUSkO/hjYAOJqlpJEc7tGXcIm4cubjVUf8LGW6eWRyxEu7gA25q02p0rQUWoCI33HNS5g==", + "requires": { + "@babel/types": "^7.12.11" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/highlight": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.10.4.tgz", + "integrity": "sha512-i6rgnR/YgPEQzZZnbTHHuZdlE8qyoBNalD6F+q4vAFlcMEcqmkoG+mPqJYJCo63qPf74+Y1UZsl3l6f7/RIkmA==", + "requires": { + "@babel/helper-validator-identifier": "^7.10.4", + "chalk": "^2.0.0", + "js-tokens": "^4.0.0" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + } + } + }, + "@babel/parser": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.12.11.tgz", + "integrity": "sha512-N3UxG+uuF4CMYoNj8AhnbAcJF0PiuJ9KHuy1lQmkYsxTer/MAH9UBNHsBoAX/4s6NvlDD047No8mYVGGzLL4hg==" + }, "@babel/plugin-proposal-class-properties": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-class-properties/-/plugin-proposal-class-properties-7.8.3.tgz", @@ -5006,6 +5259,31 @@ "@babel/plugin-syntax-numeric-separator": "^7.8.3" } }, + "@babel/plugin-proposal-object-rest-spread": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-object-rest-spread/-/plugin-proposal-object-rest-spread-7.12.1.tgz", + "integrity": "sha512-s6SowJIjzlhx8o7lsFx5zmY4At6CTtDvgNQDdPzkBQucle58A6b/TTeEBYtyDgmcXjUTM+vE8YOGHZzzbc/ioA==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-object-rest-spread": "^7.8.0", + "@babel/plugin-transform-parameters": "^7.12.1" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + }, + "@babel/plugin-transform-parameters": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-parameters/-/plugin-transform-parameters-7.12.1.tgz", + "integrity": "sha512-xq9C5EQhdPK23ZeCdMxl8bbRnAgHFrw5EOC3KJUsSylZqdkCaFEXxGSBuTSObOpiiHHNyb82es8M1QYgfQGfNg==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + } + } + } + }, "@babel/plugin-proposal-optional-chaining": { "version": "7.9.0", "resolved": "https://registry.npmjs.org/@babel/plugin-proposal-optional-chaining/-/plugin-proposal-optional-chaining-7.9.0.tgz", @@ -5015,6 +5293,83 @@ "@babel/plugin-syntax-optional-chaining": "^7.8.0" } }, + "@babel/plugin-syntax-async-generators": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-syntax-json-strings": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-syntax-jsx": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-jsx/-/plugin-syntax-jsx-7.12.1.tgz", + "integrity": "sha512-1yRi7yAtB0ETgxdY9ti/p2TivUxJkTdhu/ZbF9MshVGqOx1TdB3b7xCXs49Fupgg50N45KcAsRP/ZqWjs9SRjg==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-classes": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-classes/-/plugin-transform-classes-7.12.1.tgz", + "integrity": "sha512-/74xkA7bVdzQTBeSUhLLJgYIcxw/dpEpCdRDiHgPJ3Mv6uC11UhjpOhl72CgqbBCmt1qtssCyB2xnJm1+PFjog==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.10.4", + "@babel/helper-define-map": "^7.10.4", + "@babel/helper-function-name": "^7.10.4", + "@babel/helper-optimise-call-expression": "^7.10.4", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/helper-replace-supers": "^7.12.1", + "@babel/helper-split-export-declaration": "^7.10.4", + "globals": "^11.1.0" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-destructuring": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-destructuring/-/plugin-transform-destructuring-7.12.1.tgz", + "integrity": "sha512-fRMYFKuzi/rSiYb2uRLiUENJOKq4Gnl+6qOv5f8z0TZXg3llUwUhsNNwrwaT/6dUhJTzNpBr+CUvEWBtfNY1cw==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-new-target": { + "dependencies": { + "@babel/core": {} + } + }, "@babel/plugin-transform-react-display-name": { "version": "7.8.3", "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.8.3.tgz", @@ -5023,6 +5378,37 @@ "@babel/helper-plugin-utils": "^7.8.3" } }, + "@babel/plugin-transform-react-jsx": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-react-jsx-self": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-react-jsx-source": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-regenerator": { + "dependencies": { + "@babel/core": {} + } + }, + "@babel/plugin-transform-runtime": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-runtime/-/plugin-transform-runtime-7.9.0.tgz", + "integrity": "sha512-pUu9VSf3kI1OqbWINQ7MaugnitRss1z533436waNXp+0N3ur3zfut37sXiQMxkuCF4VUjwZucen/quskCh7NHw==", + "requires": { + "@babel/helper-module-imports": "^7.8.3", + "@babel/helper-plugin-utils": "^7.8.3", + "resolve": "^1.8.1", + "semver": "^5.5.1" + } + }, "@babel/preset-env": { "version": "7.9.0", "resolved": "https://registry.npmjs.org/@babel/preset-env/-/preset-env-7.9.0.tgz", @@ -5088,6 +5474,63 @@ "invariant": "^2.2.2", "levenary": "^1.1.1", "semver": "^5.5.0" + }, + "dependencies": { + "@babel/plugin-syntax-async-generators": { + "version": "7.8.4", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-async-generators/-/plugin-syntax-async-generators-7.8.4.tgz", + "integrity": "sha512-tycmZxkGfZaxhMRbXlPXuVFpdWlXpir2W4AMhSJgRKzk/eDlIXOhb2LHWoLpDF7TEHylV5zNhykX6KAgHJmTNw==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-json-strings": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-json-strings/-/plugin-syntax-json-strings-7.8.3.tgz", + "integrity": "sha512-lY6kdGpWHvjoe2vk4WrAapEuBR69EMxZl+RoGRhrFGNYVK8mOPAW8VfbT/ZgrFbXlDNiiaxQnAtgVCZ6jv30EA==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-syntax-optional-catch-binding": { + "version": "7.8.3", + "resolved": "https://registry.npmjs.org/@babel/plugin-syntax-optional-catch-binding/-/plugin-syntax-optional-catch-binding-7.8.3.tgz", + "integrity": "sha512-6VPD0Pc1lpTqw0aKoeRTMiB+kWhAoT24PA+ksWSBrFtl5SIRVpZlwN3NNPQjehA2E/91FV3RjLWoVTglWcSV3Q==", + "requires": { + "@babel/helper-plugin-utils": "^7.8.0" + } + }, + "@babel/plugin-transform-named-capturing-groups-regex": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-named-capturing-groups-regex/-/plugin-transform-named-capturing-groups-regex-7.12.1.tgz", + "integrity": "sha512-tB43uQ62RHcoDp9v2Nsf+dSM8sbNodbEicbQNA53zHz8pWUhsgHSJCGpt7daXxRydjb0KnfmB+ChXOv3oADp1Q==", + "requires": { + "@babel/helper-create-regexp-features-plugin": "^7.12.1" + } + }, + "@babel/plugin-transform-new-target": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-new-target/-/plugin-transform-new-target-7.12.1.tgz", + "integrity": "sha512-+eW/VLcUL5L9IvJH7rT1sT0CzkdUTvPrXC2PXTn/7z7tXLBuKvezYbGdxD5WMRoyvyaujOq2fWoKl869heKjhw==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-regenerator": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-regenerator/-/plugin-transform-regenerator-7.12.1.tgz", + "integrity": "sha512-gYrHqs5itw6i4PflFX3OdBPMQdPbF4bj2REIUxlMRUFk0/ZOAIpDFuViuxPjUL7YC8UPnf+XG7/utJvqXdPKng==", + "requires": { + "regenerator-transform": "^0.14.2" + } + } } }, "@babel/preset-react": { @@ -5101,6 +5544,95 @@ "@babel/plugin-transform-react-jsx-development": "^7.9.0", "@babel/plugin-transform-react-jsx-self": "^7.9.0", "@babel/plugin-transform-react-jsx-source": "^7.9.0" + }, + "dependencies": { + "@babel/helper-module-imports": { + "version": "7.12.5", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.12.5.tgz", + "integrity": "sha512-SR713Ogqg6++uexFRORf/+nPXMmWIn80TALu0uaFb+iQIUoR7bOC7zBWyzBs5b3tBBJXuyD0cRu1F15GyzjOWA==", + "requires": { + "@babel/types": "^7.12.5" + } + }, + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/plugin-transform-react-display-name": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-display-name/-/plugin-transform-react-display-name-7.12.1.tgz", + "integrity": "sha512-cAzB+UzBIrekfYxyLlFqf/OagTvHLcVBb5vpouzkYkBclRPraiygVnafvAoipErZLI8ANv8Ecn6E/m5qPXD26w==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-react-jsx": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx/-/plugin-transform-react-jsx-7.12.12.tgz", + "integrity": "sha512-JDWGuzGNWscYcq8oJVCtSE61a5+XAOos+V0HrxnDieUus4UMnBEosDnY1VJqU5iZ4pA04QY7l0+JvHL1hZEfsw==", + "requires": { + "@babel/helper-annotate-as-pure": "^7.12.10", + "@babel/helper-module-imports": "^7.12.5", + "@babel/helper-plugin-utils": "^7.10.4", + "@babel/plugin-syntax-jsx": "^7.12.1", + "@babel/types": "^7.12.12" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-react-jsx-self": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.12.1.tgz", + "integrity": "sha512-FbpL0ieNWiiBB5tCldX17EtXgmzeEZjFrix72rQYeq9X6nUK38HCaxexzVQrZWXanxKJPKVVIU37gFjEQYkPkA==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/plugin-transform-react-jsx-source": { + "version": "7.12.1", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.12.1.tgz", + "integrity": "sha512-keQ5kBfjJNRc6zZN1/nVHCd6LLIHq4aUKcVnvE/2l+ZZROSbqoiGFRtT5t3Is89XJxBQaP7NLZX2jgGHdZvvFQ==", + "requires": { + "@babel/helper-plugin-utils": "^7.10.4" + }, + "dependencies": { + "@babel/helper-plugin-utils": { + "version": "7.10.4", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.10.4.tgz", + "integrity": "sha512-O4KCvQA6lLiMU9l2eawBPMf1xPP8xPfB3iEQw150hOVTqj/rfXz0ThTb4HEzqQfs2Bmo5Ay8BzxfzVtBrr9dVg==" + } + } + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } } }, "@babel/runtime": { @@ -5111,6 +5643,66 @@ "regenerator-runtime": "^0.13.4" } }, + "@babel/template": { + "version": "7.12.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.12.7.tgz", + "integrity": "sha512-GkDzmHS6GV7ZeXfJZ0tLRBhZcMcY0/Lnb+eEbXDBfCAcZCjrZKe6p3J4we/D24O9Y8enxWAg1cWwof59yLh2ow==", + "requires": { + "@babel/code-frame": "^7.10.4", + "@babel/parser": "^7.12.7", + "@babel/types": "^7.12.7" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, + "@babel/traverse": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.12.12.tgz", + "integrity": "sha512-s88i0X0lPy45RrLM8b9mz8RPH5FqO9G9p7ti59cToE44xFm1Q+Pjh5Gq4SXBbtb88X7Uy7pexeqRIQDDMNkL0w==", + "requires": { + "@babel/code-frame": "^7.12.11", + "@babel/generator": "^7.12.11", + "@babel/helper-function-name": "^7.12.11", + "@babel/helper-split-export-declaration": "^7.12.11", + "@babel/parser": "^7.12.11", + "@babel/types": "^7.12.12", + "debug": "^4.1.0", + "globals": "^11.1.0", + "lodash": "^4.17.19" + }, + "dependencies": { + "@babel/helper-validator-identifier": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.12.11.tgz", + "integrity": "sha512-np/lG3uARFybkoHokJUmf1QfEvRVCPbmQeUQpKow5cQ3xWrV9i3rUHodKDJPQfTVX61qKi+UdYk8kik84n7XOw==" + }, + "@babel/types": { + "version": "7.12.12", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.12.12.tgz", + "integrity": "sha512-lnIX7piTxOH22xE7fDXDbSHg9MM1/6ORnafpJmov5rs0kX5g4BZxeXNJLXsMRiO0U5Rb8/FvMS6xlTnTHvxonQ==", + "requires": { + "@babel/helper-validator-identifier": "^7.12.11", + "lodash": "^4.17.19", + "to-fast-properties": "^2.0.0" + } + } + } + }, "@babel/types": { "version": "7.10.2", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.10.2.tgz", @@ -5121,10 +5713,76 @@ "to-fast-properties": "^2.0.0" } }, + "ansi-styles": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", + "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", + "requires": { + "color-convert": "^1.9.0" + } + }, + "babel-loader": {}, + "chalk": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", + "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", + "requires": { + "ansi-styles": "^3.2.1", + "escape-string-regexp": "^1.0.5", + "supports-color": "^5.3.0" + } + }, + "color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "requires": { + "color-name": "1.1.3" + } + }, + "color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha1-p9BVi9icQveV3UIyj3QIMcpTvCU=" + }, + "debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "requires": { + "ms": "2.1.2" + } + }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "regenerator-runtime": { "version": "0.13.5", "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.5.tgz", "integrity": "sha512-ZS5w8CpKFinUzOwW3c83oPeVXoNsrLsaCoLtJvAClH135j/R77RuymhiSErhm2lKcwSCIpmvIWSbDkIfAqKQlA==" + }, + "regexpu-core": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/regexpu-core/-/regexpu-core-4.7.1.tgz", + "integrity": "sha512-ywH2VUraA44DZQuRKzARmw6S66mr48pQVva4LBeRhcOltJ6hExvWly5ZjFLYo67xbIxb6W1q4bAGtgfEl20zfQ==", + "requires": { + "regenerate": "^1.4.0", + "regenerate-unicode-properties": "^8.2.0", + "regjsgen": "^0.5.1", + "regjsparser": "^0.6.4", + "unicode-match-property-ecmascript": "^1.0.4", + "unicode-match-property-value-ecmascript": "^1.2.0" + } + }, + "supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "requires": { + "has-flag": "^3.0.0" + } } } }, @@ -6401,6 +7059,17 @@ "tar-pack": "3.4.1", "tmp": "0.0.33", "validate-npm-package-name": "3.0.0" + }, + "dependencies": { + "cross-spawn": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", + "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", + "requires": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + } } }, "create-react-class": { @@ -6422,15 +7091,6 @@ "warning": "^4.0.3" } }, - "cross-spawn": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-4.0.2.tgz", - "integrity": "sha1-e5JHYhwjrf3ThWAEqCPL45dCTUE=", - "requires": { - "lru-cache": "^4.0.1", - "which": "^1.2.9" - } - }, "crypto-browserify": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/crypto-browserify/-/crypto-browserify-3.12.0.tgz", @@ -8204,6 +8864,13 @@ "acorn": "^7.1.1", "acorn-jsx": "^5.2.0", "eslint-visitor-keys": "^1.1.0" + }, + "dependencies": { + "acorn-jsx": { + "version": "5.3.1", + "resolved": "https://registry.npmjs.org/acorn-jsx/-/acorn-jsx-5.3.1.tgz", + "integrity": "sha512-K0Ptm/47OKfQRpNQ2J/oIN/3QYiK6FwW+eJbILhsdxh2WTLdl+30o8aGdTbm5JbffpFFAg/g+zi1E+jvJha5ng==" + } } }, "esprima": { @@ -11669,24 +12336,6 @@ "resolved": "https://registry.npmjs.org/jss-global/-/jss-global-3.0.0.tgz", "integrity": "sha512-wxYn7vL+TImyQYGAfdplg7yaxnPQ9RaXY/cIA8hawaVnmmWxDHzBK32u1y+RAvWboa3lW83ya3nVZ/C+jyjZ5Q==" }, - "jss-nested": { - "version": "6.0.1", - "resolved": "https://registry.npmjs.org/jss-nested/-/jss-nested-6.0.1.tgz", - "integrity": "sha512-rn964TralHOZxoyEgeq3hXY8hyuCElnvQoVrQwKHVmu55VRDd6IqExAx9be5HgK0yN/+hQdgAXQl/GUrBbbSTA==", - "requires": { - "warning": "^3.0.0" - }, - "dependencies": { - "warning": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/warning/-/warning-3.0.0.tgz", - "integrity": "sha1-MuU3fLVy3kqwR1O9+IIcAe1gW3w=", - "requires": { - "loose-envify": "^1.0.0" - } - } - } - }, "jss-plugin-camel-case": { "version": "10.1.1", "resolved": "https://registry.npmjs.org/jss-plugin-camel-case/-/jss-plugin-camel-case-10.1.1.tgz", @@ -12182,19 +12831,6 @@ "resolved": "https://registry.npmjs.org/material-ui-nested-menu-item/-/material-ui-nested-menu-item-1.0.2.tgz", "integrity": "sha512-LZb8xI0FrAI/A3P2vT3CB9bmSoOFWOK0dikTc1t9VvEpp1a8hZkbVUz7VhETnoLUYu3NXCkgulmXcl3zitqI9A==" }, - "material-ui-pickers": { - "version": "2.2.4", - "resolved": "https://registry.npmjs.org/material-ui-pickers/-/material-ui-pickers-2.2.4.tgz", - "integrity": "sha512-QCQh08Ylmnt+o4laW+rPs92QRAcESv3sPXl50YadLm++rAZAXAOh3K8lreGdynCMYFgZfdyu81Oz9xzTlAZNfw==", - "requires": { - "@types/react-text-mask": "^5.4.3", - "clsx": "^1.0.2", - "react-event-listener": "^0.6.6", - "react-text-mask": "^5.4.3", - "react-transition-group": "^2.5.3", - "tslib": "^1.9.3" - } - }, "md5-file": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/md5-file/-/md5-file-4.0.0.tgz", @@ -12277,6 +12913,24 @@ "warning": "^4.0.1" }, "dependencies": { + "jss-nested": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/jss-nested/-/jss-nested-6.0.1.tgz", + "integrity": "sha512-rn964TralHOZxoyEgeq3hXY8hyuCElnvQoVrQwKHVmu55VRDd6IqExAx9be5HgK0yN/+hQdgAXQl/GUrBbbSTA==", + "requires": { + "warning": "^3.0.0" + }, + "dependencies": { + "warning": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/warning/-/warning-3.0.0.tgz", + "integrity": "sha1-MuU3fLVy3kqwR1O9+IIcAe1gW3w=", + "requires": { + "loose-envify": "^1.0.0" + } + } + } + }, "react-transition-group": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-2.9.0.tgz", @@ -12339,6 +12993,37 @@ } } }, + "jss-nested": { + "dependencies": { + "warning": {} + } + }, + "material-ui-pickers": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/material-ui-pickers/-/material-ui-pickers-2.2.4.tgz", + "integrity": "sha512-QCQh08Ylmnt+o4laW+rPs92QRAcESv3sPXl50YadLm++rAZAXAOh3K8lreGdynCMYFgZfdyu81Oz9xzTlAZNfw==", + "requires": { + "@types/react-text-mask": "^5.4.3", + "clsx": "^1.0.2", + "react-event-listener": "^0.6.6", + "react-text-mask": "^5.4.3", + "react-transition-group": "^2.5.3", + "tslib": "^1.9.3" + }, + "dependencies": { + "react-transition-group": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-2.9.0.tgz", + "integrity": "sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg==", + "requires": { + "dom-helpers": "^3.4.0", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2", + "react-lifecycles-compat": "^3.0.4" + } + } + } + }, "moment": { "version": "2.24.0", "resolved": "https://registry.npmjs.org/moment/-/moment-2.24.0.tgz", @@ -12393,7 +13078,8 @@ "loose-envify": "^1.4.0", "prop-types": "^15.6.2" } - } + }, + "sass-loader": {} } }, "mdn-data": { @@ -13806,6 +14492,19 @@ "requires": { "postcss": "^7.0.2", "postcss-selector-parser": "^6.0.2" + }, + "dependencies": { + "postcss-selector-parser": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz", + "integrity": "sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw==", + "requires": { + "cssesc": "^3.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1", + "util-deprecate": "^1.0.2" + } + } } }, "postcss-browser-comments": { @@ -13826,6 +14525,17 @@ "postcss-value-parser": "^4.0.2" }, "dependencies": { + "postcss-selector-parser": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/postcss-selector-parser/-/postcss-selector-parser-6.0.4.tgz", + "integrity": "sha512-gjMeXBempyInaBqpp8gODmwZ52WaYsVOsfr4L4lDQ7n3ncD6mEyySiDtgzCT+NYC0mmeOLvtsF8iaEf0YT6dBw==", + "requires": { + "cssesc": "^3.0.0", + "indexes-of": "^1.0.1", + "uniq": "^1.0.1", + "util-deprecate": "^1.0.2" + } + }, "postcss-value-parser": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-4.1.0.tgz", @@ -14889,13 +15599,12 @@ } }, "react": { - "version": "16.10.2", - "resolved": "https://registry.npmjs.org/react/-/react-16.10.2.tgz", - "integrity": "sha512-MFVIq0DpIhrHFyqLU0S3+4dIcBhhOvBE8bJ/5kHPVOVaGdo0KuiQzpcjCPsf585WvhypqtrMILyoE2th6dT+Lw==", + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/react/-/react-17.0.1.tgz", + "integrity": "sha512-lG9c9UuMHdcAexXtigOZLX8exLWkW0Ku29qPRU8uhF2R9BN96dLCt0psvzPLlHc5OWkgymP3qwTRgbnw5BKx3w==", "requires": { "loose-envify": "^1.1.0", - "object-assign": "^4.1.1", - "prop-types": "^15.6.2" + "object-assign": "^4.1.1" } }, "react-alert": { @@ -15306,14 +16015,24 @@ } }, "react-dom": { - "version": "16.10.2", - "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-16.10.2.tgz", - "integrity": "sha512-kWGDcH3ItJK4+6Pl9DZB16BXYAZyrYQItU4OMy0jAkv5aNqc+mAKb4TpFtAteI6TJZu+9ZlNhaeNQSVQDHJzkw==", + "version": "17.0.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-17.0.1.tgz", + "integrity": "sha512-6eV150oJZ9U2t9svnsspTMrWNyHc6chX0KzDeAOXftRa8bNeOKTTfCJ7KorIwenkHd2xqVTBTCZd79yk/lx/Ug==", "requires": { "loose-envify": "^1.1.0", "object-assign": "^4.1.1", - "prop-types": "^15.6.2", - "scheduler": "^0.16.2" + "scheduler": "^0.20.1" + }, + "dependencies": { + "scheduler": { + "version": "0.20.1", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.20.1.tgz", + "integrity": "sha512-LKTe+2xNJBNxu/QhHvDR14wUXHRQbVY5ZOYpOGWRzhydZUqrLb2JBvLPY7cAqFmqrWuDED0Mjk7013SZiOz6Bw==", + "requires": { + "loose-envify": "^1.1.0", + "object-assign": "^4.1.1" + } + } } }, "react-draggable": { @@ -15584,6 +16303,49 @@ "workbox-webpack-plugin": "4.3.1" }, "dependencies": { + "@babel/core": { + "version": "7.9.0", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.9.0.tgz", + "integrity": "sha512-kWc7L0fw1xwvI0zi8OKVBuxRVefwGOrKSQMvrQ3dW+bIIavBY3/NpXmpjMy7bQnLgwgzWQZ8TlM57YHpHNHz4w==", + "requires": { + "@babel/code-frame": "^7.8.3", + "@babel/generator": "^7.9.0", + "@babel/helper-module-transforms": "^7.9.0", + "@babel/helpers": "^7.9.0", + "@babel/parser": "^7.9.0", + "@babel/template": "^7.8.6", + "@babel/traverse": "^7.9.0", + "@babel/types": "^7.9.0", + "convert-source-map": "^1.7.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.1", + "json5": "^2.1.2", + "lodash": "^4.17.13", + "resolve": "^1.3.2", + "semver": "^5.4.1", + "source-map": "^0.5.0" + }, + "dependencies": { + "@babel/code-frame": { + "version": "7.12.11", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.12.11.tgz", + "integrity": "sha512-Zt1yodBx1UcyiePMSkWnU4hPqhwq7hGi2nFL1LeA3EUl+q2LQx16MISgJ0+z7dnmgvP9QtIleuETGOiOH1RcIw==", + "requires": { + "@babel/highlight": "^7.10.4" + } + }, + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==" + }, + "source-map": { + "version": "0.5.7", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", + "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" + } + } + }, "@babel/generator": { "version": "7.11.4", "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.11.4.tgz", @@ -15887,6 +16649,46 @@ "resolve": "^1.12.0" } }, + "babel-loader": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/babel-loader/-/babel-loader-8.1.0.tgz", + "integrity": "sha512-7q7nC1tYOrqvUrN3LQK4GwSk/TQorZSOlO9C+RZDZpODgyN4ZlCqE5q9cDsyWOliN+aU9B4JX01xK9eJXowJLw==", + "requires": { + "find-cache-dir": "^2.1.0", + "loader-utils": "^1.4.0", + "mkdirp": "^0.5.3", + "pify": "^4.0.1", + "schema-utils": "^2.6.5" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "ajv-keywords": { + "version": "3.5.2", + "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", + "integrity": "sha512-5p6WTN0DdTGVQk6VjcEju19IgaHudalcfabD7yhDGeA6bcQnmL+CpveLJq/3hvfwd1aof6L386Ougkx6RfyMIQ==" + }, + "schema-utils": { + "version": "2.7.1", + "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.1.tgz", + "integrity": "sha512-SHiNtMOUGWBQJwzISiVYKu82GiV4QYGePp3odlY1tuKO7gPtphAT5R/py0fA6xtbgLL/RvtJZnU9b8s0F1q0Xg==", + "requires": { + "@types/json-schema": "^7.0.5", + "ajv": "^6.12.4", + "ajv-keywords": "^3.5.2" + } + } + } + }, "cacache": { "version": "12.0.4", "resolved": "https://registry.npmjs.org/cacache/-/cacache-12.0.4.tgz", @@ -15982,6 +16784,11 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" }, + "pify": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/pify/-/pify-4.0.1.tgz", + "integrity": "sha512-uB80kBFb/tfd68bVleG9T5GGsGPjJrLAUpR5PZIrhBnIaRTQRjqdJSsIKkOP6OAIFbj7GOrcudc5pNjZ+geV2g==" + }, "schema-utils": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-1.0.0.tgz", @@ -16115,6 +16922,19 @@ "classnames": "^2.2.6", "prop-types": "^15.7.2", "react-transition-group": "^2.6.1" + }, + "dependencies": { + "react-transition-group": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/react-transition-group/-/react-transition-group-2.9.0.tgz", + "integrity": "sha512-+HzNTCHpeQyl4MJ/bdE0u6XRMe9+XG/+aL4mCxVN4DnPBQ0/5bfHWPDuOZUzYdMj94daZaZdCCc1Dzt9R/xSSg==", + "requires": { + "dom-helpers": "^3.4.0", + "loose-envify": "^1.4.0", + "prop-types": "^15.6.2", + "react-lifecycles-compat": "^3.0.4" + } + } } }, "react-transition-group": { @@ -16852,15 +17672,6 @@ "xmlchars": "^2.1.1" } }, - "scheduler": { - "version": "0.16.2", - "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.16.2.tgz", - "integrity": "sha512-BqYVWqwz6s1wZMhjFvLfVR5WXP7ZY32M/wYPo04CcuPM7XZEbV2TBNW7Z0UkguPTl0dWMA59VbNXxK6q+pHItg==", - "requires": { - "loose-envify": "^1.1.0", - "object-assign": "^4.1.1" - } - }, "schema-utils": { "version": "2.7.0", "resolved": "https://registry.npmjs.org/schema-utils/-/schema-utils-2.7.0.tgz", diff --git a/frontend/package.json b/frontend/package.json index b627526f..d2ff0059 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,13 +1,13 @@ { "name": "shuffler", "homepage": "https://shuffler.io", - "version": "0.8.3", + "version": "0.8.53", "private": true, "dependencies": { "@material-ui/core": "^4.5.2", "@material-ui/icons": "^4.5.1", "@material-ui/styles": "^4.5.2", - "@use-it/interval": "^0.1.3", + "@use-it/interval": "^1.0.0", "babel-eslint": "^10.1.0", "class-transformer": "^0.3.1", "create-react-app": "^2.0.3", @@ -32,7 +32,7 @@ "md5-file": "^4.0.0", "mdbreact": "^4.21.1", "moment": "~2.20.1", - "react": "^16.10.2", + "react": "^17.0.1", "react-alert": "^5.5.0", "react-alert-template-basic": "^1.0.0", "react-beforeunload": "^2.2.1", @@ -40,7 +40,7 @@ "react-cookie": "^4.0.1", "react-cytoscapejs": "^1.2.0", "react-device-detect": "^1.9.10", - "react-dom": "^16.10.2", + "react-dom": "^17.0.1", "react-draggable": "^3.3.2", "react-dropzone": "^10.1.10", "react-ga": "^2.7.0", From 6f945d1a6887884def9e07aa20fb876812890735 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 18 Jan 2021 10:04:02 +0100 Subject: [PATCH 64/89] Fixed API authentication for files --- .env | 2 +- backend/go-app/files.go | 3 +- backend/go-app/main.go | 6 +-- backend/go-app/walkoff.go | 61 +++++++++++++++++++------- docker-compose.yml | 6 +-- frontend/Dockerfile | 7 +-- frontend/src/views/Admin.jsx | 37 +++++++++++----- frontend/src/views/AngularWorkflow.jsx | 11 ++++- 8 files changed, 95 insertions(+), 38 deletions(-) diff --git a/.env b/.env index 45389f1f..5fdb2e83 100644 --- a/.env +++ b/.env @@ -29,7 +29,7 @@ BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 OUTER_HOSTNAME=shuffle-backend -DB_LOCATION=./shuffle-database +DB_LOCATION=./shuffle-database-new # Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps. # PS: It will skip proxy for diff --git a/backend/go-app/files.go b/backend/go-app/files.go index 7312af1f..0779a08c 100644 --- a/backend/go-app/files.go +++ b/backend/go-app/files.go @@ -701,7 +701,8 @@ func handleCreateFile(resp http.ResponseWriter, request *http.Request) { // Loads of validation below if len(curfile.Filename) == 0 || len(curfile.OrgId) == 0 || len(curfile.WorkflowId) == 0 { - log.Printf("[ERROR] Missing field during upload.") + log.Printf("[ERROR] Missing field during fileupload. Required: filename, org_id, workflow_id") + log.Printf("INPUT: %s", string(body)) resp.WriteHeader(401) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Missing field. Required: filename, org_id, workflow_id"}`))) return diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 6e304dd2..f19679e0 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -614,13 +614,13 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U apikey := request.Header.Get("Authorization") if len(apikey) > 0 { if !strings.HasPrefix(apikey, "Bearer ") { - log.Printf("Apikey doesn't start with bearer") + log.Printf("[WARNING] Apikey doesn't start with bearer") return User{}, errors.New("No bearer token for authorization header") } apikeyCheck := strings.Split(apikey, " ") if len(apikeyCheck) != 2 { - log.Printf("Invalid format for apikey.") + log.Printf("[WARNING] Invalid format for apikey.") return User{}, errors.New("Invalid format for apikey") } @@ -643,7 +643,7 @@ func handleApiAuthentication(resp http.ResponseWriter, request *http.Request) (U if len(Userdata.Username) > 0 { return Userdata, nil } else { - return Userdata, errors.New(fmt.Sprintf("User is invalid - no username found")) + return Userdata, errors.New(fmt.Sprintf("[WARNING] User is invalid - no username found")) } } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 0c5e2fc4..b8804c39 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -909,8 +909,8 @@ func validateNewWorkerExecution(body []byte) error { return err } - log.Printf("LEN: %s", string(body)) - log.Printf("LEN: %d", len(string(body))) + //log.Printf("LEN: %s", string(body)) + //log.Printf("LEN: %d", len(string(body))) baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err) @@ -1710,7 +1710,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { user.ActiveOrg.Users = []User{} workflow.ExecutingOrg = user.ActiveOrg workflow.OrgId = user.ActiveOrg.Id - log.Printf("TRIGGERS: %d", len(workflow.Triggers)) + //log.Printf("TRIGGERS: %d", len(workflow.Triggers)) ctx := context.Background() //err = increaseStatisticsField(ctx, "total_workflows", workflow.ID, 1, workflow.OrgId) @@ -1744,7 +1744,7 @@ func setNewWorkflow(resp http.ResponseWriter, request *http.Request) { // Initialized without functions = adding a hello world node. if len(newActions) == 0 { - log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") + //log.Printf("APPENDING NEW APP FOR NEW WORKFLOW") // Adds the Testing app if it's a new workflow workflowapps, err := getAllWorkflowApps(ctx) @@ -2188,8 +2188,10 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflowapps, apperr := getAllWorkflowApps(ctx) allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) - if err == nil && len(allAuths) > 0 && len(workflowapps) > 0 && apperr == nil { + if err == nil && len(workflowapps) > 0 && apperr == nil { + log.Printf("Setting actions") actionFixing := []Action{} + appsAdded := []string{} for _, action := range newActions { setAuthentication := false if len(action.AuthenticationId) > 0 { @@ -2253,6 +2255,12 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } } + for _, added := range appsAdded { + if outerapp.ID == added { + found = true + } + } + // FIXME: Add app auth if !found { timeNow := int64(time.Now().Unix()) @@ -2281,6 +2289,8 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { err = setWorkflowAppAuthDatastore(ctx, appAuth, appAuth.Id) if err != nil { log.Printf("Failed setting appauth for with name %s", appAuth.Label) + } else { + appsAdded = append(appsAdded, outerapp.ID) } } @@ -2299,14 +2309,16 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { } newActions = actionFixing + } else { + log.Printf("Err: %s - %s", err, apperr) + //workflowapps, apperr := getAllWorkflowApps(ctx) + //allAuths, err := getAllWorkflowAppAuth(ctx, user.ActiveOrg.Id) } - //workflow.PreviouslySaved = true + workflow.PreviouslySaved = true } - //PreviouslySaved bool `json:"first_save" datastore:"first_save"` workflow.Actions = newActions - newTriggers := []Trigger{} for _, trigger := range workflow.Triggers { log.Printf("Trigger %s: %s", trigger.TriggerType, trigger.Status) @@ -3421,15 +3433,19 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf return WorkflowExecution{}, "Failed building missing Docker images", err } - b, err := json.Marshal(workflowExecution) - if err == nil { - log.Printf("%s", string(b)) - log.Printf("LEN: %d", len(string(b))) - //workflowExecution.ExecutionOrg.SyncFeatures = Org{} - } + //b, err := json.Marshal(workflowExecution) + //if err == nil { + // log.Printf("%s", string(b)) + // log.Printf("LEN: %d", len(string(b))) + // //workflowExecution.ExecutionOrg.SyncFeatures = Org{} + //} - workflowExecution.Workflow.ExecutingOrg = Org{} - workflowExecution.Workflow.Org = []Org{} + workflowExecution.Workflow.ExecutingOrg = Org{ + Id: workflowExecution.Workflow.ExecutingOrg.Id, + } + workflowExecution.Workflow.Org = []Org{ + workflowExecution.Workflow.ExecutingOrg, + } //Org []Org `json:"org,omitempty" datastore:"org"` err = setWorkflowExecution(ctx, workflowExecution, true) if err != nil { @@ -4735,6 +4751,7 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { } if config.Action == "assign_everywhere" { + log.Printf("Should set authentication config") q := datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) q = q.Order("-edited").Limit(35) @@ -4750,14 +4767,21 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { // FIXME: Add function to remove auth from other auth's actionCnt := 0 workflowCnt := 0 + authenticationUsage := []AuthenticationUsage{} for _, workflow := range workflows { newActions := []Action{} edited := false + usage := AuthenticationUsage{ + WorkflowId: workflow.ID, + Nodes: []string{}, + } + for _, action := range workflow.Actions { if action.AppName == auth.App.Name { //log.Printf("FOUND ACTION TO UPDATE: %#v", action) edited = true actionCnt += 1 + usage.Nodes = append(usage.Nodes, action.ID) } newActions = append(newActions, action) @@ -4765,6 +4789,8 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { workflow.Actions = newActions if edited { + //auth.Usage = usage + authenticationUsage = append(authenticationUsage, usage) err = setWorkflow(ctx, workflow, workflow.ID) if err != nil { log.Printf("Failed setting (authupdate) workflow: %s", err) @@ -4775,9 +4801,12 @@ func setAuthenticationConfig(resp http.ResponseWriter, request *http.Request) { } } + //Usage []AuthenticationUsage `json:"usage" datastore:"usage"` + log.Printf("Found %d workflows, %d actions", workflowCnt, actionCnt) if actionCnt > 0 && workflowCnt > 0 { auth.WorkflowCount = int64(workflowCnt) auth.NodeCount = int64(actionCnt) + auth.Usage = authenticationUsage err = setWorkflowAppAuthDatastore(ctx, *auth, auth.Id) if err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index c6632614..1ce245a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:0.8.53 container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,8 +16,8 @@ services: depends_on: - backend backend: - #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.53 + build: ./backend + image: ghcr.io/frikky/shuffle-backend:0.8.54 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: diff --git a/frontend/Dockerfile b/frontend/Dockerfile index cc791b82..f48b048d 100644 --- a/frontend/Dockerfile +++ b/frontend/Dockerfile @@ -8,7 +8,8 @@ ENV PATH /usr/src/app/node_modules/.bin:$PATH COPY package.json /usr/src/app/package.json -RUN npm install --verbose +#RUN npm install --verbose +RUN yarn install # copy only required files to not trigger rebuilding every time COPY ./certs /usr/src/app/certs/ @@ -19,9 +20,9 @@ COPY ./*.json /usr/src/app/ # There were issues with the webpack installer from package.json RUN rm -rf /usr/src/app/node_modules/webpack -RUN npm install webpack@4.42.0 +#RUN yarn add webpack@4.42.0 -RUN npm run-script build +RUN yarn build # Production environment FROM nginx:latest diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index ae7ec058..bc94a537 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -301,6 +301,7 @@ const Admin = (props) => { } else { //alert.success("Successfully password!") setSelectedUserModalOpen(false) + getAppAuthentication() } }), ) @@ -762,6 +763,7 @@ const Admin = (props) => { .then((responseJson) => { if (responseJson.success) { //console.log(responseJson.data) + console.log(responseJson) setAuthentication(responseJson.data) } else { alert.error("Failed getting authentications") @@ -2033,7 +2035,7 @@ const Admin = (props) => {

App Authentication

Control the authentication options for individual apps. Actions can be destructive! - . Learn more +  Learn more
@@ -2087,7 +2089,7 @@ const Admin = (props) => { style={{minWidth: 150, maxWidth: 150}} /> { > - { - editAuthenticationConfig(data.id) - }} - > - - + {data.defined ? + + { + editAuthenticationConfig(data.id) + }} + > + + + + : + + { + }} + > + + + + } { deleteAuthentication(data) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index a1dead44..d860d561 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -948,7 +948,16 @@ const AngularWorkflow = (props) => { }) .then((responseJson) => { if (responseJson.success) { - setAppAuthentication(responseJson.data) + var newauth = [] + for (var key in responseJson.data) { + if (responseJson.data[key].defined === false) { + continue + } + + newauth.push(responseJson.data[key]) + } + + setAppAuthentication(newauth) } else { alert.error("Failed getting authentications") } From 79b6c6444c1c8aae6d4d96c6eb04f51920dd08e9 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 18 Jan 2021 10:08:53 +0100 Subject: [PATCH 65/89] Pushed new backend --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1ce245a3..9dec03cb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.54 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} From 5338bf24859a20c323da388fec8ac7323b8bd83a Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 18 Jan 2021 16:09:01 +0100 Subject: [PATCH 66/89] #204: Added file upload baseline to app creator --- backend/app_sdk/app_base.py | 8 +++--- backend/go-app/codegen.go | 25 ++++++++++++++---- frontend/src/views/AppCreator.jsx | 43 +++++++++++++++++++++++++++++++ frontend/src/views/Apps.jsx | 16 +++++++----- 4 files changed, 78 insertions(+), 14 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 14936845..77ccdb4c 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -397,7 +397,7 @@ class AppBase: } ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) - print("RET1: %s" % ret1.text) + print("RET1 (file get): %s" % ret1.text) if ret1.status_code != 200: returns.append({ "filename": "", @@ -408,7 +408,7 @@ class AppBase: content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) - print("Ret2: %s" % ret2.text) + print("RET2 (file get): %s" % ret2.text) if ret2.status_code == 200: tmpdata = ret1.json() returndata = { @@ -418,6 +418,8 @@ class AppBase: } returns.append(returndata) + print("RET3 (file get done)") + if len(returns) == 0: return { "success": False, @@ -1687,7 +1689,7 @@ class AppBase: print("[INFO] APP_SDK DONE: Starting NORMAL execution of function") print("[INFO] Running with params (0): %s" % params) newres = await func(**params) - print("[INFO] Returned from execution.") + print("[INFO] Returned from execution:", newres) if isinstance(newres, tuple): print("[INFO] Handling return as tuple") # Handles files. diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index ba029b15..2c7c64d7 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -365,6 +365,16 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet preparedHeaders += "}" } + fileBalance := "" + fileAdder := `` + fileGrabber := `` + if method == "post" && strings.Contains(functionname, "filescan") { + fileGrabber = `filedata = self.get_file("63264006-5958-451a-bff1-1975495fb4d8")` + //fileGrabber += "\n print(filedata)" + fileAdder = `files = {"file": (filedata["filename"], filedata["data"])}` + fileBalance = ", files=files" + } + // Extra param for url if it's changeable // Extra param for authentication scheme(s) // The last weird one is the body.. Tabs & spaces sucks. @@ -375,7 +385,9 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet %s %s %s - return requests.%s(url, headers=headers%s%s%s).text + %s + %s + return requests.%s(url, headers=headers%s%s%s%s).text `, functionname, authenticationParameter, @@ -391,17 +403,20 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet authenticationSetup, queryData, bodyFormatter, + fileGrabber, + fileAdder, method, authenticationAddin, bodyAddin, verifyAddin, + fileBalance, ) //log.Printf("FUNCTION: %s", data) - //if strings.Contains(functionname, "search") { - // log.Println(data) - // log.Printf("Queries: %s", queryString) - //} + if strings.Contains(functionname, "filescan") { + log.Println(data) + log.Printf("Queries: %s", queryString) + } //log.Printf(data) return functionname, data diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 3171610a..a4fabc8e 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -226,6 +226,7 @@ const AppCreator = (props) => { const [errorCode, setErrorCode] = useState("") const [appBuilding, setAppBuilding] = useState(false) const [extraBodyFields, setExtraBodyFields] = useState([]) + const [fileUploadEnabled, setFileUploadEnabled] = useState(false) //const [actions, setActions] = useState([{ // "name": "Get workflows", @@ -254,6 +255,7 @@ const AppCreator = (props) => { const [currentActionMethod, setCurrentActionMethod] = useState(actionNonBodyRequest[0]) const [currentAction, setCurrentAction] = useState({ "name": "", + "file_field": "", "description": "", "url": "", "headers": "", @@ -494,6 +496,7 @@ const AppCreator = (props) => { "name": tmpname, "description": methodvalue.description, "url": path, + "file_field": "", "method": method.toUpperCase(), "headers": "", "queries": [], @@ -959,6 +962,11 @@ const AppCreator = (props) => { data.paths[item.url][item.method.toLowerCase()].parameters.push(newitem) } + // https://swagger.io/docs/specification/describing-request-body/file-upload/ + if (item.file_field !== undefined && item.file_field !== null && item.file_field.length > 0) { + console.log("HANDLE FILEFIELD SAVE: ", item.file_field) + } + if (item.headers.length > 0) { const required = false @@ -1171,6 +1179,7 @@ const AppCreator = (props) => { "name": "", "description": "", "url": "", + "file_field": "", "headers": "", "paths": [], "queries": [], @@ -1609,6 +1618,7 @@ const AppCreator = (props) => { "name": "", "description": "", "url": "", + "file_field": "", "headers": "", "paths": [], "queries": [], @@ -1619,6 +1629,7 @@ const AppCreator = (props) => { setCurrentActionMethod(apikeySelection[0]) setUrlPathQueries([]) setActionsModalOpen(false) + setFileUploadEnabled(false) }} > @@ -1806,6 +1817,36 @@ const AppCreator = (props) => { + {currentActionMethod === "POST" ? + + : null} + {fileUploadEnabled ? + setActionField("file_field", e.target.value)} + helperText={The File field to interact with} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style:{ + color: "white", + }, + }} + /> + : null}
Headers: static for the action { setActionsModalOpen(false) setUrlPathQueries([]) setUrlPath("") + setFileUploadEnabled(false) }}> Submit @@ -1916,6 +1958,7 @@ const AppCreator = (props) => { "name": "", "description": "", "url": "", + "file_field": "", "headers": "", "queries": [], "paths": [], diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 97689cac..bcee8fe5 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -807,12 +807,16 @@ const Apps = (props) => { const reader = new FileReader(); - reader.addEventListener('load', (e) => { - const content = e.target.result; - setOpenApiData(content); - setIsDropzone(isDropzone); - setOpenApiModal(true) - }) + try { + reader.addEventListener('load', (e) => { + const content = e.target.result; + setOpenApiData(content); + setIsDropzone(isDropzone); + setOpenApiModal(true) + }) + } catch (e) { + console.log("Error in dropzone: ", e) + } reader.readAsText(files[0]); }; From 792a0b86caacc2e5666863cd4141b67702e89cde Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 18 Jan 2021 16:18:19 +0100 Subject: [PATCH 67/89] Rolled back env --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 5fdb2e83..45389f1f 100644 --- a/.env +++ b/.env @@ -29,7 +29,7 @@ BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 OUTER_HOSTNAME=shuffle-backend -DB_LOCATION=./shuffle-database-new +DB_LOCATION=./shuffle-database # Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps. # PS: It will skip proxy for From 42bfa92c81ed432788d7eff9085a332e91fc263e Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 20 Jan 2021 16:03:01 +0100 Subject: [PATCH 68/89] Added tracker of selected executions --- backend/app_sdk/app_base.py | 2 +- backend/go-app/codegen.go | 4 ++-- backend/go-app/docker.go | 2 +- backend/go-app/walkoff.go | 14 ++++++------ docker-compose.yml | 2 +- frontend/src/views/AngularWorkflow.jsx | 30 +++++++++++++++++++++++--- frontend/src/views/AppCreator.jsx | 7 +++--- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 18 ++++++++++------ functions/onprem/worker/worker.go | 4 ++++ 10 files changed, 60 insertions(+), 25 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 77ccdb4c..23c695eb 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -601,7 +601,7 @@ class AppBase: self.full_execution = fullexecution - self.logger.info("AFTER FULLEXEC stream result") + self.logger.info("AFTER FULLEXEC stream result (init)") # Gets the value at the parenthesis level you want def parse_nested_param(string, level): diff --git a/backend/go-app/codegen.go b/backend/go-app/codegen.go index 2c7c64d7..ee182ef6 100644 --- a/backend/go-app/codegen.go +++ b/backend/go-app/codegen.go @@ -412,8 +412,8 @@ func makePythoncode(swagger *openapi3.Swagger, name, url, method string, paramet fileBalance, ) - //log.Printf("FUNCTION: %s", data) - if strings.Contains(functionname, "filescan") { + if strings.Contains(functionname, "get_list_rulesssss") { + //log.Printf("FUNCTION: %s", data) log.Println(data) log.Printf("Queries: %s", queryString) } diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index f05e4d38..0a0b4bae 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -190,7 +190,7 @@ func fixTags(tags []string) []string { */ // Custom Docker image builder wrapper in memory -func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string) error { +func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error { ctx := context.Background() client, err := client.NewEnvClient() if err != nil { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b8804c39..4a468d29 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -909,8 +909,6 @@ func validateNewWorkerExecution(body []byte) error { return err } - //log.Printf("LEN: %s", string(body)) - //log.Printf("LEN: %d", len(string(body))) baseExecution, err := getWorkflowExecution(ctx, execution.ExecutionId) if err != nil { log.Printf("[ERROR] Failed getting execution (workflowqueue) %s: %s", execution.ExecutionId, err) @@ -6421,7 +6419,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin if len(extra) == 0 { log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) for _, item := range buildLaterFirst { - err = buildImageMemory(fs, item.Tags, item.Extra) + err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { log.Printf("Failed image build memory: %s", err) } else { @@ -6435,7 +6433,7 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin log.Printf("[INFO] Starting build of %d skipped docker images", len(buildLaterList)) for _, item := range buildLaterList { - err = buildImageMemory(fs, item.Tags, item.Extra) + err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { log.Printf("[INFO] Failed image build memory: %s", err) } else { @@ -6647,15 +6645,19 @@ func getAllSchedules(ctx context.Context, orgId string) ([]ScheduleOld, error) { return schedules, nil } +//FIXME: Add cursor func getAllWorkflowApps(ctx context.Context) ([]WorkflowApp, error) { var allworkflowapps []WorkflowApp - q := datastore.NewQuery("workflowapp").Order("-edited").Limit(50) + q := datastore.NewQuery("workflowapp").Order("-edited").Limit(40) + //Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` //Activated bool `json:"activated" yaml:"activated" required:false datastore:"activated"` _, err := dbclient.GetAll(ctx, q, &allworkflowapps) if err != nil { if strings.Contains(fmt.Sprintf("%s", err), "ResourceExhausted") { - q := datastore.NewQuery("workflowapp").Limit(40).Order("-edited") + //datastore.NewQuery("workflowapp").Limit(30).Order("-edited") + q = datastore.NewQuery("workflowapp").Order("-edited").Limit(27) + //q := q.Limit(25) _, err := dbclient.GetAll(ctx, q, &allworkflowapps) if err != nil { return []WorkflowApp{}, err diff --git a/docker-compose.yml b/docker-compose.yml index 9dec03cb..1ce245a3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.54 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index d860d561..6605e386 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -153,6 +153,7 @@ const AngularWorkflow = (props) => { const [requiresAuthentication, setRequiresAuthentication] = React.useState(false) const [rightSideBarOpen, setRightSideBarOpen] = React.useState(false) const [showSkippedActions, setShowSkippedActions] = React.useState(false) + const [lastExecution, setLastExecution] = React.useState("") const [selectedResult, setSelectedResult] = React.useState({}) const [codeModalOpen, setCodeModalOpen] = React.useState(false); @@ -6380,13 +6381,14 @@ const AngularWorkflow = (props) => { - + {workflowExecutions.length > 0 ?
{workflowExecutions.map((data, index) => { @@ -6405,6 +6407,7 @@ const AngularWorkflow = (props) => { } return ( + {}} onMouseOut={() => {}} onClick={() => { if (data.result === undefined || data.result === null || data.result.length === 0) { @@ -6420,7 +6423,7 @@ const AngularWorkflow = (props) => { setExecutionData(data) }}>
-
+
{getExecutionSourceImage(data)}
@@ -6436,9 +6439,14 @@ const AngularWorkflow = (props) => { : null}
- + {lastExecution === data.execution_id ? + + : + + } + ) return })} @@ -6457,6 +6465,7 @@ const AngularWorkflow = (props) => { stop() getWorkflowExecution(props.match.params.key) setExecutionModalView(0) + setLastExecution(executionData.execution_id) }}> {}}> @@ -6617,6 +6626,21 @@ const AngularWorkflow = (props) => { validate.result = JSON.parse(validate.result) } + //if (codeModalOpen && selectedResult.result.includes("file_id")) { + // console.log("SHOW RESULT WITH FILES: ", selectedResult.result) + // //const regex = "\b[0-9a-f]{8}\b-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-\b[0-9a-f]{12}\b" + // //const regex = /^[0-9A-F]{8}-[0-9A-F]{4}-4[0-9A-F]{3}-[89AB][0-9A-F]{3}-[0-9A-F]{12}/i + // const regex = /^[A-F\d]{8}-[A-F\d]{4}-4[A-F\d]{3}-[89AB][A-F\d]{3}-[A-F\d]{12}$/i + // //const found = selectedResult.result.match(regex) + // const found = "hello how are you cf80fa70-65cf-4963-b474-b459a6dead81 what".match(regex) + // const regex = /\${(\w{8}-\w{4}-\w{3}-\w{3}-\w)}/g + // const found = placeholder.match(regex) + + // console.log("FOUND: ", found) + + // //cf80fa70-65cf-4963-b474-b459a6dead81 + //} + const codePopoutModal = !codeModalOpen ? null : { diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index a4fabc8e..82c28446 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -479,11 +479,12 @@ const AppCreator = (props) => { for (let [path, pathvalue] of Object.entries(data.paths)) { for (let [method, methodvalue] of Object.entries(pathvalue)) { if (methodvalue === null) { - alert.info("Skipped method "+method) + alert.info("Skipped method (null)"+method) continue } if (!allowedfunctions.includes(method.toUpperCase())) { + alert.info("Skipped method (not allowed) "+method) continue } @@ -556,7 +557,7 @@ const AppCreator = (props) => { } // HAHAHA wtf is this. - if (methodvalue.responses !== undefined) { + if (methodvalue.responses !== undefined && methodvalue.responses !== null) { if (methodvalue.responses.default !== undefined) { if (methodvalue.responses.default.content !== undefined) { if (methodvalue.responses.default.content["text/plain"] !== undefined) { @@ -1818,7 +1819,7 @@ const AppCreator = (props) => { addPathQuery() }}>New query {currentActionMethod === "POST" ? - {currentActionMethod === "POST" ? - From 09002f4aa316e66a702745e2cc6ef3e16988d355 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 22 Jan 2021 13:27:28 +0100 Subject: [PATCH 77/89] Moved executions to workflow view --- backend/go-app/walkoff.go | 1 - frontend/src/views/AngularWorkflow.jsx | 6 ++++++ frontend/src/views/Workflows.jsx | 6 ++++-- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 66555157..fa8389ea 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -4940,7 +4940,6 @@ func addAppAuthentication(resp http.ResponseWriter, request *http.Request) { if foundIndex >= 0 { log.Printf("[INFO] Found app %s by looping auth", appAuth.App.ID) - app = &workflowapps[foundIndex] } else { log.Printf("[ERROR] Failed finding app %s which has auth after looping", appAuth.App.ID) resp.WriteHeader(409) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 36c07f4e..9e494d91 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1536,6 +1536,12 @@ const AngularWorkflow = (props) => { getWorkflowExecution(props.match.params.key) getAvailableWorkflows(-1) getSettings() + + const cursearch = typeof window === 'undefined' || window.location === undefined ? "" : window.location.search + const tmpView = new URLSearchParams(cursearch).get("view") + if (tmpView !== undefined && tmpView !== null && tmpView === "executions") { + setExecutionModalOpen(true) + } return } diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index ca0351ec..960a7fbc 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -907,7 +907,7 @@ const Workflows = (props) => {
:

- There are no executions for this workflow yet + Executions have been moved to the Workflow itself. Click here to see them

) } @@ -1247,7 +1247,8 @@ const Workflows = (props) => {
-
+ {/* +

Execution Timeline

@@ -1265,6 +1266,7 @@ const Workflows = (props) => {
+ */}
) } From 73b7469107ed43a2d6ea9703ac0a5fe2af3c233a Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 22 Jan 2021 13:55:04 +0100 Subject: [PATCH 78/89] #223: Added import and export protections for subflow --- backend/go-app/walkoff.go | 39 ++++++++++++++++++++++++++------ frontend/src/views/Workflows.jsx | 25 ++++++++++++++------ 2 files changed, 50 insertions(+), 14 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index fa8389ea..3b4db6e5 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2332,15 +2332,40 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { trigger.Status = "stopped" } } else if trigger.TriggerType == "SUBFLOW" { - for _, param := range trigger.Parameters { + for index, param := range trigger.Parameters { if len(param.Value) == 0 && param.Name != "argument" { - workflow.IsValid = false - workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} + log.Printf("Param: %#v", param) + if param.Name == "user_apikey" { + apikey := "" + if len(user.ApiKey) > 0 { + apikey = user.ApiKey + } else { + user, err = generateApikey(ctx, user) + if err != nil { + workflow.IsValid = false + workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} - log.Printf("No type specified for user input node") - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) - return + log.Printf("No type specified for user input node") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) + return + } + + apikey = user.ApiKey + } + + log.Printf("[INFO] Set apikey in subflow trigger for user during save") + trigger.Parameters[index].Value = apikey + } else { + + workflow.IsValid = false + workflow.Errors = []string{"Trigger is missing a parameter: %s", param.Name} + + log.Printf("No type specified for user input node") + resp.WriteHeader(401) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Trigger %s is missing the parameter %s"}`, trigger.Label, param.Name))) + return + } } } } else if trigger.TriggerType == "WEBHOOK" && trigger.Status != "uninitialized" { diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 960a7fbc..83616e2b 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -234,8 +234,8 @@ const Workflows = (props) => { color: "#ffffff", width: "100%", display: "flex", - minWidth: 1366, - maxWidth: 1766, + minWidth: 1024, + maxWidth: 1024, margin: "auto", maxHeight: "90vh", } @@ -393,8 +393,15 @@ const Workflows = (props) => { data["owner"] = "" for (var key in data.triggers) { - if (data.triggers[key].status == "running") { - data.triggers[key].status = "stopped" + const trigger = data.triggers[key] + if (trigger.app_name === "Shuffle Workflow") { + if (trigger.parameters.length > 2) { + trigger.parameters[2].value = "" + } + } + + if (trigger.status == "running") { + trigger.status = "stopped" } } @@ -402,6 +409,8 @@ const Workflows = (props) => { data.actions[key].authentication_id = "" } + //return + data["org"] = [] data["org_id"] = "" data.execution_org = {"id": ""} @@ -907,7 +916,7 @@ const Workflows = (props) => {
:

- Executions have been moved to the Workflow itself. Click here to see them + Executions have been moved to the Workflow itself.
Click here to see them

) } @@ -1230,10 +1239,11 @@ const Workflows = (props) => {
-
+

Executions: {selectedWorkflow.name}

-
+ {/* +
+ */}
From c413dcece5ea6bd5f2a6cc9ae592795d800e24d0 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 22 Jan 2021 16:31:08 +0100 Subject: [PATCH 79/89] 0.8.56 build update --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index cef90c06..addef547 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: frontend: #build: ./frontend - image: ghcr.io/frikky/shuffle-frontend:0.8.55 + image: ghcr.io/frikky/shuffle-frontend:0.8.56 container_name: shuffle-frontend hostname: shuffle-frontend ports: @@ -17,7 +17,7 @@ services: - backend backend: #build: ./backend - image: ghcr.io/frikky/shuffle-backend:0.8.55 + image: ghcr.io/frikky/shuffle-backend:0.8.56 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} # Here for debugging: From 9b1f7fdce6b40d8e0abd7c3d9d79148ee79ce4d8 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 22 Jan 2021 16:46:31 +0100 Subject: [PATCH 80/89] Fixed env --- .env | 2 +- frontend/src/views/AngularWorkflow.jsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.env b/.env index 5fdb2e83..45389f1f 100644 --- a/.env +++ b/.env @@ -29,7 +29,7 @@ BACKEND_PORT=5001 FRONTEND_PORT=3001 FRONTEND_PORT_HTTPS=3443 OUTER_HOSTNAME=shuffle-backend -DB_LOCATION=./shuffle-database-new +DB_LOCATION=./shuffle-database # Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps. # PS: It will skip proxy for diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 9e494d91..f4ef05c4 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1396,7 +1396,7 @@ const AngularWorkflow = (props) => { console.log("ESCAPE") break; case 46: - removeNode() + //removeNode() console.log("DELETE") break; case 38: From 5dd78211d4c45437e7582206547747e2a3811673 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 23 Jan 2021 17:04:52 +0100 Subject: [PATCH 81/89] Added appcreator autocomplete feature for GET {PATH} --- backend/go-app/walkoff.go | 4 +- frontend/src/views/AppCreator.jsx | 24 ++++- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 151 ++++++++++++++++++++---------- 4 files changed, 121 insertions(+), 60 deletions(-) diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 3b4db6e5..d12d969d 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -3306,7 +3306,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf if len(action.Label) == 0 { action.Label = action.ID } - log.Printf("LABEL: %s", action.Label) + //log.Printf("LABEL: %s", action.Label) newActions = append(newActions, action) // If the node is NOT found, it's supposed to be set to SKIPPED, @@ -3360,7 +3360,7 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf } for _, trigger := range workflowExecution.Workflow.Triggers { - log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) + //log.Printf("[INFO] ID: %s vs %s", trigger.ID, workflowExecution.Start) if trigger.ID == workflowExecution.Start { if trigger.AppName == "User Input" { startFound = true diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 6978dd08..0c673342 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -1784,7 +1784,20 @@ const AppCreator = (props) => { }} onBlur={event => { var parsedurl = event.target.value - if (parsedurl.startsWith("curl")) { + if (parsedurl.startsWith("PUT ") || parsedurl.startsWith("GET ") ||parsedurl.startsWith("POST ") || parsedurl.startsWith("DELETE ") ||parsedurl.startsWith("PATCH ") || parsedurl.startsWith("CONNECT ")) { + const tmp = parsedurl.split(" ") + + if (tmp.length > 1) { + parsedurl = tmp[1] + setActionField("url", parsedurl) + setUrlPath(parsedurl) + + setCurrentActionMethod(tmp[0].toUpperCase()) + setActionField("method", tmp[0].toUpperCase()) + } + + setUpdate(Math.random()) + } else if (parsedurl.startsWith("curl")) { const request = parseCurl(event.target.value) if (request !== event.target.value) { if (request.method.toUpperCase() !== currentAction.Method) { @@ -1819,7 +1832,8 @@ const AppCreator = (props) => { } } - if (parsedurl !== undefined) { + console.log("PARSED: ", parsedurl) + if (parsedurl !== undefined) { if (parsedurl.includes("<") && parsedurl.includes(">")) { parsedurl = parsedurl.split("<").join("{") parsedurl = parsedurl.split(">").join("}") @@ -1920,9 +1934,9 @@ const AppCreator = (props) => { Cancel +
+ + {actionAmount > 0 && actionAmount < actions.length ? null : + + } +
diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 430645ef..15289670 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -21,6 +21,7 @@ import {Link} from 'react-router-dom'; import Breadcrumbs from '@material-ui/core/Breadcrumbs'; import ReactJson from 'react-json-view' import Chip from '@material-ui/core/Chip'; +import { useTheme } from '@material-ui/core/styles'; import CachedIcon from '@material-ui/icons/Cached'; import CloudDownloadIcon from '@material-ui/icons/CloudDownload'; @@ -113,6 +114,7 @@ const Apps = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata } = props; //const [workflows, setWorkflows] = React.useState([]); + const theme = useTheme(); const baseRepository = "https://github.com/frikky/shuffle-apps" const alert = useAlert() const [selectedApp, setSelectedApp] = React.useState({}); @@ -316,10 +318,18 @@ const Apps = (props) => { boxColor = "orange" } + if (data.invalid) { + boxColor = "red" + } + + //
+ //
var imageline = data.large_image.length === 0 ? - {data.title} + {data.title} : - {data.title} + {data.title} { + //console.log("IMG LOADED!: ", event.target) + }} /> // FIXME - add label to apps, as this might be slow with A LOT of apps var newAppname = data.name @@ -341,7 +351,7 @@ const Apps = (props) => { } var description = data.description - const maxDescLen = 56 + const maxDescLen = 51 if (description.length > maxDescLen) { description = data.description.slice(0, maxDescLen)+"..." } @@ -364,8 +374,8 @@ const Apps = (props) => { } } }}> - - + + {imageline}
@@ -520,9 +530,9 @@ const Apps = (props) => { : null var imageline = selectedApp.large_image === undefined || selectedApp.large_image.length === 0 ? - {selectedApp.title} + {selectedApp.title} : - {selectedApp.title} + {selectedApp.title} const GetAppExample = () => { if (selectedAction.returns === undefined) { @@ -639,7 +649,7 @@ const Apps = (props) => { updateAppField(selectedApp.id, "sharing", !selectedApp.sharing) //setSelectedAction(event.target.value) }} - style={{width: 150, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}} + style={{width: 150, backgroundColor: theme.palette.surfaceColor, backgroundColor: inputColor, color: "white", height: 35, marginleft: 10,}} SelectDisplayProps={{ style: { marginLeft: 10, From 6ad9a3bfaea72b022877d7d5761fc56c0e69034b Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 27 Jan 2021 15:19:45 +0100 Subject: [PATCH 89/89] Updated README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c5bfd74..f9589f40 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ https://shuffler.io -**Shuffle Apps** +[**App magicians**](https://github.com/frikky/shuffle-apps)