diff --git a/.env b/.env index 5a78c4d8..ac324e6a 100644 --- a/.env +++ b/.env @@ -50,8 +50,8 @@ SHUFFLE_PASS_APP_PROXY=FALSE TZ=Europe/Amsterdam # Timezone-handler in Orborus, Worker and Apps ORBORUS_CONTAINER_NAME= # Used to FIND the containername. cgroup v2: issue 501 -SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_NAME=frikky +SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.80" # Used for auto-cleanup of containers. REALLY important at scale. diff --git a/README.md b/README.md index 261358ee..01fd94b1 100644 --- a/README.md +++ b/README.md @@ -5,13 +5,14 @@ Shuffle Automation

-[Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. + +![Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. [_Key Features_](https://shuffler.io/docs/features) — -[_Community & Support_](https://discord.gg/B2CBzUm) +[_Community & Support_](https://discord.gg/B2CBzUm) — [_Documentation_](https://shuffler.io/docs) — [_Getting Started_](https://shuffler.io/docs/getting_started) — -[_Development_](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md) — +[_Development_](https://github.com/frikky/Shuffle/blob/master/.github/CONTRIBUTING.md) Follow us on Twitter at [@shuffleio](https://twitter.com/shuffleio). diff --git a/backend/app_sdk/README.md b/backend/app_sdk/README.md index 170781f3..478fb394 100644 --- a/backend/app_sdk/README.md +++ b/backend/app_sdk/README.md @@ -8,5 +8,15 @@ This is the SDK used for apps to behave like they should. 4. Delete the specific app's Docker image (docker rmi frikky/shuffle:...) 5. Rebuild the Docker image (click load in GUI?) +## Cloud updates +1. Go to shuffle cloud on GCP +2. Go to Cloud Storage +3. Find shuffler.appspot.com +4. Navigate to generated_apps/baseline +5. Update SDK there. This will make all new apps run with the new SDK + +## Cloud app force-updates +1. Run the "stitcher.go" program in the public shuffle-shared repository. + # LICENSE Everything in here is MIT, not AGPLv3 as indicated by the license. diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 190d983a..6889ab75 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -62,12 +62,100 @@ class AppBase: if len(self.base_url) == 0: self.base_url = self.url + # Checks output for whether it should be automatically parsed or not + def run_magic_parser(self, input_data): + if not isinstance(input_data, str): + return input_data + + # Don't touch existing JSON/lists + if (input_data.startswith("[") and input_data.endswith("]")) or (input_data.startswith("{") and input_data.endswith("}")): + return input_data + + # Don't touch large data. + if len(input_data) > 100000: + return input_data + + + new_input = input_data + try: + #new_input.strip() + new_input = input_data.split() + new_return = [] + + index = 0 + for item in new_input: + splititem = "," + if ", " in item: + splititem = ", " + elif "," in item: + splititem = "," + else: + new_return.append(item) + index += 1 + continue + + #print("FIX ITEM %s" % item) + for subitem in item.split(splititem): + new_return.insert(index, subitem) + + index += 1 + + # Prevent large data or infinite loops + if index > 10000: + return input_data + + fixed_return = [] + for item in new_return: + if not item: + continue + + if not isinstance(item, str): + fixed_return.append(item) + continue + + if item.endswith(","): + item = item[0:-1] + + fixed_return.append(item) + + new_input = fixed_return + except Exception as e: + self.logger.info(f"[DEBUG] Failed to run magic parser (2): {e}") + return input_data + + try: + new_input = input_data.split() + except Exception as e: + self.logger.info(f"[DEBUG] Failed to run magic parser (1): {e}") + return input_data + + # Won't ever touch this one? + if isinstance(input_data, list) or isinstance(input_data, object): + try: + return json.dumps(new_input) + except Exception as e: + self.logger.info(f"[DEBUG] Failed to run magic parser: {e}") + + return new_input + # FIXME: Add more info like logs in here. # Docker logs: https://forums.docker.com/t/docker-logs-inside-the-docker-container/68190/2 def send_result(self, action_result, headers, stream_path): if action_result["status"] == "EXECUTING": action_result["status"] = "FAILURE" + try: + if self.original_action["run_magic_output"] == True: + self.logger.warning("[INFO] Action result ran with Magic parser output.") + action_result["result"] = self.run_magic_parser(action_result["result"]) + else: + self.logger.warning("[ERROR] Magic output not defined.") + except Exception as e: + self.logger.warning("[ERROR] Failed to run magic autoparser: {e}") + pass + + # Try it with some magic + self.logger.info(f"""[DEBUG] Inside Send result with status {action_result["status"]}""") # FIXME: Add cleanup of parameters to not send to frontend here @@ -981,12 +1069,20 @@ class AppBase: except json.decoder.JSONDecodeError: pass - self.action_result["result"] = "Bad result from backend: %d" % ret.status_code + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"Bad result from backend during startup of app: {ret.status_code}", + "extended_reason": f"{ret.text}" + }) self.send_result(self.action_result, headers, stream_path) return except requests.exceptions.ConnectionError as e: self.logger.info("[DEBUG] FullExec Connectionerror: %s" % e) - self.action_result["result"] = "Connection error during startup: %s" % e + self.action_result["result"] = json.dumps({ + "success": False, + "reason": f"Connection error during startup: {e}" + }) + self.send_result(self.action_result, headers, stream_path) return else: @@ -2805,7 +2901,7 @@ class AppBase: # Dump the result as a string of a list #self.logger.info("RESULTS: %s" % results) if isinstance(results, list) or isinstance(results, dict): - self.logger.info("JSON OBJECT? ", json_object) + self.logger.info(f"JSON OBJECT? {json_object}") # This part is weird lol if json_object: diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 02732783..ab1c9f38 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.35 +VERSION=0.9.40 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/app_sdk/requirements.txt b/backend/app_sdk/requirements.txt index de650cea..e8a534c2 100644 --- a/backend/app_sdk/requirements.txt +++ b/backend/app_sdk/requirements.txt @@ -1,7 +1,7 @@ urllib3==1.26.5 requests==2.25.1 MarkupSafe==2.0.1 -liquidpy==0.7.2 +liquidpy==0.7.3 flask[async]==2.0.2 #waitress==2.0.0 #flask==1.1.2 diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 60c16bc6..7ff5c0b1 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -299,6 +299,7 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin } if !downloaded { + return errors.New(fmt.Sprintf("Failed to build / download images %s", strings.Join(tags, ","))) } //baseDockerName diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index a479da25..48c5c4cd 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.15 -//replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index b38f5d9f..420dbd78 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -2467,6 +2467,7 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId log.Printf("[INFO] Should send email to %s during execution.", email) } + if strings.Contains(triggerType, "sms") { action := shuffle.CloudSyncJob{ Type: "user_input", @@ -2491,7 +2492,11 @@ func handleUserInput(trigger shuffle.Trigger, organizationId string, workflowId return err } - log.Printf("Should send SMS to %s during execution.", sms) + log.Printf("[DEBUG] Should send SMS to %s during execution.", sms) + } + + if strings.Contains(triggerType, "subflow") { + log.Printf("[DEBUG] Should run a subflow with the result for user input.") } return nil @@ -2927,7 +2932,21 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os. for _, item := range buildLaterFirst { err = buildImageMemory(fs, item.Tags, item.Extra, true) if err != nil { - log.Printf("Failed image build memory: %s", err) + orgId := "" + + log.Printf("[DEBUG] Failed image build memory. Creating notification with org %#v: %s", orgId, err) + + if len(item.Tags) > 0 { + err = shuffle.CreateOrgNotification( + ctx, + fmt.Sprintf("App failed to build"), + fmt.Sprintf("The app %s with image %s failed to build. Check backend logs with docker! docker logs shuffle-backend", item.Tags[0], item.Extra), + fmt.Sprintf("/apps"), + orgId, + false, + ) + } + } else { if len(item.Tags) > 0 { log.Printf("[INFO] Successfully built image %s", item.Tags[0]) diff --git a/docker-compose.yml b/docker-compose.yml index 8a8ba025..8a1fa4bd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: - SHUFFLE_SWARM_NETWORK_NAME=shuffle-executions restart: unless-stopped opensearch: - image: opensearchproject/opensearch:1.1.0 + image: opensearchproject/opensearch:1.2.0 hostname: shuffle-opensearch container_name: shuffle-opensearch environment: diff --git a/frontend/package.json b/frontend/package.json index 2933a4bb..acc20550 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -5,6 +5,8 @@ "private": true, "dependencies": { "@babel/core": "^7.15.8", + "@emotion/react": "^11.7.0", + "@emotion/styled": "^11.6.0", "@material-ui/core": "^4.5.2", "@material-ui/data-grid": "^4.0.0-alpha.22", "@material-ui/icons": "^4.5.1", @@ -12,6 +14,8 @@ "@material-ui/styles": "^4.5.2", "@material-ui/utils": "^4.11.2", "@metamask/detect-provider": "^1.2.0", + "@mui/icons-material": "^5.2.1", + "@mui/material": "^5.2.3", "@uiw/react-codemirror": "^3.2.1", "@use-it/interval": "^1.0.0", "babel-eslint": "^10.1.0", diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index b66ed310..f4c36f28 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -164,7 +164,10 @@ const ConfigureWorkflow = (props) => { newaction.must_authenticate = true; newaction.action_ids.push(action.id); } - } + } else if (action.authentication_id !== "" && app.authentication.required === true) { + console.log("Should verify authentication ID ", action.authentication_id) + + } newaction.app = app; } diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index b0cc7e74..fd4dd5ab 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -80,7 +80,10 @@ import { LockOpen as LockOpenIcon, ExpandMore as ExpandMoreIcon, VpnKey as VpnKeyIcon, -} from "@material-ui/icons"; + AutoFixHigh as AutoFixHighIcon, +} from '@mui/icons-material'; +//} from "@material-ui/icons"; + import Autocomplete from "@material-ui/lab/Autocomplete"; import CodeMirror from "@uiw/react-codemirror"; @@ -2169,7 +2172,7 @@ const ParsedAction = (props) => { marginTop: "auto", marginBottom: "auto", height: 30, - paddingLeft: 25, + marginLeft: 15, paddingRight: 0, }} onClick={() => { @@ -2189,7 +2192,7 @@ const ParsedAction = (props) => { marginTop: "auto", marginBottom: "auto", height: 30, - paddingLeft: 25, + marginLeft: 15, paddingRight: 0, }} onClick={() => {}} @@ -2209,6 +2212,40 @@ const ParsedAction = (props) => { + { + //setAuthenticationModalOpen(true); + console.log("Should enable/disable magic!") + console.log("Action: ", selectedAction) + if (selectedAction.run_magic_output === undefined) { + selectedAction.run_magic_output = true + } else { + if (selectedAction.run_magic_output === true) { + selectedAction.run_magic_output = false + } else { + selectedAction.run_magic_output = true + } + } + + setSelectedAction(selectedAction) + setUpdate(Math.random()); + }} + > + + + +
diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 76faca3e..a229cbe6 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1754,13 +1754,17 @@ const AngularWorkflow = (props) => { var idnumber = -1; if (curElement.id.startsWith("rightside_field_")) { console.log("FOUND FIELD WITH NUMBER: ", curElement.id); + + + // Find exact position to put the text + const idsplit = curElement.id.split("_"); console.log(idsplit); if (idsplit.length === 3 && !isNaN(idsplit[2])) { console.log("ADDING TO PARAM ", idsplit[2]); console.log("PARAM: ", selectedAction); - selectedAction.parameters[idsplit[2]].value = newValue; + selectedAction.parameters[idsplit[2]].value += newValue; paramname = selectedAction.parameters[idsplit[2]].name; idnumber = idsplit[2]; } @@ -1849,7 +1853,7 @@ const AngularWorkflow = (props) => { const onNodeDrag = (event, selectedAction) => { const nodedata = event.target.data(); if (nodedata.finished === false) { - return; + console.log("NOT FINISHED - ADD EXAMPLE BRANCHES TO CLOSEST!!") } if (nodedata.app_name !== undefined) { @@ -1859,6 +1863,15 @@ const AngularWorkflow = (props) => { if (currentNode.data.attachedTo === nodedata.id) { cy.getElementById(currentNode.data.id).remove(); } + + // Calculate location + //currentNode.position.x > + //if (nodedata.position.x > 0 && nodedata.position.y > 0) { + // console.log("Positive both") + //} + + //console.log(currentNode.position) + //console.log(nodedata.position) } } else { //console.log("No appid? ", nodedata) @@ -2389,6 +2402,9 @@ const AngularWorkflow = (props) => { return ""; } + console.log("NOT REPLACING ON PURPOSE!!") + return "" + // Basically just a stupid if-else :) const synonyms = { id: [ @@ -2491,6 +2507,7 @@ const AngularWorkflow = (props) => { // Takes an action as input, then runs through and updates the relevant fields // based on previous actions' + // Uses lots of synonyms const RunAutocompleter = (dstdata) => { // **PS: The right action should already be set here** // 1. Check execution argument @@ -2617,10 +2634,13 @@ const AngularWorkflow = (props) => { (data) => data.id === edge.source ); if (targetnode === -1) { - alert.error("Can't make arrow to starting node"); - event.target.remove(); - found = true; - break; + if (targetnode.type !== "TRIGGER") { + alert.error("Can't make arrow to starting node"); + event.target.remove(); + break; + } + + found = true; } } else if (edge.source === workflow.branches[key].source_id) { // FIXME: Verify multi-target for triggers @@ -2953,16 +2973,16 @@ const AngularWorkflow = (props) => { console.log("DELETE"); break; case 38: - console.log("UP"); + //console.log("UP"); break; case 37: - console.log("LEFT"); + //console.log("LEFT"); break; case 40: - console.log("DOWN"); + //console.log("DOWN"); break; case 39: - console.log("RIGHT"); + //console.log("RIGHT"); break; case 90: if (event.ctrlKey) { @@ -3169,7 +3189,12 @@ const AngularWorkflow = (props) => { } } - setEnvironments(responseJson); + // FIXME: Don't allow multiple in cloud yet. Cloud -> Onprem isn't stable. + if (isCloud) { + setEnvironments({ name: "Cloud", type: "cloud" }); + } else { + setEnvironments(responseJson); + } }) .catch((error) => { alert.error(error.toString()); @@ -3307,7 +3332,7 @@ const AngularWorkflow = (props) => { document.title = "Workflow - " + workflow.name; registerKeys(); } - }); + }) const animationDuration = 150; const onNodeHoverOut = (event) => { @@ -3480,11 +3505,9 @@ const AngularWorkflow = (props) => { const onNodeHover = (event) => { const nodedata = event.target.data(); - /* if (nodedata.finished === false) { return; } - */ //var parentNode = cy.$("#" + event.target.data("id")); //if (parentNode.data("isButton") || parentNode.data("buttonId")) return; @@ -4794,6 +4817,7 @@ const AngularWorkflow = (props) => { parameters: parameters, isStartNode: false, large_image: app.large_image, + run_magic_output: true, authentication: [], execution_variable: undefined, example: example, @@ -6701,6 +6725,9 @@ const AngularWorkflow = (props) => {
{outlookButton} {gmailButton} + + If you have trouble using the triggers, please contact us to get access + )} @@ -10075,7 +10102,10 @@ const AngularWorkflow = (props) => { } console.log("COPY: ", copy); - var newitem = JSON.parse(base); + + //var newitem = JSON.parse(base); + var newitem = validateJson(base).result + to_be_copied = "$" + base_node_name.toLowerCase().replaceAll(" ", "_"); for (var key in copy.namespace) { if (copy.namespace[key].includes("Results for")) { @@ -10292,7 +10322,7 @@ const AngularWorkflow = (props) => { {workflowExecutions.length > 0 ? (
{workflowExecutions.map((data, index) => { - executionDelay += 75 + executionDelay += 50 const statusColor = data.status === "FINISHED" @@ -10441,7 +10471,7 @@ const AngularWorkflow = (props) => { )}
) : ( -
+
{ onClick={() => {}} > - -

{}} - > - See other Executions -

+ +

{}} + > + See other Executions +

{ marginBottom: 10, }} /> -
+

Execution info

{
{executionData.status !== undefined && executionData.status.length > 0 ? ( -
+
Status    @@ -10544,7 +10574,7 @@ const AngularWorkflow = (props) => { executionData.execution_source !== null && executionData.execution_source.length > 0 && executionData.execution_source !== "default" ? ( -
+
Source    @@ -10581,7 +10611,7 @@ const AngularWorkflow = (props) => {
) : null} {executionData.started_at !== undefined ? ( -
+
Started    @@ -10593,7 +10623,7 @@ const AngularWorkflow = (props) => { {executionData.completed_at !== undefined && executionData.completed_at !== null && executionData.completed_at > 0 ? ( -
+
{ console.log(executionData) }}> @@ -10809,18 +10839,27 @@ const AngularWorkflow = (props) => {
{ var currentnode = cy.getElementById(data.action.id); if (currentnode.length !== 0) { currentnode.addClass("shuffle-hover-highlight"); } + + // Add a hover highlight + + //var copyText = document.getElementById( + // "copy_element_shuffle" + //) }} onMouseOut={() => { var currentnode = cy.getElementById(data.action.id); diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index 2d50122c..c11970d8 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -4676,12 +4676,15 @@ const AppCreator = (props) => { for (var key in invalid) { if (e.target.value.includes(invalid[key])) { alert.error("Can't use " + invalid[key] + " in name"); + setName(e.target.value.replaceAll(".", "").replaceAll("#", "").replaceAll(":", "").replaceAll(",", "")) + return; } } if (e.target.value.length > 29) { alert.error("Choose a shorter name (max 29)."); + setName(e.target.value.slice(0,28)) return; } diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index f717f9c5..de3740bb 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -398,12 +398,31 @@ export const validateJson = (showResult) => { var result = showResult; try { - const result = jsonvalid ? JSON.parse(showResult) : showResult; + result = jsonvalid ? JSON.parse(showResult) : showResult; } catch (e) { //console.log("Failed parsing JSON even though its valid: ", e) jsonvalid = false; } + if (jsonvalid === false) { + + if (typeof showResult === 'string') { + showResult = showResult.trim() + } + + try { + var newstr = showResult.replaceAll("'", '"') + + console.log("Try replacements and trimming with new value: ", newstr) + result = JSON.parse(newstr) + jsonvalid = true + } catch (e) { + + console.log("Failed parsing JSON even though its valid (2): ", e) + jsonvalid = false + } + } + //console.log("VALID: ", jsonvalid, result) return { valid: jsonvalid, @@ -458,6 +477,8 @@ const Workflows = (props) => { const [submitLoading, setSubmitLoading] = React.useState(false); const [actionImageList, setActionImageList] = React.useState([]); + const [firstLoad, setFirstLoad] = React.useState(true); + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"; @@ -839,6 +860,11 @@ const Workflows = (props) => { setFilteredWorkflows(responseJson); setWorkflowDone(true); + + // Ensures the zooming happens only once per load + setTimeout(() => { + setFirstLoad(false) + }, 100) } else { if (isLoggedIn) { alert.error("An error occurred while loading workflows"); @@ -1764,6 +1790,7 @@ const Workflows = (props) => { if (file.type !== "application/json") { if (file.type !== undefined) { alert.error("File has to contain valid json"); + setImportLoading(false); } continue; @@ -2581,7 +2608,11 @@ const Workflows = (props) => { data.large_image = theme.palette.defaultImage; } - appDelay += 75 + if (firstLoad) { + appDelay += 75 + } else { + appDelay = 0 + } return ( @@ -2650,7 +2681,11 @@ const Workflows = (props) => { {filteredWorkflows.map((data, index) => { - workflowDelay += 75 + if (firstLoad) { + workflowDelay += 75 + } else { + workflowDelay = 0 + } return (