From 5525131e7c8e09a39eb5e3dda7511bd84cd64264 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 21 Jan 2022 15:47:22 +0100 Subject: [PATCH 01/53] Fixed openapi replacement bug in app creator --- frontend/src/components/ParsedAction.jsx | 2 +- frontend/src/views/AngularWorkflow.jsx | 4 +-- frontend/src/views/AppCreator.jsx | 33 ++++++++++++++---------- frontend/src/views/SettingsPage.jsx | 16 +++++++----- 4 files changed, 33 insertions(+), 22 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 93f582c3..978712f3 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -183,7 +183,7 @@ const ParsedAction = (props) => { //} // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find(param => param.name === "body") - console.log("LOADED! Change hideBody based on input? Action: ", selectedAction, paramcheck) + //console.log("LOADED! Change hideBody based on input? Action: ", selectedAction, paramcheck) if (paramcheck !== undefined && paramcheck !== null) { if (paramcheck.id === "TOGGLED"){ setHideBody(false) diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 8a3c5924..c587f5d1 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -673,7 +673,7 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { - console.log("RESPONSE: ", responseJson) + //console.log("RESPONSE: ", responseJson) handleUpdateResults(responseJson, executionRequest); }) .catch((error) => { @@ -7066,7 +7066,7 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { - console.log("RESPONSE: "); + //console.log("RESPONSE: "); setTriggerAuthentication(responseJson); clearInterval(id); newwin.close(); diff --git a/frontend/src/views/AppCreator.jsx b/frontend/src/views/AppCreator.jsx index d199a109..9e6563c2 100644 --- a/frontend/src/views/AppCreator.jsx +++ b/frontend/src/views/AppCreator.jsx @@ -618,6 +618,7 @@ const AppCreator = (defaultprops) => { continue; } + //console.log("METHOD: ", methodvalue) var tmpname = methodvalue.summary; if ( methodvalue.operationId !== undefined && @@ -628,7 +629,13 @@ const AppCreator = (defaultprops) => { tmpname = methodvalue.operationId; } - tmpname = tmpname.replaceAll(".", " "); + if (tmpname !== undefined && tmpname !== null) { + tmpname = tmpname.replaceAll(".", " "); + } + + if ((tmpname === undefined || tmpname === null) && methodvalue.description !== undefined && methodvalue.description !== null && methodvalue.description.length > 0) { + tmpname = methodvalue.description.replaceAll(".", " ").replaceAll("_", " ") + } var newaction = { name: tmpname, @@ -739,9 +746,9 @@ const AppCreator = (defaultprops) => { var newbody = {}; // Can handle default, required, description and type for (var propkey in retRef.properties) { - const parsedkey = propkey - .replaceAll(" ", "_") - .toLowerCase(); + console.log("replace: ", propkey) + + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); newbody[parsedkey] = "${" + parsedkey + "}"; } @@ -885,9 +892,8 @@ const AppCreator = (defaultprops) => { ) { var newbody = {}; for (var propkey in parameter.properties) { - const parsedkey = propkey - .replaceAll(" ", "_") - .toLowerCase(); + console.log("propkey2: ", propkey) + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); if (parameter.properties[propkey].type === undefined) { console.log( "Skipping (4): ", @@ -1022,9 +1028,8 @@ const AppCreator = (defaultprops) => { ) { var newbody = {}; for (var propkey in parameter.properties) { - const parsedkey = propkey - .replaceAll(" ", "_") - .toLowerCase(); + console.log("propkey3: ", propkey) + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); if ( parameter.properties[propkey].type === undefined ) { @@ -1112,9 +1117,8 @@ const AppCreator = (defaultprops) => { ) { var newbody = {}; for (var propkey in parameter.properties) { - const parsedkey = propkey - .replaceAll(" ", "_") - .toLowerCase(); + console.log("propkey4: ", propkey) + const parsedkey = propkey.replaceAll(" ", "_").toLowerCase(); if ( parameter.properties[propkey].type === undefined @@ -1197,6 +1201,7 @@ const AppCreator = (defaultprops) => { ) { var newbody = {}; for (var propkey in parameter.properties) { + console.log("propkey5: ", propkey) const parsedkey = propkey .replaceAll(" ", "_") .toLowerCase(); @@ -2068,6 +2073,7 @@ const AppCreator = (defaultprops) => { return; } + console.log("Paramname: ", parameterName) var newparamName = parameterName.replaceAll('"', ""); newparamName = newparamName.replaceAll("'", ""); @@ -2098,6 +2104,7 @@ const AppCreator = (defaultprops) => { scheme: "basic", }; } else if (authenticationOption === "Oauth2") { + console.log("oauth2: ", parameterName) var newparamName = parameterName.replaceAll('"', ""); newparamName = newparamName.replaceAll("'", ""); diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index 468b8bc0..11585331 100644 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -257,6 +257,7 @@ const Settings = (props) => { console.log("Status not 200 for WORKFLOW EXECUTION :O!"); } + return response.json(); }) .then((responseJson) => { @@ -402,11 +403,14 @@ const Settings = (props) => { userdata.eth_info.account.length > 0 && userdata.eth_info.parsed_balance !== undefined // Random names for type & autoComplete. Didn't research :^) - var imageData = file.length > 0 ? file : fileBase64; - imageData = - imageData === undefined || imageData.length === 0 - ? theme.palette.defaultImage - : imageData; + //var imageData = file.length > 0 ? file : fileBase64; + //imageData = imageData === undefined || imageData.length === 0 + // ? theme.palette.defaultImage + // : imageData; + + const imageData = userSettings.image === undefined || userSettings.image == null || userSettings.image.length === 0 ? theme.palette.defaultImage : userSettings.image + console.log("settings: ", userSettings) + console.log("Image: ", imageData) const imageInfo = ( { const client_id = "3d272b1b782b100b1e61" const username = userdata.id; - const scopes = "user:email"; + const scopes = "read:user"; const url = `https://github.com/login/oauth/authorize?access_type=offline&prompt=consent&client_id=${client_id}&redirect_uri=${redirectUri}&response_type=code&scope=${scopes}&state=username%3D${username}%26type%3Dgithub` From 67edef562496baefe42f0ccd3570f0252c5372f3 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 21 Jan 2022 16:58:48 +0100 Subject: [PATCH 02/53] Added small fixes to cloud UI --- frontend/src/components/ParsedAction.jsx | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 978712f3..a4b1f344 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -6,6 +6,7 @@ import { GetIconInfo } from "../views/Workflows.jsx"; import { sortByKey } from "../views/AngularWorkflow.jsx"; import { useTheme } from "@material-ui/core/styles"; import NestedMenuItem from "material-ui-nested-menu-item"; +import { useAlert } from "react-alert"; import theme from '../theme'; //import NestedMenuItem from "./NestedMenu.jsx"; @@ -168,6 +169,7 @@ const ParsedAction = (props) => { //const theme = useTheme(); const classes = useStyles(); + const alert = useAlert() const [expansionModalOpen, setExpansionModalOpen] = React.useState(false); const [hideBody, setHideBody] = React.useState(true); From d2c9cbec7d0b0c276c5daa925b84b26cad0fe2a5 Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 21 Jan 2022 16:59:33 +0100 Subject: [PATCH 03/53] Fixed link in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 01fd94b1..d4a13732 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Shuffle Automation

-![Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. +[Shuffle](https://shuffler.io) is an automation platform for and by the community, focusing on accessibility for anyone to automate. Security operations is complex, but it doesn't have to be. [_Key Features_](https://shuffler.io/docs/features) — [_Community & Support_](https://discord.gg/B2CBzUm) — From 9ce92d6e1fd8cce7df21fe8c222140e4a504d22c Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 21 Jan 2022 19:30:40 +0000 Subject: [PATCH 04/53] Added a bunch of CORS related fixes for new shell dev --- .env | 1 + backend/go-app/docker.go | 7 ++-- backend/go-app/go.mod | 22 ++++++------ backend/go-app/main.go | 57 +++++++++++++++++++++----------- backend/go-app/walkoff.go | 32 +++++++++--------- frontend/src/App.jsx | 13 ++++++-- frontend/src/views/Apps.jsx | 6 +++- frontend/src/views/LoginPage.jsx | 3 +- 8 files changed, 87 insertions(+), 54 deletions(-) diff --git a/.env b/.env index ac324e6a..d97418dd 100644 --- a/.env +++ b/.env @@ -2,6 +2,7 @@ ORG_ID=Shuffle ENVIRONMENT_NAME=Shuffle + # Remote github config for first load SHUFFLE_DOWNLOAD_WORKFLOW_LOCATION= SHUFFLE_DOWNLOAD_WORKFLOW_USERNAME= diff --git a/backend/go-app/docker.go b/backend/go-app/docker.go index 9b535c72..25acc65a 100644 --- a/backend/go-app/docker.go +++ b/backend/go-app/docker.go @@ -14,6 +14,7 @@ import ( "encoding/json" "errors" "fmt" + //"github.com/docker/docker" "github.com/docker/docker/api/types" //"github.com/docker/docker/api/types/container" @@ -417,7 +418,7 @@ func stopWebhook(image string, identifier string) error { // Starts a new webhook func handleStopHookDocker(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -502,7 +503,7 @@ var webhook = `{ // Starts a new webhook func handleDeleteHookDocker(resp http.ResponseWriter, request *http.Request) { ctx := context.Background() - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -619,7 +620,7 @@ func hookTest() { //https://stackoverflow.com/questions/23935141/how-to-copy-docker-images-from-one-host-to-another-without-using-a-repository func getDockerImage(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0eb5e3b3..02b59762 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -1,34 +1,34 @@ module main -go 1.15 +go 1.16 -replace github.com/shuffle/shuffle-shared => ../../../../git/shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch require ( cloud.google.com/go/datastore v1.6.0 - cloud.google.com/go/pubsub v1.17.0 + cloud.google.com/go/iam v0.1.1 // indirect + cloud.google.com/go/pubsub v1.17.1 cloud.google.com/go/storage v1.18.2 github.com/basgys/goxml2json v1.1.0 github.com/carlescere/scheduler v0.0.0-20170109141437-ee74d2f83d82 - github.com/docker/docker v20.10.9+incompatible + github.com/docker/docker v20.10.12+incompatible github.com/frikky/kin-openapi v0.41.0 - github.com/fsouza/go-dockerclient v1.7.4 + github.com/fsouza/go-dockerclient v1.7.7 github.com/ghodss/yaml v1.0.0 github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/gorilla/mux v1.8.0 - github.com/h2non/filetype v1.1.1 + github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.79 - github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect + github.com/shuffle/shuffle-shared v0.1.84 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect - golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 - google.golang.org/api v0.58.0 + golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce + google.golang.org/api v0.65.0 google.golang.org/appengine v1.6.7 - google.golang.org/grpc v1.41.0 + google.golang.org/grpc v1.43.0 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 9e9473c5..0b9b8c9a 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -1,12 +1,14 @@ package main import ( + uuid "github.com/satori/go.uuid" "github.com/shuffle/shuffle-shared" "bufio" "bytes" "context" "crypto/md5" + //"crypto/tls" //"crypto/x509" "encoding/hex" @@ -21,6 +23,7 @@ import ( "os" "os/exec" "path/filepath" + //"regexp" "strings" "time" @@ -49,12 +52,12 @@ import ( "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" + //githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" // Random xj "github.com/basgys/goxml2json" newscheduler "github.com/carlescere/scheduler" - "github.com/satori/go.uuid" "golang.org/x/crypto/bcrypt" "gopkg.in/yaml.v3" @@ -706,7 +709,7 @@ func createNewUser(username, password, role, apikey string, org shuffle.OrgMini) } func handleRegister(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -835,7 +838,7 @@ func handleCookie(request *http.Request) bool { } func handleInfo(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1118,7 +1121,7 @@ func increaseStatisticsField(ctx context.Context, fieldname, id string, amount i // FIXME - forward this to emails or whatever CRM system in use func handleContact(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1164,7 +1167,8 @@ func handleContact(resp http.ResponseWriter, request *http.Request) { } func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + log.Printf("In admin login request?") + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1193,7 +1197,7 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { } func handleLogin(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1418,8 +1422,12 @@ func fixUserOrg(ctx context.Context, user *shuffle.User) *shuffle.User { } // Used for testing only. Shouldn't impact production. -func handleCors(resp http.ResponseWriter, request *http.Request) bool { - allowedOrigins := "http://localhost:3000" +/* +func shuffle.HandleCors(resp http.ResponseWriter, request *http.Request) bool { + // Used for Codespace dev + allowedOrigins := "https://frikky-shuffle-5gvr4xx62w64-3000.githubpreview.dev" + //origin := request.Header["Origin"] + //log.Printf("Origin: %s", origin) //allowedOrigins := "http://localhost:3002" resp.Header().Set("Vary", "Origin") @@ -1436,6 +1444,7 @@ func handleCors(resp http.ResponseWriter, request *http.Request) bool { return false } +*/ func parseWorkflowParameters(resp http.ResponseWriter, request *http.Request) (map[string]interface{}, error) { body, err := ioutil.ReadAll(request.Body) @@ -1580,7 +1589,7 @@ func SearchNested(obj interface{}, key string) (interface{}, bool) { } func handleSetHook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1768,7 +1777,7 @@ func verifyHook(hook shuffle.Hook) (bool, string) { } func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1828,7 +1837,7 @@ func setSpecificSchedule(resp http.ResponseWriter, request *http.Request) { } func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1880,7 +1889,7 @@ func getSpecificWebhook(resp http.ResponseWriter, request *http.Request) { // Starts a new webhook func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1935,7 +1944,7 @@ func handleDeleteSchedule(resp http.ResponseWriter, request *http.Request) { // Starts a new webhook func handleNewSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2229,7 +2238,7 @@ func getSpecificSchedule(resp http.ResponseWriter, request *http.Request) { return } - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2295,7 +2304,7 @@ func loadYaml(fileLocation string) (ApiYaml, error) { // This should ALWAYS come from an OUTPUT func executeSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2788,7 +2797,7 @@ type Result struct { // r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS") func getOpenapi(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -3254,7 +3263,7 @@ func buildSwaggerApp(resp http.ResponseWriter, body []byte, user shuffle.User) { // Creates an app from the app builder func verifySwagger(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -4999,7 +5008,7 @@ func handleStopCloudSync(syncUrl string, org shuffle.Org) (*shuffle.Org, error) This is here to both enable and disable cloud sync features for an organization */ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -5687,10 +5696,15 @@ func initHandlers() { dbclient, err = datastore.NewClient(ctx, gceProject, option.WithGRPCDialOption(grpc.WithNoProxy())) if err != nil { if elasticConfig == "" { - log.Fatalf("[ERROR] Database client error during init: %s. Env: SHUFFLE_ELASTIC=false", err) + log.Printf("[ERROR] Database client error during init: %s. Env: SHUFFLE_ELASTIC=false", err) } else { - log.Printf("[DEBUG] Database client error during init: %s. Here for backwards compatibility: not critical.", err) + if !strings.Contains(fmt.Sprintf("%s", err), "find default credentials") { + log.Printf("[DEBUG] Database client error info during init: %s. Here for backwards compatibility: not critical.", err) + } + dbclient = &datastore.Client{} } + } else { + //log.Printf("Database client initiated: %s", dbclient) } for { @@ -5887,6 +5901,9 @@ func initHandlers() { r.HandleFunc("/api/v1/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") //r.HandleFunc("/api/v1/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications", shuffle.HandleGetNotifications).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications/clear", shuffle.HandleClearNotifications).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/users/notifications/{notificationId}/markasread", shuffle.HandleMarkAsRead).Methods("GET", "OPTIONS") http.Handle("/", r) } diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index c05c380d..27ca1615 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -21,6 +21,7 @@ import ( "github.com/docker/docker/api/types" dockerclient "github.com/docker/docker/client" + //gyaml "github.com/ghodss/yaml" "github.com/h2non/filetype" @@ -34,6 +35,7 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http" + //"github.com/gorilla/websocket" //"google.golang.org/appengine" //"google.golang.org/appengine/memcache" @@ -142,7 +144,7 @@ func createSchedule(ctx context.Context, scheduleId, workflowId, name, startNode } func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -243,7 +245,7 @@ func handleGetWorkflowqueueConfirm(resp http.ResponseWriter, request *http.Reque // FIXME: Authenticate this one? Can org ID be auth enough? // (especially since we have a default: shuffle) func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -335,7 +337,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -393,7 +395,7 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) { } func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -666,7 +668,7 @@ func handleExecutionStatistics(execution shuffle.WorkflowExecution) { } func deleteWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1034,7 +1036,7 @@ func cloudExecuteAction(execution shuffle.WorkflowExecution) error { } func executeWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1131,7 +1133,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } func stopSchedule(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1281,7 +1283,7 @@ func stopSchedule(resp http.ResponseWriter, request *http.Request) { } func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1390,7 +1392,7 @@ func deleteSchedule(ctx context.Context, id string) error { } func scheduleWorkflow(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1641,7 +1643,7 @@ func setExampleresult(ctx context.Context, result shuffle.AppExecutionExample) e } func getWorkflowApps(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1714,7 +1716,7 @@ func handleGetfile(resp http.ResponseWriter, request *http.Request) ([]byte, err // Basically a search for apps that aren't activated yet func getSpecificApps(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1789,7 +1791,7 @@ func getSpecificApps(resp http.ResponseWriter, request *http.Request) { } func validateAppInput(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1910,7 +1912,7 @@ func loadGithubWorkflows(url, username, password, userId, branch, orgId string) } func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -1972,7 +1974,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) { } func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } @@ -2290,7 +2292,7 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra } func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { - cors := handleCors(resp, request) + cors := shuffle.HandleCors(resp, request) if cors { return } diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index d27cdc39..d11886e1 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -49,6 +49,12 @@ if (window.location.port === "3000") { //globalUrl = "http://localhost:5002" } +if (globalUrl.includes("githubpreview.dev")) { + //globalUrl = globalUrl.replace("3000", "5001") + globalUrl = "https://frikky-shuffle-5gvr4xx62w64-5001.githubpreview.dev" +} +console.log("global: ", globalUrl) + const App = (message, props) => { const [userdata, setUserData] = useState({}); @@ -70,7 +76,7 @@ const App = (message, props) => { checkLogin(); setDataset(true); } - }); + }, []); if ( isLoaded && @@ -84,11 +90,12 @@ const App = (message, props) => { } const getUserNotifications = () => { - fetch(`${globalUrl}/api/v1/notifications`, { + fetch(`${globalUrl}/api/v1/users/notifications`, { credentials: "include", headers: { "Content-Type": "application/json", }, + cors: "cors", }) .then((response) => response.json()) .then((responseJson) => { @@ -109,7 +116,7 @@ const App = (message, props) => { const checkLogin = () => { var baseurl = globalUrl; - fetch(baseurl + "/api/v1/users/getinfo", { + fetch(`${globalUrl}/api/v1/getinfo`, { credentials: "include", headers: { "Content-Type": "application/json", diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 75238e70..fcd342e3 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -515,7 +515,11 @@ const Apps = (props) => { valid = "false"; } - if (data.actions === null || data.actions.length === 0) { + if (data.actions === undefined || data.actions === null) { + data.actions = [] + } + + if (data === undefined || data.actions === undefined || data.actions === null || data.actions.length === 0) { valid = "false"; } diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index 8928f166..4231a326 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -44,6 +44,7 @@ const LoginDialog = (props) => { register, checkLogin, } = props; + const [username, setUsername] = useState(""); const [password, setPassword] = useState(""); const [firstRequest, setFirstRequest] = useState(true); @@ -143,7 +144,7 @@ const LoginDialog = (props) => { var baseurl = globalUrl; if (register) { - var url = baseurl + "/api/v1/users/login"; + var url = baseurl + "/api/v1/login"; fetch(url, { mode: "cors", method: "POST", From 2230cef978f026f76f4170319fc4cbbb878d24b9 Mon Sep 17 00:00:00 2001 From: Frikky Date: Fri, 21 Jan 2022 21:20:25 +0000 Subject: [PATCH 05/53] Fixed autocompletion issue for exec and added errors for bad variables --- frontend/src/components/ParsedAction.jsx | 189 ++++++++++++++--------- frontend/src/views/AngularWorkflow.jsx | 3 +- frontend/src/views/Workflows.jsx | 24 ++- functions/onprem/worker/worker.go | 4 +- 4 files changed, 141 insertions(+), 79 deletions(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index a4b1f344..95562a7f 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1,8 +1,8 @@ import React, { useState, useEffect, useLayoutEffect } from "react"; import { makeStyles, createStyles } from "@material-ui/core/styles"; +import { validateJson, GetIconInfo } from "../views/Workflows.jsx"; import { GetParsedPaths } from "../views/Apps.jsx"; -import { GetIconInfo } from "../views/Workflows.jsx"; import { sortByKey } from "../views/AngularWorkflow.jsx"; import { useTheme } from "@material-ui/core/styles"; import NestedMenuItem from "material-ui-nested-menu-item"; @@ -306,6 +306,8 @@ const ParsedAction = (props) => { }); }; + + const defineStartnode = () => { if (cy === undefined) { return; @@ -393,14 +395,46 @@ const ParsedAction = (props) => { if (actionlist.length === 0) { // FIXME: Have previous execution values in here - actionlist.push({ - type: "Execution Argument", - name: "Execution Argument", - value: "$exec", - highlight: "exec", - autocomplete: "exec", - example: "", - }); + if (workflowExecutions.length > 0) { + for (var key in workflowExecutions) { + if ( + workflowExecutions[key].execution_argument === undefined || + workflowExecutions[key].execution_argument === null || + workflowExecutions[key].execution_argument.length === 0 + ) { + continue; + } + + console.log("EXEC: ", workflowExecutions[key].execution_argument) + + const valid = validateJson(workflowExecutions[key].execution_argument) + console.log("VALID: ", valid) + if (valid.valid) { + actionlist.push({ + type: "Execution Argument", + name: "Execution Argument", + value: "$exec", + highlight: "exec", + autocomplete: "exec", + example: valid.result, + }) + break + } + } + + } + + if (actionlist.length === 0) { + actionlist.push({ + type: "Execution Argument", + name: "Execution Argument", + value: "$exec", + highlight: "exec", + autocomplete: "exec", + example: "", + }) + } + actionlist.push({ type: "Shuffle DB", name: "Shuffle DB", @@ -477,51 +511,11 @@ const ParsedAction = (props) => { continue; } - foundResult.result = foundResult.result.trim(); - foundResult.result = foundResult.result - .split(" None") - .join(' "None"'); - foundResult.result = foundResult.result - .split(" False") - .join(" false"); - foundResult.result = foundResult.result - .split(" True") - .join(" true"); - - var jsonvalid = true; - try { - const tmp = String(JSON.parse(foundResult.result)); - if ( - !foundResult.result.includes("{") && - !foundResult.result.includes("[") - ) { - jsonvalid = false; - } - } catch (e) { - try { - foundResult.result = foundResult.result - .split("'") - .join('"'); - const tmp = String(JSON.parse(foundResult.result)); - if ( - !foundResult.result.includes("{") && - !foundResult.result.includes("[") - ) { - jsonvalid = false; - } - } catch (e) { - jsonvalid = false; - } - } - - // Finds the FIRST json only - if (jsonvalid) { + const valid = validateJson(foundResult) + if (valid.valid) { exampledata = JSON.parse(foundResult.result); break; } - //else { - // console.log("Invalid JSON: ", foundResult.result) - //} } } @@ -546,6 +540,59 @@ const ParsedAction = (props) => { } }); + + const calculateHelpertext = (input_data) => { + var helperText = "" + var looperText = "" + const found = input_data.match(/[$]{1}([a-zA-Z0-9_-]+\.?){1}([a-zA-Z0-9#_-]+\.?){0,}/g) + + if (found !== null) { + try { + // When the found array is empty. + for (var i = 0; i < found.length; i++) { + const variableSplit = found[i].split(".#") + if ((variableSplit.length-1) > 1) { + //console.log("Larger than 1: ", variableSplit) + if (looperText.length === 0) { + looperText += "PS: Double looping (.#) may cause problems." + } + } + + var foundSlice = false + for (var j = 0; j < actionlist.length; j++) { + //console.log("ACTION: ", found[i], actionlist[j]) + //console.log("ACTION :", found[i].split(".")[0].slice(1,).toLowerCase(), actionlist[j].autocomplete.toLowerCase()) + if(found[i].split(".")[0].slice(1,).toLowerCase() == actionlist[j].autocomplete.toLowerCase()){ + //console.log("Found: ", found[i]) + // Validate path? + + foundSlice = true + } + } + + if (!foundSlice) { + if (!helperText.includes("Invalid variables")) { + helperText+= "Invalid variables: " + } + helperText+= found[i] + ", " + } + } + } catch (e) { + console.log("Parsing error: ", e) + } + } + + if (looperText.length > 0) { + if (helperText.length > 0) { + helperText += ". " + } + + helperText += looperText + } + + return helperText + } + const changeActionParameter = (event, count, data) => { //console.log("Action change: ", selectedAction, data) if (data.name.startsWith("${") && data.name.endsWith("}")) { @@ -1284,21 +1331,27 @@ const ParsedAction = (props) => { const clickedFieldId = "rightside_field_" + count; - const shufflecode = + const shufflecode = fieldCount !== count ? null : + ( + + ) // 0) { + baseHelperText = calculateHelpertext(data.value) + } var datafield = ( { //changeActionParameterCodemirror(event, count, data) changeActionParameter(event, count, data); }} - helperText={ + helperText={baseHelperText.length > 0 ? baseHelperText : selectedApp.generated && selectedApp.activated && data.name === "body" ? ( @@ -1425,15 +1478,7 @@ const ParsedAction = (props) => { ) : null } onBlur={(event) => { - // Super basic check - //if (event.target.value.startsWith("{")) { - // console.log("VALIDATING JSON") - // try { - // JSON.parse(event.target.value) - // } catch (e) { - // alert.error("Failed to parse json: ", e) - // } - //} + baseHelperText = calculateHelpertext(event.target.value) }} /> ); diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index c587f5d1..a98c23bd 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -340,8 +340,7 @@ const AngularWorkflow = (defaultprops) => { const [_, setUpdate] = useState(""); // Used for rendring, don't remove const [workflowExecutions, setWorkflowExecutions] = React.useState([]); - const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = - React.useState(0); + const [defaultEnvironmentIndex, setDefaultEnvironmentIndex] = React.useState(0); // This should all be set once, not on every iteration // Use states and don't update lol diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index b1e7f13c..09e52081 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -381,9 +381,27 @@ const chipStyle = { }; export const validateJson = (showResult) => { - //showResult = showResult.split(" None").join(" \"None\"") - showResult = showResult.split(" False").join(" false"); - showResult = showResult.split(" True").join(" true"); + console.log("INPUT: ", showResult, typeof showResult) + if (typeof showResult === 'string') { + //showResult = showResult.split(" None").join(" \"None\"") + showResult = showResult.split(" False").join(" false"); + showResult = showResult.split(" True").join(" true"); + //return { + // valid: false, + // result: showResult, + //}; + } + + //if (typeof showResult === undefined) { + + //} + + if (typeof showResult === "object" || typeof showResult === "array") { + return { + valid: true, + result: showResult, + }; + } var jsonvalid = true; try { diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 7460161f..775fc8fd 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1517,7 +1517,7 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { onpremApps := []string{} toExecuteOnprem := []string{} for _, action := range workflowExecution.Workflow.Actions { - if action.Environment != environment { + if strings.ToLower(action.Environment) != strings.ToLower(environment) { continue } @@ -1599,7 +1599,7 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec err := executionInit(workflowExecution) if err != nil { - log.Printf("[INFO] Workflow setup failed: %s", workflowExecution.ExecutionId, err) + log.Printf("[INFO] Workflow setup failed for %s: %s", workflowExecution.ExecutionId, err) log.Printf("[DEBUG] Shutting down (18)") shutdown(workflowExecution, "", "", true) } From 7b673ccdb442e9cf052a1e5ddad239c1b7e4cb7b Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 22 Jan 2022 01:46:16 +0000 Subject: [PATCH 06/53] Added small fixes to orborus and sdk --- backend/app_sdk/app_base.py | 8 +++++++- docker-compose.yml | 2 +- functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/orborus.go | 10 +++++++++- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index f6a5a83e..cda19d93 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -2612,7 +2612,13 @@ class AppBase: for i in range(0, curminlength): tmpitem = json.loads(json.dumps(parameter["value"])) for key, value in replacements.items(): - replacement = json.dumps(json.loads(value)[i]) + replacement = value + try: + replacement = json.dumps(json.loads(value)[i]) + except IndexError as e: + self.logger.info(f"[ERROR] Failed handling value parsing with index: {e}") + pass + if replacement.startswith("\"") and replacement.endswith("\""): replacement = replacement[1:len(replacement)-1] #except json.decoder.JSONDecodeError as e: diff --git a/docker-compose.yml b/docker-compose.yml index d5c27571..19405365 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -64,7 +64,7 @@ services: - SHUFFLE_SWARM_CONFIG=runn restart: unless-stopped opensearch: - image: opensearchproject/opensearch:1.2.3 + image: opensearchproject/opensearch:1.2.4 hostname: shuffle-opensearch container_name: shuffle-opensearch environment: diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index f0186a6e..fa93483b 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.50 +VERSION=0.9.51 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 1c48f902..9fdc9b56 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -37,9 +37,11 @@ import ( "github.com/docker/docker/api/types/mount" "github.com/docker/docker/api/types/network" "github.com/docker/docker/api/types/swarm" + //"github.com/docker/docker/api/types/filters" dockerclient "github.com/docker/docker/client" - "github.com/satori/go.uuid" + uuid "github.com/satori/go.uuid" + //network "github.com/docker/docker/api/types/network" //natting "github.com/docker/go-connections/nat" "github.com/mackerelio/go-osstat/cpu" @@ -538,8 +540,14 @@ func deployWorker(image string, identifier string, env []string, executionReques nil, identifier+"-2", ) + if err != nil { + log.Printf("[ERROR] Failed to CREATE container (2): %s", err) + } err = dockercli.ContainerStart(context.Background(), cont.ID, containerStartOptions) + if err != nil { + log.Printf("[ERROR] Failed to start container (2): %s", err) + } } else { log.Printf("[ERROR] Failed initial container start. Quitting as this is NOT a simple network issue. Err: %s", err) } From 2f324e6862ec634cc12e90e41c4af4bce4d7e6d9 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 26 Jan 2022 17:21:33 +0100 Subject: [PATCH 07/53] More App SDK fixes around indexing in recursions, and another workflow config handler fix --- backend/app_sdk/app_base.py | 26 +++++++++---- backend/go-app/go.mod | 4 +- docker-compose.yml | 4 +- frontend/src/components/ConfigureWorkflow.jsx | 4 +- frontend/src/components/ParsedAction.jsx | 39 ++++++++++--------- frontend/src/components/ShuffleCodeEditor.jsx | 1 + frontend/src/theme.js | 1 + frontend/src/views/AngularWorkflow.jsx | 14 +++++-- frontend/src/views/LoginPage.jsx | 5 ++- frontend/src/views/SettingsPage.jsx | 9 ++--- frontend/src/views/Workflows.jsx | 4 +- functions/onprem/orborus/orborus.go | 2 +- 12 files changed, 68 insertions(+), 45 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index cda19d93..e526a2ab 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1010,8 +1010,12 @@ class AppBase: } # Simple validation of parameters in general + replace_params = False try: tmp_parameters = action["parameters"] + for param in tmp_parameters: + if param["value"] == "SHUFFLE_AUTO_REMOVED": + replace_params = True except KeyError: action["parameters"] = [] except TypeError: @@ -1509,17 +1513,23 @@ class AppBase: newvalue, is_loop = (tmpitem, parsersplit[outercnt+1:]) else: print("[INFO] In ELSE - handling %s and %s" % (firstitem, seconditem)) - if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": - firstitem = len(basejson)-1 - elif firstitem.lower() == "min" or firstitem.lower() == "first": - firstitem = 0 + if isinstance(firstitem, str): + if firstitem.lower() == "max" or firstitem.lower() == "last" or firstitem.lower() == "end": + firstitem = len(basejson)-1 + elif firstitem.lower() == "min" or firstitem.lower() == "first": + firstitem = 0 + else: + firstitem = int(firstitem) else: firstitem = int(firstitem) - if seconditem.lower() == "max" or seconditem.lower() == "last" or firstitem.lower() == "end": - seconditem = len(basejson)-1 - elif seconditem.lower() == "min" or seconditem.lower() == "first": - seconditem = 0 + if isinstance(seconditem, str): + if seconditem.lower() == "max" or seconditem.lower() == "last" or firstitem.lower() == "end": + seconditem = len(basejson)-1 + elif seconditem.lower() == "min" or seconditem.lower() == "first": + seconditem = 0 + else: + seconditem = int(seconditem) else: seconditem = int(seconditem) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 02b59762..0baf7fee 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.16 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch @@ -23,7 +23,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.84 + github.com/shuffle/shuffle-shared v0.1.86 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/docker-compose.yml b/docker-compose.yml index 19405365..0fe72b68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/components/ConfigureWorkflow.jsx b/frontend/src/components/ConfigureWorkflow.jsx index f4c36f28..fbd7eb79 100644 --- a/frontend/src/components/ConfigureWorkflow.jsx +++ b/frontend/src/components/ConfigureWorkflow.jsx @@ -556,7 +556,7 @@ const ConfigureWorkflow = (props) => { {action.must_activate ? ( diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 95562a7f..29fa0d65 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -180,23 +180,21 @@ const ParsedAction = (props) => { const [hiddenDescription, setHiddenDescription] = React.useState(true); useEffect(() => { - - //if (data.startsWith("${") && data.endsWith("}")) { - //} - // PARAM FIX - Gonna use the ID field, even though it's a hack - const paramcheck = selectedAction.parameters.find(param => param.name === "body") - //console.log("LOADED! Change hideBody based on input? Action: ", selectedAction, paramcheck) - if (paramcheck !== undefined && paramcheck !== null) { - if (paramcheck.id === "TOGGLED"){ - setHideBody(false) - setActivateHidingBodyButton(false) - console.log("TOGGLED BODY!") - } else { - setHideBody(true) - - if (paramcheck.id === "UNTOGGLED") { + if (selectedAction.parameters !== null && selectedAction.parameters !== undefined) { + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + //console.log("LOADED! Change hideBody based on input? Action: ", selectedAction, paramcheck) + if (paramcheck !== undefined && paramcheck !== null) { + if (paramcheck.id === "TOGGLED"){ + setHideBody(false) setActivateHidingBodyButton(false) - console.log("UNTOGGLED!") + console.log("TOGGLED BODY!") + } else { + setHideBody(true) + + if (paramcheck.id === "UNTOGGLED") { + setActivateHidingBodyButton(false) + console.log("UNTOGGLED!") + } } } } @@ -513,9 +511,11 @@ const ParsedAction = (props) => { const valid = validateJson(foundResult) if (valid.valid) { - exampledata = JSON.parse(foundResult.result); + exampledata = valid.result; break; - } + } else { + exampledata = foundResult; + } } } @@ -2382,9 +2382,12 @@ const ParsedAction = (props) => { { - handleScopeChange(e); - }} - fullWidth - input={} - renderValue={(selected) => selected.join(", ")} - MenuProps={MenuProps} - > - {allscopes.map((data, index) => { - return ( - - -1} /> - - - ); - })} - - +
+ + Scopes + } + renderValue={(selected) => selected.join(", ")} + MenuProps={MenuProps} + > + {allscopes.map((data, index) => { + return ( + + -1} /> + + + ); + })} + + + + + { + setOfflineAccess(!offlineAccess) + }}/> + + +
)} )} diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index 29fa0d65..badf7a6c 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -2558,6 +2558,7 @@ const ParsedAction = (props) => { SelectDisplayProps={{ style: { marginLeft: 10, + maxWidth: 250, }, }} fullWidth @@ -2631,6 +2632,7 @@ const ParsedAction = (props) => { > { setAuthenticationModalOpen(true); From a811036fb2d311305489fc1920567fb259da9920 Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 31 Jan 2022 12:22:37 +0100 Subject: [PATCH 15/53] Added fixes for public workflows with app visuals --- docker-compose.yml | 4 +- frontend/src/components/Oauth2Auth.jsx | 4 + frontend/src/views/AngularWorkflow.jsx | 326 +++++++++++++++++++------ frontend/src/views/Docs.jsx | 2 +- frontend/src/views/Workflows.jsx | 89 +++++-- 5 files changed, 337 insertions(+), 88 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f8642281..064d4a42 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,10 +47,10 @@ services: volumes: - /var/run/docker.sock:/var/run/docker.sock environment: - - SHUFFLE_WORKER_VERSION=0.9.52 + - SHUFFLE_WORKER_VERSION=nightly - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - - BASE_URL=https://frikky-shuffle-5gvr4xx62w64-5001.githubpreview.dev + - BASE_URL=https://${OUTER_HOSTNAME}:${BACKEND_PORT} - DOCKER_API_VERSION=1.40 - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME} - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY} diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index e63ee93c..ef7384f7 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -114,6 +114,10 @@ const AuthenticationOauth2 = (props) => { setButtonClicked(true); console.log("SCOPES: ", scopes); + client_id = client_id.trim() + client_secret = client_secret.trim() + oauth_url = oauth_url.trim() + var resources = ""; if (scopes !== undefined && (scopes !== null) & (scopes.length > 0)) { if (offlineAccess === true && !scopes.includes("offline_access")) { diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 94a5def4..a3d11d88 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -14,6 +14,7 @@ import theme from '../theme'; import { Zoom, + Avatar, Popover, TextField, Drawer, @@ -45,8 +46,13 @@ import { Breadcrumbs, CircularProgress, Switch, + Chip, } from "@material-ui/core"; +import { + AvatarGroup, +} from "@mui/material" + import { OpenInNew as OpenInNewIcon, Undo as UndoIcon, @@ -248,6 +254,9 @@ const AngularWorkflow = (defaultprops) => { const [subworkflowStartnode, setSubworkflowStartnode] = React.useState(""); const [leftViewOpen, setLeftViewOpen] = React.useState(true); const [leftBarSize, setLeftBarSize] = React.useState(350); + const [creatorProfile, setCreatorProfile] = React.useState({}); + const [appGroup, setAppGroup] = React.useState([]); + const [triggerGroup, setTriggerGroup] = React.useState([]); const [executionText, setExecutionText] = React.useState(""); const [executionRequestStarted, setExecutionRequestStarted] = React.useState(false); @@ -1346,7 +1355,7 @@ const AngularWorkflow = (defaultprops) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for apps :O!"); + console.log("Status not 200 for app auth :O!"); } return response.json(); @@ -1460,6 +1469,10 @@ const AngularWorkflow = (defaultprops) => { return response.json(); }) .then((responseJson) => { + if (responseJson.success === false) { + return + } + // FIXME - handle versions on left bar //handleAppVersioning(responseJson) //var tmpapps = [] @@ -1495,7 +1508,7 @@ const AngularWorkflow = (defaultprops) => { } }) .catch((error) => { - alert.error(error.toString()); + alert.error("App loading error: ", error.toString()); }); }; @@ -1530,12 +1543,51 @@ const AngularWorkflow = (defaultprops) => { responseJson.errors = []; } + if (responseJson.actions === undefined || responseJson.actions === null) { + responseJson.actions = []; + } + + if (responseJson.triggers === undefined || responseJson.triggers === null) { + responseJson.triggers = []; + } + if (responseJson.public) { alert.info( - "This workflow is public. You will have to save it to make it your own!" + "This workflow is public. Save the workflow to " ); + + console.log("RESP: ", responseJson) + if (Object.getOwnPropertyNames(creatorProfile).length === 0) { + getUserProfile("frikky") + } + + //{appGroup.map((data, index) => { + //const [appGroup, setAppGroup] = React.useState([]); + var appsFound = [] + for (var key in responseJson.actions) { + const parsedAction = responseJson.actions[key] + if (parsedAction.large_image === undefined || parsedAction.large_image === null || parsedAction.large_image === "") { + continue + } + if (appsFound.findIndex(data => data.app_name === parsedAction.app_name) < 0){ + appsFound.push(parsedAction) + } + } + + setAppGroup(appsFound) + + appsFound = [] + for (var key in responseJson.triggers) { + const parsedAction = responseJson.triggers[key] + if (appsFound.findIndex(data => data.app_name === parsedAction.app_name) < 0){ + appsFound.push(parsedAction) + } + } + + setTriggerGroup(appsFound) } + // Appends SUBFLOWS. Does NOT run during normal grabbing of workflows. if (sourcenode.id !== undefined) { console.log("WORKFLOW: ", responseJson); @@ -2107,9 +2159,11 @@ const AngularWorkflow = (defaultprops) => { if (!lastSaved) { return unloadText; } else { - //document.removeEventListener("mousemove", onMouseUpdate, true); - document.removeEventListener("keydown", handleKeyDown, true); - document.removeEventListener("paste", handlePaste, true); + if (workflow.public === false) { + //document.removeEventListener("mousemove", onMouseUpdate, true); + document.removeEventListener("keydown", handleKeyDown, true); + document.removeEventListener("paste", handlePaste, true); + } } }); @@ -3393,7 +3447,7 @@ const AngularWorkflow = (defaultprops) => { }) .then((response) => { if (response.status !== 200) { - console.log("Status not 200 for apps :O!"); + console.log("Status not 200 for envs :O!"); if (isCloud) { setEnvironments([{ Name: "Cloud", Type: "cloud" }]); } else { @@ -3820,7 +3874,7 @@ const AngularWorkflow = (defaultprops) => { //var parentNode = cy.$("#" + event.target.data("id")); //if (parentNode.data("isButton") || parentNode.data("buttonId")) return; - if (nodedata.app_name !== undefined) { + if (nodedata.app_name !== undefined && !workflow.public === true) { const allNodes = cy.nodes().jsons(); var found = false; @@ -9893,6 +9947,10 @@ const AngularWorkflow = (defaultprops) => { }; const TopCytoscapeBar = (props) => { + if (workflow.public === true) { + return null + } + return (
@@ -9916,10 +9974,6 @@ const AngularWorkflow = (defaultprops) => {

{workflow.name}

- - {workflow.public ? ( -

Public Workflow PREVIEW

- ) : null} @@ -10518,66 +10572,202 @@ const AngularWorkflow = (defaultprops) => { // executeWorkflowWebsocket() //}}>Execute websocket // + + const getUserProfile = (username) => { + fetch(`${globalUrl}/api/v1/users/creators/${username}`, { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for WORKFLOW EXECUTION :O!"); + } - const leftView = leftViewOpen ? ( -
- -
- ) : ( -
-
{ - setLeftViewOpen(true); - setLeftBarSize(350); - }} - > - - - -
-
- ); - const executionPaperStyle = { - minWidth: "95%", - maxWidth: "95%", - marginTop: "5px", - color: "white", - marginBottom: 10, - padding: 5, - backgroundColor: surfaceColor, - cursor: "pointer", - display: "flex", - minHeight: 40, - maxHeight: 40, - }; + return response.json(); + }) + .then((responseJson) => { + console.log("FOUND USER FOR: ", username, responseJson) + if (responseJson.success !== false) { + setCreatorProfile(responseJson) + } + }) + .catch((error) => { + console.log(error); + }) + } + + const leftView = workflow.public === true ? +
+ + {workflow.name} + + + This workflow is public and { + saveWorkflow() + }}>must be saved to be used in your organization. + + {Object.getOwnPropertyNames(creatorProfile).length !== 0 && creatorProfile.github_avatar !== undefined && creatorProfile.github_avatar !== null ? +
+ { + }}> + + + + + + Shared by {creatorProfile.github_username} + +
+ : null} +
+ {workflow.tags !== undefined && workflow.tags !== null && workflow.tags.length > 0 ? +
+ + Tags + +
+ {workflow.tags.map((tag, index) => { + if (index >= 3) { + return null; + } - const parsedExecutionArgument = () => { - var showResult = executionData.execution_argument.trim(); - const validate = validateJson(showResult); + return ( + + ); + })} +
+
+ : null } +
+ + Mitre Att&ck: + + + TBD + +
+ {appGroup.length > 0 ? +
+ + Apps + + + {appGroup.map((data, index) => { + return ( + + + + ) + })} + +
+ : null} + {triggerGroup.length > 0 ? +
+ + Triggers + + + {triggerGroup.map((data, index) => { + return ( + + ) + })} + +
+ : null} +
+ + Related Workflows: + + + TBD + +
+ {workflow.description !== undefined && workflow.description !== null && workflow.description.length > 0 ? +
+ + Description + + + {workflow.description} + +
+ : null} +
+ : leftViewOpen ? ( +
+ +
+) : ( +
+
{ + setLeftViewOpen(true); + setLeftBarSize(350); + }} + > + + + +
+
+); - if (validate.valid) { - if (typeof validate.result === "string") { - try { - validate.result = JSON.parse(validate.result); - } catch (e) { - console.log("Error: ", e); - validate.valid = false; - } - } +const executionPaperStyle = { + minWidth: "95%", + maxWidth: "95%", + marginTop: "5px", + color: "white", + marginBottom: 10, + padding: 5, + backgroundColor: surfaceColor, + cursor: "pointer", + display: "flex", + minHeight: 40, + maxHeight: 40, +}; + +const parsedExecutionArgument = () => { + var showResult = executionData.execution_argument.trim(); + const validate = validateJson(showResult); + + if (validate.valid) { + if (typeof validate.result === "string") { + try { + validate.result = JSON.parse(validate.result); + } catch (e) { + console.log("Error: ", e); + validate.valid = false; + } + } return (
diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 91ac42da..d595b75a 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -352,7 +352,7 @@ const Docs = (defaultprops) => { } function Img(props) { - return {props.alt}; + return {props.alt}; } function CodeHandler(props) { diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 5930dd1a..dc8b47e4 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -33,6 +33,10 @@ import { DialogContent, } from "@material-ui/core"; +import { + AvatarGroup, +} from "@mui/material" + import { GridOn as GridOnIcon, List as ListIcon, @@ -1310,6 +1314,30 @@ const Workflows = (props) => { ); }; + const getWorkflowAppgroup = (data) => { + if (data.actions === undefined || data.actions === null) { + return [] + } + + var appsFound = [] + for (var key in data.actions) { + const parsedAction = data.actions[key] + if (parsedAction.large_image === undefined || parsedAction.large_image === null || parsedAction.large_image === "") { + continue + } + + if (parsedAction.app_name === "Shuffle Tools" || parsedAction.app_id === "bc78f35c6c6351b07a09b7aed5d29652") { + continue + } + + if (appsFound.findIndex(data => data.app_name === parsedAction.app_name) < 0){ + appsFound.push(parsedAction) + } + } + + return appsFound + } + const WorkflowPaper = (props) => { const { data } = props; const [open, setOpen] = React.useState(false); @@ -1339,6 +1367,7 @@ const Workflows = (props) => { } const actions = data.actions !== null ? data.actions.length : 0; + const appGroup = getWorkflowAppgroup(data) const [triggers, subflows] = getWorkflowMeta(data); const workflowMenuButtons = ( @@ -1561,22 +1590,48 @@ const Workflows = (props) => { - - - - - {actions} - - - + {appGroup.length > 0 ? +
+ + {appGroup.map((data, index) => { + return ( +
{ + addFilter(data.app_name); + }} + > + + + +
+ ) + })} + +
+ : + + + + + {actions} + + + + } { marginTop: 5, }} > - {data.tags !== undefined + {data.tags !== undefined && data.tags !== null ? data.tags.map((tag, index) => { if (index >= 3) { return null; From 4e3f0833ad833d1196a1487ed814ef5426bf155f Mon Sep 17 00:00:00 2001 From: frikky Date: Mon, 31 Jan 2022 12:27:01 +0100 Subject: [PATCH 16/53] Minor action fix --- frontend/src/components/ParsedAction.jsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/src/components/ParsedAction.jsx b/frontend/src/components/ParsedAction.jsx index badf7a6c..0b5c31ab 100644 --- a/frontend/src/components/ParsedAction.jsx +++ b/frontend/src/components/ParsedAction.jsx @@ -1349,7 +1349,7 @@ const ParsedAction = (props) => { // 0) { + if (data !== undefined && data !== null && data.value !== undefined && data.value !== null && data.value.length > 0) { baseHelperText = calculateHelpertext(data.value) } From c11051c7ebd8cef9f0ca317dacd2666b03281c98 Mon Sep 17 00:00:00 2001 From: frikky Date: Wed, 2 Feb 2022 04:15:16 +0100 Subject: [PATCH 17/53] Added new test-build that includes OpenID connect to test with KeyCloak --- backend/go-app/go.mod | 3 +- backend/go-app/main.go | 81 ++++++- frontend/src/components/OrgHeader.jsx | 295 +++++++++++++++++++------- frontend/src/views/LoginPage.jsx | 14 +- 4 files changed, 308 insertions(+), 85 deletions(-) diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 0baf7fee..e22ce748 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -22,8 +22,9 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/gorilla/mux v1.8.0 github.com/h2non/filetype v1.1.3 + github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.86 + github.com/shuffle/shuffle-shared v0.1.94 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 0b9b8c9a..c8b147a5 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -11,6 +11,7 @@ import ( //"crypto/tls" //"crypto/x509" + "encoding/base64" "encoding/hex" "encoding/json" "errors" @@ -18,6 +19,7 @@ import ( "io" "io/ioutil" "log" + "math/rand" "net/http" "net/url" "os" @@ -53,6 +55,8 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/storage/memory" + //cv "github.com/nirasan/go-oauth-pkce-code-verifier" + //githttp "gopkg.in/src-d/go-git.v4/plumbing/transport/http" // Random @@ -1166,8 +1170,16 @@ func handleContact(resp http.ResponseWriter, request *http.Request) { resp.Write([]byte(fmt.Sprintf(`{"success": true, "message": "Thanks for reaching out. We will contact you soon!"}`))) } +func verifier() (*shuffle.CodeVerifier, error) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + b := make([]byte, 32, 32) + for i := 0; i < 32; i++ { + b[i] = byte(r.Intn(255)) + } + return shuffle.CreateCodeVerifierFromBytes(b) +} + func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { - log.Printf("In admin login request?") cors := shuffle.HandleCors(resp, request) if cors { return @@ -1190,10 +1202,63 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) { return } - //ssoUrl = org.SSOConfig.SOSOEntrypoint - redirectUri := shuffle.SSOUrl + baseSSOUrl := "" + handled := []string{} + for _, user := range users { + if shuffle.ArrayContains(handled, user.ActiveOrg.Id) { + continue + } + + handled = append(handled, user.ActiveOrg.Id) + org, err := shuffle.GetOrg(ctx, user.ActiveOrg.Id) + if err != nil { + log.Printf("[WARNING] Error getting org in admin check: %s", err) + continue + } + + // No childorg setup, only parent org + if len(org.ManagerOrgs) > 0 || len(org.CreatorOrg) > 0 { + continue + } + + // Should run calculations + if len(org.SSOConfig.OpenIdAuthorization) > 0 { + log.Printf("[DEBUG] Found OpenID url (PKCE). Extra redirect check: %s", request.URL.String()) + baseSSOUrl = org.SSOConfig.OpenIdAuthorization + + codeChallenge := uuid.NewV4().String() + //h.Write([]byte(v.Value)) + verifier, verifiererr := verifier() + if verifiererr == nil { + codeChallenge = verifier.Value + } + + //log.Printf("[DEBUG] Got challenge value %s (pre state)", codeChallenge) + + redirectUrl := url.QueryEscape("http://localhost:5001/api/v1/login_openid") + state := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("org=%s&challenge=%s&redirect=%s", org.Id, codeChallenge, redirectUrl))) + + // has to happen after initial value is stored + if verifiererr == nil { + codeChallenge = verifier.CodeChallengeS256() + } + + //log.Printf("[DEBUG] Got challenge value %s (POST state)", codeChallenge) + + baseSSOUrl += fmt.Sprintf("?client_id=%s&response_type=code&scope=openid&redirect_uri=%s&state=%s&code_challenge_method=S256&code_challenge=%s", org.SSOConfig.OpenIdClientId, redirectUrl, state, codeChallenge) + break + } + + if len(org.SSOConfig.SSOEntrypoint) > 0 { + log.Printf("[DEBUG] Found SAML SSO url") + baseSSOUrl = org.SSOConfig.SSOEntrypoint + break + } + } + + log.Printf("[DEBUG] URL: %s", baseSSOUrl) resp.WriteHeader(200) - resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, redirectUri))) + resp.Write([]byte(fmt.Sprintf(`{"success": true, "reason": "redirect", "sso_url": "%s"}`, baseSSOUrl))) } func handleLogin(resp http.ResponseWriter, request *http.Request) { @@ -3325,11 +3390,6 @@ func createFs(basepath, pathname string) (billy.Filesystem, error) { return err } - //if strings.Contains(path, "yaml") { - // log.Printf("PATH: %s -> %s", path, fullpath) - // //log.Printf("DATA: %s", string(srcData)) - //} - dst, err := fs.Create(fullpath) if err != nil { log.Printf("Dst error: %s", err) @@ -5882,7 +5942,8 @@ func initHandlers() { // Docker orborus specific - downloads an image r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/migrate_database", migrateDatabase).Methods("POST", "OPTIONS") - r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("POST", "OPTIONS") + r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS") + r.HandleFunc("/api/v1/login_openid", shuffle.HandleOpenId).Methods("GET", "OPTIONS") // Important for email, IDS etc. Create this by: // PS: For cloud, this has to use cloud storage. diff --git a/frontend/src/components/OrgHeader.jsx b/frontend/src/components/OrgHeader.jsx index 49897638..3d9f0136 100644 --- a/frontend/src/components/OrgHeader.jsx +++ b/frontend/src/components/OrgHeader.jsx @@ -97,6 +97,30 @@ const OrgHeader = (props) => { ? "" : selectedOrganization.defaults.notification_workflow ); + const [openidClientId, setOpenidClientId] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.client_id === undefined || + selectedOrganization.sso_config.client_id.length === 0 + ? "" + : selectedOrganization.sso_config.client_id + ); + const [openidAuthorization, setOpenidAuthorization] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_authorization === undefined || + selectedOrganization.sso_config.openid_authorization.length === 0 + ? "" + : selectedOrganization.sso_config.openid_authorization + ); + const [openidToken, setOpenidToken] = React.useState( + selectedOrganization.sso_config === undefined + ? "" + : selectedOrganization.sso_config.openid_token=== undefined || + selectedOrganization.sso_config.openid_token.length === 0 + ? "" + : selectedOrganization.sso_config.openidtoken_ + ) const [file, setFile] = React.useState(""); const [fileBase64, setFileBase64] = React.useState( @@ -145,6 +169,7 @@ const OrgHeader = (props) => { defaults, sso_config ) => { + const data = { name: name, description: description, @@ -216,6 +241,9 @@ const OrgHeader = (props) => { { sso_entrypoint: ssoEntrypoint, sso_certificate: ssoCertificate, + client_id: openidClientId, + openid_authorization: openidAuthorization, + openid_token: openidToken, } ) } @@ -548,79 +576,200 @@ const OrgHeader = (props) => { )} - - - SSO Entrypoint (IdP) - 0 - } - id="outlined-with-placeholder" - margin="normal" - variant="outlined" - placeholder="The entrypoint URL from your provider" - value={ssoEntrypoint} - onChange={(e) => { - setSsoEntrypoint(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - - - - SSO Certificate (X509) - { - setSsoCertificate(e.target.value); - }} - InputProps={{ - classes: { - notchedOutline: classes.notchedOutline, - }, - style: { - color: "white", - }, - }} - /> - - + {isCloud ? null : + + OpenID connect + + + + Client ID + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The OpenID client ID from the identity provider" + value={openidClientId} + onChange={(e) => { + setOpenidClientId(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Authorization URL + { + setOpenidAuthorization(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + Token URL + { + setOpenidToken(e.target.value) + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + } + {isCloud ? null : + + SAML SSO (v1.1) + + + + SSO Entrypoint (IdP) + 0 + } + id="outlined-with-placeholder" + margin="normal" + variant="outlined" + placeholder="The entrypoint URL from your provider" + value={ssoEntrypoint} + onChange={(e) => { + setSsoEntrypoint(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + SSO Certificate (X509) + { + setSsoCertificate(e.target.value); + }} + InputProps={{ + classes: { + notchedOutline: classes.notchedOutline, + }, + style: { + color: "white", + }, + }} + /> + + + + + } {/* {expanded ? diff --git a/frontend/src/views/LoginPage.jsx b/frontend/src/views/LoginPage.jsx index fd33f22e..19922e43 100644 --- a/frontend/src/views/LoginPage.jsx +++ b/frontend/src/views/LoginPage.jsx @@ -1,5 +1,5 @@ /* eslint-disable react/no-multi-comp */ -import React, { useState } from "react"; +import React, { useState, useEffect } from "react"; import { makeStyles } from "@material-ui/styles"; import { useInterval } from "react-powerhooks"; @@ -55,6 +55,7 @@ const LoginDialog = (props) => { const [MFAField, setMFAField] = useState(false); const [MFAValue, setMFAValue] = useState(""); + // Used to swap from login to register. True = login, false = register const classes = useStyles(); @@ -468,6 +469,7 @@ const LoginDialog = (props) => {
diff --git a/frontend/src/components/Oauth2Auth.jsx b/frontend/src/components/Oauth2Auth.jsx index ef7384f7..65134fbf 100644 --- a/frontend/src/components/Oauth2Auth.jsx +++ b/frontend/src/components/Oauth2Auth.jsx @@ -517,6 +517,8 @@ const AuthenticationOauth2 = (props) => { backgroundColor: theme.palette.inputColor, color: "white", padding: 5, + minWidth: 300, + maxWidth: 300, }} onChange={(e) => { handleScopeChange(e) @@ -542,7 +544,7 @@ const AuthenticationOauth2 = (props) => { title={"Automatic Refresh (default: true)"} placement="top" > - { + { setOfflineAccess(!offlineAccess) }}/> diff --git a/frontend/src/components/OrgHeader.jsx b/frontend/src/components/OrgHeader.jsx index 3d9f0136..a370af93 100644 --- a/frontend/src/components/OrgHeader.jsx +++ b/frontend/src/components/OrgHeader.jsx @@ -116,10 +116,10 @@ const OrgHeader = (props) => { const [openidToken, setOpenidToken] = React.useState( selectedOrganization.sso_config === undefined ? "" - : selectedOrganization.sso_config.openid_token=== undefined || + : selectedOrganization.sso_config.openid_token === undefined || selectedOrganization.sso_config.openid_token.length === 0 ? "" - : selectedOrganization.sso_config.openidtoken_ + : selectedOrganization.sso_config.openid_token ) const [file, setFile] = React.useState(""); diff --git a/frontend/src/views/Admin.jsx b/frontend/src/views/Admin.jsx index 347d4568..81f1ea7b 100644 --- a/frontend/src/views/Admin.jsx +++ b/frontend/src/views/Admin.jsx @@ -1154,8 +1154,8 @@ const Admin = (props) => { }) .then((respdata) => { if (respdata.length === 0) { - alert.error("Failed getting file"); - return; + alert.error("Failed getting file. Is it deleted?"); + return; } var blob = new Blob([respdata], { @@ -3101,12 +3101,12 @@ const Admin = (props) => { {fileNamespaces !== undefined && fileNamespaces !== null && fileNamespaces.length > 1 ? ( - - Namespace + + File Category { Autocomplete { console.log("VAL: ", event.target.value) @@ -2466,6 +2475,7 @@ const ParsedAction = (props) => { if (param.value.includes(baselabel)) { //if (param.value.toLowerCase().includes(baselabel)) { console.log("FOUND: ", param); + workflow.actions[key].parameters[subkey].value.replaceAll( baselabel, e.target.value @@ -2555,6 +2565,9 @@ const ParsedAction = (props) => { Authentication
{
Set execution variable (optional) { return response.json(); }) .then((responseJson) => { + if (responseJson === null) { + console.log("No response") + const pretend_apps = [{ + "name": "TBD", + "app_name": "TBD", + "app_version": "TBD", + "description": "TBD", + "version": "TBD", + "large_image": "", + }] + setApps(pretend_apps) + setFilteredApps(pretend_apps) + setPrioritizedApps(pretend_apps); + return + } + if (responseJson.success === false) { return } @@ -5657,12 +5673,14 @@ const AngularWorkflow = (defaultprops) => { } // Does this one find the wrong one? - var newSelectedAction = JSON.parse(JSON.stringify(selectedAction)) + //var newSelectedAction = JSON.parse(JSON.stringify(selectedAction)) + var newSelectedAction = selectedAction newSelectedAction.name = newaction.name; newSelectedAction.parameters = JSON.parse(JSON.stringify(newaction.parameters)) newSelectedAction.errors = []; newSelectedAction.isValid = true; newSelectedAction.is_valid = true; + console.log(newSelectedAction) // Simmple action swap autocompleter if (selectedAction.parameters !== undefined && newSelectedAction.parameters !== undefined && selectedAction.id === newSelectedAction.id) { @@ -5804,6 +5822,7 @@ const AngularWorkflow = (defaultprops) => { event.target.value = event.target.value.replaceAll(".", ""); event.target.value = event.target.value.replaceAll(",", ""); event.target.value = event.target.value.replaceAll(" ", "_"); + selectedAction.label = event.target.value; setSelectedAction(selectedAction); }; @@ -6240,6 +6259,9 @@ const AngularWorkflow = (defaultprops) => { Autocomplete {
Environment { data.schema !== null && data.schema.type === "bool" ? ( - - Accounts - - - - - - - - - - - - -
- -
- - - - - - - - -
Total Shipments
- - 763,215 - - - -
- -
- - - - - - -
Daily Sales
- - {" "} - 3,500€ - - - -
- -
- - - - - - -
Completed Tasks
- - 12,100K - - - -
- -
- - - -
); diff --git a/frontend/src/views/GettingStarted.jsx b/frontend/src/views/GettingStarted.jsx index 37d537ed..76bde71c 100644 --- a/frontend/src/views/GettingStarted.jsx +++ b/frontend/src/views/GettingStarted.jsx @@ -110,263 +110,7 @@ const useStyles = makeStyles((theme) => ({ }, })); -// Takes an action in Shuffle and -// Returns information about the icon, the color etc to be used -// This can be used for actions of all types -export const GetIconInfo = (action) => { - // Finds the icon based on the action. Should be verbs. - const iconList = [ - { key: "cache_add", values: ["set_cache"] }, - { key: "cache_get", values: ["get_cache"] }, - { key: "filter", values: ["filter", "route", "router"] }, - { key: "merge", values: ["join", "merge"] }, - { - key: "search", - values: ["search", "find", "locate", "index", "analyze", "anal", "match", "check cache", "check", "verify", "validate"], - }, - { key: "list", values: ["list", "head", "options"] }, - { - key: "download", - values: [ - "capture", - "get", - "download", - "return", - "hello_world", - "curl", - "request", - "export", - "preview", - ], - }, - { key: "add", values: ["add", "accept", ] }, - { key: "delete", values: ["delete", "remove", "clear", "clean", "dismiss",] }, - { - key: "send", - values: [ - "send", - "dispatch", - "mail", - "forward", - "post", - "submit", - "mark", - "set", - "release", - ], - }, - { - key: "repeat", - values: ["repeat", "retry", "pause", "skip", "copy", "replicat", "demo", ], - }, - { key: "execute", values: ["execute", "run", "play", "raise"] }, - { key: "extract", values: ["extract", "unpack", "decompress", "open"] }, - { key: "inflate", values: ["inflate", "pack", "compress"] }, - { - key: "edit", - values: [ - "modify", - "update", - "create", - "edit", - "put", - "patch", - "change", - "replace", - "conver", - "map", - "format", - "escape", - "describe", - ], - }, - { - key: "compare", - values: ["compare", "convert", "to", "filter", "translate", "parse"], - }, - { key: "close", values: ["close", "stop", "cancel", "block"] }, - ]; - - var selectedKey = ""; - if (action.name === undefined || action.name === null) { - } else { - const actionname = action.name.toLowerCase(); - for (var key in iconList) { - //console.log(iconList[key], actionname) - const found = iconList[key].values.find((value) => - actionname.includes(value) - ); - if (found !== null && found !== undefined) { - selectedKey = iconList[key].key; - break; - } - } - } - - // Some of these are manually parsed or created instead of material ui - //M8 0C3.58 0 0 1.79 0 4C0 6.21 3.58 8 8 8C12.42 8 16 6.21 16 4C16 1.79 12.42 0 8 0ZM0 6V9C0 11.21 3.58 13 8 13C12.42 13 16 11.21 16 9V6C16 8.21 12.42 10 8 10C3.58 10 0 8.21 0 6ZM0 11V14C0 16.21 3.58 18 8 18C9.41 18 10.79 17.81 12 17.46V14.46C10.79 14.81 9.41 15 8 15C3.58 15 0 13.21 0 11ZM17 11V14H14V16H17V19H19V16H22V14H19V11 - //https://www.figma.com/file/uCfnMs5w6wnLx6ehPHEV74/Figma-Material-Design-System-v3_0?node-id=834%3A21 - //COLORS: https://www.pinterest.co.uk/pin/326299935499972946/ - const defaultColor = "#f76b1c"; - const defaultGradient = ["#fad961", "#f76b1c"]; - const parsedIcons = { - cache_add: { - icon: "M11 3C6.58 3 3 4.79 3 7C3 9.21 6.58 11 11 11C15.42 11 19 9.21 19 7C19 4.79 15.42 3 11 3ZM3 9V12C3 14.21 6.58 16 11 16C15.42 16 19 14.21 19 12V9C19 11.21 15.42 13 11 13C6.58 13 3 11.21 3 9ZM3 14V17C3 19.21 6.58 21 11 21C12.41 21 13.79 20.81 15 20.46V17.46C13.79 17.81 12.41 18 11 18C6.58 18 3 16.21 3 14ZM20 14V17H17V19H20V22H22V19H25V17H22V14", - iconColor: "white", - iconBackgroundColor: "#8acc3f", - originalIcon: "", - fillGradient: ["#8acc3f", "#459622"], - }, - cache_get: { - icon: "M12 2C7.58 2 4 3.79 4 6C4 8.06 7.13 9.74 11.15 9.96C12.45 8.7 14.19 8 16 8C16.8 8 17.59 8.14 18.34 8.41C19.37 7.74 20 6.91 20 6C20 3.79 16.42 2 12 2ZM4 8V11C4 12.68 6.08 14.11 9 14.71C9.06 13.7 9.32 12.72 9.77 11.82C6.44 11.34 4 9.82 4 8ZM15.93 9.94C14.75 9.95 13.53 10.4 12.46 11.46C8.21 15.71 13.71 22.5 18.75 19.17L23.29 23.71L24.71 22.29L20.17 17.75C22.66 13.97 19.47 9.93 15.93 9.94ZM15.9 12C17.47 11.95 19 13.16 19 15C19 15.7956 18.6839 16.5587 18.1213 17.1213C17.5587 17.6839 16.7956 18 16 18C13.33 18 12 14.77 13.88 12.88C14.47 12.29 15.19 12 15.9 12ZM4 13V16C4 18.05 7.09 19.72 11.06 19.95C10.17 19.07 9.54 17.95 9.22 16.74C6.18 16.17 4 14.72 4 13Z", - iconColor: "white", - iconBackgroundColor: "#8acc3f", - originalIcon: "", - fillGradient: ["#8acc3f", "#459622"], - }, - repeat: { - icon: "M19 8l-4 4h3c0 3.31-2.69 6-6 6-1.01 0-1.97-.25-2.8-.7l-1.46 1.46C8.97 19.54 10.43 20 12 20c4.42 0 8-3.58 8-8h3l-4-4zM6 12c0-3.31 2.69-6 6-6 1.01 0 1.97.25 2.8.7l1.46-1.46C15.03 4.46 13.57 4 12 4c-4.42 0-8 3.58-8 8H1l4 4 4-4H6z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - add: { - icon: "M19 13h-6v6h-2v-6H5v-2h6V5h2v6h6v2z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - edit: { - icon: "M3 17.25V21h3.75L17.81 9.94l-3.75-3.75L3 17.25zM20.71 7.04c.39-.39.39-1.02 0-1.41l-2.34-2.34a.9959.9959 0 00-1.41 0l-1.83 1.83 3.75 3.75 1.83-1.83z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - filter: { - icon: "M4.25 5.61C6.27 8.2 10 13 10 13v6c0 .55.45 1 1 1h2c.55 0 1-.45 1-1v-6s3.72-4.8 5.74-7.39c.51-.66.04-1.61-.79-1.61H5.04c-.83 0-1.3.95-.79 1.61z", - iconColor: "white", - iconBackgroundColor: "#f5515f", - originalIcon: "", - fillGradient: ["#f5515f", "#a1051d"], - }, - merge: { - icon: "M17 20.41 18.41 19 15 15.59 13.59 17 17 20.41zM7.5 8H11v5.59L5.59 19 7 20.41l6-6V8h3.5L12 3.5 7.5 8z", - iconColor: "white", - iconBackgroundColor: "#f5515f", - originalIcon: "", - fillGradient: ["#f5515f", "#a1051d"], - }, - compare: { - icon: "M10 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h5v2h2V1h-2v2zm0 15H5l5-6v6zm9-15h-5v2h5v13l-5-6v9h5c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - extract: { - icon: "M3 3h18v2H3z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - inflate: { - icon: "M6 19h12v2H6z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - list: { - icon: "M3 9h14V7H3v2zm0 4h14v-2H3v2zm0 4h14v-2H3v2zm16 0h2v-2h-2v2zm0-10v2h2V7h-2zm0 6h2v-2h-2v2z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - execute: { - icon: "M8 5v14l11-7z", - iconColor: "white", - iconBackgroundColor: defaultColor, - originalIcon: , - }, - delete: { - icon: "M6 19c0 1.1.9 2 2 2h8c1.1 0 2-.9 2-2V7H6v12zM19 4h-3.5l-1-1h-5l-1 1H5v2h14V4z", - iconColor: "white", - iconBackgroundColor: "#03030e", - originalIcon: , - fillGradient: ["#03030e", "#205d66"], - }, - close: { - icon: "M19 6.41L17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z", - iconColor: "white", - iconBackgroundColor: "#03030e", - originalIcon: , - fillGradient: ["#03030e", "#205d66"], - }, - send: { - icon: "M2.01 21L23 12 2.01 3 2 10l15 2-15 2z", - iconColor: "white", - iconBackgroundColor: "#0373da", - originalIcon: , - fillGradient: ["#0bc8bf", "#0373da"], - }, - download: { - icon: "M19 9h-4V3H9v6H5l7 7 7-7zM5 18v2h14v-2H5z", - iconColor: "white", - iconBackgroundColor: "#0373da", - originalIcon: , - fillGradient: ["#0bc8bf", "#0373da"], - }, - search: { - icon: "M15.5 14h-.79l-.28-.27C15.41 12.59 16 11.11 16 9.5 16 5.91 13.09 3 9.5 3S3 5.91 3 9.5 5.91 16 9.5 16c1.61 0 3.09-.59 4.23-1.57l.27.28v.79l5 4.99L20.49 19l-4.99-5zm-6 0C7.01 14 5 11.99 5 9.5S7.01 5 9.5 5 14 7.01 14 9.5 11.99 14 9.5 14z", - iconColor: "white", - iconBackgroundColor: "green", - originalIcon: , - }, - }; - - var selectedItem = parsedIcons[selectedKey]; - if (selectedItem === undefined || selectedItem === null) { - return { - icon: "", - iconColor: "", - iconBackground: "black", - originalIcon: "", - }; - } - - if (selectedItem.fillGradient === undefined) { - selectedItem.fillGradient = defaultGradient; - selectedItem.iconBackgroundColor = defaultColor; - } - - if (selectedItem.icon === "" || selectedItem.icon === undefined) { - console.log( - `MISSING PATH FOR ${selectedKey} (find in scope): `, - selectedItem.originalIcon.type.type - ); - } - - if ( - (selectedItem.originalIcon === undefined || - selectedItem.originalIcon === "") && - selectedItem.icon !== "" && - selectedItem.icon !== undefined - ) { - const svg_pin = ( - - - - ); - selectedItem.originalIcon = svg_pin; - } - - return selectedItem; -}; - + const chipStyle = { backgroundColor: "#3d3f43", marginRight: 5, @@ -378,62 +122,6 @@ const chipStyle = { color: "white", }; -export const validateJson = (showResult) => { - //showResult = showResult.split(" None").join(" \"None\"") - showResult = showResult.split(" False").join(" false"); - showResult = showResult.split(" True").join(" true"); - - var jsonvalid = true; - try { - if (!showResult.includes("{") && !showResult.includes("[")) { - jsonvalid = false; - } - } catch (e) { - showResult = showResult.split("'").join('"'); - - try { - if (!showResult.includes("{") && !showResult.includes("[")) { - jsonvalid = false; - } - } catch (e) { - jsonvalid = false; - } - } - - var result = showResult; - try { - result = jsonvalid ? JSON.parse(showResult) : showResult; - } catch (e) { - ////console.log("Failed parsing JSON even though its valid: ", e) - jsonvalid = false; - } - - if (jsonvalid === false) { - - if (typeof showResult === 'string') { - showResult = showResult.trim() - } - - try { - var newstr = showResult.replaceAll("'", '"') - - //console.log("Try replacements and trimming with new value: ", newstr) - result = JSON.parse(newstr) - jsonvalid = true - } catch (e) { - - //console.log("Failed parsing JSON even though its valid (2): ", e) - jsonvalid = false - } - } - - //console.log("VALID: ", jsonvalid, result) - return { - valid: jsonvalid, - result: result, - }; -}; - const GettingStarted = (props) => { const { globalUrl, isLoggedIn, isLoaded, userdata } = props; @@ -461,8 +149,8 @@ const GettingStarted = (props) => { "https://github.com/frikky/shuffle-workflows" ); const [downloadBranch, setDownloadBranch] = React.useState("master"); - const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = - React.useState(false); + const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false); + const [videoViewOpen, setVideoViewOpen] = React.useState(false); const [exportModalOpen, setExportModalOpen] = React.useState(false); const [exportData, setExportData] = React.useState(""); @@ -827,6 +515,8 @@ const GettingStarted = (props) => { credentials: "include", }) .then((response) => { + setVideoViewOpen(true) + if (response.status !== 200) { console.log("Status not 200 for workflows :O!: ", response.status); @@ -881,6 +571,8 @@ const GettingStarted = (props) => { } }) .catch((error) => { + setVideoViewOpen(true) + alert.error(error.toString()); }); }; @@ -2448,58 +2140,8 @@ const GettingStarted = (props) => { maxWidth: 600, }; - const WorkflowView = () => { - /* - if (workflows.length === 0) { - return ( -
- -
-

Welcome to Shuffle

-
-
-

- Shuffle is a flexible, easy to use, automation platform - allowing users to integrate their services and devices freely. - It's made to significantly reduce the amount of manual labor, - and is focused on security applications.{" "} - - Click here to learn more. - -

-
-
- If you want to jump straight into it, click here to create your - first workflow: -
-
- - - - ..OR - - {workflowButtons} - -
- -
- ); - } - */ + const WorkflowView = () => { var workflowDelay = -150 var appDelay = -75 @@ -2584,6 +2226,53 @@ const GettingStarted = (props) => { return (
+ { + setVideoViewOpen(false) + }} + PaperProps={{ + style: { + backgroundColor: surfaceColor, + color: "white", + minWidth: 560, + minHeight: 415, + textAlign: "center", + }, + }} + > + + Welcome to Shuffle! + + + + { + e.preventDefault(); + setVideoViewOpen(false) + }} + > + + + + + +
Getting Started with Shuffle diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 2063636f..f4e445c1 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.56 +VERSION=0.9.58 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force From 0df7bdd970e0f4813d9a964a4d195ff7dccb3661 Mon Sep 17 00:00:00 2001 From: frikky Date: Thu, 17 Feb 2022 23:06:01 +0100 Subject: [PATCH 38/53] Added sample proxy server for testing with orborus --- backend/go-app/go.mod | 2 +- docker-compose.yml | 5 +- frontend/src/views/AngularWorkflow.jsx | 10 ++- frontend/src/views/GettingStarted.jsx | 94 ++++++++++++------------ functions/onprem/orborus/build.sh | 2 +- functions/onprem/orborus/proxy_server.py | 54 ++++++++++++++ functions/onprem/orborus/run.sh | 13 ++-- 7 files changed, 120 insertions(+), 60 deletions(-) create mode 100644 functions/onprem/orborus/proxy_server.py diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index fba0f78f..86e33d11 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.1.98 + github.com/shuffle/shuffle-shared v0.2.3 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/docker-compose.yml b/docker-compose.yml index a6b6fc0d..4eacbeb7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - build: ./frontend + #build: ./frontend image: ghcr.io/frikky/shuffle-frontend:nightly container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - build: ./backend + #build: ./backend image: ghcr.io/frikky/shuffle-backend:nightly container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} @@ -48,7 +48,6 @@ services: - /var/run/docker.sock:/var/run/docker.sock environment: - SHUFFLE_WORKER_VERSION=nightly - - ORG_ID=${ORG_ID} - ENVIRONMENT_NAME=${ENVIRONMENT_NAME} - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT} - DOCKER_API_VERSION=1.40 diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 99d69f0b..952cb4d1 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -659,7 +659,8 @@ const AngularWorkflow = (defaultprops) => { }); const newitem = removeParam("execution_id", cursearch); - props.history.push(curpath + newitem); + navigate(curpath + newitem) + //props.history.push(curpath + newitem); } } } @@ -3576,7 +3577,8 @@ const AngularWorkflow = (defaultprops) => { ) { setExecutionModalOpen(true) const newitem = removeParam("execution_highlight", cursearch); - props.history.push(curpath + newitem); + navigate(curpath + newitem) + //props.history.push(curpath + newitem); } const tmpView = new URLSearchParams(cursearch).get("view"); @@ -3588,7 +3590,9 @@ const AngularWorkflow = (defaultprops) => { setExecutionModalOpen(true); const newitem = removeParam("view", cursearch); - props.history.push(curpath + newitem); + navigate(curpath + newitem) + //navigate(`?execution_highlight=${parsed_url}`) + //props.history.push(curpath + newitem); } return; } diff --git a/frontend/src/views/GettingStarted.jsx b/frontend/src/views/GettingStarted.jsx index 76bde71c..03c6f280 100644 --- a/frontend/src/views/GettingStarted.jsx +++ b/frontend/src/views/GettingStarted.jsx @@ -2226,53 +2226,55 @@ const GettingStarted = (props) => { return (
- { - setVideoViewOpen(false) - }} - PaperProps={{ - style: { - backgroundColor: surfaceColor, - color: "white", - minWidth: 560, - minHeight: 415, - textAlign: "center", - }, - }} - > - - Welcome to Shuffle! - - - - { - e.preventDefault(); - setVideoViewOpen(false) - }} - > - - - - - - + + Welcome to Shuffle! + + + + { + e.preventDefault(); + setVideoViewOpen(false) + }} + > + + + + + + + : null}
Getting Started with Shuffle diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index f4e445c1..5226f32f 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.58 +VERSION=0.9.59 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/proxy_server.py b/functions/onprem/orborus/proxy_server.py new file mode 100644 index 00000000..8020616c --- /dev/null +++ b/functions/onprem/orborus/proxy_server.py @@ -0,0 +1,54 @@ +# +# curl -H "Org-id: Shuffle" --proxy "http://192.168.86.45:8081" http://192.168.86.45:5001/api/v1/workflows/queue + +import SocketServer +import SimpleHTTPServer +import requests +import json + +PORT = 8082 + +class MyProxy(SimpleHTTPServer.SimpleHTTPRequestHandler): + def do_GET(self): + url=self.path[:] + allheaders = {} + for item in ("%s" % self.headers).split("\n"): + headersplit = item.split(":") + if len(headersplit) == 2: + allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() + + ret = requests.get(url, headers=allheaders) + print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) + + self.send_response(ret.status_code) + self.end_headers() + self.wfile.write(ret.text) + + def do_POST(self): + url=self.path[:] + allheaders = {} + for item in ("%s" % self.headers).split("\n"): + headersplit = item.split(":") + if len(headersplit) == 2: + allheaders[headersplit[0]] = (headersplit[1][:-1]).strip() + + length = int(self.headers.getheader('content-length')) + try: + message = json.loads(self.rfile.read(length)) + print("Got message: %s" % message) + ret = requests.post(url, headers=allheaders, json=message) + except: + message = self.rfile.read(length) + print("Got message: %s" % message) + ret = requests.post(url, headers=allheaders, data=message) + + print("RESP (%s) - %d - %s" % (url, ret.status_code, ret.text)) + + self.send_response(ret.status_code) + self.end_headers() + self.wfile.write(ret.text) + +httpd = SocketServer.ForkingTCPServer(('', PORT), MyProxy) +print("Now serving at %d" % PORT) +httpd.serve_forever() + diff --git a/functions/onprem/orborus/run.sh b/functions/onprem/orborus/run.sh index 48e5b701..eac60eec 100644 --- a/functions/onprem/orborus/run.sh +++ b/functions/onprem/orborus/run.sh @@ -1,9 +1,10 @@ docker run \ - --env ORG_ID=$ORG_ID \ + --env DOCKER_API_VERSION=1.40 \ --env ENVIRONMENT_NAME="Shuffle" \ - --env BASE_URL=http://shuffle-backend:5001 \ - --env DOCKER_API_VERSION=1.42 \ - --env RUNNING_MODE="Docker" \ - --network "shuffle_shuffle" \ + --env BASE_URL="http://192.168.86.45:5001" \ + --env HTTP_PROXY="http://192.168.86.45:8082" \ + --env HTTPS_PROXY="https://192.168.86.45:8082" \ + --env SHUFFLE_PASS_WORKER_PROXY=true \ + --env SHUFFLE_PASS_APP_PROXY=true \ -v /var/run/docker.sock:/var/run/docker.sock \ - frikky/shuffle:orborus + ghcr.io/frikky/shuffle-orborus:nightly From 6dae6706ac184c0808963a6fb79f2ee2b753ddaf Mon Sep 17 00:00:00 2001 From: frikky Date: Fri, 18 Feb 2022 18:55:28 +0100 Subject: [PATCH 39/53] Added usecase exporter from mindmaps --- frontend/src/views/Apps.jsx | 6 +- functions/usecases/README.md | 16 + functions/usecases/categories.json | 522 ++++++++++++++++++++++++ functions/usecases/categories.rtf | 560 ++++++++++++++++++++++++++ functions/usecases/read_categories.py | 53 +++ 5 files changed, 1154 insertions(+), 3 deletions(-) create mode 100644 functions/usecases/README.md create mode 100644 functions/usecases/categories.json create mode 100644 functions/usecases/categories.rtf create mode 100644 functions/usecases/read_categories.py diff --git a/frontend/src/views/Apps.jsx b/frontend/src/views/Apps.jsx index 6626e8c4..68081191 100644 --- a/frontend/src/views/Apps.jsx +++ b/frontend/src/views/Apps.jsx @@ -291,9 +291,9 @@ const Apps = (props) => { if (response.status !== 200) { console.log("Status not 200 for apps :O!"); - if (isCloud) { - window.location.pathname = "/search"; - } + //if (isCloud) { + // window.location.pathname = "/search"; + //} } return response.json(); diff --git a/functions/usecases/README.md b/functions/usecases/README.md new file mode 100644 index 00000000..553034c2 --- /dev/null +++ b/functions/usecases/README.md @@ -0,0 +1,16 @@ +# Mindmap exporter +Shuffle has a mindmap for Workflow use-cases. These can be changed and exported, with the most important piece being that they're explorable and editable. This has and will come in handy for us as we build it into the product. + +https://www.mindmeister.com/map/2172644474 + +## Editing the Mindmap +There are a few categories. To edit them, click the small plus next to the branch you want to change. + +## Exporting the Mindmap +Click "Export as RTF" in the top left corner of the URL. Download it there. + +## Generating the Shuffle-comaptible mindmap +1. Move the rtf file here +2. Rename it categories.rtf +3. Run the read_categories.py file (python3 read_categories.py) +4. You now have a file called categories.json locally with all the categories in JSON format, ready to be used in graphs. diff --git a/functions/usecases/categories.json b/functions/usecases/categories.json new file mode 100644 index 00000000..f32d9700 --- /dev/null +++ b/functions/usecases/categories.json @@ -0,0 +1,522 @@ +{ + "1. Collect & Distribute": { + "2-way Ticket synchronization": { + "name": "", + "description": "", + "image": "" + }, + "Email management": { + "name": "", + "description": "", + "image": "", + "Attachments": { + "name": "", + "description": "", + "image": "" + }, + "Manage senders": { + "name": "", + "description": "", + "image": "" + }, + "Manage URLs": { + "name": "", + "description": "", + "image": "" + }, + "Encode & Decode URLs": { + "name": "", + "description": "", + "image": "" + }, + "Release a quarantined message": { + "name": "", + "description": "", + "image": "" + } + }, + "EDR to ticket": { + "name": "", + "description": "", + "image": "", + "Fetch incidents & events": { + "name": "", + "description": "", + "image": "" + }, + "Quarantine files": { + "name": "", + "description": "", + "image": "" + }, + "Quarantine host (respond)": { + "name": "", + "description": "", + "image": "" + }, + "Get host information": { + "name": "", + "description": "", + "image": "" + } + }, + "SIEM to ticket": { + "name": "", + "description": "", + "image": "" + }, + "ChatOps": { + "name": "", + "description": "", + "image": "" + }, + "Threat Intel received": { + "name": "", + "description": "", + "image": "" + }, + "Domain investigation with LetsEncrypt": { + "name": "", + "description": "", + "image": "" + }, + "Botnet tracker": { + "name": "", + "description": "", + "image": "" + }, + "Get running containers": { + "name": "", + "description": "", + "image": "" + }, + "Assign tickets": { + "name": "", + "description": "", + "image": "" + }, + "Firewall alerts": { + "name": "", + "description": "", + "image": "", + "Block/accept policies": { + "name": "", + "description": "", + "image": "" + }, + "Add addresses and ports to groups": { + "name": "", + "description": "", + "image": "" + }, + "Support custom URL categories": { + "name": "", + "description": "", + "image": "" + }, + "Fetch logs for specific address": { + "name": "", + "description": "", + "image": "" + }, + "URL filtering": { + "name": "", + "description": "", + "image": "" + } + }, + "IDS/IPS alerts": { + "name": "", + "description": "", + "image": "", + "Get/Fetch alerts": { + "name": "", + "description": "", + "image": "" + }, + "Receive alerts real-time": { + "name": "", + "description": "", + "image": "" + }, + "Get PCAP files": { + "name": "", + "description": "", + "image": "" + }, + "Get network logs": { + "name": "", + "description": "", + "image": "" + }, + "Manage policies": { + "name": "", + "description": "", + "image": "" + } + }, + "Deduplicate information": { + "name": "", + "description": "", + "image": "" + }, + "Correlate information": { + "name": "", + "description": "", + "image": "" + } + }, + "3. Detect": { + "Search SIEM (Sigma)": { + "name": "", + "description": "", + "image": "", + "Network": { + "name": "", + "description": "", + "image": "" + }, + "Endpoint": { + "name": "", + "description": "", + "image": "" + } + }, + "Search EDR (OSQuery)": { + "name": "", + "description": "", + "image": "" + }, + "Search emails (Phish)": { + "name": "", + "description": "", + "image": "", + "Check malware": { + "name": "", + "description": "", + "image": "" + }, + "Check targeted": { + "name": "", + "description": "", + "image": "" + }, + "Check headers and IOCs": { + "name": "", + "description": "", + "image": "" + } + }, + "Search IOCs (ioc-finder)": { + "name": "", + "description": "", + "image": "" + }, + "Search files (Yara)": { + "name": "", + "description": "", + "image": "" + }, + "Correlate tickets": { + "name": "", + "description": "", + "image": "" + }, + "Honeypot access": { + "name": "", + "description": "", + "image": "", + "S3 Honeypot": { + "name": "", + "description": "", + "image": "" + }, + "SSH Honeypot": { + "name": "", + "description": "", + "image": "" + }, + "FTP honeypot": { + "name": "", + "description": "", + "image": "" + }, + "Network honeypot": { + "name": "", + "description": "", + "image": "" + }, + "...": { + "name": "", + "description": "", + "image": "" + } + } + }, + "Verify": { + "Discover vulnerabilities": { + "name": "", + "description": "", + "image": "" + }, + "Discover assets": { + "name": "", + "description": "", + "image": "" + }, + "Ensure policies are followed": { + "name": "", + "description": "", + "image": "" + }, + "Find Inactive users": { + "name": "", + "description": "", + "image": "" + }, + "Ensure access rights match HR systems": { + "name": "", + "description": "", + "image": "" + }, + "Ensure onboarding is followed": { + "name": "", + "description": "", + "image": "" + }, + "Third party apps in SaaS": { + "name": "", + "description": "", + "image": "" + }, + "Devices used for your cloud account": { + "name": "", + "description": "", + "image": "" + }, + "Too much access in GCP/Azure/AWS/ other clouds": { + "name": "", + "description": "", + "image": "" + }, + "Certificate validation": { + "name": "", + "description": "", + "image": "" + }, + "Monitor new DNS entries for domain with passive DNS": { + "name": "", + "description": "", + "image": "" + }, + "Monitor and track password dumps": { + "name": "", + "description": "", + "image": "" + }, + "Monitor for mentions of domain on darknet sites": { + "name": "", + "description": "", + "image": "" + }, + "Reporting": { + "name": "", + "description": "", + "image": "", + "Automation time saved": { + "name": "", + "description": "", + "image": "" + }, + "Automation money saved": { + "name": "", + "description": "", + "image": "" + }, + "Incident response report": { + "name": "", + "description": "", + "image": "" + }, + "Department cost": { + "name": "", + "description": "", + "image": "" + }, + "Monthly reports": { + "name": "", + "description": "", + "image": "", + "EDR alerts": { + "name": "", + "description": "", + "image": "" + }, + "SIEM alerts": { + "name": "", + "description": "", + "image": "" + }, + "Emails quarantined": { + "name": "", + "description": "", + "image": "" + }, + "...": { + "name": "", + "description": "", + "image": "" + } + } + } + }, + "4. Respond": { + "Eradicate malware": { + "name": "", + "description": "", + "image": "" + }, + "Quarantine host(s)": { + "name": "", + "description": "", + "image": "" + }, + "Trigger scans": { + "name": "", + "description": "", + "image": "" + }, + "Update indicators (FW, EDR, SIEM...)": { + "name": "", + "description": "", + "image": "" + }, + "Autoblock activity when threat intel is received": { + "name": "", + "description": "", + "image": "" + }, + "Lock/Delete/Reset account": { + "name": "", + "description": "", + "image": "" + }, + "Lock vault": { + "name": "", + "description": "", + "image": "" + }, + "Increase authentication": { + "name": "", + "description": "", + "image": "" + }, + "Get policies from assets": { + "name": "", + "description": "", + "image": "" + } + }, + "2. Enrich": { + "Internal Enrichment": { + "name": "", + "description": "", + "image": "", + "Users": { + "name": "", + "description": "", + "image": "" + }, + "Hostnames": { + "name": "", + "description": "", + "image": "" + }, + "IPs": { + "name": "", + "description": "", + "image": "" + }, + "Departments": { + "name": "", + "description": "", + "image": "" + }, + "Role": { + "name": "", + "description": "", + "image": "" + }, + "Software": { + "name": "", + "description": "", + "image": "" + }, + "...": { + "name": "", + "description": "", + "image": "" + } + }, + "External historical Enrichment": { + "name": "", + "description": "", + "image": "", + "IPs": { + "name": "", + "description": "", + "image": "" + }, + "URLs": { + "name": "", + "description": "", + "image": "" + }, + "Hashes": { + "name": "", + "description": "", + "image": "" + }, + "Files": { + "name": "", + "description": "", + "image": "" + }, + "...": { + "name": "", + "description": "", + "image": "" + } + }, + "Realtime": { + "name": "", + "description": "", + "image": "", + "File detonation": { + "name": "", + "description": "", + "image": "" + }, + "URL detonation": { + "name": "", + "description": "", + "image": "" + }, + "PCAP analysis": { + "name": "", + "description": "", + "image": "" + }, + "Analyze screenshots": { + "name": "", + "description": "", + "image": "" + } + }, + "Ticketing webhook verification": { + "name": "", + "description": "", + "image": "" + } + } +} \ No newline at end of file diff --git a/functions/usecases/categories.rtf b/functions/usecases/categories.rtf new file mode 100644 index 00000000..e66d5fbf --- /dev/null +++ b/functions/usecases/categories.rtf @@ -0,0 +1,560 @@ +{\rtf1\ansi\deff0\deflang2057\plain\fs24\fet1 +{\fonttbl +{\f0\froman Arial;} +} +{\info +{\createim\yr2022\mo2\dy18\hr17\min49} +} + +\paperw11907\paperh16840\margl1800\margr1800\margt1440\margb1440 +\slmult0\ltrpar\li0 +{\b\fs28 +Shuffle categories +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +1. Collect & Distribute +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +2-way Ticket synchronization +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Email management +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Attachments +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Manage senders +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Manage URLs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Encode & Decode URLs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Release a quarantined message +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +EDR to ticket +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Fetch incidents & events +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Quarantine files +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Quarantine host (respond) +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get host information +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +SIEM to ticket +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +ChatOps +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Threat Intel received +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Domain investigation with LetsEncrypt +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Botnet tracker +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Get running containers +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Assign tickets +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Firewall alerts +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Block/accept policies +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Add addresses and ports to groups +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Support custom URL categories +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Fetch logs for specific address +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +URL filtering +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +IDS/IPS alerts +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get/Fetch alerts +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Receive alerts real-time +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get PCAP files +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Get network logs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Manage policies +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Deduplicate information +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Correlate information +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +3. Detect +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search SIEM (Sigma) +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Network +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Endpoint +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search EDR (OSQuery) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search emails (Phish) +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Check malware +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Check targeted +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Check headers and IOCs +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search IOCs (ioc-finder) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Search files (Yara) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Correlate tickets +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Honeypot access +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +S3 Honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +SSH Honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +FTP honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Network honeypot +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +rich +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +Verify +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Discover vulnerabilities +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Discover assets +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ensure policies are followed +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Find Inactive users +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ensure access rights match HR systems +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ensure onboarding is followed +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Third party apps in SaaS +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Devices used for your cloud account +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Too much access in GCP/Azure/AWS/ other clouds +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Certificate validation +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Monitor new DNS entries for domain with passive DNS +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Monitor and track password dumps +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Monitor for mentions of domain on darknet sites +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Reporting +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Automation time saved +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Automation money saved +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Incident response report +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Department cost +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Monthly reports +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +EDR alerts +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +SIEM alerts +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +Emails quarantined +} +\par\pard\plain +\slmult0\ltrpar\li800 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +4. Respond +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Eradicate malware +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Quarantine host(s) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Trigger scans +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Update indicators (FW, EDR, SIEM...) +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Autoblock activity when threat intel is received +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Lock/Delete/Reset account +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Lock vault +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Increase authentication +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Trigger scans +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Get policies from assets +} +\par\pard\plain +\slmult0\ltrpar\li200 +{\fs24 +2. Enrich +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Internal Enrichment +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Users +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Hostnames +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +IPs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Departments +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Role +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Software +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +External historical Enrichment +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +IPs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +URLs +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Hashes +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Files +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +... +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Realtime +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +File detonation +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +URL detonation +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +PCAP analysis +} +\par\pard\plain +\slmult0\ltrpar\li600 +{\fs24 +Analyze screenshots +} +\par\pard\plain +\slmult0\ltrpar\li400 +{\fs24 +Ticketing webhook verification +} +\par\pard\plain +} \ No newline at end of file diff --git a/functions/usecases/read_categories.py b/functions/usecases/read_categories.py new file mode 100644 index 00000000..c99c4f01 --- /dev/null +++ b/functions/usecases/read_categories.py @@ -0,0 +1,53 @@ +data = "" +with open("categories.rtf", "r") as tmp: + data = tmp.read() + +fixed_json = {} +linearity = 0 +heading = "" +subheading = "" +subsubheading = "" +for line in data.split("\n"): + if line == "rich": + continue + + if "li" in line: + lisplit = line.split("\\") + try: + linearity = int(lisplit[-1][2]) + except: + pass + + #print("Linearity: %s" % linearity) + + if line.startswith("{") or line.startswith("}"): + continue + + if line.startswith("\\"): + continue + + if linearity == 0: + continue + + if linearity == 2: + fixed_json[line] = {} + heading = line + elif linearity == 4: + fixed_json[heading][line] = {"name": "", "description": "", "image": ""} + subheading = line + elif linearity == 6: + fixed_json[heading][subheading][line] = {"name": "", "description": "", "image": ""} + subsubheading = line + elif linearity == 8: + fixed_json[heading][subheading][subsubheading][line] = {"name": "", "description": "", "image": ""} + else: + print("No handler for %s" % line) + +#print(line) +#print(data) +import json +filename = "categories.json" +with open(filename, "w+") as tmp: + tmp.write(json.dumps(fixed_json, indent=4)) + +print("Wrote to file %s" % filename) From 698b4f54a50bcb90038cb1c9c2c1842816263c00 Mon Sep 17 00:00:00 2001 From: frikky Date: Sat, 19 Feb 2022 16:45:57 +0100 Subject: [PATCH 40/53] Fixed new caching system in worker to ensure failed apps retry multiple times --- functions/onprem/worker/build.sh | 2 +- functions/onprem/worker/worker.go | 29 ++++++++++++++++++++++++++--- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 6d482956..e828cc0e 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.55 +VERSION=0.9.59 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 390a2012..9b634e90 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -358,6 +358,7 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] log.Printf("\n\n[DEBUG] Result for %s already found - returning\n\n", newExecId) return nil } + cacheData := []byte("1") err = shuffle.SetCache(ctx, newExecId, cacheData) if err != nil { @@ -371,17 +372,17 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] waitTime := time.Duration(action.ExecutionDelay) * time.Second time.AfterFunc(waitTime, func() { - DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution) + DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) }) } else { log.Printf("[DEBUG] Running app %s in docker NORMALLY as there is no delay set", action.Name) - return DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution) + return DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) } return nil } -func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *container.Config, hostConfig *container.HostConfig, identifier string, workflowExecution shuffle.WorkflowExecution) error { +func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *container.Config, hostConfig *container.HostConfig, identifier string, workflowExecution shuffle.WorkflowExecution, newExecId string) error { cont, err := cli.ContainerCreate( ctx, config, @@ -396,6 +397,11 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Container CREATE error (1): %s", err) + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + return err } else { parsedUuid := uuid.NewV4() @@ -414,6 +420,11 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (2): %s", err) + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + return err } @@ -445,6 +456,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (3): %s", err) + + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + return err } @@ -454,6 +471,12 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) + + err = shuffle.DeleteCache(ctx, newExecId) + if err != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + } + //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) return err } From ce0bfa7f89a8ab6fce38cd67bbbc921b9adbade3 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 20 Feb 2022 03:39:29 +0100 Subject: [PATCH 41/53] Added usecase reference mapping for workflows, allowing easier discovery --- backend/app_sdk/app_base.py | 2 +- backend/app_sdk/build.sh | 2 +- backend/go-app/go.mod | 2 +- backend/go-app/main.go | 5 +- frontend/src/views/AngularWorkflow.jsx | 2 +- frontend/src/views/Dashboard.jsx | 389 ++++++++++++- frontend/src/views/Workflows.jsx | 290 ++++++++-- functions/onprem/worker/worker.go | 32 +- functions/usecases/categories.json | 732 ++++++++----------------- functions/usecases/categories.rtf | 9 +- functions/usecases/read_categories.py | 25 +- 11 files changed, 907 insertions(+), 583 deletions(-) diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 34105383..9d8e6592 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -823,7 +823,7 @@ class AppBase: returns = [] for item in value: self.logger.info("VALUE: %s" % item) - if len(item) != 36: + if len(item) != 36 and not item.startswith("file_"): self.logger.info("Bad length for file value %s" % item) continue #return { diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 20372b62..a8be1c76 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.58 +VERSION=0.9.59 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 86e33d11..09a4b486 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.16 -//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 5cd17cd7..a8b52797 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5875,10 +5875,13 @@ func initHandlers() { r.HandleFunc("/api/v1/apps/authentication/{appauthId}/config", shuffle.SetAuthenticationConfig).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/apps/authentication/{appauthId}", shuffle.DeleteAppAuthentication).Methods("DELETE", "OPTIONS") - // Related to + // Related to NFT things r.HandleFunc("/api/v1/workflows/collections/load", shuffle.LoadCollections).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/collections/{key}", shuffle.HandleGetCollection).Methods("GET", "OPTIONS") + // Related to use-cases that are not directly workflows. + r.HandleFunc("/api/v1/workflows/usecases", shuffle.LoadUsecases).Methods("GET", "OPTIONS") + // Legacy app things r.HandleFunc("/api/v1/workflows/apps/validate", validateAppInput).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/apps", getWorkflowApps).Methods("GET", "OPTIONS") diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index 952cb4d1..09ef01d4 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -1351,7 +1351,7 @@ const AngularWorkflow = (defaultprops) => { // This can be used to only show prioritzed ones later // Right now, it can prioritize authenticated ones //"Testing", - const internalIds = ["Shuffle Tools", "http"]; + const internalIds = ["Shuffle Tools", "http", "email"]; const getAppAuthentication = (reset, updateAction) => { fetch(globalUrl + "/api/v1/apps/authentication", { diff --git a/frontend/src/views/Dashboard.jsx b/frontend/src/views/Dashboard.jsx index 8a2986d8..ee9327ec 100644 --- a/frontend/src/views/Dashboard.jsx +++ b/frontend/src/views/Dashboard.jsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useState, useEffect } from "react"; import { useInterval } from "react-powerhooks"; // nodejs library that concatenates classes import classNames from "classnames"; @@ -32,18 +32,316 @@ import { RadialPointSeries, RadialArea, RadialLine, + TreeMap, + TreeMapSeries, + TreeMapLabel, + TreeMapRect, } from 'reaviz'; -const keys = [ - { key: '1. Collect & Distribute', data: 2 }, - { key: '2. Enrich', data: 11 }, - { key: '3. Detect', data: 3 }, - { key: '4. Respond', data: 4 }, - { key: '5. Report', data: 12 }, - { key: '6. Validate', data: 7, color: "red",}, +const categorydata = [ + { + "name": "1. Collect & Distribute", + "list": [ + { + "name": "2-way Ticket synchronization", + "items": {} + }, + { + "name": "Email management", + "items": { + "name": "Release a quarantined message", + "items": {} + } + }, + { + "name": "EDR to ticket", + "items": { + "name": "Get host information", + "items": {} + } + }, + { + "name": "SIEM to ticket", + "items": {} + }, + { + "name": "ChatOps", + "items": {} + }, + { + "name": "Threat Intel received", + "items": {} + }, + { + "name": "Domain investigation with LetsEncrypt", + "items": {} + }, + { + "name": "Botnet tracker", + "items": {} + }, + { + "name": "Get running containers", + "items": {} + }, + { + "name": "Assign tickets", + "items": {} + }, + { + "name": "Firewall alerts", + "items": { + "name": "URL filtering", + "items": {} + } + }, + { + "name": "IDS/IPS alerts", + "items": { + "name": "Manage policies", + "items": {} + } + }, + { + "name": "Deduplicate information", + "items": {} + }, + { + "name": "Correlate information", + "items": {} + } + ] + }, + { + "name": "3. Detect", + "list": [ + { + "name": "Search SIEM (Sigma)", + "items": { + "name": "Endpoint", + "items": {} + } + }, + { + "name": "Search EDR (OSQuery)", + "items": {} + }, + { + "name": "Search emails (Phish)", + "items": { + "name": "Check headers and IOCs", + "items": {} + } + }, + { + "name": "Search IOCs (ioc-finder)", + "items": {} + }, + { + "name": "Search files (Yara)", + "items": {} + }, + { + "name": "Correlate tickets", + "items": {} + }, + { + "name": "Honeypot access", + "items": { + "name": "...", + "items": {} + } + } + ] + }, + { + "name": "Verify", + "list": [ + { + "name": "Discover vulnerabilities", + "items": {} + }, + { + "name": "Discover assets", + "items": {} + }, + { + "name": "Ensure policies are followed", + "items": {} + }, + { + "name": "Find Inactive users", + "items": {} + }, + { + "name": "Ensure access rights match HR systems", + "items": {} + }, + { + "name": "Ensure onboarding is followed", + "items": {} + }, + { + "name": "Third party apps in SaaS", + "items": {} + }, + { + "name": "Devices used for your cloud account", + "items": {} + }, + { + "name": "Too much access in GCP/Azure/AWS/ other clouds", + "items": {} + }, + { + "name": "Certificate validation", + "items": {} + }, + { + "name": "Monitor new DNS entries for domain with passive DNS", + "items": {} + }, + { + "name": "Monitor and track password dumps", + "items": {} + }, + { + "name": "Monitor for mentions of domain on darknet sites", + "items": {} + }, + { + "name": "Reporting", + "items": { + "name": "Monthly reports", + "items": { + "name": "...", + "items": {} + } + } + } + ] + }, + { + "name": "4. Respond", + "list": [ + { + "name": "Eradicate malware", + "items": {} + }, + { + "name": "Quarantine host(s)", + "items": {} + }, + { + "name": "Trigger scans", + "items": {} + }, + { + "name": "Update indicators (FW, EDR, SIEM...)", + "items": {} + }, + { + "name": "Autoblock activity when threat intel is received", + "items": {} + }, + { + "name": "Lock/Delete/Reset account", + "items": {} + }, + { + "name": "Lock vault", + "items": {} + }, + { + "name": "Increase authentication", + "items": {} + }, + { + "name": "Trigger scans", + "items": {} + }, + { + "name": "Get policies from assets", + "items": {} + } + ] + }, + { + "name": "2. Enrich", + "list": [ + { + "name": "Internal Enrichment", + "items": { + "name": "...", + "items": {} + } + }, + { + "name": "External historical Enrichment", + "items": { + "name": "...", + "items": {} + } + }, + { + "name": "Realtime", + "items": { + "name": "Analyze screenshots", + "items": {} + } + }, + { + "name": "Ticketing webhook verification", + "items": {} + } + ] + } ] -const RadialChart = () => { +const TreeChart = ({keys}) => { + const [hovered, setHovered] = useState(""); + + return ( +
{ + console.log("Click: ", hovered) + }}> + { + return info.color + }} + label={ + + } + rect={ + { + console.log("Click: ", event) + }} + /> + } + /> + } + /> +
+ ) + //axis={} +} + + +const RadialChart = ({keys}) => { const [hovered, setHovered] = useState(""); return ( @@ -137,11 +435,70 @@ const Dashboard = (props) => { const [stats, setStats] = useState({}); const [changeme, setChangeme] = useState(""); const [statsRan, setStatsRan] = useState(false); + const [keys, setKeys] = useState([]) + const [treeKeys, setTreeKeys] = useState([]) + + //const keys = [ + // { key: '1. Collect & Distribute', data: 2 }, + // { key: '2. Enrich', data: 11 }, + // { key: '3. Detect', data: 3 }, + // { key: '4. Respond', data: 4 }, + // { key: 'Validation', data: 7, color: "red",}, + //] document.title = "Shuffle - dashboard"; var dayGraphLabels = [60, 80, 65, 130, 80, 105, 90, 130, 70, 115, 60, 130]; var dayGraphData = [60, 80, 65, 130, 80, 105, 90, 130, 70, 115, 60, 130]; + const handleKeysetting = (categorydata) => { + var allCategories = [] + var treeCategories = [] + for (key in categorydata) { + const category = categorydata[key] + console.log("cat: ", category) + allCategories.push({"key": category.name, "data": category.list.length,}) + treeCategories.push({"key": category.name, "data": 100, "color": category.color,}) + for (var subkey in category.list) { + treeCategories.push({"key": category.list[subkey].name, "data": 20, "color": category.color}) + } + } + + setKeys(allCategories) + setTreeKeys(treeCategories) + } + + const fetchUsecases = () => { + fetch(globalUrl + "/api/v1/workflows/usecases", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for usecases"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false) { + console.log("Usecases: ", responseJson) + handleKeysetting(responseJson) + } + }) + .catch((error) => { + //alert.error("ERROR: " + error.toString()); + console.log("ERROR: " + error.toString()); + }); + }; + + useEffect(() => { + fetchUsecases() + }, []); + const fetchdata = (stats_id) => { fetch(globalUrl + "/api/v1/stats/" + stats_id, { method: "GET", @@ -291,8 +648,8 @@ const Dashboard = (props) => { if (firstRequest) { console.log("HELO"); setFirstRequest(false); - start(); - runUpdate(); + //start(); + //runUpdate(); } else if (!statsRan) { // FIXME: Run this under runUpdate schedule? // 1. Fix labels in dayGraphy.data @@ -391,8 +748,14 @@ const Dashboard = (props) => { ) : null; const data = ( -
- +
+ {keys.length > 0 ? + + : null} + + {treeKeys.length > 0 ? + + : null} {/* { var upload = ""; const [workflows, setWorkflows] = React.useState([]); + const [_, setUpdate] = React.useState(""); // Used for rendering, don't remove + const [selectedUsecases, setSelectedUsecases] = React.useState([]); const [filteredWorkflows, setFilteredWorkflows] = React.useState([]); const [selectedWorkflow, setSelectedWorkflow] = React.useState({}); const [workflowDone, setWorkflowDone] = React.useState(false); @@ -510,6 +520,8 @@ const Workflows = (props) => { const [actionImageList, setActionImageList] = React.useState([]); const [firstLoad, setFirstLoad] = React.useState(true); + const [showMoreClicked, setShowMoreClicked] = React.useState(false); + const [usecases, setUsecases] = React.useState([]); const isCloud = window.location.host === "localhost:3002" || @@ -869,8 +881,10 @@ const Workflows = (props) => { .then((responseJson) => { if (responseJson !== undefined) { setWorkflows(responseJson); + fetchUsecases(responseJson) if (responseJson !== undefined) { + var actionnamelist = []; var parsedactionlist = []; for (var key in responseJson) { @@ -910,6 +924,83 @@ const Workflows = (props) => { }); }; + const handleKeysetting = (categorydata, workflows) => { + console.log("Workflows: ", workflows) + //workflows[0].category = ["detect"] + //workflows[0].usecase_ids = ["Correlate tickets"] + + if (workflows !== undefined && workflows !== null) { + const newcategories = [] + for (var key in categorydata) { + var category = categorydata[key] + category.matches = [] + + for (var subcategorykey in category.list) { + var subcategory = category.list[subcategorykey] + subcategory.matches = [] + + for (var workflowkey in workflows) { + const workflow = workflows[workflowkey] + + if (workflow.usecase_ids !== undefined && workflow.usecase_ids !== null) { + for (var usecasekey in workflow.usecase_ids) { + if (workflow.usecase_ids[usecasekey].toLowerCase() === subcategory.name.toLowerCase()) { + console.log("Got match: ", workflow.usecase_ids[usecasekey]) + + category.matches.push({ + "workflow": workflow.id, + "category": subcategory.name, + }) + subcategory.matches.push(workflow.id) + break + } + } + } + + if (subcategory.matches.length > 0) { + break + } + } + } + + newcategories.push(category) + } + + console.log("Categories: ", newcategories) + setUsecases(newcategories) + } else { + setUsecases(categorydata) + } + } + + const fetchUsecases = (workflows) => { + fetch(globalUrl + "/api/v1/workflows/usecases", { + method: "GET", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + credentials: "include", + }) + .then((response) => { + if (response.status !== 200) { + console.log("Status not 200 for usecases"); + } + + return response.json(); + }) + .then((responseJson) => { + if (responseJson.success !== false) { + console.log("Usecases: ", responseJson) + handleKeysetting(responseJson, workflows) + } + }) + .catch((error) => { + //alert.error("ERROR: " + error.toString()); + console.log("ERROR: " + error.toString()); + }); + }; + // eslint-disable-next-line react-hooks/exhaustive-deps useEffect(() => { if (workflows.length <= 0) { @@ -918,7 +1009,6 @@ const Workflows = (props) => { setView(tmpView); } - //setFirstrequest(false); getAvailableWorkflows(); } }, []) @@ -1392,6 +1482,11 @@ const Workflows = (props) => { if (data.tags !== undefined && data.tags !== null) { setNewWorkflowTags(JSON.parse(JSON.stringify(data.tags))); } + + console.log("Editing: ", data) + if (data.usecase_ids !== undefined && data.usecase_ids !== null && data.usecase_ids.length > 0) { + setSelectedUsecases(data.usecase_ids) + } }} key={"change"} > @@ -1785,7 +1880,8 @@ const Workflows = (props) => { tags, defaultReturnValue, editingWorkflow, - redirect + redirect, + currentUsecases, ) => { var method = "POST"; var extraData = ""; @@ -1812,6 +1908,12 @@ const Workflows = (props) => { workflowdata["default_return_value"] = defaultReturnValue; } + if (currentUsecases !== undefined && currentUsecases !== null) { + workflowdata["usecase_ids"] = currentUsecases + //workflows[0].category = ["detect"] + //workflows[0].usecase_ids = ["Correlate tickets"] + } + return fetch(globalUrl + "/api/v1/workflows" + extraData, { method: method, headers: { @@ -2289,6 +2391,7 @@ const Workflows = (props) => { return
{workflowData}
; }; + var total_count = 0 const modalView = modalOpen ? ( { }} color="primary" placeholder="Name" + required margin="dense" defaultValue={newWorkflowName} autoFocus @@ -2346,47 +2450,115 @@ const Workflows = (props) => { color="primary" defaultValue={newWorkflowDescription} placeholder="Description" - rows="3" multiline margin="dense" fullWidth /> - { - newWorkflowTags.push(chip); - setNewWorkflowTags(newWorkflowTags); - }} - onDelete={(chip, index) => { - newWorkflowTags.splice(index, 1); - setNewWorkflowTags(newWorkflowTags); - }} - /> +
+ { + newWorkflowTags.push(chip); + setNewWorkflowTags(newWorkflowTags); + }} + onDelete={(chip, index) => { + newWorkflowTags.splice(index, 1); + setNewWorkflowTags(newWorkflowTags); + }} + /> + + Usecases + + +
+ + {showMoreClicked ? + + setDefaultReturnValue(event.target.value)} + InputProps={{ + style: { + color: "white", + }, + }} + color="primary" + defaultValue={defaultReturnValue} + placeholder="Default return value (used for Subflows if the subflow fails)" + rows="3" + multiline + margin="dense" + fullWidth + /> + + : null} + + { + setShowMoreClicked(!showMoreClicked); + }} + > + {showMoreClicked ? : } + + - setDefaultReturnValue(event.target.value)} - InputProps={{ - style: { - color: "white", - }, - }} - color="primary" - defaultValue={defaultReturnValue} - placeholder="Default return value (used for Subflows if the subflow fails)" - rows="3" - multiline - margin="dense" - fullWidth - /> diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index 9b634e90..e3d988ef 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -375,8 +375,10 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) }) } else { - log.Printf("[DEBUG] Running app %s in docker NORMALLY as there is no delay set", action.Name) - return DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) + log.Printf("[DEBUG] Running app %s in docker NORMALLY as there is no delay set with identifier %s", action.Name, identifier) + returnvalue := DeployContainer(ctx, cli, config, hostConfig, identifier, workflowExecution, newExecId) + log.Printf("[DEBUG] Normal deploy ret: %s", returnvalue) + return returnvalue } return nil @@ -397,9 +399,9 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if !strings.Contains(err.Error(), "Conflict. The container name") { log.Printf("[ERROR] Container CREATE error (1): %s", err) - err = shuffle.DeleteCache(ctx, newExecId) - if err != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) } return err @@ -420,9 +422,10 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (2): %s", err) - err = shuffle.DeleteCache(ctx, newExecId) - if err != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) } return err @@ -457,9 +460,9 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Container create error (3): %s", err) - err = shuffle.DeleteCache(ctx, newExecId) - if err != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) } return err @@ -472,9 +475,9 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont if err != nil { log.Printf("[ERROR] Failed to start container in environment %s: %s", environment, err) - err = shuffle.DeleteCache(ctx, newExecId) - if err != nil { - log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, err) + cacheErr := shuffle.DeleteCache(ctx, newExecId) + if cacheErr != nil { + log.Printf("[ERROR] FAILED Deleting cache for %s: %s", newExecId, cacheErr) } //shutdown(workflowExecution, workflowExecution.Workflow.ID, true) @@ -1352,6 +1355,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } else { err = deployApp(dockercli, images[0], identifier, env, workflowExecution, action) + log.Printf("[DEBUG] Failed deploying app? %s", err) if err != nil && !strings.Contains(err.Error(), "Conflict. The container name") { if strings.Contains(err.Error(), "exited prematurely") { log.Printf("[DEBUG] Shutting down (9)") diff --git a/functions/usecases/categories.json b/functions/usecases/categories.json index f32d9700..e438a380 100644 --- a/functions/usecases/categories.json +++ b/functions/usecases/categories.json @@ -1,522 +1,260 @@ -{ - "1. Collect & Distribute": { - "2-way Ticket synchronization": { - "name": "", - "description": "", - "image": "" - }, - "Email management": { - "name": "", - "description": "", - "image": "", - "Attachments": { - "name": "", - "description": "", - "image": "" +[ + { + "name": "1. Collect & Distribute", + "color": "#c51152", + "list": [ + { + "name": "2-way Ticket synchronization", + "items": {} }, - "Manage senders": { - "name": "", - "description": "", - "image": "" + { + "name": "Email management", + "items": { + "name": "Release a quarantined message", + "items": {} + } }, - "Manage URLs": { - "name": "", - "description": "", - "image": "" + { + "name": "EDR to ticket", + "items": { + "name": "Get host information", + "items": {} + } }, - "Encode & Decode URLs": { - "name": "", - "description": "", - "image": "" + { + "name": "SIEM to ticket", + "items": {} }, - "Release a quarantined message": { - "name": "", - "description": "", - "image": "" + { + "name": "ChatOps", + "items": {} + }, + { + "name": "Threat Intel received", + "items": {} + }, + { + "name": "Domain investigation with LetsEncrypt", + "items": {} + }, + { + "name": "Botnet tracker", + "items": {} + }, + { + "name": "Get running containers", + "items": {} + }, + { + "name": "Assign tickets", + "items": {} + }, + { + "name": "Firewall alerts", + "items": { + "name": "URL filtering", + "items": {} + } + }, + { + "name": "IDS/IPS alerts", + "items": { + "name": "Manage policies", + "items": {} + } + }, + { + "name": "Deduplicate information", + "items": {} + }, + { + "name": "Correlate information", + "items": {} } - }, - "EDR to ticket": { - "name": "", - "description": "", - "image": "", - "Fetch incidents & events": { - "name": "", - "description": "", - "image": "" - }, - "Quarantine files": { - "name": "", - "description": "", - "image": "" - }, - "Quarantine host (respond)": { - "name": "", - "description": "", - "image": "" - }, - "Get host information": { - "name": "", - "description": "", - "image": "" - } - }, - "SIEM to ticket": { - "name": "", - "description": "", - "image": "" - }, - "ChatOps": { - "name": "", - "description": "", - "image": "" - }, - "Threat Intel received": { - "name": "", - "description": "", - "image": "" - }, - "Domain investigation with LetsEncrypt": { - "name": "", - "description": "", - "image": "" - }, - "Botnet tracker": { - "name": "", - "description": "", - "image": "" - }, - "Get running containers": { - "name": "", - "description": "", - "image": "" - }, - "Assign tickets": { - "name": "", - "description": "", - "image": "" - }, - "Firewall alerts": { - "name": "", - "description": "", - "image": "", - "Block/accept policies": { - "name": "", - "description": "", - "image": "" - }, - "Add addresses and ports to groups": { - "name": "", - "description": "", - "image": "" - }, - "Support custom URL categories": { - "name": "", - "description": "", - "image": "" - }, - "Fetch logs for specific address": { - "name": "", - "description": "", - "image": "" - }, - "URL filtering": { - "name": "", - "description": "", - "image": "" - } - }, - "IDS/IPS alerts": { - "name": "", - "description": "", - "image": "", - "Get/Fetch alerts": { - "name": "", - "description": "", - "image": "" - }, - "Receive alerts real-time": { - "name": "", - "description": "", - "image": "" - }, - "Get PCAP files": { - "name": "", - "description": "", - "image": "" - }, - "Get network logs": { - "name": "", - "description": "", - "image": "" - }, - "Manage policies": { - "name": "", - "description": "", - "image": "" - } - }, - "Deduplicate information": { - "name": "", - "description": "", - "image": "" - }, - "Correlate information": { - "name": "", - "description": "", - "image": "" - } + ] }, - "3. Detect": { - "Search SIEM (Sigma)": { - "name": "", - "description": "", - "image": "", - "Network": { - "name": "", - "description": "", - "image": "" + { + "name": "2. Enrich", + "color": "#f4c20d", + "list": [ + { + "name": "Internal Enrichment", + "items": { + "name": "...", + "items": {} + } }, - "Endpoint": { - "name": "", - "description": "", - "image": "" + { + "name": "External historical Enrichment", + "items": { + "name": "...", + "items": {} + } + }, + { + "name": "Realtime", + "items": { + "name": "Analyze screenshots", + "items": {} + } + }, + { + "name": "Ticketing webhook verification", + "items": {} } - }, - "Search EDR (OSQuery)": { - "name": "", - "description": "", - "image": "" - }, - "Search emails (Phish)": { - "name": "", - "description": "", - "image": "", - "Check malware": { - "name": "", - "description": "", - "image": "" - }, - "Check targeted": { - "name": "", - "description": "", - "image": "" - }, - "Check headers and IOCs": { - "name": "", - "description": "", - "image": "" - } - }, - "Search IOCs (ioc-finder)": { - "name": "", - "description": "", - "image": "" - }, - "Search files (Yara)": { - "name": "", - "description": "", - "image": "" - }, - "Correlate tickets": { - "name": "", - "description": "", - "image": "" - }, - "Honeypot access": { - "name": "", - "description": "", - "image": "", - "S3 Honeypot": { - "name": "", - "description": "", - "image": "" - }, - "SSH Honeypot": { - "name": "", - "description": "", - "image": "" - }, - "FTP honeypot": { - "name": "", - "description": "", - "image": "" - }, - "Network honeypot": { - "name": "", - "description": "", - "image": "" - }, - "...": { - "name": "", - "description": "", - "image": "" - } - } + ] }, - "Verify": { - "Discover vulnerabilities": { - "name": "", - "description": "", - "image": "" - }, - "Discover assets": { - "name": "", - "description": "", - "image": "" - }, - "Ensure policies are followed": { - "name": "", - "description": "", - "image": "" - }, - "Find Inactive users": { - "name": "", - "description": "", - "image": "" - }, - "Ensure access rights match HR systems": { - "name": "", - "description": "", - "image": "" - }, - "Ensure onboarding is followed": { - "name": "", - "description": "", - "image": "" - }, - "Third party apps in SaaS": { - "name": "", - "description": "", - "image": "" - }, - "Devices used for your cloud account": { - "name": "", - "description": "", - "image": "" - }, - "Too much access in GCP/Azure/AWS/ other clouds": { - "name": "", - "description": "", - "image": "" - }, - "Certificate validation": { - "name": "", - "description": "", - "image": "" - }, - "Monitor new DNS entries for domain with passive DNS": { - "name": "", - "description": "", - "image": "" - }, - "Monitor and track password dumps": { - "name": "", - "description": "", - "image": "" - }, - "Monitor for mentions of domain on darknet sites": { - "name": "", - "description": "", - "image": "" - }, - "Reporting": { - "name": "", - "description": "", - "image": "", - "Automation time saved": { - "name": "", - "description": "", - "image": "" + { + "name": "3. Detect", + "color": "#3cba54", + "list": [ + { + "name": "Search SIEM (Sigma)", + "items": { + "name": "Endpoint", + "items": {} + } }, - "Automation money saved": { - "name": "", - "description": "", - "image": "" + { + "name": "Search EDR (OSQuery)", + "items": {} }, - "Incident response report": { - "name": "", - "description": "", - "image": "" + { + "name": "Search emails (Phish)", + "items": { + "name": "Check headers and IOCs", + "items": {} + } }, - "Department cost": { - "name": "", - "description": "", - "image": "" + { + "name": "Search IOCs (ioc-finder)", + "items": {} }, - "Monthly reports": { - "name": "", - "description": "", - "image": "", - "EDR alerts": { - "name": "", - "description": "", - "image": "" - }, - "SIEM alerts": { - "name": "", - "description": "", - "image": "" - }, - "Emails quarantined": { - "name": "", - "description": "", - "image": "" - }, - "...": { - "name": "", - "description": "", - "image": "" + { + "name": "Search files (Yara)", + "items": {} + }, + { + "name": "Correlate tickets", + "items": {} + }, + { + "name": "Honeypot access", + "items": { + "name": "...", + "items": {} } } - } + ] }, - "4. Respond": { - "Eradicate malware": { - "name": "", - "description": "", - "image": "" - }, - "Quarantine host(s)": { - "name": "", - "description": "", - "image": "" - }, - "Trigger scans": { - "name": "", - "description": "", - "image": "" - }, - "Update indicators (FW, EDR, SIEM...)": { - "name": "", - "description": "", - "image": "" - }, - "Autoblock activity when threat intel is received": { - "name": "", - "description": "", - "image": "" - }, - "Lock/Delete/Reset account": { - "name": "", - "description": "", - "image": "" - }, - "Lock vault": { - "name": "", - "description": "", - "image": "" - }, - "Increase authentication": { - "name": "", - "description": "", - "image": "" - }, - "Get policies from assets": { - "name": "", - "description": "", - "image": "" - } + { + "name": "4. Respond", + "color": "#4a148c", + "list": [ + { + "name": "Eradicate malware", + "items": {} + }, + { + "name": "Quarantine host(s)", + "items": {} + }, + { + "name": "Trigger scans", + "items": {} + }, + { + "name": "Update indicators (FW, EDR, SIEM...)", + "items": {} + }, + { + "name": "Autoblock activity when threat intel is received", + "items": {} + }, + { + "name": "Lock/Delete/Reset account", + "items": {} + }, + { + "name": "Lock vault", + "items": {} + }, + { + "name": "Increase authentication", + "items": {} + }, + { + "name": "Get policies from assets", + "items": {} + } + ] }, - "2. Enrich": { - "Internal Enrichment": { - "name": "", - "description": "", - "image": "", - "Users": { - "name": "", - "description": "", - "image": "" + { + "name": "5. Verify", + "color": "#4885ed", + "list": [ + { + "name": "Discover vulnerabilities", + "items": {} }, - "Hostnames": { - "name": "", - "description": "", - "image": "" + { + "name": "Discover assets", + "items": {} }, - "IPs": { - "name": "", - "description": "", - "image": "" + { + "name": "Ensure policies are followed", + "items": {} }, - "Departments": { - "name": "", - "description": "", - "image": "" + { + "name": "Find Inactive users", + "items": {} }, - "Role": { - "name": "", - "description": "", - "image": "" + { + "name": "Ensure access rights match HR systems", + "items": {} }, - "Software": { - "name": "", - "description": "", - "image": "" + { + "name": "Ensure onboarding is followed", + "items": {} }, - "...": { - "name": "", - "description": "", - "image": "" + { + "name": "Third party apps in SaaS", + "items": {} + }, + { + "name": "Devices used for your cloud account", + "items": {} + }, + { + "name": "Too much access in GCP/Azure/AWS/ other clouds", + "items": {} + }, + { + "name": "Certificate validation", + "items": {} + }, + { + "name": "Monitor new DNS entries for domain with passive DNS", + "items": {} + }, + { + "name": "Monitor and track password dumps", + "items": {} + }, + { + "name": "Monitor for mentions of domain on darknet sites", + "items": {} + }, + { + "name": "Reporting", + "items": { + "name": "Monthly reports", + "items": { + "name": "...", + "items": {} + } + } } - }, - "External historical Enrichment": { - "name": "", - "description": "", - "image": "", - "IPs": { - "name": "", - "description": "", - "image": "" - }, - "URLs": { - "name": "", - "description": "", - "image": "" - }, - "Hashes": { - "name": "", - "description": "", - "image": "" - }, - "Files": { - "name": "", - "description": "", - "image": "" - }, - "...": { - "name": "", - "description": "", - "image": "" - } - }, - "Realtime": { - "name": "", - "description": "", - "image": "", - "File detonation": { - "name": "", - "description": "", - "image": "" - }, - "URL detonation": { - "name": "", - "description": "", - "image": "" - }, - "PCAP analysis": { - "name": "", - "description": "", - "image": "" - }, - "Analyze screenshots": { - "name": "", - "description": "", - "image": "" - } - }, - "Ticketing webhook verification": { - "name": "", - "description": "", - "image": "" - } + ] } -} \ No newline at end of file +] \ No newline at end of file diff --git a/functions/usecases/categories.rtf b/functions/usecases/categories.rtf index e66d5fbf..e19cf26a 100644 --- a/functions/usecases/categories.rtf +++ b/functions/usecases/categories.rtf @@ -3,7 +3,7 @@ {\f0\froman Arial;} } {\info -{\createim\yr2022\mo2\dy18\hr17\min49} +{\createim\yr2022\mo2\dy20\hr1\min15} } \paperw11907\paperh16840\margl1800\margr1800\margt1440\margb1440 @@ -279,7 +279,7 @@ rich \par\pard\plain \slmult0\ltrpar\li200 {\fs24 -Verify +5. Verify } \par\pard\plain \slmult0\ltrpar\li400 @@ -444,11 +444,6 @@ Increase authentication \par\pard\plain \slmult0\ltrpar\li400 {\fs24 -Trigger scans -} -\par\pard\plain -\slmult0\ltrpar\li400 -{\fs24 Get policies from assets } \par\pard\plain diff --git a/functions/usecases/read_categories.py b/functions/usecases/read_categories.py index c99c4f01..65df43c9 100644 --- a/functions/usecases/read_categories.py +++ b/functions/usecases/read_categories.py @@ -2,11 +2,16 @@ data = "" with open("categories.rtf", "r") as tmp: data = tmp.read() -fixed_json = {} +fixed_json = [] linearity = 0 heading = "" subheading = "" subsubheading = "" + +cnt = -1 +subcnt = -1 + +colors = ["#c51152", "#3cba54", "#4885ed", "#4a148c", "#f4c20d"] for line in data.split("\n"): if line == "rich": continue @@ -30,16 +35,23 @@ for line in data.split("\n"): continue if linearity == 2: - fixed_json[line] = {} + #if cnt >= 0: + # for key, value in fixed_json[cnt].items(): + # print(key, value) + + + cnt += 1 + subcnt = -1 + fixed_json.append({"name": line, "color": colors[cnt], "list": []}) heading = line elif linearity == 4: - fixed_json[heading][line] = {"name": "", "description": "", "image": ""} subheading = line + fixed_json[cnt]["list"].append({"name": line, "items": {}}) + subcnt += 1 elif linearity == 6: - fixed_json[heading][subheading][line] = {"name": "", "description": "", "image": ""} - subsubheading = line + fixed_json[cnt]["list"][subcnt]["items"] = {"name": line, "items": {}} elif linearity == 8: - fixed_json[heading][subheading][subsubheading][line] = {"name": "", "description": "", "image": ""} + fixed_json[cnt]["list"][subcnt]["items"]["items"] = {"name": line, "items": {}} else: print("No handler for %s" % line) @@ -47,6 +59,7 @@ for line in data.split("\n"): #print(data) import json filename = "categories.json" +fixed_json.sort(key=lambda x: x["name"]) with open(filename, "w+") as tmp: tmp.write(json.dumps(fixed_json, indent=4)) From 36f9ed26c279c3f1875ff16f51a84307eac3ec13 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 20 Feb 2022 19:03:58 +0100 Subject: [PATCH 42/53] #709: Added usecase discovery and mapping on /usecases path --- frontend/src/App.jsx | 4 +- frontend/src/components/Header.js | 11 ++ frontend/src/views/Dashboard.jsx | 152 ++++++++++++--- frontend/src/views/GettingStarted.jsx | 7 +- frontend/src/views/SettingsPage.jsx | 29 +-- frontend/src/views/Workflows.jsx | 271 ++++++++++++++------------ 6 files changed, 301 insertions(+), 173 deletions(-) diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 5448777e..1fe81088 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -85,7 +85,7 @@ const App = (message, props) => { !window.location.pathname.startsWith("/docs") && !window.location.pathname.startsWith("/detectionframework") && !window.location.pathname.startsWith("/adminsetup") && - !window.location.pathname.startsWith("/dashboard") + !window.location.pathname.startsWith("/usecases") ) { window.location = "/login"; } @@ -407,7 +407,7 @@ const App = (message, props) => { /> { Get Started + { + event.preventDefault(); + handleClose(); + }} + > + + Use Cases + + { event.preventDefault(); diff --git a/frontend/src/views/Dashboard.jsx b/frontend/src/views/Dashboard.jsx index ee9327ec..1cb9f4a7 100644 --- a/frontend/src/views/Dashboard.jsx +++ b/frontend/src/views/Dashboard.jsx @@ -10,15 +10,18 @@ import { useAlert } from "react-alert"; import { Typography, + Grid, + Paper, + Chip, } from "@material-ui/core"; // core components -import { - chartExample1, - chartExample2, - chartExample3, - chartExample4, -} from "../charts.js"; +//import { +// chartExample1, +// chartExample2, +// chartExample3, +// chartExample4, +//} from "../charts.js"; import { RadialBarChart, @@ -298,6 +301,47 @@ const categorydata = [ } ] +const UsecaseListComponent = ({keys}) => { + if (keys === undefined || keys === null || keys.length === 0) { + return null + } + + return ( +
+ + Shuffle usecases + + + Usecases in Shuffle are divided into {keys.length} type{keys.length === 1 ? "" : "s"}. + + {keys.map((usecase, index) => { + return ( +
+ + {usecase.name} + + + {usecase.list.map((subcase, subindex) => { + return ( + + { + console.log("Clicked: ", subcase.name) + }}> + + {subcase.name} + + + + ) + })} + +
+ ) + })} +
+ ) +} + const TreeChart = ({keys}) => { const [hovered, setHovered] = useState(""); @@ -307,8 +351,6 @@ const TreeChart = ({keys}) => { }}> { } -const RadialChart = ({keys}) => { +const RadialChart = ({keys, setSelectedCategory}) => { const [hovered, setHovered] = useState(""); return (
{ console.log("Click: ", hovered) + if (setSelectedCategory !== undefined) { + setSelectedCategory(hovered) + } }}> { data={keys} axis={} series={ - { + console.log("Color: ", colorInput) + return '#f86a3e' + }} animated={false} id="workflow_series_id" style={{cursor: "pointer",}} @@ -381,7 +428,6 @@ const RadialChart = ({keys}) => { }} isRadial={true} onValueEnter={(event) => { - console.log("Entered: ", event.value.x) if (hovered !== event.value.x) { setHovered(event.value.x) } @@ -394,7 +440,7 @@ const RadialChart = ({keys}) => { }} content={(data, color) => { return ( -
+
{data.x} @@ -438,15 +484,24 @@ const Dashboard = (props) => { const [keys, setKeys] = useState([]) const [treeKeys, setTreeKeys] = useState([]) - //const keys = [ - // { key: '1. Collect & Distribute', data: 2 }, - // { key: '2. Enrich', data: 11 }, - // { key: '3. Detect', data: 3 }, - // { key: '4. Respond', data: 4 }, - // { key: 'Validation', data: 7, color: "red",}, - //] + const [selectedUsecaseCategory, setSelectedUsecaseCategory] = useState(""); + const [selectedUsecases, setSelectedUsecases] = useState([]); + const [usecases, setUsecases] = useState([]); - document.title = "Shuffle - dashboard"; + useEffect(() => { + console.log("Changed: ", selectedUsecaseCategory) + if (selectedUsecaseCategory.length === 0) { + setSelectedUsecases(usecases) + } else { + const foundUsecase = usecases.find(data => data.name === selectedUsecaseCategory) + if (foundUsecase !== undefined && foundUsecase !== null) { + console.log("FOUND: ", foundUsecase) + setSelectedUsecases([foundUsecase]) + } + } + }, [selectedUsecaseCategory]) + + document.title = "Shuffle - usecases"; var dayGraphLabels = [60, 80, 65, 130, 80, 105, 90, 130, 70, 115, 60, 130]; var dayGraphData = [60, 80, 65, 130, 80, 105, 90, 130, 70, 115, 60, 130]; @@ -455,8 +510,7 @@ const Dashboard = (props) => { var treeCategories = [] for (key in categorydata) { const category = categorydata[key] - console.log("cat: ", category) - allCategories.push({"key": category.name, "data": category.list.length,}) + allCategories.push({"key": category.name, "data": category.list.length, "color": category.color}) treeCategories.push({"key": category.name, "data": 100, "color": category.color,}) for (var subkey in category.list) { treeCategories.push({"key": category.list[subkey].name, "data": 20, "color": category.color}) @@ -487,6 +541,8 @@ const Dashboard = (props) => { if (responseJson.success !== false) { console.log("Usecases: ", responseJson) handleKeysetting(responseJson) + setUsecases(responseJson) + setSelectedUsecases(responseJson) } }) .catch((error) => { @@ -748,20 +804,52 @@ const Dashboard = (props) => { ) : null; const data = ( -
- {keys.length > 0 ? - +
+
+ {keys.length > 0 ? + + : null} +
+ + {usecases !== null && usecases !== undefined && usecases.length > 0 ? +
+ {usecases.map((usecase, index) => { + return ( + { + console.log("Clicked: ", usecase.name) + if (selectedUsecaseCategory === usecase.name) { + setSelectedUsecaseCategory("") + } else { + setSelectedUsecaseCategory(usecase.name) + } + //addFilter(usecase.name.slice(3,usecase.name.length)) + }} + variant="outlined" + color="primary" + /> + ) + })} +
: null} + + {treeKeys.length > 0 ? : null} - {/* - - */} {newdata}
diff --git a/frontend/src/views/GettingStarted.jsx b/frontend/src/views/GettingStarted.jsx index 03c6f280..0821ac7e 100644 --- a/frontend/src/views/GettingStarted.jsx +++ b/frontend/src/views/GettingStarted.jsx @@ -2160,7 +2160,7 @@ const GettingStarted = (props) => { }) } }}> - Discover your apps by following our simple security model! + Find relevant apps and start your automation journey ), tutorial: "find_integrations", @@ -2168,7 +2168,8 @@ const GettingStarted = (props) => { { html: - Discover { + Discover Use Case ideas and  + { if (isCloud) { navigate(`/search?tab=workflows`) @@ -2193,7 +2194,7 @@ const GettingStarted = (props) => { alert.success("TBD: Coming in version 1.0.0"); } }}> - use-cases made by other creators! + workflows made by other creators! , tutorial: "discover_workflows", }, diff --git a/frontend/src/views/SettingsPage.jsx b/frontend/src/views/SettingsPage.jsx index 3d218761..46325bb4 100644 --- a/frontend/src/views/SettingsPage.jsx +++ b/frontend/src/views/SettingsPage.jsx @@ -731,17 +731,22 @@ const Settings = (props) => {
{isCloud ? - + + + By connecting your Github account, you agree to our Terms of Service, and acknowledge that your non-sensitive data will be turned into a creator account. This enables you to earn a passive income from Shuffle. This IS reversible. + + + : null}
@@ -861,7 +866,7 @@ const Settings = (props) => { handleEthereumConnection(); }} > - Authenticate + Authenticate Metamask Wallet )}
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 98b8ee0c..d09ed42b 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -2475,58 +2475,60 @@ const Workflows = (props) => { setNewWorkflowTags(newWorkflowTags); }} /> - - Usecases - selected.join(', ')} + onChange={(event) => { + console.log("Changed: ", event) + }} + > + + None + + {usecases.map((usecase, index) => { + console.log(usecase) + return ( + + + {usecase.name} + + {usecase.list.map((subcase, subindex) => { + //console.log(subcase) + total_count += 1 + return ( + { + if (selectedUsecases.includes(subcase.name)) { + const itemIndex = selectedUsecases.indexOf(subcase.name) + if (itemIndex > -1) { + selectedUsecases.splice(itemIndex, 1) + } + } else { + selectedUsecases.push(subcase.name) } - } else { - selectedUsecases.push(subcase.name) - } - setUpdate(Math.random()); - setSelectedUsecases(selectedUsecases) - }}> - - - - ) - })} - - ) - })} - - + setUpdate(Math.random()); + setSelectedUsecases(selectedUsecases) + }}> + + + + ) + })} + + ) + })} + + + : null}
{showMoreClicked ? @@ -2643,7 +2645,7 @@ const Workflows = (props) => { {view === "list" && ( ) : ( -
+ {isMobile ? + + + + + + : null} +
); }; @@ -10315,7 +10442,7 @@ const AngularWorkflow = (defaultprops) => { return null; } - const boxSize = 100; + const boxSize = isMobile ? 50 : 100; const executionButton = executionRunning ? ( @@ -10327,7 +10454,7 @@ const AngularWorkflow = (defaultprops) => { abortExecution(); }} > - + @@ -10345,7 +10472,7 @@ const AngularWorkflow = (defaultprops) => { executeWorkflow(executionText, workflow.start, lastSaved); }} > - + @@ -10356,13 +10483,17 @@ const AngularWorkflow = (defaultprops) => { {executionButton}
- {workflow.public ? null : ( + {isMobile || workflow.public ? null : ( { return null; } + var defaultReturn = null if (Object.getOwnPropertyNames(selectedAction).length > 0) { if (Object.getOwnPropertyNames(selectedAction).length === 0) { return null; } - return ( - -
- -
- - ); + defaultReturn = + } else if (Object.getOwnPropertyNames(selectedComment).length > 0) { - return ( -
- -
- ); + defaultReturn = } else if (Object.getOwnPropertyNames(selectedTrigger).length > 0) { if (selectedTrigger.trigger_type === "SCHEDULE") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "WEBHOOK") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "SUBFLOW") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "EMAIL") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === "USERINPUT") { - return ( -
- -
- ); + defaultReturn = } else if (selectedTrigger.trigger_type === undefined) { + //defaultReturn = return null; } else { console.log( @@ -10696,14 +10800,46 @@ const AngularWorkflow = (defaultprops) => { return null; } } else if (Object.getOwnPropertyNames(selectedEdge).length > 0) { - return ( -
- -
- ); + defaultReturn = } - return null; + const iOS = typeof navigator !== 'undefined' && /iPad|iPhone|iPod/.test(navigator.userAgent); + + const drawerBleeding = 56; + return ( + isMobile ? + { + console.log("Close!") + //setRightSideBarOpen(false) + cy.elements().unselect() + }} + disableSwipeToOpen={false} + ModalProps={{ + keepMounted: true, + }} + PaperProps={{ + style: { + maxHeight: "70%", + overflow: "auto", + } + }} + > + {defaultReturn} + + : + +
+ {defaultReturn} +
+ + ); + + //return null; }; // This can execute a workflow with firestore. Used for test, as datastore is old and stuff @@ -10850,17 +10986,25 @@ const AngularWorkflow = (defaultprops) => {
: null}
+ : isMobile && leftViewOpen ? +
+ +
: leftViewOpen ? ( -
- -
-) : ( +
+ +
+ ) : (
{ var executionDelay = -75 const executionModal = ( - setExecutionModalOpen(false)} + onClose={() => { + setExecutionModalOpen(false) + }} style={{ resize: "both", overflow: "auto", zIndex: 10005 }} hideBackdrop={false} variant="temporary" @@ -11300,8 +11446,8 @@ const parsedExecutionArgument = () => { style: { resize: "both", overflow: "auto", - minWidth: 420, - maxWidth: 420, + minWidth: isMobile ? "100%" : 420, + maxWidth: isMobile ? "100%" : 420, backgroundColor: "#1F2023", color: "white", fontSize: 18, @@ -11310,8 +11456,25 @@ const parsedExecutionArgument = () => { }, }} > + {isMobile ? + + { + e.preventDefault(); + setExecutionModalOpen(false) + }} + > + + + + : null} {executionModalView === 0 ? ( -
+
{ )}
) : ( -
+
{ )}
)} - + ); // This sucks :) @@ -12194,7 +12357,7 @@ const parsedExecutionArgument = () => { pointerEvents: "auto", backgroundColor: inputColor, color: "white", - minWidth: 650, + minWidth: isMobile ? "90%" : 650, padding: 30, maxHeight: 550, overflowY: "auto", @@ -12444,6 +12607,7 @@ const parsedExecutionArgument = () => {
+ {/*isMobile ? null : leftView*/} {leftView} {workflow.id === undefined || workflow.id === null || @@ -12476,7 +12640,7 @@ const parsedExecutionArgument = () => { maxZoom={2.0} wheelSensitivity={0.25} style={{ - width: bodyWidth - leftBarSize - 25, + width: cytoscapeWidth, height: bodyHeight - appBarSize - 5, backgroundColor: surfaceColor, }} @@ -12529,7 +12693,7 @@ const parsedExecutionArgument = () => { requiresAuthentication={requiresAuthentication} /> - +
); @@ -12563,6 +12727,7 @@ const parsedExecutionArgument = () => { backgroundColor: surfaceColor, color: "white", border: theme.palette.defaultBorder, + maxWidth: "100%", }, }} > @@ -12718,6 +12883,7 @@ const parsedExecutionArgument = () => { backgroundColor: surfaceColor, color: "white", border: theme.palette.defaultBorder, + maxWidth: isMobile ? bodyWidth-100 : "100%", }, }} > @@ -13308,10 +13474,10 @@ const parsedExecutionArgument = () => { style={{ flex: 2, padding: 0, - minHeight: 650, - maxHeight: 650, + minHeight: isMobile ? "90%" : 650, + maxHeight: isMobile ? "90%" : 650, overflowY: "auto", - overflowX: "hidden", + overflowX: isMobile ? "auto" : "hidden", }} > {authenticationType.type === "oauth2" ? ( diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 434a9287..9ccf18b5 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -6,6 +6,7 @@ import { Navigate } from "react-router-dom"; import SecurityFramework from '../components/SecurityFramework.jsx'; import { ShepherdTour, ShepherdTourContext } from 'react-shepherd' +import { isMobile } from "react-device-detect" import { @@ -1017,8 +1018,8 @@ const Workflows = (props) => { color: "#ffffff", width: "100%", display: "flex", - minWidth: 1024, - maxWidth: 1024, + minWidth: isMobile ? "100%" : 1024, + maxWidth: isMobile ? "100%" : 1024, margin: "auto", }; @@ -1038,6 +1039,7 @@ const Workflows = (props) => { flexDirection: "column", }; + //flexDirection: !isMobile ? "column" : "row", const paperAppContainer = { display: "flex", flexWrap: "wrap", @@ -1382,7 +1384,7 @@ const Workflows = (props) => { }; return ( - + { style: { backgroundColor: surfaceColor, color: "white", - minWidth: "800px", + minWidth: isMobile ? "90%" : "800px", + maxWidth: isMobile ? "90%" : "800px", }, }} > @@ -2905,30 +2908,32 @@ const Workflows = (props) => {
*/} -
-
- { - addFilter(chip); - }} - onDelete={(_, index) => { - removeFilter(index); - }} - /> + {isMobile ? null : +
+
+ { + addFilter(chip); + }} + onDelete={(_, index) => { + removeFilter(index); + }} + /> +
-
+ }
{workflowButtons}
@@ -2982,8 +2987,8 @@ const Workflows = (props) => { */}
- {usecases !== null && usecases !== undefined && usecases.length > 0 ? -
+ {!isMobile && usecases !== null && usecases !== undefined && usecases.length > 0 ? +
{usecases.map((usecase, index) => { //console.log(usecase) return ( @@ -3136,7 +3141,7 @@ const Workflows = (props) => { workflowDelay += 75 } else { return ( - + ) @@ -3144,7 +3149,7 @@ const Workflows = (props) => { return ( - + @@ -3355,7 +3360,7 @@ const Workflows = (props) => { */} 1366 ? 1366 : 1200, + maxWidth: window.innerWidth > 1366 ? 1366 : isMobile ? "100%" : 1200, margin: "auto", padding: 20, }}