diff --git a/.env b/.env index c962701d..e139fdda 100644 --- a/.env +++ b/.env @@ -60,6 +60,7 @@ SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true SHUFFLE_LOGS_DISABLED=false SHUFFLE_CHAT_DISABLED=false +SHUFFLE_RERUN_SCHEDULE=300 # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 diff --git a/backend/app_sdk/Dockerfile_ubuntu b/backend/app_sdk/Dockerfile_ubuntu index 228d5a41..3f34d4bd 100644 --- a/backend/app_sdk/Dockerfile_ubuntu +++ b/backend/app_sdk/Dockerfile_ubuntu @@ -13,6 +13,9 @@ COPY requirements.txt /requirements.txt RUN pip install --prefix="/install" -r /requirements.txt FROM base +RUN apt-get update +RUN apt-get dist-upgrade -y +RUN apt install build-essential libffi-dev musl-dev openssl python3 python3-pip -y COPY --from=builder /install /usr/local COPY __init__.py /app/walkoff_app_sdk/__init__.py diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index 5ec71be8..3c42c42e 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -273,7 +273,7 @@ class AppBase: self.logger.warning(f"[INFO] Action result ran with Magic parser output.") action_result["result"] = self.run_magic_parser(action_result["result"]) else: - self.logger.warning(f"[ERROR] Magic output not defined.") + self.logger.warning(f"[WARNING] Magic output not defined.") except KeyError as e: self.logger.warning(f"[DEBUG] Failed to run magic autoparser (send result) - keyerror: {e}") except Exception as e: diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 4200ca10..22416f8b 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -2,7 +2,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.69 +VERSION=0.9.70 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly @@ -21,7 +21,11 @@ docker build . -f Dockerfile_ubuntu -t frikky/shuffle:app_sdk_ubuntu -t frikky/$ docker push frikky/shuffle:app_sdk_ubuntu docker push ghcr.io/frikky/$NAME:$VERSION - +#### Alpine GRPC +NAME=shuffle-app_sdk_grpc +docker build . -f Dockerfile_alpine_grpc -t frikky/shuffle:app_sdk_grpc -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION +docker push frikky/shuffle:app_sdk_grpc +docker push ghcr.io/frikky/$NAME:$VERSION diff --git a/backend/go-app/main.go b/backend/go-app/main.go index f4149c3a..9447c1cb 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -9,6 +9,7 @@ import ( "bytes" "context" "crypto/md5" + "strconv" //"crypto/tls" //"crypto/x509" @@ -1029,6 +1030,7 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) { } } + chatDisabled := false if os.Getenv("SHUFFLE_CHAT_DISABLED") == "true" { chatDisabled = true } @@ -4083,6 +4085,19 @@ func runInitEs(ctx context.Context) { // FIXME: Have this for all envs in all orgs (loop and find). if len(parsedApikey) > 0 { cleanupSchedule := 300 + + if len(os.Getenv("SHUFFLE_RERUN_SCHEDULE")) > 0 { + newfrequency, err := strconv.Atoi(os.Getenv("SHUFFLE_RERUN_SCHEDULE")) + if err == nil { + cleanupSchedule = newfrequency + + if cleanupSchedule < 300 { + log.Printf("[WARNING] A Cleanupschedule of less than 300 seconds won't help.") + cleanupSchedule = 300 + } + } + } + environments := []string{"Shuffle"} log.Printf("[DEBUG] Starting schedule setup for execution cleanup every %d seconds. Running first immediately.", cleanupSchedule) cleanupJob := func() func() { diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 20b43e8d..ffdf9b4f 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -321,8 +321,8 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) { } } - if len(executionRequests.Data) > 10 { - executionRequests.Data = executionRequests.Data[0:9] + if len(executionRequests.Data) > 50 { + executionRequests.Data = executionRequests.Data[0:49] } } @@ -895,8 +895,8 @@ func handleExecution(id string, workflow shuffle.Workflow, request *http.Request workflowExecution, execInfo, _, err := shuffle.PrepareWorkflowExecution(ctx, workflow, request, 10) if err != nil { - log.Printf("[WARNING] Failed in prepareExecution: %s", err) - return shuffle.WorkflowExecution{}, fmt.Sprintf("Failed preparration: %s", err), err + log.Printf("[WARNING] Failed in prepareExecution for execution Id %s: %s", workflowExecution.ExecutionId, err) + return workflowExecution, fmt.Sprintf("Failed preparration: %s", err), err } err = imageCheckBuilder(execInfo.ImageNames) @@ -1134,7 +1134,6 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { user.ActiveOrg.Users = []shuffle.UserMini{} workflow.ExecutingOrg = user.ActiveOrg workflowExecution, executionResp, err := handleExecution(fileId, *workflow, request) - if err == nil { resp.WriteHeader(200) resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization))) @@ -1142,7 +1141,7 @@ func executeWorkflow(resp http.ResponseWriter, request *http.Request) { } resp.WriteHeader(500) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, executionResp))) + resp.Write([]byte(fmt.Sprintf(`{"success": false, "execution_id": "%s", "authorization": "%s", "reason": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization, executionResp))) } func stopSchedule(resp http.ResponseWriter, request *http.Request) { diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 72764b33..e1b800a4 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -84,6 +84,7 @@ const App = (message, props) => { !isLoggedIn && !window.location.pathname.startsWith("/login") && !window.location.pathname.startsWith("/docs") && + !window.location.pathname.startsWith("/support") && !window.location.pathname.startsWith("/detectionframework") && !window.location.pathname.startsWith("/appframework") && !window.location.pathname.startsWith("/adminsetup") && @@ -562,6 +563,19 @@ const App = (message, props) => { /> } /> + + } + /> { // This is the data FROM the database, not what's being saved const parseIncomingOpenapiData = (data) => { - console.log("Data: ", data) var parsedDecoded = "" try { const decoded = base64_decode(data.openapi) diff --git a/frontend/src/views/Docs.jsx b/frontend/src/views/Docs.jsx index 91d5ba3f..8dba1aa2 100644 --- a/frontend/src/views/Docs.jsx +++ b/frontend/src/views/Docs.jsx @@ -41,6 +41,11 @@ const hrefStyle = { textDecoration: "none", }; +const hrefStyle2 = { + color: "#f86a3e", + textDecoration: "none", +}; + const innerHrefStyle = { color: "rgba(255, 255, 255, 0.75)", textDecoration: "none", @@ -57,7 +62,6 @@ const Docs = (defaultprops) => { var props = JSON.parse(JSON.stringify(defaultprops)) props.match = {} props.match.params = params - console.log("Props: ", props.match.params) useEffect(() => { //if (params["key"] === undefined) { @@ -231,7 +235,6 @@ const Docs = (defaultprops) => { if (!serverside) { fetchDocList(); - console.log("PROPS: ", props) //const propkey = props.match.params.key //if (propkey === undefined) { // navigate("/docs/about") @@ -552,8 +555,13 @@ const Docs = (defaultprops) => { const [hover, setHover] = useState(false); + console.log("Link: ", link) + if (link === undefined || link === null) { + return null + } + return ( - +
{ setHover(true) @@ -570,6 +578,115 @@ const Docs = (defaultprops) => { ) } + const headerStyle = { + marginTop: 25, + } + + const mainpageInfo = +
+ + Documentation + +
+ link="https://support.shuffler.io" /> + link="https://discord.gg/B2CBzUm" /> +
+ +
+ Tutorial + + Dive in. Hands-on is the best approach to see how Shuffle can transform your security operations. Our set of tutorials and videos teach you how to build your skills. Check out the getting started section to give it a go! + + + Why Shuffle? + + Security first. We incentivize trying before buying, and give you the full set of tools you need to automate your operations. What's more is we also help you find usecases that fit your your unique needs. Accessibility is key, and we intend to help every SOC globally use and share their usecases. + + + Get help + + Our promise is to make it easier and easier to automate your operations. In some cases however, it may be good with a helping hand. That's where Shuffle's consultancy and support services come in handy. We help you build and automate your operational processes to a level you haven't seen before with the help of our usecases. + + + APIs + + Learn. We're all about learning, and are continuously creating documentation and video tutorials to better understand how to get started. APIs are an extremely important part of how the internet works today, and our goal is helping every security professional learn about them. + + + Workflow building + + Build. Creating workflows has never been easier. Jump into things with our getting Started section and build to your hearts content. Workflows make it all come together, with an easy to use area. + + + Managing Shuffle + + Organize. Whether an organization of 1000 or 1, management tools are necessary. In Shuffle we offer full user management, MFA and single-signon options, multi-tenancy and a lot more - for free! + +
+ + {/* + + {list.map((data, index) => { + const item = data.name; + if (item === undefined) { + return null; + } + + const path = "/docs/" + item; + const newname = + item.charAt(0).toUpperCase() + + item.substring(1).split("_").join(" ").split("-").join(" "); + + const itemMatching = props.match.params.key === undefined ? false : + props.match.params.key.toLowerCase() === item.toLowerCase(); + + return ( + + + + ) + })} + + */} + + {/* + { + console.log("Change: ", event.target.value) + }} + /> + */} +
+ const postDataBrowser = list === undefined || list === null ? null : (
@@ -641,75 +758,7 @@ const Docs = (defaultprops) => {
{props.match.params.key === undefined ? -
- - Documentation - -
- link="http://172.17.14.113" /> - link="https://discord.gg/B2CBzUm" /> -
- - - {list.map((data, index) => { - const item = data.name; - if (item === undefined) { - return null; - } - - const path = "/docs/" + item; - const newname = - item.charAt(0).toUpperCase() + - item.substring(1).split("_").join(" ").split("-").join(" "); - - const itemMatching = props.match.params.key === undefined ? false : - props.match.params.key.toLowerCase() === item.toLowerCase(); - - return ( - - - - ) - })} - - - {/* - { - console.log("Change: ", event.target.value) - }} - /> - */} -
+ mainpageInfo :
{ })}
-
- -
+ {props.match.params.key === undefined ? + mainpageInfo + : +
+ +
+ } { : null}
- {actionImageList !== undefined && + {!isMobile && + actionImageList !== undefined && actionImageList !== null && actionImageList.length > 0 ? (
workerTimeout { go zombiecheck(ctx, workerTimeout)