diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index e7604c5e..f6a5a83e 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -1755,6 +1755,10 @@ class AppBase: self.logger.info("[DEBUG] Skipping liquid - size too big (%d)" % len(template)) return template + if "${" in template and "}$" in template: + self.logger.info("[DEBUG] Shuffle loop shouldn't run in liquid. Data length: %d" % len(template)) + return template + #if not "{{" in template or not "}}" in template: # if not "{%" in template or not "%}" in template: # self.logger.info("Skipping liquid - missing {{ }} and {% %}") diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 0eeeb7c7..198177c4 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.49 +VERSION=0.9.50 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 2eef7eb3..9e9473c5 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -5804,6 +5804,8 @@ func initHandlers() { r.HandleFunc("/api/v1/workflows/download_remote", loadSpecificWorkflows).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/execute", executeWorkflow).Methods("GET", "POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/schedule/{schedule}", stopSchedule).Methods("DELETE", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/stream", shuffle.HandleStreamWorkflow).Methods("GET", "OPTIONS") + r.HandleFunc("/api/v1/workflows/{key}/stream", shuffle.HandleStreamWorkflowUpdate).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", deleteWorkflow).Methods("DELETE", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", shuffle.SaveWorkflow).Methods("PUT", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}", shuffle.GetSpecificWorkflow).Methods("GET", "OPTIONS") diff --git a/frontend/src/App.jsx b/frontend/src/App.jsx index 7f048a56..d27cdc39 100644 --- a/frontend/src/App.jsx +++ b/frontend/src/App.jsx @@ -26,6 +26,7 @@ import SetAuthenticationSSO from "./views/SetAuthenticationSSO"; import LandingPageNew from "./views/LandingpageNew"; import LoginPage from "./views/LoginPage"; import SettingsPage from "./views/SettingsPage"; +import KeepAlive from "./views/KeepAlive.jsx"; import MyView from "./views/MyView"; @@ -572,6 +573,22 @@ const App = (message, props) => { /> } /> + + } + /> { const [expansionModalOpen, setExpansionModalOpen] = React.useState(false); const [hideBody, setHideBody] = React.useState(true); - const [activateHidingBody, setActivateHidingBody] = React.useState(false); + const [activateHidingBodyButton, setActivateHidingBodyButton] = React.useState(false); + const [codedata, setcodedata] = React.useState(""); const [fieldCount, setFieldCount] = React.useState(0); const [hiddenDescription, setHiddenDescription] = React.useState(true); + useEffect(() => { + + //if (data.startsWith("${") && data.endsWith("}")) { + //} + // PARAM FIX - Gonna use the ID field, even though it's a hack + const paramcheck = selectedAction.parameters.find(param => param.name === "body") + console.log("LOADED! Change hideBody based on input? Action: ", selectedAction, paramcheck) + if (paramcheck !== undefined && paramcheck !== null) { + if (paramcheck.id === "TOGGLED"){ + setHideBody(false) + setActivateHidingBodyButton(false) + console.log("TOGGLED BODY!") + } else { + setHideBody(true) + + if (paramcheck.id === "UNTOGGLED") { + setActivateHidingBodyButton(false) + console.log("UNTOGGLED!") + } + } + } + }, []) + const keywords = [ "len(", "lower(", @@ -521,7 +545,7 @@ const ParsedAction = (props) => { }); const changeActionParameter = (event, count, data) => { - console.log("Action change: ", selectedAction, data) + //console.log("Action change: ", selectedAction, data) if (data.name.startsWith("${") && data.name.endsWith("}")) { // PARAM FIX - Gonna use the ID field, even though it's a hack const paramcheck = selectedAction.parameters.find( @@ -533,6 +557,7 @@ const ParsedAction = (props) => { .trim() .replaceAll('\\"', '"') .replaceAll('"', '\\"'); + console.log("REPLACE WITH: ", toReplace); if ( paramcheck["value_replace"] === undefined || @@ -1155,14 +1180,25 @@ const ParsedAction = (props) => { color: theme.palette.primary.secondary, }} onChange={(event) => { - setHideBody(!hideBody); + var tag = "TOGGLED" + if (hideBody) { + tag = "UNTOGGLED" + } + setHideBody(!hideBody); + for (var key in selectedActionParameters) { var currentItem = selectedActionParameters[key]; if (currentItem.name === "ssl_verify") { } + if (currentItem.name === "body") { + // FIXME: Workaround for toggling, as actions don't have IDs. + // May screw up something in the future. + currentItem.id = tag + } + if (currentItem.description === openApiFieldDesc) { currentItem.field_active = !hideBody; console.log("Changing", currentItem); @@ -1176,7 +1212,7 @@ const ParsedAction = (props) => { /> - ); + ) if (selectedApp.generated && data.name === "body") { const regex = /\${(\w+)}/g; @@ -1184,10 +1220,10 @@ const ParsedAction = (props) => { hideBodyButton = hideBodyButtonValue; if (found === null || !hideBody) { - //setExtraBodyFields([]) + console.log("Should hide body? ", found) // if (found === null) { - setActivateHidingBody(true); + setActivateHidingBodyButton(true); } } else { //console.log("SHOW BUTTON"); @@ -1240,7 +1276,7 @@ const ParsedAction = (props) => { } } - if (activateHidingBody === true) { + if (activateHidingBodyButton === true) { hideBodyButton = ""; } @@ -2777,6 +2813,19 @@ const ParsedAction = (props) => { ); }} renderInput={(params) => { + if (params.inputProps !== undefined && params.inputProps !== null && params.inputProps.value !== undefined && params.inputProps.value !== null) { + const prefixes = ["Post", "Put", "Patch"] + for (var key in prefixes) { + if (params.inputProps.value.startsWith(prefixes[key])) { + params.inputProps.value = params.inputProps.value.replace(prefixes[key]+" ", "", -1) + if (params.inputProps.value.length > 1) { + params.inputProps.value = params.inputProps.value.charAt(0).toUpperCase()+params.inputProps.value.substring(1) + } + break + } + } + } + return ( { } }; + const sendStreamRequest = (body) => { + console.log("Stream not activated yet.") + return + // Session may be important here huh + body.user_id = userdata.id + + fetch(globalUrl + "/api/v1/workflows/" + props.match.params.key + "/stream", { + method: "POST", + headers: { + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify(body), + credentials: "include", + }) + .then((response) => { + setSavingState(0); + if (response.status !== 200) { + console.log("Status not 200 for setting workflows :O!"); + } + + return response.json(); + }) + .then((responseJson) => { + console.log("RESP: ", responseJson) + }) + .catch((error) => { + console.log("Stream error: ", error.toString()) + //alert.error(error.toString()); + }) + + } + const saveWorkflow = (curworkflow, executionArgument, startNode) => { var success = false; @@ -1929,6 +1962,13 @@ const AngularWorkflow = (defaultprops) => { x: 0, y: 0, }; + + sendStreamRequest({ + "item": "node", + "type": "move", + "id": nodedata.id, + "location": {"x": event.target.position("x"), "y": event.target.position("y")} + }) }; const onNodeDrag = (event, selectedAction) => { @@ -2708,6 +2748,18 @@ const AngularWorkflow = (defaultprops) => { setLastSaved(false); const edge = event.target.data(); + const sourcenode = cy.getElementById(edge.source) + const destinationnode = cy.getElementById(edge.target) + if (sourcenode === undefined || sourcenode === null || destinationnode === undefined || destinationnode === null) { + } else { + const edgeCurve = calculateEdgeCurve(sourcenode.position(), destinationnode.position()) + const currentedge = cy.getElementById(edge.id) + if (currentedge !== undefined && currentedge !== null) { + currentedge.style('control-point-distance', edgeCurve.distance) + currentedge.style('control-point-weight', edgeCurve.weight) + } + } + var targetnode = workflow.triggers.findIndex( (data) => data.id === edge.target ); @@ -3460,6 +3512,7 @@ const AngularWorkflow = (defaultprops) => { if (el.isNode() && !el.data("isButton") && !el.data("isDescriptor") && + !el.data("isSuggestion") && el.data("type") !== "COMMENT") { return true } @@ -3476,7 +3529,7 @@ const AngularWorkflow = (defaultprops) => { cy.fit(null, 200); cy.on("boxselect", "node", (e) => { - if (e.target.data("isButton") || e.target.data("isDescriptor")) { + if (e.target.data("isButton") || e.target.data("isDescriptor") || e.target.data("isSuggestion")) { e.target.unselect(); } @@ -3662,6 +3715,41 @@ const AngularWorkflow = (defaultprops) => { }); }; + const addSuggestionButtons = (event) => { + var parentNode = cy.$("#" + event.target.data("id")); + if (parentNode.data("isButton") || parentNode.data("buttonId")) return; + + const px = parentNode.position("x") + 300; + const py = parentNode.position("y") + 0; + const circleId = (newNodeId = uuidv4()); + + parentNode.data("circleId", circleId); + + var appid = "1234" + var suggestions = [{ + app_name: "TheHive", + app_version: "1.1.0", + app_id: appid, + sharing: false, + private_id: false, + isStartNode: false, + label: "Suggestion 1", + large_image: "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAK4AAACuCAYAAACvDDbuAAAABGdBTUEAALGPC/xhBQAAACBjSFJNAAB6JgAAgIQAAPoAAACA6AAAdTAAAOpgAAA6mAAAF3CculE8AAAABmJLR0QA/wD/AP+gvaeTAAAAB3RJTUUH5AgXDjM6hEZGWwAAD+lJREFUeNrtXb/vJTcRH7/v3iVBCqRBiCAQAtHwq4AWRElHwX8AoqbmXwDRpiH/QyQkGoogUSAhKIKUAE1IdSRSREhQQk7c3XtD8X55vePxjNfe3bk3H+nu+96uPf54POtnj8fe8OQX30JwOIxhtzYBh6MGOsPF0z9p2iWwpd8LjX6W5vWUYaiqlBuvLT5b5TQDPlRwmMSAABBg+kCer+XuAeQf4tL9tAxJ/hIfZGSm8rhyEfjytfxr9FeSX+KjvVfipNVpWlaPNhsAEPCS7Ao8FYnRlbO4ksLnjiSQvIanv4FNjwJ5pXIlMq6MQpIqqPnQKQKbjuPDtZlG55o6UHXWtVncZZTbbNBVB1P5dJYguCbJJ1WjOG8PVOioSm5HPrVt1rwuyN+K+PSZnNV1M/MmEFubfFjjU9tmK9XBJ2cOk3DDdZiEG67DJOrGuA7HyvAe12ESAxa73KPrN1z8gUikCCdvcD5NXnpQpA8nNhh9m5Yn4ZMrV8dHV/8a/dRA0x419a3lI9GBtM2GcrGYFXRNUU5TyluTOpdXwqeUt6YOpby9DUTLZylOcRlzdBTf2yV3ZBFOmKSHQh5KpjSSSpqG4s6VkUubqw8W8knTSnWk0Y+2jF5tlmuDUloJn6T8gRVcEpJ+3srChHSNt8RJsq4p+S41LC13KTcu/RJt1pLPKY1Pzhwm4YbrMAk3XIdJTMe4aeCjJhBVk0YiQ1MWZHhLgmO5QNVWfKRlavlIIQnurQmcnaMjSbBxhtMwYUxODpLcl2tUhvPlNE6VkiuoFVLXKT6ZfBjxRIIzOSlgWpLSB8uZ0g3BjeVDlFGEos0mfKKL7CQrY2ES7pM2i/OX22w4/sWReEhEnUOTxx3a+FrawQGZh04/rWe6oJBKo5zT4zLjPHE9ZHym5YzToogzfQcmfLgOhuLF/Sjm2izVDyXnrKtcmmmdaKumf+RyCw5Xn7OmzQaJF0fiEZG6BjXpYUYaSVkaPrXeHe4eVaZEr3Prqrmmrbc2T8lrmOMjn5xJHeJLYkk+PfzNTxOflrwF0EeHbU0Zt2wsW+PTkncB7g5zmMSwzUfS4eDhPa7DJK5jXGorsnZxonbRIbeAoOUjkUvlp+qxFp9YNuWL0nBqsVCkqUsrHQnuX+Nx5/qcJDI0kWgtJh7ihYCN8aG+13DqOXlbWUfD+fN0AUEmp3RcUWlVEwCynb5ssYLnxHViJT6ULCykb8EnzUfpqBWfVAdcnt5tprGhIe10WnjHpB2FtMPWcpM66yXyOad4Lz4Srq34SHhwZfRos1w9Y/jkzGESvj3dYRLe4zpMwg3XYRJuuA6T4M/Hzfk/OGd9OP2HOE2f8wtBlCebJrkfp+Gc3AGmiSiuaVlpwkmajL4osPUm9FMqIzBOJolfjGuzEtdUwWl53Dm7Eh9pzIdps+FiYJyi1N+Rvs/6OLCQBul8Ip8R08ik3EwhLZz1Wv8XmU7ZZqX7OT2gUIB2oaRBm+2ovDm5nM+ulEeiD8yka8UnJ1PCP82r9YWW8iCU5XO8W/PhPmvllNKW7lEyszsgNKuzkspJFZFL15uPtIweq7A1xiKpz1J8tGXP+dE53/fJmcMk6hcgJO8XqokEKi5uYzTG29LqSev95JqyKsoOOxjNpKQBD7VFc5GBJRsi+NQHkkv6+7m/UxTufwLCCy+CbAruyOLDdwEf/uf6vbbNJukzlogZC6wMdhAcM7ohHPawe/GrcO+HPwe4u782G7sIAE9++0vYv/YKwO6usfCaka0etgwXAGB3D8JznwIYnlmbiW0M92FbQy0d+MmZ3Xo5JDDcvuXJ2ZYqtyUuTwuM6nSXctcufHCOZqkjPScXhbIcdeD0XUpfKyNNy8nlyhuozLkM8XxR6pjm7tc4Fdx620I7lWq10JCm0ZanWoBwm3FsBe1WznpadbTg4A9PI2xx7FUKHopQjg7TKqNnpbioIUcFUGUsy1CS8fFYBYdJuOE6TMIN12ESgyiKiwO1bQOJe1w+6p42Etmhwmi6kLZXfC2G9IUj2vulY2wIPrv4onRhIXcRqS0DiWxkhF0uIb37wG22LRCSuVCyekC2GSXj9CG3YyT+krWh+KPAhkTvgGDKqbqnWbBwY+2Pnm3Wy4aMRYc1MuPDvp0skwgAh8PaJGbh5k4kx0f/hce/ewnw/QenXQCTFJDfQy45PzFNn5NHsoPy/u6gzE+nObzz91P9Z+6kWAm2zg6bDMoq8OQxHN78Axze/htAaB1EbQhhdzyfgRqIGoCxoUIjhDuA3ZDpcR0W4C3nMInbNVw7v4oOAsehArVFPL0uOjMM+DlM+pk7t7/BDuwcJsM6gcM7WweOX05nFCHNi12ASRfLo3QaX9O0GWTylOTnZIMwf4YPPTlD4iMm7aZwAGOUf3Rf48wjHNzVOMkKFA8pp0RHZ1mjdihs5R61PWbsWlphgs/E5gptNvFfSLY8QPk7dVbh+UNg8qfnJsZ8Bo0hzF0Y2Nqvc0s+Vbs5YL5OLfPRcorT2hvjtuxyHWZhzHCX6AMcFtB2B0RvtKZqqe6OEYz1uA7HEbdruN7ZmsZtGq4brXnQhlsbLFkDrY9mC9giH41/dSlONfeEIBcgss7nXopInPdkYN95J3XD1bMgkJUNFOxsDNLgyiynhYyX5dnAhnLyhzmO4V7IO8+xyZEgx5UqvJ41rOUTdhBOr2w6KjZc+B1FBkLGVUoAABQEcmPu6rPPw73v/gh2n/wMANYEhAd4/NqvYf/Wn5pEyPW2IUrOzQWSHyHdkEJgN8D97/0Edp/7GgDu9fnDDvD9t+HRqy8BPvxQ9i6xEXUEuPcMDF//Puw+/aVqDewfvA77f/zx9M40e7jNeNw5CDu4++K34e4r36kWcXj3TYDfvwz8D79ml1clDPuxx9FhuUik0rblVihFWLX+7ZFEXE2ioLBNg9fUSRopVsOjJbioskZlDuyAvmflpOWsOUNu/cBQ8jW/1A0np11RG+GjwG36cQHqFWnBcG4Axgx37d/I1uXXcvCnx6BXoQXf3mOAzvVpooJzaOcWdKBH1fZ07dCsFZpNgmfZbaOJ2dxnpwkNFC3C9MBcGxo0OugxwV8LWKm5lg9sFQdszKGhLAla2dCuduuOZcypx+UXdk0OK5e/hXKNTc4cjiPGhtvTX1njI6Z2+vbuKtaKspLooXdkXs1u5yUR7/LdROMsraSSIfTa6pqWodE9Mvla6sCI8d7uUMEXIEzjdg3XYRr2osOePIbDR+9BGO7re78QAD/+AODwpK5sBDg6dGyGAtL1sYnLGDe3+2BNTNycYQf7B2/Aw5d/XB9HejjA4YN3jgHUNQ132MOTv/wG9v98A+CgFBCO/+FH/wJ89PBaSY1OULZzQyQL2skayVwg/7Dk3Ky2IlcEgEcfw/7dt+YJnRP1f9jDoz+/AvM0FU4c1u8mes59e+ZXDhXmPE+tForD+lH73Q6EluiozfaldnzWQUWQzdprPk87lg44nkTKN+DT/10S7lW4VYz8wWucOTAPtl5e4mgfjmu0/b3HdZiEG67DJNxwbxlGhwkAuZeXAJS3Qpfemq7dds1tS5dsbc6dAyQpS5uGe+lKrJLSGUqlCb2GcwUuCxBzt71T2/g7t9mQniofv0yjWOtMYdSLM6Sy0pd5iLdFSQtUyiJtRnjmGOdhqq5bo5WzUXAYzns2Lu2tjaqb0WaTHRBrR9cvEVG4VF3WkLsGnzXqohzjbk3dt4hG/jDDxy8BLL5y5miBZi1wa9vT14dJ0o2qft6/1GhQZ1SV9uJxd3cQ7j+XD7RJ40JK38/XAPKz4ly+OG+KwOTDwn0uDSKEZ58/vgH+hmHLcA97uPvCN+G5H/wMoCaQ/KkAAtzdg/DCZ9cmsipsGS4ce5u7z38DYHhmbTL2YfjBH28DOM80s+MoxllVvfkwKudSbiL0dB0NTya2iGpNYmIzl+/EdexjQ8PEGE4FhdPHMAlbLhcsdWaPnfDEAxQJnbx53TEPJ51j3N7CrEfbSNt+arzXt57X2RBx94LsUGHOGRQtF7Fa8HFQQOabJmc5XQ8b8iAbh0mYNFzvdefD+nRhyPowqWitc2VbRyutGCF18+ilU2mEXWX51zFuKbqlZ/RLy0gixzagiS6sgL2hghuwAywarsMBxgzXO9u2sBzZWHwHRLwrQ5rWYQBIfuwCKnZJEpvEYSg9dRoncnejtdxFbBRLqFQzr5fSudH3nDmOaH26yHIwNcZ1NIZNmwWArYU1Fg8HDLB/7wH879VfAey2Rd0a9g/+2ubUyZUOdAz//umXjT136GPd2cDNnM9bC4Pd1gbOx3WsDh/jOkzCDddhEpcjmKiFhvGLQwDitJNrYTz05H7MS+N56hiq0mbYCfeIj2STb2s+cSJEOrguJ4fScaneOW7kOWZJm4VCmaPFg8wKgcSGuLpzR49Rerm8vIRaaECgvyB1Tbl9qOZoMiykHeVhVoZKwW9N+CSJuPwsH4YY12aTa5TxYyZPpsxSDG/Rhgp1lyxUnK/7UMFhEm64DpNIlnzTAdXcsJml8rdO1yt/K+R45EJUluS9zHaWITuQJb9rsVT+HvuKe+RvhdIIcE3ey4Rj+VDBYRJuuA6TcMN1mMT15SWMZ5h10Oc86+dr50s14QWch7rEh5PHef+psgsyqB0iI2e+hE+pDlpvvkQ/uVUMDfdSnTq12TA58injFUdOMPB5AeiALtHcUrstXrqSINnaoVjxyE5ra1ZipHMsTV2kMiQ8NDw7tdmqQ4WtzNEd9uBjXIdJuOE6TMLoy0sct46KHndNS6d2pW5tp+rW+Jw5rVl2qpP5Oqrcnr52w9RMgbfA8db5tAsp8DGuwyTaGW6DB7ppn9CCzxKnvKz9Kz7j/prUi0cwqQLQDBtvrp5uvMc/Wf00oFAT5FjscbcwMloCt1LPWvTUT41sH+M6TMIN12ESw3UPd8gPtrh7JeTyXvZGn0KD0jSlMms5Sfhw92vkUvXT5tPWt3WbSfjMsSFl3ujlJdy+4xkjnFze+PWrNWXWclqaT6t82vq2bjMJnzk2pMzrQwWHSbjhOkzCDdchxpZchpezwySQvHhiyVMLevPRctXwqeWmfcv5GaVTGKRy557YIHnhpETeoCl05grhbPlL89HK1vCp5darvZbgo+XEwYcKDpNww3WYxC6/U5PY5oun66MzPHH8L05PpqHKghn+TpjyictkZQLPh4u6yeknvXeWU+JD6TDHJ/cbn93Bi8nnDKdJm8EG2+zIZwBudlbjUOYOpj1frClPwyf3OZuXuaEx3lgWZixKxIfZ911rvJO65PRFVmZjbYY+VHDYhBuuwyTccB0mcdkB0cr5z70pW/pm7Bo+LesgqUsrPjVye9WXkqld8FiizRCi6LBWjmTRPGGG/JZ5ejvoa1ai1qwvlWarbeZDBYdJuOE6TKKP4W7xJdFb4+R8ZvH5P852gxhpwOZ9AAAAJXRFWHRkYXRlOmNyZWF0ZQAyMDIwLTA4LTIzVDE0OjUyOjAwKzAyOjAwetRgVgAAACV0RVh0ZGF0ZTptb2RpZnkAMjAyMC0wOC0yM1QxNDo1MTo1OCswMjowMJuxI+oAAAAASUVORK5CYII=", + finished: false, + is_valid: true, + isSuggestion: true, + attachedTo: event.target.data("id"), + }] + //isButton: true, + + cy.add({ + group: "nodes", + data: suggestions[0], + position: { x: px, y: py }, + locked: true, + }); + } + const addDeleteButton = (event) => { var parentNode = cy.$("#" + event.target.data("id")); if (parentNode.data("isButton") || parentNode.data("buttonId")) return; @@ -3717,8 +3805,6 @@ const AngularWorkflow = (defaultprops) => { if (nodedata.app_name !== undefined) { const allNodes = cy.nodes().jsons(); - //if (parentNode.data("isButton") || parentNode.data("buttonId")) return; - var found = false; for (var key in allNodes) { const currentNode = allNodes[key]; @@ -3744,6 +3830,8 @@ const AngularWorkflow = (defaultprops) => { addCopyButton(event); addStartnodeButton(event); } + + //addSuggestionButtons(event) } } @@ -3811,6 +3899,8 @@ const AngularWorkflow = (defaultprops) => { const targetcolor = cy .getElementById(event.target.data("target")) .style("border-color"); + + //console.log(sourcecolor, targetcolor) if ( sourcecolor !== null && sourcecolor !== undefined && @@ -5476,10 +5566,21 @@ const AngularWorkflow = (defaultprops) => { //console.log("OLD: ", selectedAction, "NEW: ", newSelectedAction) for (var paramkey in selectedAction.parameters) { const param = selectedAction.parameters[paramkey]; + if (param.value === null || param.value === undefined || param.value.length === 0) { continue } + if (param.name === "body") { + //console.log("Param: ", param) + continue + } + + if (param.name === "headers") { + console.log("Swap header?") + //newSelectedAction.parameters[newParamIndex].value = param.value + } + const newParamIndex = newSelectedAction.parameters.findIndex(paramdata => paramdata.name === param.name) if (newParamIndex < 0) { continue @@ -10950,14 +11051,37 @@ const AngularWorkflow = (defaultprops) => { setExecutionRequestStarted(false); } + // Ensuring we have the latest version of the result. + // Especially important IF the result is > 1 Mb in cloud + var checkStarted = false + if (isCloud && data.results !== undefined && data.results !== null && data.results.length > 0) { + for (var key in data.results) { + if (data.results[key].status !== "SUCCESS") { + continue + } + + if (data.results[key].result.includes("too large")) { + setExecutionData({}); + checkStarted = true + start(); + setExecutionRunning(true); + setExecutionRequestStarted(false); + break + } + } + } + const cur_execution = { execution_id: data.execution_id, authorization: data.authorization, }; setExecutionRequest(cur_execution); setExecutionModalView(1); - setExecutionData(data); - handleUpdateResults(data, cur_execution); + + if (!checkStarted) { + handleUpdateResults(data, cur_execution); + setExecutionData(data); + } }} >
diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index e79d4a61..f0186a6e 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.49 +VERSION=0.9.50 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 2b2c9f54..498e8d3d 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -222,16 +222,16 @@ func deployServiceWorkers(image string) { defaultNetworkAttach := false if containerId != "" { - log.Printf("[WARNING] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) + log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId) // https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings networkConfig := &network.EndpointSettings{} err := dockercli.NetworkConnect(ctx, networkName, containerId, networkConfig) if err != nil { - log.Printf("[WARNING] Failed connecting to Orborus to docker network %s: %s", networkName, err) + log.Printf("[ERROR] Failed connecting to Orborus to docker network %s: %s", networkName, err) } if len(containerId) == 64 && baseUrl == "http://shuffle-backend:5001" { - log.Printf("[WARNING] Network MAY not work due to backend being %s and container length 64. Will try to attach shuffle_shuffle network", baseUrl) + log.Printf("[ERROR] Network MAY not work due to backend being %s and container length 64. Will try to attach shuffle_shuffle network", baseUrl) defaultNetworkAttach = true } } @@ -782,7 +782,8 @@ func main() { //workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix) workerImage := fmt.Sprintf("%s/%s/shuffle-worker:%s", baseimageregistry, baseimagename, workerVersion) - go deployServiceWorkers(workerImage) + //go deployServiceWorkers(workerImage) + deployServiceWorkers(workerImage) log.Printf("[INFO] Finished configuring docker environment") diff --git a/functions/onprem/worker/build.sh b/functions/onprem/worker/build.sh index 5ebeb89a..87919a26 100644 --- a/functions/onprem/worker/build.sh +++ b/functions/onprem/worker/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-worker -VERSION=0.9.49 +VERSION=0.9.50 echo "Running docker build with $NAME:$VERSION" #CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin . diff --git a/functions/onprem/worker/go.mod b/functions/onprem/worker/go.mod index 32dc8d2e..2f3ea588 100644 --- a/functions/onprem/worker/go.mod +++ b/functions/onprem/worker/go.mod @@ -10,6 +10,6 @@ require ( github.com/docker/go-connections v0.4.0 // indirect github.com/gorilla/mux v1.8.0 github.com/patrickmn/go-cache v2.1.0+incompatible - github.com/shuffle/shuffle-shared v0.1.78 + github.com/shuffle/shuffle-shared v0.1.83 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect ) diff --git a/functions/onprem/worker/go.sum b/functions/onprem/worker/go.sum index b704377d..5a1d006a 100644 --- a/functions/onprem/worker/go.sum +++ b/functions/onprem/worker/go.sum @@ -593,6 +593,8 @@ github.com/shuffle/shuffle-shared v0.1.73 h1:1rMOXAvxm/nDemwN/L8qWacswVMvdi6NjLf github.com/shuffle/shuffle-shared v0.1.73/go.mod h1:2ndjLm4ZOvY6arGFwOgGnkQ457Ke7gka9HDF/EkdIxQ= github.com/shuffle/shuffle-shared v0.1.78 h1://YsgQ85Ep40AA3pLUXb+85BUrNz5sqGqh0e3twKRy4= github.com/shuffle/shuffle-shared v0.1.78/go.mod h1:cW8LBv8P24rCPyJqGV6czxqrpnrv/R1d97EOqNgIvSk= +github.com/shuffle/shuffle-shared v0.1.81 h1:/lOt7NSuMTWlRzgOKg2e7j95eakg3MgW6i/4Fp30kd4= +github.com/shuffle/shuffle-shared v0.1.81/go.mod h1:cW8LBv8P24rCPyJqGV6czxqrpnrv/R1d97EOqNgIvSk= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index baa2b94b..7460161f 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -99,7 +99,7 @@ type UserInputSubflow struct { // removes every container except itself (worker) func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason string, handleResultSend bool) { - log.Printf("[INFO][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend) + log.Printf("[INFO][%s] Shutdown (%s) started with reason %#v. Result amount: %d. ResultsSent: %d, Send result: %#v, Parenent: %#v", workflowExecution.ExecutionId, workflowExecution.Status, reason, len(workflowExecution.Results), requestsSent, handleResultSend, workflowExecution.ExecutionParent) //reason := "Error in execution" sleepDuration := 1 @@ -351,6 +351,21 @@ func deployApp(cli *dockerclient.Client, image string, identifier string, env [] Env: env, } + // Checking as late as possible, just in case. + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) + _, err := shuffle.GetCache(ctx, newExecId) + if err == nil { + log.Printf("\n\n[DEBUG] Result for %s already found - returning\n\n", newExecId) + return nil + } + cacheData := []byte("1") + err = shuffle.SetCache(ctx, newExecId, cacheData) + if err != nil { + log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) + } else { + log.Printf("[DEBUG] Adding %s to cache. Name: %s", newExecId, action.Name) + } + if action.ExecutionDelay > 0 { log.Printf("[DEBUG] Running app %s in docker with delay of %d", action.Name, action.ExecutionDelay) waitTime := time.Duration(action.ExecutionDelay) * time.Second @@ -687,7 +702,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if isSkipped { //log.Printf("Skipping %s as all parents are done", item.Action.Label) if !arrayContains(visited, item.Action.ID) { - log.Printf("[INFO][%s] Adding visited (1): %s", workflowExecution.ExecutionId, item.Action.Label) + //log.Printf("[INFO][%s] Adding visited (1): %s", workflowExecution.ExecutionId, item.Action.Label) visited = append(visited, item.Action.ID) } } else { @@ -696,7 +711,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } } else { if item.Status == "FINISHED" { - log.Printf("[INFO][%s] Adding visited (2): %s", workflowExecution.ExecutionId, item.Action.Label) + //log.Printf("[INFO][%s] Adding visited (2): %s", workflowExecution.ExecutionId, item.Action.Label) visited = append(visited, item.Action.ID) } } @@ -874,22 +889,27 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { continue } else { log.Printf("[INFO][%s] %s:%s has no status result yet. Should execute.", workflowExecution.ExecutionId, action.Name, action.ID) + + // Check cache here too. } + // Rerunning this multiple places, as timing is the hardest part here. newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, nextAction) _, err := shuffle.GetCache(ctx, newExecId) if err == nil { - //log.Printf("\n\n[DEBUG] Already found %s - returning\n\n", newExecId) + log.Printf("\n\n[DEBUG] Already found %s (1) - returning\n\n", newExecId) continue } - cacheData := []byte("1") - err = shuffle.SetCache(ctx, newExecId, cacheData) - if err != nil { - log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) - } else { - log.Printf("\n\n[DEBUG] Adding %s to cache. Name: %s\n\n", newExecId, action.Name) - } + /* + cacheData := []byte("1") + err = shuffle.SetCache(ctx, newExecId, cacheData) + if err != nil { + log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) + } else { + log.Printf("\n\n[DEBUG] Adding %s to cache. Name: %s\n\n", newExecId, action.Name) + } + */ if action.AppName == "Shuffle Tools" && (action.Name == "skip_me" || action.Name == "router" || action.Name == "route") { topClient := &http.Client{ @@ -899,7 +919,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if err != nil { log.Printf("[DEBUG][%s] Error in skipme for %s: %s", workflowExecution.ExecutionId, action.Label, err) } else { - log.Printf("[INFO][%s] Adding visited (4): %s", workflowExecution.ExecutionId, action.Label) + //log.Printf("[INFO][%s] Adding visited (4): %s", workflowExecution.ExecutionId, action.Label) visited = append(visited, action.ID) executed = append(executed, action.ID) @@ -1373,7 +1393,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { } } - log.Printf("[INFO][%s] Adding visited (3): %s (%s). Actions: %d, Results: %d", workflowExecution.ExecutionId, action.Label, action.ID, len(workflowExecution.Workflow.Actions), len(workflowExecution.Results)) + //log.Printf("[INFO][%s] Adding visited (3): %s (%s). Actions: %d, Results: %d", workflowExecution.ExecutionId, action.Label, action.ID, len(workflowExecution.Workflow.Actions), len(workflowExecution.Results)) visited = append(visited, action.ID) executed = append(executed, action.ID) @@ -1396,7 +1416,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { shutdownCheck := true for _, result := range workflowExecution.Results { - if result.Status == "EXECUTING" { + if result.Status == "EXECUTING" || result.Status == "WAITING" { // Cleaning up executing stuff shutdownCheck = false // USED TO BE CONTAINER REMOVAL @@ -1436,7 +1456,7 @@ func executionInit(workflowExecution shuffle.WorkflowExecution) error { // Setting up extra counter for _, trigger := range workflowExecution.Workflow.Triggers { - log.Printf("[DEBUG] Appname trigger (0): %s", trigger.AppName) + //log.Printf("[DEBUG] Appname trigger (0): %s", trigger.AppName) if trigger.AppName == "User Input" || trigger.AppName == "Shuffle Workflow" { extra += 1 } @@ -2049,7 +2069,8 @@ func runWorkflowExecutionTransaction(ctx context.Context, attempts int64, workfl return } } - log.Printf(`[DEBUG][%s] Got result %s from %s. Execution status: %s. Save: %#v`, actionResult.ExecutionId, actionResult.Status, actionResult.Action.ID, workflowExecution.Status, dbSave) + + log.Printf(`[DEBUG][%s] Got result %s from %s. Execution status: %s. Save: %#v. Parent: %#v`, actionResult.ExecutionId, actionResult.Status, actionResult.Action.ID, workflowExecution.Status, dbSave, workflowExecution.ExecutionParent) //dbSave := false //if len(results) != len(workflowExecution.Results) { @@ -2163,10 +2184,19 @@ func validateFinished(workflowExecution shuffle.WorkflowExecution) bool { //startAction, extra, children, parents, visited, executed, nextActions, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) _, extra, _, _, _, _, _, environments := shuffle.GetExecutionVariables(ctx, workflowExecution.ExecutionId) - log.Printf("[INFO][%s] VALIDATION. Status: %s, shuffle.Actions: %d, Extra: %d, Results: %d\n", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results)) + log.Printf("[INFO][%s] VALIDATION. Status: %s, shuffle.Actions: %d, Extra: %d, Results: %d. Parent: %#v\n", workflowExecution.ExecutionId, workflowExecution.Status, len(workflowExecution.Workflow.Actions), extra, len(workflowExecution.Results), workflowExecution.ExecutionParent) //if len(workflowExecution.Results) == len(workflowExecution.Workflow.Actions)+extra { if (len(environments) == 1 && requestsSent == 0 && len(workflowExecution.Results) >= 1 && os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm") || (len(workflowExecution.Results) >= len(workflowExecution.Workflow.Actions)+extra && len(workflowExecution.Workflow.Actions) > 0) { + if workflowExecution.Status == "FINISHED" { + for _, result := range workflowExecution.Results { + if result.Status == "EXECUTING" || result.Status == "WAITING" { + log.Printf("[WARNING] NOT returning full result, as a result may be unfinished: %s (%s) - %s", result.Action.Label, result.Action.ID, result.Status) + return false + } + } + } + if os.Getenv("SHUFFLE_SWARM_CONFIG") != "run" && os.Getenv("SHUFFLE_SWARM_CONFIG") != "swarm" { requestsSent += 1 } @@ -2754,6 +2784,8 @@ func sendAppRequest(incomingUrl, appName string, port int, action shuffle.Action } else { // Callback to worker parsedRequest.Url = fmt.Sprintf("%s:%d", parsedBaseurl, baseport) + + //parsedRequest.Url } //log.Printf("[DEBUG][%s] Should add a baseurl for the app to get back to: %s", workflowExecution.ExecutionId, parsedRequest.Url) @@ -2764,6 +2796,8 @@ func sendAppRequest(incomingUrl, appName string, port int, action shuffle.Action parsedRequest.Url = parsedRequest.BaseUrl parsedRequest.BaseUrl = tmp + //http://3e05d1e7d7a0:33333, + // Run with proper hostname, but set to shuffle-worker to avoid specific host target. // This means running with VIP instead. if len(hostname) > 0 { @@ -2793,15 +2827,39 @@ func sendAppRequest(incomingUrl, appName string, port int, action shuffle.Action return err } + // Checking as LATE as possible, ensuring we don't rerun what's already ran + ctx := context.Background() + newExecId := fmt.Sprintf("%s_%s", workflowExecution.ExecutionId, action.ID) + _, err = shuffle.GetCache(ctx, newExecId) + if err == nil { + log.Printf("\n\n[DEBUG] Result for %s already found (PRE REQUEST) - returning\n\n", newExecId) + return nil + } + + cacheData := []byte("1") + err = shuffle.SetCache(ctx, newExecId, cacheData) + if err != nil { + log.Printf("[WARNING] Failed setting cache for action %s: %s", newExecId, err) + } else { + log.Printf("[DEBUG] Adding %s to cache. Name: %s", newExecId, action.Name) + } + + // FIXME: + newresp, err := client.Do(req) if err != nil { + if strings.Contains(fmt.Sprintf("%s", err), "timeout awaiting response") { + return nil + } + log.Printf("[ERROR] Error running app run request: %s", err) + return err } body, err := ioutil.ReadAll(newresp.Body) if err != nil { - log.Printf("[ERROR] Failed reading body: %s", err) + log.Printf("[ERROR] Failed reading app request body body: %s", err) return err } else { log.Printf("[INFO][%s] NEWRESP (from app): %s", workflowExecution.ExecutionId, string(body)) @@ -3060,7 +3118,7 @@ func main() { //workflowExecution.StartedAt = int64(time.Now().Unix()) cacheKey := fmt.Sprintf("workflowexecution-%s", workflowExecution.ExecutionId) - requestCache = cache.New(5*time.Minute, 10*time.Minute) + requestCache = cache.New(60*time.Minute, 120*time.Minute) requestCache.Set(cacheKey, &workflowExecution, cache.DefaultExpiration) for _, action := range workflowExecution.Workflow.Actions { found := false @@ -3331,7 +3389,7 @@ func runWebserver(listener net.Listener) { if err != nil {} */ - requestCache = cache.New(5*time.Minute, 10*time.Minute) + requestCache = cache.New(60*time.Minute, 120*time.Minute) log.Printf("[DEBUG] Running webserver config for SWARM and K8s") r.HandleFunc("/api/v1/execute", handleRunExecution).Methods("POST", "OPTIONS") }