From feae9de6269fc479bf681f1d773b7c8b48d008c9 Mon Sep 17 00:00:00 2001 From: frikky Date: Sun, 6 Dec 2020 17:17:58 +0100 Subject: [PATCH] Majorly improved app loading from Github --- backend/go-app/main.go | 2 +- backend/go-app/walkoff.go | 106 +++++++++++++++++++++---- docker-compose.yml | 4 +- frontend/src/views/AngularWorkflow.jsx | 103 ++++++++++++------------ frontend/src/views/Workflows.jsx | 80 ++++++++++++------- 5 files changed, 196 insertions(+), 99 deletions(-) diff --git a/backend/go-app/main.go b/backend/go-app/main.go index 742d3b65..babee811 100644 --- a/backend/go-app/main.go +++ b/backend/go-app/main.go @@ -6530,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error { } //log.Printf("Reading app folder: %#v", dir) - err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) + _, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate) if err != nil { log.Printf("Err: %s", err) return err diff --git a/backend/go-app/walkoff.go b/backend/go-app/walkoff.go index 70307a82..6093edf6 100644 --- a/backend/go-app/walkoff.go +++ b/backend/go-app/walkoff.go @@ -1403,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) { q := datastore.NewQuery("workflow").Filter("owner =", user.Id) if user.Role == "admin" { q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id) + log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id) } var workflows []Workflow @@ -2235,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { // Handles check for required params if !found && param.Required { log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name) - resp.WriteHeader(401) - resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) - return + action.Errors = append(action.Errors, "Parameter %s is required", param.Name) + //newActions = append(newActions, action) + //resp.WriteHeader(401) + //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name))) + //return } } @@ -2252,6 +2255,12 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { workflow.IsValid = true log.Printf("Tags: %#v", workflow.Tags) + // FIXME: Is this too drastic? May lead to issues in the future. + // Should maybe make a copy for the old org. + if workflow.OrgId != user.ActiveOrg.Id { + workflow.OrgId = user.ActiveOrg.Id + } + err = setWorkflow(ctx, workflow, fileId) if err != nil { log.Printf("Failed saving workflow to database: %s", err) @@ -2277,7 +2286,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) { Errors: workflow.Errors, } - log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId) + log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId) resp.WriteHeader(200) newBody, err := json.Marshal(returndata) if err != nil { @@ -5379,11 +5388,23 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra return err } +type buildLaterStruct struct { + Tags []string + Extra string +} + // Onlyname is used to -func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error { +func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) { var err error allapps := []WorkflowApp{} + reservedNames := []string{ + "OWA", + "NLP", + } + + buildLaterFirst := []buildLaterStruct{} + buildLaterList := []buildLaterStruct{} // It's here to prevent getting them in every iteration ctx := context.Background() @@ -5403,11 +5424,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin } // Go routine? Hmm, this can be super quick I guess - err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) + buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate) if err != nil { log.Printf("Error reading folder: %s", err) continue } + + for _, item := range buildFirst { + buildLaterFirst = append(buildLaterFirst, item) + } + + for _, item := range buildLast { + buildLaterList = append(buildLaterList, item) + } + case mode.IsRegular(): // Check the file filename := file.Name() @@ -5605,22 +5635,66 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin //log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion) - /// Only upload if successful and no errors - err = buildImageMemory(fs, tags, extra) - if err != nil { - log.Printf("Failed image build memory: %s", err) - } else { - if len(tags) > 0 { - log.Printf("Successfully built image %s", tags[0]) - } else { - log.Printf("Successfully built Docker image") + reservedFound := false + buildLater := buildLaterStruct{ + Tags: tags, + Extra: extra, + } + for _, appname := range reservedNames { + if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) { + buildLaterList = append(buildLaterList, buildLater) + + reservedFound = true + break } } + + /// Only upload if successful and no errors + if !reservedFound { + buildLaterFirst = append(buildLaterFirst, buildLater) + } else { + log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name) + } } } } - return err + if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 { + return buildLaterFirst, buildLaterList, err + } + + //log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList)) + if len(extra) == 0 { + log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst)) + for _, item := range buildLaterFirst { + err = buildImageMemory(fs, item.Tags, item.Extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(item.Tags) > 0 { + log.Printf("Successfully built image %s", item.Tags[0]) + } else { + log.Printf("Successfully built Docker image") + } + } + } + + log.Printf("Starting build of %d skipped docker images", len(buildLaterList)) + for _, item := range buildLaterList { + err = buildImageMemory(fs, item.Tags, item.Extra) + if err != nil { + log.Printf("Failed image build memory: %s", err) + } else { + if len(item.Tags) > 0 { + log.Printf("Successfully built image %s", item.Tags[0]) + } else { + log.Printf("Successfully built Docker image") + } + } + } + } + + return buildLaterFirst, buildLaterList, err } func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) { diff --git a/docker-compose.yml b/docker-compose.yml index 5ed77582..62894b11 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ version: '3' services: frontend: - #build: ./frontend + build: ./frontend image: ghcr.io/frikky/shuffle-frontend:0.8.3 container_name: shuffle-frontend hostname: shuffle-frontend @@ -16,7 +16,7 @@ services: depends_on: - backend backend: - #build: ./backend + build: ./backend image: ghcr.io/frikky/shuffle-backend:0.8.3 container_name: shuffle-backend hostname: ${BACKEND_HOSTNAME} diff --git a/frontend/src/views/AngularWorkflow.jsx b/frontend/src/views/AngularWorkflow.jsx index b07a0c17..e47c7c6c 100644 --- a/frontend/src/views/AngularWorkflow.jsx +++ b/frontend/src/views/AngularWorkflow.jsx @@ -191,7 +191,7 @@ const AngularWorkflow = (props) => { const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false //const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments - const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" + const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" const triggerEnvironments = isCloud ? ["cloud"] : ["cloud", "onprem"] const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?' @@ -1334,6 +1334,7 @@ const AngularWorkflow = (props) => { } else { setEnvironments({"name": "Onprem", "type": "onprem"}) } + return } @@ -4677,7 +4678,6 @@ const AngularWorkflow = (props) => { placeholder={selectedTrigger.label} onChange={selectedTriggerChange} /> - {showEnvironment ?
Environment @@ -4725,7 +4725,6 @@ const AngularWorkflow = (props) => { })}
- : null}
@@ -5221,59 +5220,57 @@ const AngularWorkflow = (props) => { placeholder={selectedTrigger.label} onChange={selectedTriggerChange} /> - {showEnvironment ? -
- - Environment - - { - selectedTrigger.environment = e.target.value - setSelectedTrigger(selectedTrigger) - if (e.target.value === "cloud") { - console.log("Set cloud config") - workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" + } + }} + fullWidth + onChange={(e) => { + selectedTrigger.environment = e.target.value + setSelectedTrigger(selectedTrigger) + if (e.target.value === "cloud") { + console.log("Set cloud config") + workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" - //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") - //const urlpath = tmpvalue.slice(3, tmpvalue.length) - //const newurl = "https://shuffler.io/"+urlpath.join("/") - //workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl - } else { - console.log("Set cloud config") - //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") - //const urlpath = tmpvalue.slice(3, tmpvalue.length) - //const newurl = window.location.origin+"/"+urlpath.join("/") - workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" - } + //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") + //const urlpath = tmpvalue.slice(3, tmpvalue.length) + //const newurl = "https://shuffler.io/"+urlpath.join("/") + //workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl + } else { + console.log("Set cloud config") + //var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/") + //const urlpath = tmpvalue.slice(3, tmpvalue.length) + //const newurl = window.location.origin+"/"+urlpath.join("/") + workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" + } - setWorkflow(workflow) - setUpdate(Math.random()) - }} - style={{backgroundColor: inputColor, color: "white", height: "50px"}} - > - {triggerEnvironments.map(data => { - if (data.archived) { - return null - } - - return ( - - {data} - - ) - })} - -
- : null} + setWorkflow(workflow) + setUpdate(Math.random()) + }} + style={{backgroundColor: inputColor, color: "white", height: "50px"}} + > + {triggerEnvironments.map(data => { + if (data.archived) { + return null + } + + return ( + + {data} + + ) + })} + +
diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 98583320..56ef2f86 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -15,6 +15,7 @@ import FormControlLabel from '@material-ui/core/FormControlLabel'; import Chip from '@material-ui/core/Chip'; import Switch from '@material-ui/core/Switch'; import Typography from '@material-ui/core/Typography'; +import Zoom from '@material-ui/core/Zoom'; import CircularProgress from '@material-ui/core/CircularProgress'; import CachedIcon from '@material-ui/icons/Cached'; @@ -365,6 +366,7 @@ const Workflows = (props) => { } } data["org"] = [] + data["org_id"] = "" data.execution_org = {"id": ""} console.log(data) @@ -1028,8 +1030,19 @@ const Workflows = (props) => { }, }} > + +
+ {editingWorkflow.id !== undefined ? "Editing" : "New"} workflow +
+ + + +
+
+ -
{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow
 setNewWorkflowName(event.target.value)} @@ -1127,6 +1140,35 @@ const Workflows = (props) => { workflowViewStyle.display = "none" } + const workflowButtons = + + {workflows.length > 0 ? + + + + : null} + + + + upload = ref} onChange={importFiles} /> + {workflows.length > 0 ? + + + + : null} + + + + + const WorkflowView = () => { if (workflows.length === 0) { return ( @@ -1143,15 +1185,21 @@ const Workflows = (props) => {
If you want to jump straight into it, click here to create your first workflow:
-
+
+ + + ..OR + + {workflowButtons} +
) } - return ( + return (
@@ -1159,29 +1207,7 @@ const Workflows = (props) => {

Workflows

- - - - {/* - - - - */} - - - - - - - upload = ref} onChange={importFiles} /> + {workflowButtons}
@@ -1189,7 +1215,7 @@ const Workflows = (props) => {
{workflows.map((data, index) => { return ( - + ) })}