Majorly improved app loading from Github

2020-12-06 17:17:58 +01:00
parent 8355adb7b0
commit feae9de626
5 changed files with 196 additions and 99 deletions
@@ -6530,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error {
 	}

 	//log.Printf("Reading app folder: %#v", dir)
-	err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
+	_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
 	if err != nil {
 		log.Printf("Err: %s", err)
 		return err
@@ -1403,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) {
 	q := datastore.NewQuery("workflow").Filter("owner =", user.Id)
 	if user.Role == "admin" {
 		q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id)
+		log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id)
 	}

 	var workflows []Workflow
@@ -2235,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 					// Handles check for required params
 					if !found && param.Required {
 						log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
-						resp.WriteHeader(401)
-						resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
-						return
+						action.Errors = append(action.Errors, "Parameter %s is required", param.Name)
+						//newActions = append(newActions, action)
+						//resp.WriteHeader(401)
+						//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
+						//return
 					}

 				}
@@ -2252,6 +2255,12 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 	workflow.IsValid = true
 	log.Printf("Tags: %#v", workflow.Tags)

+	// FIXME: Is this too drastic? May lead to issues in the future.
+	// Should maybe make a copy for the old org.
+	if workflow.OrgId != user.ActiveOrg.Id {
+		workflow.OrgId = user.ActiveOrg.Id
+	}
+
 	err = setWorkflow(ctx, workflow, fileId)
 	if err != nil {
 		log.Printf("Failed saving workflow to database: %s", err)
@@ -2277,7 +2286,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 		Errors:  workflow.Errors,
 	}

-	log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId)
+	log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
 	resp.WriteHeader(200)
 	newBody, err := json.Marshal(returndata)
 	if err != nil {
@@ -5379,11 +5388,23 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
 	return err
 }

+type buildLaterStruct struct {
+	Tags  []string
+	Extra string
+}
+
 // Onlyname is used to
-func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error {
+func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) {
 	var err error

 	allapps := []WorkflowApp{}
+	reservedNames := []string{
+		"OWA",
+		"NLP",
+	}
+
+	buildLaterFirst := []buildLaterStruct{}
+	buildLaterList := []buildLaterStruct{}

 	// It's here to prevent getting them in every iteration
 	ctx := context.Background()
@@ -5403,11 +5424,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 			}

 			// Go routine? Hmm, this can be super quick I guess
-			err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
+			buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
 			if err != nil {
 				log.Printf("Error reading folder: %s", err)
 				continue
 			}
+
+			for _, item := range buildFirst {
+				buildLaterFirst = append(buildLaterFirst, item)
+			}
+
+			for _, item := range buildLast {
+				buildLaterList = append(buildLaterList, item)
+			}
+
 		case mode.IsRegular():
 			// Check the file
 			filename := file.Name()
@@ -5605,22 +5635,66 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin

 				//log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)

+				reservedFound := false
+				buildLater := buildLaterStruct{
+					Tags:  tags,
+					Extra: extra,
+				}
+				for _, appname := range reservedNames {
+					if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
+						buildLaterList = append(buildLaterList, buildLater)
+
+						reservedFound = true
+						break
+					}
+				}
+
 				/// Only upload if successful and no errors
-				err = buildImageMemory(fs, tags, extra)
+				if !reservedFound {
+					buildLaterFirst = append(buildLaterFirst, buildLater)
+				} else {
+					log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name)
+				}
+			}
+		}
+	}
+
+	if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 {
+		return buildLaterFirst, buildLaterList, err
+	}
+
+	//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
+	if len(extra) == 0 {
+		log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
+		for _, item := range buildLaterFirst {
+			err = buildImageMemory(fs, item.Tags, item.Extra)
 			if err != nil {
 				log.Printf("Failed image build memory: %s", err)
 			} else {
-					if len(tags) > 0 {
-						log.Printf("Successfully built image %s", tags[0])
+				if len(item.Tags) > 0 {
+					log.Printf("Successfully built image %s", item.Tags[0])
+				} else {
+					log.Printf("Successfully built Docker image")
+				}
+			}
+		}
+
+		log.Printf("Starting build of %d skipped docker images", len(buildLaterList))
+		for _, item := range buildLaterList {
+			err = buildImageMemory(fs, item.Tags, item.Extra)
+			if err != nil {
+				log.Printf("Failed image build memory: %s", err)
+			} else {
+				if len(item.Tags) > 0 {
+					log.Printf("Successfully built image %s", item.Tags[0])
 				} else {
 					log.Printf("Successfully built Docker image")
 				}
 			}
 		}
 	}
-	}

-	return err
+	return buildLaterFirst, buildLaterList, err
 }

 func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {
@@ -1,7 +1,7 @@
 version: '3'
 services:
  frontend:
-    #build: ./frontend
+    build: ./frontend
    image: ghcr.io/frikky/shuffle-frontend:0.8.3
    container_name: shuffle-frontend
    hostname: shuffle-frontend
@@ -16,7 +16,7 @@ services:
    depends_on:
      - backend
  backend:
-    #build: ./backend
+    build: ./backend
    image: ghcr.io/frikky/shuffle-backend:0.8.3
    container_name: shuffle-backend
    hostname: ${BACKEND_HOSTNAME}
@@ -1334,6 +1334,7 @@ const AngularWorkflow = (props) => {
 				} else {
 					setEnvironments({"name": "Onprem", "type": "onprem"})
 				}
+
 				return
 			}

@@ -4677,7 +4678,6 @@ const AngularWorkflow = (props) => {
 						placeholder={selectedTrigger.label}
 						onChange={selectedTriggerChange}
 					/>
-					{showEnvironment ? 
 						<div style={{marginTop: "20px"}}>
 							<Typography>
 								Environment
@@ -4725,7 +4725,6 @@ const AngularWorkflow = (props) => {
 								})}
 							</Select>
 						</div>
-						: null}
 					<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 					<div style={{flex: "6", marginTop: "20px"}}>
 						<div>
@@ -5221,7 +5220,6 @@ const AngularWorkflow = (props) => {
 						placeholder={selectedTrigger.label}
 						onChange={selectedTriggerChange}
 					/>
-					{showEnvironment ? 
 					<div style={{marginTop: "20px"}}>
 						<Typography>
 							Environment
@@ -5273,7 +5271,6 @@ const AngularWorkflow = (props) => {
 							})}
 						</Select>
 					</div>
-						: null}
 					<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 					<div style={{flex: "6", marginTop: "20px"}}>
 						<div>
@@ -15,6 +15,7 @@ import FormControlLabel from '@material-ui/core/FormControlLabel';
 import Chip from '@material-ui/core/Chip';
 import Switch from '@material-ui/core/Switch';
 import Typography from '@material-ui/core/Typography';
+import Zoom from '@material-ui/core/Zoom';

 import CircularProgress from '@material-ui/core/CircularProgress';
 import CachedIcon from '@material-ui/icons/Cached';
@@ -365,6 +366,7 @@ const Workflows = (props) => {
 			}
 		}
 		data["org"] = []
+		data["org_id"] = ""
 		data.execution_org = {"id": ""}
 		console.log(data)

@@ -1028,8 +1030,19 @@ const Workflows = (props) => {
 				},
 			}}
 		>
+			<DialogTitle>
+				<div style={{color: "rgba(255,255,255,0.9)"}}>
+					{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow
+					<div style={{float: "right"}}>
+						<Tooltip color="primary" title={"Import manually"} placement="top">
+							<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+								<PublishIcon />
+							</Button> 				
+						</Tooltip>
+					</div>
+				</div>
+			</DialogTitle>
 			<FormControl>
-			<DialogTitle><div style={{color: "white"}}>{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow</div></DialogTitle>
 				<DialogContent>
 					<TextField
 						onBlur={(event) => setNewWorkflowName(event.target.value)}
@@ -1127,6 +1140,35 @@ const Workflows = (props) => {
 			workflowViewStyle.display = "none"
 		}

+		const workflowButtons = 
+			<span>
+				{workflows.length > 0 ?
+					<Tooltip color="primary" title={"Create new workflow"} placement="top">
+						<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
+					</Tooltip>
+				: null}
+				<Tooltip color="primary" title={"Import workflows"} placement="top">
+					<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+						<PublishIcon />
+					</Button> 				
+				</Tooltip>
+				<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
+				{workflows.length > 0 ? 
+					<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
+						<Button color="primary" style={{}} variant="text" onClick={() => {
+							exportAllWorkflows()
+						}}>
+							<GetAppIcon />
+						</Button> 				
+					</Tooltip>
+				: null}
+				<Tooltip color="primary" title={"Download workflows"} placement="top">
+					<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
+						<CloudDownloadIcon />
+					</Button> 				
+				</Tooltip>
+			</span>
+
 		const WorkflowView = () => {
 			if (workflows.length === 0) {
 				return (
@@ -1143,8 +1185,14 @@ const Workflows = (props) => {
 							<div>
 								If you want to jump straight into it, click here to create your first workflow: 
 							</div>
-							<div>
+							<div style={{display: "flex"}}>
 								<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button> 				
+								<span style={{paddingTop: 20, display: "flex",}}>
+									<Typography style={{marginTop: 5, marginLeft: 30, marginRight: 15}}>
+										..OR
+									</Typography>
+									{workflowButtons}
+								</span>
 							</div>
 						</Paper>
 					</div>
@@ -1159,29 +1207,7 @@ const Workflows = (props) => {
 							<h2>Workflows</h2> 
 						</div>
 						<div style={{marginTop: 20}}>
-						 	<Tooltip color="primary" title={"Create new workflow"} placement="top">
-								<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
-							</Tooltip>
-							{/*
-						 	<Tooltip color="primary" title={"Import workflows"} placement="top">
-								<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
-									<PublishIcon />
-								</Button> 				
-							</Tooltip>
-							*/}
-							<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
-								<Button color="primary" style={{}} variant="text" onClick={() => {
-									exportAllWorkflows()
-								}}>
-									<GetAppIcon />
-								</Button> 				
-							</Tooltip>
-						 	<Tooltip color="primary" title={"Import workflows"} placement="top">
-								<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
-									<CloudDownloadIcon />
-								</Button> 				
-							</Tooltip>
-							<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
+							{workflowButtons}
 						</div>
 					</div>
 					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>