Majorly improved app loading from Github

backend/go-app/main.go

@@ -6530,7 +6530,7 @@ func handleAppHotload(location string, forceUpdate bool) error {
 	}
 
 	//log.Printf("Reading app folder: %#v", dir)
-	err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
+	_, _, err = iterateAppGithubFolders(fs, dir, "", "", forceUpdate)
 	if err != nil {
 		log.Printf("Err: %s", err)
 		return err

backend/go-app/walkoff.go

@@ -1403,6 +1403,7 @@ func getWorkflows(resp http.ResponseWriter, request *http.Request) {
 	q := datastore.NewQuery("workflow").Filter("owner =", user.Id)
 	if user.Role == "admin" {
 		q = datastore.NewQuery("workflow").Filter("org_id =", user.ActiveOrg.Id)
+		log.Printf("[INFO] Getting workflows (ADMIN) for organization %s", user.ActiveOrg.Id)
 	}
 
 	var workflows []Workflow
@@ -2235,9 +2236,11 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 					// Handles check for required params
 					if !found && param.Required {
 						log.Printf("Appaction %s with required param %s doesn't exist.", action.Name, param.Name)
-						resp.WriteHeader(401)
+						action.Errors = append(action.Errors, "Parameter %s is required", param.Name)
-						resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
+						//newActions = append(newActions, action)
-						return
+						//resp.WriteHeader(401)
+						//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Appaction %s with required param '%s' is empty."}`, action.Name, param.Name)))
+						//return
 					}
 
 				}
@@ -2252,6 +2255,12 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 	workflow.IsValid = true
 	log.Printf("Tags: %#v", workflow.Tags)
 
+	// FIXME: Is this too drastic? May lead to issues in the future.
+	// Should maybe make a copy for the old org.
+	if workflow.OrgId != user.ActiveOrg.Id {
+		workflow.OrgId = user.ActiveOrg.Id
+	}
+
 	err = setWorkflow(ctx, workflow, fileId)
 	if err != nil {
 		log.Printf("Failed saving workflow to database: %s", err)
@@ -2277,7 +2286,7 @@ func saveWorkflow(resp http.ResponseWriter, request *http.Request) {
 		Errors:  workflow.Errors,
 	}
 
-	log.Printf("Saved new version of workflow %s (%s)", workflow.Name, fileId)
+	log.Printf("Saved new version of workflow %s (%s) for org %s", workflow.Name, fileId, workflow.OrgId)
 	resp.WriteHeader(200)
 	newBody, err := json.Marshal(returndata)
 	if err != nil {
@@ -5379,11 +5388,23 @@ func iterateWorkflowGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra
 	return err
 }
 
+type buildLaterStruct struct {
+	Tags  []string
+	Extra string
+}
+
 // Onlyname is used to
-func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) error {
+func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra string, onlyname string, forceUpdate bool) ([]buildLaterStruct, []buildLaterStruct, error) {
 	var err error
 
 	allapps := []WorkflowApp{}
+	reservedNames := []string{
+		"OWA",
+		"NLP",
+	}
+
+	buildLaterFirst := []buildLaterStruct{}
+	buildLaterList := []buildLaterStruct{}
 
 	// It's here to prevent getting them in every iteration
 	ctx := context.Background()
@@ -5403,11 +5424,20 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 			}
 
 			// Go routine? Hmm, this can be super quick I guess
-			err = iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
+			buildFirst, buildLast, err := iterateAppGithubFolders(fs, dir, tmpExtra, "", forceUpdate)
 			if err != nil {
 				log.Printf("Error reading folder: %s", err)
 				continue
 			}
+
+			for _, item := range buildFirst {
+				buildLaterFirst = append(buildLaterFirst, item)
+			}
+
+			for _, item := range buildLast {
+				buildLaterList = append(buildLaterList, item)
+			}
+
 		case mode.IsRegular():
 			// Check the file
 			filename := file.Name()
@@ -5605,22 +5635,66 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
 
 				//log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
 
-				/// Only upload if successful and no errors
+				reservedFound := false
-				err = buildImageMemory(fs, tags, extra)
+				buildLater := buildLaterStruct{
-				if err != nil {
+					Tags:  tags,
-					log.Printf("Failed image build memory: %s", err)
+					Extra: extra,
-				} else {
+				}
-					if len(tags) > 0 {
+				for _, appname := range reservedNames {
-						log.Printf("Successfully built image %s", tags[0])
+					if strings.ToUpper(workflowapp.Name) == strings.ToUpper(appname) {
-					} else {
+						buildLaterList = append(buildLaterList, buildLater)
-						log.Printf("Successfully built Docker image")
+
+						reservedFound = true
+						break
 					}
 				}
+
+				/// Only upload if successful and no errors
+				if !reservedFound {
+					buildLaterFirst = append(buildLaterFirst, buildLater)
+				} else {
+					log.Printf("\n\n[WARNING] Skipping build of %s to later\n\n", workflowapp.Name)
+				}
 			}
 		}
 	}
 
-	return err
+	if len(buildLaterFirst) == 0 && len(buildLaterList) == 0 {
+		return buildLaterFirst, buildLaterList, err
+	}
+
+	//log.Printf("BUILDLATERFIRST: %d, BUILDLATERLIST: %d", len(buildLaterFirst), len(buildLaterList))
+	if len(extra) == 0 {
+		log.Printf("[INFO] Starting build of %d containers (FIRST)", len(buildLaterFirst))
+		for _, item := range buildLaterFirst {
+			err = buildImageMemory(fs, item.Tags, item.Extra)
+			if err != nil {
+				log.Printf("Failed image build memory: %s", err)
+			} else {
+				if len(item.Tags) > 0 {
+					log.Printf("Successfully built image %s", item.Tags[0])
+				} else {
+					log.Printf("Successfully built Docker image")
+				}
+			}
+		}
+
+		log.Printf("Starting build of %d skipped docker images", len(buildLaterList))
+		for _, item := range buildLaterList {
+			err = buildImageMemory(fs, item.Tags, item.Extra)
+			if err != nil {
+				log.Printf("Failed image build memory: %s", err)
+			} else {
+				if len(item.Tags) > 0 {
+					log.Printf("Successfully built image %s", item.Tags[0])
+				} else {
+					log.Printf("Successfully built Docker image")
+				}
+			}
+		}
+	}
+
+	return buildLaterFirst, buildLaterList, err
 }
 
 func setNewWorkflowApp(resp http.ResponseWriter, request *http.Request) {

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   frontend:
-    #build: ./frontend
+    build: ./frontend
     image: ghcr.io/frikky/shuffle-frontend:0.8.3
     container_name: shuffle-frontend
     hostname: shuffle-frontend
@@ -16,7 +16,7 @@ services:
     depends_on:
       - backend
   backend:
-    #build: ./backend
+    build: ./backend
     image: ghcr.io/frikky/shuffle-backend:0.8.3
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}

frontend/src/views/AngularWorkflow.jsx

@@ -191,7 +191,7 @@ const AngularWorkflow = (props) => {
 
 	const cloudSyncEnabled = props.userdata !== undefined && props.userdata.active_org !== null && props.userdata.active_org !== undefined ? props.userdata.active_org.cloud_sync === true : false 
 	//const triggerEnvironments = cloudSyncEnabled ? ["cloud", "onprem"] : environments
-	const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io"
+	const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io" 
 	const triggerEnvironments = isCloud ? ["cloud"] : ["cloud", "onprem"] 
 
 	const unloadText = 'Are you sure you want to leave without saving (CTRL+S)?'
@@ -1334,6 +1334,7 @@ const AngularWorkflow = (props) => {
 				} else {
 					setEnvironments({"name": "Onprem", "type": "onprem"})
 				}
+
 				return
 			}
 
@@ -4677,7 +4678,6 @@ const AngularWorkflow = (props) => {
 						placeholder={selectedTrigger.label}
 						onChange={selectedTriggerChange}
 					/>
-					{showEnvironment ? 
 						<div style={{marginTop: "20px"}}>
 							<Typography>
 								Environment
@@ -4725,7 +4725,6 @@ const AngularWorkflow = (props) => {
 								})}
 							</Select>
 						</div>
-						: null}
 					<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 					<div style={{flex: "6", marginTop: "20px"}}>
 						<div>
@@ -5221,59 +5220,57 @@ const AngularWorkflow = (props) => {
 						placeholder={selectedTrigger.label}
 						onChange={selectedTriggerChange}
 					/>
-					{showEnvironment ? 
+					<div style={{marginTop: "20px"}}>
-						<div style={{marginTop: "20px"}}>
+						<Typography>
-							<Typography>
+							Environment
-								Environment
+						</Typography>
-							</Typography>
+						<Select
-							<Select
+							value={selectedTrigger.environment}
-								value={selectedTrigger.environment}
+							disabled={selectedTrigger.status === "running"}
-								disabled={selectedTrigger.status === "running"}
+							SelectDisplayProps={{
-								SelectDisplayProps={{
+								style: {
-									style: {
+									marginLeft: 10,
-										marginLeft: 10,
 
-									}
+								}
-								}}
+							}}
-								fullWidth
+							fullWidth
-								onChange={(e) => {
+							onChange={(e) => {
-									selectedTrigger.environment = e.target.value
+								selectedTrigger.environment = e.target.value
-									setSelectedTrigger(selectedTrigger)
+								setSelectedTrigger(selectedTrigger)
-									if (e.target.value === "cloud") {
+								if (e.target.value === "cloud") {
-										console.log("Set cloud config")
+									console.log("Set cloud config")
-										workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" 
+									workflow.triggers[selectedTriggerIndex].parameters[0].value = "*/2 * * * *" 
 
-										//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
+									//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
-										//const urlpath = tmpvalue.slice(3, tmpvalue.length)
+									//const urlpath = tmpvalue.slice(3, tmpvalue.length)
-										//const newurl = "https://shuffler.io/"+urlpath.join("/")
+									//const newurl = "https://shuffler.io/"+urlpath.join("/")
-										//workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl
+									//workflow.triggers[selectedTriggerIndex].parameters[0].value = newurl
-									} else {
+								} else {
-										console.log("Set cloud config")
+									console.log("Set cloud config")
-										//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
+									//var tmpvalue = workflow.triggers[selectedTriggerIndex].parameters[0].value.split("/")
-										//const urlpath = tmpvalue.slice(3, tmpvalue.length)
+									//const urlpath = tmpvalue.slice(3, tmpvalue.length)
-										//const newurl = window.location.origin+"/"+urlpath.join("/")
+									//const newurl = window.location.origin+"/"+urlpath.join("/")
-										workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" 
+									workflow.triggers[selectedTriggerIndex].parameters[0].value = "120" 
-									}
+								}
 
-									setWorkflow(workflow)
+								setWorkflow(workflow)
-									setUpdate(Math.random())
+								setUpdate(Math.random())
-								}}
+							}}
-								style={{backgroundColor: inputColor, color: "white", height: "50px"}}
+							style={{backgroundColor: inputColor, color: "white", height: "50px"}}
-							>
+						>
-								{triggerEnvironments.map(data => {
+							{triggerEnvironments.map(data => {
-									if (data.archived) {
+								if (data.archived) {
-										return null
+									return null
-									}
+								}
-									
+								
-									return (
+								return (
-										<MenuItem key={data} style={{backgroundColor: inputColor, color: "white"}} value={data}>
+									<MenuItem key={data} style={{backgroundColor: inputColor, color: "white"}} value={data}>
-											{data}
+										{data}
-										</MenuItem>
+									</MenuItem>
-									)
+								)
-								})}
+							})}
-							</Select>
+						</Select>
-						</div>
+					</div>
-						: null}
 					<Divider style={{marginTop: "20px", height: "1px", width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 					<div style={{flex: "6", marginTop: "20px"}}>
 						<div>

frontend/src/views/Workflows.jsx

@@ -15,6 +15,7 @@ import FormControlLabel from '@material-ui/core/FormControlLabel';
 import Chip from '@material-ui/core/Chip';
 import Switch from '@material-ui/core/Switch';
 import Typography from '@material-ui/core/Typography';
+import Zoom from '@material-ui/core/Zoom';
 
 import CircularProgress from '@material-ui/core/CircularProgress';
 import CachedIcon from '@material-ui/icons/Cached';
@@ -365,6 +366,7 @@ const Workflows = (props) => {
 			}
 		}
 		data["org"] = []
+		data["org_id"] = ""
 		data.execution_org = {"id": ""}
 		console.log(data)
 
@@ -1028,8 +1030,19 @@ const Workflows = (props) => {
 				},
 			}}
 		>
+			<DialogTitle>
+				<div style={{color: "rgba(255,255,255,0.9)"}}>
+					{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow
+					<div style={{float: "right"}}>
+						<Tooltip color="primary" title={"Import manually"} placement="top">
+							<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+								<PublishIcon />
+							</Button> 				
+						</Tooltip>
+					</div>
+				</div>
+			</DialogTitle>
 			<FormControl>
-			<DialogTitle><div style={{color: "white"}}>{editingWorkflow.id !== undefined ? "Editing" : "New"} workflow</div></DialogTitle>
 				<DialogContent>
 					<TextField
 						onBlur={(event) => setNewWorkflowName(event.target.value)}
@@ -1127,6 +1140,35 @@ const Workflows = (props) => {
 			workflowViewStyle.display = "none"
 		}
 
+		const workflowButtons = 
+			<span>
+				{workflows.length > 0 ?
+					<Tooltip color="primary" title={"Create new workflow"} placement="top">
+						<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
+					</Tooltip>
+				: null}
+				<Tooltip color="primary" title={"Import workflows"} placement="top">
+					<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
+						<PublishIcon />
+					</Button> 				
+				</Tooltip>
+				<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
+				{workflows.length > 0 ? 
+					<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
+						<Button color="primary" style={{}} variant="text" onClick={() => {
+							exportAllWorkflows()
+						}}>
+							<GetAppIcon />
+						</Button> 				
+					</Tooltip>
+				: null}
+				<Tooltip color="primary" title={"Download workflows"} placement="top">
+					<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
+						<CloudDownloadIcon />
+					</Button> 				
+				</Tooltip>
+			</span>
+
 		const WorkflowView = () => {
 			if (workflows.length === 0) {
 				return (
@@ -1143,15 +1185,21 @@ const Workflows = (props) => {
 							<div>
 								If you want to jump straight into it, click here to create your first workflow: 
 							</div>
-							<div>
+							<div style={{display: "flex"}}>
 								<Button color="primary" style={{marginTop: "20px",}} variant="outlined" onClick={() => setModalOpen(true)}>New workflow</Button> 				
+								<span style={{paddingTop: 20, display: "flex",}}>
+									<Typography style={{marginTop: 5, marginLeft: 30, marginRight: 15}}>
+										..OR
+									</Typography>
+									{workflowButtons}
+								</span>
 							</div>
 						</Paper>
 					</div>
 				)
 			}
 
-			return (
+		return (
 			<div style={viewStyle}>	
 				<div style={workflowViewStyle}>
 					<div style={{display: "flex"}}>
@@ -1159,29 +1207,7 @@ const Workflows = (props) => {
 							<h2>Workflows</h2> 
 						</div>
 						<div style={{marginTop: 20}}>
-						 	<Tooltip color="primary" title={"Create new workflow"} placement="top">
+							{workflowButtons}
-								<Button color="primary" style={{}} variant="text" onClick={() => setModalOpen(true)}><AddIcon /></Button> 				
-							</Tooltip>
-							{/*
-						 	<Tooltip color="primary" title={"Import workflows"} placement="top">
-								<Button color="primary" style={{}} variant="text" onClick={() => upload.click()}>
-									<PublishIcon />
-								</Button> 				
-							</Tooltip>
-							*/}
-							<Tooltip color="primary" title={`Download ALL workflows (${workflows.length})`} placement="top">
-								<Button color="primary" style={{}} variant="text" onClick={() => {
-									exportAllWorkflows()
-								}}>
-									<GetAppIcon />
-								</Button> 				
-							</Tooltip>
-						 	<Tooltip color="primary" title={"Import workflows"} placement="top">
-								<Button color="primary" style={{}} variant="text" onClick={() => setLoadWorkflowsModalOpen(true)}>
-									<CloudDownloadIcon />
-								</Button> 				
-							</Tooltip>
-							<input hidden type="file" multiple="multiple" ref={(ref) => upload = ref} onChange={importFiles} />
 						</div>
 					</div>
 					<Divider style={{marginBottom: "10px", height: "1px", width: "100%", backgroundColor: dividerColor}}/>
@@ -1189,7 +1215,7 @@ const Workflows = (props) => {
 					<div style={scrollStyle}>
 						{workflows.map((data, index) => {
 							return (
-								<WorkflowPaper key={index} data={data} />
+									<WorkflowPaper key={index} data={data} />
 							)
 						})}
 					</div>