Merge branch 'launch' of https://github.com/frikky/shuffle into launch

.dockerignore

@@ -0,0 +1,15 @@
+**/.git*
+**/.gitlab-ci.yml
+**/.env
+**/.dockerignore
+**/Dockerfile*
+**/node_modules
+buildSrc/libs
+.gradle/
+build/
+**build/
+.idea/
+!.idea/codeStyles/codeStyleConfig.xml
+.DS_Store
+*.log
+out/

.env

@@ -34,3 +34,7 @@ DB_LOCATION=./shuffle-database
 SHUFFLE_HTTP_PROXY=
 SHUFFLE_HTTPS_PROXY=
 SHUFFLE_PASS_WORKER_PROXY=TRUE
+
+SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
+SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
+SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.6.0"

backend/app_gen/openapi/test.go

@@ -180,7 +180,7 @@ func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
 	api.Name = swagger.Info.Title
 	api.Description = swagger.Info.Description
 	api.IsValid = true
-	api.Link = swagger.Servers[0].URL // host doesnt exist lol
+	api.Link = swagger.Servers[0].URL // host does not exist lol
 	api.AppVersion = "1.0.0"
 	api.Environment = "cloud"
 	api.ID = ""

backend/app_gen/openapi/testGCP.go

@@ -216,7 +216,7 @@ func generateYaml(swagger *openapi3.Swagger) (WorkflowApp, []string, error) {
 	api.Name = swagger.Info.Title
 	api.Description = swagger.Info.Description
 	api.IsValid = true
-	api.Link = swagger.Servers[0].URL // host doesnt exist lol
+	api.Link = swagger.Servers[0].URL // host does not exist lol
 	api.AppVersion = "1.0.0"
 	api.Environment = "cloud"
 	api.ID = ""

backend/app_gen/python-lib/generator.py

@@ -246,7 +246,7 @@ def loop_modules(modules, data):
                     param["position"] = cnt
 
                     if not found:
-                        # FIXME - waht here?
+                        # FIXME - what here?
                         pass
                         #print("HANDLE NOT FOUND")
                         #print(param)

backend/app_sdk/app_base.py

@@ -150,7 +150,7 @@ class AppBase:
 
         self.logger.info("AFTER FULLEXEC stream result")
 
-        # Gets the value at the paranthesis level you want
+        # Gets the value at the parenthesis level you want
         def parse_nested_param(string, level):
             """
             Generate strings contained in nested (), indexing i = level
@@ -173,7 +173,7 @@ class AppBase:
             except IndexError:
                 return [string[LeftRightIndex[level+1][0]:LeftRightIndex[level+1][1]]]
         
-        # Finds the deepest level paranthesis in a string
+        # Finds the deepest level parenthesis in a string
         def maxDepth(S): 
             current_max = 0
             max = 0

backend/go-app/codegen.go

@@ -426,7 +426,7 @@ func generateYaml(swagger *openapi3.Swagger, newmd5 string) (*openapi3.Swagger,
 	//uuid.NewV4().String()
 
 	api.IsValid = true
-	api.Link = swagger.Servers[0].URL // host doesnt exist lol
+	api.Link = swagger.Servers[0].URL // host does not exist lol
 	if strings.HasSuffix(api.Link, "/") {
 		api.Link = api.Link[:len(api.Link)-1]
 	}

backend/go-app/main.go

@@ -6838,7 +6838,7 @@ func handleCloudSetup(resp http.ResponseWriter, request *http.Request) {
 	}
 }
 
-func init() {
+func initHandlers() {
 	var err error
 	ctx := context.Background()
 
@@ -6964,7 +6964,7 @@ func init() {
 
 // Had to move away from mux, which means Method is fucked up right now.
 func main() {
-	//init()
+	initHandlers()
 	hostname, err := os.Hostname()
 	if err != nil {
 		hostname = "MISSING"

backend/go-app/walkoff.go

@@ -28,6 +28,7 @@ import (
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
 	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/config"
 	"github.com/go-git/go-git/v5/plumbing"
 	"github.com/go-git/go-git/v5/storage/memory"
 	http2 "gopkg.in/src-d/go-git.v4/plumbing/transport/http"
@@ -1551,6 +1552,12 @@ func deleteWorkflow(resp http.ResponseWriter, request *http.Request) {
 		log.Printf("Failed to increase total workflows: %s", err)
 	}
 
+	// recalculate authenticators stats
+	err = recalculateAppAuthentications()
+	if err != nil {
+		log.Printf("Authentications recalculation failed: %s", err)
+	}
+
 	//memcacheName := fmt.Sprintf("%s_%s", user.Username, fileId)
 	//memcache.Delete(ctx, memcacheName)
 	//memcacheName = fmt.Sprintf("%s_workflows", user.Username)
@@ -4396,6 +4403,7 @@ func deployWebhookFunction(ctx context.Context, name, localization, applocation
 func loadGithubWorkflows(url, username, password, userId, branch string) error {
 	fs := memfs.New()
 
+	// FIXME: add more git options lol
 	if strings.Contains(url, "github") || strings.Contains(url, "gitlab") || strings.Contains(url, "bitbucket") {
 		cloneOptions := &git.CloneOptions{
 			URL: url,
@@ -4404,16 +4412,11 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error {
 		// FIXME: Better auth.
 		if len(username) > 0 && len(password) > 0 {
 			cloneOptions.Auth = &http2.BasicAuth{
-
 				Username: username,
 				Password: password,
 			}
 		}
 
-		if len(branch) > 0 {
-			cloneOptions.ReferenceName = plumbing.ReferenceName(branch)
-		}
-
 		storer := memory.NewStorage()
 		r, err := git.Clone(storer, fs, cloneOptions)
 		if err != nil {
@@ -4421,9 +4424,31 @@ func loadGithubWorkflows(url, username, password, userId, branch string) error {
 			return err
 		}
 
+		if len(branch) > 0 {
+			log.Printf("Checkout to branch: %s", branch)
+
+			w, _ := r.Worktree()
+
+			err := r.Fetch(&git.FetchOptions{
+				RefSpecs: []config.RefSpec{"refs/*:refs/*", "HEAD:refs/heads/HEAD"},
+			})
+			if err != nil {
+				log.Printf("Failed fetch for git repo: %s", err)
+			}
+
+			err = w.Checkout(&git.CheckoutOptions{
+				Branch: plumbing.ReferenceName(fmt.Sprintf("refs/heads/%s", branch)),
+				Force:  true,
+			})
+			if err != nil {
+				log.Printf("Failed checkout for git repo: %s", err)
+				return errors.New(fmt.Sprintf("Failed checking out to branch %s - does it exist?", branch))
+			}
+		}
+
 		dir, err := fs.ReadDir("/")
 		if err != nil {
-			log.Printf("FAiled reading folder: %s", err)
+			log.Printf("Failed reading folder: %s", err)
 		}
 		_ = r
 
@@ -4498,7 +4523,7 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
 	if err != nil {
 		log.Printf("Error with unmarshal tmpBody: %s", err)
 		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false}`))
+		resp.Write([]byte(`{"success": false, "reason": "json decode error"}`))
 		return
 	}
 
@@ -4507,14 +4532,82 @@ func loadSpecificWorkflows(resp http.ResponseWriter, request *http.Request) {
 	if err != nil {
 		log.Printf("Failed to update workflows: %s", err)
 		resp.WriteHeader(401)
-		resp.Write([]byte(`{"success": false}`))
+		resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
 		return
 	}
 
+	// recalculate authenticators stats
+	err = recalculateAppAuthentications()
+	if err != nil {
+		log.Printf("Authentications recalculation failed: %s", err)
+	}
+
 	resp.WriteHeader(200)
 	resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
 }
 
+func recalculateAppAuthentications() error {
+	// create context
+	ctx := context.Background()
+
+	// form workflows list
+	workflows, err := getAllWorkflows(ctx)
+	if err != nil {
+		log.Printf("Error: Failed getting workflows: %s", err)
+		return err
+	}
+
+	// form authenticators list
+	auths, err := getAllWorkflowAppAuth(ctx)
+	if err != nil {
+		log.Printf("Error: Failed getting auths: %s", err)
+		return err
+	}
+
+	// iterate through auths
+	for _, auth := range auths {
+		// reset calculated values
+		auth.WorkflowCount = 0
+		auth.NodeCount = 0
+		auth.Usage = []AuthenticationUsage{}
+
+		// iterate through workflows to find which uses this auth
+		for _, workflow := range workflows {
+			hasCurrentAuth := false
+			usageItem := AuthenticationUsage{
+				WorkflowId: workflow.ID,
+				Nodes:      []string{},
+			}
+
+			// iterate through actions
+			for _, action := range workflow.Actions {
+				if action.AuthenticationId == auth.Id {
+					// this workflow should be added to "usage" field
+					hasCurrentAuth = true
+
+					// add this action to list
+					usageItem.Nodes = append(usageItem.Nodes, action.ID)
+				}
+			}
+
+			// update current auth with found workflow
+			if hasCurrentAuth {
+				auth.WorkflowCount += 1
+				auth.NodeCount += int64(len(usageItem.Nodes))
+				auth.Usage = append(auth.Usage, usageItem)
+			}
+		}
+
+		// update record in database
+		err := setWorkflowAppAuthDatastore(ctx, auth, auth.Id)
+		if err != nil {
+			log.Printf("Failed setting up app auth %s: %s", auth.Id, err)
+		}
+	}
+
+	return nil
+}
+
 func handleAppHotloadRequest(resp http.ResponseWriter, request *http.Request) {
 	cors := handleCors(resp, request)
 	if cors {

docker-compose.yml

@@ -61,6 +61,9 @@ services:
       - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
       - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
       - SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600
+      - SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
+      - SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
+      - SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
     restart: unless-stopped
   database:
     #build: ./backend/database

frontend/src/views/Admin.jsx

@@ -1159,7 +1159,7 @@ const Admin = (props) => {
 										console.log("Show apps with this category")
 									}}
 								>
-									Find app ({data.apps.length})
+									Find app ({data.apps === null ? 0 : data.apps.length})
 								</Button>
 							</ListItemText>
 						</ListItem>
@@ -1222,7 +1222,7 @@ const Admin = (props) => {
 								style={{minWidth: 150, maxWidth: 150}}
 							/>
 							<ListItemText
-								primary={data.usage.length}
+								primary={data.usage === null ? 0 : data.usage.length}
 								style={{minWidth: 150, maxWidth: 150, overflow: "hidden"}}
 							/>
 							<ListItemText

frontend/src/views/EditWebhook.jsx

@@ -336,7 +336,7 @@ const EditWebhook = (props) => {
 		</div>
 		: null
 	
-	// FIXME - needs refresh everytime you add a new workflow
+	// FIXME - needs refresh every time you add a new workflow
 	const workflowdata = Object.getOwnPropertyNames(webhookData).length > 0 && selectedWorkflows.length > 0 ?
 		<EditWorkflow globalUrl={globalUrl} inputworkflows={selectedWorkflows} inputname={webhookData.info.name} inputtype={webhookData.type} /> : null	
 

frontend/src/views/Workflows.jsx

@@ -65,6 +65,7 @@ const Workflows = (props) => {
 	const [field1, setField1] = React.useState("")
 	const [field2, setField2] = React.useState("")
 	const [downloadUrl, setDownloadUrl] = React.useState("https://github.com/frikky/shuffle-workflows")
+	const [downloadBranch, setDownloadBranch] = React.useState("master")
 	const [loadWorkflowsModalOpen, setLoadWorkflowsModalOpen] = React.useState(false)
 
 	const [modalOpen, setModalOpen] = React.useState(false);
@@ -1186,6 +1187,7 @@ const Workflows = (props) => {
 
 		const parsedData = {
 			"url": url,
+			"field_3": downloadBranch || 'master'
 		}
 
 		if (field1.length > 0) {
@@ -1280,6 +1282,26 @@ const Workflows = (props) => {
 					fullWidth
 					/>
 
+					<span style={{marginTop: 10}}>Branch (default value is "master"):</span>
+					<div style={{display: "flex"}}>
+						<TextField
+							style={{backgroundColor: inputColor}}
+							variant="outlined"
+							margin="normal"
+							value={downloadBranch}
+							InputProps={{
+								style:{
+									color: "white",
+									height: "50px",
+									fontSize: "1em",
+								},
+							}}
+							onChange={e => setDownloadBranch(e.target.value)}
+							placeholder="master"
+							fullWidth
+							/>
+					</div>
+
 				<span style={{marginTop: 10}}>Authentication (optional - private repos etc):</span>
 				<div style={{display: "flex"}}>
 					<TextField

functions/onprem/orborus/orborus.go

@@ -36,7 +36,10 @@ var workerVersion = os.Getenv("SHUFFLE_WORKER_VERSION")
 
 //var baseimagename = "docker.pkg.github.com/frikky/shuffle"
 //var baseimagename = "ghcr.io/frikky"
-var baseimagename = "frikky/shuffle"
+// var baseimagename = "frikky/shuffle"
+var baseimagename = os.Getenv("SHUFFLE_BASE_IMAGE_NAME")
+var baseimageregistry = os.Getenv("SHUFFLE_BASE_IMAGE_REGISTRY")
+var baseimagetagsuffix = os.Getenv("SHUFFLE_BASE_IMAGE_TAG_SUFFIX")
 
 var orgId = os.Getenv("ORG_ID")
 var baseUrl = os.Getenv("BASE_URL")
@@ -213,10 +216,19 @@ func initializeImages() {
 		log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
 	}
 
-	// check whether theyre the same first
+	if baseimageregistry == "" {
+		baseimageregistry = "docker.io"
+	}
+	if baseimagename == "" {
+		baseimagename = "frikky/shuffle"
+	}
+
+	// check whether they are the same first
 	images := []string{
-		fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
+		fmt.Sprintf("%s/%s:app_sdk%s", baseimageregistry, baseimagename, baseimagetagsuffix),
-		fmt.Sprintf("docker.io/%s:worker", baseimagename),
+		fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix),
+		// fmt.Sprintf("docker.io/%s:app_sdk", baseimagename),
+		// fmt.Sprintf("docker.io/%s:worker", baseimagename),
 
 		//fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion),
 		//fmt.Sprintf("%s/app_sdk:%s", baseimagename, appSdkVersion),
@@ -284,7 +296,8 @@ func main() {
 	//workerVersion := "0.1.0"
 	//workerImage := fmt.Sprintf("docker.pkg.github.com/frikky/shuffle/%s:%s", workerName, workerVersion)
 	//workerImage := fmt.Sprintf("%s/worker:%s", baseimagename, workerVersion)
-	workerImage := fmt.Sprintf("docker.io/%s:worker", baseimagename)
+	// workerImage := fmt.Sprintf("docker.io/%s:worker", baseimagename)
+	workerImage := fmt.Sprintf("%s/%s:worker%s", baseimageregistry, baseimagename, baseimagetagsuffix)
 
 	log.Printf("[INFO] Finished configuring docker environment")
 

functions/onprem/worker/Dockerfile

@@ -10,6 +10,11 @@ COPY worker.go /app/worker.go
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker .
 
 FROM alpine:3.12
+
+ENV SHUFFLE_BASE_IMAGE_REGISTRY=docker.io
+ENV SHUFFLE_BASE_IMAGE_NAME=frikky/shuffle
+ENV SHUFFLE_BASE_IMAGE_TAG_SUFFIX=0.6.0
+
 RUN apk add --no-cache bash
 COPY --from=builder /app/ /