Merge pull request #88 from frikky/1.0.0

Fixed Docker proxy issues

.env

@@ -22,3 +22,9 @@ FRONTEND_PORT=3001
 FRONTEND_PORT_HTTPS=3443
 OUTER_HOSTNAME=shuffle-backend
 DB_LOCATION=./shuffle-database
+
+# Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps.
+# PS: It will skip proxy for 
+SHUFFLE_HTTP_PROXY=
+SHUFFLE_HTTPS_PROXY=
+SHUFFLE_PASS_WORKER_PROXY=TRUE

backend/go-app/main.go

@@ -6157,6 +6157,10 @@ func runInit(ctx context.Context) {
 	if len(httpProxy) > 0 {
 		log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
 	}
+	httpsProxy := os.Getenv("HTTPS_PROXY")
+	if len(httpsProxy) > 0 {
+		log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
+	}
 
 	/*
 			proxyUrl, err := url.Parse(httpProxy)

docker-compose.yml

@@ -34,6 +34,8 @@ services:
       - SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME}
       - SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD}
       - SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY}
+      - HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
+      - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
     restart: unless-stopped
     depends_on:
       - database
@@ -51,6 +53,9 @@ services:
       - ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
       - BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}
       - DOCKER_API_VERSION=1.40
+      - HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
+      - HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
+      - SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
     restart: unless-stopped
   database:
     #build: ./backend/database

frontend/src/AngularWorkflow.js

@@ -1543,7 +1543,10 @@ const AngularWorkflow = (props) => {
 						)
 					})}
 					<div style={{flex: "1"}}>
-						<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button> 				
+						<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => {
+							setVariablesModalOpen(true)
+							setLastSaved(false)
+						}}>New workflow variable</Button> 				
 					</div>
 					<Divider style={{marginBottom: 20, marginTop: 20, height: 1, width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
 						What are <a href="https://shuffler.io/docs/workflows#execution_variables" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>EXECUTION variables?</a>
@@ -1603,7 +1606,10 @@ const AngularWorkflow = (props) => {
 						)
 					})}
 					<div style={{flex: "1"}}>
-						<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setExecutionVariablesModalOpen(true)}>New execution variable</Button> 				
+						<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => {
+							setExecutionVariablesModalOpen(true)
+							setLastSaved(false)
+						}}>New execution variable</Button> 				
 					</div>
 				</div>
 			</div>
@@ -2240,6 +2246,8 @@ const AngularWorkflow = (props) => {
 	const AppActionArguments = (props) => {
 		const [selectedActionParameters, setSelectedActionParameters] = React.useState([])
 		const [selectedVariableParameter, setSelectedVariableParameter] = React.useState()
+		const [showDropdown, setShowDropdown] = React.useState(false)
+		const [actionlist, setActionlist] = React.useState([])
 
 		useEffect(() => {
 			if (selectedActionParameters !== null && selectedActionParameters.length === 0) {
@@ -2257,9 +2265,51 @@ const AngularWorkflow = (props) => {
 				setSelectedVariableParameter(workflow.workflow_variables[0].name)
 			} 
 
+			if (actionlist.length === 0) {
+				actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "$exec"})
+				if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) {
+					for (var key in workflow.workflow_variables) {
+						const item = workflow.workflow_variables[key]
+						actionlist.push({"type": "workflow_variable", "name": item.name, "value": item.value, "id": item.id, "autocomplete": `${item.name.split(" ").join("_")}`})
+					}
+				}
+
+				// FIXME: Add values from previous executions if they exist
+				if (workflow.execution_variables !== null && workflow.execution_variables !== undefined && workflow.execution_variables.length > 0) {
+					for (var key in workflow.execution_variables) {
+						const item = workflow.execution_variables[key]
+						actionlist.push({"type": "execution_variable", "name": item.name, "value": item.value, "id": item.id, "autocomplete": `${item.name.split(" ").join("_")}`})
+					}
+				}
+
+				var parents = getParents(selectedAction)
+				if (parents.length > 1) {
+					for (var key in parents) {
+						const item = parents[key]
+						if (item.label === "Execution Argument") {
+							continue
+						}
+						actionlist.push({"type": "action", "id": item.id, "name": item.label, "autocomplete": `${item.label.split(" ").join("_")}`})
+					}
+				}
+
+				setActionlist(actionlist)
+			}
 		})
 
 		const changeActionParameter = (event, count) => {
+			console.log("EVENT: ", event.target.value)
+			if (event.target.value[event.target.value.length-1] === "$") {
+				console.log("LAST IS $ - SHOULD SHOW DROPDOWN")
+				if (!showDropdown) {
+					setShowDropdown(true)
+				}
+			} else {
+				if (showDropdown) {
+					setShowDropdown(false)
+				}
+			}
+
 			selectedActionParameters[count].value = event.target.value
 			selectedAction.parameters[count].value = event.target.value
 			setSelectedAction(selectedAction)
@@ -2344,6 +2394,38 @@ const AngularWorkflow = (props) => {
 		if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) {
 			return (
 				<div style={{marginTop: "30px"}}>
+
+					{showDropdown ?
+						<Select
+							SelectDisplayProps={{
+								style: {
+									marginLeft: 10,
+								}
+							}}
+							fullWidth
+							onChange={(e) => {
+								console.log("SELECTED: ", e.target.value)
+								const count = 0
+
+								console.log(selectedActionParameters)
+								selectedActionParameters[count].value += e.target.value.autocomplete
+								selectedAction.parameters[count].value = selectedActionParameters[count].value 
+								setSelectedAction(selectedAction)
+								setUpdate("action"+e.target.value.name)
+
+								setShowDropdown(false)
+							}}
+							style={{backgroundColor: surfaceColor, color: "white", height: "50px"}}
+							>
+							{actionlist.map(data => (
+								<MenuItem key={data.name} style={{backgroundColor: inputColor, color: "white"}} value={data}>
+									{data.name}
+								</MenuItem>
+							))}
+						</Select>
+					: null}
+
+
 					<b>Arguments</b>
 					{selectedActionParameters.map((data, count) => {
 						if (data.variant === "") {
@@ -2419,45 +2501,46 @@ const AngularWorkflow = (props) => {
 
 							datafield = 
 							<div>
-							<Select
+								<Select
-								SelectDisplayProps={{
+									SelectDisplayProps={{
-									style: {
+										style: {
-										marginLeft: 10,
+											marginLeft: 10,
-									}
+										}
-								}}
+									}}
-								value={selectedActionParameters[count].action_field}
+									value={selectedActionParameters[count].action_field}
-								fullWidth
+									fullWidth
-								onChange={(e) => {
+									onChange={(e) => {
-									changeActionParameterActionResult(e.target.value, count) 
+										changeActionParameterActionResult(e.target.value, count) 
-								}}
+									}}
-								style={{backgroundColor: surfaceColor, color: "white", height: "50px"}}
+									style={{backgroundColor: surfaceColor, color: "white", height: "50px"}}
-								>
+									>
-								{parents.map(data => (
+									{parents.map(data => (
-									<MenuItem key={data.label} style={{backgroundColor: inputColor, color: "white"}} value={data.label}>
+										<MenuItem key={data.label} style={{backgroundColor: inputColor, color: "white"}} value={data.label}>
-										{data.label}
+											{data.label}
-									</MenuItem>
+										</MenuItem>
-								))}
+									))}
-							</Select>
+								</Select>
-							<TextField
+								<TextField
-								style={{backgroundColor: inputColor}} 
+									style={{backgroundColor: inputColor}} 
-								InputProps={{
+									InputProps={{
-									style:{
+										style:{
-										color: "white",
+											color: "white",
-										marginLeft: "5px",
+											marginLeft: "5px",
-										maxWidth: "95%",
+											maxWidth: "95%",
-										height: "50px", 
+											height: "50px", 
-										fontSize: "1em",
+											fontSize: "1em",
-									},
+										},
-								}}
+									}}
-								fullWidth
+									fullWidth
-								color="primary"
+									color="primary"
-								defaultValue={data.value}
+									defaultValue={data.value}
-								helperText={<div style={{marginLeft: "5px", color:"white", marginBottom: "2px",}}>Example: $.body will get "data" from {'{"body": "data"}'}</div>}
+									helperText={<div style={{marginLeft: "5px", color:"white", marginBottom: "2px",}}>Example: $.body will get "data" from {'{"body": "data"}'}</div>}
-								placeholder="Action variable ($.)" 
+									placeholder="Action variable ($.)" 
-								onChange={(event) => {
+									onChange={(event) => {
-									changeActionParameter(event, count)
+										changeActionParameter(event, count)
-								}}
+									}}
-							/></div>
+								/>
+							</div>
 
 						} else if (data.variant === "WORKFLOW_VARIABLE") {
 							varcolor = "#f85a3e"
@@ -2469,7 +2552,10 @@ const AngularWorkflow = (props) => {
 									Looks like you don't have any variables yet.  
 								</div>
 								<div style={{width: "100%", margin: "auto"}}>
-									<Button style={{margin: "auto", marginTop: "10px"}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button> 				
+									<Button style={{margin: "auto", marginTop: "10px"}} color="primary" variant="outlined" onClick={() => {
+										setVariablesModalOpen(true)
+										setLastSaved(false)
+									}}>New workflow variable</Button> 				
 								</div>
 								</div>
 							} else {

functions/onprem/orborus/orborus.go

@@ -52,8 +52,19 @@ type ExecutionRequest struct {
 	Type              string `json:"type"`
 }
 
+var dockercli *dockerclient.Client
+
+func init() {
+	var err error
+
+	dockercli, err = dockerclient.NewEnvClient()
+	if err != nil {
+		panic(fmt.Sprintf("Unable to create docker client: %s", err))
+	}
+}
+
 // Deploys the internal worker whenever something happens
-func deployWorker(cli *dockerclient.Client, image string, identifier string, env []string) {
+func deployWorker(image string, identifier string, env []string) {
 	// Binds is the actual "-v" volume.
 	hostConfig := &container.HostConfig{
 		LogConfig: container.LogConfig{
@@ -82,7 +93,7 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
 			},
 		}
 	} else {
-		//log.Printf("Bad config: %s. Using default.", baseUrl)
+		// USE PROXY
 	}
 
 	//test := &network.EndpointSettings{
@@ -91,7 +102,7 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
 	//NetworkID
 	//if connect.EndpointConfig.NetworkID != "NetworkID" {
 
-	cont, err := cli.ContainerCreate(
+	cont, err := dockercli.ContainerCreate(
 		context.Background(),
 		config,
 		hostConfig,
@@ -105,7 +116,7 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
 		return
 	}
 
-	err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
+	err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
 	if err != nil {
 		log.Printf("Failed to start container in environment %s: %s", environment, err)
 		return
@@ -141,17 +152,11 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
 func stopWorker(containername string) error {
 	ctx := context.Background()
 
-	cli, err := dockerclient.NewEnvClient()
-	if err != nil {
-		log.Println("Unable to create docker client")
-		return err
-	}
-
 	//	containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
 	//		All: true,
 	//	})
 
-	if err := cli.ContainerStop(ctx, containername, nil); err != nil {
+	if err := dockercli.ContainerStop(ctx, containername, nil); err != nil {
 		log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err)
 	}
 
@@ -160,14 +165,14 @@ func stopWorker(containername string) error {
 		Force:         true,
 	}
 
-	if err := cli.ContainerRemove(ctx, containername, removeOptions); err != nil {
+	if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil {
 		log.Printf("Unable to remove container: %s", err)
 	}
 
 	return nil
 }
 
-func initializeImages(dockercli *dockerclient.Client) {
+func initializeImages() {
 	ctx := context.Background()
 
 	// check whether theyre the same first
@@ -209,6 +214,8 @@ func main() {
 	}
 
 	log.Printf("Running towards %s with Org %s", baseUrl, orgId)
+	httpProxy := os.Getenv("HTTP_PROXY")
+	httpsProxy := os.Getenv("HTTPS_PROXY")
 
 	if environment == "" {
 		environment = "onprem"
@@ -217,14 +224,8 @@ func main() {
 
 	// FIXME - during init, BUILD and/or LOAD worker and app_sdk
 	// Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk
-	dockercli, err := dockerclient.NewEnvClient()
-	if err != nil {
-		fmt.Println("Unable to create docker client")
-		os.Exit(3)
-	}
-
 	log.Printf("--- Setting up Docker environment. Downloading worker and App SDK! ---")
-	initializeImages(dockercli)
+	initializeImages()
 
 	//workerName := "worker"
 	//workerVersion := "0.1.0"
@@ -234,7 +235,22 @@ func main() {
 	log.Printf("--- Finished configuring docker environment ---\n")
 
 	// FIXME - time limit
-	client := &http.Client{}
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: nil,
+		},
+	}
+
+	if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
+		client = &http.Client{}
+	} else {
+		if len(httpProxy) > 0 {
+			log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
+		}
+		if len(httpsProxy) > 0 {
+			log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
+		}
+	}
 
 	fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl)
 	req, err := http.NewRequest(
@@ -341,15 +357,18 @@ func main() {
 				fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId),
 				fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
 				fmt.Sprintf("BASE_URL=%s", baseUrl),
-				fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")),
+			}
-				fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")),
+
+			if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" {
+				env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")))
+				env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")))
 			}
 
 			if dockerApiVersion != "" {
 				env = append(env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion))
 			}
 
-			go deployWorker(dockercli, workerImage, containerName, env)
+			go deployWorker(workerImage, containerName, env)
 
 			log.Printf("%s is deployed and to be removed from queue.", execution.ExecutionId)
 			zombiecounter += 1
@@ -416,19 +435,18 @@ func main() {
 // FIXME - add this to remove exited workers
 // Should it check what happened to the execution? idk
 func zombiecheck() error {
-	log.Println("Running zombiecheck")
+	log.Println("Looking for old containers")
 	ctx := context.Background()
 
-	dockercli, err := dockerclient.NewEnvClient()
-	if err != nil {
-		log.Println("Unable to create docker client")
-		return err
-	}
-
 	containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
 		All: true,
 	})
 
+	if err != nil {
+		log.Printf("Failed creating Containerlist: %s", err)
+		return err
+	}
+
 	containerNames := map[string]string{}
 
 	stopContainers := []string{}

functions/onprem/worker/worker.go

@@ -304,7 +304,25 @@ func shutdown(executionId, workflowId string) {
 
 	req.Header.Add("Content-Type", "application/json")
 	//req.Header.Add("Authorization", authorization)
-	client := &http.Client{}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: nil,
+		},
+	}
+
+	httpProxy := os.Getenv("HTTP_PROXY")
+	httpsProxy := os.Getenv("HTTPS_PROXY")
+	if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
+		client = &http.Client{}
+	} else {
+		if len(httpProxy) > 0 {
+			log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
+		}
+		if len(httpsProxy) > 0 {
+			log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
+		}
+	}
 	_, err = client.Do(req)
 	if err != nil {
 		log.Printf("Failed abort request: %s", err)
@@ -984,7 +1002,25 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s
 func main() {
 	log.Printf("Setting up worker environment")
 	sleepTime := 5
-	client := &http.Client{}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			Proxy: nil,
+		},
+	}
+
+	httpProxy := os.Getenv("HTTP_PROXY")
+	httpsProxy := os.Getenv("HTTPS_PROXY")
+	if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
+		client = &http.Client{}
+	} else {
+		if len(httpProxy) > 0 {
+			log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
+		}
+		if len(httpsProxy) > 0 {
+			log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
+		}
+	}
 
 	// WORKER_TESTING_WORKFLOW should be a workflow ID
 	authorization := ""