marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #88 from frikky/1.0.0

Browse Source 
Fixed Docker proxy issues
This commit is contained in:
Frikky
2020-07-07 14:52:31 +09:00
committed by GitHub
parent 6fc8fc3e67 00b17f9617
commit 4b51175058
6 changed files with 230 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+6
View File
@@ -22,3 +22,9 @@ FRONTEND_PORT=3001
FRONTEND_PORT_HTTPS=3443
OUTER_HOSTNAME=shuffle-backend
DB_LOCATION=./shuffle-database
# Proxy configurations. SHUFFLE_PASS_WORKER_PROXY must be FALSE to not pass the proxy information to sub-apps.
# PS: It will skip proxy for
SHUFFLE_HTTP_PROXY=
SHUFFLE_HTTPS_PROXY=
SHUFFLE_PASS_WORKER_PROXY=TRUE
backend/go-app/main.go
+4
View File
@@ -6157,6 +6157,10 @@ func runInit(ctx context.Context) {
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
httpsProxy := os.Getenv("HTTPS_PROXY")
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
/*
proxyUrl, err := url.Parse(httpProxy)
docker-compose.yml
+5
View File
@@ -34,6 +34,8 @@ services:
- SHUFFLE_DEFAULT_USERNAME=${SHUFFLE_DEFAULT_USERNAME}
- SHUFFLE_DEFAULT_PASSWORD=${SHUFFLE_DEFAULT_PASSWORD}
- SHUFFLE_DEFAULT_APIKEY=${SHUFFLE_DEFAULT_APIKEY}
- HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
- HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
restart: unless-stopped
depends_on:
- database
@@ -51,6 +53,9 @@ services:
- ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
- BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}
- DOCKER_API_VERSION=1.40
- HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
- HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
restart: unless-stopped
database:
#build: ./backend/database
frontend/src/AngularWorkflow.js
+128 -42
View File
@@ -1543,7 +1543,10 @@ const AngularWorkflow = (props) => {
)
})}
<div style={{flex: "1"}}>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => {
setVariablesModalOpen(true)
setLastSaved(false)
}}>New workflow variable</Button>
</div>
<Divider style={{marginBottom: 20, marginTop: 20, height: 1, width: "100%", backgroundColor: "rgb(91, 96, 100)"}}/>
What are <a href="https://shuffler.io/docs/workflows#execution_variables" target="_blank" style={{textDecoration: "none", color: "#f85a3e"}}>EXECUTION variables?</a>
@@ -1603,7 +1606,10 @@ const AngularWorkflow = (props) => {
)
})}
<div style={{flex: "1"}}>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => setExecutionVariablesModalOpen(true)}>New execution variable</Button>
<Button fullWidth style={{margin: "auto", marginTop: "10px",}} color="primary" variant="outlined" onClick={() => {
setExecutionVariablesModalOpen(true)
setLastSaved(false)
}}>New execution variable</Button>
</div>
</div>
</div>
@@ -2240,6 +2246,8 @@ const AngularWorkflow = (props) => {
const AppActionArguments = (props) => {
const [selectedActionParameters, setSelectedActionParameters] = React.useState([])
const [selectedVariableParameter, setSelectedVariableParameter] = React.useState()
const [showDropdown, setShowDropdown] = React.useState(false)
const [actionlist, setActionlist] = React.useState([])
useEffect(() => {
if (selectedActionParameters !== null && selectedActionParameters.length === 0) {
@@ -2257,9 +2265,51 @@ const AngularWorkflow = (props) => {
setSelectedVariableParameter(workflow.workflow_variables[0].name)
}
if (actionlist.length === 0) {
actionlist.push({"type": "Execution Argument", "name": "Execution Argument", "value": "$exec", "highlight": "exec", "autocomplete": "$exec"})
if (workflow.workflow_variables !== null && workflow.workflow_variables !== undefined && workflow.workflow_variables.length > 0) {
for (var key in workflow.workflow_variables) {
const item = workflow.workflow_variables[key]
actionlist.push({"type": "workflow_variable", "name": item.name, "value": item.value, "id": item.id, "autocomplete": `${item.name.split(" ").join("_")}`})
}
}
// FIXME: Add values from previous executions if they exist
if (workflow.execution_variables !== null && workflow.execution_variables !== undefined && workflow.execution_variables.length > 0) {
for (var key in workflow.execution_variables) {
const item = workflow.execution_variables[key]
actionlist.push({"type": "execution_variable", "name": item.name, "value": item.value, "id": item.id, "autocomplete": `${item.name.split(" ").join("_")}`})
}
}
var parents = getParents(selectedAction)
if (parents.length > 1) {
for (var key in parents) {
const item = parents[key]
if (item.label === "Execution Argument") {
continue
}
actionlist.push({"type": "action", "id": item.id, "name": item.label, "autocomplete": `${item.label.split(" ").join("_")}`})
}
}
setActionlist(actionlist)
}
})
const changeActionParameter = (event, count) => {
console.log("EVENT: ", event.target.value)
if (event.target.value[event.target.value.length-1] === "$") {
console.log("LAST IS $ - SHOULD SHOW DROPDOWN")
if (!showDropdown) {
setShowDropdown(true)
}
} else {
if (showDropdown) {
setShowDropdown(false)
}
}
selectedActionParameters[count].value = event.target.value
selectedAction.parameters[count].value = event.target.value
setSelectedAction(selectedAction)
@@ -2344,6 +2394,38 @@ const AngularWorkflow = (props) => {
if (Object.getOwnPropertyNames(selectedAction).length > 0 && selectedActionParameters.length > 0) {
return (
<div style={{marginTop: "30px"}}>
{showDropdown ?
<Select
SelectDisplayProps={{
style: {
marginLeft: 10,
}
}}
fullWidth
onChange={(e) => {
console.log("SELECTED: ", e.target.value)
const count = 0
console.log(selectedActionParameters)
selectedActionParameters[count].value += e.target.value.autocomplete
selectedAction.parameters[count].value = selectedActionParameters[count].value
setSelectedAction(selectedAction)
setUpdate("action"+e.target.value.name)
setShowDropdown(false)
}}
style={{backgroundColor: surfaceColor, color: "white", height: "50px"}}
>
{actionlist.map(data => (
<MenuItem key={data.name} style={{backgroundColor: inputColor, color: "white"}} value={data}>
{data.name}
</MenuItem>
))}
</Select>
: null}
<b>Arguments</b>
{selectedActionParameters.map((data, count) => {
if (data.variant === "") {
@@ -2419,45 +2501,46 @@ const AngularWorkflow = (props) => {
datafield =
<div>
<Select
SelectDisplayProps={{
style: {
marginLeft: 10,
}
}}
value={selectedActionParameters[count].action_field}
fullWidth
onChange={(e) => {
changeActionParameterActionResult(e.target.value, count)
}}
style={{backgroundColor: surfaceColor, color: "white", height: "50px"}}
>
{parents.map(data => (
<MenuItem key={data.label} style={{backgroundColor: inputColor, color: "white"}} value={data.label}>
{data.label}
</MenuItem>
))}
</Select>
<TextField
style={{backgroundColor: inputColor}}
InputProps={{
style:{
color: "white",
marginLeft: "5px",
maxWidth: "95%",
height: "50px",
fontSize: "1em",
},
}}
fullWidth
color="primary"
defaultValue={data.value}
helperText={<div style={{marginLeft: "5px", color:"white", marginBottom: "2px",}}>Example: $.body will get "data" from {'{"body": "data"}'}</div>}
placeholder="Action variable ($.)"
onChange={(event) => {
changeActionParameter(event, count)
}}
/></div>
<Select
SelectDisplayProps={{
style: {
marginLeft: 10,
}
}}
value={selectedActionParameters[count].action_field}
fullWidth
onChange={(e) => {
changeActionParameterActionResult(e.target.value, count)
}}
style={{backgroundColor: surfaceColor, color: "white", height: "50px"}}
>
{parents.map(data => (
<MenuItem key={data.label} style={{backgroundColor: inputColor, color: "white"}} value={data.label}>
{data.label}
</MenuItem>
))}
</Select>
<TextField
style={{backgroundColor: inputColor}}
InputProps={{
style:{
color: "white",
marginLeft: "5px",
maxWidth: "95%",
height: "50px",
fontSize: "1em",
},
}}
fullWidth
color="primary"
defaultValue={data.value}
helperText={<div style={{marginLeft: "5px", color:"white", marginBottom: "2px",}}>Example: $.body will get "data" from {'{"body": "data"}'}</div>}
placeholder="Action variable ($.)"
onChange={(event) => {
changeActionParameter(event, count)
}}
/>
</div>
} else if (data.variant === "WORKFLOW_VARIABLE") {
varcolor = "#f85a3e"
@@ -2469,7 +2552,10 @@ const AngularWorkflow = (props) => {
Looks like you don't have any variables yet.
</div>
<div style={{width: "100%", margin: "auto"}}>
<Button style={{margin: "auto", marginTop: "10px"}} color="primary" variant="outlined" onClick={() => setVariablesModalOpen(true)}>New workflow variable</Button>
<Button style={{margin: "auto", marginTop: "10px"}} color="primary" variant="outlined" onClick={() => {
setVariablesModalOpen(true)
setLastSaved(false)
}}>New workflow variable</Button>
</div>
</div>
} else {
functions/onprem/orborus/orborus.go
+49 -31
View File
@@ -52,8 +52,19 @@ type ExecutionRequest struct {
Type string `json:"type"`
}
var dockercli *dockerclient.Client
func init() {
var err error
dockercli, err = dockerclient.NewEnvClient()
if err != nil {
panic(fmt.Sprintf("Unable to create docker client: %s", err))
}
}
// Deploys the internal worker whenever something happens
func deployWorker(cli *dockerclient.Client, image string, identifier string, env []string) {
func deployWorker(image string, identifier string, env []string) {
// Binds is the actual "-v" volume.
hostConfig := &container.HostConfig{
LogConfig: container.LogConfig{
@@ -82,7 +93,7 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
},
}
} else {
//log.Printf("Bad config: %s. Using default.", baseUrl)
// USE PROXY
}
//test := &network.EndpointSettings{
@@ -91,7 +102,7 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
//NetworkID
//if connect.EndpointConfig.NetworkID != "NetworkID" {
cont, err := cli.ContainerCreate(
cont, err := dockercli.ContainerCreate(
context.Background(),
config,
hostConfig,
@@ -105,7 +116,7 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
return
}
err = cli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
err = dockercli.ContainerStart(context.Background(), cont.ID, types.ContainerStartOptions{})
if err != nil {
log.Printf("Failed to start container in environment %s: %s", environment, err)
return
@@ -141,17 +152,11 @@ func deployWorker(cli *dockerclient.Client, image string, identifier string, env
func stopWorker(containername string) error {
ctx := context.Background()
cli, err := dockerclient.NewEnvClient()
if err != nil {
log.Println("Unable to create docker client")
return err
}
// containers, err := cli.ContainerList(ctx, types.ContainerListOptions{
// All: true,
// })
if err := cli.ContainerStop(ctx, containername, nil); err != nil {
if err := dockercli.ContainerStop(ctx, containername, nil); err != nil {
log.Printf("Unable to stop container %s - running removal anyway, just in case: %s", containername, err)
}
@@ -160,14 +165,14 @@ func stopWorker(containername string) error {
Force: true,
}
if err := cli.ContainerRemove(ctx, containername, removeOptions); err != nil {
if err := dockercli.ContainerRemove(ctx, containername, removeOptions); err != nil {
log.Printf("Unable to remove container: %s", err)
}
return nil
}
func initializeImages(dockercli *dockerclient.Client) {
func initializeImages() {
ctx := context.Background()
// check whether theyre the same first
@@ -209,6 +214,8 @@ func main() {
}
log.Printf("Running towards %s with Org %s", baseUrl, orgId)
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if environment == "" {
environment = "onprem"
@@ -217,14 +224,8 @@ func main() {
// FIXME - during init, BUILD and/or LOAD worker and app_sdk
// Build/load app_sdk so it can be loaded as 127.0.0.1:5000/walkoff_app_sdk
dockercli, err := dockerclient.NewEnvClient()
if err != nil {
fmt.Println("Unable to create docker client")
os.Exit(3)
}
log.Printf("--- Setting up Docker environment. Downloading worker and App SDK! ---")
initializeImages(dockercli)
initializeImages()
//workerName := "worker"
//workerVersion := "0.1.0"
@@ -234,7 +235,22 @@ func main() {
log.Printf("--- Finished configuring docker environment ---\n")
// FIXME - time limit
client := &http.Client{}
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl)
req, err := http.NewRequest(
@@ -341,15 +357,18 @@ func main() {
fmt.Sprintf("EXECUTIONID=%s", execution.ExecutionId),
fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
fmt.Sprintf("BASE_URL=%s", baseUrl),
fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")),
fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")),
}
if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" {
env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")))
env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")))
}
if dockerApiVersion != "" {
env = append(env, fmt.Sprintf("DOCKER_API_VERSION=%s", dockerApiVersion))
}
go deployWorker(dockercli, workerImage, containerName, env)
go deployWorker(workerImage, containerName, env)
log.Printf("%s is deployed and to be removed from queue.", execution.ExecutionId)
zombiecounter += 1
@@ -416,19 +435,18 @@ func main() {
// FIXME - add this to remove exited workers
// Should it check what happened to the execution? idk
func zombiecheck() error {
log.Println("Running zombiecheck")
log.Println("Looking for old containers")
ctx := context.Background()
dockercli, err := dockerclient.NewEnvClient()
if err != nil {
log.Println("Unable to create docker client")
return err
}
containers, err := dockercli.ContainerList(ctx, types.ContainerListOptions{
All: true,
})
if err != nil {
log.Printf("Failed creating Containerlist: %s", err)
return err
}
containerNames := map[string]string{}
stopContainers := []string{}
functions/onprem/worker/worker.go
+38 -2
View File
@@ -304,7 +304,25 @@ func shutdown(executionId, workflowId string) {
req.Header.Add("Content-Type", "application/json")
//req.Header.Add("Authorization", authorization)
client := &http.Client{}
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
_, err = client.Do(req)
if err != nil {
log.Printf("Failed abort request: %s", err)
@@ -984,7 +1002,25 @@ func runTestExecution(client *http.Client, workflowId, apikey string) (string, s
func main() {
log.Printf("Setting up worker environment")
sleepTime := 5
client := &http.Client{}
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
// WORKER_TESTING_WORKFLOW should be a workflow ID
authorization := ""