marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

#189: Fixed proxy and tagging issues with Docker build and runs

Browse Source
This commit is contained in:
frikky
2021-04-26 18:55:52 +02:00
parent 166787e212
commit 1adf20b43c
11 changed files with 48 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+1
View File
@@ -38,6 +38,7 @@ DB_LOCATION=./shuffle-database
SHUFFLE_HTTP_PROXY=
SHUFFLE_HTTPS_PROXY=
SHUFFLE_PASS_WORKER_PROXY=TRUE
SHUFFLE_PASS_APP_PROXY=FALSE
SHUFFLE_BASE_IMAGE_REGISTRY=ghcr.io
SHUFFLE_BASE_IMAGE_NAME=frikky
backend/app_sdk/app_base.py
+2 -1
View File
@@ -1621,13 +1621,14 @@ class AppBase:
return True, ""
# THE START IS ACTUALLY RIGHT HERE :O
# Checks whether conditions are met, otherwise set
branchcheck, tmpresult = check_branch_conditions(action, fullexecution)
if isinstance(tmpresult, object) or isinstance(tmpresult, list):
print("Fixing branch return as object -> string")
try:
tmpresult = tmpresult.replace("'", "\"")
#tmpresult = tmpresult.replace("'", "\"")
tmpresult = json.dumps(tmpresult)
except json.decoder.JSONDecodeError as e:
print(f"[WARNING] Failed condition parsing {tmpresult} to string")
backend/go-app/docker.go
+19 -1
View File
@@ -214,11 +214,28 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
// Dockerfile is inside the TAR itself. Not local context
// docker build --build-arg http_proxy=http://my.proxy.url
// Attempt at setting name according to #359: https://github.com/frikky/Shuffle/issues/359
labels := map[string]string{}
target := ""
if len(tags) > 0 {
if strings.Contains(tags[0], ":") {
version := strings.Split(tags[0], ":")
if len(version) == 2 {
target = fmt.Sprintf("shuffle-build-%s", version[1])
tags = append(tags, target)
labels["name"] = target
}
}
}
_ = labels
buildOptions := types.ImageBuildOptions{
Remove: true,
Tags: tags,
BuildArgs: map[string]*string{},
Labels: labels,
}
// NetworkMode: "host",
httpProxy := os.Getenv("HTTP_PROXY")
@@ -231,13 +248,14 @@ func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder strin
}
// Build the actual image
log.Printf("[INFO] Building %s. This may take up to a few minutes.", dockerfileFolder)
log.Printf(`[INFO] Building %s with proxy "%s". Tags: "%s". This may take up to a few minutes.`, dockerfileFolder, httpsProxy, strings.Join(tags, ","))
imageBuildResponse, err := client.ImageBuild(
ctx,
dockerFileTarReader,
buildOptions,
)
log.Printf("RESPONSE: %#v", imageBuildResponse)
//log.Printf("Response: %#v", imageBuildResponse.Body)
//log.Printf("IMAGERESPONSE: %#v", imageBuildResponse.Body)
backend/go-app/go.mod
+1 -1
View File
@@ -2,7 +2,7 @@ module shuffle
go 1.13
//replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
replace github.com/frikky/shuffle-shared => ../../../../git/shuffle-shared
//replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi
backend/go-app/go.sum
+2
View File
@@ -111,6 +111,8 @@ github.com/frikky/shuffle-shared v0.0.28 h1:VQqL3+ePwKSUxCOiCC8DpOEgbb2GhXI8XzFB
github.com/frikky/shuffle-shared v0.0.28/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ=
github.com/frikky/shuffle-shared v0.0.32 h1:Uy/zcAetSVYtRr3HEkUb7aE7Ggm0oSFxVeUNsi6q4uc=
github.com/frikky/shuffle-shared v0.0.32/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ=
github.com/frikky/shuffle-shared v0.0.37 h1:6nN1Im22TBuWUCG5L619xTgEPIPXCY68QThIfNiuG8k=
github.com/frikky/shuffle-shared v0.0.37/go.mod h1:H7SqOta/EAYnfYuWzwzYSh/oWfF0kgnuaJTQNKQBvoQ=
github.com/getkin/kin-openapi v0.8.0 h1:a6TQjTqwkyscC4/hShJX7WhCVE+4bi9lzw61XHQW5hE=
github.com/getkin/kin-openapi v0.8.0/go.mod h1:zZQMFkVgRHCdhgb6ihCTIo9dyDZFvX0k/xAKqw1FhPw=
github.com/getkin/kin-openapi v0.52.0 h1:6WqsF5d6PfJ8AscdD+9Rtb2RP2iBWyC7V6GcjssWg7M=
docker-compose.yml
+5 -4
View File
@@ -47,7 +47,7 @@ services:
- database
orborus:
#build: ./functions/onprem/orborus
image: ghcr.io/frikky/shuffle-orborus:0.8.75
image: ghcr.io/frikky/shuffle-orborus:0.8.76
container_name: shuffle-orborus
hostname: shuffle-orborus
networks:
@@ -61,13 +61,14 @@ services:
- ENVIRONMENT_NAME=${ENVIRONMENT_NAME}
- BASE_URL=http://${OUTER_HOSTNAME}:${BACKEND_PORT}
- DOCKER_API_VERSION=1.40
- HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
- HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_ORBORUS_EXECUTION_TIMEOUT=600
- SHUFFLE_BASE_IMAGE_NAME=${SHUFFLE_BASE_IMAGE_NAME}
- SHUFFLE_BASE_IMAGE_REGISTRY=${SHUFFLE_BASE_IMAGE_REGISTRY}
- SHUFFLE_BASE_IMAGE_TAG_SUFFIX=${SHUFFLE_BASE_IMAGE_TAG_SUFFIX}
- HTTP_PROXY=${SHUFFLE_HTTP_PROXY}
- HTTPS_PROXY=${SHUFFLE_HTTPS_PROXY}
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
- CLEANUP=${SHUFFLE_CONTAINER_AUTO_CLEANUP}
restart: unless-stopped
database:
frontend/src/views/Workflows.jsx
+2
View File
@@ -1467,6 +1467,8 @@ const Workflows = (props) => {
color="primary"
defaultValue={newWorkflowDescription}
placeholder="Description"
rows="6"
multiline
margin="dense"
fullWidth
/>
functions/onprem/orborus/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-orborus
VERSION=0.8.75
VERSION=0.8.76
echo "Running docker build with $NAME:$VERSION"
#docker rmi frikky/shuffle:$NAME --force
functions/onprem/orborus/orborus.go
+6 -4
View File
@@ -244,11 +244,11 @@ func initializeImages() {
ctx := context.Background()
if appSdkVersion == "" {
appSdkVersion = "0.8.60"
appSdkVersion = "0.8.75"
log.Printf("[WARNING] SHUFFLE_APP_SDK_VERSION not defined. Defaulting to %s", appSdkVersion)
}
if workerVersion == "" {
workerVersion = "0.8.72"
workerVersion = "0.8.76"
log.Printf("[WARNING] SHUFFLE_WORKER_VERSION not defined. Defaulting to %s", workerVersion)
}
@@ -553,9 +553,11 @@ func main() {
fmt.Sprintf("ENVIRONMENT_NAME=%s", environment),
fmt.Sprintf("BASE_URL=%s", baseUrl),
fmt.Sprintf("CLEANUP=%s", cleanupEnv),
fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY")),
}
if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) != "false" {
//log.Printf("Running worker with proxy? %s", os.Getenv("SHUFFLE_PASS_WORKER_PROXY"))
if strings.ToLower(os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) == "true" {
env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")))
env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")))
}
@@ -715,7 +717,7 @@ func zombiecheck(ctx context.Context, workerTimeout int) error {
// Check image name
if !shuffleFound {
log.Printf("Skipping: %s, %s", container.Labels, container.Image)
log.Printf("[WARNING] Zombie container skip: %#v, %s", container.Labels, container.Image)
continue
}
//} else {
functions/onprem/worker/build.sh
+1 -1
View File
@@ -1,5 +1,5 @@
NAME=shuffle-worker
VERSION=0.8.75
VERSION=0.8.76
echo "Running docker build with $NAME:$VERSION"
#CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o worker.bin .
functions/onprem/worker/worker.go
+8 -2
View File
@@ -934,6 +934,12 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
fmt.Sprintf("BASE_URL=%s", appCallbackUrl),
}
if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" {
//log.Printf("APPENDING PROXY TO THE APP!")
env = append(env, fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")))
env = append(env, fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")))
}
// Fixes issue:
// standard_init_linux.go:185: exec user process caused "argument list too long"
// https://devblogs.microsoft.com/oldnewthing/20100203-00/?p=15083
@@ -1255,7 +1261,7 @@ func handleDefaultExecution(client *http.Client, req *http.Request, workflowExec
for {
//fullUrl := fmt.Sprintf("%s/api/v1/workflows/%s/executions/%s/abort", baseUrl, workflowExecution.Workflow.ID, workflowExecution.ExecutionId)
fullUrl := fmt.Sprintf("%s/api/v1/streams/results", baseUrl)
log.Printf("URL: %s", fullUrl)
//log.Printf("[INFO] URL: %s", fullUrl)
req, err := http.NewRequest(
"POST",
fullUrl,
@@ -1597,7 +1603,7 @@ func getWorkflowExecution(ctx context.Context, id string) (*shuffle.WorkflowExec
func sendResult(workflowExecution shuffle.WorkflowExecution, data []byte) {
if workflowExecution.ExecutionSource == "default" {
log.Printf("Not sending backend info since source is default")
log.Printf("[INFO] Not sending backend info since source is default")
return
}