diff --git a/.env b/.env index ff6b1b54..b0e85331 100644 --- a/.env +++ b/.env @@ -58,6 +58,7 @@ SHUFFLE_BASE_IMAGE_TAG_SUFFIX="-0.8.80" # Used for auto-cleanup of containers. REALLY important at scale. SHUFFLE_CONTAINER_AUTO_CLEANUP=false SHUFFLE_ELASTIC=true +SHUFFLE_LOGS_DISABLED=false # DATABASE CONFIGURATIONS DATASTORE_EMULATOR_HOST=shuffle-database:8000 diff --git a/backend/app_sdk/app_base.py b/backend/app_sdk/app_base.py index bcbfdf4a..3e889533 100644 --- a/backend/app_sdk/app_base.py +++ b/backend/app_sdk/app_base.py @@ -127,11 +127,12 @@ class AppBase: def __init__(self, redis=None, logger=None, console_logger=None):#, docker_client=None): self.logger = logger if logger is not None else logging.getLogger("AppBaseLogger") - #self.log_capture_string = StringBuffer() - #ch = logging.StreamHandler(self.log_capture_string) - #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - #ch.setFormatter(formatter) - #logger.addHandler(ch) + if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": + self.log_capture_string = StringBuffer() + ch = logging.StreamHandler(self.log_capture_string) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) self.redis=redis self.console_logger = logger if logger is not None else logging.getLogger("AppBaseLogger") @@ -299,8 +300,9 @@ class AppBase: self.logger.info(f"[INFO] URL FOR RESULT (URL): {url}") try: - #log_contents = self.log_capture_string.getvalue() log_contents = "disabled" + if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": + log_contents = self.log_capture_string.getvalue() #print("RESULTS: %s" % log_contents) self.logger.info(f"[WARNING] Got logs of length {len(log_contents)}") @@ -377,12 +379,14 @@ class AppBase: except urllib3.exceptions.ProtocolError as e: self.logger.info(f"[DEBUG] Expected ProtocolError happened: {e}") - #try: - # self.log_capture_string.flush() - # self.log_capture_string.close() - #except Exception as e: - # print(f"[WARNING] Failed to flush logs: {e}") - # pass + + if not os.getenv("SHUFFLE_LOGS_DISABLED") == "true": + try: + self.log_capture_string.flush() + self.log_capture_string.close() + except Exception as e: + print(f"[WARNING] Failed to flush logs: {e}") + pass #async def cartesian_product(self, L): def cartesian_product(self, L): @@ -3201,7 +3205,7 @@ class AppBase: # https://ptb.discord.com/channels/747075026288902237/882017498550112286/882043773138382890 except (requests.exceptions.RequestException, TimeoutError) as e: - self.logger.info(f"Failed to execute request: {e}") + self.logger.info(f"Failed to execute request (requests): {e}") self.logger.exception(f"Failed to execute {e}-{action['id']}") self.action_result["status"] = "SUCCESS" try: diff --git a/backend/app_sdk/build.sh b/backend/app_sdk/build.sh index 1b26783d..c889c20b 100644 --- a/backend/app_sdk/build.sh +++ b/backend/app_sdk/build.sh @@ -3,7 +3,7 @@ ### DEFAULT NAME=shuffle-app_sdk -VERSION=0.9.63 +VERSION=0.9.64 docker rmi docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION --force docker build . -f Dockerfile -t frikky/shuffle:app_sdk -t frikky/$NAME:$VERSION -t docker.pkg.github.com/frikky/shuffle/$NAME:$VERSION -t ghcr.io/frikky/$NAME:$VERSION -t ghcr.io/frikky/$NAME:nightly diff --git a/backend/go-app/go.mod b/backend/go-app/go.mod index 9324512a..5fea5ec6 100644 --- a/backend/go-app/go.mod +++ b/backend/go-app/go.mod @@ -2,7 +2,7 @@ module main go 1.16 -replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared +//replace github.com/shuffle/shuffle-shared => ../../../shuffle-shared //replace github.com/frikky/kin-openapi => ../../../../git/kin-openapi //replace github.com/frikky/go-elasticsearch => ../../../../git/go-elasticsearch @@ -24,7 +24,7 @@ require ( github.com/h2non/filetype v1.1.3 github.com/nirasan/go-oauth-pkce-code-verifier v0.0.0-20170819232839-0fbfe93532da // indirect github.com/satori/go.uuid v1.2.0 - github.com/shuffle/shuffle-shared v0.2.9 + github.com/shuffle/shuffle-shared v0.2.11 go4.org v0.0.0-20201209231011-d4a079459e60 // indirect golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce google.golang.org/api v0.65.0 diff --git a/frontend/src/views/Workflows.jsx b/frontend/src/views/Workflows.jsx index 9a3249d9..66232fb0 100644 --- a/frontend/src/views/Workflows.jsx +++ b/frontend/src/views/Workflows.jsx @@ -1699,7 +1699,7 @@ const Workflows = (props) => { {data.image !== undefined && data.image !== null && data.image.length > 0 ? - {data.name} + {data.name} : null} Edit {data.name} diff --git a/functions/onprem/orborus/build.sh b/functions/onprem/orborus/build.sh index 75620b0e..7e4d42fb 100644 --- a/functions/onprem/orborus/build.sh +++ b/functions/onprem/orborus/build.sh @@ -1,5 +1,5 @@ NAME=shuffle-orborus -VERSION=0.9.62 +VERSION=0.9.64 echo "Running docker build with $NAME:$VERSION" #docker rmi frikky/shuffle:$NAME --force diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go index 8c5327e2..56831574 100644 --- a/functions/onprem/orborus/orborus.go +++ b/functions/onprem/orborus/orborus.go @@ -366,6 +366,7 @@ func deployServiceWorkers(image string) { fmt.Sprintf("SHUFFLE_SWARM_NETWORK_NAME=%s", networkName), fmt.Sprintf("SHUFFLE_APP_REPLICAS=%d", cnt), fmt.Sprintf("TZ=%s", timezone), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), }, Hosts: []string{ innerContainerName, @@ -1032,6 +1033,7 @@ func main() { fmt.Sprintf("TZ=%s", timezone), fmt.Sprintf("SHUFFLE_PASS_APP_PROXY=%s", os.Getenv("SHUFFLE_PASS_APP_PROXY")), fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), } //log.Printf("Running worker with proxy? %s", os.Getenv("SHUFFLE_PASS_WORKER_PROXY")) diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go index d7a5d6e4..29e911ff 100644 --- a/functions/onprem/worker/worker.go +++ b/functions/onprem/worker/worker.go @@ -1196,6 +1196,7 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) { fmt.Sprintf("CALLBACK_URL=%s", baseUrl), fmt.Sprintf("BASE_URL=%s", appCallbackUrl), fmt.Sprintf("TZ=%s", timezone), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), } if len(actionData) >= 100000 { @@ -1932,6 +1933,7 @@ func runUserInput(client *http.Client, action shuffle.Action, workflowId string, fmt.Sprintf("CALLBACK_URL=%s", baseUrl), fmt.Sprintf("BASE_URL=%s", appCallbackUrl), fmt.Sprintf("TZ=%s", timezone), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), } if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" { @@ -2641,8 +2643,9 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl ContainerSpec: &swarm.ContainerSpec{ Image: image, Env: []string{ - fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), fmt.Sprintf("SHUFFLE_APP_EXPOSED_PORT=%d", deployport), + fmt.Sprintf("SHUFFLE_SWARM_CONFIG=%s", os.Getenv("SHUFFLE_SWARM_CONFIG")), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), }, Hosts: []string{ containerName, @@ -3006,6 +3009,7 @@ func baseDeploy() { fmt.Sprintf("CALLBACK_URL=%s", baseUrl), fmt.Sprintf("BASE_URL=%s", appCallbackUrl), fmt.Sprintf("TZ=%s", timezone), + fmt.Sprintf("SHUFFLE_LOGS_DISABLED=%s", os.Getenv("SHUFFLE_LOGS_DISABLED")), } if strings.ToLower(os.Getenv("SHUFFLE_PASS_APP_PROXY")) == "true" {