marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed app building and worker/orborus algorithms

Browse Source
This commit is contained in:
frikky
2023-02-16 22:41:31 +01:00
parent f3292bc43e
commit 808c3befbf
8 changed files with 22 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
-1
View File
@@ -65,7 +65,6 @@ SHUFFLE_CHAT_DISABLED=false # Controls support chat
SHUFFLE_RERUN_SCHEDULE=300
SHUFFLE_DISABLE_RERUN_AND_ABORT=false
SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server
SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server
SHUFFLE_ORBORUS_PULL_TIME= # Definition in case Orborus is pulling too often/not often enough
# DATABASE CONFIGURATIONS
backend/app_sdk/app_base.py
+15 -14
View File
@@ -467,7 +467,7 @@ class AppBase:
finished = False
for i in range (0, 10):
try:
ret = requests.post(url, headers=headers, json=action_result, timeout=10)
ret = requests.post(url, headers=headers, json=action_result, timeout=10, verify=False)
self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200 or 201)")
if ret.status_code == 200 or ret.status_code == 201:
@@ -514,7 +514,7 @@ class AppBase:
action_result["status"] = "FAILURE"
action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url})
self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests")
ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result)
ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""")
except requests.exceptions.ConnectionError as e:
@@ -524,7 +524,7 @@ class AppBase:
action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url})
self.logger.info(f"[DEBUG] Before typeerror stream result: {e}")
ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result)
ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
#self.logger.info(f"[DEBUG] Result: {ret.status_code}")
#if ret.status_code != 200:
# pr
@@ -653,7 +653,7 @@ class AppBase:
#self.logger.info(f"RET: {ret.text}")
#self.logger.info(f"ID: {ret.status_code}")
url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values"
ret = requests.post(url, json=data)
ret = requests.post(url, json=data, verify=False)
if ret.status_code == 200:
json_value = ret.json()
if len(json_value["found"]) > 0:
@@ -1129,7 +1129,7 @@ class AppBase:
"User-Agent": "Shuffle 1.1.0",
}
ret = requests.get("%s%s" % (self.url, get_path), headers=headers)
ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
return ret.json()
#if ret1.status_code != 200:
# return {
@@ -1155,7 +1155,7 @@ class AppBase:
"User-Agent": "Shuffle 1.1.0",
}
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
if ret1.status_code != 200:
return None
@@ -1218,7 +1218,7 @@ class AppBase:
"User-Agent": "Shuffle 1.1.0",
}
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers)
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
self.logger.info("RET1 (file get): %s" % ret1.text)
if ret1.status_code != 200:
returns.append({
@@ -1229,7 +1229,7 @@ class AppBase:
continue
content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers)
ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False)
self.logger.info("RET2 (file get) done")
if ret2.status_code == 200:
tmpdata = ret1.json()
@@ -1265,7 +1265,7 @@ class AppBase:
"value": str(value),
}
response = requests.post(url, json=data)
response = requests.post(url, json=data, verify=False)
try:
allvalues = response.json()
allvalues["key"] = key
@@ -1287,7 +1287,7 @@ class AppBase:
"key": key,
}
value = requests.post(url, json=data)
value = requests.post(url, json=data, verify=False)
try:
allvalues = value.json()
self.logger.info("VAL1: ", allvalues)
@@ -1341,7 +1341,7 @@ class AppBase:
self.logger.info(f"KeyError in file setup: {e}")
pass
ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data)
ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False)
#self.logger.info(f"Ret CREATE: {ret.text}")
cur_id = ""
if ret.status_code == 200:
@@ -1373,7 +1373,7 @@ class AppBase:
files={"shuffle_file": (filename, curfile["data"])}
#open(filename,'rb')}
ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers)
ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False)
self.logger.info("Ret UPLOAD: %s" % ret.text)
self.logger.info("Ret2 UPLOAD: %d" % ret.status_code)
@@ -1440,7 +1440,7 @@ class AppBase:
# FIXME: Shouldn't skip this, but it's good for minimzing API calls
#try:
# ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result)
# ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
# self.logger.info("Workflow: %d" % ret.status_code)
# if ret.status_code != 200:
# self.logger.info(ret.text)
@@ -1469,7 +1469,8 @@ class AppBase:
ret = requests.post(
"%s/api/v1/streams/results" % (self.base_url),
headers=headers,
json=tmpdata
json=tmpdata,
verify=False
)
if ret.status_code == 200:
backend/go-app/go.mod
+1 -1
View File
@@ -19,7 +19,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.3.52
github.com/shuffle/shuffle-shared v0.3.59
golang.org/x/crypto v0.3.0
google.golang.org/api v0.103.0
google.golang.org/appengine v1.6.7
docker-compose.yml
+1 -1
View File
@@ -58,7 +58,7 @@ services:
security_opt:
- seccomp:unconfined
opensearch:
image: opensearchproject/opensearch:2.4.0
image: opensearchproject/opensearch:2.5.0
hostname: shuffle-opensearch
container_name: shuffle-opensearch
environment:
functions/onprem/orborus/go.mod
+1 -1
View File
@@ -8,7 +8,7 @@ require (
github.com/docker/docker v23.0.0+incompatible
github.com/mackerelio/go-osstat v0.2.3
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.3.52
github.com/shuffle/shuffle-shared v0.3.59
)
require (
functions/onprem/orborus/orborus.go
+1 -23
View File
@@ -929,8 +929,6 @@ func main() {
zombiecheck(ctx, workerTimeout)
log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment)
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if environment == "" {
environment = "onprem"
@@ -965,27 +963,7 @@ func main() {
log.Printf("[INFO] Finished configuring docker environment")
// FIXME - time limit
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
//getStats()
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
client.Timeout = 30 * time.Second
client := shuffle.GetExternalClient(baseUrl)
fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl)
req, err := http.NewRequest(
functions/onprem/worker/go.mod
+1 -1
View File
@@ -11,7 +11,7 @@ require (
github.com/gorilla/mux v1.8.0
github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.3.52
github.com/shuffle/shuffle-shared v0.3.59
)
require (
functions/onprem/worker/worker.go
+2 -36
View File
@@ -144,24 +144,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
}
req.Header.Add("Content-Type", "application/json")
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("[INFO][%s] Running with HTTP proxy %s (env: HTTP_PROXY)", workflowExecution.ExecutionId, httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("[INFO][%s] Running with HTTPS proxy %s (env: HTTPS_PROXY)", workflowExecution.ExecutionId, httpsProxy)
}
}
client := shuffle.GetExternalClient(baseUrl)
//log.Printf("[DEBUG][%s] All App Logs: %#v", workflowExecution.ExecutionId, allLogs)
_, err = client.Do(req)
@@ -2023,24 +2006,7 @@ func main() {
log.Printf("[INFO] Setting up worker environment")
sleepTime := 5
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
client := shuffle.GetExternalClient(baseUrl)
if timezone == "" {
timezone = "Europe/Amsterdam"