marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed app building and worker/orborus algorithms

Browse Source
This commit is contained in:
frikky
2023-02-16 22:41:31 +01:00
parent f3292bc43e
commit 808c3befbf
8 changed files with 22 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
-1
View File
@@ -65,7 +65,6 @@ SHUFFLE_CHAT_DISABLED=false # Controls support chat
SHUFFLE_RERUN_SCHEDULE=300 SHUFFLE_RERUN_SCHEDULE=300
SHUFFLE_DISABLE_RERUN_AND_ABORT=false SHUFFLE_DISABLE_RERUN_AND_ABORT=false
SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server
SHUFFLE_WORKER_SERVER_URL= # Definition in case Worker & Orborus is talking to the wrong server
SHUFFLE_ORBORUS_PULL_TIME= # Definition in case Orborus is pulling too often/not often enough SHUFFLE_ORBORUS_PULL_TIME= # Definition in case Orborus is pulling too often/not often enough
# DATABASE CONFIGURATIONS # DATABASE CONFIGURATIONS
backend/app_sdk/app_base.py
+15 -14
View File
@@ -467,7 +467,7 @@ class AppBase:
finished = False finished = False
for i in range (0, 10): for i in range (0, 10):
try: try:
ret = requests.post(url, headers=headers, json=action_result, timeout=10) ret = requests.post(url, headers=headers, json=action_result, timeout=10, verify=False)
self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200 or 201)") self.logger.info(f"[DEBUG] Result: {ret.status_code} (break on 200 or 201)")
if ret.status_code == 200 or ret.status_code == 201: if ret.status_code == 200 or ret.status_code == 201:
@@ -514,7 +514,7 @@ class AppBase:
action_result["status"] = "FAILURE" action_result["status"] = "FAILURE"
action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url}) action_result["result"] = json.dumps({"success": False, "reason": "POST error: Failed connecting to %s over 10 retries to the backend" % url})
self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests") self.logger.info(f"[DEBUG] Before typeerror stream result - NOT finished after 10 requests")
ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""") self.logger.info(f"""[DEBUG] Successful request result request: Status= {ret.status_code} & Response= {ret.text}. Action status: {action_result["status"]}""")
except requests.exceptions.ConnectionError as e: except requests.exceptions.ConnectionError as e:
@@ -524,7 +524,7 @@ class AppBase:
action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url}) action_result["result"] = json.dumps({"success": False, "reason": "Typeerror when sending to backend URL %s" % url})
self.logger.info(f"[DEBUG] Before typeerror stream result: {e}") self.logger.info(f"[DEBUG] Before typeerror stream result: {e}")
ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
#self.logger.info(f"[DEBUG] Result: {ret.status_code}") #self.logger.info(f"[DEBUG] Result: {ret.status_code}")
#if ret.status_code != 200: #if ret.status_code != 200:
# pr # pr
@@ -653,7 +653,7 @@ class AppBase:
#self.logger.info(f"RET: {ret.text}") #self.logger.info(f"RET: {ret.text}")
#self.logger.info(f"ID: {ret.status_code}") #self.logger.info(f"ID: {ret.status_code}")
url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values" url = f"{self.url}/api/v1/orgs/{org_id}/validate_app_values"
ret = requests.post(url, json=data) ret = requests.post(url, json=data, verify=False)
if ret.status_code == 200: if ret.status_code == 200:
json_value = ret.json() json_value = ret.json()
if len(json_value["found"]) > 0: if len(json_value["found"]) > 0:
@@ -1129,7 +1129,7 @@ class AppBase:
"User-Agent": "Shuffle 1.1.0", "User-Agent": "Shuffle 1.1.0",
} }
ret = requests.get("%s%s" % (self.url, get_path), headers=headers) ret = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
return ret.json() return ret.json()
#if ret1.status_code != 200: #if ret1.status_code != 200:
# return { # return {
@@ -1155,7 +1155,7 @@ class AppBase:
"User-Agent": "Shuffle 1.1.0", "User-Agent": "Shuffle 1.1.0",
} }
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
if ret1.status_code != 200: if ret1.status_code != 200:
return None return None
@@ -1218,7 +1218,7 @@ class AppBase:
"User-Agent": "Shuffle 1.1.0", "User-Agent": "Shuffle 1.1.0",
} }
ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers) ret1 = requests.get("%s%s" % (self.url, get_path), headers=headers, verify=False)
self.logger.info("RET1 (file get): %s" % ret1.text) self.logger.info("RET1 (file get): %s" % ret1.text)
if ret1.status_code != 200: if ret1.status_code != 200:
returns.append({ returns.append({
@@ -1229,7 +1229,7 @@ class AppBase:
continue continue
content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"]) content_path = "/api/v1/files/%s/content?execution_id=%s" % (item, full_execution["execution_id"])
ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers) ret2 = requests.get("%s%s" % (self.url, content_path), headers=headers, verify=False)
self.logger.info("RET2 (file get) done") self.logger.info("RET2 (file get) done")
if ret2.status_code == 200: if ret2.status_code == 200:
tmpdata = ret1.json() tmpdata = ret1.json()
@@ -1265,7 +1265,7 @@ class AppBase:
"value": str(value), "value": str(value),
} }
response = requests.post(url, json=data) response = requests.post(url, json=data, verify=False)
try: try:
allvalues = response.json() allvalues = response.json()
allvalues["key"] = key allvalues["key"] = key
@@ -1287,7 +1287,7 @@ class AppBase:
"key": key, "key": key,
} }
value = requests.post(url, json=data) value = requests.post(url, json=data, verify=False)
try: try:
allvalues = value.json() allvalues = value.json()
self.logger.info("VAL1: ", allvalues) self.logger.info("VAL1: ", allvalues)
@@ -1341,7 +1341,7 @@ class AppBase:
self.logger.info(f"KeyError in file setup: {e}") self.logger.info(f"KeyError in file setup: {e}")
pass pass
ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data) ret = requests.post("%s%s" % (self.url, create_path), headers=headers, json=data, verify=False)
#self.logger.info(f"Ret CREATE: {ret.text}") #self.logger.info(f"Ret CREATE: {ret.text}")
cur_id = "" cur_id = ""
if ret.status_code == 200: if ret.status_code == 200:
@@ -1373,7 +1373,7 @@ class AppBase:
files={"shuffle_file": (filename, curfile["data"])} files={"shuffle_file": (filename, curfile["data"])}
#open(filename,'rb')} #open(filename,'rb')}
ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers) ret = requests.post("%s%s" % (self.url, upload_path), files=files, headers=new_headers, verify=False)
self.logger.info("Ret UPLOAD: %s" % ret.text) self.logger.info("Ret UPLOAD: %s" % ret.text)
self.logger.info("Ret2 UPLOAD: %d" % ret.status_code) self.logger.info("Ret2 UPLOAD: %d" % ret.status_code)
@@ -1440,7 +1440,7 @@ class AppBase:
# FIXME: Shouldn't skip this, but it's good for minimzing API calls # FIXME: Shouldn't skip this, but it's good for minimzing API calls
#try: #try:
# ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result) # ret = requests.post("%s%s" % (self.base_url, stream_path), headers=headers, json=action_result, verify=False)
# self.logger.info("Workflow: %d" % ret.status_code) # self.logger.info("Workflow: %d" % ret.status_code)
# if ret.status_code != 200: # if ret.status_code != 200:
# self.logger.info(ret.text) # self.logger.info(ret.text)
@@ -1469,7 +1469,8 @@ class AppBase:
ret = requests.post( ret = requests.post(
"%s/api/v1/streams/results" % (self.base_url), "%s/api/v1/streams/results" % (self.base_url),
headers=headers, headers=headers,
json=tmpdata json=tmpdata,
verify=False
) )
if ret.status_code == 200: if ret.status_code == 200:
backend/go-app/go.mod
+1 -1
View File
@@ -19,7 +19,7 @@ require (
github.com/gorilla/mux v1.8.0 github.com/gorilla/mux v1.8.0
github.com/h2non/filetype v1.1.3 github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0 github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.3.52 github.com/shuffle/shuffle-shared v0.3.59
golang.org/x/crypto v0.3.0 golang.org/x/crypto v0.3.0
google.golang.org/api v0.103.0 google.golang.org/api v0.103.0
google.golang.org/appengine v1.6.7 google.golang.org/appengine v1.6.7
docker-compose.yml
+1 -1
View File
@@ -58,7 +58,7 @@ services:
security_opt: security_opt:
- seccomp:unconfined - seccomp:unconfined
opensearch: opensearch:
image: opensearchproject/opensearch:2.4.0 image: opensearchproject/opensearch:2.5.0
hostname: shuffle-opensearch hostname: shuffle-opensearch
container_name: shuffle-opensearch container_name: shuffle-opensearch
environment: environment:
functions/onprem/orborus/go.mod
+1 -1
View File
@@ -8,7 +8,7 @@ require (
github.com/docker/docker v23.0.0+incompatible github.com/docker/docker v23.0.0+incompatible
github.com/mackerelio/go-osstat v0.2.3 github.com/mackerelio/go-osstat v0.2.3
github.com/satori/go.uuid v1.2.0 github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.3.52 github.com/shuffle/shuffle-shared v0.3.59
) )
require ( require (
functions/onprem/orborus/orborus.go
+1 -23
View File
@@ -929,8 +929,6 @@ func main() {
zombiecheck(ctx, workerTimeout) zombiecheck(ctx, workerTimeout)
log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment) log.Printf("[INFO] Running towards %s (BASE_URL) with environment name %s", baseUrl, environment)
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if environment == "" { if environment == "" {
environment = "onprem" environment = "onprem"
@@ -965,27 +963,7 @@ func main() {
log.Printf("[INFO] Finished configuring docker environment") log.Printf("[INFO] Finished configuring docker environment")
// FIXME - time limit client := shuffle.GetExternalClient(baseUrl)
client := &http.Client{
Transport: &http.Transport{
Proxy: nil,
},
}
//getStats()
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("[INFO] Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("[INFO] Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
client.Timeout = 30 * time.Second
fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl) fullUrl := fmt.Sprintf("%s/api/v1/workflows/queue", baseUrl)
req, err := http.NewRequest( req, err := http.NewRequest(
functions/onprem/worker/go.mod
+1 -1
View File
@@ -11,7 +11,7 @@ require (
github.com/gorilla/mux v1.8.0 github.com/gorilla/mux v1.8.0
github.com/patrickmn/go-cache v2.1.0+incompatible github.com/patrickmn/go-cache v2.1.0+incompatible
github.com/satori/go.uuid v1.2.0 github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.3.52 github.com/shuffle/shuffle-shared v0.3.59
) )
require ( require (
functions/onprem/worker/worker.go
+2 -36
View File
@@ -144,24 +144,7 @@ func shutdown(workflowExecution shuffle.WorkflowExecution, nodeId string, reason
} }
req.Header.Add("Content-Type", "application/json") req.Header.Add("Content-Type", "application/json")
client := &http.Client{ client := shuffle.GetExternalClient(baseUrl)
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("[INFO][%s] Running with HTTP proxy %s (env: HTTP_PROXY)", workflowExecution.ExecutionId, httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("[INFO][%s] Running with HTTPS proxy %s (env: HTTPS_PROXY)", workflowExecution.ExecutionId, httpsProxy)
}
}
//log.Printf("[DEBUG][%s] All App Logs: %#v", workflowExecution.ExecutionId, allLogs) //log.Printf("[DEBUG][%s] All App Logs: %#v", workflowExecution.ExecutionId, allLogs)
_, err = client.Do(req) _, err = client.Do(req)
@@ -2023,24 +2006,7 @@ func main() {
log.Printf("[INFO] Setting up worker environment") log.Printf("[INFO] Setting up worker environment")
sleepTime := 5 sleepTime := 5
client := &http.Client{ client := shuffle.GetExternalClient(baseUrl)
Transport: &http.Transport{
Proxy: nil,
},
}
httpProxy := os.Getenv("HTTP_PROXY")
httpsProxy := os.Getenv("HTTPS_PROXY")
if (len(httpProxy) > 0 || len(httpsProxy) > 0) && baseUrl != "http://shuffle-backend:5001" {
client = &http.Client{}
} else {
if len(httpProxy) > 0 {
log.Printf("Running with HTTP proxy %s (env: HTTP_PROXY)", httpProxy)
}
if len(httpsProxy) > 0 {
log.Printf("Running with HTTPS proxy %s (env: HTTPS_PROXY)", httpsProxy)
}
}
if timezone == "" { if timezone == "" {
timezone = "Europe/Amsterdam" timezone = "Europe/Amsterdam"