marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Ton of minor fixes all pulled in. More info in the latest release for 1.2.0

Browse Source
This commit is contained in:
frikky
2023-05-15 04:42:09 +02:00
parent 70edb366a9
commit ee48543350
25 changed files with 1497 additions and 335 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/walkoff.go
+190 -16
View File
@@ -11,6 +11,7 @@ import (
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"net/url"
"os"
@@ -251,16 +252,42 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
return
}
id := request.Header.Get("Org-Id")
if len(id) == 0 {
log.Printf("[INFO] No org-id header set")
// This is really the environment's name - NOT org-id
orgId := request.Header.Get("Org-Id")
if len(orgId) == 0 {
log.Printf("[AUDIT] No org-id header set")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`)))
return
}
ctx := context.Background()
env, err := shuffle.GetEnvironment(ctx, id, "")
environment := request.Header.Get("org")
if len(environment) == 0 {
log.Printf("[AUDIT] No 'org' header set (get workflow queue). Required for cloud.")
/*
resp.WriteHeader(403)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org header. This can be done by setting the 'ORG' environment variable for Orborus to your Org ID in Shuffle"}`)))
return
*/
}
orborusLabel := request.Header.Get("x-orborus-label")
// This section is cloud custom for now
auth := request.Header.Get("Authorization")
if len(auth) == 0 {
log.Printf("[AUDIT] No Authorization header set. Required for cloud. Env: %s, org: %s", orgId, environment)
/*
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the auth header (only applicable for cloud for now)."}`)))
return
*/
}
//log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel)
ctx := shuffle.GetContext(request)
env, err := shuffle.GetEnvironment(ctx, orgId, "")
timeNow := time.Now().Unix()
if err == nil && len(env.Id) > 0 && len(env.Name) > 0 {
if time.Now().Unix() > env.Edited+60 {
@@ -273,7 +300,154 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
}
}
executionRequests, err := shuffle.GetWorkflowQueue(ctx, id, 100)
//log.Printf("Found env: %#v", env)
if len(env.OrgId) > 0 {
environment = env.OrgId
}
if request.Method == "POST" {
if rand.Intn(1) == 0 {
// Parse out body
body, err := ioutil.ReadAll(request.Body)
if err == nil {
// Parse out CPU, memory and disk.
var envData shuffle.OrborusStats
err = json.Unmarshal(body, &envData)
if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) {
// Set the input in memory
envData.OrgId = orgId
envData.Environment = environment
envData.OrborusLabel = orborusLabel
envData.Timestamp = time.Now().Unix()
if envData.CPU > 0 && envData.MaxCPU > 0 {
envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU)
}
if envData.Memory > 0 && envData.MaxMemory > 0 {
envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory)
}
// Check if CPU percent constantly has stayed above X% for the last Y requests
percentageCheck := 90
concurrentChecks := 0
//if int(envData.CPUPercent) > percentageCheck {
// Get cached data
percentages := []float64{}
cacheKey := fmt.Sprintf("%s_%s_percent", orgId, strings.ToLower(environment))
// Marshal float list into []byte
cacheData := []byte{}
cache, err := shuffle.GetCache(ctx, cacheKey)
if err == nil {
// Unmarshal into percentages
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &percentages)
if err != nil {
log.Printf("[INFO] error in cache unmarshal for percentages: %s", err)
}
if len(percentages) > concurrentChecks {
percentages = percentages[:concurrentChecks]
}
percentages = append(percentages, envData.CPUPercent)
if len(percentages) > concurrentChecks {
//log.Printf("[INFO] Checking percentages: %v", percentages)
// percentageCheck := 1
sendAlert := true
for _, p := range percentages {
if int(p) < percentageCheck {
//log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p))
sendAlert = false
break
}
}
if sendAlert {
log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId)
// Set notification + alert for organization
err = shuffle.CreateOrgNotification(
ctx,
fmt.Sprintf("CPU percent has been above %d percent", percentageCheck),
fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck),
fmt.Sprintf("/admin?tab=environments"),
environment,
true,
)
if err != nil {
log.Printf("[ERROR] error creating notification: %s", err)
}
org, err := shuffle.GetOrg(ctx, environment)
if err == nil {
foundRecommendation := false
for _, recommendation := range org.Priorities {
if strings.Contains(recommendation.Name, "CPU") {
foundRecommendation = true
break
}
}
if !foundRecommendation {
// Add to start of org.Priorities
org.Priorities = append(org.Priorities, shuffle.Priority{
Name: fmt.Sprintf("High CPU in environment %s", orgId),
Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU.", orgId, percentageCheck),
Type: "scale",
Active: true,
URL: fmt.Sprintf("/admin?tab=environments"),
})
//Make last item the first item
org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...)
err = shuffle.SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[ERROR] Problem setting org: %s", err)
}
}
}
}
if len(percentages) > 1 {
percentages = percentages[1:]
}
}
// Marshal float list into []byte
} else {
//log.Printf("[ERROR] Failed getting cache: %s", err)
percentages = append(percentages, envData.CPUPercent)
}
if len(percentages) > 0 {
//log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages)
cacheData, err = json.Marshal(percentages)
if err != nil {
log.Printf("[INFO] error in cache marshal: %s", err)
}
// Add the new data
go shuffle.SetCache(ctx, cacheKey, cacheData, 5)
}
}
//log.Printf("CPU percent: %f", envData.CPUPercent)
//log.Printf("Memory percent: %f", envData.MemoryPercent*100)
go shuffle.SetenvStats(ctx, envData)
}
}
}
executionRequests, err := shuffle.GetWorkflowQueue(ctx, orgId, 100)
if err != nil {
// Skipping as this comes up over and over
//log.Printf("(2) Failed reading body for workflowqueue: %s", err)
@@ -290,21 +464,21 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
// Try again :)
if len(env.Id) == 0 && len(env.Name) == 0 {
orgId := ""
foundId := ""
for _, requestData := range executionRequests.Data {
execution, err := shuffle.GetWorkflowExecution(ctx, requestData.ExecutionId)
if err == nil {
if len(execution.ExecutionOrg) > 0 {
orgId = execution.ExecutionOrg
foundId = execution.ExecutionOrg
break
}
}
}
if len(orgId) > 0 {
env, err := shuffle.GetEnvironment(ctx, id, orgId)
env, err := shuffle.GetEnvironment(ctx, orgId, foundId)
if err != nil {
log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", id, err)
log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err)
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id)))
//return
@@ -361,9 +535,9 @@ func handleGetStreamResults(resp http.ResponseWriter, request *http.Request) {
err = json.Unmarshal(body, &actionResult)
if err != nil {
log.Printf("[WARNING] Failed ActionResult unmarshaling (stream result): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
//return
}
ctx := context.Background()
@@ -476,9 +650,9 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
err = json.Unmarshal(body, &actionResult)
if err != nil {
log.Printf("[WARNING] Failed ActionResult unmarshaling (queue): %s", err)
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
return
//resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "%s"}`, err)))
//return
}
//log.Printf("Received action: %#v", actionResult)