marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'nightly' of https://github.com/shuffle/shuffle into nightly

Browse Source
This commit is contained in:
Frikky
2025-07-11 16:04:20 +02:00
parent 7e9cdce4b6 895ff2eff5
commit 2029c25563
4 changed files with 143 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/tagged-nightly-release.yaml
+92
View File
@@ -0,0 +1,92 @@
name: Tagged Nightly Release
on:
release:
types: [published]
branches:
- nightly
# there is a very clear point to this existing.
# we want to also release versions that look like this:
# v2.1.0-nightly-date, v2.1.0-nightly-date-1, v2.1.0-nightly-date-2
# we NEVER want to send customers a "nightly" tag. We always want to send them
# a tagged nightly tag. So that when something breaks, They can always
# point to it.
jobs:
main:
runs-on: ubuntu-latest
continue-on-error: ${{ matrix.experimental }}
strategy:
fail-fast: false
matrix:
include:
- app: frontend
path: frontend
experimental: true
- app: backend
path: backend
experimental: true
- app: app_sdk
path: backend/app_sdk
experimental: true
- app: orborus
path: functions/onprem/orborus
experimental: true
- app: worker
path: functions/onprem/worker
experimental: true
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Set version
id: set_version
run: |
if [[ ${{ github.event_name }} == 'release' ]]; then
echo "VERSION=${{ github.event.release.tag_name }}" >> $GITHUB_OUTPUT
else
echo "VERSION=nightly-untagged-latest" >> $GITHUB_OUTPUT
fi
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
with:
platforms: "amd64,arm64,arm"
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USERNAME }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Login to Ghcr
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Ghcr Build and push
id: docker_build
uses: docker/build-push-action@v4
env:
BUILDX_NO_DEFAULT_LOAD: true
with:
logout: false
context: ${{ matrix.path }}/
file: ${{ matrix.path }}/Dockerfile
platforms: linux/amd64,linux/arm64
push: true
cache-from: type=local,src=/tmp/.buildx-cache
cache-to: type=local,dest=/tmp/.buildx-cache
tags: |
ghcr.io/shuffle/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
${{ secrets.DOCKERHUB_USERNAME }}/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
frikky/shuffle-${{ matrix.app }}:${{ steps.set_version.outputs.VERSION }}
frikky/shuffle:${{ matrix.app }}
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
backend/go-app/main.go
+2 -2
View File
@@ -5308,8 +5308,8 @@ func initHandlers() {
// EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021) // EVERYTHING below here is NEW for 0.8.0 (written 25.05.2021)
r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/workflows/{key}/publish", makeWorkflowPublic).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/cloud/setup", handleCloudSetup).Methods("POST", "OPTIONS")
//r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
//r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/", shuffle.HandleGetOrgs).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleGetOrg).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgId}", shuffle.HandleEditOrg).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/orgs/{orgid}/forms", shuffle.HandleGetOrgForms).Methods("GET", "OPTIONS") r.HandleFunc("/api/v1/orgs/{orgid}/forms", shuffle.HandleGetOrgForms).Methods("GET", "OPTIONS")
backend/go-app/walkoff.go
+31 -167
View File
@@ -11,7 +11,7 @@ import (
"io" "io"
"io/ioutil" "io/ioutil"
"log" "log"
"math/rand" //"math/rand"
"net/http" "net/http"
"net/url" "net/url"
"os" "os"
@@ -246,16 +246,16 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
} }
// This is really the environment's name - NOT org-id // This is really the environment's name - NOT org-id
orgId := request.Header.Get("Org-Id") environment := request.Header.Get("Org-Id")
if len(orgId) == 0 { if len(environment) == 0 {
log.Printf("[AUDIT] No org-id header set") log.Printf("[AUDIT] No org-id header set")
resp.WriteHeader(401) resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`))) resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Specify the org-id header."}`)))
return return
} }
environment := request.Header.Get("org") orgId := request.Header.Get("Org")
if len(environment) == 0 { if len(orgId) == 0 {
//log.Printf("[AUDIT] No 'org' header set (get workflow queue). ") //log.Printf("[AUDIT] No 'org' header set (get workflow queue). ")
/* /*
resp.WriteHeader(403) resp.WriteHeader(403)
@@ -265,6 +265,7 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
} }
orborusLabel := request.Header.Get("x-orborus-label") orborusLabel := request.Header.Get("x-orborus-label")
_ = orborusLabel
// This section is cloud custom for now // This section is cloud custom for now
auth := request.Header.Get("Authorization") auth := request.Header.Get("Authorization")
@@ -280,177 +281,40 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
//log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel) //log.Printf("[AUDIT] Get workflow queue for org %s, env %s, orborus label %s", orgId, environment, orborusLabel)
ctx := shuffle.GetContext(request) ctx := shuffle.GetContext(request)
env, err := shuffle.GetEnvironment(ctx, orgId, "") envs, err := shuffle.GetEnvironments(ctx, orgId)
if err != nil { if err != nil || len(envs) == 0 {
log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err)
}
var env *shuffle.Environment
found := false
for i := range envs {
if envs[i].Name == environment {
env = &envs[i]
found = true
break
}
}
if !found {
log.Printf("[ERROR] Failed to find environment(%s) for org(%s)", environment, orgId)
resp.WriteHeader(400)
resp.Write([]byte(`{"success":false,"reason":"environment not found"}`))
return
} }
timeNow := time.Now().Unix() timeNow := time.Now().Unix()
err = shuffle.HandleOrborusFailover(ctx, request, resp, env) err = shuffle.HandleOrborusFailover(ctx, request, resp, env)
if err != nil { if err != nil {
if !strings.Contains(err.Error(), "mismatch") {
log.Printf("[WARNING] Failed handling Orborus failover: %s", err) log.Printf("[WARNING] Failed handling Orborus failover: %s", err)
} }
log.Printf("[DEBUG] Issue with environment ID: %s", orgId)
return
}
//log.Printf("Found env: %#v", env) //log.Printf("Found env: %#v", env)
if len(env.OrgId) > 0 { if len(env.OrgId) > 0 {
environment = env.OrgId orgId = env.OrgId
} }
// FIXME: Workflow stats disabled for now executionRequests, err := shuffle.GetWorkflowQueue(ctx, environment, 100)
// as it caused too many problems
// goal: track docker stuff once a minute and graph it
// For now: Disable this as it caused too many problems
if request.Method == "POST" && true == false {
//log.Printf("[DEBUG] POST to workflowqueue")
if rand.Intn(10) == 0 {
// Parse out body
body, err := ioutil.ReadAll(request.Body)
if err == nil {
// Parse out CPU, memory and disk.
var envData shuffle.OrborusStats
err = json.Unmarshal(body, &envData)
if err == nil && !envData.Swarm && !envData.Kubernetes && (envData.CPU > 0 || envData.Memory > 0 || envData.Disk > 0) {
// Set the input in memory
envData.OrgId = orgId
envData.Environment = environment
envData.OrborusLabel = orborusLabel
envData.Timestamp = time.Now().Unix()
if envData.CPU > 0 && envData.MaxCPU > 0 {
envData.CPUPercent = float64(envData.CPU) / float64(envData.MaxCPU)
}
if envData.Memory > 0 && envData.MaxMemory > 0 {
envData.MemoryPercent = float64(envData.Memory) / float64(envData.MaxMemory)
}
// Check if CPU percent constantly has stayed above X% for the last Y requests
percentageCheck := 90
concurrentChecks := 2
//if int(envData.CPUPercent) > percentageCheck {
// Get cached data
percentages := []float64{}
cacheKey := fmt.Sprintf("%s_%s_percent", orgId, strings.ToLower(environment))
// Marshal float list into []byte
cacheData := []byte{}
cache, err := shuffle.GetCache(ctx, cacheKey)
if err == nil {
// Unmarshal into percentages
cacheData := []byte(cache.([]uint8))
err = json.Unmarshal(cacheData, &percentages)
if err != nil {
log.Printf("[INFO] error in cache unmarshal for percentages: %s", err)
}
if len(percentages) > concurrentChecks {
percentages = percentages[:concurrentChecks]
}
percentages = append(percentages, envData.CPUPercent)
if len(percentages) > concurrentChecks {
//log.Printf("[INFO] Checking percentages: %v", percentages)
// percentageCheck := 1
sendAlert := true
for _, p := range percentages {
if int(p) < percentageCheck {
//log.Printf("[AUDIT] CPU percent is below %d: %d", percentageCheck, int(p))
sendAlert = false
break
}
}
if sendAlert {
log.Printf("[INFO] CPU percent has been above %d percent for the last 5 requests. Sending alert. Env: %s, org: %s", percentageCheck, environment, orgId)
// Set notification + alert for organization
err = shuffle.CreateOrgNotification(
ctx,
fmt.Sprintf("CPU percent has been above %d percent", percentageCheck),
fmt.Sprintf("A environment %s has been using more than %d\\% CPU for the last 5 requests.", environment, percentageCheck),
fmt.Sprintf("/admin?tab=environments"),
environment,
true,
)
if err != nil {
log.Printf("[ERROR] error creating notification: %s", err)
}
org, err := shuffle.GetOrg(ctx, environment)
if err == nil {
foundRecommendation := false
for _, recommendation := range org.Priorities {
if strings.Contains(recommendation.Name, "CPU") {
foundRecommendation = true
break
}
}
if !foundRecommendation {
// Add to start of org.Priorities
org, _ = shuffle.AddPriority(*org, shuffle.Priority{
Name: fmt.Sprintf("High CPU in environment %s", orgId),
Description: fmt.Sprintf("The environment %s has been using more than %d percent CPU. This indicates you may need to look at scaling.", orgId, percentageCheck),
Type: "scale",
Active: true,
URL: fmt.Sprintf("/admin?tab=environments"),
Severity: 1,
}, false)
//Make last item the first item
org.Priorities = append([]shuffle.Priority{org.Priorities[len(org.Priorities)-1]}, org.Priorities[:len(org.Priorities)-1]...)
err = shuffle.SetOrg(ctx, *org, org.Id)
if err != nil {
log.Printf("[ERROR] Problem setting org: %s", err)
}
}
}
}
if len(percentages) > 1 {
percentages = percentages[1:]
}
}
// Marshal float list into []byte
} else {
//log.Printf("[ERROR] Failed getting cache: %s", err)
percentages = append(percentages, envData.CPUPercent)
}
if len(percentages) > 0 {
//log.Printf("[DEBUG] Setting cache for %s: %#v", cacheKey, percentages)
cacheData, err = json.Marshal(percentages)
if err != nil {
log.Printf("[INFO] error in cache marshal: %s", err)
}
// Add the new data
go shuffle.SetCache(ctx, cacheKey, cacheData, 5)
}
}
//log.Printf("CPU percent: %f", envData.CPUPercent)
//log.Printf("Memory percent: %f", envData.MemoryPercent*100)
go shuffle.SetenvStats(ctx, envData)
}
}
}
executionRequests, err := shuffle.GetWorkflowQueue(ctx, orgId, 100)
if err != nil { if err != nil {
// Skipping as this comes up over and over // Skipping as this comes up over and over
//log.Printf("(2) Failed reading body for workflowqueue: %s", err) //log.Printf("(2) Failed reading body for workflowqueue: %s", err)
@@ -478,10 +342,10 @@ func handleGetWorkflowqueue(resp http.ResponseWriter, request *http.Request) {
} }
} }
if len(orgId) > 0 { if len(environment) > 0 {
env, err := shuffle.GetEnvironment(ctx, orgId, foundId) env, err := shuffle.GetEnvironment(ctx, environment, foundId)
if err != nil { if err != nil {
log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", orgId, err) log.Printf("[WARNING] No env found matching %s - continuing without updating orborus anyway: %s", environment, err)
//resp.WriteHeader(401) //resp.WriteHeader(401)
//resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id))) //resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "No env found matching %s"}`, id)))
//return //return
functions/onprem/orborus/orborus.go
+16 -15
View File
@@ -813,21 +813,21 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
newImages = append(newImages, curimage) newImages = append(newImages, curimage)
// Force remove the current image to avoid cached layers // Force remove the current image to avoid cached layers
if swarmConfig == "run" || swarmConfig == "swarm" { // if swarmConfig == "run" || swarmConfig == "swarm" {
_, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{ // _, err := dockercli.ImageRemove(ctx, curimage, image.RemoveOptions{
Force: true, // Force: true,
PruneChildren: true, // PruneChildren: true,
}) // })
//
if err != nil { // if err != nil {
log.Printf("[ERROR] Failed removing image for re-download: %s", err) // log.Printf("[ERROR] Failed removing image for re-download: %s", err)
} else { // } else {
log.Printf("[DEBUG] Removed image: %s", curimage) // log.Printf("[DEBUG] Removed image: %s", curimage)
} // }
} else { // } else {
//log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig) // //log.Printf("[DEBUG] Skipping image removal for %s as swarmConfig is not set to run or swarm. Value: %#v", curimage, swarmConfig)
} // }
//
err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage) err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, curimage)
if err != nil { if err != nil {
//log.Printf("[ERROR] Failed downloading image: %s", err) //log.Printf("[ERROR] Failed downloading image: %s", err)
@@ -874,6 +874,7 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
// Update the service to run with the new image // Update the service to run with the new image
//docker service update --image username/imagename:latest servicename --force //docker service update --image username/imagename:latest servicename --force
serviceUpdateOptions := types.ServiceUpdateOptions{} serviceUpdateOptions := types.ServiceUpdateOptions{}
service.Spec.TaskTemplate.ForceUpdate++
resp, err := dockercli.ServiceUpdate( resp, err := dockercli.ServiceUpdate(
ctx, ctx,
service.ID, service.ID,