marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'nightly' of https://github.com/shuffle/shuffle into nightly

Browse Source
This commit is contained in:
Frikky
2025-11-12 11:39:21 +01:00
parent 822e7c1eb3 a6e1efa5e2
commit 9e36d9d39b
11 changed files with 172 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+2
View File
@@ -96,6 +96,8 @@ SHUFFLE_WORKER_SERVER_URL=
SHUFFLE_ORBORUS_PULL_TIME=
# Max recursion depth for subflows
SHUFFLE_MAX_EXECUTION_DEPTH=
# Amount of app replicas
SHUFFLE_APP_REPLICAS=3
# DATABASE CONFIGURATIONS
DATASTORE_EMULATOR_HOST=shuffle-database:8000
backend/Dockerfile
Executable → Regular
+1 -1
View File
@@ -27,7 +27,7 @@ FROM alpine:latest as certs
RUN apk add --update ca-certificates
# Sets up the final image
FROM alpine:3.22.1
FROM alpine:3.22.2
# FIXME: Install cgo because CGO_ENABLED=1 during build
RUN apk add --no-cache libc6-compat
backend/go-app/go.mod
+1 -1
View File
@@ -26,7 +26,7 @@ require (
github.com/gorilla/mux v1.8.1
github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.51
github.com/shuffle/shuffle-shared v0.9.52
github.com/shuffle/singul v0.0.20
golang.org/x/crypto v0.40.0
google.golang.org/api v0.236.0
backend/go-app/walkoff.go
+25 -16
View File
@@ -1505,30 +1505,39 @@ func stopScheduleGCP(resp http.ResponseWriter, request *http.Request) {
return
}
func deleteSchedule(ctx context.Context, id string) error {
log.Printf("[DEBUG] Should stop schedule %s!", id)
func deleteKeySchedule(ctx context.Context, id string) error {
err := shuffle.DeleteKey(ctx, "schedules", id)
if err != nil {
log.Printf("[ERROR] Failed to delete schedule: %s", err)
return err
}
return nil
}
func deleteSchedule(ctx context.Context, id string) error {
log.Printf("[DEBUG] Should stop schedule %s!", id)
if value, exists := scheduledJobs[id]; exists {
// Stops the schedule properly
value.Lock()
} else {
if value, exists := scheduledJobs[id]; exists {
// Stops the schedule properly
value.Lock()
} else {
// FIXME - allow it to kind of stop anyway?
if j, ok := cronJobs[id]; ok {
err := CronScheduler.RemoveByID(j)
if err != nil {
log.Printf("[ERROR] Failed to remove the scheduler %s", err)
return err
}
} else {
return errors.New("Can't find the schedule.")
// FIXME - allow it to kind of stop anyway?
if j, ok := cronJobs[id]; ok {
err := CronScheduler.RemoveByID(j)
if err != nil {
log.Printf("[ERROR] Failed to remove the scheduler %s", err)
return err
}
} else {
// Just stop and delete anyway if not in memory
deleteKeySchedule(ctx, id)
return errors.New("Can't find the schedule.")
}
}
err := deleteKeySchedule(ctx, id)
if err != nil {
log.Printf("[ERROR] Failed to stop schedule in db %s: %s", id, err)
return err
}
return nil
}
docker-compose.yml
+6 -7
View File
@@ -42,8 +42,6 @@ services:
- /var/run/docker.sock:/var/run/docker.sock
environment:
- SHUFFLE_APP_SDK_TIMEOUT=300
- SHUFFLE_ORBORUS_EXECUTION_CONCURRENCY=7 # The amount of concurrent executions Orborus can handle.
#- DOCKER_HOST=tcp://docker-socket-proxy:2375
- ENVIRONMENT_NAME=Shuffle
- ORG_ID=Shuffle
- BASE_URL=http://${OUTER_HOSTNAME}:5001
@@ -52,9 +50,11 @@ services:
- HTTPS_PROXY=${HTTPS_PROXY}
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
- SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS}
- SHUFFLE_STATS_DISABLED=true
- SHUFFLE_LOGS_DISABLED=true
- SHUFFLE_SWARM_CONFIG=run
- SHUFFLE_APP_REPLICAS=3
- CLEANUP=false
- SHUFFLE_WORKER_IMAGE=ghcr.io/shuffle/shuffle-worker:latest
env_file: .env
@@ -122,11 +122,10 @@ services:
# - MEMCACHED_MAX_CONNECTIONS=2500
# ports:
# - 11211:11211
# deploy:
# resources:
# limits:
# memory: 2048M
# networks:
# - shuffle
# - swarm_executions
#docker-socket-proxy:
# image: tecnativa/docker-socket-proxy
# container_name: docker-socket-proxy
frontend/Dockerfile
Executable → Regular
+1 -1
View File
@@ -27,7 +27,7 @@ COPY ./*.json /usr/src/app/
RUN npm run build --loglevel verbose 2>&1
# Production environment
FROM nginx:1.29.0
FROM nginx:1.29.3
RUN mkdir -p /usr/share/nginx/html/build
RUN mkdir -p /usr/share/nginx/html/css
functions/kubernetes/charts/shuffle/templates/orborus/orborus-dpl.yaml
-2
View File
@@ -86,8 +86,6 @@ spec:
value: kubernetes
- name: IS_KUBERNETES
value: "true"
- name: SHUFFLE_SWARM_CONFIG
value: run
{{- if .Values.orborus.extraEnvVars }}
{{- include "common.tplvalues.render" (dict "value" .Values.orborus.extraEnvVars "context" $) | nindent 12 }}
{{- end }}
functions/onprem/orborus/go.mod
+1 -1
View File
@@ -10,7 +10,7 @@ require (
github.com/docker/docker v28.3.3+incompatible
github.com/docker/go-connections v0.5.0
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.51
github.com/shuffle/shuffle-shared v0.9.52
k8s.io/api v0.33.1
k8s.io/apimachinery v0.33.1
)
functions/onprem/orborus/orborus.go
+70 -17
View File
@@ -441,12 +441,32 @@ func deployServiceWorkers(image string) {
if err != nil {
if strings.Contains(fmt.Sprintf("%s", err), "already exists") {
// Try patching for attachable
if debug {
log.Printf("[DEBUG] Network %s already exists", networkName)
}
} else {
log.Printf("[DEBUG] Failed to create network %s for workers: %s. This is not critical, and containers will still be added", networkName, err)
}
}
networkID := ""
// find network ID
networks, err := dockercli.NetworkList(ctx, network.ListOptions{})
if err == nil {
for _, net := range networks {
if net.Name == networkName {
if net.Scope == "swarm" {
log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID)
networkID = net.ID
} else {
log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope)
}
break
}
}
}
/*
isMemcachedRunning, err := checkMemcached(ctx, dockercli)
if err != nil {
@@ -463,12 +483,17 @@ func deployServiceWorkers(image string) {
}
*/
if networkID == "" {
log.Printf("[ERROR] Network %s does not exist", networkName)
networkID = networkName
}
defaultNetworkAttach := false
if containerId != "" {
log.Printf("[DEBUG] Should connect orborus container to worker network as it's running in Docker with name %#v!", containerId)
// https://pkg.go.dev/github.com/docker/docker@v20.10.12+incompatible/api/types/network#EndpointSettings
networkConfig := &network.EndpointSettings{}
err := dockercli.NetworkConnect(ctx, networkName, containerId, networkConfig)
err := dockercli.NetworkConnect(ctx, networkID, containerId, networkConfig)
if err != nil {
log.Printf("[ERROR] Failed connecting Orborus to docker network %s: %s", networkName, err)
}
@@ -491,7 +516,7 @@ func deployServiceWorkers(image string) {
for _, container := range containers {
if strings.Contains(strings.ToLower(container.Image), "docker-socket-proxy") {
networkConfig := &network.EndpointSettings{}
err := dockercli.NetworkConnect(ctx, networkName, container.ID, networkConfig)
err := dockercli.NetworkConnect(ctx, networkID, container.ID, networkConfig)
if err != nil {
log.Printf("[ERROR] Failed connecting Docker socket proxy to docker network %s: %s", networkName, err)
} else {
@@ -571,7 +596,7 @@ func deployServiceWorkers(image string) {
},
Networks: []swarm.NetworkAttachmentConfig{
swarm.NetworkAttachmentConfig{
Target: networkName,
Target: networkID,
},
swarm.NetworkAttachmentConfig{
Target: "ingress",
@@ -740,7 +765,36 @@ func deployServiceWorkers(image string) {
if err == nil {
log.Printf("[DEBUG] Successfully deployed workers with %d replica(s) on %d node(s)", replicas, cnt)
// wait for service to be ready
time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second)
//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
// patch service network
// this is an edgecase that we noticed on docker version 29
// and API version 1.44
services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{})
if serr == nil {
for _, svc := range services {
if svc.Spec.Annotations.Name == innerContainerName {
log.Printf("[DEBUG] Found service %s (%s) — patching network attach", innerContainerName, svc.ID)
spec := svc.Spec
spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{
Target: networkID,
})
_, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{})
if uerr != nil {
log.Printf("[WARNING] Failed to patch service %s with network %s: %v", innerContainerName, networkID, uerr)
} else {
log.Printf("[INFO] Successfully attached network %s to service %s", networkID, innerContainerName)
}
break
}
}
} else {
log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr)
}
} else {
if !strings.Contains(fmt.Sprintf("%s", err), "Already Exists") && !strings.Contains(fmt.Sprintf("%s", err), "is already in use by service") {
log.Printf("[ERROR] Failed making service: %s", err)
@@ -2109,12 +2163,12 @@ func cleanup() {
func StartAgent() {
log.Printf("[INFO] Starting Orborus agent mode")
auditLogEnabled := os.Getenv("SHUFFLE_AUDIT_LOG_ENABLED") == "true"
if auditLogEnabled {
log.Printf("[INFO] Audit log monitoring is enabled")
// Initialize telemetry configuration
telemetryConfig := shuffle.TelemetryConfig{
Enabled: true,
@@ -2130,7 +2184,7 @@ func StartAgent() {
Exclude: patterns,
})
}
if includePatterns := os.Getenv("SHUFFLE_AUDIT_LOG_INCLUDE"); includePatterns != "" {
patterns := strings.Split(includePatterns, ",")
telemetryConfig.Filters = append(telemetryConfig.Filters, shuffle.TelemetryFilter{
@@ -2151,7 +2205,7 @@ func StartAgent() {
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-sigChan
log.Printf("[INFO] Received shutdown signal, stopping audit log collector...")
@@ -2184,7 +2238,7 @@ func main() {
if os.Getenv("SHUFFLE_PIPELINE_STANDALONE") == "true" {
log.Printf("[INFO] Allowing use of standalone pipeline (tenzir). URL: %s", pipelineUrl)
//if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" {
// os.Setenv("SHUFFLE_SKIP_PIPELINES", "true")
//}
@@ -3016,10 +3070,7 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error {
func deployTenzirNode() error {
// Disabled all pipeline features
if os.Getenv("SHUFFLE_SKIP_PIPELINES") == "false" || os.Getenv("SHUFFLE_PIPELINE_ENABLED") == "true" {
// return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES")
//log.Printf("[INFO] Pipelines are enabled by user")
} else {
if os.Getenv("SHUFFLE_SKIP_PIPELINES") != "true" {
return errors.New("Pipelines are disabled by user with SHUFFLE_SKIP_PIPELINES")
}
@@ -3249,7 +3300,7 @@ func createAndStartTenzirNode(ctx context.Context, containerName, imageName stri
EndpointsConfig: map[string]*network.EndpointSettings{
"tenzir-network": {
IPAMConfig: nil,
Aliases: []string{"tenzir-node"},
Aliases: []string{"tenzir-node"},
},
},
}
@@ -3904,12 +3955,14 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) {
err := deployTenzirNode()
if err != nil {
if (!strings.Contains(err.Error(), "SHUFFLE_SKIP_PIPELINES") && !strings.Contains(err.Error(), "Kubernetes not implemented for Tenzir node")) && !strings.Contains(err.Error(), "Tenzir Node is already running") && !strings.Contains(err.Error(), "docker daemon") {
log.Printf("[ERROR] Tenzir node connection problem: %s", err)
} else {
//tenzirDisabled = true
log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
if debug {
log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
}
}
return pipelinePayload, err
functions/onprem/worker/go.mod
+3 -3
View File
@@ -11,8 +11,8 @@ require (
github.com/docker/docker v28.3.3+incompatible
github.com/gorilla/mux v1.8.1
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.39
github.com/shuffle/singul v0.0.19
github.com/shuffle/shuffle-shared v0.9.52
github.com/shuffle/singul v0.0.20
k8s.io/api v0.33.1
k8s.io/apimachinery v0.33.1
k8s.io/client-go v0.33.1
@@ -60,7 +60,7 @@ require (
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/frikky/kin-openapi v0.42.0 // indirect
github.com/frikky/schemaless v0.0.23 // indirect
github.com/frikky/schemaless v0.0.24 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
functions/onprem/worker/worker.go
+62 -1
View File
@@ -2555,6 +2555,11 @@ func getWorkerBackendExecution(auth string, executionId string) (*shuffle.Workfl
return workflowExecution, err
}
if debug {
log.Printf("[INFO] Here is the result we got back from backend: %s", workflowExecution.Results)
}
setWorkflowExecution(context.Background(), *workflowExecution, false)
return workflowExecution, nil
}
@@ -3454,7 +3459,6 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
_ = service
if err != nil {
if strings.Contains(fmt.Sprintf("%s", err), "network") && strings.Contains(fmt.Sprintf("%s", err), "not found") {
log.Printf("[DEBUG] Network %s not found. Trying to initialize it.", networkName)
networkErr := initSwarmNetwork()
@@ -3481,6 +3485,63 @@ func deploySwarmService(dockercli *dockerclient.Client, name, image string, depl
log.Printf("[DEBUG] Failed deploying %s with image %s: %s", name, image, err)
return err
} else {
// wait for service to be ready
time.Sleep(time.Duration(rand.Intn(4)+1) * time.Second)
//log.Printf("[DEBUG] Servicecreate request: %#v %#v", service, err)
// patch service network
// this is an edgecase that we noticed on docker version 29
// and API version 1.44
// get networkID of swarmNetworkName
networkID := ""
ctx := context.Background()
// find network ID
networks, err := dockercli.NetworkList(ctx, network.ListOptions{})
if err == nil {
for _, net := range networks {
if net.Name == networkName {
if net.Scope == "swarm" {
log.Printf("[DEBUG] Found swarm-scoped network: %s (%s)", networkName, net.ID)
networkID = net.ID
} else {
log.Printf("[WARNING] Network %s exists but is not swarm scoped (scope=%s)", networkName, net.Scope)
}
break
}
}
}
if networkID == "" {
log.Printf("[ERROR] Network %s not found", networkName)
networkID = networkName
}
services, serr := dockercli.ServiceList(ctx, types.ServiceListOptions{})
if serr == nil {
for _, svc := range services {
if svc.ID == service.ID {
log.Printf("[DEBUG] Found service %s (%s) — patching network attach", service.ID, svc.ID)
spec := svc.Spec
spec.TaskTemplate.Networks = append(spec.TaskTemplate.Networks, swarm.NetworkAttachmentConfig{
Target: networkID,
})
_, uerr := dockercli.ServiceUpdate(ctx, svc.ID, svc.Version, spec, types.ServiceUpdateOptions{})
if uerr != nil {
log.Printf("[WARNING] Failed to patch service %s with network %s: %v", service.ID, networkID, uerr)
} else {
log.Printf("[INFO] Successfully attached network %s to service %s", networkID, service.ID)
}
break
}
}
} else {
log.Printf("[WARNING] Failed to list services for patching network attach: %v", serr)
}
}
log.Printf("[DEBUG] Successfully deployed service %s with image %s on port %d", name, image, deployport)