marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Minor sync

Browse Source
This commit is contained in:
Frikky
2025-03-18 00:53:26 +01:00
parent 74f3ea46de
commit 5395ad40ec
7 changed files with 133 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
functions/onprem/orborus/go.mod
+3 -2
View File
@@ -1,9 +1,10 @@
module orborus
go 1.22.0
go 1.22.7
toolchain go1.23.6
//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
require (
github.com/docker/docker v27.5.0+incompatible
functions/onprem/orborus/go.sum
+2 -8
View File
@@ -123,12 +123,10 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8=
github.com/go-git/go-billy/v5 v5.6.0/go.mod h1:sFDq7xD3fn3E0GOwUSZqHo9lrkmx8xJhA0ZrfvjBRGM=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
github.com/go-git/go-git/v5 v5.13.0 h1:vLn5wlGIh/X78El6r3Jr+30W16Blk0CTcxTYcYPWi5E=
github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkvVkiXRR/zw=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -305,10 +303,11 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/shuffle-shared v0.6.99 h1:sPGmZo+8JMgUH9Q2O659za2w4sF/NXiWFCrSPm1nTAU=
github.com/shuffle/shuffle-shared v0.6.99/go.mod h1:bBXhEsPKjxln0mFnSeri7gIJ3tL/636Sh5NyTyNrvIQ=
github.com/shuffle/shuffle-shared v0.8.7 h1:+UdRx7b/KUy/E92ODNr87UvCI2Dxfnv0samQWuggJu4=
github.com/shuffle/shuffle-shared v0.8.7/go.mod h1:bBXhEsPKjxln0mFnSeri7gIJ3tL/636Sh5NyTyNrvIQ=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
@@ -369,7 +368,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -427,7 +425,6 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -437,7 +434,6 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -474,14 +470,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
functions/onprem/orborus/orborus.go
+51 -48
View File
@@ -456,6 +456,7 @@ func deployServiceWorkers(image string) {
}
}
/*
isMemcachedRunning, err := checkMemcached(ctx, dockercli)
if err != nil {
log.Printf("[ERROR] Failed checking memcached: %s", err)
@@ -469,6 +470,7 @@ func deployServiceWorkers(image string) {
if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 {
os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip))
}
*/
defaultNetworkAttach := false
if containerId != "" {
@@ -767,10 +769,8 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
// Remove the image
handled := []string{}
log.Printf("[DEBUG] Should remove existing image (s): %s. Waiting 30 seconds to ensure backend has the latest images built and ready to distribute.", images)
removeOptions := image.RemoveOptions{}
time.Sleep(time.Duration(30) * time.Second)
log.Printf("[DEBUG] Removing existing image (s): %s. Waiting 30 seconds before starting to ensure backend has the latest images built and ready to distribute.", images)
//time.Sleep(time.Duration(30) * time.Second)
newImages := []string{}
for _, image := range strings.Split(images, ",") {
@@ -786,23 +786,12 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
newImages = append(newImages, image)
// There is no real point in actual removal. This may however be a good idea, as Worker will force download the new one anyway
resp, err := dockercli.ImageRemove(ctx, image, removeOptions)
log.Printf("[DEBUG] Downloading image: %s", image)
err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
if err != nil {
log.Printf("[ERROR] Failed removing image: %s. Resp: %#v", err, resp)
// Goroutining images that don't already exist, as they are most likely not the correct one
go shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
log.Printf("[ERROR] Failed downloading image: %s", err)
} else {
log.Printf("[DEBUG] Removed image: %s", image)
err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
if err != nil {
log.Printf("[ERROR] Failed downloading image: %s", err)
} else {
log.Printf("[DEBUG] Downloaded image: %s", image)
//break
}
log.Printf("[DEBUG] Downloaded image: %s", image)
}
}
@@ -821,20 +810,18 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
)
if err != nil {
log.Printf("[ERROR] Failed finding containers: %s", err)
log.Printf("[ERROR] Failed finding services: %s", err)
} else {
log.Printf("[DEBUG] Found %d services", len(services))
found := false
for _, service := range services {
log.Printf("Imagename: %s", service.Spec.TaskTemplate.ContainerSpec.Image)
//log.Printf("Service image: %s", service.Spec.TaskTemplate.ContainerSpec.Image)
for _, image := range newImages {
if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, image) {
continue
}
log.Printf("[DEBUG] Found service for image %#v: %#v", service.Spec.Annotations.Name)
log.Printf("[DEBUG] Found service for image: %#v", service.Spec.Annotations.Name)
// Update the service to run with the new image
//docker service update --image username/imagename:latest servicename --force
@@ -854,9 +841,20 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
if !strings.Contains(fmt.Sprintf("%s", resp), "error") {
break
} else {
found = true
log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp)
}
}
}
if found {
break
}
}
if !found {
log.Printf("[DEBUG] Failed to find service to update for service %s", newImages)
}
}
@@ -1480,29 +1478,34 @@ func initializeImages() {
}
// check whether they are the same first
images := []string{
fmt.Sprintf("frikky/shuffle:app_sdk"),
fmt.Sprintf("shuffle/shuffle:app_sdk"),
fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion),
newWorker,
}
pullOptions := image.PullOptions{}
for _, image := range images {
if isKubernetes == "true" {
log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", image)
} else {
log.Printf("[DEBUG] Pulling image %s", image)
reader, err := dockercli.ImagePull(ctx, image, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", image, err)
continue
}
io.Copy(os.Stdout, reader)
log.Printf("[DEBUG] Successfully downloaded and built %s", image)
if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") != "true" {
images := []string{
fmt.Sprintf("frikky/shuffle:app_sdk"),
fmt.Sprintf("shuffle/shuffle:app_sdk"),
fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion),
newWorker,
}
pullOptions := image.PullOptions{}
for _, image := range images {
if isKubernetes == "true" {
log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", image)
} else {
log.Printf("[DEBUG] Pulling image %s", image)
reader, err := dockercli.ImagePull(ctx, image, pullOptions)
if err != nil {
log.Printf("[ERROR] Failed getting image %s: %s", image, err)
continue
}
io.Copy(os.Stdout, reader)
log.Printf("[DEBUG] Successfully downloaded and built %s", image)
}
}
} else {
log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to true")
}
}
@@ -2246,7 +2249,7 @@ func main() {
toBeRemoved.Data = append(toBeRemoved.Data, incRequest)
} else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" {
log.Printf("[INFO] Should delete -> download new images: %#v", incRequest.ExecutionArgument)
log.Printf("[INFO] Re-downloading new image(s): %#v", incRequest.ExecutionArgument)
if len(incRequest.ExecutionArgument) > 0 {
// FIXME: Wait X seconds before running this as the image build may not be done yet. This is shitty, but may be ok to do in Orborus. Easy fix for the future: Just let it run through jobs 5-10 times before actually picking it up
@@ -3463,7 +3466,7 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) {
} else {
tenzirDisabled = true
log.Printf("[ERROR] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
}
return pipelinePayload, err
functions/onprem/worker/worker.go
-2
View File
@@ -106,9 +106,7 @@ var window = shuffle.NewTimeWindow(10 * time.Second)
// Images to be autodeployed in the latest version of Shuffle.
var autoDeploy = map[string]string{
"http:1.4.0": "frikky/shuffle:http_1.4.0",
"http:1.3.0": "frikky/shuffle:http_1.3.0",
"shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0",
"shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0",
"shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0",
// "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0",
}