+ 
+
)
}
@@ -278,8 +280,6 @@ export const CodeHandler = (props) => {
const Docs = (defaultprops) => {
const { globalUrl, selectedDoc, serverside, serverMobile, isLoggedIn, isLoaded, userdata } = defaultprops;
- console.log("\n\nSELECTED DOC\n", selectedDoc, "\n\n")
-
let navigate = useNavigate();
const location = useLocation();
// Quickfix for react router 5 -> 6
@@ -321,14 +321,22 @@ const Docs = (defaultprops) => {
useEffect(() => {
fetchDocList();
setSidebarOpen(false);
- }, [location]);
- useEffect(() => {
- //if (params["key"] === undefined) {
- // navigate("/docs/about")
- // return
- //}
- }, [])
+ if (props.match.params === undefined || props.match.params === null) {
+ return
+ }
+
+ const propkey = props.match.params.key
+ if (propkey === undefined || propkey === null) {
+ return
+ }
+
+ if (location.pathname.includes("/docs/")) {
+ if (propkey === "cookie_policy" || propkey === "compliance" || propkey === "privacy_policy" || propkey === "terms_of_service") {
+ navigate(`/legal/${propkey}`)
+ }
+ }
+ }, [location]);
const handleClick = (event) => {
setAnchorEl(event.currentTarget);
@@ -761,7 +769,13 @@ const Docs = (defaultprops) => {
}
const fetchDocList = (resetCache = false) => {
- const url = isArticlePage ? `${globalUrl}/api/v1/articles?resetCache=${resetCache}` : `${globalUrl}/api/v1/docs`;
+ var url = `${globalUrl}/api/v1/docs`
+ if (location.pathname.includes("/legal")) {
+ url = `${globalUrl}/api/v1/docs?folder=legal&resetCache=${resetCache}`
+ } else if (location.pathname.includes("/articles")) {
+ url = `${globalUrl}/api/v1/docs?folder=articles&resetCache=${resetCache}`
+ }
+
fetch(url, {
method: "GET",
headers: {
@@ -782,8 +796,18 @@ const Docs = (defaultprops) => {
.catch((error) => { });
};
- const fetchDocs = (docId) => {
- const url = isArticlePage ? `${globalUrl}/api/v1/articles/${docId}` : `${globalUrl}/api/v1/docs/${docId}`;
+ const fetchDoc = (docId) => {
+ if (docId === undefined) {
+ return
+ }
+
+ var url = `${globalUrl}/api/v1/docs/${docId}`
+ if (location.pathname.includes("/legal")) {
+ url = `${globalUrl}/api/v1/docs/${docId}?folder=legal`
+ } else if (location.pathname.includes("/articles")) {
+ url = `${globalUrl}/api/v1/docs/${docId}?folder=articles`
+ }
+
fetch(url, {
method: "GET",
headers: {
@@ -902,7 +926,7 @@ const Docs = (defaultprops) => {
});
} else {
console.log("DOCID: ", props.match.params.key)
- fetchDocs(props.match.params.key)
+ fetchDoc(props.match.params.key)
}
}
}
@@ -911,7 +935,7 @@ const Docs = (defaultprops) => {
// Handles search-based changes that origin from outside this file
if (serverside !== true && window.location.href !== baseUrl) {
setBaseUrl(window.location.href);
- fetchDocs(props.match.params.key);
+ fetchDoc(props.match.params.key);
}
const markdownStyle = {
@@ -1106,7 +1130,14 @@ const Docs = (defaultprops) => {
if (item === undefined) {
return null;
}
- const path = "/docs/" + item;
+
+ var path = "/docs/" + item;
+ if (location.pathname.includes("/legal")) {
+ path = "/legal/" + item
+ } else if (location.pathname.includes("/articles")) {
+ path = "/articles/" + item
+ }
+
const newname =
item.charAt(0).toUpperCase() +
item.substring(1).split("_").join(" ").split("-").join(" ");
diff --git a/functions/onprem/orborus/go.mod b/functions/onprem/orborus/go.mod
index 7909c2ee..bb0fcb9e 100644
--- a/functions/onprem/orborus/go.mod
+++ b/functions/onprem/orborus/go.mod
@@ -1,9 +1,10 @@
module orborus
-go 1.22.0
+go 1.22.7
+
toolchain go1.23.6
-//replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
+replace github.com/shuffle/shuffle-shared => ../../../../shuffle-shared
require (
github.com/docker/docker v27.5.0+incompatible
diff --git a/functions/onprem/orborus/go.sum b/functions/onprem/orborus/go.sum
index 2bb0c986..7fc88f31 100644
--- a/functions/onprem/orborus/go.sum
+++ b/functions/onprem/orborus/go.sum
@@ -123,12 +123,10 @@ github.com/gliderlabs/ssh v0.3.8 h1:a4YXD1V7xMF9g5nTkdfnja3Sxy1PVDCj1Zg4Wb8vY6c=
github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI=
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic=
-
github.com/go-git/go-billy/v5 v5.6.0 h1:w2hPNtoehvJIxR00Vb4xX94qHQi/ApZfX+nBE2Cjio8=
github.com/go-git/go-billy/v5 v5.6.0/go.mod h1:sFDq7xD3fn3E0GOwUSZqHo9lrkmx8xJhA0ZrfvjBRGM=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4=
github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399/go.mod h1:1OCfN199q1Jm3HZlxleg+Dw/mwps2Wbk9frAWm+4FII=
-
github.com/go-git/go-git/v5 v5.13.0 h1:vLn5wlGIh/X78El6r3Jr+30W16Blk0CTcxTYcYPWi5E=
github.com/go-git/go-git/v5 v5.13.0/go.mod h1:Wjo7/JyVKtQgUNdXYXIepzWfJQkUEIGvkvVkiXRR/zw=
github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
@@ -305,10 +303,11 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/shuffle-shared v0.6.99 h1:sPGmZo+8JMgUH9Q2O659za2w4sF/NXiWFCrSPm1nTAU=
github.com/shuffle/shuffle-shared v0.6.99/go.mod h1:bBXhEsPKjxln0mFnSeri7gIJ3tL/636Sh5NyTyNrvIQ=
+github.com/shuffle/shuffle-shared v0.8.7 h1:+UdRx7b/KUy/E92ODNr87UvCI2Dxfnv0samQWuggJu4=
+github.com/shuffle/shuffle-shared v0.8.7/go.mod h1:bBXhEsPKjxln0mFnSeri7gIJ3tL/636Sh5NyTyNrvIQ=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
-
github.com/skeema/knownhosts v1.3.0 h1:AM+y0rI04VksttfwjkSTNQorvGqmwATnvnAHpSgc0LY=
github.com/skeema/knownhosts v1.3.0/go.mod h1:sPINvnADmT/qYH1kfv+ePMmOBTH6Tbl7b5LvTDjFK7M=
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
@@ -369,7 +368,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-
golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -427,7 +425,6 @@ golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-
golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -437,7 +434,6 @@ golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4Iltr
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.23.0 h1:PbgcYx2W7i4LvjJWEbf0ngHV6qJYr86PkAV3bXdLEbs=
golang.org/x/oauth2 v0.23.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
-
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -474,14 +470,12 @@ golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-
golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-
golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg=
golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
diff --git a/functions/onprem/orborus/orborus.go b/functions/onprem/orborus/orborus.go
index ffb21af4..dbc79434 100755
--- a/functions/onprem/orborus/orborus.go
+++ b/functions/onprem/orborus/orborus.go
@@ -456,6 +456,7 @@ func deployServiceWorkers(image string) {
}
}
+ /*
isMemcachedRunning, err := checkMemcached(ctx, dockercli)
if err != nil {
log.Printf("[ERROR] Failed checking memcached: %s", err)
@@ -469,6 +470,7 @@ func deployServiceWorkers(image string) {
if len(os.Getenv("SHUFFLE_MEMCACHED")) == 0 {
os.Setenv("SHUFFLE_MEMCACHED", fmt.Sprintf("%s:11211", ip))
}
+ */
defaultNetworkAttach := false
if containerId != "" {
@@ -767,10 +769,8 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
// Remove the image
handled := []string{}
- log.Printf("[DEBUG] Should remove existing image (s): %s. Waiting 30 seconds to ensure backend has the latest images built and ready to distribute.", images)
- removeOptions := image.RemoveOptions{}
-
- time.Sleep(time.Duration(30) * time.Second)
+ log.Printf("[DEBUG] Removing existing image (s): %s. Waiting 30 seconds before starting to ensure backend has the latest images built and ready to distribute.", images)
+ //time.Sleep(time.Duration(30) * time.Second)
newImages := []string{}
for _, image := range strings.Split(images, ",") {
@@ -786,23 +786,12 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
newImages = append(newImages, image)
- // There is no real point in actual removal. This may however be a good idea, as Worker will force download the new one anyway
- resp, err := dockercli.ImageRemove(ctx, image, removeOptions)
+ log.Printf("[DEBUG] Downloading image: %s", image)
+ err := shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
if err != nil {
- log.Printf("[ERROR] Failed removing image: %s. Resp: %#v", err, resp)
-
- // Goroutining images that don't already exist, as they are most likely not the correct one
- go shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
+ log.Printf("[ERROR] Failed downloading image: %s", err)
} else {
- log.Printf("[DEBUG] Removed image: %s", image)
-
- err = shuffle.DownloadDockerImageBackend(&http.Client{Timeout: imagedownloadTimeout}, image)
- if err != nil {
- log.Printf("[ERROR] Failed downloading image: %s", err)
- } else {
- log.Printf("[DEBUG] Downloaded image: %s", image)
- //break
- }
+ log.Printf("[DEBUG] Downloaded image: %s", image)
}
}
@@ -821,20 +810,18 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
)
if err != nil {
- log.Printf("[ERROR] Failed finding containers: %s", err)
+ log.Printf("[ERROR] Failed finding services: %s", err)
} else {
- log.Printf("[DEBUG] Found %d services", len(services))
-
+ found := false
for _, service := range services {
-
- log.Printf("Imagename: %s", service.Spec.TaskTemplate.ContainerSpec.Image)
+ //log.Printf("Service image: %s", service.Spec.TaskTemplate.ContainerSpec.Image)
for _, image := range newImages {
if !strings.Contains(service.Spec.TaskTemplate.ContainerSpec.Image, image) {
continue
}
- log.Printf("[DEBUG] Found service for image %#v: %#v", service.Spec.Annotations.Name)
+ log.Printf("[DEBUG] Found service for image: %#v", service.Spec.Annotations.Name)
// Update the service to run with the new image
//docker service update --image username/imagename:latest servicename --force
@@ -854,9 +841,20 @@ func handleBackendImageDownload(ctx context.Context, images string) error {
if !strings.Contains(fmt.Sprintf("%s", resp), "error") {
break
+ } else {
+ found = true
+ log.Printf("[ERROR] Failed updating service %s with the new image %s: %s. Resp: %#v", service.Spec.Annotations.Name, image, err, resp)
}
}
}
+
+ if found {
+ break
+ }
+ }
+
+ if !found {
+ log.Printf("[DEBUG] Failed to find service to update for service %s", newImages)
}
}
@@ -1480,29 +1478,34 @@ func initializeImages() {
}
// check whether they are the same first
- images := []string{
- fmt.Sprintf("frikky/shuffle:app_sdk"),
- fmt.Sprintf("shuffle/shuffle:app_sdk"),
- fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion),
- newWorker,
- }
- pullOptions := image.PullOptions{}
- for _, image := range images {
- if isKubernetes == "true" {
- log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", image)
- } else {
- log.Printf("[DEBUG] Pulling image %s", image)
- reader, err := dockercli.ImagePull(ctx, image, pullOptions)
- if err != nil {
- log.Printf("[ERROR] Failed getting image %s: %s", image, err)
-
- continue
- }
-
- io.Copy(os.Stdout, reader)
- log.Printf("[DEBUG] Successfully downloaded and built %s", image)
+ if os.Getenv("SHUFFLE_AUTO_IMAGE_DOWNLOAD") != "true" {
+ images := []string{
+ fmt.Sprintf("frikky/shuffle:app_sdk"),
+ fmt.Sprintf("shuffle/shuffle:app_sdk"),
+ fmt.Sprintf("%s/%s/shuffle-app_sdk:%s", baseimageregistry, baseimagename, appSdkVersion),
+ newWorker,
}
+
+ pullOptions := image.PullOptions{}
+ for _, image := range images {
+ if isKubernetes == "true" {
+ log.Printf("[DEBUG] Skipping image pull of '%s' because Kubernetes does it in realtime instead", image)
+ } else {
+ log.Printf("[DEBUG] Pulling image %s", image)
+ reader, err := dockercli.ImagePull(ctx, image, pullOptions)
+ if err != nil {
+ log.Printf("[ERROR] Failed getting image %s: %s", image, err)
+
+ continue
+ }
+
+ io.Copy(os.Stdout, reader)
+ log.Printf("[DEBUG] Successfully downloaded and built %s", image)
+ }
+ }
+ } else {
+ log.Printf("[DEBUG] Skipping image download as SHUFFLE_AUTO_IMAGE_DOWNLOAD is set to true")
}
}
@@ -2246,7 +2249,7 @@ func main() {
toBeRemoved.Data = append(toBeRemoved.Data, incRequest)
} else if incRequest.Type == "DOCKER_IMAGE_DOWNLOAD" {
- log.Printf("[INFO] Should delete -> download new images: %#v", incRequest.ExecutionArgument)
+ log.Printf("[INFO] Re-downloading new image(s): %#v", incRequest.ExecutionArgument)
if len(incRequest.ExecutionArgument) > 0 {
// FIXME: Wait X seconds before running this as the image build may not be done yet. This is shitty, but may be ok to do in Orborus. Easy fix for the future: Just let it run through jobs 5-10 times before actually picking it up
@@ -3463,7 +3466,7 @@ func sendPipelineHealthStatus() (shuffle.LakeConfig, error) {
} else {
tenzirDisabled = true
- log.Printf("[ERROR] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
+ log.Printf("[WARNING] Disabling pipelines: %s. You will need to restart the Orborus to fix this.", err)
}
return pipelinePayload, err
diff --git a/functions/onprem/worker/worker.go b/functions/onprem/worker/worker.go
index dca31904..943dcd2f 100644
--- a/functions/onprem/worker/worker.go
+++ b/functions/onprem/worker/worker.go
@@ -106,9 +106,7 @@ var window = shuffle.NewTimeWindow(10 * time.Second)
// Images to be autodeployed in the latest version of Shuffle.
var autoDeploy = map[string]string{
"http:1.4.0": "frikky/shuffle:http_1.4.0",
- "http:1.3.0": "frikky/shuffle:http_1.3.0",
"shuffle-tools:1.2.0": "frikky/shuffle:shuffle-tools_1.2.0",
- "shuffle-subflow:1.0.0": "frikky/shuffle:shuffle-subflow_1.0.0",
"shuffle-subflow:1.1.0": "frikky/shuffle:shuffle-subflow_1.1.0",
// "shuffle-tools-fork:1.0.0": "frikky/shuffle:shuffle-tools-fork_1.0.0",
}