Merge pull request #1684 from Shuffle/nightly

Fixed Rerun systems to ensure Latest allows for them to work well

functions/kubernetes/charts/shuffle/README.md

@@ -105,6 +105,15 @@ SHUFFLE_DEFAULT_APIKEY: "72E41083-A6F6-4A1B-8538-B06B577F47F0" # Shuffle uses uu
 SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier"
 ```
 
+## OpenSearch
+
+Shuffle uses OpenSearch as its database. This helm chart installs a single-node OpenSearch cluster using [the Bitnami Helm Chart](https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml).
+You can customize the helm chart using the values of the Bitnami helm chart under the `opensearch` prefix (e.g. `opensearch.master.replicaCOunt`).
+
+Alternatively, you can disable the built-in OpenSearch installation using `opensearch.enabled=false`.
+Provide your own OpenSearch url and username with `backend.openSearch.url` and `backend.openSearch.username`.
+The password should be provided with the `SHUFFLE_OPENSEARCH_PASSWORD` env variable to the backend.
+
 ## Parameters
 
 ### Global parameters
@@ -598,4 +607,3 @@ SHUFFLE_ENCRYPTION_MODIFIER: "MyShuffleEncryptionModifier"
 
 ### Other Parameters
 
-

functions/kubernetes/charts/shuffle/templates/shuffle-worker/shuffle-worker-network-policy.yaml

@@ -62,6 +62,12 @@ spec:
               kubernetes.io/metadata.name: {{ .Release.Namespace }}
           podSelector:
             matchLabels: {{ include "shuffle.orborus.matchLabels" . | nindent 14 }}
+        # Allow traffic from apps
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: {{ .Release.Namespace }}
+          podSelector:
+            matchLabels: {{ include "shuffle.app.matchLabels" . | nindent 14 }}
       {{- end }}
     {{- if .Values.worker.networkPolicy.extraIngress }}
     {{- include "common.tplvalues.render" ( dict "value" .Values.worker.networkPolicy.extraIngress "context" $ ) | nindent 4 }}

functions/kubernetes/charts/shuffle/values.yaml

@@ -1744,9 +1744,24 @@ volumePermissions:
 ## OpenSearch chart configuration
 ## ref: https://github.com/bitnami/charts/blob/main/bitnami/opensearch/values.yaml
 ## @param opensearch.enabled Switch to enable or disable the opensearch helm chart
+## @skip opensearch.master
+## @skip opensearch.data
+## @skip opensearch.coordinating
+## @skip opensearch.ingest
+## @skip opensearch.dashboards
 ##
 opensearch:
   enabled: true
+  master:
+    replicaCount: 1
+  data:
+    replicaCount: 1
+  coordinating:
+    replicaCount: 1
+  ingest:
+    replicaCount: 1
+  dashboards:
+    enabled: true
 
 ## @section Vault Parameters
 ##