marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1888 from Shuffle/nightly

Browse Source 
v2.1.3
This commit is contained in:
Yash Singh
2025-12-19 17:15:22 +05:30
committed by GitHub
parent d21a20c8c7 9f410f0877
commit dd2242583f
21 changed files with 669 additions and 543 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.env
+1
View File
@@ -115,5 +115,6 @@ OPENSEARCH_INITIAL_ADMIN_PASSWORD="StrongShufflePassword321!" # In use for the f
#Tenzir related
SHUFFLE_TENZIR_URL=
SHUFFLE_PROTECTED_CLEANUP_DISABLED=true
DEBUG_MODE=false
.github/workflows/quick-testing.yml
+123 -120
View File
@@ -15,142 +15,145 @@ jobs:
os: [ubuntu-latest]
architecture: [x64, arm64]
steps:
- name: Checkout code
uses: actions/checkout@v2
- name: Checkout code
uses: actions/checkout@v2
- name: Install Docker Compose
run: |
sudo apt-get update
sudo apt-get install -y docker-compose
docker-compose --version
- name: Shai-Hulud 2.0 Detector
uses: gensecaihq/Shai-Hulud-2.0-Detector@v1.0.0
- name: Set up opensearch directory
run: chmod -R 777 shuffle-database
- name: Install Docker Compose
run: |
sudo apt-get update
sudo apt-get install -y docker-compose
docker-compose --version
- name: Build the stack
run: docker compose up -d
- name: Set up opensearch directory
run: chmod -R 777 shuffle-database
- name: Wait for 30 seconds
run: sleep 30
- name: Build the stack
run: docker compose up -d
- name: Check for restarting containers in a loop and fixing perms again
run: |
# echo "Changing permissions on shuffle-database directory again"
# chmod -R 777 shuffle-database
- name: Wait for 30 seconds
run: sleep 30
ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds
for i in $(seq 1 $ATTEMPTS); do
RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}")
if [ -n "$RESTARTING_CONTAINERS" ]; then
echo "The following containers are restarting:"
echo "$RESTARTING_CONTAINERS"
- name: Check for restarting containers in a loop and fixing perms again
run: |
# echo "Changing permissions on shuffle-database directory again"
# chmod -R 777 shuffle-database
ATTEMPTS=30 # Total time = ATTEMPTS * 5 seconds = 30 seconds
for i in $(seq 1 $ATTEMPTS); do
RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}")
if [ -n "$RESTARTING_CONTAINERS" ]; then
echo "The following containers are restarting:"
echo "$RESTARTING_CONTAINERS"
exit 1
fi
echo "No containers are restarting. Attempt $i/$ATTEMPTS."
sleep 1
done
echo "No containers were found in a restarting state after $ATTEMPTS checks."
- name: Check if the response from the frontend contains the word "Shuffle"
run: |
RESPONSE=$(curl -s http://localhost:3001)
if echo "$RESPONSE" | grep -q "Shuffle"; then
echo "The word 'Shuffle' was found in the response."
else
echo "The word 'Shuffle' was not found in the response."
exit 1
fi
echo "No containers are restarting. Attempt $i/$ATTEMPTS."
sleep 1
done
echo "No containers were found in a restarting state after $ATTEMPTS checks."
- name: Register a user and check the status code
run: |
MAX_RETRIES=30
RETRY_INTERVAL=10
CONTAINER_NAME="shuffle-backend"
- name: Check if the response from the frontend contains the word "Shuffle"
run: |
RESPONSE=$(curl -s http://localhost:3001)
if echo "$RESPONSE" | grep -q "Shuffle"; then
echo "The word 'Shuffle' was found in the response."
else
echo "The word 'Shuffle' was not found in the response."
for (( i=1; i<=$MAX_RETRIES; i++ ))
do
STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \
-H 'Accept: */*' \
-H 'Accept-Language: en-US,en;q=0.9' \
-H 'Connection: keep-alive' \
-H 'Content-Type: application/json' \
--data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}')
if [ "$STATUS_CODE" -eq 200 ]; then
echo "User registration was successful with status code 200."
exit 0
elif [ "$STATUS_CODE" -ne 502 ]; then
echo "User registration failed with status code $STATUS_CODE."
exit 1
fi
echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)"
echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..."
logs_output=$(docker logs --tail 30 "$CONTAINER_NAME")
echo "$logs_output"
echo "Fetching last 30 lines of logs from container shuffle-opensearch..."
opensearch_logs=$(docker logs --tail 30 shuffle-opensearch)
echo "$opensearch_logs"
sleep $RETRY_INTERVAL
done
echo "User registration failed after $MAX_RETRIES attempts."
exit 1
fi
- name: Register a user and check the status code
run: |
MAX_RETRIES=30
RETRY_INTERVAL=10
CONTAINER_NAME="shuffle-backend"
for (( i=1; i<=$MAX_RETRIES; i++ ))
do
STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \
-H 'Accept: */*' \
-H 'Accept-Language: en-US,en;q=0.9' \
-H 'Connection: keep-alive' \
-H 'Content-Type: application/json' \
--data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}')
- name: Run Selenium testing for frontend
run: |
cd $GITHUB_WORKSPACE/frontend
# write some log to see the current directory
chmod +x frontend-testing.sh
./frontend-testing.sh
if [ "$STATUS_CODE" -eq 200 ]; then
echo "User registration was successful with status code 200."
- name: Get the API key and run a health check
id: health_check
run: |
RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search')
echo "Raw Response: $RESPONSE"
API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey')
if [ -n "$API_KEY" ] && [ "$API_KEY" != "null" ]; then
echo "Admin API key: $API_KEY"
echo "API_KEY=$API_KEY" >> $GITHUB_ENV
else
echo "Failed to retrieve the API key for the admin user."
exit 1
fi
echo "Waiting 1 minute before sending the health API request..."
sleep 60
echo "Checking health API..."
HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \
-H "Authorization: Bearer $API_KEY")
echo "Health API Response: $HEALTH_RESPONSE"
WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create')
WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run')
WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished')
WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status')
WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete')
echo "WORKFLOWS_CREATE: $WORKFLOWS_CREATE"
echo "WORKFLOWS_RUN: $WORKFLOWS_RUN"
echo "WORKFLOWS_RUN_FINISHED: $WORKFLOWS_RUN_FINISHED"
echo "WORKFLOWS_RUN_STATUS: $WORKFLOWS_RUN_STATUS"
echo "WORKFLOWS_DELETE: $WORKFLOWS_DELETE"
if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then
echo "Health endpoint check was successful."
exit 0
elif [ "$STATUS_CODE" -ne 502 ]; then
echo "User registration failed with status code $STATUS_CODE."
else
echo "Health check failed."
exit 1
fi
echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)"
echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..."
logs_output=$(docker logs --tail 30 "$CONTAINER_NAME")
echo "$logs_output"
echo "Fetching last 30 lines of logs from container shuffle-opensearch..."
opensearch_logs=$(docker logs --tail 30 shuffle-opensearch)
echo "$opensearch_logs"
sleep $RETRY_INTERVAL
done
echo "User registration failed after $MAX_RETRIES attempts."
exit 1
- name: Run Selenium testing for frontend
run: |
cd $GITHUB_WORKSPACE/frontend
# write some log to see the current directory
chmod +x frontend-testing.sh
./frontend-testing.sh
- name: Get the API key and run a health check
id: health_check
run: |
RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search')
echo "Raw Response: $RESPONSE"
API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey')
if [ -n "$API_KEY" ] && [ "$API_KEY" != "null" ]; then
echo "Admin API key: $API_KEY"
echo "API_KEY=$API_KEY" >> $GITHUB_ENV
else
echo "Failed to retrieve the API key for the admin user."
exit 1
fi
echo "Waiting 1 minute before sending the health API request..."
sleep 60
echo "Checking health API..."
HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \
-H "Authorization: Bearer $API_KEY")
echo "Health API Response: $HEALTH_RESPONSE"
WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create')
WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run')
WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished')
WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status')
WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete')
echo "WORKFLOWS_CREATE: $WORKFLOWS_CREATE"
echo "WORKFLOWS_RUN: $WORKFLOWS_RUN"
echo "WORKFLOWS_RUN_FINISHED: $WORKFLOWS_RUN_FINISHED"
echo "WORKFLOWS_RUN_STATUS: $WORKFLOWS_RUN_STATUS"
echo "WORKFLOWS_DELETE: $WORKFLOWS_DELETE"
if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then
echo "Health endpoint check was successful."
exit 0
else
echo "Health check failed."
exit 1
fi
notify:
needs: build
if: failure()
backend/Dockerfile
+1 -1
View File
@@ -24,7 +24,7 @@ RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o shufflebackend .
# Certificate build - gets required certs
FROM alpine:latest as certs
RUN apk add --update ca-certificates
RUN apk add --update --no-scripts ca-certificates
# Sets up the final image
FROM alpine:3.22.2
backend/go-app/docker.go
+9 -6
View File
@@ -21,7 +21,6 @@ import (
"github.com/docker/docker/api/types"
//"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/client"
newdockerclient "github.com/fsouza/go-dockerclient"
"github.com/go-git/go-billy/v5"
@@ -35,12 +34,13 @@ import (
"os"
"strings"
"time"
batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"time"
// "k8s.io/client-go/tools/clientcmd"
// "k8s.io/client-go/util/homedir"
)
@@ -206,7 +206,8 @@ func fixTags(tags []string) []string {
// Custom Docker image builder wrapper in memory
func buildImageMemory(fs billy.Filesystem, tags []string, dockerfileFolder string, downloadIfFail bool) error {
ctx := context.Background()
client, err := client.NewEnvClient()
// client, err := client.NewEnvClient()
client, _, err := shuffle.GetDockerClient()
defer client.Close()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
@@ -493,7 +494,8 @@ func buildImage(tags []string, dockerfileLocation string) error {
}
ctx := context.Background()
client, err := client.NewEnvClient()
// client, err := client.NewEnvClient()
client, _, err := shuffle.GetDockerClient()
defer client.Close()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
@@ -581,7 +583,8 @@ func imageCheckBuilder(images []string) error {
return nil
ctx := context.Background()
client, err := client.NewEnvClient()
// client, err := client.NewEnvClient()
client, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
return err
@@ -670,7 +673,7 @@ func getDockerImage(resp http.ResponseWriter, request *http.Request) {
}
//log.Printf("[DEBUG] Image to load: %s", version.Name)
dockercli, err := client.NewEnvClient()
dockercli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[WARNING] Unable to create docker client: %s", err)
resp.WriteHeader(422)
backend/go-app/go.mod
+27 -21
View File
@@ -26,15 +26,15 @@ require (
github.com/gorilla/mux v1.8.1
github.com/h2non/filetype v1.1.3
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.54
github.com/shuffle/shuffle-shared v0.9.69
github.com/shuffle/singul v0.0.20
golang.org/x/crypto v0.40.0
golang.org/x/crypto v0.45.0
google.golang.org/api v0.236.0
google.golang.org/grpc v1.72.2
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.33.1
k8s.io/apimachinery v0.33.1
k8s.io/client-go v0.33.1
k8s.io/api v0.34.2
k8s.io/apimachinery v0.34.2
k8s.io/client-go v0.34.2
)
require (
@@ -70,13 +70,13 @@ require (
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.2 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/frikky/schemaless v0.0.24 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/frikky/schemaless v0.0.25 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-jose/go-jose/v4 v4.0.5 // indirect
github.com/go-logr/logr v1.4.2 // indirect
@@ -87,8 +87,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/go-github/v28 v28.1.1 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
@@ -110,9 +109,10 @@ require (
github.com/moby/sys/userns v0.1.0 // indirect
github.com/moby/term v0.5.2 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/openai/openai-go/v3 v3.8.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/osteele/liquid v1.7.0 // indirect
@@ -132,6 +132,10 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/zeebo/errs v1.4.0 // indirect
@@ -145,13 +149,15 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/term v0.33.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/time v0.11.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
google.golang.org/genproto v0.0.0-20250505200425-f936aa4a68b2 // indirect
@@ -163,10 +169,10 @@ require (
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)
backend/go-app/go.sum
+4 -4
View File
@@ -138,8 +138,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/fsouza/go-dockerclient v1.12.1 h1:FMoLq+Zhv9Oz/rFmu6JWkImfr6CBgZOPcL+bHW4gS0o=
github.com/fsouza/go-dockerclient v1.12.1/go.mod h1:OqsgJJcpCwqyM3JED7TdfM9QVWS5O7jSYwXxYKmOooY=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
@@ -344,8 +344,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI=
github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
backend/go-app/main.go
+8 -5
View File
@@ -3,7 +3,7 @@ package main
import (
uuid "github.com/satori/go.uuid"
"github.com/shuffle/shuffle-shared"
"github.com/shuffle/singul/pkg"
singul "github.com/shuffle/singul/pkg"
"net/http/pprof"
@@ -44,11 +44,12 @@ import (
"github.com/go-git/go-git/v5/storage/memory"
// Random
"sort"
xj "github.com/basgys/goxml2json"
newscheduler "github.com/carlescere/scheduler"
"golang.org/x/crypto/bcrypt"
"gopkg.in/yaml.v3"
"sort"
// Web
"github.com/gorilla/mux"
@@ -1312,7 +1313,9 @@ func checkAdminLogin(resp http.ResponseWriter, request *http.Request) {
// Should run calculations
if len(org.SSOConfig.OpenIdAuthorization) > 0 {
baseSSOUrl = shuffle.GetOpenIdUrl(request, *org)
if err != nil {
log.Printf("[ERROR] Failed getting OpenID URL for org %s: %s", org.Name, err)
}
break
}
@@ -5433,7 +5436,7 @@ func initHandlers() {
r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS")
r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS")
// r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS")
r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/users/{key}/get2fa", shuffle.HandleGet2fa).Methods("GET", "OPTIONS")
@@ -5657,7 +5660,7 @@ func initHandlers() {
// Docker orborus specific - downloads an image
r.HandleFunc("/api/v1/get_docker_image", getDockerImage).Methods("POST", "GET", "OPTIONS")
r.HandleFunc("/api/v1/login_sso", shuffle.HandleSSO).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/login_sso", shuffle.HandleSAML).Methods("GET", "POST", "OPTIONS")
r.HandleFunc("/api/v1/login_openid", shuffle.HandleOpenId).Methods("GET", "POST", "OPTIONS")
// Important for email, IDS etc. Create this by:
backend/go-app/walkoff.go
+7 -2
View File
@@ -11,6 +11,7 @@ import (
"io"
"io/ioutil"
"log"
//"math/rand"
"net/http"
"net/url"
@@ -21,7 +22,6 @@ import (
"time"
"github.com/docker/docker/api/types/image"
dockerclient "github.com/docker/docker/client"
"github.com/h2non/filetype"
uuid "github.com/satori/go.uuid"
@@ -2997,6 +2997,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
"email",
"shuffle-ai",
"shuffle-subflow",
"yara",
"sigma",
}
// It's here to prevent getting them in every iteration
@@ -3393,6 +3395,8 @@ func IterateAppGithubFolders(ctx context.Context, fs billy.Filesystem, dir []os.
fmt.Sprintf("/apps"),
orgId,
false,
"HIGH",
"APP_BUID",
)
}
@@ -3527,7 +3531,8 @@ func LoadSpecificApps(resp http.ResponseWriter, request *http.Request) {
// As it's not even Docker
if tmpBody.ForceUpdate {
dockercli, err := dockerclient.NewEnvClient()
// dockercli, err := dockerclient.NewEnvClient()
dockercli, _, err := shuffle.GetDockerClient()
if err == nil {
appSdk := os.Getenv("SHUFFLE_APP_SDK_VERSION")
docker-compose.yml
+1 -1
View File
@@ -50,7 +50,7 @@ services:
- HTTPS_PROXY=${HTTPS_PROXY}
- SHUFFLE_PASS_WORKER_PROXY=${SHUFFLE_PASS_WORKER_PROXY}
- SHUFFLE_PASS_APP_PROXY=${SHUFFLE_PASS_APP_PROXY}
- SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS}
# - SHUFFLE_APP_REPLICAS=${SHUFFLE_APP_REPLICAS}
- SHUFFLE_STATS_DISABLED=true
- SHUFFLE_LOGS_DISABLED=true
- SHUFFLE_SWARM_CONFIG=run
frontend/package.json
+3
View File
@@ -26,10 +26,13 @@
"cpx": "^1.5.0",
"create-react-app": "^5.0.1",
"cytoscape": "^3.29.2",
"cytoscape-bubblesets": "^4.1.0",
"cytoscape-clipboard": "^2.2.1",
"cytoscape-cose-bilkent": "^4.1.0",
"cytoscape-cxtmenu": "^3.4.0",
"cytoscape-edgehandles": "^3.5.1",
"cytoscape-grid-guide": "~2.3.3",
"cytoscape-layers": "^3.1.0",
"cytoscape-node-html-label": "^1.2.2",
"cytoscape-panzoom": "^2.5.3",
"cytoscape-undo-redo": "^1.3.3",
frontend/src/components/LeftSideBar.jsx
+3 -1
View File
@@ -128,6 +128,8 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => {
}
}, [userdata]);
const CustomPopper = (props) => {
return (
<Portal>
@@ -1949,7 +1951,7 @@ const LeftSideBar = ({ userdata, serverside, globalUrl, notifications, }) => {
</div>
}
{!isCloud ? (
{!isCloud && activeOrgData?.old_org? (
<div
style={{
display: "flex",
frontend/src/components/ShuffleCodeEditor1.jsx
+242 -238
View File
@@ -18,7 +18,7 @@ import {
Collapse,
} from '@mui/material';
import { Context } from '../context/ContextApi.jsx';
import {getTheme} from '../theme.jsx';
import { getTheme } from '../theme.jsx';
import Checkbox from '@mui/material/Checkbox';
import { isMobile } from "react-device-detect"
import { NestedMenuItem } from "mui-nested-menu"
@@ -179,7 +179,7 @@ const CodeEditor = (props) => {
const [localcodedata, setlocalcodedata] = React.useState(codedata === undefined || codedata === null || codedata.length === 0 ? "" : codedata);
// const {codelang, setcodelang} = props
const {themeMode, supportEmail} = useContext(Context)
const { themeMode, supportEmail } = useContext(Context)
const theme = getTheme(themeMode)
const [validation, setValidation] = React.useState(false);
@@ -370,9 +370,9 @@ const CodeEditor = (props) => {
setMainVariables(tmpVariables)
}
const handleKeyDown = (event) => {
if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') {
event.preventDefault()
const handleKeyDown = (event) => {
if ((event.metaKey || event.ctrlKey) && event.key === 'Enter') {
event.preventDefault()
const tryItButton = document.getElementById("try-it-button")
if (tryItButton !== undefined && tryItButton !== null) {
tryItButton.click()
@@ -382,7 +382,7 @@ const CodeEditor = (props) => {
// Remove the original useEffect for actionlist since we'll update on action/trigger changes
useEffect(() => {
document.addEventListener("keydown", handleKeyDown)
document.addEventListener("keydown", handleKeyDown)
updateAvailableVariables(actionlist)
}, [])
@@ -972,7 +972,11 @@ const CodeEditor = (props) => {
// Resolve the child paths/variables first,
// So that "$a.b.c" is handled before "$a.b" and then "$a"
// Replace "$parent.child" before "$parent" so "$parent.child" doesn't become "parentValue.child" (which can't be matched later).
found.sort((a, b) => b.length - a.length)
console.log("FOUND PROD BEFORE SORT:", found)
found.sort((a, b) => {
return Number(b.length) - Number(a.length)
})
console.log("FOUND PROD AFTER SORT:", found)
//console.log("FOUND: ", found)
try {
@@ -983,8 +987,8 @@ const CodeEditor = (props) => {
var valuefound = false
// First check if it's a workflow variable
if (actionlist !== undefined && actionlist.length > 0) {
// First check if it's a workflow variable
if (actionlist !== undefined && actionlist.length > 0) {
const workflowVar = actionlist?.find(item =>
item.type === "workflow_variable" &&
`$${item.autocomplete.toLowerCase()}` === fixedVariable.toLowerCase()
@@ -995,7 +999,7 @@ const CodeEditor = (props) => {
try {
// Try to parse the example value if it's stored as a JSON string
if (typeof workflowVar.example === "string" &&
(workflowVar.example.startsWith("[") || workflowVar.example.startsWith("{"))) {
(workflowVar.example.startsWith("[") || workflowVar.example.startsWith("{"))) {
const parsedExample = JSON.parse(workflowVar.example)
input = input.replace(found[i], JSON.stringify(parsedExample), -1)
} else {
@@ -1083,24 +1087,24 @@ const CodeEditor = (props) => {
for (var k = 0; k < actionlist.length; k++) {
var parsedPaths = []
// Handle both workflow variables and regular actions
if (actionlist[k].type === "workflow_variable") {
// Try to parse the value if it's a string that could be JSON
if (typeof actionlist[k].value === "string") {
try {
const parsedValue = JSON.parse(actionlist[k].value)
if (typeof parsedValue === "object") {
parsedPaths = GetParsedPaths(parsedValue, "");
}
} catch (e) {
// Not valid JSON, use the value directly
parsedPaths = GetParsedPaths(actionlist[k].value, "");
// Handle both workflow variables and regular actions
if (actionlist[k].type === "workflow_variable") {
// Try to parse the value if it's a string that could be JSON
if (typeof actionlist[k].value === "string") {
try {
const parsedValue = JSON.parse(actionlist[k].value)
if (typeof parsedValue === "object") {
parsedPaths = GetParsedPaths(parsedValue, "");
}
} else if (typeof actionlist[k].value === "object") {
} catch (e) {
// Not valid JSON, use the value directly
parsedPaths = GetParsedPaths(actionlist[k].value, "");
}
} else if (typeof actionlist[k].example === "object") {
parsedPaths = GetParsedPaths(actionlist[k].example, "");
} else if (typeof actionlist[k].value === "object") {
parsedPaths = GetParsedPaths(actionlist[k].value, "");
}
} else if (typeof actionlist[k].example === "object") {
parsedPaths = GetParsedPaths(actionlist[k].example, "");
}
for (var key in parsedPaths) {
@@ -1116,19 +1120,19 @@ const CodeEditor = (props) => {
var new_input = ""
try {
const sourceData = actionlist[k].type === "workflow_variable" ?
(() => {
// Try to parse the value if it's a JSON string
if (typeof actionlist[k].value === "string") {
try {
return JSON.parse(actionlist[k].value);
} catch (e) {
// If parsing fails, return the original string value
return actionlist[k].value;
(() => {
// Try to parse the value if it's a JSON string
if (typeof actionlist[k].value === "string") {
try {
return JSON.parse(actionlist[k].value);
} catch (e) {
// If parsing fails, return the original string value
return actionlist[k].value;
}
}
}
return actionlist[k].value;
})() :
actionlist[k].example;
return actionlist[k].value;
})() :
actionlist[k].example;
new_input = FindJsonPath(fullpath, sourceData)
} catch (e) {
@@ -1437,83 +1441,83 @@ const CodeEditor = (props) => {
return (
<Draggable
style={{
position: "absolute",
zIndex: 15000,
}}
onDrag={(e) => {
e.preventDefault()
// Check if inside div.ace_content
if (e.srcElement.className === "ace_content") {
// Input on the correct line. Each line is:
// Show some tooltip at mouse cursor that shows "Insert Action"
style={{
position: "absolute",
zIndex: 15000,
}}
onDrag={(e) => {
e.preventDefault()
// Check if inside div.ace_content
if (e.srcElement.className === "ace_content") {
// Input on the correct line. Each line is:
// Show some tooltip at mouse cursor that shows "Insert Action"
// Append the text to the DOM
} else {
//console.log("PAGEX: ", e.pageX, e.pageY)
//console.log("OffsetX: ", e.offsetX, e.offsetY)
//console.log("E: ", e)
}
// Append the text to the DOM
} else {
//console.log("PAGEX: ", e.pageX, e.pageY)
//console.log("OffsetX: ", e.offsetX, e.offsetY)
//console.log("E: ", e)
}
if (!dragging) {
setDragging(true)
}
}}
onStop={(e) => {
if (e.srcElement.className === "ace_content") {
console.log("DRAG STOP IN CONTENT!", e.srcElement.className)
if (!dragging) {
setDragging(true)
}
}}
onStop={(e) => {
if (e.srcElement.className === "ace_content") {
console.log("DRAG STOP IN CONTENT!", e.srcElement.className)
const usedposition = e.offsetY
if (usedposition === undefined || usedposition === null) {
toast.info(`Error: LayerY is undefined or null. Please contact ${supportEmail}`)
return
}
const usedposition = e.offsetY
if (usedposition === undefined || usedposition === null) {
toast.info(`Error: LayerY is undefined or null. Please contact ${supportEmail}`)
return
}
if (usedposition === 0) {
usedposition = 1
}
if (usedposition === 0) {
usedposition = 1
}
const lineheight = 15
const codedatasplit = localcodedata.split('\n')
if (codedatasplit === undefined || codedatasplit === null || codedatasplit.length === 0) {
return
}
const lineheight = 15
const codedatasplit = localcodedata.split('\n')
if (codedatasplit === undefined || codedatasplit === null || codedatasplit.length === 0) {
return
}
// Int
const lineposition = parseInt(usedposition/lineheight)
// Int
const lineposition = parseInt(usedposition / lineheight)
// Find the correct line
if (lineposition > codedatasplit.length) {
codedatasplit[codedatasplit.length-1] += value
} else {
codedatasplit[lineposition] += value
}
// Find the correct line
if (lineposition > codedatasplit.length) {
codedatasplit[codedatasplit.length - 1] += value
} else {
codedatasplit[lineposition] += value
}
//e.srcElement.layerY
setlocalcodedata(codedatasplit.join('\n'))
}
//e.srcElement.layerY
setlocalcodedata(codedatasplit.join('\n'))
}
setDragging(false)
}}
dragging={dragging}
position={{
x: 0,
y: 0,
}}
onMouseHover={() => {
setHovering(true)
}}
onMouseLeave={() => {
setHovering(false)
}}
setDragging(false)
}}
dragging={dragging}
position={{
x: 0,
y: 0,
}}
onMouseHover={() => {
setHovering(true)
}}
onMouseLeave={() => {
setHovering(false)
}}
>
<div
style={{
cursor: dragging ? "grabbing" : "grab",
minWidth: 100,
border: "1px solid rgba(255,255,255,0.5)",
borderRadius: theme.palette.borderRadius/2,
borderRadius: theme.palette.borderRadius / 2,
marginRight: 10,
padding: 5,
}}
@@ -1525,56 +1529,56 @@ const CodeEditor = (props) => {
}
const IndentJsonLikeString = (input, indentSize = 2) => {
const indent = ' '.repeat(indentSize);
let level = 0;
let inString = false;
let escapeNext = false;
let result = '';
const indent = ' '.repeat(indentSize);
let level = 0;
let inString = false;
let escapeNext = false;
let result = '';
for (let i = 0; i < input.length; i++) {
let char = input[i];
for (let i = 0; i < input.length; i++) {
let char = input[i];
if (escapeNext) {
result += char;
escapeNext = false;
continue;
if (escapeNext) {
result += char;
escapeNext = false;
continue;
}
if (char === '\\') {
escapeNext = true;
result += char;
continue;
}
if (char === '"') {
inString = !inString;
result += char;
continue;
}
if (!inString) {
if (char === '{' || char === '[') {
result += char + '\n' + indent.repeat(++level);
continue;
} else if (char === '}' || char === ']') {
result += '\n' + indent.repeat(--level) + char;
continue;
} else if (char === ',') {
result += char + '\n' + indent.repeat(level);
continue;
} else if (char === ':') {
result += ': ';
continue;
} else if (char === ' ' || char === '\t' || char === '\n' || char === '\r') {
// Skip whitespace characters when not in string
continue;
}
}
result += char;
}
if (char === '\\') {
escapeNext = true;
result += char;
continue;
}
if (char === '"') {
inString = !inString;
result += char;
continue;
}
if (!inString) {
if (char === '{' || char === '[') {
result += char + '\n' + indent.repeat(++level);
continue;
} else if (char === '}' || char === ']') {
result += '\n' + indent.repeat(--level) + char;
continue;
} else if (char === ',') {
result += char + '\n' + indent.repeat(level);
continue;
} else if (char === ':') {
result += ': ';
continue;
} else if (char === ' ' || char === '\t' || char === '\n' || char === '\r') {
// Skip whitespace characters when not in string
continue;
}
}
result += char;
}
return result;
return result;
}
@@ -1585,7 +1589,7 @@ const CodeEditor = (props) => {
return (
<div style={{minHeight: 40, marginTop: 10, }}>
<div style={{ minHeight: 40, marginTop: 10, }}>
<div style={{
cursor: "pointer",
display: "flex",
@@ -1594,7 +1598,7 @@ const CodeEditor = (props) => {
setExpanded(!expanded)
}}>
<ValueBox name={innerdata?.name} value={innerdata?.value} />
<Typography style={{marginTop: 5, maxWidth: 150, maxHeight: 40, overflow: "hidden", }}>
<Typography style={{ marginTop: 5, maxWidth: 150, maxHeight: 40, overflow: "hidden", }}>
{innerdata?.name}
</Typography>
</div>
@@ -1781,7 +1785,7 @@ const CodeEditor = (props) => {
};
const handleActionHover = (inside, actionId) => {
};
};
const handleMouseover = () => {
if (innerdata.type === "Execution Argument") {
@@ -1802,20 +1806,20 @@ const CodeEditor = (props) => {
var parsedPaths = [];
if (innerdata.type === "workflow_variable") {
// Try to parse the value if it's a string that could be JSON
if (typeof innerdata.value === "string") {
if (typeof innerdata.value === "string") {
try {
const parsedValue = JSON.parse(innerdata.value)
if (typeof parsedValue === "object") {
parsedPaths = GetParsedPaths(parsedValue, "");
}
const parsedValue = JSON.parse(innerdata.value)
if (typeof parsedValue === "object") {
parsedPaths = GetParsedPaths(parsedValue, "");
}
} catch (e) {
// Not valid JSON, use the value directly
parsedPaths = GetParsedPaths(innerdata.value, "");
// Not valid JSON, use the value directly
parsedPaths = GetParsedPaths(innerdata.value, "");
}
} else if (typeof innerdata.value === "object") {
} else if (typeof innerdata.value === "object") {
parsedPaths = GetParsedPaths(innerdata.value, "");
}
} else if (typeof innerdata.example === "object") {
}
} else if (typeof innerdata.example === "object") {
parsedPaths = GetParsedPaths(innerdata.example, "");
}
@@ -1830,7 +1834,7 @@ const CodeEditor = (props) => {
)
})}
</div>
: null}
: null}
<div style={{ flex: 3, }}>
{isFileEditor ?
@@ -1921,7 +1925,7 @@ const CodeEditor = (props) => {
>
<AddIcon /> Show Source Data
</Button>
: null}
: null}
<Button
id="basic-button"
@@ -2018,18 +2022,18 @@ const CodeEditor = (props) => {
}}
key={index} onClick={() => {
if (item.disabled) {
toast.error("This feature may not work in your environment until you update your Shuffle Tools app.", { autoClose: 10000 })
}
if (item.disabled) {
toast.error("This feature may not work in your environment until you update your Shuffle Tools app.", { autoClose: 10000 })
}
if (selectedAction.name !== "execute_python") {
var newitem = JSON.parse(JSON.stringify(item))
newitem.value = `{% python %}\n${item.value}\n{% endpython %}`
handleClick(newitem)
} else {
handleClick(item)
}
}}>{item.name}</MenuItem>
if (selectedAction.name !== "execute_python") {
var newitem = JSON.parse(JSON.stringify(item))
newitem.value = `{% python %}\n${item.value}\n{% endpython %}`
handleClick(newitem)
} else {
handleClick(item)
}
}}>{item.name}</MenuItem>
)
})}
</Menu>
@@ -2071,9 +2075,9 @@ const CodeEditor = (props) => {
<FavoriteBorderIcon style={{ marginRight: 10 }} />
) :
innerdata.type === "Shuffle DB" ?
<StorageIcon style={{ marginRight: 10, }} />
:
<ScheduleIcon style={{ marginRight: 10 }} />
<StorageIcon style={{ marginRight: 10, }} />
:
<ScheduleIcon style={{ marginRight: 10 }} />
const handleExecArgumentHover = (inside) => {
var exec_text_field = document.getElementById(
@@ -2110,20 +2114,20 @@ const CodeEditor = (props) => {
var parsedPaths = [];
if (innerdata.type === "workflow_variable") {
// Try to parse the value if it's a string that could be JSON
if (typeof innerdata.value === "string") {
if (typeof innerdata.value === "string") {
try {
const parsedValue = JSON.parse(innerdata.value)
if (typeof parsedValue === "object") {
parsedPaths = GetParsedPaths(parsedValue, "");
}
const parsedValue = JSON.parse(innerdata.value)
if (typeof parsedValue === "object") {
parsedPaths = GetParsedPaths(parsedValue, "");
}
} catch (e) {
// Not valid JSON, use the value directly
parsedPaths = GetParsedPaths(innerdata.value, "");
// Not valid JSON, use the value directly
parsedPaths = GetParsedPaths(innerdata.value, "");
}
} else if (typeof innerdata.value === "object") {
} else if (typeof innerdata.value === "object") {
parsedPaths = GetParsedPaths(innerdata.value, "");
}
} else if (typeof innerdata.example === "object") {
}
} else if (typeof innerdata.example === "object") {
parsedPaths = GetParsedPaths(innerdata.example, "");
}
@@ -2486,29 +2490,29 @@ const CodeEditor = (props) => {
}}
>
<div>
{actionId === null && triggerId === null ?
<div style={{display: "flex", alignItems: "center"}}>
{`Condition ${selectedEdge?.conditions?.findIndex(cond => cond.condition.id === conditionId) + 1 || "0"}`}
{/* Source node image */}
{selectedEdge?.source ?
<img
src={sourceImage || ""}
alt="Source"
style={{
width: 30,
height: 30,
marginRight: 10,
borderRadius: "50%",
marginLeft: 10,
border: conditionField === "source" ? `3px solid #FF8544` : null,
}}
/>
: null
}
{actionId === null && triggerId === null ?
<div style={{ display: "flex", alignItems: "center" }}>
{`Condition ${selectedEdge?.conditions?.findIndex(cond => cond.condition.id === conditionId) + 1 || "0"}`}
{/* Source node image */}
{selectedEdge?.source ?
<img
src={sourceImage || ""}
alt="Source"
style={{
width: 30,
height: 30,
marginRight: 10,
borderRadius: "50%",
marginLeft: 10,
border: conditionField === "source" ? `3px solid #FF8544` : null,
}}
/>
: null
}
{/* Add arrow icon */}
{
selectedEdge && Object.keys(selectedEdge).length > 0 ?
{/* Add arrow icon */}
{
selectedEdge && Object.keys(selectedEdge).length > 0 ?
<ArrowForwardIcon style={{
color: theme.palette.textPrimary,
fontSize: 18,
@@ -2516,26 +2520,26 @@ const CodeEditor = (props) => {
marginRight: -5,
}} />
: null
}
}
{/* Destination node image */}
{selectedEdge?.target ?
<img
src={targetImage || ""}
alt="Destination"
style={{
width: 30,
height: 30,
marginLeft: 10,
borderRadius: "50%",
border: conditionField === "destination" ? `3px solid #FF8544` : null,
}}
/>
: null
}
</div>
:
<span style={{ color: theme.palette.text.primary }}>
{/* Destination node image */}
{selectedEdge?.target ?
<img
src={targetImage || ""}
alt="Destination"
style={{
width: 30,
height: 30,
marginLeft: 10,
borderRadius: "50%",
border: conditionField === "destination" ? `3px solid #FF8544` : null,
}}
/>
: null
}
</div>
:
<span style={{ color: theme.palette.text.primary }}>
{selectedAction.name === "execute_python" || selectedAction.name === "execute_bash" ?
"Code to run" :
triggerId ?
@@ -2592,18 +2596,18 @@ const CodeEditor = (props) => {
<span>
<PlayArrowIcon style={{ height: 18, width: 18, marginBottom: -4, marginLeft: 5, }} />
{selectedAction === undefined ? <Typography style={{color: "inherit"}}>Try it</Typography> : selectedAction.name === "execute_python" ? "Run Python Code" : selectedAction.name === "execute_bash" ? "Run Bash" : "Try it"}
<span
style={{
color: "#C8C8C8",
fontSize: "12px",
whiteSpace: "nowrap",
marginLeft: 5,
marginRight: 10,
}}
>
<kbd>Ctrl</kbd> + <kbd><KeyboardReturnIcon style={{width: 13, position: "absolute", marginLeft: 3, top: 5, }}/></kbd>
</span>
{selectedAction === undefined ? <Typography style={{ color: "inherit" }}>Try it</Typography> : selectedAction.name === "execute_python" ? "Run Python Code" : selectedAction.name === "execute_bash" ? "Run Bash" : "Try it"}
<span
style={{
color: "#C8C8C8",
fontSize: "12px",
whiteSpace: "nowrap",
marginLeft: 5,
marginRight: 10,
}}
>
<kbd>Ctrl</kbd> + <kbd><KeyboardReturnIcon style={{ width: 13, position: "absolute", marginLeft: 3, top: 5, }} /></kbd>
</span>
</span>
}
@@ -2713,7 +2717,7 @@ const CodeEditor = (props) => {
Output is based on the last VALID run of the node(s) you are referencing. Refresh the page to get updated Variable values.&nbsp;
{selectedAction?.name === "execute_python" ?
"For Python: exit() to stop a python script ANYWHERE."
: null}
: null}
</Typography>
<Typography variant="body2" style={{ maxHeight: 150, overflow: "auto", marginTop: 20, }}>
No test output yet.
@@ -2735,7 +2739,7 @@ const CodeEditor = (props) => {
</div>
<div style={{ display: 'flex', width: fullScreenModeEnabled ? "92%" : isWorkflowEditor ? "90%" : "100%", }}>
<div style={{ display: 'flex', width: fullScreenModeEnabled ? "92%" : isWorkflowEditor ? "90%" : "100%", }}>
<Button
style={{
height: 35,
frontend/src/views/Workflows2.jsx
+3 -1
View File
@@ -812,6 +812,7 @@ const Workflows2 = (props) => {
const [isLoadingWorkflow, setIsLoadingWorkflow] = useState(false);
const [isLoadingPublicWorkflow, setIsLoadingPublicWorkflow] = useState(false);
const [view, setView] = useState(localStorage?.getItem("workflowView") || "grid");
const [currentOrg, setCurrentOrg] = useState(null);
const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io";
const [showExecutionStats, setShowExecutionStats] = React.useState(localStorage?.getItem("showExecutionStats") === "true")
@@ -971,6 +972,7 @@ const Workflows2 = (props) => {
.then((org) => {
if (!fetched && org) {
if (!isCloud) {
setCurrentOrg(org);
if (org?.cloud_sync && org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) {
setIsProdStatusOn(true);
} else if (org?.subscriptions[0]?.name?.toLowerCase().includes("enterprise") && org?.subscriptions[0]?.active) {
@@ -5518,7 +5520,7 @@ const Workflows2 = (props) => {
</div>
}
{!isCloud ? (
{!isCloud && currentOrg?.old_org ? (
<div
style={{
display: "flex",
functions/kubernetes/README.md
+29
View File
@@ -10,3 +10,32 @@ More details in the [kubernetes/Charts/Shuffle folder.](https://github.com/Shuff
Here is the default architecture it follows, with the "Frontend" being the exposed container you interact with.
<img width="1006" height="1069" alt="image" src="https://github.com/user-attachments/assets/263369a1-6944-4ef5-8f19-14bc234130d8" />
## GKE testing
For testing on GKE, you can use the following command:
```bash
PROJECT_NAME="project"
CLUSTER_NAME="cluster"
# feel free to change the region and machine type
gcloud container clusters create "$CLUSTER_NAME" \
--project "$PROJECT_NAME" \
--region "asia-south2" \
--release-channel "regular" \
--machine-type "e2-standard-2" \
--num-nodes "1" \
--node-locations "asia-south2-b,asia-south2-a,asia-south2-c"
# given that your gcloud cli is connected with kubectl
kubectl config set-context $CLUSTER_NAME
# apply shuffle k8s helm chart
helm install shuffle oci://ghcr.io/shuffle/charts/shuffle \
--version 2.1.1 \
--namespace shuffle \
--create-namespace
# verify
kubectl get pods -n shuffle
```
functions/kubernetes/charts/shuffle/values.yaml
-9
View File
@@ -1558,15 +1558,6 @@ app:
##
annotations: {}
## @param app.serviceAccount.automountServiceAccountToken Automount service account token for the app service account
## NOTE: You likely want to allow access to cluster-proxies, e.g:
## extraEgress:
## - to:
## - namespaceSelector:
## matchLabels:
## kubernetes.io/metadata.name: istio-system
## podSelector:
## matchLabels:
## istio: pilot
##
automountServiceAccountToken: true
## @param app.serviceAccount.imagePullSecrets Add image pull secrets to the app service account
functions/onprem/orborus/go.mod
+28 -22
View File
@@ -10,9 +10,9 @@ require (
github.com/docker/docker v28.3.3+incompatible
github.com/docker/go-connections v0.5.0
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.53
k8s.io/api v0.33.1
k8s.io/apimachinery v0.33.1
github.com/shuffle/shuffle-shared v0.9.69
k8s.io/api v0.34.2
k8s.io/apimachinery v0.34.2
)
require (
@@ -52,14 +52,14 @@ require (
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.2 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/frikky/kin-openapi v0.42.0 // indirect
github.com/frikky/schemaless v0.0.24 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/frikky/schemaless v0.0.25 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.6.2 // indirect
@@ -73,8 +73,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/go-github/v28 v28.1.1 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
@@ -89,8 +88,9 @@ require (
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/sys/sequential v0.6.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/openai/openai-go/v3 v3.8.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/osteele/liquid v1.7.0 // indirect
@@ -106,8 +106,12 @@ require (
github.com/shuffle/opensearch-go/v4 v4.0.0 // indirect
github.com/skeema/knownhosts v1.3.1 // indirect
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/zeebo/errs v1.4.0 // indirect
@@ -121,14 +125,16 @@ require (
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
go.opentelemetry.io/proto/otlp v1.5.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/crypto v0.40.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/term v0.33.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/time v0.11.0 // indirect
google.golang.org/api v0.236.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
@@ -142,12 +148,12 @@ require (
gopkg.in/warnings.v0 v0.1.2 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/client-go v0.33.1 // indirect
k8s.io/client-go v0.34.2 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)
functions/onprem/orborus/go.sum
+4 -4
View File
@@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -309,8 +309,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
github.com/shuffle/shuffle-shared v0.9.61 h1:osyJqgRx68m9l+kssB1fpHA4pmkNXK4ZAVVEHL5dW3Q=
github.com/shuffle/shuffle-shared v0.9.61/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
functions/onprem/orborus/orborus.go
+42 -4
View File
@@ -136,11 +136,34 @@ var window = shuffle.NewTimeWindow(1 * time.Minute)
func init() {
var err error
dockercli, err = dockerclient.NewEnvClient()
// dockercli, err = dockerclient.NewEnvClient()
dockercli, dockerApiVersion, err = shuffle.GetDockerClient()
if err != nil {
log.Printf("Unable to create docker client: %s", err)
}
if os.Getenv("SHUFFLE_EC2_INSTANCE") == "true" {
log.Printf("[INFO] Detected AWS EC2 instance. Setting up Docker Swarm with AWS optimizations.")
containers, err := dockercli.ContainerList(context.Background(), container.ListOptions{})
if err == nil {
for _, container := range containers {
if strings.Contains(container.Image, "shuffle-orborus") {
if len(container.Names) != 0 {
if strings.Contains(container.Names[0], "shuffle-orborus") {
containerName = container.Names[0]
containerName = strings.TrimPrefix(containerName, "/")
os.Setenv("ORBORUS_CONTAINER_NAME", containerName)
log.Printf("[DEBUG] Found orborus container name: %s", containerName)
break
}
}
}
}
} else {
log.Printf("[ERROR] Failed to find orborus container: %s", err)
}
}
getThisContainerId()
if len(pipelineApikey) == 0 {
@@ -1426,6 +1449,9 @@ func deployK8sWorker(image string, identifier string, env []string) error {
}
replicaNumberInt32 := int32(replicaNumber)
// worker makes authenticated requests to the k8s api to create app deployments.
// Therefore, it needs to have access to the service account token.
automountServiceAccountToken := true
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
@@ -1445,9 +1471,10 @@ func deployK8sWorker(image string, identifier string, env []string) error {
Containers: []corev1.Container{
containerAttachment,
},
DNSPolicy: corev1.DNSClusterFirst,
ServiceAccountName: workerServiceAccountName,
SecurityContext: podSecurityContext,
DNSPolicy: corev1.DNSClusterFirst,
ServiceAccountName: workerServiceAccountName,
AutomountServiceAccountToken: &automountServiceAccountToken,
SecurityContext: podSecurityContext,
},
},
},
@@ -2411,6 +2438,7 @@ func main() {
}
if swarmConfig == "run" || swarmConfig == "swarm" || isKubernetes == "true" {
if isKubernetes != "true" {
checkSwarmService(ctx)
}
@@ -2745,6 +2773,8 @@ func main() {
fmt.Sprintf("/detections/Sigma"),
org,
true,
"LOW",
"TENZIR_START",
)
if err != nil {
@@ -2755,6 +2785,10 @@ func main() {
}
} else {
if debug {
log.Printf("[DEBUG] Passing execution ID request to normal queue: %#v", incRequest.ExecutionId)
}
newrequests = append(newrequests, incRequest)
}
}
@@ -4289,6 +4323,10 @@ func sendWorkerRequest(workflowExecution shuffle.ExecutionRequest, image string,
Timeout: time.Duration(120 * time.Second),
}
if debug {
log.Printf("[DEBUG][%s] Worker request to be sent to URL: %s", workflowExecution.ExecutionId, streamUrl)
}
req, err := http.NewRequest(
"POST",
streamUrl,
functions/onprem/worker/go.mod
+27 -21
View File
@@ -11,11 +11,11 @@ require (
github.com/docker/docker v28.3.3+incompatible
github.com/gorilla/mux v1.8.1
github.com/satori/go.uuid v1.2.0
github.com/shuffle/shuffle-shared v0.9.53
github.com/shuffle/shuffle-shared v0.9.69
github.com/shuffle/singul v0.0.20
k8s.io/api v0.33.1
k8s.io/apimachinery v0.33.1
k8s.io/client-go v0.33.1
k8s.io/api v0.34.2
k8s.io/apimachinery v0.34.2
k8s.io/client-go v0.34.2
)
require (
@@ -54,14 +54,14 @@ require (
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/emicklei/go-restful/v3 v3.11.0 // indirect
github.com/emicklei/go-restful/v3 v3.12.2 // indirect
github.com/emirpasic/gods v1.18.1 // indirect
github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/frikky/kin-openapi v0.42.0 // indirect
github.com/frikky/schemaless v0.0.24 // indirect
github.com/fxamacker/cbor/v2 v2.7.0 // indirect
github.com/frikky/schemaless v0.0.25 // indirect
github.com/fxamacker/cbor/v2 v2.9.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
github.com/go-git/go-billy/v5 v5.6.2 // indirect
@@ -75,8 +75,7 @@ require (
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
github.com/golang/protobuf v1.5.4 // indirect
github.com/google/gnostic-models v0.6.9 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/gnostic-models v0.7.0 // indirect
github.com/google/go-github/v28 v28.1.1 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/s2a-go v0.1.9 // indirect
@@ -91,8 +90,9 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/openai/openai-go/v3 v3.8.1 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.1 // indirect
github.com/osteele/liquid v1.7.0 // indirect
@@ -110,6 +110,10 @@ require (
github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
github.com/spf13/pflag v1.0.6 // indirect
github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect
github.com/tidwall/gjson v1.18.0 // indirect
github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.1 // indirect
github.com/tidwall/sjson v1.2.5 // indirect
github.com/x448/float16 v0.8.4 // indirect
github.com/xanzy/ssh-agent v0.3.3 // indirect
github.com/zeebo/errs v1.4.0 // indirect
@@ -121,14 +125,16 @@ require (
go.opentelemetry.io/otel/sdk v1.36.0 // indirect
go.opentelemetry.io/otel/sdk/metric v1.36.0 // indirect
go.opentelemetry.io/otel/trace v1.36.0 // indirect
go.yaml.in/yaml/v2 v2.4.2 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
go4.org v0.0.0-20230225012048-214862532bf5 // indirect
golang.org/x/crypto v0.40.0 // indirect
golang.org/x/net v0.41.0 // indirect
golang.org/x/crypto v0.45.0 // indirect
golang.org/x/net v0.47.0 // indirect
golang.org/x/oauth2 v0.30.0 // indirect
golang.org/x/sync v0.16.0 // indirect
golang.org/x/sys v0.34.0 // indirect
golang.org/x/term v0.33.0 // indirect
golang.org/x/text v0.27.0 // indirect
golang.org/x/sync v0.18.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/term v0.37.0 // indirect
golang.org/x/text v0.31.0 // indirect
golang.org/x/time v0.11.0 // indirect
google.golang.org/api v0.236.0 // indirect
google.golang.org/appengine v1.6.8 // indirect
@@ -143,10 +149,10 @@ require (
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
k8s.io/utils v0.0.0-20241104100929-3ea5e8cea738 // indirect
sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 // indirect
sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
sigs.k8s.io/randfill v1.0.0 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
sigs.k8s.io/yaml v1.4.0 // indirect
sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect
sigs.k8s.io/yaml v1.6.0 // indirect
)
functions/onprem/worker/go.sum
+4 -4
View File
@@ -128,8 +128,8 @@ github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/frikky/kin-openapi v0.42.0 h1:d5Z6vnuQ6RnCCPIxZaDL+TH2ODLxT8abytOt+Zh+Kd0=
github.com/frikky/kin-openapi v0.42.0/go.mod h1:ev9OZAw7Bv5p0w93j91++6a1ElPzGcCofst+kmrWsj4=
github.com/frikky/schemaless v0.0.24 h1:1bNg96gm82yLbRlDS0haQH/96X3GlCAEuTCSasXIJZE=
github.com/frikky/schemaless v0.0.24/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/frikky/schemaless v0.0.25 h1:qXjrKT54LWl3tzDFIUmevZ86F7FiusPpQKfa40xxNl8=
github.com/frikky/schemaless v0.0.25/go.mod h1:m9s+6gALXhA5ZERCrJw+jI2rRtTPNa8mkl4vav9sxnY=
github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E=
github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ=
github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
@@ -311,8 +311,8 @@ github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN
github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4=
github.com/shuffle/opensearch-go/v4 v4.0.0 h1:Mh85CD1MwOgXiFFYlzS1llnvdqL3CztRdR1ZT/SLIjU=
github.com/shuffle/opensearch-go/v4 v4.0.0/go.mod h1:gVLZKQE5khQWMb68XBtgKrhu78oLGL2zHwAGnFMDwC0=
github.com/shuffle/shuffle-shared v0.9.53 h1:ykziiRA82FZX0YiuErSmFLQMxvAFi6yUajgRjd3J5xE=
github.com/shuffle/shuffle-shared v0.9.53/go.mod h1:g03P/ZM8pUwBbdxW0l9A1siMMuemEQshgDpO9hF23eE=
github.com/shuffle/shuffle-shared v0.9.60 h1:8NiovcsSsVX8i0UHmbFBANjUQ/u+4PDDpJXg9JfMAUo=
github.com/shuffle/shuffle-shared v0.9.60/go.mod h1:vK6t1WY5Nfg5vOAk6taT788jIGKs6/4iN1d8Argyn4o=
github.com/shuffle/singul v0.0.20 h1:Lz+K4l2GJQ5W6o8ePr/2qlIcGuZuMC7W8Wj/JFw1XDI=
github.com/shuffle/singul v0.0.20/go.mod h1:qY3ZmwaNwmRApLWMTqZaNEMXO6X8bjr/zY44xDU62hQ=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
functions/onprem/worker/worker.go
+38 -14
View File
@@ -625,6 +625,8 @@ func deployk8sApp(image string, identifier string, env []string) error {
}
replicaNumberInt32 := int32(replicaNumber)
// apps do not need access the k8s api.
automountServiceAccountToken := false
deployment := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
@@ -656,9 +658,10 @@ func deployk8sApp(image string, identifier string, env []string) error {
Resources: buildResourcesFromEnv(),
},
},
DNSPolicy: corev1.DNSClusterFirst,
ServiceAccountName: appServiceAccountName,
SecurityContext: podSecurityContext,
DNSPolicy: corev1.DNSClusterFirst,
ServiceAccountName: appServiceAccountName,
AutomountServiceAccountToken: &automountServiceAccountToken,
SecurityContext: podSecurityContext,
},
},
},
@@ -1243,7 +1246,8 @@ func DeployContainer(ctx context.Context, cli *dockerclient.Client, config *cont
func removeContainer(containername string) error {
ctx := context.Background()
cli, err := dockerclient.NewEnvClient()
// cli, err := dockerclient.NewEnvClient()
cli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[DEBUG] Unable to create docker client: %s", err)
return err
@@ -1341,7 +1345,8 @@ func getWorkerURLs() ([]string, error) {
}
// Create a new Docker client
cli, err := dockerclient.NewEnvClient()
// cli, err := dockerclient.NewEnvClient()
cli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Println("[ERROR] Failed to create Docker client:", err)
return workerUrls, err
@@ -1458,7 +1463,8 @@ func handleExecutionResult(workflowExecution shuffle.WorkflowExecution) {
var err error
if isKubernetes != "true" {
dockercli, err = dockerclient.NewEnvClient()
// dockercli, err = dockerclient.NewEnvClient()
dockercli, _, err = shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (3): %s", err)
return
@@ -3578,7 +3584,8 @@ func findAppInfo(image, name string, redeploy bool) (int, error) {
//Filters:
if exposedPort == -1 || redeploy {
dockercli, err := dockerclient.NewEnvClient()
// dockercli, err := dockerclient.NewEnvClient()
dockercli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (2): %s", err)
return -1, err
@@ -3690,7 +3697,8 @@ func findAppInfo(image, name string, redeploy bool) (int, error) {
if exposedPort >= 0 {
//log.Printf("[INFO] Found service %s on port %d - no need to deploy another", name, exposedPort)
} else {
dockercli, err := dockerclient.NewEnvClient()
// dockercli, err := dockerclient.NewEnvClient()
dockercli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (2): %s", err)
return -1, err
@@ -3768,7 +3776,8 @@ func findAppInfoKubernetes(image, name string, env []string) error {
// Backups in case networks are removed
func initSwarmNetwork() error {
ctx := context.Background()
dockercli, err := dockerclient.NewEnvClient()
// dockercli, err := dockerclient.NewEnvClient()
dockercli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (2): %s", err)
return err
@@ -4074,7 +4083,8 @@ func baseDeploy() {
//var err error
if isKubernetes != "true" {
cli, err := dockerclient.NewEnvClient()
// cli, err := dockerclient.NewEnvClient()
cli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (3): %s", err)
return
@@ -4815,7 +4825,8 @@ func handleDownloadImage(resp http.ResponseWriter, request *http.Request) {
return
}
client, err := dockerclient.NewEnvClient()
// client, err := dockerclient.NewEnvClient()
client, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (4): %s", err)
resp.WriteHeader(401)
@@ -4979,7 +4990,13 @@ func AutoScaleApps(ctx context.Context, client *dockerclient.Client, maxExecutio
}
func scaleApps(ctx context.Context, client *dockerclient.Client, replicas uint64) error {
client, err := dockerclient.NewEnvClient()
// client, err := dockerclient.NewEnvClient()
client, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (scaleApps): %s", err)
return err
}
services, err := client.ServiceList(ctx, types.ServiceListOptions{})
if err != nil {
log.Printf("[ERROR] Failed to find services in the swarm: %s", err)
@@ -5057,7 +5074,8 @@ func numberOfApps(ctx context.Context, dockercli *dockerclient.Client) int {
}
if dockercli == nil {
dockercli, err = dockerclient.NewEnvClient()
// dockercli, err = dockerclient.NewEnvClient()
dockercli, _, err = shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (5): %s", err)
return 0
@@ -5144,7 +5162,13 @@ func IsServiceRunning(ctx context.Context, cli *dockerclient.Client) bool {
}
func numberOfWorkers(ctx context.Context, cli *dockerclient.Client) int {
cli, err := dockerclient.NewEnvClient()
// cli, err := dockerclient.NewEnvClient()
cli, _, err := shuffle.GetDockerClient()
if err != nil {
log.Printf("[ERROR] Unable to create docker client (5): %s", err)
return 0
}
service, _, err := cli.ServiceInspectWithRaw(ctx, "shuffle-workers", types.ServiceInspectOptions{})
if err != nil {
return 0