Merge branch '2.0.0' of https://github.com/shuffle/shuffle into 2.0.0

.github/workflows/quick-testing.yml

@@ -0,0 +1,135 @@
+name: Docker Compose Up and Ping
+
+on:
+  workflow_run:
+    workflows: ["dockerbuild"]
+    types:
+      - completed
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest]
+        architecture: [x64, arm64]
+
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    
+    - name: Set up opensearch directory
+      run: mkdir shuffle-database && chmod -R 777 shuffle-database
+
+    - name: Build the stack
+      run: docker-compose up -d
+
+    - name: Wait for 30 seconds
+      run: sleep 30
+
+    - name: Check for restarting containers in a loop and fixing perms again 
+      run: |
+        # echo "Changing permissions on shuffle-database directory again"
+        # chmod -R 777 shuffle-database
+
+        ATTEMPTS=30  # Total time = ATTEMPTS * 5 seconds = 30 seconds
+        for i in $(seq 1 $ATTEMPTS); do
+          RESTARTING_CONTAINERS=$(docker ps --filter "status=restarting" --format "{{.Names}}")
+          if [ -n "$RESTARTING_CONTAINERS" ]; then
+            echo "The following containers are restarting:"
+            echo "$RESTARTING_CONTAINERS"
+            exit 1
+          fi
+          echo "No containers are restarting. Attempt $i/$ATTEMPTS."
+          sleep 1
+        done
+        echo "No containers were found in a restarting state after $ATTEMPTS checks."
+
+    - name: Check if the response from the frontend contains the word "Shuffle"
+      run: |
+        RESPONSE=$(curl -s http://localhost:3001)
+        if echo "$RESPONSE" | grep -q "Shuffle"; then
+          echo "The word 'Shuffle' was found in the response."
+        else
+          echo "The word 'Shuffle' was not found in the response."
+          exit 1
+        fi
+
+    - name: Register a user and check the status code
+      run: |
+        MAX_RETRIES=30
+        RETRY_INTERVAL=10
+        CONTAINER_NAME="shuffle-backend"
+
+        for (( i=1; i<=$MAX_RETRIES; i++ ))
+        do
+          STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" 'http://localhost:3001/api/v1/register' \
+            -H 'Accept: */*' \
+            -H 'Accept-Language: en-US,en;q=0.9' \
+            -H 'Connection: keep-alive' \
+            -H 'Content-Type: application/json' \
+            --data-raw '{"username":"demo@demo.io","password":"supercoolpassword"}')
+
+          if [ "$STATUS_CODE" -eq 200 ]; then
+            echo "User registration was successful with status code 200."
+            exit 0
+          elif [ "$STATUS_CODE" -ne 502 ]; then
+            echo "User registration failed with status code $STATUS_CODE."
+            exit 1
+          fi
+
+          echo "Received status code $STATUS_CODE. Retrying in $RETRY_INTERVAL seconds... ($i/$MAX_RETRIES)"
+          echo "Fetching last 30 lines of logs from container $CONTAINER_NAME..."
+
+          logs_output=$(docker logs --tail 30 "$CONTAINER_NAME")
+          echo "$logs_output"
+
+          echo "Fetching last 30 lines of logs from container shuffle-opensearch..."
+
+          opensearch_logs=$(docker logs --tail 30 shuffle-opensearch)
+          echo "$opensearch_logs"
+
+          sleep $RETRY_INTERVAL
+        done
+
+        echo "User registration failed after $MAX_RETRIES attempts."
+        exit 1
+
+    - name: Get the API key and run a health check
+      run: |
+        RESPONSE=$(curl -s -k -u admin:StrongShufflePassword321! 'https://localhost:9200/users/_search')
+        API_KEY=$(echo "$RESPONSE" | jq -r '.hits.hits[0]._source.apikey')
+        if [ -n "$API_KEY" ]; then
+          echo "Admin API key: $API_KEY"
+        else
+          echo "Failed to retrieve the API key for the admin user."
+          exit 1
+        fi
+
+        HEALTH_RESPONSE=$(curl -s 'http://localhost:3001/api/v1/health?force=true' \
+          -H 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7' \
+          -H 'Accept-Language: en-US,en;q=0.9' \
+          -H 'Connection: keep-alive' \
+          -H 'Sec-Fetch-Dest: document' \
+          -H 'Sec-Fetch-Mode: navigate' \
+          -H 'Sec-Fetch-Site: none' \
+          -H "Authorization: Bearer $API_KEY" \
+          -H 'Sec-Fetch-User: ?1' \
+          -H 'Upgrade-Insecure-Requests: 1' \
+          -H 'User-Agent: Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' \
+          -H 'sec-ch-ua: "Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"' \
+          -H 'sec-ch-ua-mobile: ?0' \
+          -H 'sec-ch-ua-platform: "macOS"')
+
+        WORKFLOWS_CREATE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.create')
+        WORKFLOWS_RUN=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run')
+        WORKFLOWS_RUN_FINISHED=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_finished')
+        WORKFLOWS_RUN_STATUS=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.run_status')
+        WORKFLOWS_DELETE=$(echo "$HEALTH_RESPONSE" | jq -r '.workflows.delete')
+
+        if [ "$WORKFLOWS_CREATE" = "true" ] && [ "$WORKFLOWS_RUN" = "true" ] && [ "$WORKFLOWS_RUN_FINISHED" = "true" ] && [ "$WORKFLOWS_RUN_STATUS" = "FINISHED" ] && [ "$WORKFLOWS_DELETE" = "true" ]; then
+          echo "Health endpoint check was successful."
+        else
+          echo "Health endpoint check failed. Response did not meet expected criteria."
+          exit 1
+        fi

backend/go-app/go.mod

@@ -20,11 +20,10 @@ require (
 	github.com/gorilla/mux v1.8.1
 	github.com/h2non/filetype v1.1.3
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.6.27
+	github.com/shuffle/shuffle-shared v0.6.31
 	golang.org/x/crypto v0.22.0
 	google.golang.org/api v0.176.1
 	google.golang.org/grpc v1.63.2
-
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/api v0.30.0
@@ -42,12 +41,10 @@ require (
 	github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
-
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
 	github.com/adrg/strutil v0.2.3 // indirect
 	github.com/algolia/algoliasearch-client-go/v3 v3.18.1 // indirect
 	github.com/bitly/go-simplejson v0.5.1 // indirect
-
 	github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 // indirect
 	github.com/bradfitz/slice v0.0.0-20180809154707-2b758aa73013 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
@@ -60,10 +57,8 @@ require (
 	github.com/docker/go-units v0.5.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
-
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/frikky/schemaless v0.0.11 // indirect
-
 	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
@@ -77,12 +72,10 @@ require (
 	github.com/google/go-github/v28 v28.1.1 // indirect
 	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
-
 	github.com/google/s2a-go v0.1.7 // indirect
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.3 // indirect
-
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -106,11 +99,9 @@ require (
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/sashabaranov/go-openai v1.19.2 // indirect
-
 	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
 	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skeema/knownhosts v1.2.2 // indirect
-
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e // indirect
 	github.com/src-d/gcfg v1.4.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
@@ -128,7 +119,6 @@ require (
 	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/term v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
-
 	golang.org/x/time v0.5.0 // indirect
 	golang.org/x/tools v0.18.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
@@ -136,7 +126,6 @@ require (
 	google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be // indirect
 	google.golang.org/protobuf v1.33.0 // indirect
-
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

backend/go-app/go.sum

@@ -441,6 +441,8 @@ github.com/shuffle/shuffle-shared v0.6.26 h1:UZ4o3s+GPULLv/gy31SdhUw1ttvF+f1FfP1
 github.com/shuffle/shuffle-shared v0.6.26/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4=
 github.com/shuffle/shuffle-shared v0.6.27 h1:q4qZD6bGZFIvZ5Y10unGr3N3rZ7OryWyvvaGgANZJZU=
 github.com/shuffle/shuffle-shared v0.6.27/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4=
+github.com/shuffle/shuffle-shared v0.6.31 h1:MK1SW1pwjIP7hznq+mMlTPM1R3LIOfi1/bUL5xFSg/8=
+github.com/shuffle/shuffle-shared v0.6.31/go.mod h1:rWkh1eWdIx7OqQzJ1+JzF3Hck1X/Ty1WkUtjLrp+CU4=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=

backend/go-app/main.go

@@ -759,6 +759,8 @@ func handleInfo(resp http.ResponseWriter, request *http.Request) {
 		}
 	}
 
+	go shuffle.CheckSessionOrgs(ctx, userInfo)
+
 	//log.Printf("%s  %s", session.Session, UserInfo.Session)
 	//if session.Session != userInfo.Session {
 	//	log.Printf("Session %s is not the same as %s for %s. %s", userInfo.Session, session.Session, userInfo.Username, err)
@@ -4887,7 +4889,7 @@ func initHandlers() {
 	}
 
 	for {
-		_, err = shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), gceProject, "onprem", true, elasticConfig)
+		_, err = shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), gceProject, "onprem", true, elasticConfig, false, 0)
 		if err != nil {
 			log.Printf("[ERROR] Error in initial database connection. Retrying in 5 seconds. %s", err)
 			time.Sleep(5 * time.Second)
@@ -4926,6 +4928,7 @@ func initHandlers() {
 	r.HandleFunc("/api/v1/users/getsettings", shuffle.HandleSettings).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/users/getusers", shuffle.HandleGetUsers).Methods("GET", "OPTIONS")
 	r.HandleFunc("/api/v1/users/updateuser", shuffle.HandleUpdateUser).Methods("PUT", "OPTIONS")
+	r.HandleFunc("/api/v1/users/{userID}/remove", shuffle.HandleDeleteUsersAccount).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/users/{user}", shuffle.DeleteUser).Methods("DELETE", "OPTIONS")
 	r.HandleFunc("/api/v1/users/passwordchange", shuffle.HandlePasswordChange).Methods("POST", "OPTIONS")
 	r.HandleFunc("/api/v1/users/{key}/get2fa", shuffle.HandleGet2fa).Methods("GET", "OPTIONS")

backend/go-app/main_test.go

@@ -34,7 +34,7 @@ func init() {
 		log.Fatalf("[DEBUG] Database client error during init: %s", err)
 	}
 
-	_, err = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true, "elasticsearch")
+	_, err = shuffle.RunInit(*dbclient, storage.Client{}, gceProject, "onprem", true, "elasticsearch", false, 0)
 	log.Printf("INIT")
 }
 

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3'
 services:
   frontend:
-    image: ghcr.io/shuffle/shuffle-frontend:nightly
+    image: ghcr.io/shuffle/shuffle-frontend:latest
     container_name: shuffle-frontend
     hostname: shuffle-frontend
     ports:
@@ -15,7 +15,7 @@ services:
     depends_on:
       - backend
   backend:
-    image: ghcr.io/shuffle/shuffle-backend:nightly
+    image: ghcr.io/shuffle/shuffle-backend:latest
     container_name: shuffle-backend
     hostname: ${BACKEND_HOSTNAME}
     # Here for debugging:
@@ -34,7 +34,7 @@ services:
       - SHUFFLE_FILE_LOCATION=/shuffle-files
     restart: unless-stopped
   orborus:
-    image: ghcr.io/shuffle/shuffle-orborus:nightly
+    image: ghcr.io/shuffle/shuffle-orborus:latest
     container_name: shuffle-orborus
     hostname: shuffle-orborus
     networks:

frontend/src/App.jsx

@@ -9,6 +9,8 @@ import GettingStarted from "./views/GettingStarted";
 import AngularWorkflow from "./views/AngularWorkflow.jsx";
 
 import Header from "./components/NewHeader.jsx";
+import HealthPage from "./components/HealthPage.jsx";
+
 //import Header from "./components/Header.jsx";
 import theme from "./theme";
 import Apps from "./views/Apps";
@@ -272,6 +274,38 @@ const App = (message, props) => {
         	    />
         	  }
         	/>
+			<Route
+				exact
+				path="/health"
+				element={
+					<HealthPage
+						cookies={cookies}
+						removeCookie={removeCookie}
+						isLoaded={isLoaded}
+						isLoggedIn={isLoggedIn}
+						globalUrl={globalUrl}
+						cookies={cookies}
+						userdata={userdata}
+						{...props}
+					/>
+				}
+			/>
+			<Route
+				exact
+				path="/status"
+				element={
+					<HealthPage
+						cookies={cookies}
+						removeCookie={removeCookie}
+						isLoaded={isLoaded}
+						isLoggedIn={isLoggedIn}
+						globalUrl={globalUrl}
+						cookies={cookies}
+						userdata={userdata}
+						{...props}
+					/>
+				}
+			/>
         	{userdata.id !== undefined ? (
         	  <Route
         	    exact

frontend/src/components/HealthBarChart.jsx

@@ -0,0 +1,22 @@
+import React from 'react';
+import { Bar } from 'react-chartjs-2';
+
+const HealthBarChart = (props) => {
+    const { globalUrl, filteredData, options, onBarClick  } = props;
+
+    return (
+        <Bar
+            data={filteredData}
+            options={options}
+            height="35.5rem"
+            width={filteredData.width}
+            getElementAtEvent={(elements) => {
+                if (elements && elements.length > 0) {
+                    onBarClick(elements);
+                }
+            }}
+        />
+    );
+};
+
+export default HealthBarChart;

frontend/src/components/HealthPage.jsx

@@ -0,0 +1,317 @@
+import React, { useEffect, useState, useCallback } from 'react';
+import { Bar } from 'react-chartjs-2';
+//import healthData1 from '../healthstats1.json';
+import { toast } from "react-toastify"
+import CheckOutlinedIcon from '@mui/icons-material/CheckOutlined';
+import {
+    Button,
+    ButtonGroup,
+    Typography,
+} from "@mui/material";
+import HealthBarChart from '../components/HealthBarChart.jsx';
+
+const HealthPage = (props) => {
+    const { userdata } = props;
+    const [healthData, setHealthData] = useState(null);
+    const [selectedRange, setSelectedRange] = useState('30d');
+    const [filteredData, setFilteredData] = useState([]);
+    const [averageUptime, setAverageUptime] = useState(0);
+
+    const isCloud = window.location.host === "localhost:3002" || window.location.host === "shuffler.io";
+    const globalUrl = `https://shuffler.io` 
+
+	console.log("HEALTHPAGE 1")
+
+    const fetchHealthStats = useCallback(async () => {
+		//const after = new Date().getTime() - 90 * 24 * 60 * 60 
+        try {
+            //const response = await fetch(`${globalUrl}/api/v1/health/stats?after=${after}`, {
+            const response = await fetch(`${globalUrl}/api/v1/health/stats`, {
+                method: "GET",
+                headers: {
+                    "Content-Type": "application/json",
+                    Accept: "application/json",
+                },
+                credentials: "include",
+            });
+            if (!response.ok) {
+                throw new Error("Failed to fetch health stats");
+            }
+            const data = await response.json();
+            setHealthData(data);
+        } catch (error) {
+            console.error("Error fetching health stats:", error);
+            toast.error("Failed loading health stats");
+        }
+    }, [globalUrl]);
+
+    useEffect(() => {
+        fetchHealthStats();
+    }, [fetchHealthStats]);
+
+    const extractRunFinished = (data, range) => {
+        if (!data || !Array.isArray(data)) return [];
+    
+        const currentDate = new Date().getTime();
+        const rangeInMillis = {
+            '24hr': 24 * 60 * 60 * 1000,
+            '7day': 7 * 24 * 60 * 60 * 1000,
+            '30d': 30 * 24 * 60 * 60 * 1000,
+            '90d': 90 * 24 * 60 * 60 * 1000
+        }
+        const filteredData = data.filter(item => currentDate - item.updated * 1000 <= rangeInMillis[range])
+    
+        const aggregatedData = new Map()
+    
+        filteredData.forEach(item => {
+            const timestamp = item.updated * 1000; // Convert Unix timestamp to milliseconds
+            let key;
+    
+            switch (range) {
+                case '24hr':
+                    const date = new Date(timestamp);
+                    const hour = date.getHours();
+                    const formattedDate = `${date.toLocaleDateString()} ${hour}:00`;
+                    key = formattedDate;
+                    break;
+                case '7day':
+                    const date1 = new Date(timestamp);
+                    const hour1 = date1.getHours();
+                    const formattedDate1 = `${date1.toLocaleDateString()} ${hour1}:00`;
+                    key = formattedDate1;
+                    break;
+                default:
+                    key = new Date(timestamp).toLocaleDateString();
+            }
+    
+            // Check if date already exists in the map
+            if (aggregatedData.has(key)) {
+                // Update aggregated values
+                const existingData = aggregatedData.get(key);
+                existingData.totalEntries++;
+                existingData.totalRunFinished += item.workflows.run_finished ? 1 : 0;
+                existingData.executionIds.push(item.workflows.execution_id);
+            } else {
+                // Add new entry to the map
+                aggregatedData.set(key, {
+                    totalEntries: 1,
+                    totalRunFinished: item.workflows.run_finished ? 1 : 0,
+                    executionIds: [item.workflows.execution_id]
+                });
+            }
+        });
+    
+        // Calculate averages and assign colors
+        const result = Array.from(aggregatedData.entries()).map(([key, { totalEntries, totalRunFinished, executionIds }]) => {
+            const avg = totalEntries > 0 ? totalRunFinished / totalEntries : 0;
+            const FinalAvg = avg * 100;
+            let color;
+    
+            if (FinalAvg >= 100) {
+                color = '#00F670';
+            } else if (FinalAvg >= 98.50 && FinalAvg <= 99.99) {
+                color = '#FFD700';
+            } else if (FinalAvg <= 98.49) {
+                color = '#FF354C';
+            }
+    
+            return {
+                date: range === '24hr' ? `${key}:00` : key,
+                avgRunFinished: FinalAvg,
+                color,
+                executionIds
+            };
+        });
+    
+        return result;
+    };
+    
+    
+    useEffect(() => {
+        if (healthData) {
+            const newData = extractRunFinished(healthData, selectedRange);
+            setFilteredData(newData);
+
+            const totalUptime = newData.reduce((acc, curr) => acc + curr.avgRunFinished, 0);
+            const avgUptime = totalUptime / newData.length;
+            setAverageUptime(avgUptime);
+        }
+    }, [selectedRange, healthData]);
+
+    const filterDataByRange = (range) => {
+        setSelectedRange(range);
+    };
+
+    const updateChartData = () => {
+        if (!filteredData) {
+            return {
+                labels: [],
+                datasets: [{
+                    label: "",
+                    data: [],
+                    backgroundColor: [],
+                    borderWidth: 1,
+                    barThickness: 7, // Default bar thickness
+                }],
+            };
+        }
+        let barThickness = 7;
+
+        const labels = filteredData.map((value, i) => {
+            if (selectedRange === '24hr') {
+                const [datePart, hourPart] = value.date.split(' ');
+                const [day, month, year] = datePart.split('/');
+                const monthIndex = parseInt(month, 10) - 1;
+                const date = new Date(year, monthIndex, day);
+                let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`;
+
+                // Add hour part if available
+                if (hourPart) {
+                    formattedDate += `, ${hourPart.split(':').slice(0, 2).join(':')}`;
+                }
+                return `${formattedDate} \nUptime: ${value.avgRunFinished.toFixed(2)}%`;
+            } else if (selectedRange === '7day') {
+                const [datePart, hourPart] = value.date.split(' ');
+                const [day, month, year] = datePart.split('/');
+                const monthIndex = parseInt(month, 10) - 1;
+                const date = new Date(year, monthIndex, day);
+                let formattedDate = `${date.toLocaleString('en-US', { month: 'short', day: '2-digit' })}`;
+
+                // Add hour part if available
+                if (hourPart) {
+                    formattedDate += `, ${hourPart.split(':').slice(0, 2).join(':')}`;
+                }
+                return `${formattedDate} \nUptime: ${value.avgRunFinished.toFixed(2)}%`;
+            }
+            else {
+                const dateParts = value.date.split('/'); // Assuming the date format is "DD/MM/YYYY"
+                const date = new Date(`${dateParts[2]}-${dateParts[1]}-${dateParts[0]}`); // Reformat the date string to "YYYY-MM-DD"
+                return `${date.toLocaleDateString('en-US', { month: 'short', day: '2-digit', year: 'numeric' })} \nUptime: ${value.avgRunFinished.toFixed(2)}%`;
+            }
+        });
+
+
+        if (selectedRange === '24hr') {
+            barThickness = 35;
+        }
+        else if (selectedRange === '7day') {
+            barThickness = 5;
+        }
+        else if (selectedRange === '30d') {
+            barThickness = 25;
+        }
+        // let width = 800; // Default chart width
+        // if (selectedRange === '7day') {
+        //     width = 400; // Adjust chart width for 7 days
+        // } else if (selectedRange === '30d') {
+        //     width = 600; // Adjust chart width for 30 days
+        // }
+
+        const datasets = [{
+            label: "",
+            data: filteredData.map(item => 1),
+            backgroundColor: filteredData.map(item => item.color),
+            borderWidth: 1,
+            barThickness: barThickness,
+            // barPercentage: barPercentage,
+        }];
+
+        return { labels, datasets };
+    };
+
+	console.log("HEALTHPAGE 2")
+
+    const options = {
+        legend: {
+            display: false
+        },
+        layout: {
+            padding: {
+                top: 0, // Adjust the top padding as needed
+                bottom: 20, // Adjust the bottom padding as needed
+                left: 20, // Adjust the left padding as needed
+                right: 20 // Adjust the right padding as needed
+            }
+        },
+        scales: {
+            yAxes: [{
+                ticks: {
+                    display: false
+                }
+            }],
+            xAxes: [{
+                ticks: {
+                    display: false
+                }
+            }]
+        },
+        tooltips: {
+            callbacks: {
+                label: function (tooltipItem, data) {
+                    const label = data.labels[tooltipItem.index];
+                    return label.split('\n')[0]; // Return only the date part
+                },
+                afterLabel: function (tooltipItem, data) {
+                    const label = data.labels[tooltipItem.index];
+                    // console.log(label)
+                    const uptime = label.match(/Uptime:\s*(\d+(?:\.\d+)?)/)[1]; // Extract uptime value using regex
+                    return `Success Rate: ${uptime}%`; // Customize the uptime display
+                },
+                title: function () {
+                    return 'Fully Oprational'; // Hide the tooltip title
+                }
+            }
+        }
+    };
+
+    const handleBarClick = (event, elements) => {
+        if (event && event.length > 0) {
+            const clickedIndex = event[0]._index
+            const clickedData = filteredData[clickedIndex]
+            const executionIds = clickedData.executionIds
+                .filter(executionId => {
+                    const item = healthData.find(dataItem => dataItem.workflows.execution_id === executionId);
+                    return item && item.workflows.run_finished === false;
+                });
+
+            // console.log("Filtered Execution IDs:", executionIds);
+            if (executionIds.length > 0) {
+                const url = `${globalUrl}/api/v1/health/stats?execution_id=${executionIds.join(',')}`;
+                window.open(url, '_blank');
+            } else {
+                toast.success("All executions in selected period succeeded");
+            }
+        }
+    };
+
+
+    return (
+        <div style={{ padding: 30, width: '100%', display: 'flex', flexDirection: 'column', alignItems: 'center', justifyContent: 'center' }}>
+
+            <ButtonGroup style={{ display: 'flex', margin: "auto", marginBottom: 10, width: 300, borderRadius: 30, background: "#000000" }}>
+                <Button variant="contained" style={{ flex: 1, borderBottom: selectedRange === '24hr' ? '2px solid #FF8444' : 'none', background: "#000000", color: selectedRange === '24hr' ? '#FF8444' : '#cfd8dc', textTransform: 'none' }} onClick={() => filterDataByRange('24hr')}>24h</Button>
+                <Button variant="contained" style={{ flex: 1, borderBottom: selectedRange === '7day' ? '2px solid #FF8444' : 'none', background: "#000000", color: selectedRange === '7day' ? '#FF8444' : '#cfd8dc', textTransform: 'none' }} onClick={() => filterDataByRange('7day')}>7d</Button>
+                <Button variant="contained" style={{ flex: 1, borderBottom: selectedRange === '30d' ? '2px solid #FF8444' : 'none', background: "#000000", color: selectedRange === '30d' ? '#FF8444' : '#cfd8dc', textTransform: 'none' }} onClick={() => filterDataByRange('30d')}>30d</Button>
+                <Button disabled variant="contained" style={{ flex: 1, borderBottom: selectedRange === '90d' ? '2px solid #FF8444' : 'none', background: "#000000", color: false === false ? "grey" : selectedRange === '90d' ? '#FF8444' : '#cfd8dc', textTransform: 'none' }} onClick={() => filterDataByRange('90d')}>90d</Button>
+                <Button disabled variant="contained" style={{ flex: 1, borderBottom: selectedRange === '180d' ? '2px solid #FF8444' : 'none', background: "#000000", color: false === false ? "grey" : selectedRange === '180d' ? '#FF8444' : '#cfd8dc', textTransform: 'none' }} onClick={() => filterDataByRange('180d')}>180d</Button>
+            </ButtonGroup>
+
+            <div style={{ margin: '0 auto', padding: 20, width: 1000, justifyContent: "center", color: '#ffffff', backgroundColor: '#000000', fontSize: '16px', borderRadius: '16px' }}>
+                <div style={{ display: "flex", alignItems: "center" }}>
+                    <CheckOutlinedIcon style={{ borderRadius: 20, fontSize: 24, backgroundColor: '#00e600', marginLeft: 25 }} />
+                    <div>
+                        <Typography style={{ marginLeft: 10 }}>Workflow Health</Typography>
+                        <Typography style={{ marginLeft: 10, fontWeight: 100, fontSize: 13, color: "#00FF00" }}>Operational</Typography>
+                    </div>
+                    <div style={{ marginLeft: 720 }}>
+                        <Typography style={{}}>{averageUptime.toFixed(2)}%</Typography>
+                        <Typography style={{ fontWeight: 100, fontSize: 13, textAlign: "end" }}>Success Rate</Typography>
+                    </div>
+                </div>
+                <HealthBarChart filteredData={updateChartData()} options={options} onBarClick={handleBarClick} />
+            </div>
+        </div>
+    );
+};
+
+export default HealthPage;

frontend/src/components/OrgHeaderexpanded.jsx

@@ -299,6 +299,7 @@ const OrgHeaderexpanded = (props) => {
 								client_secret: openidClientSecret,
 								openid_authorization: openidAuthorization,
 								openid_token: openidToken,
+								SSORequired: SSORequired
 							}
 						)
 					}
@@ -310,7 +311,16 @@ const OrgHeaderexpanded = (props) => {
 	);
 
 	const toggleBetweenRequiredOrOptional = (event) => {
-		setSSORequired(event.target.checked);
+		if (ssoEntrypoint === "" && openidAuthorization === "" && openidToken === "") {
+			if (!SSORequired) {
+				toast.error("Please fill in fields for either OpenID connect or SSO before continuing. ")
+				return
+			}
+		} else {
+			toast.info("Toggled SSO. Remember to save.")
+		}
+
+		setSSORequired(event.target.checked)
 	};
 
 	return (
@@ -503,23 +513,26 @@ const OrgHeaderexpanded = (props) => {
 					</span>
 				</Grid>
 				{/* {isCloud ? null : */}
+				<Typography variant="h4" style={{ textAlign: "left", marginTop: 75, marginLeft: 20, }}>Single Signon - SAML/OpenID</Typography>
 				<div style={{ display: 'flex', flexDirection: 'column', marginTop: 20, marginLeft: 10, width: '100%', borderBottom: '1px solid #414347' }}>
+					<Typography variant="body1" style={{ margin: '5px 0px 5px 10px' }}>Make SAML SSO or OpenID Authentication required/optional for your organization. Tip: Do not make it required until you have tested the URL directly.</Typography>
 					<div>
 						<Switch
 							checked={SSORequired}
-							onChange={toggleBetweenRequiredOrOptional}
+							onChange={(e) => {
+								toggleBetweenRequiredOrOptional(e)
+							}}
 							name="onOffSwitch"
 							color="primary"
 							title="Make SAML SSO or OpenID Authentication Required or Optional for Your Organization"
 						/>
 						{SSORequired ? 'Required' : 'Optional'}
 					</div>
-					<Typography variant="body2" style={{ margin: '5px 0px 5px 10px' }}>Make SAML SSO or OpenID Authentication Required or Optional for Your Organization.</Typography>
 				</div>
 				<div>
 				</div>
 				<Grid item xs={12} style={{}}>
-					<Typography variant="h4" style={{ textAlign: "center", }}>OpenID connect</Typography>
+					<Typography variant="h6" style={{ textAlign: "left", }}>OpenID connect</Typography>
 					<Grid container style={{ marginTop: 10, }}>
 						<Grid item xs={6} style={{}}>
 							<span>
@@ -536,11 +549,6 @@ const OrgHeaderexpanded = (props) => {
 									type="name"
 									multiline={true}
 									rows={2}
-									disabled={
-										selectedOrganization.manager_orgs !== undefined &&
-										selectedOrganization.manager_orgs !== null &&
-										selectedOrganization.manager_orgs.length > 0
-									}
 									id="outlined-with-placeholder"
 									margin="normal"
 									variant="outlined"
@@ -575,11 +583,6 @@ const OrgHeaderexpanded = (props) => {
 									type="name"
 									multiline={true}
 									rows={2}
-									disabled={
-										selectedOrganization.manager_orgs !== undefined &&
-										selectedOrganization.manager_orgs !== null &&
-										selectedOrganization.manager_orgs.length > 0
-									}
 									id="outlined-with-placeholder"
 									margin="normal"
 									variant="outlined"
@@ -691,11 +694,6 @@ const OrgHeaderexpanded = (props) => {
 									type="name"
 									multiline={true}
 									rows={2}
-									disabled={
-										selectedOrganization.manager_orgs !== undefined &&
-										selectedOrganization.manager_orgs !== null &&
-										selectedOrganization.manager_orgs.length > 0
-									}
 									id="outlined-with-placeholder"
 									margin="normal"
 									variant="outlined"

frontend/src/views/Admin.jsx

@@ -805,7 +805,7 @@ If you're interested, please let me know a time that works for you, or set up a
       .then((responseJson) => {
         setWebHooks(responseJson.webhooks || []); // Handling the case where the result is null or undefined
         setAllSchedules(responseJson.schedules || []);
-        // setPipelines(responseJson.pipelines || []);
+        setPipelines(responseJson.pipelines || []);
       })
       .catch((error) => {
         // toast(error.toString());
@@ -990,11 +990,13 @@ If you're interested, please let me know a time that works for you, or set up a
     }
   
     const data = {
+      command: pipeline.command,
       name: pipeline.name,
       type: state,
       environment: pipeline.environment,
       workflow_id: pipeline.workflow_id,
       trigger_id: pipeline.trigger_id,
+      start_node: pipeline.start_node,
     };
   
     if (state === "start") toast("starting the pipeline");
@@ -1025,6 +1027,7 @@ If you're interested, please let me know a time that works for you, or set up a
           if (state === "start") toast("Successfully created pipeline");
           else toast("Sucessfully stopped the pipeline");
         }
+        setTimeout(handleGetAllTriggers, 1000);
       })
       .catch((error) => {
         //toast(error.toString());
@@ -4938,7 +4941,7 @@ If you're interested, please let me know a time that works for you, or set up a
   
                   <ListItemText>
                     <Button
-                      style={{ marginLeft: "18%" }}
+                      style={{ marginLeft: "140px" }}
                       variant={
                         webhook.status === "running" ? "contained" : "outlined"
                       }
@@ -4960,10 +4963,10 @@ If you're interested, please let me know a time that works for you, or set up a
           </List>
         )}
   
-        {/* <div style={{ marginTop: 20, marginBottom: 20 }}>
+        <div style={{ marginTop: 20, marginBottom: 20 }}>
           <h2 style={{ display: "inline" }}>Tenzir Pipelines</h2>
           <span style={{ marginLeft: 25 }}>
-            Controls a pipeline to run things.{" "}
+             Controls the Tenzir pipeline operations.{" "}
             <a
               target="_blank"
               rel="noopener noreferrer"
@@ -5010,7 +5013,7 @@ If you're interested, please let me know a time that works for you, or set up a
                 primary="Workflow"
                 style={{ maxWidth: 315, minWidth: 315 }}
               />
-              <ListItemText primary="Actions" />
+            <ListItemText primary="Actions" style={{ marginLeft: '120px' }} />
             </ListItem>
             {pipelines.map((pipeline, index) => {
               var bgColor = "#27292d";
@@ -5063,7 +5066,7 @@ If you're interested, please let me know a time that works for you, or set up a
               );
             })}
           </List>
-        )}*/}
+        )}
       </div>
     ) : null;
 

functions/onprem/orborus/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/docker/docker v26.1.0+incompatible
 	github.com/docker/go-connections v0.5.0
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.6.27
+	github.com/shuffle/shuffle-shared v0.6.29
 	k8s.io/api v0.30.0
 	k8s.io/apimachinery v0.30.0
 	k8s.io/client-go v0.30.0

functions/onprem/orborus/orborus.go

@@ -2050,8 +2050,6 @@ func handlePipeline(incRequest shuffle.ExecutionRequest) error {
 		if err != nil {
 			log.Printf("[ERROR] Failed Deleting Pipeline %s", err)
 			return err
-		} else {
-			log.Printf("[INFO] successfully deleted the Pipeline: %s", pipelineId)
 		} 
 	} else if incRequest.Type == "PIPELINE_STOP" {
 		log.Printf("[INFO] Should stop the pipeline %#v", identifier)
@@ -2260,7 +2258,7 @@ func createPipeline(command, identifier string) (string, error) {
 
 	if err != nil {
 		if strings.Contains(fmt.Sprintf("%s", err), "no existing pipeline found") {
-			log.Printf("[INFO] No existing pipeline found with name: %s. Creating a new one!", identifier)
+			log.Printf("[INFO] No existing pipeline found with id: %s. Creating a new one!", identifier)
 		} else {
 			log.Printf("[ERROR] Failed to search for existing pipeline but continuing anyway : %s", err)
 		}
@@ -2286,7 +2284,6 @@ func createPipeline(command, identifier string) (string, error) {
 			command = command[:startIndex] + baseUrl + command[endIndex:]
 		}
 	}
-
 	requestBody := map[string]interface{}{
 		"definition": command,
 		"name":       identifier,

functions/onprem/worker/go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/docker/docker v26.1.0+incompatible
 	github.com/gorilla/mux v1.8.1
 	github.com/satori/go.uuid v1.2.0
-	github.com/shuffle/shuffle-shared v0.6.27
+	github.com/shuffle/shuffle-shared v0.6.30
 	k8s.io/api v0.30.0
 	k8s.io/apimachinery v0.30.0
 	k8s.io/client-go v0.30.0

functions/onprem/worker/worker.go

@@ -2981,7 +2981,7 @@ func getStreamResultsWrapper(client *http.Client, req *http.Request, workflowExe
 func main() {
 	// Elasticsearch necessary to ensure we'ren ot running with Datastore configurations for minimal/maximal data sizes
 	// Recursive import kind of :)
-	_, err := shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), "", "worker", true, "elasticsearch")
+	_, err := shuffle.RunInit(*shuffle.GetDatastore(), *shuffle.GetStorage(), "", "worker", true, "elasticsearch", false, 0)
 	if err != nil {
 		if !strings.Contains(fmt.Sprintf("%s", err), "no such host") {
 			log.Printf("[ERROR] Failed to run worker init: %s", err)

shuffle-apps/README.md

@@ -1,5 +1,5 @@
 # Shuffle Apps 
 * This folder is by default meant to be empty
 * This folder is meant for quick development of apps (single button hot-loading) 
-* Some shuffle apps can be found at https://github.com/frikky/shuffle-apps
+* Some shuffle apps can be found at https://github.com/shuffle/python-apps