Added backend statistics for dashboard

This commit is contained in:
frikky
2020-05-14 08:48:20 +02:00
parent e3dca4c17e
commit 7c33b91f1f
9 changed files with 297 additions and 53 deletions
+1
View File
@@ -19,6 +19,7 @@ Documentation can be found on https://shuffler.io/docs/about or in your own inst
## In the works
* User run statistics - Dashboard
* Debug view for manual executions
* App versioning
### Setup - Local
Frontend - requires [npm](https://nodejs.org/en/download/)/[yarn](https://yarnpkg.com/lang/en/docs/install/#debian-stable)/your preferred manager. Runs independently from backend - edit frontend/src/App.yaml to change from localhost to prod setting.
+99 -6
View File
@@ -79,6 +79,35 @@ type ExecutionInfo struct {
DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"`
}
type StatisticsData struct {
Timestamp int64 `json:"timestamp" datastore:"timestamp"`
Id string `json:"id" datastore:"id"`
Amount int64 `json:"amount" datastore:"amount"`
}
type StatisticsItem struct {
Total int64 `json:"total" datastore:"total"`
Fieldname string `json:"field_name" datastore:"field_name"`
Data []StatisticsData `json:"data" datastore:"data"`
}
// "Execution by status"
// Execution history
//type GlobalStatistics struct {
// BackendExecutions int64 `json:"backend_executions" datastore:"backend_executions"`
// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"`
// ExecutionCount int64 `json:"execution_count" datastore:"execution_count"`
// ExecutionSuccessCount int64 `json:"execution_success_count" datastore:"execution_success_count"`
// ExecutionAbortCount int64 `json:"execution_abort_count" datastore:"execution_abort_count"`
// ExecutionFailureCount int64 `json:"execution_failure_count" datastore:"execution_failure_count"`
// ExecutionPendingCount int64 `json:"execution_pending_count" datastore:"execution_pending_count"`
// AppUsageCount int64 `json:"app_usage_count" datastore:"app_usage_count"`
// TotalAppsCount int64 `json:"total_apps_count" datastore:"total_apps_count"`
// SelfMadeAppCount int64 `json:"self_made_app_count" datastore:"self_made_app_count"`
// WebhookUsageCount int64 `json:"webhook_usage_count" datastore:"webhook_usage_count"`
// Baseline map[string]int64 `json:"baseline" datastore:"baseline"`
//}
type ParsedOpenApi struct {
Body string `datastore:"body,noindex" json:"body"`
ID string `datastore:"id" json:"id"`
@@ -2783,6 +2812,11 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
workflowExecution, executionResp, err := handleExecution(item, workflow, request)
if err == nil {
err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization)))
return
@@ -4481,6 +4515,55 @@ func handleGetOutlookFolders(resp http.ResponseWriter, request *http.Request) {
resp.Write(b)
}
func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
_, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in getting specific workflow: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var statsId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
statsId = location[4]
}
ctx := context.Background()
statisticsId := "global_statistics"
nameKey := statsId
key := datastore.NameKey(statisticsId, nameKey, nil)
statisticsItem := StatisticsItem{}
if err := dbclient.Get(ctx, key, &statisticsItem); err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
b, err := json.Marshal(statisticsItem)
if err != nil {
log.Println("Failed to marshal data: %s", err)
resp.WriteHeader(401)
return
}
resp.WriteHeader(200)
resp.Write([]byte(b))
}
func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -5503,6 +5586,15 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
}
setOpenApiDatastore(ctx, api.ID, parsed)
err = increaseStatisticsField(ctx, "total_apps_created", api.ID, 1)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
err = increaseStatisticsField(ctx, "openapi_apps_created", api.ID, 1)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
@@ -5522,6 +5614,11 @@ func init() {
log.Fatalf("DBclient error during init: %s", err)
}
err = increaseStatisticsField(ctx, "backend_executions", "", 1)
if err != nil {
log.Printf("Failed increasing local stats: %s", err)
}
count, err := getEnvironmentCount()
if count == 0 && err == nil {
item := Environment{
@@ -5554,15 +5651,9 @@ func init() {
r.HandleFunc("/api/v1/getenvironments", handleGetEnvironments).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/setenvironments", handleSetEnvironments).Methods("PUT", "OPTIONS")
//r.HandleFunc("/api/v1/register/{key}", handleRegisterVerification).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/getinfo", handleInfo).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/getsettings", handleSettings).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/generateapikey", handleApiGeneration).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/passwordchange", handlePasswordChange).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/passwordresetmail", handlePasswordResetMail).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/passwordreset", handlePasswordReset).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/contact", handleContact).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/docs", getDocList).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS")
@@ -5614,6 +5705,8 @@ func init() {
// Trigger hmm
r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS")
// OpenAPI configuration
r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS")
+99
View File
@@ -274,6 +274,62 @@ type ExecutionRequestWrapper struct {
Data []ExecutionRequest `json:"data"`
}
// This might be... a bit off, but that's fine :)
// This might also be stupid, as we want timelines and such
// Anyway, these are super basic stupid stats.
func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64) error {
// 1. Get current stats
// 2. Increase field(s)
// 3. Put new stats
statisticsId := "global_statistics"
nameKey := fieldname
key := datastore.NameKey(statisticsId, nameKey, nil)
statisticsItem := StatisticsItem{}
newData := StatisticsData{
Timestamp: int64(time.Now().Unix()),
Amount: amount,
Id: id,
}
if err := dbclient.Get(ctx, key, &statisticsItem); err != nil {
// Should init
if strings.Contains(fmt.Sprintf("%s", err), "entity") {
statisticsItem = StatisticsItem{
Total: amount,
Fieldname: fieldname,
Data: []StatisticsData{
newData,
},
}
if _, err := dbclient.Put(ctx, key, &statisticsItem); err != nil {
log.Printf("Error setting base stats: %s", err)
return err
}
return nil
}
//log.Printf("STATSERR: %s", err)
return err
}
statisticsItem.Total += amount
statisticsItem.Data = append(statisticsItem.Data, newData)
// New struct, to not add body, author etc
if _, err := dbclient.Put(ctx, key, &statisticsItem); err != nil {
log.Printf("Error stats to %s: %s", fieldname, err)
return err
}
log.Printf("Stats: %#v", statisticsItem)
return nil
}
func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWrapper, id string) error {
key := datastore.NameKey("workflowqueue", id, nil)
@@ -607,6 +663,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
if workflowExecution.Status == "FINISHED" {
log.Printf("Workflowexecution is already FINISHED. No further action can be taken")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status)))
return
@@ -615,6 +672,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
// Not sure what's up here
// FIXME - remove comment
if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" {
log.Printf("Workflowexecution is already aborted. No further action can be taken")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status)))
@@ -645,6 +703,18 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
workflowExecution.Result = lastResult
workflowExecution.Results = newResults
if workflowExecution.Status == "ABORTED" {
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase aborted execution stats: %s", err)
}
} else if workflowExecution.Status == "FAILURE" {
err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase failure execution stats: %s", err)
}
}
}
// This means it should continue I think :)
@@ -722,6 +792,11 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
if workflowExecution.LastNode == "" {
workflowExecution.LastNode = actionResult.Action.ID
}
err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
}
}
@@ -1491,6 +1566,11 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) {
return
}
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase aborted execution stats: %s", err)
}
// FIXME - allowed to edit it? idk
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
@@ -1834,6 +1914,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
}
}
err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase stats execution stats: %s", err)
}
return workflowExecution, "", nil
}
@@ -2460,6 +2545,10 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
}
}
err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
//err = memcache.Delete(request.Context(), sessionToken)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
@@ -3043,6 +3132,16 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
return err
}
err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1)
if err != nil {
log.Printf("Failed to increase total apps created stats: %s", err)
}
err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
//memcache.Delete(ctx, "all_apps")
//os.Exit(3)
+11 -16
View File
@@ -23,19 +23,14 @@
#curl -X POST "https://europe-west1-shuffle-241517.cloudfunctions.net/webhook_3ceff795-ce9a-43a2-a2f5-d4401a6e772d" -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" --data 'wut'
#curl POST "https://europe-west1-shuffler.cloudfunctions.net/outlooktrigger_be4dbb0a-d396-4544-bc36-e57d1bdb2e40" -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" --data 'wut' -vvv
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
+1 -1
View File
@@ -4016,7 +4016,7 @@ const AngularWorkflow = (props) => {
},
}}
color="secondary"
placeholder={"Optional argument"}
placeholder={"Execution Argument"}
onBlur={(e) => {
setExecutionText(e.target.value)
}}
+1 -7
View File
@@ -76,14 +76,8 @@ const App = (message, props) => {
if (dataset === false) {
checkLogin()
setDataset(true)
initializeReactGA()
}
})
}})
function initializeReactGA() {
}
console.log(window.location)
if (isLoaded && !isLoggedIn && (!window.location.pathname.startsWith("/login") && (!window.location.pathname.startsWith("/docs") && (!window.location.pathname.startsWith("/adminsetup"))))) {
window.location = "login"
}
+76
View File
@@ -3,6 +3,7 @@ import React, {useState} from 'react';
import classNames from "classnames";
// react plugin used to create charts
import { Line, Bar } from "react-chartjs-2";
import { useAlert } from "react-alert";
// https://demos.creative-tim.com/black-dashboard-react/?ref=appseed#/admin/dashboard
@@ -38,12 +39,87 @@ import {
// This is the start of a dashboard that can be used.
// What data do we fill in here? Idk
const Dashboard = (props) => {
const { globalUrl } = props;
const alert = useAlert()
const [bigChartData, setBgChartData] = useState("data1");
const [firstRequest, setFirstRequest] = useState(true);
const [stats, setStats] = useState({})
const [changeme, setChangeme] = useState("")
document.title = "Shuffle - dashboard"
const fetchdata = (stats_id) => {
fetch(globalUrl+"/api/v1/stats/"+stats_id, {
method: 'GET',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
credentials: "include",
})
.then((response) => {
if (response.status !== 200) {
console.log("Status not 200 for stream results :O!")
}
return response.json()
})
.then((responseJson) => {
console.log("DATA: ", responseJson)
stats[stats_id] = responseJson
setStats(stats)
// Used to force updates
setChangeme(stats_id)
})
.catch(error => {
alert.error("ERROR: "+error.toString())
});
}
const variables = [
"backend_executions",
"workflow_executions",
"workflow_executions_aborted",
"workflow_executions_success",
"total_apps_created",
"total_apps_loaded",
"openapi_apps_created",
"total_apps_deleted",
"total_webhooks_ran",
]
if (firstRequest) {
setFirstRequest(false)
for (var key in variables) {
fetchdata(variables[key])
}
}
const newdata = Object.getOwnPropertyNames(stats).length > 0 ?
<div>
{variables.map(data => {
if (stats[data] === undefined || stats[data] === null) {
return null
}
if (stats[data].total === undefined) {
return null
}
return (
<div>
{data}: {stats[data].total}
</div>
)
})}
</div>
: null
const data =
<div className="content">
{newdata}
<Row>
<Col xs="12">
<Card className="card-chart">
-14
View File
@@ -103,20 +103,6 @@ const Header = props => {
const loginTextBrowser = !isLoggedIn ?
<div style={{display: "flex"}}>
<List style={{display: "flex", flexDirect: "row"}} component="nav">
<ListItem style={{textAlign: "center", minWidth: "120px"}}>
<Link to="/" style={hrefStyle}>
<div onMouseOver={handleHomeHover} onMouseOut={handleHomeHoverOut} style={{color: HomeHoverColor, cursor: "pointer"}}>
<Grid container direction="row" alignItems="center">
<Grid item>
<HomeIcon style={{marginTop: "3px", marginRight: "5px"}} />
</Grid>
<Grid item>
Shuffle
</Grid>
</Grid>
</div>
</Link>
</ListItem>
<ListItem style={{textAlign: "center", marginLeft: "0px"}}>
<Link to ="/docs/about" style={hrefStyle}>
<div onMouseOver={handleSoarHover} onMouseOut={handleSoarHoverOut} style={{color: SoarHoverColor, cursor: "pointer"}}>
+9 -9
View File
@@ -357,15 +357,15 @@ const Workflows = (props) => {
>
<MoreVertIcon />
</IconButton>
<Menu
id="long-menu"
anchorEl={anchorEl}
keepMounted
open={open}
onClose={() => {
setOpen(false)
setAnchorEl(null)
}}
<Menu
id="long-menu"
anchorEl={anchorEl}
keepMounted
open={open}
onClose={() => {
setOpen(false)
setAnchorEl(null)
}}
>
<MenuItem style={{backgroundColor: surfaceColor, color: "white"}} onClick={() => {