marker/shuffle-cracked
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added backend statistics for dashboard

Browse Source
This commit is contained in:
frikky
2020-05-14 08:48:20 +02:00
parent e3dca4c17e
commit 7c33b91f1f
9 changed files with 297 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/go-app/main.go
+99 -6
View File
@@ -79,6 +79,35 @@ type ExecutionInfo struct {
DailyOnpremExecutions int64 `json:"daily_onprem_executions" datastore:"daily_onprem_executions"`
}
type StatisticsData struct {
Timestamp int64 `json:"timestamp" datastore:"timestamp"`
Id string `json:"id" datastore:"id"`
Amount int64 `json:"amount" datastore:"amount"`
}
type StatisticsItem struct {
Total int64 `json:"total" datastore:"total"`
Fieldname string `json:"field_name" datastore:"field_name"`
Data []StatisticsData `json:"data" datastore:"data"`
}
// "Execution by status"
// Execution history
//type GlobalStatistics struct {
// BackendExecutions int64 `json:"backend_executions" datastore:"backend_executions"`
// WorkflowCount int64 `json:"workflow_count" datastore:"workflow_count"`
// ExecutionCount int64 `json:"execution_count" datastore:"execution_count"`
// ExecutionSuccessCount int64 `json:"execution_success_count" datastore:"execution_success_count"`
// ExecutionAbortCount int64 `json:"execution_abort_count" datastore:"execution_abort_count"`
// ExecutionFailureCount int64 `json:"execution_failure_count" datastore:"execution_failure_count"`
// ExecutionPendingCount int64 `json:"execution_pending_count" datastore:"execution_pending_count"`
// AppUsageCount int64 `json:"app_usage_count" datastore:"app_usage_count"`
// TotalAppsCount int64 `json:"total_apps_count" datastore:"total_apps_count"`
// SelfMadeAppCount int64 `json:"self_made_app_count" datastore:"self_made_app_count"`
// WebhookUsageCount int64 `json:"webhook_usage_count" datastore:"webhook_usage_count"`
// Baseline map[string]int64 `json:"baseline" datastore:"baseline"`
//}
type ParsedOpenApi struct {
Body string `datastore:"body,noindex" json:"body"`
ID string `datastore:"id" json:"id"`
@@ -2783,6 +2812,11 @@ func handleWebhookCallback(resp http.ResponseWriter, request *http.Request) {
workflowExecution, executionResp, err := handleExecution(item, workflow, request)
if err == nil {
err = increaseStatisticsField(ctx, "total_webhooks_ran", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true, "execution_id": "%s", "authorization": "%s"}`, workflowExecution.ExecutionId, workflowExecution.Authorization)))
return
@@ -4481,6 +4515,55 @@ func handleGetOutlookFolders(resp http.ResponseWriter, request *http.Request) {
resp.Write(b)
}
func handleGetSpecificStats(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
return
}
_, err := handleApiAuthentication(resp, request)
if err != nil {
log.Printf("Api authentication failed in getting specific workflow: %s", err)
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
location := strings.Split(request.URL.String(), "/")
var statsId string
if location[1] == "api" {
if len(location) <= 4 {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
statsId = location[4]
}
ctx := context.Background()
statisticsId := "global_statistics"
nameKey := statsId
key := datastore.NameKey(statisticsId, nameKey, nil)
statisticsItem := StatisticsItem{}
if err := dbclient.Get(ctx, key, &statisticsItem); err != nil {
resp.WriteHeader(401)
resp.Write([]byte(`{"success": false}`))
return
}
b, err := json.Marshal(statisticsItem)
if err != nil {
log.Println("Failed to marshal data: %s", err)
resp.WriteHeader(401)
return
}
resp.WriteHeader(200)
resp.Write([]byte(b))
}
func handleGetSpecificTrigger(resp http.ResponseWriter, request *http.Request) {
cors := handleCors(resp, request)
if cors {
@@ -5503,6 +5586,15 @@ func verifySwagger(resp http.ResponseWriter, request *http.Request) {
}
setOpenApiDatastore(ctx, api.ID, parsed)
err = increaseStatisticsField(ctx, "total_apps_created", api.ID, 1)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
err = increaseStatisticsField(ctx, "openapi_apps_created", api.ID, 1)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
@@ -5522,6 +5614,11 @@ func init() {
log.Fatalf("DBclient error during init: %s", err)
}
err = increaseStatisticsField(ctx, "backend_executions", "", 1)
if err != nil {
log.Printf("Failed increasing local stats: %s", err)
}
count, err := getEnvironmentCount()
if count == 0 && err == nil {
item := Environment{
@@ -5554,15 +5651,9 @@ func init() {
r.HandleFunc("/api/v1/getenvironments", handleGetEnvironments).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/setenvironments", handleSetEnvironments).Methods("PUT", "OPTIONS")
//r.HandleFunc("/api/v1/register/{key}", handleRegisterVerification).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/getinfo", handleInfo).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/getsettings", handleSettings).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/generateapikey", handleApiGeneration).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/passwordchange", handlePasswordChange).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/passwordresetmail", handlePasswordResetMail).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/passwordreset", handlePasswordReset).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/contact", handleContact).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/docs", getDocList).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/docs/{key}", getDocs).Methods("GET", "OPTIONS")
@@ -5614,6 +5705,8 @@ func init() {
// Trigger hmm
r.HandleFunc("/api/v1/triggers/{key}", handleGetSpecificTrigger).Methods("GET", "OPTIONS")
r.HandleFunc("/api/v1/stats/{key}", handleGetSpecificStats).Methods("GET", "OPTIONS")
// OpenAPI configuration
r.HandleFunc("/api/v1/verify_swagger", verifySwagger).Methods("POST", "OPTIONS")
r.HandleFunc("/api/v1/verify_openapi", verifySwagger).Methods("POST", "OPTIONS")
backend/go-app/walkoff.go
+99
View File
@@ -274,6 +274,62 @@ type ExecutionRequestWrapper struct {
Data []ExecutionRequest `json:"data"`
}
// This might be... a bit off, but that's fine :)
// This might also be stupid, as we want timelines and such
// Anyway, these are super basic stupid stats.
func increaseStatisticsField(ctx context.Context, fieldname, id string, amount int64) error {
// 1. Get current stats
// 2. Increase field(s)
// 3. Put new stats
statisticsId := "global_statistics"
nameKey := fieldname
key := datastore.NameKey(statisticsId, nameKey, nil)
statisticsItem := StatisticsItem{}
newData := StatisticsData{
Timestamp: int64(time.Now().Unix()),
Amount: amount,
Id: id,
}
if err := dbclient.Get(ctx, key, &statisticsItem); err != nil {
// Should init
if strings.Contains(fmt.Sprintf("%s", err), "entity") {
statisticsItem = StatisticsItem{
Total: amount,
Fieldname: fieldname,
Data: []StatisticsData{
newData,
},
}
if _, err := dbclient.Put(ctx, key, &statisticsItem); err != nil {
log.Printf("Error setting base stats: %s", err)
return err
}
return nil
}
//log.Printf("STATSERR: %s", err)
return err
}
statisticsItem.Total += amount
statisticsItem.Data = append(statisticsItem.Data, newData)
// New struct, to not add body, author etc
if _, err := dbclient.Put(ctx, key, &statisticsItem); err != nil {
log.Printf("Error stats to %s: %s", fieldname, err)
return err
}
log.Printf("Stats: %#v", statisticsItem)
return nil
}
func setWorkflowQueue(ctx context.Context, executionRequests ExecutionRequestWrapper, id string) error {
key := datastore.NameKey("workflowqueue", id, nil)
@@ -607,6 +663,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
if workflowExecution.Status == "FINISHED" {
log.Printf("Workflowexecution is already FINISHED. No further action can be taken")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is already finished because of %s with status %s"}`, workflowExecution.LastNode, workflowExecution.Status)))
return
@@ -615,6 +672,7 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
// Not sure what's up here
// FIXME - remove comment
if workflowExecution.Status == "ABORTED" || workflowExecution.Status == "FAILURE" {
log.Printf("Workflowexecution is already aborted. No further action can be taken")
resp.WriteHeader(401)
resp.Write([]byte(fmt.Sprintf(`{"success": false, "reason": "Workflowexecution is aborted because of %s with result %s and status %s"}`, workflowExecution.LastNode, workflowExecution.Result, workflowExecution.Status)))
@@ -645,6 +703,18 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
workflowExecution.Result = lastResult
workflowExecution.Results = newResults
if workflowExecution.Status == "ABORTED" {
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase aborted execution stats: %s", err)
}
} else if workflowExecution.Status == "FAILURE" {
err = increaseStatisticsField(ctx, "workflow_executions_failure", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase failure execution stats: %s", err)
}
}
}
// This means it should continue I think :)
@@ -722,6 +792,11 @@ func handleWorkflowQueue(resp http.ResponseWriter, request *http.Request) {
if workflowExecution.LastNode == "" {
workflowExecution.LastNode = actionResult.Action.ID
}
err = increaseStatisticsField(ctx, "workflow_executions_success", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase success execution stats: %s", err)
}
}
}
@@ -1491,6 +1566,11 @@ func abortExecution(resp http.ResponseWriter, request *http.Request) {
return
}
err = increaseStatisticsField(ctx, "workflow_executions_aborted", workflowExecution.Workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase aborted execution stats: %s", err)
}
// FIXME - allowed to edit it? idk
resp.WriteHeader(200)
resp.Write([]byte(fmt.Sprintf(`{"success": true}`)))
@@ -1834,6 +1914,11 @@ func handleExecution(id string, workflow Workflow, request *http.Request) (Workf
}
}
err = increaseStatisticsField(ctx, "workflow_executions", workflow.ID, 1)
if err != nil {
log.Printf("Failed to increase stats execution stats: %s", err)
}
return workflowExecution, "", nil
}
@@ -2460,6 +2545,10 @@ func deleteWorkflowApp(resp http.ResponseWriter, request *http.Request) {
}
}
err = increaseStatisticsField(ctx, "total_apps_deleted", fileId, 1)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
//err = memcache.Delete(request.Context(), sessionToken)
resp.WriteHeader(200)
resp.Write([]byte(`{"success": true}`))
@@ -3043,6 +3132,16 @@ func iterateAppGithubFolders(fs billy.Filesystem, dir []os.FileInfo, extra strin
return err
}
err = increaseStatisticsField(ctx, "total_apps_created", workflowapp.ID, 1)
if err != nil {
log.Printf("Failed to increase total apps created stats: %s", err)
}
err = increaseStatisticsField(ctx, "total_apps_loaded", workflowapp.ID, 1)
if err != nil {
log.Printf("Failed to increase total apps loaded stats: %s", err)
}
log.Printf("Added %s:%s to the database", workflowapp.Name, workflowapp.AppVersion)
//memcache.Delete(ctx, "all_apps")
//os.Exit(3)
backend/tests/hooks.sh
+11 -16
View File
@@ -23,19 +23,14 @@
#curl -X POST "https://europe-west1-shuffle-241517.cloudfunctions.net/webhook_3ceff795-ce9a-43a2-a2f5-d4401a6e772d" -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" --data 'wut'
#curl POST "https://europe-west1-shuffler.cloudfunctions.net/outlooktrigger_be4dbb0a-d396-4544-bc36-e57d1bdb2e40" -H "Authorization: Bearer 144308d0-6aab-4d4f-8bb2-75189281ee26" --data 'wut' -vvv
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_9a4efb03-089c-48b7-82f3-f76500a3ebc0" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v
curl -X POST "http://localhost:5001/api/v1/hooks/webhook_e38e0768-191c-42ec-a021-cf17a4339473" -H "Content-Type:application/json" --data '{"test": {"hello": "HEYOOOO"}}' -v