Мониторинг
Платформа предоставляет эндпоинт с метриками в формате Prometheus, позволяющий отслеживать состояние приложения и его внутренние процессы. Prometheus — это система сбора и анализа метрик с открытым исходным кодом, построенная по принципу pull-модели: мониторинговая система периодически опрашивает целевые приложения, запрашивая у них актуальные показатели работы. Метрики представляются в текстовом формате и могут быть использованы для визуализации (например, в Grafana), настройки алертов и анализа производительности.
Для сбора метрик в платформе используются библиотеки:
- prometheus_client - клиентская библиотека для работы с метриками;
- prometheus_flask_exporter - интеграция с Flask-приложением для автоматического сбора HTTP-метрик.
Описание, что такое Prometheus и как он работает можно посмотреть здесь https://prometheus.io/docs/introduction/overview/.
Из приложения метрики доступны по API http:/application/metrics
Доступ к метрикам может быть защищен паролем с использованием механизма basic HTTP auth.
Список доступных метрик:
Метрики по умолчанию (prometheus_flask_exporter)
# HELP flask_http_request_duration_seconds Multiprocess metric
# TYPE flask_http_request_duration_seconds histogram
flask_http_request_duration_seconds_sum{endpoint="versions.version",method="GET",status="200"} 0.8981670029461384
flask_http_request_duration_seconds_sum{endpoint="application.get_styles",method="GET",status="200"} 1.4094492476433516
flask_http_request_duration_seconds_sum{endpoint="auth.status_post",method="POST",status="200"} 1.340100983157754
flask_http_request_duration_seconds_sum{endpoint="admin.get_license",method="GET",status="200"} 0.7679055598564446
flask_http_request_duration_seconds_sum{endpoint="tasks.get_tasks",method="GET",status="200"} 1.2158617144450545
flask_http_request_duration_seconds_sum{endpoint="tags.get_tags",method="GET",status="200"} 0.14136453811079264
flask_http_request_duration_seconds_sum{endpoint="tasks.get_tasks_last_opened",method="GET",status="200"} 0.1545978100039065
flask_http_request_duration_seconds_sum{endpoint="scenarios.get_all_tasks_with_scenario",method="GET",status="200"} 0.9224562519229949
flask_http_request_duration_seconds_sum{endpoint="progressbar.calc_graph_get_progress",method="POST",status="200"} 258.88298390991986
flask_http_request_duration_seconds_sum{endpoint="graph.get_blocks_state",method="POST",status="200"} 0.1778613799251616
flask_http_request_duration_seconds_sum{endpoint="debug.get_block_debug_input_and_output",method="GET",status="200"} 61.21176861273125
flask_http_request_duration_seconds_sum{endpoint="library.get_libraries",method="GET",status="200"} 0.5758497272618115
flask_http_request_duration_seconds_sum{endpoint="tasks.get_task",method="GET",status="200"} 0.995703399181366
flask_http_request_duration_seconds_sum{endpoint="graph.get_tasks",method="GET",status="200"} 1.8132902421057224
flask_http_request_duration_seconds_sum{endpoint="presets.get_presets",method="GET",status="200"} 16.25772683136165
flask_http_request_duration_seconds_sum{endpoint="scenarios.get_scenario_by_task",method="GET",status="200"} 0.4887871718965471
flask_http_request_duration_seconds_sum{endpoint="debug.get_debug_vis",method="GET",status="200"} 44.124159295111895
flask_http_request_duration_seconds_sum{endpoint="presets.update_preset",method="PUT",status="200"} 0.44251092430204153
flask_http_request_duration_seconds_sum{endpoint="debug.get_block_debug_info",method="GET",status="200"} 0.7266020360402763
flask_http_request_duration_seconds_sum{endpoint="system.get_menu_styles_list",method="GET",status="200"} 0.23752294667065144
flask_http_request_duration_seconds_sum{endpoint="admin.get_users",method="GET",status="200"} 3.246767120435834
flask_http_request_duration_seconds_sum{endpoint="admin.get_roles",method="GET",status="200"} 0.09917925577610731
flask_http_request_duration_seconds_sum{endpoint="admin.get_groups",method="GET",status="200"} 3.4793411530554295
flask_http_request_duration_seconds_sum{endpoint="admin.get_users_tasks",method="GET",status="200"} 0.540447766892612
flask_http_request_duration_seconds_sum{endpoint="states.find_states_by_block",method="POST",status="200"} 0.49723414750769734
flask_http_request_duration_seconds_sum{endpoint="states.get_states",method="GET",status="200"} 0.26628985488787293
flask_http_request_duration_seconds_sum{endpoint="screenshots.get_screenshots",method="GET",status="200"} 0.26606357656419277
flask_http_request_duration_seconds_sum{endpoint="presets.get_preset",method="GET",status="200"} 1.7903634295798838
flask_http_request_duration_seconds_sum{endpoint="library.get_blocks_preset",method="GET",status="200"} 0.2519649784080684
flask_http_request_duration_seconds_sum{endpoint="progressbar.calc_event",method="POST",status="200"} 1.165907091461122
flask_http_request_duration_seconds_sum{endpoint="system.get_connections_list",method="GET",status="200"} 0.07805519551038742
flask_http_request_duration_seconds_sum{endpoint="system.check_connection",method="POST",status="200"} 4.445009535178542
flask_http_request_duration_seconds_sum{endpoint="versions.version",method="HEAD",status="200"} 0.018980565946549177
flask_http_request_duration_seconds_sum{endpoint="globals.get_menu_styles",method="GET",status="200"} 0.11861382890492678
flask_http_request_duration_seconds_sum{endpoint="globals.get_connections",method="GET",status="200"} 0.6563644618727267
flask_http_request_duration_seconds_sum{endpoint="globals.get_connection_types",method="GET",status="200"} 0.030361573211848736
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.025",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.05",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.075",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.1",method="GET",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.25",method="GET",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.5",method="GET",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.75",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="1.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="2.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="5.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="7.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="10.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="+Inf",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_count{endpoint="versions.version",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.05",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.075",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.1",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.25",method="GET",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.5",method="GET",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="0.75",method="GET",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="1.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="2.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="5.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="7.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="10.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="application.get_styles",le="+Inf",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_count{endpoint="application.get_styles",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.005",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.01",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.025",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.05",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.075",method="POST",status="200"} 5.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.1",method="POST",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.25",method="POST",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.5",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="0.75",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="1.0",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="2.5",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="5.0",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="7.5",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="10.0",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="auth.status_post",le="+Inf",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_count{endpoint="auth.status_post",method="POST",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.025",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.05",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.075",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.1",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.25",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="0.75",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="1.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="2.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="5.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="7.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="10.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_license",le="+Inf",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_count{endpoint="admin.get_license",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.05",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.075",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.1",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.25",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="0.75",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="1.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="2.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="5.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="7.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="10.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks",le="+Inf",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_count{endpoint="tasks.get_tasks",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.05",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.075",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.1",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.25",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="0.75",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="1.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="2.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="5.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="7.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="10.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tags.get_tags",le="+Inf",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_count{endpoint="tags.get_tags",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.075",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.1",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.25",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="0.75",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="1.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="2.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="5.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="7.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="10.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_tasks_last_opened",le="+Inf",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_count{endpoint="tasks.get_tasks_last_opened",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.05",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.075",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.1",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.25",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="0.75",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="1.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="2.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="5.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="7.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="10.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_all_tasks_with_scenario",le="+Inf",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="scenarios.get_all_tasks_with_scenario",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.005",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.01",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.025",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.05",method="POST",status="200"} 299.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.075",method="POST",status="200"} 2455.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.1",method="POST",status="200"} 2473.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.25",method="POST",status="200"} 2516.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.5",method="POST",status="200"} 2550.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="0.75",method="POST",status="200"} 2551.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="1.0",method="POST",status="200"} 2551.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="2.5",method="POST",status="200"} 2553.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="5.0",method="POST",status="200"} 2555.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="7.5",method="POST",status="200"} 2555.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="10.0",method="POST",status="200"} 2555.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_graph_get_progress",le="+Inf",method="POST",status="200"} 2559.0
flask_http_request_duration_seconds_count{endpoint="progressbar.calc_graph_get_progress",method="POST",status="200"} 2559.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.005",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.01",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.025",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.05",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.075",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.1",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.25",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.5",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="0.75",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="1.0",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="2.5",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="5.0",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="7.5",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="10.0",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_blocks_state",le="+Inf",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_count{endpoint="graph.get_blocks_state",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.1",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.25",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="0.75",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="1.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="2.5",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="5.0",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="7.5",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="10.0",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_input_and_output",le="+Inf",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_count{endpoint="debug.get_block_debug_input_and_output",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.1",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.25",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="0.75",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="1.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="2.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="5.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="7.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="10.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_libraries",le="+Inf",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="library.get_libraries",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.075",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.1",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.25",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.5",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="0.75",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="1.0",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="2.5",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="5.0",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="7.5",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="10.0",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="tasks.get_task",le="+Inf",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_count{endpoint="tasks.get_task",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.1",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.25",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="0.75",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="1.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="2.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="5.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="7.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="10.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="graph.get_tasks",le="+Inf",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="graph.get_tasks",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.075",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.1",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.25",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.5",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="0.75",method="GET",status="200"} 9.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="1.0",method="GET",status="200"} 10.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="2.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="5.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="7.5",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="10.0",method="GET",status="200"} 11.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_presets",le="+Inf",method="GET",status="200"} 12.0
flask_http_request_duration_seconds_count{endpoint="presets.get_presets",method="GET",status="200"} 12.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.05",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.075",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.1",method="GET",status="200"} 7.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.25",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.5",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="0.75",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="1.0",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="2.5",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="5.0",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="7.5",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="10.0",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="scenarios.get_scenario_by_task",le="+Inf",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_count{endpoint="scenarios.get_scenario_by_task",method="GET",status="200"} 8.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.1",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.25",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.5",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="0.75",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="1.0",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="2.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="5.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="7.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="10.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_debug_vis",le="+Inf",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_count{endpoint="debug.get_debug_vis",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.005",method="PUT",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.01",method="PUT",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.025",method="PUT",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.05",method="PUT",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.075",method="PUT",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.1",method="PUT",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.25",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.5",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="0.75",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="1.0",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="2.5",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="5.0",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="7.5",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="10.0",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.update_preset",le="+Inf",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_count{endpoint="presets.update_preset",method="PUT",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.05",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.075",method="GET",status="200"} 5.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.1",method="GET",status="200"} 5.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.25",method="GET",status="200"} 5.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="0.75",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="1.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="2.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="5.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="7.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="10.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="debug.get_block_debug_info",le="+Inf",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_count{endpoint="debug.get_block_debug_info",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.05",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.075",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.1",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.25",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="0.75",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="1.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="2.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="5.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="7.5",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="10.0",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_menu_styles_list",le="+Inf",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_count{endpoint="system.get_menu_styles_list",method="GET",status="200"} 6.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.1",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.25",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.5",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="0.75",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="1.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="2.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="5.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="7.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="10.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users",le="+Inf",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_count{endpoint="admin.get_users",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.05",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.075",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.1",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.25",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="0.75",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="1.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="2.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="5.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="7.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="10.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_roles",le="+Inf",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_count{endpoint="admin.get_roles",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.1",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.25",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.5",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="0.75",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="1.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="2.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="5.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="7.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="10.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_groups",le="+Inf",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_count{endpoint="admin.get_groups",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.1",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.25",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="0.75",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="1.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="2.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="5.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="7.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="10.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="admin.get_users_tasks",le="+Inf",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_count{endpoint="admin.get_users_tasks",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.005",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.01",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.025",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.05",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.075",method="POST",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.1",method="POST",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.25",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.5",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="0.75",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="1.0",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="2.5",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="5.0",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="7.5",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="10.0",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.find_states_by_block",le="+Inf",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="states.find_states_by_block",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.075",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.1",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.25",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="0.75",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="1.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="2.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="5.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="7.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="10.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="states.get_states",le="+Inf",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="states.get_states",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.05",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.075",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.1",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.25",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="0.75",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="1.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="2.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="5.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="7.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="10.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="screenshots.get_screenshots",le="+Inf",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="screenshots.get_screenshots",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.1",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.25",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.5",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="0.75",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="1.0",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="2.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="5.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="7.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="10.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="presets.get_preset",le="+Inf",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="presets.get_preset",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.05",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.075",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.1",method="GET",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.25",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="0.75",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="1.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="2.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="5.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="7.5",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="10.0",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="library.get_blocks_preset",le="+Inf",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="library.get_blocks_preset",method="GET",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.005",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.01",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.025",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.05",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.075",method="POST",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.1",method="POST",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.25",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.5",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="0.75",method="POST",status="200"} 3.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="1.0",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="2.5",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="5.0",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="7.5",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="10.0",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="progressbar.calc_event",le="+Inf",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_count{endpoint="progressbar.calc_event",method="POST",status="200"} 4.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.05",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.075",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.1",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.25",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="0.75",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="1.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="2.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="5.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="7.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="10.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.get_connections_list",le="+Inf",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_count{endpoint="system.get_connections_list",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.005",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.01",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.025",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.05",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.075",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.1",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.25",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.5",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="0.75",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="1.0",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="2.5",method="POST",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="5.0",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="7.5",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="10.0",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="system.check_connection",le="+Inf",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_count{endpoint="system.check_connection",method="POST",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.005",method="HEAD",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.01",method="HEAD",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.025",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.05",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.075",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.1",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.25",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.5",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="0.75",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="1.0",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="2.5",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="5.0",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="7.5",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="10.0",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="versions.version",le="+Inf",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_count{endpoint="versions.version",method="HEAD",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.075",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.1",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.25",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="0.75",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="1.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="2.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="5.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="7.5",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="10.0",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_menu_styles",le="+Inf",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_count{endpoint="globals.get_menu_styles",method="GET",status="200"} 2.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.05",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.075",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.1",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.25",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.5",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="0.75",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="1.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="2.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="5.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="7.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="10.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connections",le="+Inf",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_count{endpoint="globals.get_connections",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.005",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.01",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.025",method="GET",status="200"} 0.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.05",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.075",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.1",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.25",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="0.75",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="1.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="2.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="5.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="7.5",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="10.0",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_bucket{endpoint="globals.get_connection_types",le="+Inf",method="GET",status="200"} 1.0
flask_http_request_duration_seconds_count{endpoint="globals.get_connection_types",method="GET",status="200"} 1.0
# HELP flask_http_request_total Multiprocess metric
# TYPE flask_http_request_total counter
flask_http_request_total{method="GET",status="200"} 139.0
flask_http_request_total{method="POST",status="200"} 2582.0
flask_http_request_total{method="PUT",status="200"} 3.0
flask_http_request_total{method="HEAD",status="200"} 1.0
# HELP vismind_metrics Multiprocess metric
# TYPE vismind_metrics gauge
vismind_metrics{name="auth_correct"} 0.0
vismind_metrics{name="auth_not_correct"} 0.0
vismind_metrics{name="open_presets"} 0.0
vismind_metrics{name="tokens"} 95.0
# HELP vismind_metrics_users Multiprocess metric
# TYPE vismind_metrics_users gauge
# HELP vismind_metrics_users_operation Multiprocess metric
# TYPE vismind_metrics_users_operation gauge
# HELP vismind_metrics_calculations Multiprocess metric
# TYPE vismind_metrics_calculations gauge
# HELP vismind_build Multiprocess metric
# TYPE vismind_build gauge
vismind_build{version="0.38.0.18855-0a34bb26-vismind-10386-metrics-27.09.2023 14:15:32"} 1.0
Пользовательские метрики приложения, добавленные в дополнение к стандартным метрикам Prometheus:
- статические (собираются при запуске приложения):
vismind_build{version}- информация о версии приложении;vismind_metrics_calculations- количество активных расчетов с разбивкой по типу расчета (type_calculation);- Обновляется в момент сбора метрики;
- Соответствует количеству активных расчетов на странице "Текущие расчеты";
- Возможные варианты
Graph- расчет графаBlock- расчет блокаBlocks- расчет блоковEvent- расчет событияBranch- расчет веткиBranches- расчет ветокPreset- расчет пресетаPreview- расчет предпросмотраBlockAllPath- расчет блока и перерасчет всех блоков перед нимMasterAutoFactor- расчет мастера автоподбора в линейной регрессииMasterArimaAnalysis- анализ временных рядов для аримыMasterArimaAuto- расчет мастера аналитики аримыPresetOptimizationAnalysis- анализ оптимизации пресетаBlockSave- сохраненияBlockSaveAttribute- сохранение только атрибутов
vismind_metrics{name="tokens"}- количество активных токенов- Обновляется в момент сбора метрики.
- динамические (собираются при запуске приложения и обновляются во время использования приложения):
vismind_metrics{name="open_presets"}- количество открытых пресетов:- Обновляется при входе пользователя на пресет или при выходе из него
vismind_metrics{name="auth_correct"}- количество корректных авторизаций:- Обновляется при корректной авторизации пользователя в системе;
- При получении метрики показатель обнуляется;
vismind_metrics{name="auth_not_correct"}- количество ошибочных авторизаций:- Обновляется при неуспешной авторизации пользователя в системе;
- При получении метрики показатель обнуляется;
vismind_metrics_users- количество пользователей в комнате:- login - логин пользователя;
- room - комната, в которой находится пользователь:
- если начитается с P - это комната пресета и его id;
- если начитается с T - это комната графа и его id;
- если начитается с A - это комната администрирования;
- Обновляется при входе пользователя в комнату или выходе из нее;
vismind_metrics_users_operation- количество операций по пользователю:- login - логин пользователя ;
- Обновляется при вызове пользователем любого API;
- При получении метрики показатель обнуляется;
Метрики приложения (prometheus_flask_exporter)
# HELP vismind_metrics Multiprocess metric
# TYPE vismind_metrics gauge
vismind_metrics{name="auth_correct"} 0.0
vismind_metrics{name="auth_not_correct"} 0.0
vismind_metrics{name="open_presets"} 0.0
vismind_metrics{name="tokens"} 95.0
# HELP vismind_metrics_users Multiprocess metric
# TYPE vismind_metrics_users gauge
vismind_metrics_users{login="nkataev",room="T-review-vismind-10386-metrics"} 1.0
vismind_metrics_users{login="nkataev",room="P-2c1a3386-0d67-4cef-b497-009331500509"} 1.0
# HELP vismind_metrics_users_operation Multiprocess metric
# TYPE vismind_metrics_users_operation gauge
vismind_metrics_users_operation{login="nkataev"} 5.0
# HELP vismind_metrics_calculations Multiprocess metric
# TYPE vismind_metrics_calculations gauge
vismind_metrics_calculations{type_calculation="RouterEvent"} 1.0
# HELP vismind_build Multiprocess metric
# TYPE vismind_build gauge
vismind_build{version="0.38.0.18855-0a34bb26-vismind-10386-metrics-27.09.2023 14:15:32"} 1.0
Настройка сбора метрик в prometheus
scrape_configs:
# Сбор мерик приложения по http с указанием "login"|"password" доступа к API `http:/application/metrics`
- job_name: 'vmapp'
scrape_interval: 60s
metrics_path: '/metrics'
fallback_scrape_protocol: "PrometheusText1.0.0"
static_configs:
- targets: ['server1:80']
labels:
label: "server1"
basic_auth:
username: "login"
password: "password"
# Сбор мерик приложения по https с указанием "login"|"password" доступа к API `https:/application/metrics`
# В случаи использования самоподписаных сертификатов, для Prometheus необходим корневой сертификат локального центр сертификации в формате PEM
- job_name: 'vmapp-https'
scrape_interval: 60s
metrics_path: '/metrics'
fallback_scrape_protocol: "PrometheusText1.0.0"
static_configs:
- targets: ['server2.domain.local']
labels:
label: "sever2"
scheme: https
tls_config:
ca_file: ./certs/ca_cert.crt
server_name: 'server2.domain.local'
basic_auth:
username: "login"
password: "password"
Важно
При настройке сбора метрик необходимо создать только одно задание по указанному выше шаблону — либо HTTP, либо HTTPS — в зависимости от конкретной реализации в инфраструктуре.
Пример dashboard для импорта в Grafana
{
"__inputs": [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus"
}
],
"__elements": {},
"__requires": [
{
"type": "panel",
"id": "bargauge",
"name": "Bar gauge",
"version": ""
},
{
"type": "grafana",
"id": "grafana",
"name": "Grafana",
"version": "12.3.6"
},
{
"type": "panel",
"id": "heatmap",
"name": "Heatmap",
"version": ""
},
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0"
},
{
"type": "panel",
"id": "stat",
"name": "Stat",
"version": ""
},
{
"type": "panel",
"id": "timeseries",
"name": "Time series",
"version": ""
}
],
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"links": [],
"panels": [
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 0
},
"id": 21,
"panels": [],
"title": "Vismind metrics",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.3,
"drawStyle": "bars",
"fillOpacity": 0,
"gradientMode": "hue",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 10,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 4,
"w": 24,
"x": 0,
"y": 1
},
"id": 35,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"disableTextWrap": false,
"editorMode": "code",
"exemplar": true,
"expr": "vismind_build{instance=~\"$instance\"}",
"format": "heatmap",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"interval": "",
"legendFormat": "{{version}}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "App version by time",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 3,
"x": 0,
"y": 5
},
"id": 26,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "count(count(vismind_metrics_users{instance=~\"$instance\"} > 0) by (login))",
"format": "time_series",
"hide": false,
"interval": "",
"legendFormat": "__auto",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "sum by (login) (vismind_metrics_users{instance=~\"$instance\"} > 0)",
"hide": true,
"instant": false,
"legendFormat": "{{login}}",
"range": true,
"refId": "B"
}
],
"title": "Кол-во пользователей в системе",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 3,
"x": 3,
"y": 5
},
"id": 12,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "vismind_metrics{instance=~\"$instance\",name=\"tokens\"}",
"interval": "",
"legendFormat": "tokens",
"range": true,
"refId": "A"
}
],
"title": "Кол-во активных токенов",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 3,
"x": 6,
"y": 5
},
"id": 13,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "vismind_metrics{instance=~\"$instance\",name=\"open_presets\"}",
"interval": "",
"legendFormat": "open_presets",
"range": true,
"refId": "A"
}
],
"title": "Кол-во открытых пресетов",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 3,
"x": 9,
"y": 5
},
"id": 9,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"last"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"text": {},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "vismind_metrics_calculations{instance=~\"$instance\"}",
"instant": true,
"interval": "",
"legendFormat": "{{type_calculation}}",
"refId": "A"
}
],
"title": "Кол-во активных расчетов",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 50,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 12,
"y": 5
},
"id": 14,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "vismind_metrics_users_operation{instance=~\"$instance\"}",
"interval": "",
"legendFormat": "{{login}}",
"range": true,
"refId": "A"
}
],
"title": "Операций/сек в разрезе пользователей",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 50,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 14
},
"id": 19,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "vismind_metrics_calculations{instance=~\"$instance\"}",
"interval": "",
"legendFormat": "calc",
"range": true,
"refId": "A"
}
],
"title": "Количество расчетов",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 14
},
"id": 15,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum by(room)(vismind_metrics_users{instance=~\"$instance\"})",
"interval": "",
"legendFormat": "{{room}}",
"range": true,
"refId": "A"
}
],
"title": "Количество пользователей в комнатах",
"type": "timeseries"
},
{
"datasource": {
"uid": "${datasource}"
},
"description": "",
"fieldConfig": {
"defaults": {
"custom": {
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"scaleDistribution": {
"type": "linear"
}
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 22
},
"id": 27,
"options": {
"calculate": false,
"cellGap": 1,
"color": {
"exponent": 0.5,
"fill": "semi-dark-green",
"mode": "opacity",
"reverse": false,
"scale": "exponential",
"scheme": "Oranges",
"steps": 64
},
"exemplars": {
"color": "rgba(255,0,255,0.7)"
},
"filterValues": {
"le": 1e-9
},
"legend": {
"show": true
},
"rowsFrame": {
"layout": "auto"
},
"tooltip": {
"mode": "single",
"showColorScale": false,
"yHistogram": false
},
"yAxis": {
"axisPlacement": "left",
"reverse": false
}
},
"pluginVersion": "12.3.6",
"targets": [
{
"editorMode": "code",
"expr": "vismind_metrics_users_operation{instance=~\"$instance\"}",
"legendFormat": "{{login}}",
"range": true,
"refId": "A",
"datasource": {
"uid": "${datasource}"
}
}
],
"title": "Активность пользователя в системе",
"type": "heatmap"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 22
},
"id": 17,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "(vismind_metrics_users{instance=~\"$instance\"} > 0)",
"interval": "",
"legendFormat": "{{login}} - {{room}}",
"range": true,
"refId": "A"
}
],
"title": "Количество пользователей в комнатах в разрезе пользователей",
"type": "timeseries"
},
{
"collapsed": false,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 30
},
"id": 22,
"panels": [],
"title": "Flask metrics",
"type": "row"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 7,
"x": 0,
"y": 31
},
"id": 23,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum (flask_http_request_total{instance=~\"$instance\"})",
"interval": "",
"legendFormat": "",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum (flask_http_request_total{instance=~\"$instance\"})",
"hide": false,
"instant": true,
"legendFormat": "__auto",
"range": false,
"refId": "B"
}
],
"timeFrom": "24h",
"title": "Total Requests",
"transformations": [
{
"id": "seriesToRows",
"options": {}
},
{
"id": "sortBy",
"options": {
"fields": {},
"sort": [
{
"field": "Time"
}
]
}
}
],
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "thresholds"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 7,
"x": 7,
"y": 31
},
"id": 24,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "sum (flask_http_request_total{instance=~\"$instance\"}) by(status)",
"interval": "",
"legendFormat": "HTTP Status: {{status}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": false,
"expr": "sum (flask_http_request_total{instance=~\"$instance\"}) by(status)",
"hide": false,
"instant": true,
"legendFormat": "HTTP Status: {{status}}",
"range": false,
"refId": "B"
}
],
"timeFrom": "24h",
"title": "Requests Per Status Code",
"transformations": [
{
"id": "seriesToRows",
"options": {}
},
{
"id": "sortBy",
"options": {
"fields": {},
"sort": [
{
"field": "Time"
}
]
}
},
{
"id": "partitionByValues",
"options": {
"fields": [
"Metric"
],
"keepFields": false
}
}
],
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "continuous-GrYlRd"
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 10,
"x": 14,
"y": 31
},
"id": 25,
"options": {
"displayMode": "lcd",
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"maxVizHeight": 300,
"minVizHeight": 16,
"minVizWidth": 8,
"namePlacement": "auto",
"orientation": "horizontal",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showUnfilled": true,
"sizing": "auto",
"valueMode": "color"
},
"pluginVersion": "12.3.6",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "flask_http_request_duration_seconds_sum{instance=~\"$instance\"} / flask_http_request_duration_seconds_count{instance=~\"$instance\"}",
"interval": "",
"legendFormat": "{{method}} {{endpoint}}",
"range": true,
"refId": "A"
}
],
"title": "Requests Average Duration total",
"type": "bargauge"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 39
},
"id": 29,
"options": {
"legend": {
"calcs": [
"last"
],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "topk(10,flask_http_request_duration_seconds_count{instance=~\"$instance\"})",
"interval": "",
"legendFormat": "{{method}} {{endpoint}}",
"range": true,
"refId": "A"
}
],
"title": "Requests Count Top 10",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "s"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 39
},
"id": 28,
"options": {
"legend": {
"calcs": [
"last"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "topk(10,(flask_http_request_duration_seconds_sum{instance=~\"$instance\"} / flask_http_request_duration_seconds_count{instance=~\"$instance\"}))",
"hide": false,
"interval": "",
"legendFormat": "{{endpoint}}",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"disableTextWrap": false,
"editorMode": "code",
"exemplar": true,
"expr": "sum by(endpoint) (rate(flask_http_request_duration_seconds_sum{instance=~\"$instance\", status=\"200\"}[2m])) / sum by(endpoint) (rate(flask_http_request_duration_seconds_count{instance=~\"$instance\", status=\"200\"}[2m]))",
"fullMetaSearch": false,
"hide": true,
"includeNullMetadata": true,
"interval": "",
"legendFormat": "{{endpoint}}",
"range": true,
"refId": "B",
"useBackend": false
}
],
"title": "Request Average Duration Top 10",
"transformations": [
{
"id": "seriesToColumns",
"options": {}
}
],
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 47
},
"id": 33,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"uid": "$datasource"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "sum by(endpoint) (rate(flask_http_request_duration_seconds_count{instance=~\"$instance\", status=\"200\"}[2m]))",
"format": "time_series",
"fullMetaSearch": false,
"includeNullMetadata": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ path }}",
"range": true,
"refId": "A",
"useBackend": false
}
],
"title": "Requests per second by endpoint",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 47
},
"id": 34,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max",
"min"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"uid": "$datasource"
},
"disableTextWrap": false,
"editorMode": "code",
"expr": "topk(10, histogram_quantile(0.90, sum by(le, endpoint) (rate(flask_http_request_duration_seconds_bucket{instance=~\"$instance\", status=\"200\"}[2m]))))",
"format": "time_series",
"fullMetaSearch": false,
"hide": false,
"includeNullMetadata": true,
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{ path }}",
"range": true,
"refId": "C",
"useBackend": false
}
],
"title": "Request duration [s] by endpoint Top 10 slowly",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "bars",
"fillOpacity": 100,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "normal"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "HTTP 500"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#bf1b00",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 55
},
"id": 30,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"uid": "$datasource"
},
"editorMode": "code",
"expr": "increase(flask_http_request_total{instance=~\"$instance\"}[2m])",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "HTTP {{ status }}",
"range": true,
"refId": "A"
}
],
"title": "Total requests per minute",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 55
},
"id": 18,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "right",
"showLegend": true
},
"tooltip": {
"hideZeros": false,
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "rate(flask_http_request_total{instance=~\"$instance\",method=\"POST\"}[2m])",
"interval": "",
"legendFormat": "POST",
"range": true,
"refId": "A"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "rate(flask_http_request_total{instance=~\"$instance\",method=\"GET\"}[2m])",
"hide": false,
"interval": "",
"legendFormat": "GET",
"range": true,
"refId": "B"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"editorMode": "code",
"exemplar": true,
"expr": "rate(flask_http_request_total{instance=~\"$instance\",method=\"PUT\"}[2m])",
"hide": false,
"interval": "",
"legendFormat": "PUT",
"range": true,
"refId": "C"
}
],
"title": "Total requests per minute by method",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "errors"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#c15c17",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 62
},
"id": 31,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"uid": "$datasource"
},
"editorMode": "code",
"expr": "sum(rate(flask_http_request_duration_seconds_count{instance=~\"$instance\",status=\"200\"}[2m]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "requests",
"range": true,
"refId": "A"
}
],
"title": "Requests per second",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "${datasource}"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 10,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "never",
"showValues": false,
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": 0
},
{
"color": "red",
"value": 80
}
]
},
"unit": "short"
},
"overrides": [
{
"matcher": {
"id": "byName",
"options": "errors"
},
"properties": [
{
"id": "color",
"value": {
"fixedColor": "#c15c17",
"mode": "fixed"
}
}
]
}
]
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 62
},
"id": 32,
"options": {
"legend": {
"calcs": [
"mean",
"lastNotNull",
"max"
],
"displayMode": "list",
"placement": "bottom",
"showLegend": false
},
"tooltip": {
"hideZeros": false,
"mode": "multi",
"sort": "none"
}
},
"pluginVersion": "12.3.0",
"targets": [
{
"datasource": {
"uid": "$datasource"
},
"editorMode": "code",
"expr": "sum(rate(flask_http_request_duration_seconds_count{instance=~\"$instance\",status!=\"200\"}[2m]))",
"format": "time_series",
"interval": "",
"intervalFactor": 1,
"legendFormat": "errors",
"range": true,
"refId": "A"
}
],
"title": "Errors per second",
"type": "timeseries"
}
],
"preload": false,
"refresh": "5s",
"schemaVersion": 42,
"tags": [],
"templating": {
"list": [
{
"current": {},
"definition": "label_values({job=~\"vmapp|vmapp-https\"},instance)",
"label": "instance",
"name": "instance",
"options": [],
"query": {
"qryType": 1,
"query": "label_values({job=~\"vmapp|vmapp-https\"},instance)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"type": "query"
},
{
"current": {},
"definition": "label_values({instance=\"$instance\"},label)",
"description": "",
"label": "app",
"name": "app",
"options": [],
"query": {
"qryType": 1,
"query": "label_values({instance=\"$instance\"},label)",
"refId": "PrometheusVariableQueryEditor-VariableQuery"
},
"refresh": 1,
"regex": "",
"type": "query"
},
{
"current": {
"text": "",
"value": "${DS_PROMETHEUS}",
"selected": true
},
"label": "Datasource",
"name": "datasource",
"options": [],
"query": "prometheus",
"refresh": 1,
"regex": "",
"type": "datasource"
}
]
},
"time": {
"from": "now-7d",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "[app] vismind",
"uid": "vismind",
"version": 9,
"weekStart": "",
"id": null
}
Поскольку приложение функционирует с использованием нескольких gunicorn workers,
динамические метрики могут обновляться с задержкой. Данные метрик хранятся в Redis по ключу metrics:{env окружения}