diff --git a/backend/app/core/metrics/__init__.py b/backend/app/core/metrics/__init__.py index 068610db..b4e5d53c 100644 --- a/backend/app/core/metrics/__init__.py +++ b/backend/app/core/metrics/__init__.py @@ -1,6 +1,6 @@ from app.core.metrics.base import BaseMetrics from app.core.metrics.connections import ConnectionMetrics -from app.core.metrics.database import DatabaseMetrics +from app.core.metrics.database import IdempotencyMetrics from app.core.metrics.dlq import DLQMetrics from app.core.metrics.events import EventMetrics from app.core.metrics.execution import ExecutionMetrics @@ -14,7 +14,7 @@ __all__ = [ "BaseMetrics", "ConnectionMetrics", - "DatabaseMetrics", + "IdempotencyMetrics", "DLQMetrics", "EventMetrics", "ExecutionMetrics", diff --git a/backend/app/core/metrics/database.py b/backend/app/core/metrics/database.py index 4590309c..a5498299 100644 --- a/backend/app/core/metrics/database.py +++ b/backend/app/core/metrics/database.py @@ -1,29 +1,10 @@ from app.core.metrics.base import BaseMetrics -class DatabaseMetrics(BaseMetrics): - """Metrics for database operations.""" +class IdempotencyMetrics(BaseMetrics): + """Metrics for idempotency operations.""" def _create_instruments(self) -> None: - # MongoDB operation metrics - self.mongodb_event_operations = self._meter.create_counter( - name="mongodb.event.operations.total", description="Total MongoDB operations for events", unit="1" - ) - - self.mongodb_event_query_duration = self._meter.create_histogram( - name="mongodb.event.query.duration", description="Duration of MongoDB event queries in seconds", unit="s" - ) - - # Event store specific metrics - self.event_store_operations = self._meter.create_counter( - name="event.store.operations.total", description="Total event store operations", unit="1" - ) - - self.event_store_failures = self._meter.create_counter( - name="event.store.failures.total", description="Total event store operation failures", unit="1" - ) - - # Idempotency metrics self.idempotency_cache_hits = self._meter.create_counter( name="idempotency.cache.hits.total", description="Total idempotency cache hits", unit="1" ) @@ -46,37 +27,6 @@ def _create_instruments(self) -> None: name="idempotency.keys.active", description="Number of active idempotency keys", unit="1" ) - # Database connection metrics - self.database_connections_active = self._meter.create_up_down_counter( - name="database.connections.active", description="Number of active database connections", unit="1" - ) - - self.database_connection_errors = self._meter.create_counter( - name="database.connection.errors.total", description="Total database connection errors", unit="1" - ) - - def record_mongodb_operation(self, operation: str, status: str) -> None: - self.mongodb_event_operations.add(1, attributes={"operation": operation, "status": status}) - - def record_mongodb_query_duration(self, duration_seconds: float, operation: str) -> None: - self.mongodb_event_query_duration.record(duration_seconds, attributes={"operation": operation}) - - def record_event_store_duration(self, duration_seconds: float, operation: str, collection: str) -> None: - self.mongodb_event_query_duration.record( - duration_seconds, attributes={"operation": f"store_{operation}", "collection": collection} - ) - - # Also record in event store specific counter - self.event_store_operations.add(1, attributes={"operation": operation, "collection": collection}) - - def record_event_query_duration(self, duration_seconds: float, operation: str, collection: str) -> None: - self.mongodb_event_query_duration.record( - duration_seconds, attributes={"operation": f"query_{operation}", "collection": collection} - ) - - def record_event_store_failed(self, event_type: str, error_type: str) -> None: - self.event_store_failures.add(1, attributes={"event_type": event_type, "error_type": error_type}) - def record_idempotency_cache_hit(self, event_type: str, operation: str) -> None: self.idempotency_cache_hits.add(1, attributes={"event_type": event_type, "operation": operation}) @@ -97,16 +47,5 @@ def decrement_idempotency_keys(self, prefix: str) -> None: """Decrement active idempotency keys count when a key is removed.""" self.idempotency_keys_active.add(-1, attributes={"key_prefix": prefix}) - def record_idempotent_event_processed(self, event_type: str, result: str) -> None: - self.event_store_operations.add( - 1, attributes={"operation": "idempotent_process", "event_type": event_type, "result": result} - ) - def record_idempotent_processing_duration(self, duration_seconds: float, event_type: str) -> None: self.idempotency_processing_duration.record(duration_seconds, attributes={"event_type": event_type}) - - def update_database_connections(self, delta: int) -> None: - self.database_connections_active.add(delta) - - def record_database_connection_error(self, error_type: str) -> None: - self.database_connection_errors.add(1, attributes={"error_type": error_type}) diff --git a/backend/app/core/providers.py b/backend/app/core/providers.py index 99f6706e..05cdcab1 100644 --- a/backend/app/core/providers.py +++ b/backend/app/core/providers.py @@ -13,10 +13,10 @@ from app.core.logging import setup_logger from app.core.metrics import ( ConnectionMetrics, - DatabaseMetrics, DLQMetrics, EventMetrics, ExecutionMetrics, + IdempotencyMetrics, KubernetesMetrics, NotificationMetrics, QueueMetrics, @@ -184,9 +184,9 @@ def get_idempotency_manager( self, repo: RedisIdempotencyRepository, logger: structlog.stdlib.BoundLogger, - database_metrics: DatabaseMetrics, + idempotency_metrics: IdempotencyMetrics, ) -> IdempotencyManager: - return IdempotencyManager(IdempotencyConfig(), repo, logger, database_metrics) + return IdempotencyManager(IdempotencyConfig(), repo, logger, idempotency_metrics) class DLQProvider(Provider): @@ -251,8 +251,8 @@ def get_execution_metrics(self, settings: Settings) -> ExecutionMetrics: return ExecutionMetrics(settings) @provide - def get_database_metrics(self, settings: Settings) -> DatabaseMetrics: - return DatabaseMetrics(settings) + def get_idempotency_metrics(self, settings: Settings) -> IdempotencyMetrics: + return IdempotencyMetrics(settings) @provide def get_kubernetes_metrics(self, settings: Settings) -> KubernetesMetrics: diff --git a/backend/app/dlq/manager.py b/backend/app/dlq/manager.py index ab65c1ac..94e8836e 100644 --- a/backend/app/dlq/manager.py +++ b/backend/app/dlq/manager.py @@ -1,3 +1,4 @@ +import time from datetime import datetime, timezone from typing import Callable @@ -78,6 +79,7 @@ async def process_monitoring_cycle(self) -> None: async def handle_message(self, message: DLQMessage) -> None: """Process a single DLQ message: filter -> store -> decide retry/discard.""" + start = time.monotonic() for filter_func in self._filters: if not filter_func(message): self.logger.info("Message filtered out", event_id=message.event.event_id) @@ -86,6 +88,9 @@ async def handle_message(self, message: DLQMessage) -> None: message.status = DLQMessageStatus.PENDING message.last_updated = datetime.now(timezone.utc) await self.repository.save_message(message) + self.metrics.record_dlq_message_received(message.original_topic, message.event.event_type) + age_seconds = (datetime.now(timezone.utc) - message.failed_at).total_seconds() + self.metrics.record_dlq_message_age(age_seconds) await self._broker.publish( DLQMessageReceivedEvent( @@ -120,11 +125,14 @@ async def handle_message(self, message: DLQMessage) -> None: if retry_policy.strategy == RetryStrategy.IMMEDIATE: await self.retry_message(message) + self.metrics.record_dlq_processing_duration(time.monotonic() - start, "handle") + async def retry_message(self, message: DLQMessage) -> None: """Retry a DLQ message by republishing to the original topic. FastStream handles JSON serialization of Pydantic models natively. """ + start = time.monotonic() await self._broker.publish( message=message.event, topic=message.original_topic, @@ -158,10 +166,12 @@ async def retry_message(self, message: DLQMessage) -> None: ), topic=EventType.DLQ_MESSAGE_RETRIED, ) + self.metrics.record_dlq_processing_duration(time.monotonic() - start, "retry") self.logger.info("Successfully retried message", event_id=message.event.event_id) async def discard_message(self, message: DLQMessage, reason: str) -> None: """Discard a DLQ message, updating status and emitting an event.""" + start = time.monotonic() self.metrics.record_dlq_message_discarded(message.original_topic, message.event.event_type, reason) await self.repository.update_status( @@ -188,6 +198,7 @@ async def discard_message(self, message: DLQMessage, reason: str) -> None: ), topic=EventType.DLQ_MESSAGE_DISCARDED, ) + self.metrics.record_dlq_processing_duration(time.monotonic() - start, "discard") self.logger.warning("Discarded message", event_id=message.event.event_id, reason=reason) async def process_due_retries(self) -> int: diff --git a/backend/app/events/handlers.py b/backend/app/events/handlers.py index 81aedf3b..0a2e732a 100644 --- a/backend/app/events/handlers.py +++ b/backend/app/events/handlers.py @@ -6,6 +6,7 @@ from faststream import AckPolicy from faststream.kafka import KafkaBroker +from app.core.metrics import EventMetrics from app.domain.enums import EventType from app.domain.events import ( CreatePodCommandEvent, @@ -30,6 +31,24 @@ _sse_field_names: frozenset[str] = frozenset(f.name for f in dataclasses.fields(SSEExecutionEventData)) +async def _track_consumed( + metrics: EventMetrics, event: DomainEvent, consumer_group: str, coro: Awaitable[None], +) -> None: + """Record consumption metric, await *coro*, and record failure metric on error.""" + metrics.record_kafka_message_consumed(topic=event.event_type, consumer_group=consumer_group) + try: + await coro + except Exception as e: + metrics.record_events_processing_failed( + topic=event.event_type, event_type=event.event_type, + consumer_group=consumer_group, error_type=type(e).__name__, + ) + metrics.record_kafka_consumption_error( + topic=event.event_type, consumer_group=consumer_group, error_type=type(e).__name__, + ) + raise + + # --8<-- [start:with_idempotency] async def with_idempotency( event: DomainEvent, @@ -68,10 +87,10 @@ async def on_create_pod( worker: FromDishka[KubernetesWorker], idem: FromDishka[IdempotencyManager], logger: FromDishka[structlog.stdlib.BoundLogger], + event_metrics: FromDishka[EventMetrics], ) -> None: - await with_idempotency( - body, worker.handle_create_pod_command, idem, KeyStrategy.CONTENT_HASH, 3600, logger, - ) + await _track_consumed(event_metrics, body, "k8s-worker", + with_idempotency(body, worker.handle_create_pod_command, idem, KeyStrategy.CONTENT_HASH, 3600, logger)) @broker.subscriber( EventType.DELETE_POD_COMMAND, @@ -83,10 +102,10 @@ async def on_delete_pod( worker: FromDishka[KubernetesWorker], idem: FromDishka[IdempotencyManager], logger: FromDishka[structlog.stdlib.BoundLogger], + event_metrics: FromDishka[EventMetrics], ) -> None: - await with_idempotency( - body, worker.handle_delete_pod_command, idem, KeyStrategy.CONTENT_HASH, 3600, logger, - ) + await _track_consumed(event_metrics, body, "k8s-worker", + with_idempotency(body, worker.handle_delete_pod_command, idem, KeyStrategy.CONTENT_HASH, 3600, logger)) def register_result_processor_subscriber(broker: KafkaBroker) -> None: @@ -102,10 +121,10 @@ async def on_execution_completed( processor: FromDishka[ResultProcessor], idem: FromDishka[IdempotencyManager], logger: FromDishka[structlog.stdlib.BoundLogger], + event_metrics: FromDishka[EventMetrics], ) -> None: - await with_idempotency( - body, processor.handle_execution_completed, idem, KeyStrategy.CONTENT_HASH, 7200, logger, - ) + await _track_consumed(event_metrics, body, "result-processor", + with_idempotency(body, processor.handle_execution_completed, idem, KeyStrategy.CONTENT_HASH, 7200, logger)) @broker.subscriber( EventType.EXECUTION_FAILED, @@ -119,10 +138,10 @@ async def on_execution_failed( processor: FromDishka[ResultProcessor], idem: FromDishka[IdempotencyManager], logger: FromDishka[structlog.stdlib.BoundLogger], + event_metrics: FromDishka[EventMetrics], ) -> None: - await with_idempotency( - body, processor.handle_execution_failed, idem, KeyStrategy.CONTENT_HASH, 7200, logger, - ) + await _track_consumed(event_metrics, body, "result-processor", + with_idempotency(body, processor.handle_execution_failed, idem, KeyStrategy.CONTENT_HASH, 7200, logger)) @broker.subscriber( EventType.EXECUTION_TIMEOUT, @@ -136,10 +155,10 @@ async def on_execution_timeout( processor: FromDishka[ResultProcessor], idem: FromDishka[IdempotencyManager], logger: FromDishka[structlog.stdlib.BoundLogger], + event_metrics: FromDishka[EventMetrics], ) -> None: - await with_idempotency( - body, processor.handle_execution_timeout, idem, KeyStrategy.CONTENT_HASH, 7200, logger, - ) + await _track_consumed(event_metrics, body, "result-processor", + with_idempotency(body, processor.handle_execution_timeout, idem, KeyStrategy.CONTENT_HASH, 7200, logger)) def register_saga_subscriber(broker: KafkaBroker) -> None: @@ -153,8 +172,10 @@ def register_saga_subscriber(broker: KafkaBroker) -> None: async def on_execution_requested( body: ExecutionRequestedEvent, orchestrator: FromDishka[SagaOrchestrator], + event_metrics: FromDishka[EventMetrics], ) -> None: - await orchestrator.handle_execution_requested(body) + await _track_consumed(event_metrics, body, "saga-orchestrator", + orchestrator.handle_execution_requested(body)) @broker.subscriber( EventType.EXECUTION_COMPLETED, @@ -164,8 +185,10 @@ async def on_execution_requested( async def on_execution_completed( body: ExecutionCompletedEvent, orchestrator: FromDishka[SagaOrchestrator], + event_metrics: FromDishka[EventMetrics], ) -> None: - await orchestrator.handle_execution_completed(body) + await _track_consumed(event_metrics, body, "saga-orchestrator", + orchestrator.handle_execution_completed(body)) @broker.subscriber( EventType.EXECUTION_FAILED, @@ -175,8 +198,10 @@ async def on_execution_completed( async def on_execution_failed( body: ExecutionFailedEvent, orchestrator: FromDishka[SagaOrchestrator], + event_metrics: FromDishka[EventMetrics], ) -> None: - await orchestrator.handle_execution_failed(body) + await _track_consumed(event_metrics, body, "saga-orchestrator", + orchestrator.handle_execution_failed(body)) @broker.subscriber( EventType.EXECUTION_TIMEOUT, @@ -186,8 +211,10 @@ async def on_execution_failed( async def on_execution_timeout( body: ExecutionTimeoutEvent, orchestrator: FromDishka[SagaOrchestrator], + event_metrics: FromDishka[EventMetrics], ) -> None: - await orchestrator.handle_execution_timeout(body) + await _track_consumed(event_metrics, body, "saga-orchestrator", + orchestrator.handle_execution_timeout(body)) @broker.subscriber( EventType.EXECUTION_CANCELLED, @@ -197,8 +224,10 @@ async def on_execution_timeout( async def on_execution_cancelled( body: ExecutionCancelledEvent, orchestrator: FromDishka[SagaOrchestrator], + event_metrics: FromDishka[EventMetrics], ) -> None: - await orchestrator.handle_execution_cancelled(body) + await _track_consumed(event_metrics, body, "saga-orchestrator", + orchestrator.handle_execution_cancelled(body)) _SSE_EVENT_TYPES = [ @@ -233,7 +262,9 @@ def register_sse_subscriber(broker: KafkaBroker, settings: Settings) -> None: async def on_sse_event( body: DomainEvent, sse_bus: FromDishka[SSERedisBus], + event_metrics: FromDishka[EventMetrics], ) -> None: + event_metrics.record_kafka_message_consumed(topic=body.event_type, consumer_group="sse-bridge-pool") execution_id = getattr(body, "execution_id", None) if execution_id: sse_data = SSEExecutionEventData(**{ @@ -253,8 +284,10 @@ def register_notification_subscriber(broker: KafkaBroker) -> None: async def on_execution_completed( body: ExecutionCompletedEvent, service: FromDishka[NotificationService], + event_metrics: FromDishka[EventMetrics], ) -> None: - await service.handle_execution_completed(body) + await _track_consumed(event_metrics, body, "notification-service", + service.handle_execution_completed(body)) @broker.subscriber( EventType.EXECUTION_FAILED, @@ -266,8 +299,10 @@ async def on_execution_completed( async def on_execution_failed( body: ExecutionFailedEvent, service: FromDishka[NotificationService], + event_metrics: FromDishka[EventMetrics], ) -> None: - await service.handle_execution_failed(body) + await _track_consumed(event_metrics, body, "notification-service", + service.handle_execution_failed(body)) @broker.subscriber( EventType.EXECUTION_TIMEOUT, @@ -279,7 +314,9 @@ async def on_execution_failed( async def on_execution_timeout( body: ExecutionTimeoutEvent, service: FromDishka[NotificationService], + event_metrics: FromDishka[EventMetrics], ) -> None: - await service.handle_execution_timeout(body) + await _track_consumed(event_metrics, body, "notification-service", + service.handle_execution_timeout(body)) diff --git a/backend/app/services/event_replay/replay_service.py b/backend/app/services/event_replay/replay_service.py index 5dc65db6..f54058cf 100644 --- a/backend/app/services/event_replay/replay_service.py +++ b/backend/app/services/event_replay/replay_service.py @@ -1,5 +1,6 @@ import asyncio import json +import time from collections.abc import AsyncIterator from datetime import datetime, timedelta, timezone from uuid import uuid4 @@ -92,9 +93,11 @@ async def start_session(self, session_id: str) -> ReplayOperationResult: misfire_grace_time=None, ) + previous_status = session.status session.status = ReplayStatus.RUNNING session.started_at = datetime.now(timezone.utc) self._metrics.increment_active_replays() + self._metrics.record_status_change(session_id, previous_status, ReplayStatus.RUNNING) self._metrics.record_speed_multiplier(session.config.speed_multiplier, session.config.replay_type) await self._repository.update_session_status(session_id, ReplayStatus.RUNNING) return ReplayOperationResult( @@ -105,7 +108,9 @@ async def pause_session(self, session_id: str) -> ReplayOperationResult: session = self.get_session(session_id) if session.status != ReplayStatus.RUNNING: raise ReplayOperationError(session_id, "pause", "Session is not running") + previous_status = session.status session.status = ReplayStatus.PAUSED + self._metrics.record_status_change(session_id, previous_status, ReplayStatus.PAUSED) scheduler = self._schedulers.get(session_id) if scheduler: scheduler.remove_all_jobs() @@ -118,7 +123,9 @@ async def resume_session(self, session_id: str) -> ReplayOperationResult: session = self.get_session(session_id) if session.status != ReplayStatus.PAUSED: raise ReplayOperationError(session_id, "resume", "Session is not paused") + previous_status = session.status session.status = ReplayStatus.RUNNING + self._metrics.record_status_change(session_id, previous_status, ReplayStatus.RUNNING) scheduler = self._schedulers.get(session_id) if scheduler: scheduler.add_job( @@ -137,9 +144,7 @@ async def resume_session(self, session_id: str) -> ReplayOperationResult: async def cancel_session(self, session_id: str) -> ReplayOperationResult: session = self.get_session(session_id) - session.status = ReplayStatus.CANCELLED await self._finalize_session(session, ReplayStatus.CANCELLED) - await self._repository.update_session_status(session_id, ReplayStatus.CANCELLED) return ReplayOperationResult( session_id=session_id, status=ReplayStatus.CANCELLED, message="Replay session cancelled" ) @@ -188,10 +193,17 @@ async def _dispatch_next(self, session: ReplaySessionState) -> None: await self._finalize_session(session, ReplayStatus.COMPLETED) return + buf = self._event_buffers.get(session.session_id, []) + idx = self._buffer_indices.get(session.session_id, 0) + self._metrics.update_replay_queue_size(session.session_id, len(buf) - idx) + success = False + t0 = time.monotonic() try: success = await self._replay_event(session, event) except Exception as e: + processing_time = time.monotonic() - t0 + self._metrics.record_event_processing_time(processing_time, event.event_type) session.errors.append( ReplayError(timestamp=datetime.now(timezone.utc), event_id=str(event.event_id), error=str(e)) ) @@ -199,6 +211,9 @@ async def _dispatch_next(self, session: ReplaySessionState) -> None: session.failed_events += 1 await self._finalize_session(session, ReplayStatus.FAILED) return + else: + processing_time = time.monotonic() - t0 + self._metrics.record_event_processing_time(processing_time, event.event_type) if success: session.replayed_events += 1 @@ -254,17 +269,25 @@ async def _load_next_batch(self, session_id: str) -> bool: batch = await batch_iter.__anext__() self._event_buffers[session_id] = batch self._buffer_indices[session_id] = 0 + session = self._sessions.get(session_id) + if session: + self._metrics.record_batch_size(len(batch), session.config.replay_type) + self._metrics.update_replay_queue_size(session_id, len(batch)) return True except StopAsyncIteration: return False async def _finalize_session(self, session: ReplaySessionState, final_status: ReplayStatus) -> None: + previous_status = session.status session.status = final_status session.completed_at = datetime.now(timezone.utc) if final_status == ReplayStatus.COMPLETED and session.started_at: duration = (session.completed_at - session.started_at).total_seconds() - self._metrics.record_replay_duration(duration, session.config.replay_type) + total_events = session.replayed_events + session.failed_events + self._metrics.record_replay_duration(duration, session.config.replay_type, total_events=total_events) + self._metrics.record_status_change(session.session_id, previous_status, final_status) self._metrics.decrement_active_replays() + self._metrics.update_replay_queue_size(session.session_id, 0) await self._update_session_in_db(session) scheduler = self._schedulers.pop(session.session_id, None) if scheduler: diff --git a/backend/app/services/idempotency/idempotency_manager.py b/backend/app/services/idempotency/idempotency_manager.py index 353d94f7..29e3fcbc 100644 --- a/backend/app/services/idempotency/idempotency_manager.py +++ b/backend/app/services/idempotency/idempotency_manager.py @@ -1,12 +1,13 @@ import hashlib import json +import time from datetime import datetime, timedelta, timezone import structlog from pydantic import BaseModel from pymongo.errors import DuplicateKeyError -from app.core.metrics import DatabaseMetrics +from app.core.metrics import IdempotencyMetrics from app.domain.enums import EventType from app.domain.events import BaseEvent from app.domain.idempotency import IdempotencyRecord, IdempotencyStatus, KeyStrategy @@ -40,10 +41,10 @@ def __init__( config: IdempotencyConfig, repository: RedisIdempotencyRepository, logger: structlog.stdlib.BoundLogger, - database_metrics: DatabaseMetrics, + idempotency_metrics: IdempotencyMetrics, ) -> None: self.config = config - self.metrics = database_metrics + self.metrics = idempotency_metrics self._repo = repository self.logger = logger self.logger.info("Idempotency manager initialized") @@ -78,16 +79,21 @@ async def check_and_reserve( ttl_seconds: int | None = None, fields: set[str] | None = None, ) -> IdempotencyResult: + start = time.monotonic() full_key = self._generate_key(event, key_strategy, custom_key, fields) ttl = ttl_seconds or self.config.default_ttl_seconds existing = await self._repo.find_by_key(full_key) if existing: self.metrics.record_idempotency_cache_hit(event.event_type, "check_and_reserve") - return await self._handle_existing_key(existing, full_key, event.event_type) + result = await self._handle_existing_key(existing, full_key, event.event_type) + self.metrics.record_idempotency_processing_duration(time.monotonic() - start, "check_and_reserve") + return result self.metrics.record_idempotency_cache_miss(event.event_type, "check_and_reserve") - return await self._create_new_key(full_key, event, ttl) + result = await self._create_new_key(full_key, event, ttl) + self.metrics.record_idempotency_processing_duration(time.monotonic() - start, "check_and_reserve") + return result async def _handle_existing_key( self, diff --git a/backend/grafana/provisioning/alerting/alerting.yml b/backend/grafana/provisioning/alerting/alerting.yml index c23b337e..c4d41d72 100644 --- a/backend/grafana/provisioning/alerting/alerting.yml +++ b/backend/grafana/provisioning/alerting/alerting.yml @@ -22,6 +22,7 @@ groups: - orgId: 1 name: infrastructure-alerts folder: 'Integr8sCode' + folderUid: 'integr8scode' interval: 1m rules: - uid: host-memory-warning @@ -34,7 +35,7 @@ groups: to: 0 datasourceUid: victoria-metrics model: - expr: system_memory_utilization{state="used"} * 100 + expr: avg_over_time(system_memory_utilization{state="used"}[5m]) * 100 intervalMs: 15000 maxDataPoints: 43200 - refId: C @@ -65,7 +66,7 @@ groups: to: 0 datasourceUid: victoria-metrics model: - expr: system_memory_utilization{state="used"} * 100 + expr: avg_over_time(system_memory_utilization{state="used"}[5m]) * 100 intervalMs: 15000 maxDataPoints: 43200 - refId: C diff --git a/backend/grafana/provisioning/dashboards/coordinator-execution.json b/backend/grafana/provisioning/dashboards/coordinator-execution.json index 373496ac..f4d84c3e 100644 --- a/backend/grafana/provisioning/dashboards/coordinator-execution.json +++ b/backend/grafana/provisioning/dashboards/coordinator-execution.json @@ -1,6 +1,16 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, "description": "Queue & Execution", "editable": true, @@ -24,6 +34,7 @@ "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -37,23 +48,80 @@ }, { "datasource": "Victoria Metrics", + "description": "P95 queue wait time broken down by execution priority", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "s" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, "y": 1 }, "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(queue_wait_time_seconds_bucket[5m])) by (le, priority))", "legendFormat": "p95 {{priority}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Queue Wait Time (by Priority)", @@ -61,28 +129,86 @@ }, { "datasource": "Victoria Metrics", + "description": "Number of pending and actively running executions", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, "y": 1 }, "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "queue_depth", "legendFormat": "Depth", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "queue_active", "legendFormat": "Active", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Queue Depth & Active", @@ -90,22 +216,61 @@ }, { "datasource": "Victoria Metrics", + "description": "Current number of actively running executions", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, "unit": "short" } }, "gridPos": { "h": 4, - "w": 6, + "w": 12, "x": 0, - "y": 7 + "y": 9 }, "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "queue_active", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Active Executions", @@ -113,22 +278,61 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of executions scheduled in the last 5 minutes", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "red", + "value": 100 + } + ] + }, "unit": "short" } }, "gridPos": { "h": 4, - "w": 6, - "x": 6, - "y": 7 + "w": 12, + "x": 12, + "y": 9 }, "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "rate(queue_schedule_total[5m])", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Scheduled (5m)", @@ -136,11 +340,12 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 11 + "y": 13 }, "id": 101, "panels": [], @@ -149,23 +354,80 @@ }, { "datasource": "Victoria Metrics", + "description": "P95 queue wait time for all executions", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "s" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 14 }, "id": 5, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(queue_wait_time_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Queue Wait Time (Execution)", @@ -173,28 +435,86 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of executions being assigned from queue vs enqueued", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 14 }, "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "rate(queue_schedule_total[5m])", "legendFormat": "Assigned", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(queue_enqueue_total[5m])", "legendFormat": "Queued", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Assigned & Queued", @@ -202,11 +522,12 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 18 + "y": 22 }, "id": 102, "panels": [], @@ -215,28 +536,86 @@ }, { "datasource": "Victoria Metrics", + "description": "Script memory usage percentiles (MiB)", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "mbytes" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 19 + "y": 23 }, "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(script_memory_usage_MiB_bucket[5m])) by (le))", "legendFormat": "p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "histogram_quantile(0.95, sum(rate(script_memory_usage_MiB_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Memory Usage", @@ -244,28 +623,86 @@ }, { "datasource": "Victoria Metrics", + "description": "Script memory utilization as percentage of limit", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "percent" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 19 + "y": 23 }, "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(script_memory_utilization_percent_bucket[5m])) by (le))", "legendFormat": "p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "histogram_quantile(0.95, sum(rate(script_memory_utilization_percent_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Memory Utilization", @@ -286,9 +723,23 @@ "from": "now-3h", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, "timezone": "", "title": "Queue & Execution", "uid": "coordinator-execution", - "version": 1 + "version": 1, + "weekStart": "" } diff --git a/backend/grafana/provisioning/dashboards/dlq-monitoring.json b/backend/grafana/provisioning/dashboards/dlq-monitoring.json index c949c108..e595100d 100644 --- a/backend/grafana/provisioning/dashboards/dlq-monitoring.json +++ b/backend/grafana/provisioning/dashboards/dlq-monitoring.json @@ -99,7 +99,8 @@ "targets": [ { "expr": "sum(dlq_queue_size) or vector(0)", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Total DLQ Size", @@ -174,9 +175,10 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "rate(dlq_messages_received_total[5m])", + "expr": "rate(dlq_messages_received_total[5m]) or vector(0)", "legendFormat": "Incoming Rate", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "DLQ Incoming Rate", @@ -190,7 +192,17 @@ "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "N/A", + "color": "green" + } + } + } + ], "max": 100, "min": 0, "thresholds": { @@ -236,8 +248,9 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "(sum(rate(dlq_messages_retried_total{result=\"success\"}[5m])) / (sum(rate(dlq_messages_retried_total[5m])) + 0.001)) * 100", - "refId": "A" + "expr": "((sum(rate(dlq_messages_retried_total{result=\"success\"}[5m])) or vector(0)) / ((sum(rate(dlq_messages_retried_total[5m])) or vector(0)) + 0.001)) * 100", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Retry Success Rate", @@ -290,8 +303,9 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "sum(increase(dlq_messages_discarded_total[24h]))", - "refId": "A" + "expr": "sum(increase(dlq_messages_discarded_total[24h])) or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Messages Discarded (24h)", @@ -366,7 +380,8 @@ { "expr": "sum by (original_topic) (dlq_queue_size)", "legendFormat": "{{original_topic}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "DLQ Size by Topic", @@ -443,19 +458,22 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "sum(rate(dlq_messages_received_total[5m]))", + "expr": "sum(rate(dlq_messages_received_total[5m])) or vector(0)", "legendFormat": "Received", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { - "expr": "sum(rate(dlq_messages_retried_total[5m]))", + "expr": "sum(rate(dlq_messages_retried_total[5m])) or vector(0)", "legendFormat": "Retried", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" }, { - "expr": "sum(rate(dlq_messages_discarded_total[5m]))", + "expr": "sum(rate(dlq_messages_discarded_total[5m])) or vector(0)", "legendFormat": "Discarded", - "refId": "C" + "refId": "C", + "datasource": "Victoria Metrics" } ], "title": "DLQ Message Flow", @@ -517,7 +535,8 @@ { "expr": "sum by (reason) (increase(dlq_messages_discarded_total[24h]))", "legendFormat": "{{reason}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Discard Reasons (24h)", @@ -617,19 +636,22 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "histogram_quantile(0.50, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { - "expr": "histogram_quantile(0.90, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p90", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" }, { - "expr": "histogram_quantile(0.99, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p99", - "refId": "C" + "refId": "C", + "datasource": "Victoria Metrics" } ], "title": "Message Age in DLQ", @@ -709,12 +731,14 @@ { "expr": "histogram_quantile(0.50, sum(rate(dlq_processing_duration_seconds_bucket[5m])) by (le, operation))", "legendFormat": "p50 {{operation}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "histogram_quantile(0.95, sum(rate(dlq_processing_duration_seconds_bucket[5m])) by (le, operation))", "legendFormat": "p95 {{operation}}", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "DLQ Processing Duration", @@ -785,8 +809,8 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 8, + "w": 24, + "x": 0, "y": 26 }, "id": 11, @@ -807,7 +831,8 @@ { "expr": "sum by (result) (increase(dlq_messages_retried_total[1h]))", "legendFormat": "{{result}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Retry Results (1h)", @@ -903,7 +928,8 @@ { "expr": "sum by (error_type) (rate(dlq_processing_errors_total[5m]))", "legendFormat": "{{error_type}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "DLQ Processing Errors", @@ -964,7 +990,8 @@ "expr": "topk(10, sum by (original_topic, event_type, error_type) (increase(dlq_processing_errors_total[1h])))", "format": "table", "instant": true, - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Top Error Sources (1h)", @@ -1048,7 +1075,8 @@ "expr": "sum by (original_topic, event_type) (increase(dlq_messages_received_total[24h]))", "format": "table", "instant": true, - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "DLQ Messages by Topic & Type (24h)", @@ -1143,7 +1171,8 @@ { "expr": "sum by (original_topic) (dlq_queue_size)", "legendFormat": "{{original_topic}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Queue Size by Topic", @@ -1217,8 +1246,9 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "sum(dlq_queue_size) > 100", - "refId": "A" + "expr": "sum(dlq_queue_size) or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "High DLQ Size Alert", @@ -1232,7 +1262,17 @@ "color": { "mode": "thresholds" }, - "mappings": [], + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "N/A", + "color": "green" + } + } + } + ], "max": 100, "min": 0, "thresholds": { @@ -1280,8 +1320,9 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "(sum(rate(dlq_messages_retried_total{result=\"success\"}[5m])) / (sum(rate(dlq_messages_retried_total[5m])) + 0.001)) * 100", - "refId": "A" + "expr": "((sum(rate(dlq_messages_retried_total{result=\"success\"}[5m])) or vector(0)) / ((sum(rate(dlq_messages_retried_total[5m])) or vector(0)) + 0.001)) * 100", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Low Success Rate Alert", @@ -1341,8 +1382,9 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "sum(rate(dlq_messages_discarded_total[1m])) * 60", - "refId": "A" + "expr": "(sum(rate(dlq_messages_discarded_total[5m])) or vector(0)) * 60", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "High Discard Rate Alert", @@ -1402,8 +1444,9 @@ "pluginVersion": "8.3.3", "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le))", - "refId": "A" + "expr": "histogram_quantile(0.99, sum(rate(dlq_message_age_seconds_bucket[5m])) by (le)) or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Old Messages Alert", diff --git a/backend/grafana/provisioning/dashboards/event-replay.json b/backend/grafana/provisioning/dashboards/event-replay.json index 1aeba408..d4f282c8 100644 --- a/backend/grafana/provisioning/dashboards/event-replay.json +++ b/backend/grafana/provisioning/dashboards/event-replay.json @@ -1,8 +1,18 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, - "description": "Event Replay", + "description": "Event Replay monitoring dashboard for tracking replay sessions, event processing, performance, and target delivery", "editable": true, "gnetId": null, "graphTooltip": 1, @@ -24,6 +34,7 @@ "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -37,8 +48,50 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of new replay sessions being created", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -49,11 +102,23 @@ "y": 1 }, "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "rate(replay_sessions_created_total[5m])", + "expr": "rate(replay_sessions_created_total[5m]) or vector(0)", "legendFormat": "Created", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Sessions Created", @@ -61,22 +126,61 @@ }, { "datasource": "Victoria Metrics", + "description": "Number of currently active replay sessions", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, "unit": "short" } }, "gridPos": { - "h": 4, + "h": 6, "w": 6, "x": 12, "y": 1 }, "id": 2, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "replay_sessions_active", - "refId": "A" + "expr": "replay_sessions_active or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Active Sessions", @@ -84,22 +188,61 @@ }, { "datasource": "Victoria Metrics", + "description": "Replay sessions broken down by current status", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 10 + } + ] + }, "unit": "short" } }, "gridPos": { - "h": 4, + "h": 6, "w": 6, "x": 18, "y": 1 }, "id": 3, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "replay_sessions_by_status", - "refId": "A" + "expr": "replay_sessions_by_status or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "By Status", @@ -107,6 +250,7 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -120,33 +264,89 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of events processed, failed, and skipped during replay", "fieldConfig": { "defaults": { - "unit": "short" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, "y": 8 }, "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "rate(replay_events_processed_total[5m])", + "expr": "rate(replay_events_processed_total[5m]) or vector(0)", "legendFormat": "Processed", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { - "expr": "rate(replay_events_failed_total[5m])", + "expr": "rate(replay_events_failed_total[5m]) or vector(0)", "legendFormat": "Failed", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" }, { - "expr": "rate(replay_events_skipped_total[5m])", + "expr": "rate(replay_events_skipped_total[5m]) or vector(0)", "legendFormat": "Skipped", - "refId": "C" + "refId": "C", + "datasource": "Victoria Metrics" } ], "title": "Events Processed", @@ -154,23 +354,77 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of replay session status transitions", "fieldConfig": { "defaults": { - "unit": "short" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, "y": 8 }, "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "rate(replay_status_changes_total[5m])", + "expr": "rate(replay_status_changes_total[5m]) or vector(0)", "legendFormat": "Changes", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Status Changes", @@ -178,11 +432,12 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 14 + "y": 16 }, "id": 102, "panels": [], @@ -191,23 +446,80 @@ }, { "datasource": "Victoria Metrics", + "description": "P95 total duration of replay sessions from start to completion", "fieldConfig": { "defaults": { - "unit": "short" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 15 + "y": 17 }, "id": 6, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(replay_duration_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(replay_duration_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Replay Duration", @@ -215,23 +527,80 @@ }, { "datasource": "Victoria Metrics", + "description": "P95 time to process individual replay events", "fieldConfig": { "defaults": { - "unit": "short" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "s" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 15 + "y": 17 }, "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(replay_event_processing_time_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(replay_event_processing_time_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Event Processing Time", @@ -239,23 +608,77 @@ }, { "datasource": "Victoria Metrics", + "description": "Median events replayed per second (throughput)", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 25 }, "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "histogram_quantile(0.50, sum(rate(replay_throughput_event_per_second_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(replay_throughput_event_per_second_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Throughput", @@ -263,23 +686,77 @@ }, { "datasource": "Victoria Metrics", + "description": "Median number of events loaded per batch", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 25 }, "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "histogram_quantile(0.50, sum(rate(replay_batch_size_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(replay_batch_size_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Batch Size", @@ -287,11 +764,12 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 33 }, "id": 103, "panels": [], @@ -300,28 +778,83 @@ }, { "datasource": "Victoria Metrics", + "description": "Replay event delivery rate by target type and errors", "fieldConfig": { "defaults": { - "unit": "short" + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ops" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 34 }, "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "rate(replay_by_target_total[5m])", + "expr": "rate(replay_by_target_total[5m]) or vector(0)", "legendFormat": "{{target}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { - "expr": "rate(replay_target_errors_total[5m])", + "expr": "rate(replay_target_errors_total[5m]) or vector(0)", "legendFormat": "Errors", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "By Target", @@ -329,28 +862,83 @@ }, { "datasource": "Victoria Metrics", + "description": "Speed multiplier and inter-event delay applied during replay", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 34 }, "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "histogram_quantile(0.50, sum(rate(replay_speed_multiplier_x_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(replay_speed_multiplier_x_bucket[5m])) by (le)) or vector(0)", "legendFormat": "Multiplier p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { - "expr": "histogram_quantile(0.50, sum(rate(replay_delay_applied_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(replay_delay_applied_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "Delay p50", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Speed Control", @@ -358,8 +946,30 @@ }, { "datasource": "Victoria Metrics", + "description": "Number of events buffered in the replay queue awaiting dispatch", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 500 + }, + { + "color": "red", + "value": 1000 + } + ] + }, "unit": "short" } }, @@ -367,13 +977,30 @@ "h": 4, "w": 12, "x": 0, - "y": 34 + "y": 42 }, "id": 12, + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { - "expr": "replay_queue_size", - "refId": "A" + "expr": "replay_queue_size or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Queue Size", @@ -384,7 +1011,8 @@ "schemaVersion": 33, "style": "dark", "tags": [ - "replay" + "replay", + "events" ], "templating": { "list": [] @@ -393,9 +1021,23 @@ "from": "now-3h", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, "timezone": "", "title": "Event Replay", "uid": "event-replay", - "version": 1 + "version": 1, + "weekStart": "" } diff --git a/backend/grafana/provisioning/dashboards/event-stream-monitoring.json b/backend/grafana/provisioning/dashboards/event-stream-monitoring.json index 93f1cd3f..9682023a 100644 --- a/backend/grafana/provisioning/dashboards/event-stream-monitoring.json +++ b/backend/grafana/provisioning/dashboards/event-stream-monitoring.json @@ -103,7 +103,7 @@ }, "gridPos": { "h": 6, - "w": 4, + "w": 6, "x": 0, "y": 1 }, @@ -170,8 +170,8 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 4, + "w": 6, + "x": 6, "y": 1 }, "id": 52, @@ -260,7 +260,7 @@ }, "gridPos": { "h": 6, - "w": 6, + "w": 12, "x": 12, "y": 1 }, @@ -662,7 +662,7 @@ "h": 8, "w": 12, "x": 0, - "y": 34 + "y": 17 }, "id": 66, "options": { @@ -759,7 +759,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 17 }, "id": 67, "options": { @@ -797,7 +797,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 25 }, "id": 68, "panels": [], @@ -855,29 +855,13 @@ }, "unit": "ops" }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Failed" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 26 }, "id": 69, "options": { @@ -905,15 +889,6 @@ "legendFormat": "{{operation}}", "range": true, "refId": "A" - }, - { - "datasource": "Victoria Metrics", - "editorMode": "code", - "expr": "sum(rate(event_store_failures_total[5m]))", - "instant": false, - "legendFormat": "Failed", - "range": true, - "refId": "B" } ], "title": "Event Store Operations", @@ -984,7 +959,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 26 }, "id": 70, "options": { @@ -1031,7 +1006,7 @@ "h": 1, "w": 24, "x": 0, - "y": 51 + "y": 34 }, "id": 71, "panels": [], @@ -1095,7 +1070,7 @@ "h": 8, "w": 12, "x": 12, - "y": 52 + "y": 35 }, "id": 73, "options": { diff --git a/backend/grafana/provisioning/dashboards/http-middleware.json b/backend/grafana/provisioning/dashboards/http-middleware.json index d0929c9f..f5305117 100644 --- a/backend/grafana/provisioning/dashboards/http-middleware.json +++ b/backend/grafana/provisioning/dashboards/http-middleware.json @@ -2,7 +2,7 @@ "annotations": { "list": [] }, - "description": "HTTP & Middleware", + "description": "HTTP and middleware monitoring dashboard for tracking request rates, event processing, and system health metrics", "editable": true, "gnetId": null, "graphTooltip": 1, @@ -19,11 +19,24 @@ "tooltip": "Return to main dashboard", "type": "link", "url": "/d/integr8scode-overview" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "MongoDB Detail", + "tooltip": "Full MongoDB monitoring dashboard", + "type": "link", + "url": "/d/mongodb-monitoring" } ], "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -43,7 +56,7 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, "y": 1 @@ -51,9 +64,10 @@ "id": 1, "targets": [ { - "expr": "rate(http_requests_total[5m])", + "expr": "rate(http_requests_total[5m]) or vector(0)", "legendFormat": "Requests", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Request Rate", @@ -67,7 +81,7 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, "y": 1 @@ -75,9 +89,10 @@ "id": 2, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Request Duration", @@ -91,16 +106,17 @@ } }, "gridPos": { - "h": 4, - "w": 8, + "h": 8, + "w": 12, "x": 0, - "y": 7 + "y": 9 }, "id": 3, "targets": [ { - "expr": "http_requests_active_requests", - "refId": "A" + "expr": "http_requests_active_requests or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Active Requests", @@ -114,22 +130,24 @@ } }, "gridPos": { - "h": 6, - "w": 16, - "x": 8, - "y": 7 + "h": 8, + "w": 12, + "x": 12, + "y": 9 }, "id": 4, "targets": [ { - "expr": "histogram_quantile(0.50, sum(rate(http_request_size_bytes_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(http_request_size_bytes_bucket[5m])) by (le)) or vector(0)", "legendFormat": "Request p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { - "expr": "histogram_quantile(0.50, sum(rate(http_response_size_bytes_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(http_response_size_bytes_bucket[5m])) by (le)) or vector(0)", "legendFormat": "Response p50", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Request/Response Size", @@ -137,46 +155,18 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 13 + "y": 17 }, "id": 101, "panels": [], - "title": "Database & Event Store", + "title": "Middleware", "type": "row" }, - { - "datasource": "Victoria Metrics", - "fieldConfig": { - "defaults": { - "unit": "short" - } - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 14 - }, - "id": 5, - "targets": [ - { - "expr": "rate(mongodb_event_operations_total[5m])", - "legendFormat": "Operations", - "refId": "A" - }, - { - "expr": "rate(database_connection_errors_total[5m])", - "legendFormat": "Connection Errors", - "refId": "B" - } - ], - "title": "MongoDB Operations", - "type": "timeseries" - }, { "datasource": "Victoria Metrics", "fieldConfig": { @@ -185,88 +175,18 @@ } }, "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 14 - }, - "id": 6, - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(mongodb_event_query_duration_seconds_bucket[5m])) by (le))", - "legendFormat": "p95", - "refId": "A" - } - ], - "title": "MongoDB Query Duration", - "type": "timeseries" - }, - { - "datasource": "Victoria Metrics", - "fieldConfig": { - "defaults": { - "unit": "short" - } - }, - "gridPos": { - "h": 4, - "w": 6, + "h": 8, + "w": 24, "x": 0, - "y": 20 - }, - "id": 7, - "targets": [ - { - "expr": "database_connections_active", - "refId": "A" - } - ], - "title": "Active DB Connections", - "type": "stat" - }, - { - "datasource": "Victoria Metrics", - "fieldConfig": { - "defaults": { - "unit": "short" - } - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 6, - "y": 20 - }, - "id": 8, - "targets": [ - { - "expr": "rate(event_store_operations_total[5m])", - "legendFormat": "Store Ops", - "refId": "A" - } - ], - "title": "Event Store Operations", - "type": "timeseries" - }, - { - "datasource": "Victoria Metrics", - "fieldConfig": { - "defaults": { - "unit": "s" - } - }, - "gridPos": { - "h": 6, - "w": 9, - "x": 15, - "y": 20 + "y": 18 }, "id": 13, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(idempotency_processing_duration_seconds_bucket[5m])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(idempotency_processing_duration_seconds_bucket[5m])) by (le)) or vector(0)", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Idempotency Processing Duration", @@ -274,6 +194,7 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -293,7 +214,7 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, "y": 27 @@ -301,14 +222,16 @@ "id": 9, "targets": [ { - "expr": "rate(kafka_production_errors_total[5m])", + "expr": "rate(kafka_production_errors_total[5m]) or vector(0)", "legendFormat": "Production", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { - "expr": "rate(kafka_consumption_errors_total[5m])", + "expr": "rate(kafka_consumption_errors_total[5m]) or vector(0)", "legendFormat": "Consumption", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Kafka Errors", @@ -322,7 +245,7 @@ } }, "gridPos": { - "h": 4, + "h": 8, "w": 12, "x": 12, "y": 27 @@ -330,8 +253,9 @@ "id": 10, "targets": [ { - "expr": "event_bus_queue_size", - "refId": "A" + "expr": "event_bus_queue_size or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Event Bus Queue Size", @@ -339,11 +263,12 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 33 + "y": 35 }, "id": 103, "panels": [], @@ -359,15 +284,16 @@ }, "gridPos": { "h": 4, - "w": 8, + "w": 12, "x": 0, - "y": 34 + "y": 36 }, "id": 11, "targets": [ { - "expr": "system_cpu_percent", - "refId": "A" + "expr": "system_cpu_percent or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "CPU", @@ -382,15 +308,16 @@ }, "gridPos": { "h": 4, - "w": 8, - "x": 8, - "y": 34 + "w": 12, + "x": 12, + "y": 36 }, "id": 12, "targets": [ { - "expr": "process_metrics_mixed", - "refId": "A" + "expr": "process_metrics_mixed or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Process Metrics", @@ -402,8 +329,7 @@ "style": "dark", "tags": [ "http", - "middleware", - "database" + "middleware" ], "templating": { "list": [] @@ -412,7 +338,20 @@ "from": "now-3h", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, "timezone": "", "title": "HTTP & Middleware", "uid": "http-middleware", diff --git a/backend/grafana/provisioning/dashboards/integr8scode.json b/backend/grafana/provisioning/dashboards/integr8scode.json index 2f7b8c50..3c6e3632 100644 --- a/backend/grafana/provisioning/dashboards/integr8scode.json +++ b/backend/grafana/provisioning/dashboards/integr8scode.json @@ -12,9 +12,10 @@ } ] }, + "description": "System overview dashboard — execution rates, latency percentiles, queue depth, errors, and resource utilization", "editable": true, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "id": null, "links": [ { @@ -84,13 +85,16 @@ } }, "gridPos": { - "h": 6, + "h": 4, "w": 6, "x": 0, "y": 8 }, "id": 1, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -99,11 +103,10 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} + "text": {}, + "textMode": "auto" }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -147,13 +150,16 @@ } }, "gridPos": { - "h": 6, + "h": 4, "w": 6, "x": 6, "y": 8 }, "id": 2, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -162,11 +168,10 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} + "text": {}, + "textMode": "auto" }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -207,13 +212,16 @@ } }, "gridPos": { - "h": 6, + "h": 4, "w": 6, "x": 12, "y": 8 }, "id": 3, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -222,11 +230,10 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} + "text": {}, + "textMode": "auto" }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -267,13 +274,16 @@ } }, "gridPos": { - "h": 6, + "h": 4, "w": 6, "x": 18, "y": 8 }, "id": 4, "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", "orientation": "auto", "reduceOptions": { "calcs": [ @@ -282,11 +292,10 @@ "fields": "", "values": false }, - "showThresholdLabels": false, - "showThresholdMarkers": true, - "text": {} + "text": {}, + "textMode": "auto" }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -299,14 +308,13 @@ }, { "datasource": "Victoria Metrics", + "description": "Execution rate broken down by language and version", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "Executions/sec", "axisPlacement": "auto", "barAlignment": 0, @@ -325,7 +333,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -351,7 +359,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 12 }, "id": 5, "options": { @@ -368,6 +376,7 @@ "sort": "desc" } }, + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -382,14 +391,13 @@ }, { "datasource": "Victoria Metrics", + "description": "P50, P95, and P99 execution latency over time", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "Duration (s)", "axisPlacement": "auto", "barAlignment": 0, @@ -408,7 +416,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -434,7 +442,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 12 }, "id": 6, "options": { @@ -451,6 +459,7 @@ "sort": "desc" } }, + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -476,14 +485,13 @@ }, { "datasource": "Victoria Metrics", + "description": "Number of executions waiting in the queue", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "Queue Depth", "axisPlacement": "auto", "barAlignment": 0, @@ -502,7 +510,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "none" @@ -536,7 +544,7 @@ "h": 8, "w": 8, "x": 0, - "y": 22 + "y": 20 }, "id": 7, "options": { @@ -553,6 +561,7 @@ "sort": "desc" } }, + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -566,14 +575,13 @@ }, { "datasource": "Victoria Metrics", + "description": "Error rate broken down by error type", "fieldConfig": { "defaults": { "color": { "mode": "continuous-GrYlRd" }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", "axisLabel": "Errors/min", "axisPlacement": "auto", "barAlignment": 0, @@ -592,7 +600,7 @@ "type": "linear" }, "showPoints": "never", - "spanNulls": false, + "spanNulls": true, "stacking": { "group": "A", "mode": "normal" @@ -618,7 +626,7 @@ "h": 8, "w": 8, "x": 8, - "y": 22 + "y": 20 }, "id": 8, "options": { @@ -632,6 +640,7 @@ "sort": "desc" } }, + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -645,6 +654,7 @@ }, { "datasource": "Victoria Metrics", + "description": "System memory utilization percentage", "fieldConfig": { "defaults": { "color": { @@ -682,7 +692,7 @@ "h": 8, "w": 8, "x": 16, - "y": 22 + "y": 20 }, "id": 9, "options": { @@ -699,12 +709,9 @@ "text": { "titleSize": 16, "valueSize": 24 - }, - "neutral": 50, - "minVizHeight": 75, - "minVizWidth": 75 + } }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -748,7 +755,7 @@ "h": 4, "w": 6, "x": 6, - "y": 30 + "y": 28 }, "id": 11, "options": { @@ -766,7 +773,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -810,7 +817,7 @@ "h": 4, "w": 6, "x": 12, - "y": 30 + "y": 28 }, "id": 12, "options": { @@ -828,7 +835,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -873,7 +880,7 @@ "h": 4, "w": 6, "x": 18, - "y": 30 + "y": 28 }, "id": 13, "options": { @@ -891,7 +898,7 @@ "text": {}, "textMode": "auto" }, - "pluginVersion": "10.2.0", + "pluginVersion": "8.3.3", "targets": [ { "datasource": "Victoria Metrics", @@ -904,7 +911,7 @@ } ], "refresh": "10s", - "schemaVersion": 38, + "schemaVersion": 33, "style": "dark", "tags": [ "overview", diff --git a/backend/grafana/provisioning/dashboards/kafka-events-monitoring.json b/backend/grafana/provisioning/dashboards/kafka-events-monitoring.json index 46077493..f16a07e5 100644 --- a/backend/grafana/provisioning/dashboards/kafka-events-monitoring.json +++ b/backend/grafana/provisioning/dashboards/kafka-events-monitoring.json @@ -15,6 +15,7 @@ } ] }, + "description": "Kafka events monitoring dashboard for tracking message production, consumption, and event processing metrics", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, @@ -66,6 +67,7 @@ "x": 0, "y": 0 }, + "datasource": null, "id": 1, "panels": [], "title": "Kafka Overview", @@ -103,7 +105,7 @@ }, "gridPos": { "h": 6, - "w": 4, + "w": 6, "x": 0, "y": 1 }, @@ -128,7 +130,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(rate(kafka_messages_produced_total[1m]))", + "expr": "sum(rate(kafka_messages_produced_total[1m])) or vector(0)", "instant": false, "legendFormat": "Messages/sec", "range": true, @@ -170,8 +172,8 @@ }, "gridPos": { "h": 6, - "w": 4, - "x": 8, + "w": 6, + "x": 6, "y": 1 }, "id": 4, @@ -195,7 +197,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(dlq_queue_size)", + "expr": "sum(dlq_queue_size) or vector(0)", "instant": false, "legendFormat": "DLQ Size", "range": true, @@ -203,7 +205,14 @@ } ], "title": "DLQ Messages", - "type": "stat" + "type": "stat", + "links": [ + { + "title": "DLQ Dashboard", + "url": "/d/dlq-monitoring", + "targetBlank": true + } + ] }, { "datasource": "Victoria Metrics", @@ -315,7 +324,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(rate(kafka_messages_produced_total[1m]))", + "expr": "sum(rate(kafka_messages_produced_total[1m])) or vector(0)", "instant": false, "legendFormat": "Produced", "range": true, @@ -324,7 +333,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(rate(kafka_messages_consumed_total[1m]))", + "expr": "sum(rate(kafka_messages_consumed_total[1m])) or vector(0)", "instant": false, "legendFormat": "Consumed", "range": true, @@ -342,6 +351,7 @@ "x": 0, "y": 7 }, + "datasource": null, "id": 6, "panels": [], "title": "Producer Metrics", @@ -426,7 +436,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(rate(kafka_messages_produced_total[1m])) by (topic)", + "expr": "sum(rate(kafka_messages_produced_total[1m])) by (topic) or vector(0)", "instant": false, "legendFormat": "{{topic}}", "range": true, @@ -523,7 +533,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "histogram_quantile(0.99, sum(rate(event_processing_duration_seconds_bucket[5m])) by (le)) * 1000", + "expr": "(histogram_quantile(0.99, sum(rate(event_processing_duration_seconds_bucket[5m])) by (le)) or vector(0)) * 1000", "instant": false, "legendFormat": "p99", "range": true, @@ -532,7 +542,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(event_processing_duration_seconds_bucket[5m])) by (le)) * 1000", + "expr": "(histogram_quantile(0.95, sum(rate(event_processing_duration_seconds_bucket[5m])) by (le)) or vector(0)) * 1000", "instant": false, "legendFormat": "p95", "range": true, @@ -541,7 +551,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "histogram_quantile(0.50, sum(rate(event_processing_duration_seconds_bucket[5m])) by (le)) * 1000", + "expr": "(histogram_quantile(0.50, sum(rate(event_processing_duration_seconds_bucket[5m])) by (le)) or vector(0)) * 1000", "instant": false, "legendFormat": "p50", "range": true, @@ -559,6 +569,7 @@ "x": 0, "y": 16 }, + "datasource": null, "id": 9, "panels": [], "title": "Consumer Metrics", @@ -619,8 +630,8 @@ }, "gridPos": { "h": 8, - "w": 12, - "x": 12, + "w": 24, + "x": 0, "y": 17 }, "id": 11, @@ -643,7 +654,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum by (consumer_group) (rate(kafka_messages_consumed_total[1m]))", + "expr": "sum by (consumer_group) (rate(kafka_messages_consumed_total[1m])) or vector(0)", "instant": false, "legendFormat": "{{consumer_group}}", "range": true, @@ -661,6 +672,7 @@ "x": 0, "y": 25 }, + "datasource": null, "id": 12, "panels": [], "title": "Event Processing", @@ -753,7 +765,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "histogram_quantile(0.95, sum(rate(event_processing_duration_seconds_bucket[5m])) by (event_type, le))", + "expr": "histogram_quantile(0.95, sum(rate(event_processing_duration_seconds_bucket[5m])) by (event_type, le)) or vector(0)", "instant": false, "legendFormat": "{{event_type}}", "range": true, @@ -841,7 +853,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "rate(event_processing_errors_total[1m]) * 60", + "expr": "(rate(event_processing_errors_total[1m]) or vector(0)) * 60", "instant": false, "legendFormat": "{{event_type}}", "range": true, @@ -859,336 +871,7 @@ "x": 0, "y": 34 }, - "id": 15, - "panels": [], - "title": "Dead Letter Queue", - "type": "row" - }, - { - "datasource": "Victoria Metrics", - "description": "DLQ size by original topic", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Messages", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 10 - }, - { - "color": "red", - "value": 50 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 35 - }, - "id": 16, - "options": { - "legend": { - "calcs": [ - "lastNotNull", - "max" - ], - "displayMode": "table", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } - }, - "targets": [ - { - "datasource": "Victoria Metrics", - "editorMode": "code", - "expr": "dlq_queue_size", - "instant": false, - "legendFormat": "{{original_topic}}", - "range": true, - "refId": "A" - } - ], - "title": "DLQ Size by Topic", - "type": "timeseries" - }, - { - "datasource": "Victoria Metrics", - "description": "DLQ message processing", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Messages/min", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "Received" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "red", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Retried" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "yellow", - "mode": "fixed" - } - } - ] - }, - { - "matcher": { - "id": "byName", - "options": "Discarded" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-red", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 8, - "y": 35 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - "mean", - "sum" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "Victoria Metrics", - "editorMode": "code", - "expr": "sum(rate(dlq_messages_received_total[1m]) * 60)", - "instant": false, - "legendFormat": "Received", - "range": true, - "refId": "A" - }, - { - "datasource": "Victoria Metrics", - "editorMode": "code", - "expr": "sum(rate(dlq_messages_retried_total[1m]) * 60)", - "instant": false, - "legendFormat": "Retried", - "range": true, - "refId": "B" - }, - { - "datasource": "Victoria Metrics", - "editorMode": "code", - "expr": "sum(rate(dlq_messages_discarded_total[1m]) * 60)", - "instant": false, - "legendFormat": "Discarded", - "range": true, - "refId": "C" - } - ], - "title": "DLQ Processing Rate", - "type": "timeseries" - }, - { - "datasource": "Victoria Metrics", - "description": "Age of messages in DLQ", - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "yellow", - "value": 3600 - }, - { - "color": "red", - "value": 86400 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 16, - "y": 35 - }, - "id": 18, - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "max" - ], - "fields": "", - "values": false - }, - "textMode": "auto", - "wideLayout": true - }, - "pluginVersion": "10.2.0", - "targets": [ - { - "datasource": "Victoria Metrics", - "editorMode": "code", - "expr": "max(dlq_message_age_seconds_sum / dlq_message_age_seconds_count)", - "instant": false, - "legendFormat": "Max Age", - "range": true, - "refId": "A" - } - ], - "title": "DLQ Message Max Age", - "type": "stat" - }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 43 - }, + "datasource": null, "id": 19, "panels": [], "title": "Topic & Partition Metrics", @@ -1251,7 +934,7 @@ "h": 8, "w": 12, "x": 0, - "y": 44 + "y": 35 }, "id": 20, "options": { @@ -1273,7 +956,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "rate(kafka_messages_produced_total[1m])", + "expr": "rate(kafka_messages_produced_total[1m]) or vector(0)", "instant": false, "legendFormat": "{{topic}}", "range": true, @@ -1285,7 +968,7 @@ }, { "datasource": "Victoria Metrics", - "description": "Partition distribution", + "description": "Cumulative messages produced per topic for partition balance analysis", "fieldConfig": { "defaults": { "color": { @@ -1340,7 +1023,7 @@ "h": 8, "w": 12, "x": 12, - "y": 44 + "y": 35 }, "id": 21, "options": { @@ -1361,7 +1044,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(kafka_messages_produced_total) by (topic)", + "expr": "sum(kafka_messages_produced_total) by (topic) or vector(0)", "instant": false, "legendFormat": "{{topic}}-{{partition}}", "range": true, @@ -1377,101 +1060,14 @@ "h": 1, "w": 24, "x": 0, - "y": 52 + "y": 43 }, + "datasource": null, "id": 22, "panels": [], - "title": "Event Replay & Idempotency", + "title": "Idempotency", "type": "row" }, - { - "datasource": "Victoria Metrics", - "description": "Event replay operations", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Operations/min", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 8, - "x": 0, - "y": 53 - }, - "id": 23, - "options": { - "legend": { - "calcs": [ - "sum" - ], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "targets": [ - { - "datasource": "Victoria Metrics", - "editorMode": "code", - "expr": "sum(rate(event_replay_operations_total[1m]) * 60)", - "instant": false, - "legendFormat": "Events Replayed", - "range": true, - "refId": "A" - } - ], - "title": "Event Replay Rate", - "type": "timeseries" - }, { "datasource": "Victoria Metrics", "description": "Idempotency cache performance", @@ -1573,9 +1169,9 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 8, - "y": 53 + "w": 12, + "x": 0, + "y": 44 }, "id": 24, "options": { @@ -1597,7 +1193,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(rate(idempotency_cache_hits_total[1m]))", + "expr": "sum(rate(idempotency_cache_hits_total[1m])) or vector(0)", "instant": false, "legendFormat": "Cache Hits", "range": true, @@ -1606,7 +1202,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(rate(idempotency_cache_misses_total[1m]))", + "expr": "sum(rate(idempotency_cache_misses_total[1m])) or vector(0)", "instant": false, "legendFormat": "Cache Misses", "range": true, @@ -1615,7 +1211,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(rate(idempotency_duplicates_blocked_total[1m]))", + "expr": "sum(rate(idempotency_duplicates_blocked_total[1m])) or vector(0)", "instant": false, "legendFormat": "Duplicates Blocked", "range": true, @@ -1657,9 +1253,9 @@ }, "gridPos": { "h": 8, - "w": 8, - "x": 16, - "y": 53 + "w": 12, + "x": 12, + "y": 44 }, "id": 25, "options": { @@ -1682,7 +1278,7 @@ { "datasource": "Victoria Metrics", "editorMode": "code", - "expr": "sum(idempotency_keys_active)", + "expr": "sum(idempotency_keys_active) or vector(0)", "instant": false, "legendFormat": "Active Keys", "range": true, @@ -1706,7 +1302,20 @@ "from": "now-30m", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, "timezone": "", "title": "Kafka Events", "uid": "kafka-events-monitoring", diff --git a/backend/grafana/provisioning/dashboards/kubernetes-pods.json b/backend/grafana/provisioning/dashboards/kubernetes-pods.json index 54f231c1..8d0d82a7 100644 --- a/backend/grafana/provisioning/dashboards/kubernetes-pods.json +++ b/backend/grafana/provisioning/dashboards/kubernetes-pods.json @@ -53,12 +53,14 @@ { "expr": "rate(pod_creations_total[5m])", "legendFormat": "Created", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(pod_creation_failures_total[5m])", "legendFormat": "Failed", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Pod Creations", @@ -68,7 +70,7 @@ "datasource": "Victoria Metrics", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "s" } }, "gridPos": { @@ -82,7 +84,8 @@ { "expr": "histogram_quantile(0.95, sum(rate(pod_creation_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Pod Creation Duration", @@ -105,7 +108,8 @@ "targets": [ { "expr": "pod_creations_active", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Active Creations", @@ -128,7 +132,8 @@ "targets": [ { "expr": "increase(configmaps_created_total[24h])", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "ConfigMaps Created (24h)", @@ -165,7 +170,8 @@ { "expr": "rate(pod_phase_transitions_total[5m])", "legendFormat": "{{phase}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Phase Transitions", @@ -175,7 +181,7 @@ "datasource": "Victoria Metrics", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "s" } }, "gridPos": { @@ -189,12 +195,14 @@ { "expr": "histogram_quantile(0.50, sum(rate(pod_lifetime_seconds_bucket[5m])) by (le))", "legendFormat": "p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "histogram_quantile(0.95, sum(rate(pod_lifetime_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Pod Lifetime", @@ -217,7 +225,8 @@ "targets": [ { "expr": "pods_by_phase", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Pods by Phase", @@ -240,7 +249,8 @@ "targets": [ { "expr": "pods_monitored", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Pods Monitored", @@ -277,12 +287,14 @@ { "expr": "rate(pod_monitor_events_total[5m])", "legendFormat": "Events", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(pod_monitor_reconciliations_total[5m])", "legendFormat": "Reconciliations", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Monitor Events", @@ -292,7 +304,7 @@ "datasource": "Victoria Metrics", "fieldConfig": { "defaults": { - "unit": "short" + "unit": "s" } }, "gridPos": { @@ -306,7 +318,8 @@ { "expr": "histogram_quantile(0.95, sum(rate(pod_monitor_processing_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Monitor Processing Duration", @@ -330,12 +343,14 @@ { "expr": "rate(pod_monitor_watch_errors_total[5m])", "legendFormat": "Errors", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(pod_monitor_watch_reconnects_total[5m])", "legendFormat": "Reconnects", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Watch Errors", diff --git a/backend/grafana/provisioning/dashboards/mongodb-monitoring.json b/backend/grafana/provisioning/dashboards/mongodb-monitoring.json new file mode 100644 index 00000000..9bc373cc --- /dev/null +++ b/backend/grafana/provisioning/dashboards/mongodb-monitoring.json @@ -0,0 +1,907 @@ +{ + "annotations": { + "list": [] + }, + "description": "Comprehensive MongoDB monitoring: health, storage, memory, cache, network, locking, latency, and index performance", + "editable": true, + "gnetId": null, + "graphTooltip": 1, + "id": null, + "links": [ + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": false, + "title": "Back to Overview", + "tooltip": "Return to main dashboard", + "type": "link", + "url": "/d/integr8scode-overview" + }, + { + "asDropdown": false, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [], + "targetBlank": true, + "title": "HTTP & Middleware", + "tooltip": "HTTP and middleware monitoring dashboard", + "type": "link", + "url": "/d/http-middleware" + } + ], + "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Health Overview", + "type": "row" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [ + { + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + }, + "type": "value" + } + ] + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "targets": [ + { + "expr": "mongodb_health or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "MongoDB Health", + "type": "stat" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "s" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "targets": [ + { + "expr": "(mongodb_uptime / 1000) or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Server Uptime", + "type": "stat" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 3, + "targets": [ + { + "expr": "mongodb_session_count or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Active Sessions", + "type": "stat" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 4, + "targets": [ + { + "expr": "mongodb_cursor_count or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Open Cursors", + "type": "stat" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 5, + "targets": [ + { + "expr": "mongodb_active_reads or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Active Reads", + "type": "stat" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 6, + "targets": [ + { + "expr": "mongodb_active_writes or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Active Writes", + "type": "stat" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 7, + "targets": [ + { + "expr": "rate(mongodb_cursor_timeout_count[5m]) or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Cursor Timeouts", + "type": "stat" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 8, + "targets": [ + { + "expr": "mongodb_database_count or vector(0)", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Total Databases", + "type": "stat" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 101, + "panels": [], + "title": "Storage", + "type": "row" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "id": 10, + "targets": [ + { + "expr": "sum(mongodb_data_size) by (db_namespace) or vector(0)", + "legendFormat": "{{db_namespace}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Data Size by Database", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 11, + "targets": [ + { + "expr": "sum(mongodb_storage_size) by (db_namespace) or vector(0)", + "legendFormat": "{{db_namespace}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Storage Size by Database", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 12, + "targets": [ + { + "expr": "sum(mongodb_index_size) by (db_namespace) or vector(0)", + "legendFormat": "{{db_namespace}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Index Size by Database", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 13, + "targets": [ + { + "expr": "sum(mongodb_object_count) by (db_namespace) or vector(0)", + "legendFormat": "{{db_namespace}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Object Count by Database", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 14, + "targets": [ + { + "expr": "sum(mongodb_collection_count) by (db_namespace) or vector(0)", + "legendFormat": "{{db_namespace}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Collection Count by Database", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 102, + "panels": [], + "title": "Memory & Cache", + "type": "row" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "id": 20, + "targets": [ + { + "expr": "sum(mongodb_memory_usage) by (memory_type) or vector(0)", + "legendFormat": "{{memory_type}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Memory Usage by Type", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "orange", "value": 0.8 }, + { "color": "yellow", "value": 0.9 }, + { "color": "green", "value": 0.95 } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "id": 21, + "targets": [ + { + "expr": "rate(mongodb_cache_operations{type=\"hit\"}[5m]) / (rate(mongodb_cache_operations{type=\"hit\"}[5m]) + rate(mongodb_cache_operations{type=\"miss\"}[5m]) + 0.001) or vector(0)", + "legendFormat": "Hit Ratio", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Cache Hit Ratio", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 1 }, + { "color": "red", "value": 10 } + ] + } + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 35 + }, + "id": 22, + "targets": [ + { + "expr": "rate(mongodb_page_faults[5m]) or vector(0)", + "legendFormat": "Page Faults/s", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Page Faults", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 35 + }, + "id": 23, + "targets": [ + { + "expr": "rate(mongodb_cache_operations{type=\"hit\"}[5m]) or vector(0)", + "legendFormat": "Hits", + "refId": "A", + "datasource": "Victoria Metrics" + }, + { + "expr": "rate(mongodb_cache_operations{type=\"miss\"}[5m]) or vector(0)", + "legendFormat": "Misses", + "refId": "B", + "datasource": "Victoria Metrics" + } + ], + "title": "Cache Operations Rate", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "bytes" + } + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 24, + "targets": [ + { + "expr": "rate(mongodb_wtcache_bytes_read[5m]) or vector(0)", + "legendFormat": "WT Cache Bytes Read/s", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "WiredTiger Cache Read Throughput", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 51 + }, + "id": 103, + "panels": [], + "title": "Network I/O", + "type": "row" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "Bps" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 52 + }, + "id": 30, + "targets": [ + { + "expr": "rate(mongodb_network_io_receive[5m]) or vector(0)", + "legendFormat": "Receive", + "refId": "A", + "datasource": "Victoria Metrics" + }, + { + "expr": "rate(mongodb_network_io_transmit[5m]) or vector(0)", + "legendFormat": "Transmit", + "refId": "B", + "datasource": "Victoria Metrics" + } + ], + "title": "Network Throughput", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "reqps" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 52 + }, + "id": 31, + "targets": [ + { + "expr": "rate(mongodb_network_request_count[5m]) or vector(0)", + "legendFormat": "Requests/s", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Network Request Rate", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 60 + }, + "id": 104, + "panels": [], + "title": "Locking & Latency", + "type": "row" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "ms" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 61 + }, + "id": 40, + "targets": [ + { + "expr": "rate(mongodb_global_lock_time[5m]) or vector(0)", + "legendFormat": "Global Lock Time", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Global Lock Hold Time", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "µs" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 61 + }, + "id": 41, + "targets": [ + { + "expr": "mongodb_operation_latency_time or vector(0)", + "legendFormat": "{{operation_latency}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Operation Latency", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 61 + }, + "id": 42, + "targets": [ + { + "expr": "rate(mongodb_lock_acquire_count[5m]) or vector(0)", + "legendFormat": "Acquires/s", + "refId": "A", + "datasource": "Victoria Metrics" + }, + { + "expr": "rate(mongodb_lock_acquire_wait_count[5m]) or vector(0)", + "legendFormat": "Waits/s", + "refId": "B", + "datasource": "Victoria Metrics" + }, + { + "expr": "rate(mongodb_lock_deadlock_count[5m]) or vector(0)", + "legendFormat": "Deadlocks/s", + "refId": "C", + "datasource": "Victoria Metrics" + } + ], + "title": "Lock Acquisition", + "type": "timeseries" + }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 69 + }, + "id": 105, + "panels": [], + "title": "Operations & Index Performance", + "type": "row" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 70 + }, + "id": 50, + "targets": [ + { + "expr": "mongodb_commands_rate or vector(0)", + "legendFormat": "Commands", + "refId": "A", + "datasource": "Victoria Metrics" + }, + { + "expr": "mongodb_queries_rate or vector(0)", + "legendFormat": "Queries", + "refId": "B", + "datasource": "Victoria Metrics" + }, + { + "expr": "mongodb_inserts_rate or vector(0)", + "legendFormat": "Inserts", + "refId": "C", + "datasource": "Victoria Metrics" + }, + { + "expr": "mongodb_updates_rate or vector(0)", + "legendFormat": "Updates", + "refId": "D", + "datasource": "Victoria Metrics" + }, + { + "expr": "mongodb_deletes_rate or vector(0)", + "legendFormat": "Deletes", + "refId": "E", + "datasource": "Victoria Metrics" + }, + { + "expr": "mongodb_getmores_rate or vector(0)", + "legendFormat": "Getmores", + "refId": "F", + "datasource": "Victoria Metrics" + } + ], + "title": "Operation Rates", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "ops" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 70 + }, + "id": 51, + "targets": [ + { + "expr": "mongodb_flushes_rate or vector(0)", + "legendFormat": "Flushes/s", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Flush Rate", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 52, + "targets": [ + { + "expr": "topk(10, sum(rate(mongodb_index_access_count[5m])) by (collection, db_namespace)) or vector(0)", + "legendFormat": "{{db_namespace}}.{{collection}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Index Access Frequency (Top 10)", + "type": "timeseries" + }, + { + "datasource": "Victoria Metrics", + "fieldConfig": { + "defaults": { + "unit": "short" + } + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 78 + }, + "id": 53, + "targets": [ + { + "expr": "sum(mongodb_index_count) by (db_namespace) or vector(0)", + "legendFormat": "{{db_namespace}}", + "refId": "A", + "datasource": "Victoria Metrics" + } + ], + "title": "Index Count by Database", + "type": "timeseries" + } + ], + "refresh": "10s", + "schemaVersion": 33, + "style": "dark", + "tags": [ + "mongodb", + "database" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, + "timezone": "", + "title": "MongoDB Monitoring", + "uid": "mongodb-monitoring", + "version": 1 +} diff --git a/backend/grafana/provisioning/dashboards/notifications.json b/backend/grafana/provisioning/dashboards/notifications.json index 54b632c5..88a367e4 100644 --- a/backend/grafana/provisioning/dashboards/notifications.json +++ b/backend/grafana/provisioning/dashboards/notifications.json @@ -53,17 +53,20 @@ { "expr": "rate(notifications_sent_total[5m])", "legendFormat": "Sent", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(notifications_failed_total[5m])", "legendFormat": "Failed", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" }, { "expr": "rate(notifications_read_total[5m])", "legendFormat": "Read", - "refId": "C" + "refId": "C", + "datasource": "Victoria Metrics" } ], "title": "Notification Flow", @@ -77,8 +80,8 @@ } }, "gridPos": { - "h": 4, - "w": 4, + "h": 6, + "w": 6, "x": 12, "y": 1 }, @@ -86,7 +89,8 @@ "targets": [ { "expr": "notifications_pending", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Pending", @@ -100,16 +104,17 @@ } }, "gridPos": { - "h": 4, - "w": 4, - "x": 16, + "h": 6, + "w": 6, + "x": 18, "y": 1 }, "id": 3, "targets": [ { "expr": "notifications_queued", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Queued", @@ -136,7 +141,7 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, "y": 8 @@ -146,7 +151,8 @@ { "expr": "rate(notifications_by_channel_total[5m])", "legendFormat": "{{channel}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "By Channel", @@ -160,7 +166,7 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, "y": 8 @@ -170,7 +176,8 @@ { "expr": "rate(notifications_by_severity_total[5m])", "legendFormat": "{{severity}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "By Severity", @@ -184,17 +191,18 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 16 }, "id": 7, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(notification_channel_delivery_time_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Channel Delivery Time", @@ -208,17 +216,18 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 16 }, "id": 8, "targets": [ { "expr": "rate(notification_channel_failures_total[5m])", "legendFormat": "{{channel}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Channel Failures", @@ -230,7 +239,7 @@ "h": 1, "w": 24, "x": 0, - "y": 20 + "y": 24 }, "id": 102, "panels": [], @@ -245,17 +254,18 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 21 + "y": 25 }, "id": 9, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(notification_delivery_time_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Overall Delivery Time", @@ -269,17 +279,18 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 21 + "y": 25 }, "id": 10, "targets": [ { "expr": "rate(notification_status_changes_total[5m])", "legendFormat": "Changes", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Status Changes", @@ -291,7 +302,7 @@ "h": 1, "w": 24, "x": 0, - "y": 27 + "y": 33 }, "id": 103, "panels": [], @@ -306,22 +317,24 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 28 + "y": 34 }, "id": 11, "targets": [ { "expr": "rate(notifications_throttled_total[5m])", "legendFormat": "Throttled", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(notification_throttle_window_hits_total[5m])", "legendFormat": "Window Hits", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Throttling", @@ -335,22 +348,24 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 28 + "y": 34 }, "id": 12, "targets": [ { "expr": "rate(notification_retries_total[5m])", "legendFormat": "Retries", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "histogram_quantile(0.50, sum(rate(notification_retry_success_rate_percent_bucket[5m])) by (le))", "legendFormat": "Success Rate p50", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Retries", @@ -362,7 +377,7 @@ "h": 1, "w": 24, "x": 0, - "y": 34 + "y": 42 }, "id": 104, "panels": [], @@ -377,22 +392,24 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 35 + "y": 43 }, "id": 13, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(notification_webhook_delivery_time_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(notification_webhook_response_status_total[5m])", "legendFormat": "{{status}}", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Webhook Delivery", @@ -406,22 +423,24 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 12, - "y": 35 + "y": 43 }, "id": 14, "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(notification_slack_delivery_time_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(notification_slack_api_errors_total[5m])", "legendFormat": "Errors", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Slack", @@ -433,7 +452,7 @@ "h": 1, "w": 24, "x": 0, - "y": 41 + "y": 51 }, "id": 105, "panels": [], @@ -448,17 +467,18 @@ } }, "gridPos": { - "h": 6, + "h": 8, "w": 12, "x": 0, - "y": 42 + "y": 52 }, "id": 15, "targets": [ { "expr": "rate(notification_subscription_changes_total[5m])", "legendFormat": "Changes/s", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Subscriptions", diff --git a/backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json b/backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json index cb45ce15..be01119b 100644 --- a/backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json +++ b/backend/grafana/provisioning/dashboards/rate-limiting-dashboard.json @@ -12,6 +12,7 @@ } ] }, + "description": "Rate limiting monitoring — request rates, rejections, bypasses, and algorithm distribution", "editable": true, "gnetId": null, "graphTooltip": 1, @@ -31,8 +32,23 @@ } ], "panels": [ + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 100, + "panels": [], + "title": "Overview", + "type": "row" + }, { "datasource": "Victoria Metrics", + "description": "Total incoming request rate across all endpoints", "fieldConfig": { "defaults": { "color": { @@ -45,6 +61,14 @@ { "color": "green", "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "red", + "value": 500 } ] }, @@ -55,7 +79,7 @@ "h": 4, "w": 6, "x": 0, - "y": 0 + "y": 1 }, "id": 1, "options": { @@ -64,18 +88,21 @@ "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_requests_total[5m]))", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Request Rate", @@ -83,6 +110,7 @@ }, { "datasource": "Victoria Metrics", + "description": "Percentage of requests rejected by rate limiting", "fieldConfig": { "defaults": { "color": { @@ -113,7 +141,7 @@ "h": 4, "w": 6, "x": 6, - "y": 0 + "y": 1 }, "id": 2, "options": { @@ -122,18 +150,21 @@ "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "(sum(rate(rate_limit_rejected_total[5m])) / sum(rate(rate_limit_requests_total[5m]))) * 100", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Rejection Rate", @@ -141,6 +172,7 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of requests allowed through rate limiting", "fieldConfig": { "defaults": { "color": { @@ -163,7 +195,7 @@ "h": 4, "w": 6, "x": 12, - "y": 0 + "y": 1 }, "id": 3, "options": { @@ -172,18 +204,21 @@ "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_allowed_total[5m]))", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Allowed Rate", @@ -191,6 +226,7 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of requests that bypassed rate limiting", "fieldConfig": { "defaults": { "color": { @@ -213,7 +249,7 @@ "h": 4, "w": 6, "x": 18, - "y": 0 + "y": 1 }, "id": 4, "options": { @@ -222,25 +258,43 @@ "justifyMode": "center", "orientation": "auto", "reduceOptions": { - "calcs": ["lastNotNull"], + "calcs": [ + "lastNotNull" + ], "fields": "", "values": false }, "text": {}, "textMode": "auto" }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_bypass_total[5m]))", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Bypass Rate", "type": "stat" }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 101, + "panels": [], + "title": "Request Analysis", + "type": "row" + }, { "datasource": "Victoria Metrics", + "description": "Incoming request rate broken down by API endpoint", "fieldConfig": { "defaults": { "color": { @@ -291,12 +345,15 @@ "h": 8, "w": 12, "x": 0, - "y": 4 + "y": 6 }, "id": 5, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean", + "max" + ], "displayMode": "list", "placement": "bottom" }, @@ -304,17 +361,19 @@ "mode": "multi" } }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_requests_total{endpoint!~\"/api/v1/result/.*\"}[5m])) by (endpoint)", "legendFormat": "{{endpoint}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "sum(rate(rate_limit_requests_total{endpoint=~\"/api/v1/result/.*\"}[5m]))", "legendFormat": "/api/v1/result/*", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Request Rate by Endpoint", @@ -322,6 +381,7 @@ }, { "datasource": "Victoria Metrics", + "description": "Request rate split by authentication status", "fieldConfig": { "defaults": { "color": { @@ -372,12 +432,15 @@ "h": 8, "w": 12, "x": 12, - "y": 4 + "y": 6 }, "id": 6, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean", + "max" + ], "displayMode": "list", "placement": "bottom" }, @@ -385,19 +448,35 @@ "mode": "multi" } }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_requests_total[5m])) by (authenticated)", "legendFormat": "authenticated={{authenticated}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Authenticated vs Anonymous Requests", "type": "timeseries" }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 102, + "panels": [], + "title": "Distribution & Groups", + "type": "row" + }, { "datasource": "Victoria Metrics", + "description": "Distribution of rate limiting algorithms used in the last hour", "fieldConfig": { "defaults": { "color": { @@ -411,6 +490,15 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -418,31 +506,39 @@ "h": 8, "w": 12, "x": 0, - "y": 12 + "y": 15 }, "id": 7, "options": { - "displayLabels": ["percent"], + "displayLabels": [ + "percent" + ], "legend": { "displayMode": "table", "placement": "right", - "values": ["value", "percent"] + "values": [ + "value", + "percent" + ] }, "pieType": "donut", "reduceOptions": { "values": false, - "calcs": ["lastNotNull"] + "calcs": [ + "lastNotNull" + ] }, "tooltip": { "mode": "single" } }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(increase(rate_limit_requests_total[1h])) by (algorithm)", "legendFormat": "{{algorithm}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Algorithm Distribution (Last Hour)", @@ -450,6 +546,7 @@ }, { "datasource": "Victoria Metrics", + "description": "Allowed vs rejected request rates per endpoint group", "fieldConfig": { "defaults": { "color": { @@ -500,12 +597,15 @@ "h": 8, "w": 12, "x": 12, - "y": 12 + "y": 15 }, "id": 8, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean", + "max" + ], "displayMode": "list", "placement": "bottom" }, @@ -513,24 +613,41 @@ "mode": "multi" } }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_allowed_total[5m])) by (group)", "legendFormat": "allowed: {{group}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "sum(rate(rate_limit_rejected_total[5m])) by (group)", "legendFormat": "rejected: {{group}}", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Allowed/Rejected by Endpoint Group", "type": "timeseries" }, + { + "collapsed": false, + "datasource": null, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 103, + "panels": [], + "title": "Bypass & Rejections", + "type": "row" + }, { "datasource": "Victoria Metrics", + "description": "Rate of bypassed requests broken down by endpoint", "fieldConfig": { "defaults": { "color": { @@ -581,12 +698,15 @@ "h": 8, "w": 12, "x": 0, - "y": 20 + "y": 24 }, "id": 9, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean", + "max" + ], "displayMode": "list", "placement": "bottom" }, @@ -594,12 +714,13 @@ "mode": "multi" } }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_bypass_total[5m])) by (endpoint)", "legendFormat": "{{endpoint}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Bypassed Requests by Endpoint", @@ -607,6 +728,7 @@ }, { "datasource": "Victoria Metrics", + "description": "Rate of rejected requests broken down by endpoint", "fieldConfig": { "defaults": { "color": { @@ -657,12 +779,15 @@ "h": 8, "w": 12, "x": 12, - "y": 20 + "y": 24 }, "id": 10, "options": { "legend": { - "calcs": [], + "calcs": [ + "mean", + "max" + ], "displayMode": "list", "placement": "bottom" }, @@ -670,21 +795,25 @@ "mode": "multi" } }, - "pluginVersion": "8.0.0", + "pluginVersion": "8.3.3", "targets": [ { "expr": "sum(rate(rate_limit_rejected_total[5m])) by (endpoint)", "legendFormat": "{{endpoint}}", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Rejected Requests by Endpoint", "type": "timeseries" } ], - "schemaVersion": 27, + "refresh": "10s", + "schemaVersion": 33, "style": "dark", - "tags": ["rate-limiting"], + "tags": [ + "rate-limiting" + ], "templating": { "list": [] }, @@ -692,9 +821,23 @@ "from": "now-1h", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, "timezone": "", "title": "Rate Limiting", "uid": "rate-limiting", - "version": 1 + "version": 1, + "weekStart": "" } diff --git a/backend/grafana/provisioning/dashboards/security-auth.json b/backend/grafana/provisioning/dashboards/security-auth.json index ca9e1a5b..74b4f280 100644 --- a/backend/grafana/provisioning/dashboards/security-auth.json +++ b/backend/grafana/provisioning/dashboards/security-auth.json @@ -1,8 +1,18 @@ { "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, - "description": "Security & Authentication", + "description": "Security and authentication monitoring — login attempts, token operations, CSRF protection, and authorization checks", "editable": true, "gnetId": null, "graphTooltip": 1, @@ -24,6 +34,7 @@ "panels": [ { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -37,8 +48,50 @@ }, { "datasource": "Victoria Metrics", + "description": "Authentication attempt and failure rates by method", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -49,16 +102,32 @@ "y": 1 }, "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "rate(authentication_attempts_total[5m])", "legendFormat": "Attempts", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(authentication_failures_total[5m])", "legendFormat": "Failures", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Authentication Attempts", @@ -66,8 +135,50 @@ }, { "datasource": "Victoria Metrics", + "description": "95th percentile authentication latency", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "s" } }, @@ -78,11 +189,26 @@ "y": 1 }, "id": 2, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(authentication_duration_seconds_bucket[5m])) by (le))", "legendFormat": "p95", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Authentication Duration", @@ -90,8 +216,22 @@ }, { "datasource": "Victoria Metrics", + "description": "Current number of active user sessions", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -102,10 +242,27 @@ "y": 7 }, "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "authentication_sessions_active", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Active Sessions", @@ -113,8 +270,30 @@ }, { "datasource": "Victoria Metrics", + "description": "Number of accounts locked in the last 24 hours", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 5 + } + ] + }, "unit": "short" } }, @@ -125,10 +304,27 @@ "y": 7 }, "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "increase(accounts_locked_total[24h])", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Accounts Locked (24h)", @@ -136,8 +332,30 @@ }, { "datasource": "Victoria Metrics", + "description": "Detected brute force attempts in the last hour", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 10 + } + ] + }, "unit": "short" } }, @@ -148,10 +366,27 @@ "y": 7 }, "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "increase(brute_force_attempts_total[1h])", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Brute Force Attempts (1h)", @@ -159,8 +394,30 @@ }, { "datasource": "Victoria Metrics", + "description": "Weak password attempts in the last 24 hours", "fieldConfig": { "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 5 + }, + { + "color": "red", + "value": 20 + } + ] + }, "unit": "short" } }, @@ -171,10 +428,27 @@ "y": 7 }, "id": 6, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "increase(weak_password_attempts_total[24h])", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Weak Password Attempts", @@ -182,6 +456,7 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -195,8 +470,50 @@ }, { "datasource": "Victoria Metrics", + "description": "Token generation, revocation, and validation failure rates", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -207,21 +524,38 @@ "y": 12 }, "id": 7, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "rate(tokens_generated_total[5m])", "legendFormat": "Generated", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(tokens_revoked_total[5m])", "legendFormat": "Revoked", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" }, { "expr": "rate(token_validation_failures_total[5m])", "legendFormat": "Validation Failures", - "refId": "C" + "refId": "C", + "datasource": "Victoria Metrics" } ], "title": "Token Operations", @@ -229,8 +563,50 @@ }, { "datasource": "Victoria Metrics", + "description": "Median token expiry time distribution", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "s" } }, @@ -241,11 +617,26 @@ "y": 12 }, "id": 8, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "histogram_quantile(0.50, sum(rate(token_expiry_time_seconds_bucket[5m])) by (le))", "legendFormat": "p50", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Token Expiry Time", @@ -253,8 +644,50 @@ }, { "datasource": "Victoria Metrics", + "description": "CSRF token generation and validation failure rates", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -265,16 +698,32 @@ "y": 18 }, "id": 9, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "rate(csrf_tokens_generated_total[5m])", "legendFormat": "Generated", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(csrf_validation_failures_total[5m])", "legendFormat": "Failures", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "CSRF", @@ -282,6 +731,7 @@ }, { "collapsed": false, + "datasource": null, "gridPos": { "h": 1, "w": 24, @@ -295,8 +745,50 @@ }, { "datasource": "Victoria Metrics", + "description": "Authorization check and denial rates", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -307,16 +799,32 @@ "y": 25 }, "id": 10, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "rate(authorization_checks_total[5m])", "legendFormat": "Checks", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" }, { "expr": "rate(authorization_denials_total[5m])", "legendFormat": "Denials", - "refId": "B" + "refId": "B", + "datasource": "Victoria Metrics" } ], "title": "Authorization", @@ -324,8 +832,50 @@ }, { "datasource": "Victoria Metrics", + "description": "Password reset request rate", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, "unit": "short" } }, @@ -336,11 +886,26 @@ "y": 25 }, "id": 11, + "options": { + "legend": { + "calcs": [ + "mean", + "max" + ], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi" + } + }, + "pluginVersion": "8.3.3", "targets": [ { "expr": "rate(password_reset_requests_total[5m])", "legendFormat": "Resets", - "refId": "A" + "refId": "A", + "datasource": "Victoria Metrics" } ], "title": "Password Resets", @@ -361,9 +926,23 @@ "from": "now-3h", "to": "now" }, - "timepicker": {}, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ] + }, "timezone": "", "title": "Security & Authentication", "uid": "security-auth", - "version": 1 + "version": 1, + "weekStart": "" } diff --git a/backend/otel-collector-config.yaml b/backend/otel-collector-config.yaml index cc0895fe..10ca2c6c 100644 --- a/backend/otel-collector-config.yaml +++ b/backend/otel-collector-config.yaml @@ -33,6 +33,51 @@ receivers: - topics - consumers + mongodb: + hosts: + - endpoint: mongo:27017 + username: ${env:MONGO_ROOT_USER} + password: ${env:MONGO_ROOT_PASSWORD} + collection_interval: 15s + initial_delay: 10s + metrics: + mongodb.health: + enabled: true + mongodb.uptime: + enabled: true + mongodb.page_faults: + enabled: true + mongodb.operation.latency.time: + enabled: true + mongodb.active.reads: + enabled: true + mongodb.active.writes: + enabled: true + mongodb.commands.rate: + enabled: true + mongodb.deletes.rate: + enabled: true + mongodb.flushes.rate: + enabled: true + mongodb.getmores.rate: + enabled: true + mongodb.inserts.rate: + enabled: true + mongodb.queries.rate: + enabled: true + mongodb.updates.rate: + enabled: true + mongodb.lock.acquire.count: + enabled: true + mongodb.lock.acquire.time: + enabled: true + mongodb.lock.acquire.wait_count: + enabled: true + mongodb.lock.deadlock.count: + enabled: true + mongodb.wtcache.bytes.read: + enabled: true + processors: batch: timeout: 10s @@ -117,7 +162,7 @@ service: exporters: [otlp/jaeger, logging] metrics: - receivers: [otlp, hostmetrics, kafkametrics] + receivers: [otlp, hostmetrics, kafkametrics, mongodb] processors: [memory_limiter, batch, resource, attributes] exporters: [prometheusremotewrite, logging] diff --git a/backend/tests/e2e/idempotency/test_idempotency.py b/backend/tests/e2e/idempotency/test_idempotency.py index f458475c..2d8273e5 100644 --- a/backend/tests/e2e/idempotency/test_idempotency.py +++ b/backend/tests/e2e/idempotency/test_idempotency.py @@ -6,7 +6,7 @@ import pytest import redis.asyncio as redis -from app.core.metrics import DatabaseMetrics +from app.core.metrics import IdempotencyMetrics from app.domain.enums import EventType from app.domain.idempotency import IdempotencyRecord, IdempotencyStatus, KeyStrategy from app.services.idempotency import IdempotencyConfig, IdempotencyManager, RedisIdempotencyRepository @@ -34,8 +34,8 @@ def manager(self, redis_client: redis.Redis, test_settings: Settings) -> Idempot max_result_size_bytes=1024, ) repo = RedisIdempotencyRepository(redis_client, key_prefix=prefix) - database_metrics = DatabaseMetrics(test_settings) - return IdempotencyManager(cfg, repo, _test_logger, database_metrics=database_metrics) + idempotency_metrics = IdempotencyMetrics(test_settings) + return IdempotencyManager(cfg, repo, _test_logger, idempotency_metrics=idempotency_metrics) @pytest.mark.asyncio async def test_complete_flow_new_event(self, manager: IdempotencyManager) -> None: diff --git a/backend/tests/unit/conftest.py b/backend/tests/unit/conftest.py index b62993b4..54752169 100644 --- a/backend/tests/unit/conftest.py +++ b/backend/tests/unit/conftest.py @@ -6,7 +6,7 @@ import pytest from app.core.metrics import ( ConnectionMetrics, - DatabaseMetrics, + IdempotencyMetrics, DLQMetrics, EventMetrics, ExecutionMetrics, @@ -228,8 +228,8 @@ def queue_metrics(test_settings: Settings) -> QueueMetrics: @pytest.fixture -def database_metrics(test_settings: Settings) -> DatabaseMetrics: - return DatabaseMetrics(test_settings) +def idempotency_metrics(test_settings: Settings) -> IdempotencyMetrics: + return IdempotencyMetrics(test_settings) @pytest.fixture diff --git a/backend/tests/unit/core/metrics/test_database_and_dlq_metrics.py b/backend/tests/unit/core/metrics/test_database_and_dlq_metrics.py index 7484a573..d62f8990 100644 --- a/backend/tests/unit/core/metrics/test_database_and_dlq_metrics.py +++ b/backend/tests/unit/core/metrics/test_database_and_dlq_metrics.py @@ -1,29 +1,20 @@ import pytest -from app.core.metrics import DatabaseMetrics, DLQMetrics +from app.core.metrics import IdempotencyMetrics, DLQMetrics from app.settings import Settings pytestmark = pytest.mark.unit -def test_database_metrics_methods(test_settings: Settings) -> None: - """Test DatabaseMetrics methods with no-op metrics.""" - m = DatabaseMetrics(test_settings) - m.record_mongodb_operation("insert", "ok") - m.record_mongodb_query_duration(0.1, "find") - m.record_event_store_duration(0.2, "insert", "events") - m.record_event_query_duration(0.3, "by_type", "events") - m.record_event_store_failed("etype", "error") +def test_idempotency_metrics_methods(test_settings: Settings) -> None: + """Test IdempotencyMetrics methods with no-op metrics.""" + m = IdempotencyMetrics(test_settings) m.record_idempotency_cache_hit("etype", "check") m.record_idempotency_cache_miss("etype", "check") m.record_idempotency_duplicate_blocked("etype") m.record_idempotency_processing_duration(0.4, "process") m.increment_idempotency_keys("prefix") m.decrement_idempotency_keys("prefix") - m.record_idempotent_event_processed("etype", "blocked") m.record_idempotent_processing_duration(0.5, "etype") - m.update_database_connections(1) - m.update_database_connections(-1) - m.record_database_connection_error("timeout") def test_dlq_metrics_methods(test_settings: Settings) -> None: diff --git a/backend/tests/unit/core/metrics/test_metrics_classes.py b/backend/tests/unit/core/metrics/test_metrics_classes.py index 5206b5de..5fcf7b6a 100644 --- a/backend/tests/unit/core/metrics/test_metrics_classes.py +++ b/backend/tests/unit/core/metrics/test_metrics_classes.py @@ -1,7 +1,7 @@ import pytest from app.core.metrics import ( ConnectionMetrics, - DatabaseMetrics, + IdempotencyMetrics, DLQMetrics, EventMetrics, ExecutionMetrics, @@ -51,7 +51,7 @@ def test_event_metrics_smoke(test_settings: Settings) -> None: def test_other_metrics_classes_smoke(test_settings: Settings) -> None: """Test other metrics classes smoke test with no-op metrics.""" QueueMetrics(test_settings).record_enqueue() - DatabaseMetrics(test_settings).record_mongodb_operation("read", "ok") + IdempotencyMetrics(test_settings).record_idempotency_cache_hit("etype", "check") DLQMetrics(test_settings).record_dlq_message_received("topic", "type") ExecutionMetrics(test_settings).record_script_execution(ExecutionStatus.QUEUED, "python") KubernetesMetrics(test_settings).record_k8s_pod_created("success", "python") diff --git a/backend/tests/unit/services/idempotency/test_idempotency_manager.py b/backend/tests/unit/services/idempotency/test_idempotency_manager.py index c4fa9ebe..a8208cb9 100644 --- a/backend/tests/unit/services/idempotency/test_idempotency_manager.py +++ b/backend/tests/unit/services/idempotency/test_idempotency_manager.py @@ -2,7 +2,7 @@ from unittest.mock import MagicMock import pytest -from app.core.metrics import DatabaseMetrics +from app.core.metrics import IdempotencyMetrics from app.domain.events import BaseEvent from app.domain.idempotency import KeyStrategy from app.services.idempotency import IdempotencyConfig, IdempotencyManager @@ -37,9 +37,9 @@ def test_custom_config(self) -> None: assert config.max_result_size_bytes == 2048 -def test_manager_generate_key_variants(database_metrics: DatabaseMetrics) -> None: +def test_manager_generate_key_variants(idempotency_metrics: IdempotencyMetrics) -> None: repo = MagicMock() - mgr = IdempotencyManager(IdempotencyConfig(), repo, _test_logger, database_metrics=database_metrics) + mgr = IdempotencyManager(IdempotencyConfig(), repo, _test_logger, idempotency_metrics=idempotency_metrics) ev = MagicMock(spec=BaseEvent) ev.event_type = "t" ev.event_id = "e" diff --git a/docker-compose.yaml b/docker-compose.yaml index 99386ab8..da1ab447 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -494,6 +494,8 @@ services: environment: - HOST_PROC=/hostfs/proc - HOST_SYS=/hostfs/sys + - MONGO_ROOT_USER=${MONGO_ROOT_USER:-root} + - MONGO_ROOT_PASSWORD=${MONGO_ROOT_PASSWORD:-rootpassword} ports: - "127.0.0.1:4317:4317" # OTLP gRPC - "127.0.0.1:4318:4318" # OTLP HTTP @@ -507,6 +509,8 @@ services: condition: service_started kafka: condition: service_healthy + mongo: + condition: service_healthy restart: unless-stopped # --8<-- [start:dev_volumes]