- add Alembic migration c1d2e3f4g5h6 for client monitoring:
- create client_logs table with FK to clients.uuid and performance indexes
- extend clients with process/health tracking fields
- extend data model with ClientLog, LogLevel, ProcessStatus, and ScreenHealthStatus
- enhance listener MQTT handling:
- subscribe to logs and health topics
- persist client logs from infoscreen/{uuid}/logs/{level}
- process health payloads and enrich heartbeat-derived client state
- add monitoring API blueprint server/routes/client_logs.py:
- GET /api/client-logs/<uuid>/logs
- GET /api/client-logs/summary
- GET /api/client-logs/recent-errors
- GET /api/client-logs/test
- register client_logs blueprint in server/wsgi.py
- align compose/dev runtime for listener live-code execution
- add client-side implementation docs:
- CLIENT_MONITORING_SPECIFICATION.md
- CLIENT_MONITORING_IMPLEMENTATION_GUIDE.md
- update TECH-CHANGELOG.md and copilot-instructions.md:
- document monitoring changes
- codify post-release technical-notes/no-version-bump convention
256 lines
8.4 KiB
Python
256 lines
8.4 KiB
Python
from flask import Blueprint, jsonify, request
|
|
from server.database import Session
|
|
from server.permissions import admin_or_higher
|
|
from models.models import ClientLog, Client, LogLevel
|
|
from sqlalchemy import desc, func
|
|
from datetime import datetime, timedelta, timezone
|
|
import json
|
|
|
|
client_logs_bp = Blueprint("client_logs", __name__, url_prefix="/api/client-logs")
|
|
|
|
|
|
@client_logs_bp.route("/test", methods=["GET"])
|
|
def test_client_logs():
|
|
"""Test endpoint to verify logging infrastructure (no auth required)"""
|
|
session = Session()
|
|
try:
|
|
# Count total logs
|
|
total_logs = session.query(func.count(ClientLog.id)).scalar()
|
|
|
|
# Count by level
|
|
error_count = session.query(func.count(ClientLog.id)).filter_by(level=LogLevel.ERROR).scalar()
|
|
warn_count = session.query(func.count(ClientLog.id)).filter_by(level=LogLevel.WARN).scalar()
|
|
info_count = session.query(func.count(ClientLog.id)).filter_by(level=LogLevel.INFO).scalar()
|
|
|
|
# Get last 5 logs
|
|
recent_logs = session.query(ClientLog).order_by(desc(ClientLog.timestamp)).limit(5).all()
|
|
|
|
recent = []
|
|
for log in recent_logs:
|
|
recent.append({
|
|
"client_uuid": log.client_uuid,
|
|
"level": log.level.value if log.level else None,
|
|
"message": log.message,
|
|
"timestamp": log.timestamp.isoformat() if log.timestamp else None
|
|
})
|
|
|
|
session.close()
|
|
return jsonify({
|
|
"status": "ok",
|
|
"infrastructure": "working",
|
|
"total_logs": total_logs,
|
|
"counts": {
|
|
"ERROR": error_count,
|
|
"WARN": warn_count,
|
|
"INFO": info_count
|
|
},
|
|
"recent_5": recent
|
|
})
|
|
except Exception as e:
|
|
session.close()
|
|
return jsonify({"status": "error", "message": str(e)}), 500
|
|
|
|
|
|
@client_logs_bp.route("/<uuid>/logs", methods=["GET"])
|
|
@admin_or_higher
|
|
def get_client_logs(uuid):
|
|
"""
|
|
Get logs for a specific client
|
|
Query params:
|
|
- level: ERROR, WARN, INFO, DEBUG (optional)
|
|
- limit: number of entries (default 50, max 500)
|
|
- since: ISO timestamp (optional)
|
|
|
|
Example: /api/client-logs/abc-123/logs?level=ERROR&limit=100
|
|
"""
|
|
session = Session()
|
|
try:
|
|
# Verify client exists
|
|
client = session.query(Client).filter_by(uuid=uuid).first()
|
|
if not client:
|
|
session.close()
|
|
return jsonify({"error": "Client not found"}), 404
|
|
|
|
# Parse query parameters
|
|
level_param = request.args.get('level')
|
|
limit = min(int(request.args.get('limit', 50)), 500)
|
|
since_param = request.args.get('since')
|
|
|
|
# Build query
|
|
query = session.query(ClientLog).filter_by(client_uuid=uuid)
|
|
|
|
# Filter by log level
|
|
if level_param:
|
|
try:
|
|
level_enum = LogLevel[level_param.upper()]
|
|
query = query.filter_by(level=level_enum)
|
|
except KeyError:
|
|
session.close()
|
|
return jsonify({"error": f"Invalid level: {level_param}. Must be ERROR, WARN, INFO, or DEBUG"}), 400
|
|
|
|
# Filter by timestamp
|
|
if since_param:
|
|
try:
|
|
# Handle both with and without 'Z' suffix
|
|
since_str = since_param.replace('Z', '+00:00')
|
|
since_dt = datetime.fromisoformat(since_str)
|
|
if since_dt.tzinfo is None:
|
|
since_dt = since_dt.replace(tzinfo=timezone.utc)
|
|
query = query.filter(ClientLog.timestamp >= since_dt)
|
|
except ValueError:
|
|
session.close()
|
|
return jsonify({"error": "Invalid timestamp format. Use ISO 8601"}), 400
|
|
|
|
# Execute query
|
|
logs = query.order_by(desc(ClientLog.timestamp)).limit(limit).all()
|
|
|
|
# Format results
|
|
result = []
|
|
for log in logs:
|
|
entry = {
|
|
"id": log.id,
|
|
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
|
"level": log.level.value if log.level else None,
|
|
"message": log.message,
|
|
"context": {}
|
|
}
|
|
|
|
# Parse context JSON
|
|
if log.context:
|
|
try:
|
|
entry["context"] = json.loads(log.context)
|
|
except json.JSONDecodeError:
|
|
entry["context"] = {"raw": log.context}
|
|
|
|
result.append(entry)
|
|
|
|
session.close()
|
|
return jsonify({
|
|
"client_uuid": uuid,
|
|
"logs": result,
|
|
"count": len(result),
|
|
"limit": limit
|
|
})
|
|
|
|
except Exception as e:
|
|
session.close()
|
|
return jsonify({"error": f"Server error: {str(e)}"}), 500
|
|
|
|
|
|
@client_logs_bp.route("/summary", methods=["GET"])
|
|
@admin_or_higher
|
|
def get_logs_summary():
|
|
"""
|
|
Get summary of errors/warnings across all clients in last 24 hours
|
|
Returns count of ERROR, WARN, INFO logs per client
|
|
|
|
Example response:
|
|
{
|
|
"summary": {
|
|
"client-uuid-1": {"ERROR": 5, "WARN": 12, "INFO": 45},
|
|
"client-uuid-2": {"ERROR": 0, "WARN": 3, "INFO": 20}
|
|
},
|
|
"period_hours": 24,
|
|
"timestamp": "2026-03-09T21:00:00Z"
|
|
}
|
|
"""
|
|
session = Session()
|
|
try:
|
|
# Get hours parameter (default 24, max 168 = 1 week)
|
|
hours = min(int(request.args.get('hours', 24)), 168)
|
|
since = datetime.now(timezone.utc) - timedelta(hours=hours)
|
|
|
|
# Query log counts grouped by client and level
|
|
stats = session.query(
|
|
ClientLog.client_uuid,
|
|
ClientLog.level,
|
|
func.count(ClientLog.id).label('count')
|
|
).filter(
|
|
ClientLog.timestamp >= since
|
|
).group_by(
|
|
ClientLog.client_uuid,
|
|
ClientLog.level
|
|
).all()
|
|
|
|
# Build summary dictionary
|
|
summary = {}
|
|
for stat in stats:
|
|
uuid = stat.client_uuid
|
|
if uuid not in summary:
|
|
# Initialize all levels to 0
|
|
summary[uuid] = {
|
|
"ERROR": 0,
|
|
"WARN": 0,
|
|
"INFO": 0,
|
|
"DEBUG": 0
|
|
}
|
|
|
|
summary[uuid][stat.level.value] = stat.count
|
|
|
|
# Get client info for enrichment
|
|
clients = session.query(Client.uuid, Client.hostname, Client.description).all()
|
|
client_info = {c.uuid: {"hostname": c.hostname, "description": c.description} for c in clients}
|
|
|
|
# Enrich summary with client info
|
|
enriched_summary = {}
|
|
for uuid, counts in summary.items():
|
|
enriched_summary[uuid] = {
|
|
"counts": counts,
|
|
"info": client_info.get(uuid, {})
|
|
}
|
|
|
|
session.close()
|
|
return jsonify({
|
|
"summary": enriched_summary,
|
|
"period_hours": hours,
|
|
"since": since.isoformat(),
|
|
"timestamp": datetime.now(timezone.utc).isoformat()
|
|
})
|
|
|
|
except Exception as e:
|
|
session.close()
|
|
return jsonify({"error": f"Server error: {str(e)}"}), 500
|
|
|
|
|
|
@client_logs_bp.route("/recent-errors", methods=["GET"])
|
|
@admin_or_higher
|
|
def get_recent_errors():
|
|
"""
|
|
Get recent ERROR logs across all clients
|
|
Query params:
|
|
- limit: number of entries (default 20, max 100)
|
|
|
|
Useful for system-wide error monitoring
|
|
"""
|
|
session = Session()
|
|
try:
|
|
limit = min(int(request.args.get('limit', 20)), 100)
|
|
|
|
# Get recent errors from all clients
|
|
logs = session.query(ClientLog).filter_by(
|
|
level=LogLevel.ERROR
|
|
).order_by(
|
|
desc(ClientLog.timestamp)
|
|
).limit(limit).all()
|
|
|
|
result = []
|
|
for log in logs:
|
|
entry = {
|
|
"id": log.id,
|
|
"client_uuid": log.client_uuid,
|
|
"timestamp": log.timestamp.isoformat() if log.timestamp else None,
|
|
"message": log.message,
|
|
"context": json.loads(log.context) if log.context else {}
|
|
}
|
|
result.append(entry)
|
|
|
|
session.close()
|
|
return jsonify({
|
|
"errors": result,
|
|
"count": len(result)
|
|
})
|
|
|
|
except Exception as e:
|
|
session.close()
|
|
return jsonify({"error": f"Server error: {str(e)}"}), 500
|