Quality Dashboards

def calculate_quality_metrics(responses, window_hours=24):
    """Calculate quality metrics for a time window."""
    cutoff = time.time() - window_hours * 3600
    recent = [r for r in responses if r["timestamp"] > cutoff]

    if not recent:
        return None

    metrics = {
        "groundedness_rate": np.mean([r.get("groundedness", 0) for r in recent]),
        "refusal_rate": np.mean([1 if r.get("refused") else 0 for r in recent]),
        "citation_rate": np.mean([1 if r.get("cited_source") else 0 for r in recent]),
        "avg_response_length": np.mean([len(r["answer"]) for r in recent]),
    }

    # User feedback
    feedback = [r for r in recent if r.get("feedback") is not None]
    if feedback:
        positive = sum(1 for f in feedback if f["feedback"] == "positive")
        metrics["user_satisfaction"] = positive / len(feedback)
    else:
        metrics["user_satisfaction"] = None

    return metrics

def calculate_cost_metrics(traces, incidents, window_hours=24):
    """Calculate cost efficiency metrics."""
    cutoff = time.time() - window_hours * 3600
    recent_traces = [t for t in traces if t["timestamp"] > cutoff]
    recent_incidents = [i for i in incidents if i["timestamp"] > cutoff]

    total_cost = sum(t["cost_usd"] for t in recent_traces)
    total_tokens = sum(t["total_tokens"] for t in recent_traces)

    return {
        "total_cost_usd": total_cost,
        "cost_per_request": total_cost / len(recent_traces) if recent_traces else 0,
        "cost_per_incident": total_cost / len(recent_incidents) if recent_incidents else 0,
        "total_tokens": total_tokens,
        "tokens_per_request": total_tokens / len(recent_traces) if recent_traces else 0,
        "cache_hit_savings": calculate_cache_savings(recent_traces),
    }

def calculate_operational_metrics(traces, window_hours=24):
    """Calculate operational health metrics."""
    cutoff = time.time() - window_hours * 3600
    recent = [t for t in traces if t["timestamp"] > cutoff]

    latencies = [t["latency_ms"] for t in recent]

    return {
        "p50_latency_ms": np.percentile(latencies, 50) if latencies else 0,
        "p95_latency_ms": np.percentile(latencies, 95) if latencies else 0,
        "p99_latency_ms": np.percentile(latencies, 99) if latencies else 0,
        "error_rate": sum(1 for t in recent if t.get("error")) / len(recent) if recent else 0,
        "timeout_rate": sum(1 for t in recent if t.get("timeout")) / len(recent) if recent else 0,
        "fallback_rate": sum(1 for t in recent if t.get("used_fallback")) / len(recent) if recent else 0,
        "requests_per_minute": len(recent) / (window_hours * 60),
    }

class QualityAlerts:
    def __init__(self):
        self.thresholds = {
            "groundedness_rate": {"min": 0.85, "window_hours": 1},
            "user_satisfaction": {"min": 0.70, "window_hours": 24},
            "error_rate": {"max": 0.05, "window_hours": 1},
            "p99_latency_ms": {"max": 10000, "window_hours": 1},
            "cost_per_hour": {"max": 50, "window_hours": 1},
        }

    def check_alerts(self, metrics):
        """Check metrics against thresholds and fire alerts."""
        alerts = []
        for metric, config in self.thresholds.items():
            value = metrics.get(metric)
            if value is None:
                continue

            if "min" in config and value < config["min"]:
                alerts.append({
                    "metric": metric,
                    "value": value,
                    "threshold": config["min"],
                    "severity": "warning"
                })

            if "max" in config and value > config["max"]:
                alerts.append({
                    "metric": metric,
                    "value": value,
                    "threshold": config["max"],
                    "severity": "warning"
                })

        return alerts