Serving & Latency SLOs

import openai
from concurrent.futures import ThreadPoolExecutor
import time

def stream_with_timeout(messages, timeout_seconds=10):
    """Stream LLM response with a timeout."""
    start_time = time.time()
    full_response = ""

    for chunk in openai.ChatCompletion.create(
        model="gpt-4",
        messages=messages,
        stream=True
    ):
        elapsed = time.time() - start_time
        if elapsed > timeout_seconds:
            return {
                "response": full_response,
                "truncated": True,
                "elapsed": elapsed
            }

        delta = chunk["choices"][0].get("delta", {})
        if "content" in delta:
            full_response += delta["content"]

    return {
        "response": full_response,
        "truncated": False,
        "elapsed": time.time() - start_time
    }

import time
import random

def call_with_retry(fn, max_retries=3, base_timeout=10, backoff_factor=2):
    """Call a function with timeout and retry logic."""
    for attempt in range(max_retries):
        timeout = base_timeout * (backoff_factor ** attempt)
        try:
            result = fn(timeout=timeout)
            return {"success": True, "result": result, "attempts": attempt + 1}
        except TimeoutError:
            if attempt == max_retries - 1:
                return {"success": False, "error": "Timeout after retries", "attempts": attempt + 1}
            time.sleep(random.uniform(0.5, 1.5))
        except Exception as e:
            return {"success": False, "error": str(e), "attempts": attempt + 1}

class ModelFallbackChain:
    def __init__(self):
        self.chain = [
            {"model": "gpt-4-turbo", "timeout": 10, "cost_per_1k": 0.03},
            {"model": "gpt-4", "timeout": 15, "cost_per_1k": 0.06},
            {"model": "gpt-3.5-turbo", "timeout": 5, "cost_per_1k": 0.002},
        ]

    def call(self, messages, quality_check=None):
        """Try models in order of preference, falling back on failure."""
        for config in self.chain:
            try:
                start = time.time()
                response = openai.ChatCompletion.create(
                    model=config["model"],
                    messages=messages,
                    timeout=config["timeout"]
                )
                elapsed = time.time() - start

                # Optional quality check
                answer = response["choices"][0]["message"]["content"]
                if quality_check and not quality_check(answer):
                    continue

                return {
                    "answer": answer,
                    "model": config["model"],
                    "latency": elapsed,
                    "tokens": response["usage"]["total_tokens"],
                }
            except (TimeoutError, Exception) as e:
                log_fallback(config["model"], str(e))
                continue

        return {"error": "All models failed"}

class LatencyTracker:
    def __init__(self):
        self.measurements = []

    def record(self, feature, latency_ms, slo_ms):
        """Record a latency measurement."""
        self.measurements.append({
            "feature": feature,
            "latency_ms": latency_ms,
            "slo_ms": slo_ms,
            "within_slo": latency_ms <= slo_ms,
            "timestamp": time.time()
        })

    def get_slo_status(self, feature, window_hours=24):
        """Calculate SLO compliance for a feature."""
        cutoff = time.time() - window_hours * 3600
        relevant = [m for m in self.measurements
                   if m["feature"] == feature and m["timestamp"] > cutoff]

        if not relevant:
            return {"feature": feature, "compliance": None}

        within_slo = sum(1 for m in relevant if m["within_slo"])
        compliance = within_slo / len(relevant)

        return {
            "feature": feature,
            "compliance": compliance,
            "target": 0.99,  # 99% SLO
            "breached": compliance < 0.99,
            "p50_latency": sorted([m["latency_ms"] for m in relevant])[len(relevant)//2],
            "p99_latency": sorted([m["latency_ms"] for m in relevant])[int(len(relevant)*0.99)],
        }