Stage 7 · Master
HTTP APIs & REST
requests & httpx
GET/POST/PUT/DELETE, auth, sessions, retries, and async HTTP with httpx.
requests Fundamentals
requests is the most-used HTTP library in Python. It is simple, reliable, and covers every DevOps use case. For new code that needs async, use httpx instead.
import requests
# GET request
resp = requests.get("https://api.example.com/health")
print(resp.status_code) # 200
print(resp.json()) # Parse JSON response
print(resp.text) # Raw text
# POST with JSON body
resp = requests.post(
"https://api.example.com/deploy",
json={"service": "web-01", "replicas": 3},
headers={"Authorization": "Bearer token123"},
)
# PUT and DELETE
requests.put("https://api.example.com/config", json={"key": "value"})
requests.delete("https://api.example.com/servers/web-02")requests.get/post/put/delete are convenience wrappers around requests.request(). The json= parameter sets Content-Type to application/json automatically.
Sessions and Retries
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
def create_session() -> requests.Session:
"""Create a session with automatic retries."""
session = requests.Session()
retry = Retry(
total=3,
backoff_factor=0.5, # 0.5s, 1s, 2s
status_forcelist=[429, 500, 502, 503, 504],
)
adapter = HTTPAdapter(max_retries=retry)
session.mount("http://", adapter)
session.mount("https://", adapter)
return session
# Reuse the session across requests (connection pooling)
session = create_session()
for server in ["web-01", "web-02", "web-03"]:
resp = session.get(f"http://{server}:8080/health")
print(f"{server}: {resp.status_code}")Sessions reuse TCP connections, which is faster than creating new connections for each request. Retries handle transient failures automatically.
Authentication
import requests
from requests.auth import HTTPBasicAuth
# Bearer token (most common for APIs)
resp = requests.get(
"https://api.example.com/data",
headers={"Authorization": "Bearer eyJhbGciOiJIUzI1NiJ9..."},
)
# Basic auth
resp = requests.get(
"https://api.example.com/admin",
auth=HTTPBasicAuth("admin", "password"),
)
# API key in header
resp = requests.get(
"https://api.example.com/data",
headers={"X-API-Key": "my-api-key-123"},
)
# API key as query parameter
resp = requests.get(
"https://api.example.com/data",
params={"api_key": "my-api-key-123"},
)
# Token from environment variable
import os
token = os.environ["API_TOKEN"]
resp = requests.get(
"https://api.example.com/data",
headers={"Authorization": f"Bearer {token}"},
)Never hardcode credentials. Read tokens from environment variables, key rings, or a secrets manager. requests supports auth through the auth= parameter or custom headers.
Error Handling
import requests
def api_request(method: str, url: str, **kwargs) -> dict:
"""Make an API request with proper error handling."""
try:
resp = requests.request(method, url, timeout=10, **kwargs)
resp.raise_for_status() # Raises HTTPError for 4xx/5xx
return resp.json()
except requests.exceptions.Timeout:
print(f"Request timed out: {url}")
raise
except requests.exceptions.ConnectionError:
print(f"Connection failed: {url}")
raise
except requests.exceptions.HTTPError as e:
print(f"HTTP {resp.status_code}: {resp.text}")
raise
except requests.exceptions.JSONDecodeError:
print(f"Invalid JSON response from {url}")
raise
# Usage
try:
data = api_request("GET", "https://api.example.com/health")
except requests.exceptions.RequestException as e:
print(f"API request failed: {e}")raise_for_status() converts HTTP error codes into exceptions. Always set a timeout — requests without a timeout can hang forever. catch RequestException as the base class for all requests errors.
httpx — Async HTTP
httpx is a modern HTTP client with async support and a requests-compatible API. Use it when you need concurrent HTTP requests or HTTP/2 support.
import httpx
import asyncio
# Sync usage (drop-in replacement for requests)
with httpx.Client() as client:
resp = client.get("https://api.example.com/health")
print(resp.json())
# Async usage
async def check_services(urls: list[str]) -> list[dict]:
async with httpx.AsyncClient(timeout=10) as client:
tasks = [client.get(url) for url in urls]
responses = await asyncio.gather(*tasks, return_exceptions=True)
results = []
for url, resp in zip(urls, responses):
if isinstance(resp, Exception):
results.append({"url": url, "error": str(resp)})
else:
results.append({"url": url, "status": resp.status_code})
return results
# Run async function
results = asyncio.run(check_services([
"http://web-01:8080/health",
"http://web-02:8080/health",
]))httpx.AsyncClient is the async equivalent of requests.Session. Use async with for automatic connection cleanup. httpx supports HTTP/2, which requests does not.
Production Patterns
import requests
import time
def paginated_get(url: str, token: str, page_size: int = 100) -> list[dict]:
"""Fetch all pages from a paginated API."""
all_items = []
headers = {"Authorization": f"Bearer {token}"}
params = {"per_page": page_size, "page": 1}
while True:
resp = requests.get(url, headers=headers, params=params, timeout=10)
resp.raise_for_status()
data = resp.json()
items = data.get("items", [])
all_items.extend(items)
if len(items) < page_size:
break
params["page"] += 1
return all_items
def rate_limited_request(session: requests.Session, url: str, max_retries: int = 3) -> dict:
"""Handle rate limiting with exponential backoff."""
for attempt in range(max_retries):
resp = session.get(url, timeout=10)
if resp.status_code == 429:
retry_after = int(resp.headers.get("Retry-After", 2 ** attempt))
time.sleep(retry_after)
continue
resp.raise_for_status()
return resp.json()
raise Exception(f"Rate limited after {max_retries} retries")Paginated APIs return a subset of results per request. Follow the pagination until you get fewer items than page_size. Rate limiting requires retry logic with backoff.
httpx has a requests-compatible API and supports async, HTTP/2, and timeouts on the client. For new code, prefer httpx. Use requests only when working with existing codebases.
Mark this lesson complete to store local progress and unlock a cleaner resume path the next time you visit.