Secrets Management in the Platform

apiVersion: external-secrets.io/v1beta1
kind: ClusterSecretStore
metadata:
  name: vault-platform
spec:
  provider:
    vault:
      server: "https://vault.platform.example.com"
      path: "secret"
      version: "v2"
      auth:
        kubernetes:
          mountPath: "kubernetes"
          role: "platform-eso"
      # Namespace isolation: each team gets own path
      # namespace: "{{ .metadata.namespace }}"
---
apiVersion: external-secrets.io/v1beta1
kind: ExternalSecret
metadata:
  name: payment-api-secrets
  namespace: payments-payment-api
spec:
  refreshInterval: 1h
  secretStoreRef:
    name: vault-platform
    kind: ClusterSecretStore
  target:
    name: payment-api-secrets
    creationPolicy: Owner
    template:
      engineVersion: v2
      data:
        # Transform secret keys to env var format
        DATABASE_URL: "{{ .postgresql_url | toString }}"
        REDIS_URL: "{{ .redis_url | toString }}"
        KAFKA_BROKERS: "{{ .kafka_brokers | toString }}"
        JWT_SECRET: "{{ .jwt_secret | toString }}"
  data:
    - secretKey: postgresql_url
      remoteRef:
        key: payments/payment-api
        property: database_url
    - secretKey: redis_url
      remoteRef:
        key: payments/payment-api
        property: redis_url
    - secretKey: kafka_brokers
      remoteRef:
        key: shared/kafka
        property: brokers
    - secretKey: jwt_secret
      remoteRef:
        key: platform/jwt
        property: signing_key

# Platform team policy
path "secret/data/platform/*" {
  capabilities = ["create", "read", "update", "delete", "list"]
}

path "secret/metadata/platform/*" {
  capabilities = ["list"]
}

# Payments team policy
path "secret/data/payments/*" {
  capabilities = ["create", "read", "update", "delete", "list"]
}

path "secret/metadata/payments/*" {
  capabilities = ["list"]
}

# Database dynamic secrets
path "database/creds/payments-readonly" {
  capabilities = ["read"]
}

path "database/creds/payments-readwrite" {
  capabilities = ["read"]
}

# Transit encryption
path "transit/encrypt/payments-pii" {
  capabilities = ["update"]
}

path "transit/decrypt/payments-pii" {
  capabilities = ["update"]
}

# Install kubeseal CLI
brew install kubeseal

# Fetch public key from cluster
kubeseal --fetch-cert --controller-name=sealed-secrets --controller-namespace=kube-system > pub-cert.pem

# Create secret locally (never commit raw secret!)
cat > secret.yaml <<EOF
apiVersion: v1
kind: Secret
metadata:
  name: payment-api-secrets
  namespace: payments-payment-api
type: Opaque
stringData:
  DATABASE_URL: "postgresql://user:pass@host:5432/db"
  JWT_SECRET: "super-secret-key"
EOF

# Seal it (encrypt with cluster public key)
kubeseal --cert pub-cert.pem < secret.yaml > sealed-secret.yaml

# sealed-secret.yaml is safe to commit to Git!
cat sealed-secret.yaml
# apiVersion: bitnami.com/v1alpha1
# kind: SealedSecret
# metadata:
#   name: payment-api-secrets
#   namespace: payments-payment-api
# spec:
#   encryptedData:
#     DATABASE_URL: AgBy3i... (encrypted)
#     JWT_SECRET: AgBy3i... (encrypted)
#   template:
#     metadata:
#       name: payment-api-secrets
#       namespace: payments-payment-api

# Commit sealed-secret.yaml to Git
git add sealed-secret.yaml
git commit -m "Add sealed secrets for payment-api"
git push

# ArgoCD applies SealedSecret -> Controller decrypts -> Creates Secret

apiVersion: apps/v1
kind: Deployment
metadata:
  name: payment-api
  annotations:
    reloader.stakater.com/auto: "true"
    # Or specific secrets
    # secret.reloader.stakater.com/reload: "payment-api-secrets,jwt-secret"
spec:
  replicas: 3
  strategy:
    type: RollingUpdate
    rollingUpdate:
      maxSurge: 1
      maxUnavailable: 0
  template:
    spec:
      containers:
        - name: payment-api
          envFrom:
            - secretRef:
                name: payment-api-secrets
          readinessProbe:
            httpGet:
              path: /health
              port: 8080
            initialDelaySeconds: 5
            periodSeconds: 10

# List secrets for a service (no values shown)
$ platform secret list payment-api
NAME              TYPE        LAST_ROTATED   TTL
database_url      dynamic     2024-01-10     24h
redis_url         static      2024-01-05     90d
kafka_brokers     static      2024-01-01     90d
jwt_secret        static      2024-01-15     90d

# Get secret value (requires permission, audited)
$ platform secret get payment-api database_url
postgresql://user:pass@host:5432/db

# Rotate a static secret immediately
$ platform secret rotate payment-api jwt_secret
Rotated jwt_secret for payment-api. New version deployed. Rollout in progress...

# Create a new secret (platform provisions in Vault + syncs to K8s)
$ platform secret create payment-api new_api_key --type=static --ttl=90d
Created new_api_key for payment-api. Available in 30 seconds.

# View rotation history
$ platform secret history payment-api jwt_secret
VERSION  ROTATED_AT          ROTATED_BY      REASON
v3       2024-01-15 10:30    alice@company   Scheduled rotation
v2       2023-10-15 10:30    bob@company     Scheduled rotation
v1       2023-07-15 10:30    charlie@company Initial creation