Kubernetes Network Debugging

# Get pod IPs
kubectl get pods -o wide

# Test from one pod to another
kubectl exec -it pod-a -- curl http://pod-b-ip:8080

# Check pod network namespace
kubectl exec -it debug-pod -- ip addr show
kubectl exec -it debug-pod -- ip route show

# Check if CNI plugin is functioning
kubectl get pods -n kube-system -o wide | grep -E "(calico|flannel|cilium|weave)"

# Test cross-node connectivity
kubectl exec -it pod-node1 -- ping pod-node2-ip

# Check pod's network interface
kubectl exec -it debug-pod -- ip link show eth0

# Check ARP table
kubectl exec -it debug-pod -- ip neigh

# Check iptables rules inside pod
kubectl exec -it debug-pod -- iptables -L -n

# Capture traffic from a pod (requires privileged or NET_ADMIN)
kubectl exec -it debug-pod -- tcpdump -i eth0 -c 10

# Check CNI configuration
kubectl exec -it debug-pod -- cat /etc/cni/net.d/*.conf

# Check service endpoints
kubectl get endpoints my-service

# If endpoints are empty, pods are not matching selectors
kubectl get pods --show-labels | grep app=my-app

# Test service DNS resolution
kubectl exec -it debug-pod -- nslookup my-service

# Test service IP directly
kubectl get svc my-service -o jsonpath='{.spec.clusterIP}'
kubectl exec -it debug-pod -- curl http://<cluster-ip>:80

# Check kube-proxy rules
kubectl -n kube-system logs -l k8s-app=kube-proxy --tail=20

# Verify service selector matches pod labels
kubectl describe svc my-service

# Check CoreDNS pods
kubectl -n kube-system get pods -l k8s-app=kube-dns

# Check CoreDNS logs
kubectl -n kube-system logs -l k8s-app=kube-dns --tail=50

# Test DNS from inside a pod
kubectl run -it --rm debug --image=busybox --restart=Never -- \
  nslookup kubernetes.default

kubectl run -it --rm debug --image=busybox --restart=Never -- \
  nslookup my-service.my-namespace.svc.cluster.local

# Check CoreDNS ConfigMap
kubectl -n kube-system get configmap coredns -o yaml

# Common DNS issues:
# 1. CoreDNS pods not running
# 2. /etc/resolv.conf pointing to wrong DNS
# 3. Search domains missing
# 4. CoreDNS unable to reach upstream DNS

# Check CNI plugin pods
kubectl -n kube-system get pods -o wide

# Calico specific
kubectl -n kube-system logs -l k8s-app=calico-node --tail=20
calicoctl node status

# Cilium specific
kubectl -n kube-system logs -l k8s-app=cilium --tail=20
cilium status
cilium endpoint list

# Flannel specific
kubectl -n kube-system logs -l app=flannel --tail=20

# Check CNI binary
ls -la /opt/cni/bin/

# Check node CNI config
cat /etc/cni/net.d/10-calico.conflist

# List NetworkPolicies
kubectl get networkpolicy -A

# Check if a policy is applied to your pods
kubectl describe networkpolicy my-policy

# Test if traffic is being blocked
# From a debug pod, try to reach the target
kubectl exec -it debug-pod -- curl -v http://my-service:80

# If blocked, check if there is a deny-all policy
kubectl get networkpolicy -A -o yaml | grep "policyTypes.*Ingress"

# Temporarily remove policies for debugging
kubectl delete networkpolicy my-policy
# Test connectivity
# Recreate the policy

# Calico-specific policy debug
kubectl -n kube-system logs -l k8s-app=calico-node | grep -i policy