Kubernetes Resource Management and Capacity Planning

# The resource block that actually matters
resources:
  requests:
    cpu: "500m"       # 0.5 CPU cores — used for scheduling
    memory: "512Mi"   # 512 MB — used for scheduling
  limits:
    cpu: "2000m"      # 2 CPU cores — throttled if exceeded
    memory: "1Gi"     # 1 GB — OOMKilled if exceeded

# The anti-pattern: no requests, no limits
# Result: pods scheduled arbitrarily, no resource accounting,
# nodes get overloaded, OOM killer makes random choices

# Check current resource utilization for a deployment
kubectl top pods -l app=my-service --containers

# Get utilization at the container level (not just pod)
kubectl top pods --all-namespaces --containers | sort -k4 -rh | head -20

# For historical data, query Prometheus
# Average CPU usage over the past 7 days, per pod

# Actual CPU usage vs requested (lower = over-provisioned)
sum(rate(container_cpu_usage_seconds_total{container!="", namespace="production"}[5m])) by (pod, container)
/
sum(kube_pod_container_resource_requests{resource="cpu", namespace="production"}) by (pod, container)

# Memory: ratio of actual to requested
sum(container_memory_working_set_bytes{container!="", namespace="production"}) by (pod, container)
/
sum(kube_pod_container_resource_requests{resource="memory", namespace="production"}) by (pod, container)

apiVersion: autoscaling.k8s.io/v1
kind: VerticalPodAutoscaler
metadata:
  name: my-service-vpa
spec:
  targetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-service
  updatePolicy:
    updateMode: "Off"   # Start with "Off" — recommendations only, no auto-apply
    # Options: Off (recommend only), Initial (apply on pod creation), Auto (evict and recreate)
  resourcePolicy:
    containerPolicies:
      - containerName: my-service
        minAllowed:
          cpu: "100m"
          memory: "128Mi"
        maxAllowed:
          cpu: "4"
          memory: "8Gi"
        controlledResources: ["cpu", "memory"]

# Check VPA recommendations without applying them
kubectl describe vpa my-service-vpa

# Output includes:
# Lower Bound: minimum safe values
# Target: recommended values based on observed usage
# Upper Bound: maximum based on usage spikes
# Uncapped Target: recommendation without min/max constraints

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: my-service-hpa
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: my-service
  minReplicas: 3
  maxReplicas: 50
  metrics:
    # Scale on CPU — simple but often wrong
    - type: Resource
      resource:
        name: cpu
        target:
          type: Utilization
          averageUtilization: 70

    # Scale on requests per second — usually better for web services
    - type: Pods
      pods:
        metric:
          name: http_requests_per_second
        target:
          type: AverageValue
          averageValue: "1000"

    # Scale on queue depth for async workers
    - type: External
      external:
        metric:
          name: sqs_messages_visible
          selector:
            matchLabels:
              queue: my-worker-queue
        target:
          type: AverageValue
          averageValue: "100"  # Scale so each replica handles ~100 messages

  behavior:
    scaleUp:
      stabilizationWindowSeconds: 60     # Wait 60s before scaling up further
      policies:
        - type: Pods
          value: 4                        # Add at most 4 pods per scaling event
          periodSeconds: 60
    scaleDown:
      stabilizationWindowSeconds: 300    # Wait 5 minutes before scaling down
      policies:
        - type: Percent
          value: 10                       # Remove at most 10% of replicas per event
          periodSeconds: 60

# Common CA configuration gotchas

# 1. PodDisruptionBudgets must be set correctly — CA respects them
apiVersion: policy/v1
kind: PodDisruptionBudget
metadata:
  name: my-service-pdb
spec:
  minAvailable: 2         # Always keep 2 pods running
  selector:
    matchLabels:
      app: my-service

# 2. Scale-down annotations on nodes you want to protect
kubectl annotate node my-critical-node \
  cluster-autoscaler.kubernetes.io/scale-down-disabled=true

# 3. CA logs for debugging why it's not scaling
kubectl logs -n kube-system -l app=cluster-autoscaler | grep -i "scale\|cannot\|unschedulable"

# Capacity planning projection from historical data
import pandas as pd
import numpy as np
from sklearn.linear_model import LinearRegression

def project_cluster_capacity(utilization_history_df, horizon_days=180):
    """
    Project resource needs based on historical utilization trends.
    
    utilization_history_df: DataFrame with columns [date, cpu_cores_used, memory_gb_used]
    """
    df = utilization_history_df.copy()
    df['day_index'] = (df['date'] - df['date'].min()).dt.days
    
    projections = {}
    for resource in ['cpu_cores_used', 'memory_gb_used']:
        X = df[['day_index']].values
        y = df[resource].values
        
        model = LinearRegression()
        model.fit(X, y)
        
        future_day = df['day_index'].max() + horizon_days
        projected_value = model.predict([[future_day]])[0]
        
        # Add 30% headroom for variance + safety margin
        projections[resource] = projected_value * 1.30
        
        print(f"{resource}: currently {y[-1]:.1f}, projected {projected_value:.1f} "
              f"(+{(projected_value/y[-1] - 1)*100:.0f}%), with headroom: {projections[resource]:.1f}")
    
    return projections

apiVersion: v1
kind: ResourceQuota
metadata:
  name: team-backend-quota
  namespace: team-backend
spec:
  hard:
    requests.cpu: "20"           # Max 20 CPU cores requested across all pods
    requests.memory: 40Gi
    limits.cpu: "80"
    limits.memory: 160Gi
    count/pods: "100"            # Max 100 pods in this namespace
    count/persistentvolumeclaims: "20"

---
apiVersion: v1
kind: LimitRange
metadata:
  name: team-backend-defaults
  namespace: team-backend
spec:
  limits:
    - type: Container
      default:                   # Applied when no limits are specified
        cpu: "500m"
        memory: "512Mi"
      defaultRequest:            # Applied when no requests are specified
        cpu: "100m"
        memory: "128Mi"
      max:                       # No single container can exceed these
        cpu: "8"
        memory: "16Gi"