Performance Profiling

flame_graph_reading:
  x_axis: "Width of bar = percentage of time"
  y_axis: "Stack depth (caller above, callee below)"

  hot_path: "Wide bar at the top = function spending most time"
  optimization_target: "Widest bars in the middle of the stack"

  common_patterns:
    cpu_hot_path:
      description: "Wide bar in application code"
      action: "Optimize the algorithm or data structure"

    gc_pressure:
      description: "Wide bar in garbage collector"
      action: "Reduce allocations, reuse objects"

    lock_contention:
      description: "Wide bar in mutex/futex"
      action: "Reduce lock scope, use read-write locks"

    io_wait:
      description: "Wide bar in syscall/read/write"
      action: "Batch I/O, use connection pools, async I/O"

  tools:
    go: "go tool pprof -http=:8080 profile.pb.gz"
    python: "py-spy top --pid PID"
    java: "async-profiler, JFR"
    node: "clinic.js, 0x"

# Capture CPU profile
go tool pprof http://localhost:6060/debug/pprof/profile?seconds=30

# Capture memory profile
go tool pprof http://localhost:6060/debug/pprof/heap

# Capture goroutine dump
go tool pprof http://localhost:6060/debug/pprof/goroutine

# Interactive analysis
go tool pprof profile.pb.gz
> top 20          # Show top 20 functions by CPU usage
> list handler    # Show source code for handler function
> web             # Show call graph in browser

# Generate flame graph
go tool pprof -http=:8080 profile.pb.gz

# OpenTelemetry Collector configuration
receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

processors:
  batch:
    timeout: 5s
    send_batch_size: 1024

  tail_sampling:
    decision_wait: 10s
    policies:
      # Always sample errors
      - name: error-policy
        type: status_code
        status_code:
          status_codes: [ERROR]

      # Sample 10% of successful requests
      - name: probabilistic-policy
        type: probabilistic
        probabilistic:
          sampling_percentage: 10

      # Always sample slow requests
      - name: latency-policy
        type: latency
        latency:
          threshold_ms: 1000

exporters:
  jaeger:
    endpoint: jaeger.internal:14250
  prometheus:
    endpoint: 0.0.0.0:8889

continuous_profiling:
  tool: "Pyroscope"
  overhead: "1-2% CPU"
  storage: "S3 + PostgreSQL"

  configuration:
    application_name: "user-api"
    server_address: "http://pyroscope.internal:4040"
    sample_rate: 10  # 10 Hz
    tags:
      environment: "production"
      region: "us-east-1"

  profiling_types:
    - "cpu"
    - "heap"
    - "goroutine"
    - "mutex"
    - "block"

  retention:
    hot: "7 days"
    warm: "30 days"
    cold: "90 days"

  alerts:
    - name: "CPU spike detected"
      condition: "cpu_usage > 2x baseline for 5 minutes"
      action: "Notify on-call, attach flame graph"

    - name: "Memory leak detected"
      condition: "heap_usage increasing for 24 hours"
      action: "Notify on-call, create investigation ticket"

optimization_patterns:
  caching:
    when: "Profiling shows repeated expensive computations or I/O"
    patterns:
      - "In-memory cache with TTL"
      - "Distributed cache (Redis/Memcached)"
      - "HTTP cache headers"
    tradeoff: "Stale data risk, memory usage"

  connection_pooling:
    when: "Profiling shows connection establishment overhead"
    patterns:
      - "Database connection pool"
      - "HTTP client connection pool"
      - "gRPC connection reuse"
    tradeoff: "Pool size tuning, idle connection management"

  batching:
    when: "Profiling shows per-request I/O overhead"
    patterns:
      - "Batch database inserts"
      - "Batch API calls"
      - "Batch log writes"
    tradeoff: "Latency increase, complexity"

  async_processing:
    when: "Profiling shows synchronous blocking"
    patterns:
      - "Message queues for background work"
      - "Goroutines/threads for parallel processing"
      - "Async I/O with callbacks/promises"
    tradeoff: "Complexity, error handling",

  precomputation:
    when: "Profiling shows expensive computations per request"
    patterns:
      - "Precompute results at startup"
      - "Cache computed values"
      - "Use lookup tables"
    tradeoff: "Memory usage, freshness"