Node.js Production Troubleshooting - Debug the Shit That Breaks at 3AM

Memory Leaks: The Silent Killers

Your app starts at 200MB RAM. Six hours later it's at 1.8GB and climbing. The V8 heap limit is ~2GB on 64-bit systems - hit it and your app dies with FATAL ERROR: Reached heap limit. Use heap profiling tools and Chrome DevTools to track down the leaks before they kill your production server.

Most Common Culprits:

Global variables that never get cleared:

// WRONG - creates a memory leak
let userCache = new Map();
app.get('/users/:id', (req, res) => {
    userCache.set(req.params.id, userData); // Never cleaned up
});

// RIGHT - use TTL cache with cleanup
const NodeCache = require('node-cache');
const userCache = new NodeCache({ stdTTL: 600 }); // 10-minute expiry

Event listeners that pile up:

// WRONG - adds listener on every request
app.get('/data', (req, res) => {
    req.on('close', handleClose); // Memory leak
});

// RIGHT - remove listeners
app.get('/data', (req, res) => {
    req.on('close', handleClose);
    res.on('finish', () => {
        req.removeListener('close', handleClose);
    });
});

Debugging Memory Leaks - Tools That Actually Work:

**Clinic.js Doctor** - Free and catches most leaks:

npm install -g @nodejs/clinic
clinic doctor -- node app.js
## Let it run for 10+ minutes under load
## Kill with Ctrl+C and check the generated report

**0x Profiler** - Shows exactly what's eating memory:

npm install -g 0x
0x app.js
## Generate load, then kill process
## Opens flame graph showing memory hotspots

Production Memory Monitoring:

// Real production memory monitoring
const memoryUsage = () => {
    const usage = process.memoryUsage();
    console.log({
        rss: Math.round(usage.rss / 1024 / 1024) + 'MB',
        heapUsed: Math.round(usage.heapUsed / 1024 / 1024) + 'MB',
        heapTotal: Math.round(usage.heapTotal / 1024 / 1024) + 'MB',
        external: Math.round(usage.external / 1024 / 1024) + 'MB'
    });
    
    // Kill process if heap usage > 1.5GB (before hitting 2GB limit)
    if (usage.heapUsed > 1.5 * 1024 * 1024 * 1024) {
        console.error('Memory usage too high, restarting...');
        process.exit(1);
    }
};

setInterval(memoryUsage, 30000); // Check every 30 seconds

Event Loop Blocking - When Everything Stops

The event loop is single-threaded. Block it and your entire API becomes unresponsive. I've seen 2-second API responses turn into 30-second timeouts because someone processed a CSV file synchronously.

Event Loop Lag Detection:

const { performance } = require('perf_hooks');

let previousNow = performance.now();
setInterval(() => {
    const now = performance.now();
    const lag = now - previousNow - 1000; // Expected 1000ms interval
    
    if (lag > 100) {
        console.warn(`Event loop lag: ${lag.toFixed(2)}ms`);
        
        // Log stack trace to find the blocking code
        console.trace('Event loop blocked here');
    }
    previousNow = now;
}, 1000);

Common Event Loop Blockers:

Synchronous file operations - Never use these in production:

// WRONG - blocks the event loop completely
const fs = require('fs');
const data = fs.readFileSync('./large-file.json'); // BLOCKS EVERYTHING

// RIGHT - async file operations
const fs = require('fs').promises;
const data = await fs.readFile('./large-file.json'); // Non-blocking

JSON.parse() on large payloads:

// WRONG - blocks on large JSON
app.post('/upload', (req, res) => {
    const data = JSON.parse(req.body); // Can block for seconds
});

// RIGHT - stream processing or worker threads
const { Worker } = require('worker_threads');

app.post('/upload', (req, res) => {
    const worker = new Worker(`
        const { parentPort } = require('worker_threads');
        parentPort.on('message', (data) => {
            try {
                const parsed = JSON.parse(data);
                parentPort.postMessage({ success: true, data: parsed });
            } catch (error) {
                parentPort.postMessage({ success: false, error: error.message });
            }
        });
    `, { eval: true });
    
    worker.postMessage(req.body);
    worker.on('message', (result) => {
        res.json(result);
        worker.terminate();
    });
});

Database Connection Hell

Database connections are where most production Node.js apps die. Connection pools run out, queries hang forever, and suddenly your API returns 500 errors.

Connection Pool Debugging:

// Most apps get this wrong
const mysql = require('mysql2');

const pool = mysql.createPool({
    host: 'localhost',
    user: 'app',
    password: 'secret',
    database: 'production',
    connectionLimit: 10, // Too low for production load
    acquireTimeout: 60000, // Default timeout often too high
    timeout: 60000,
    reconnect: true
});

// RIGHT - production-ready pool with monitoring
const pool = mysql.createPool({
    host: process.env.DB_HOST,
    user: process.env.DB_USER,
    password: process.env.DB_PASSWORD,
    database: process.env.DB_NAME,
    connectionLimit: 50, // Higher limit for production
    acquireTimeout: 10000, // Fail fast on connection issues
    timeout: 30000, // Reasonable query timeout
    reconnect: true,
    multipleStatements: false // Security
});

// Monitor pool health
setInterval(() => {
    console.log('DB Pool Stats:', {
        totalConnections: pool.config.connectionLimit,
        activeConnections: pool._allConnections.length,
        freeConnections: pool._freeConnections.length,
        queuedConnections: pool._connectionQueue.length
    });
    
    // Alert if pool is running low
    const freeConnections = pool._freeConnections.length;
    const totalConnections = pool.config.connectionLimit;
    
    if (freeConnections / totalConnections < 0.2) {
        console.error('Database connection pool running low!');
    }
}, 30000);

Query Timeout Hell:

// Production query with proper timeout handling
const executeQuery = (query, params) => {
    return new Promise((resolve, reject) => {
        const timeout = setTimeout(() => {
            reject(new Error('Query timeout'));
        }, 15000); // 15 second timeout
        
        pool.execute(query, params, (error, results) => {
            clearTimeout(timeout);
            
            if (error) {
                console.error('Query failed:', {
                    query: query.substring(0, 100) + '...',
                    error: error.message,
                    code: error.code,
                    errno: error.errno
                });
                reject(error);
            } else {
                resolve(results);
            }
        });
    });
};

// Usage with error handling
app.get('/users/:id', async (req, res) => {
    try {
        const results = await executeQuery(
            'SELECT * FROM users WHERE id = ?',
            [req.params.id]
        );
        
        if (results.length === 0) {
            return res.status(404).json({ error: 'User not found' });
        }
        
        res.json(results[0]);
    } catch (error) {
        console.error('Database error:', error);
        
        if (error.message === 'Query timeout') {
            res.status(504).json({ error: 'Database timeout' });
        } else {
            res.status(500).json({ error: 'Database error' });
        }
    }
});

Process Crashes and Recovery

Your Node.js process will crash. The question is whether you'll recover gracefully or leave users staring at error pages.

Graceful Shutdown Handling:

// Production-ready graceful shutdown
const gracefulShutdown = (signal) => {
    console.log(`Received ${signal}, starting graceful shutdown...`);
    
    // Stop accepting new connections
    server.close((err) => {
        if (err) {
            console.error('Error during server close:', err);
            process.exit(1);
        }
        
        console.log('HTTP server closed');
        
        // Close database connections
        if (pool) {
            pool.end(() => {
                console.log('Database pool closed');
                process.exit(0);
            });
        } else {
            process.exit(0);
        }
    });
    
    // Force exit after 30 seconds
    setTimeout(() => {
        console.error('Forced exit after timeout');
        process.exit(1);
    }, 30000);
};

// Handle shutdown signals
process.on('SIGTERM', () => gracefulShutdown('SIGTERM'));
process.on('SIGINT', () => gracefulShutdown('SIGINT'));

// Handle uncaught exceptions
process.on('uncaughtException', (error) => {
    console.error('Uncaught Exception:', error);
    
    // Log to external service (Sentry, LogRocket, etc.)
    if (typeof logError === 'function') {
        logError(error);
    }
    
    // Graceful shutdown after logging
    setTimeout(() => {
        process.exit(1);
    }, 1000);
});

// Handle unhandled promise rejections
process.on('unhandledRejection', (reason, promise) => {
    console.error('Unhandled Promise Rejection at:', promise, 'reason:', reason);
    
    // Log the error but don't exit immediately
    if (typeof logError === 'function') {
        logError(reason);
    }
});

The key to production Node.js troubleshooting is preparation. Set up monitoring before things break, because when your app crashes at 3AM, you need data immediately - not time to install debugging tools.

Production Monitoring That Actually Works

Most Node.js monitoring is garbage - generic dashboards that tell you your app is slow after users already hate you. You need specific metrics that catch problems before they become disasters.

Critical Metrics to Monitor

Memory Metrics:

// Custom monitoring that actually helps
const monitorHealth = () => {
    const mem = process.memoryUsage();
    const cpu = process.cpuUsage();
    const eventLoopLag = process.hrtime.bigint();
    
    // Track memory growth rate
    const heapUsedMB = Math.round(mem.heapUsed / 1024 / 1024);
    const rssUsedMB = Math.round(mem.rss / 1024 / 1024);
    
    // Alert thresholds
    const alerts = [];
    
    if (heapUsedMB > 1600) { // 80% of 2GB heap limit
        alerts.push({ type: 'MEMORY_HIGH', heap: heapUsedMB });
    }
    
    if (rssUsedMB > 1800) { // Near container limits
        alerts.push({ type: 'RSS_HIGH', rss: rssUsedMB });
    }
    
    // Track memory growth rate over time
    const now = Date.now();
    const timeDelta = now - (global.lastMemCheck || now);
    const memDelta = heapUsedMB - (global.lastHeapUsed || heapUsedMB);
    
    if (timeDelta > 0) {
        const growthRate = memDelta / (timeDelta / 1000); // MB per second
        
        if (growthRate > 5) { // Growing >5MB/second
            alerts.push({ 
                type: 'MEMORY_LEAK', 
                growthRate: growthRate.toFixed(2) 
            });
        }
    }
    
    global.lastMemCheck = now;
    global.lastHeapUsed = heapUsedMB;
    
    return {
        memory: {
            heap: heapUsedMB,
            rss: rssUsedMB,
            external: Math.round(mem.external / 1024 / 1024)
        },
        cpu: {
            user: Math.round(cpu.user / 1000), // Convert to ms
            system: Math.round(cpu.system / 1000)
        },
        alerts
    };
};

// Check health every 30 seconds
setInterval(() => {
    const health = monitorHealth();
    
    if (health.alerts.length > 0) {
        console.error('HEALTH ALERTS:', health.alerts);
        
        // Send to your alerting system (Slack, PagerDuty, etc.)
        health.alerts.forEach(alert => sendAlert(alert));
    }
    
    // Log metrics for dashboards
    console.log('Health:', JSON.stringify(health, null, 2));
}, 30000);

Database Connection Monitoring

Database issues kill more Node.js apps than bad code. Monitor pool health aggressively:

// Monitor database pool health
const monitorDbPool = (pool) => {
    const stats = {
        total: pool.config.connectionLimit,
        active: pool._allConnections ? pool._allConnections.length : 0,
        free: pool._freeConnections ? pool._freeConnections.length : 0,
        queued: pool._connectionQueue ? pool._connectionQueue.length : 0
    };
    
    // Calculate utilization percentage
    stats.utilization = ((stats.active / stats.total) * 100).toFixed(1);
    
    // Alert if pool is stressed
    const alerts = [];
    
    if (stats.utilization > 80) {
        alerts.push({
            type: 'DB_POOL_HIGH',
            utilization: stats.utilization,
            queued: stats.queued
        });
    }
    
    if (stats.queued > 10) {
        alerts.push({
            type: 'DB_QUEUE_BACKLOG',
            queued: stats.queued
        });
    }
    
    return { stats, alerts };
};

// Monitor every minute
setInterval(() => {
    const dbHealth = monitorDbPool(pool);
    
    if (dbHealth.alerts.length > 0) {
        console.error('DB ALERTS:', dbHealth.alerts);
        dbHealth.alerts.forEach(alert => sendAlert(alert));
    }
    
    console.log('DB Pool:', dbHealth.stats);
}, 60000);

Event Loop Monitoring

The event loop is your app's heartbeat. When it stops, everything dies:

// Event loop lag monitoring
const monitorEventLoop = () => {
    let previousHrtime = process.hrtime.bigint();
    
    setInterval(() => {
        const currentHrtime = process.hrtime.bigint();
        const delta = Number(currentHrtime - previousHrtime);
        const lag = Math.max(0, delta - 1000000000); // Expected 1 second
        const lagMs = lag / 1000000; // Convert to milliseconds
        
        previousHrtime = currentHrtime;
        
        if (lagMs > 100) { // Event loop lag > 100ms
            console.warn(`Event loop lag: ${lagMs.toFixed(2)}ms`);
            
            // Critical lag alerts
            if (lagMs > 1000) {
                sendAlert({
                    type: 'EVENT_LOOP_BLOCKED',
                    lagMs: lagMs.toFixed(2)
                });
            }
        }
        
        // Track event loop utilization
        const elu = performance.eventLoopUtilization();
        const utilization = (elu.active / (elu.active + elu.idle)) * 100;
        
        if (utilization > 90) {
            sendAlert({
                type: 'EVENT_LOOP_SATURATED',
                utilization: utilization.toFixed(1)
            });
        }
        
    }, 1000);
};

monitorEventLoop();

Performance Debugging in Production

Debugging performance issues in production without killing your app requires the right tools and techniques.

Using 0x Profiler Safely

0x is the only profiler I trust in production. It has low overhead and shows real bottlenecks:

## Install globally
npm install -g 0x

## Profile with minimal impact (flame graphs)
0x --open -o ./profiles app.js

## Profile memory allocation patterns
0x --collect-memory-profile app.js

## Profile for specific duration then auto-stop
timeout 300 0x app.js  # Profile for 5 minutes then stop

The flame graphs it generates show exactly which functions consume CPU time. Look for:

• Wide bars (functions that take lots of CPU time)
• Deep stacks (excessive function call depth)
• Red sections (hot paths that need optimization)

Request-Level Debugging

Track slow requests to identify bottlenecks:

// Request performance tracking
app.use((req, res, next) => {
    const startTime = process.hrtime.bigint();
    const startMemory = process.memoryUsage().heapUsed;
    
    // Track when request finishes
    res.on('finish', () => {
        const endTime = process.hrtime.bigint();
        const endMemory = process.memoryUsage().heapUsed;
        
        const duration = Number(endTime - startTime) / 1000000; // Convert to ms
        const memoryDelta = endMemory - startMemory;
        
        const logData = {
            method: req.method,
            url: req.url,
            statusCode: res.statusCode,
            duration: Math.round(duration),
            memory: Math.round(memoryDelta / 1024), // KB
            userAgent: req.get('User-Agent'),
            ip: req.ip
        };
        
        // Log slow requests
        if (duration > 1000) { // >1 second
            console.warn('SLOW REQUEST:', logData);
        }
        
        // Log memory-heavy requests
        if (memoryDelta > 50 * 1024 * 1024) { // >50MB
            console.warn('MEMORY-HEAVY REQUEST:', logData);
        }
        
        // Normal request logging
        console.log('REQUEST:', logData);
    });
    
    next();
});

Error Handling and Recovery Patterns

Production Node.js apps crash. The key is recovering gracefully and learning from failures.

Graceful Degradation

When external services fail, degrade gracefully instead of crashing:

// Circuit breaker pattern for external services
class CircuitBreaker {
    constructor(service, threshold = 5, resetTime = 60000) {
        this.service = service;
        this.failureThreshold = threshold;
        this.resetTime = resetTime;
        this.failureCount = 0;
        this.lastFailureTime = null;
        this.state = 'CLOSED'; // CLOSED, OPEN, HALF_OPEN
    }
    
    async call(request) {
        if (this.state === 'OPEN') {
            const timeSinceLastFailure = Date.now() - this.lastFailureTime;
            
            if (timeSinceLastFailure >= this.resetTime) {
                this.state = 'HALF_OPEN';
                this.failureCount = 0;
            } else {
                throw new Error('Circuit breaker is OPEN');
            }
        }
        
        try {
            const result = await this.service(request);
            
            // Success - reset failure count
            if (this.state === 'HALF_OPEN') {
                this.state = 'CLOSED';
            }
            this.failureCount = 0;
            
            return result;
        } catch (error) {
            this.failureCount++;
            this.lastFailureTime = Date.now();
            
            if (this.failureCount >= this.failureThreshold) {
                this.state = 'OPEN';
                console.error(`Circuit breaker opened after ${this.failureCount} failures`);
            }
            
            throw error;
        }
    }
}

// Usage with external API
const apiCircuitBreaker = new CircuitBreaker(
    async (request) => {
        const response = await fetch(request.url, {
            timeout: 5000, // 5 second timeout
            signal: AbortSignal.timeout(5000)
        });
        
        if (!response.ok) {
            throw new Error(`API error: ${response.status}`);
        }
        
        return response.json();
    },
    3, // Open after 3 failures
    30000 // Reset after 30 seconds
);

// Use with fallback
app.get('/api/user-data/:id', async (req, res) => {
    try {
        const userData = await apiCircuitBreaker.call({
            url: `${process.env.API_URL}/users/${req.params.id}`
        });
        
        res.json(userData);
    } catch (error) {
        console.warn('External API failed, using cached data:', error.message);
        
        // Fallback to cached data
        const cachedData = await getCachedUserData(req.params.id);
        
        if (cachedData) {
            res.json({
                ...cachedData,
                _cached: true,
                _warning: 'Using cached data due to service unavailability'
            });
        } else {
            res.status(503).json({
                error: 'User data temporarily unavailable',
                retryAfter: 30
            });
        }
    }
});

Centralized Error Reporting

Never debug production issues without centralized error reporting:

// Production error reporting
const reportError = (error, context = {}) => {
    const errorData = {
        message: error.message,
        stack: error.stack,
        timestamp: new Date().toISOString(),
        nodeVersion: process.version,
        pid: process.pid,
        memory: process.memoryUsage(),
        uptime: process.uptime(),
        ...context
    };
    
    // Log locally
    console.error('ERROR REPORTED:', JSON.stringify(errorData, null, 2));
    
    // Send to external service (Sentry, Bugsnag, etc.)
    try {
        // Replace with your error reporting service
        sendToErrorService(errorData);
    } catch (reportingError) {
        console.error('Failed to report error:', reportingError);
    }
};

// Global error handlers
process.on('uncaughtException', (error) => {
    reportError(error, {
        type: 'uncaughtException',
        fatal: true
    });
    
    // Graceful shutdown
    process.exit(1);
});

process.on('unhandledRejection', (reason, promise) => {
    reportError(reason, {
        type: 'unhandledRejection',
        promise: promise.toString(),
        fatal: false
    });
});

// Request error handling
app.use((error, req, res, next) => {
    reportError(error, {
        type: 'requestError',
        method: req.method,
        url: req.url,
        userAgent: req.get('User-Agent'),
        ip: req.ip,
        body: req.body
    });
    
    // Don't expose internal errors to users
    if (process.env.NODE_ENV === 'production') {
        res.status(500).json({
            error: 'Internal server error',
            requestId: req.id // Include request ID for support
        });
    } else {
        res.status(500).json({
            error: error.message,
            stack: error.stack
        });
    }
});

Production Node.js troubleshooting is 80% preparation and 20% panic. Set up monitoring, error reporting, and health checks before you need them. When your app crashes at 3AM, you'll thank yourself for the preparation.