Architecting Resilient Background Jobs: Dead-Letter Queues and Exponential Backoff

+-------------------------------------------------------------------------------+
| TRANSIENT FAILURES (Recoverable via retry)                                     |
| Network blips, momentary 503 Service Unavailable, connection timeouts.        |
| Strategy: Exponential backoff with randomized jitter.                         |
+-------------------------------------------------------------------------------+
                                      |
+-------------------------------------------------------------------------------+
| RATE LIMITING & BACKPRESSURE (Recoverable with delay)                         |
| HTTP 429 Too Many Requests, Stripe/SendGrid quota exhaustion.                 |
| Strategy: Respect Retry-After headers, delay bucket consumption.              |
+-------------------------------------------------------------------------------+
                                      |
+-------------------------------------------------------------------------------+
| FATAL & MALFORMED ERRORS (Non-recoverable / Poison Pills)                     |
| JSON parsing exceptions, missing mandatory IDs, schema mismatches.           |
| Strategy: Immediate quarantine to Dead-Letter Queue (DLQ); do NOT retry.      |
+-------------------------------------------------------------------------------+

WITHOUT JITTER (Periodic synchronized spikes):
Spike (t=2s)    -----> [ 5,000 requests hit simultaneously ]
Spike (t=4s)    -----> [ 5,000 requests hit simultaneously ]
Spike (t=8s)    -----> [ 5,000 requests hit simultaneously ]

WITH FULL JITTER (Smooth distributed traffic distribution):
Traffic spread  -----> [ Continuous flat load between 0s and 8s, enabling vendor recovery ]

// workers/payment-settlement-worker.ts
import { Worker, Job } from "bullmq";
import { redisConnection } from "@/lib/redis";

export class NonRetryableError extends Error {
  constructor(message: string) {
    super(message);
    this.name = "NonRetryableError";
  }
}

export function calculateFullJitterBackoff(attemptsMade: number): number {
  const baseDelayMs = 1000; // 1 second
  const maxDelayMs = 60000; // 60 seconds
  const exponentialCap = Math.min(maxDelayMs, baseDelayMs * Math.pow(2, attemptsMade));

  // Full jitter: uniformly distributed random delay
  return Math.floor(Math.random() * exponentialCap);
}

export const paymentWorker = new Worker(
  "payment-settlements",
  async (job: Job) => {
    const { orderId, amount, customerId } = job.data;

    // Fast-fail schema validation (poison pill protection)
    if (!orderId || !amount || typeof amount !== "number" || amount <= 0) {
      throw new NonRetryableError(
        `Invalid payment payload for job ${job.id}. Missing orderId or non-positive amount.`
      );
    }

    try {
      console.log(`[Job ${job.id}] Processing settlement for Order ${orderId}...`);
      await settlePaymentWithGateway(orderId, amount, customerId);
      console.log(`[Job ${job.id}] Successfully processed.`);
    } catch (err: any) {
      // Differentiate between transient network issues and permanent authorization failures
      if (err.statusCode === 400 || err.statusCode === 422) {
        throw new NonRetryableError(`Gateway rejected invalid transaction: ${err.message}`);
      }

      // Check for upstream rate limits
      if (err.statusCode === 429) {
        const retryAfter = parseInt(err.headers?.["retry-after"] || "10", 10);
        console.warn(`[Job ${job.id}] Rate limited. Backing off for ${retryAfter}s...`);
        throw err; // Allow worker backoff mechanism to reschedule
      }

      throw err; // Transient error: allow exponential retry
    }
  },
  {
    connection: redisConnection,
    concurrency: 15,
    settings: {
      backoffStrategies: {
        fullJitter(attemptsMade: number) {
          return calculateFullJitterBackoff(attemptsMade);
        },
      },
    },
  }
);

[ Ingestion API ] ----> [ Primary Work Queue: 'billing-jobs' ]
                                     |
                                     v
                           [ Worker Processing ]
                               /              \
           (Success: ACK)     /                \  (Failure: NACK with attempt count)
                             v                  v
                       [ Completed ]      [ Retry with Jitter Backoff ]
                                                |
                                                v (Attempts >= 5 OR NonRetryableError)
                                   +--------------------------------+
                                   | Dead-Letter Queue (DLQ)        |
                                   | 'billing-jobs-dead-letter'     |
                                   +--------------------------------+
                                                |
                                                v
                                  [ PagerDuty Alert & Admin Dashboard ]

// scripts/replay-dlq-jobs.ts
import { Queue } from "bullmq";
import { redisConnection } from "@/lib/redis";

async function replayQuarantinedJobs(queueName: string, batchSize = 100) {
  const primaryQueue = new Queue(queueName, { connection: redisConnection });
  const dlqQueue = new Queue(`${queueName}-dlq`, { connection: redisConnection });

  console.log(`[DLQ Replay] Inspecting failed jobs for queue: ${queueName}...`);
  const failedJobs = await dlqQueue.getFailed(0, batchSize);

  console.log(`[DLQ Replay] Found ${failedJobs.length} failed jobs to evaluate.`);

  for (const job of failedJobs) {
    console.log(`[Replaying] Job ${job.id} - Original Failure Reason: ${job.failedReason}`);

    // Re-inject payload into primary queue with fresh retry counts
    await primaryQueue.add(job.name, job.data, {
      attempts: 3,
      backoff: { type: "fullJitter" },
      removeOnComplete: true,
    });

    // Remove from DLQ quarantine
    await job.remove();
  }

  console.log(`[DLQ Replay] Batch processing complete.`);
}