| 1 | import { Worker, WorkerOptions, Processor } from "bullmq";
|
| 2 | import { getQueue, QueueName } from "@app/lib/queue/QueueRegistry";
|
| 3 | import { mailProcessor } from "@app/lib/queue/processors/mail";
|
| 4 | import { insightProcessor } from "@app/lib/queue/processors/insights";
|
| 5 | import { InsightJobName } from "@app/lib/queue/types";
|
| 6 |
|
| 7 | type ProcessorMap = Record<QueueName, Processor>;
|
| 8 |
|
| 9 | const processorMap: ProcessorMap = {
|
| 10 | mail: mailProcessor,
|
| 11 | insights: insightProcessor,
|
| 12 | };
|
| 13 |
|
| 14 | // Per-queue concurrency
|
| 15 | // TODO: When we process images, we need to bump for that queue
|
| 16 | const concurrencyMap: Record<QueueName, number> = {
|
| 17 | mail: 10,
|
| 18 | insights: 5,
|
| 19 | };
|
| 20 |
|
| 21 | const workers: Worker[] = [];
|
| 22 |
|
| 23 | export function bootWorkers(): void {
|
| 24 | console.log("Booting workers...");
|
| 25 | for (const [name, processor] of Object.entries(processorMap)) {
|
| 26 | const queueName = name as QueueName;
|
| 27 |
|
| 28 | console.log(`Worker ${queueName} booting...`);
|
| 29 | const worker = new Worker(queueName, processor, {
|
| 30 | connection: {
|
| 31 | host: process.env.VALKEY_HOST,
|
| 32 | port: Number(process.env.VALKEY_PORT),
|
| 33 | },
|
| 34 | concurrency: concurrencyMap[queueName],
|
| 35 | } satisfies WorkerOptions);
|
| 36 |
|
| 37 | worker.on("failed", (job, err) => {
|
| 38 | console.error(`[${queueName}] job ${job?.id} failed:`, err.message);
|
| 39 | });
|
| 40 |
|
| 41 | workers.push(worker);
|
| 42 | }
|
| 43 | }
|
| 44 |
|
| 45 | export async function scheduleInsights(): Promise<void> {
|
| 46 | const queue = getQueue("insights");
|
| 47 | await queue.add(
|
| 48 | InsightJobName.FanOut,
|
| 49 | {},
|
| 50 | {
|
| 51 | repeat: { pattern: "0 3 * * 1" }, // Mondays at 3am
|
| 52 | jobId: "insight-fan-out-weekly", // prevents duplicate repeatable jobs on restart
|
| 53 | },
|
| 54 | );
|
| 55 | }
|
| 56 |
|
| 57 | export async function closeAllWorkers(): Promise<void> {
|
| 58 | await Promise.all(workers.map((w) => w.close()));
|
| 59 | }
|