api: introduce queues and workers for insights

Pedro Lucas Porcellis porcellis@eletrotupi.com 4 days ago c4e059e5c5f2c31a451ad9b27204e46056841ed3

Parents: 7345b48

10 file(s) changed

api/src/lib/queue/QueueRegistry.ts +5 -5
api/src/lib/queue/WorkerRegistry.ts +26 -8
api/src/lib/queue/index.ts +8 -4
api/src/lib/queue/processors/insights/energySleep.ts +100 -0
api/src/lib/queue/processors/insights/index.ts +62 -0
api/src/lib/queue/processors/insights/moodTrend.ts +81 -0
api/src/lib/queue/processors/insights/triggerPattern.ts +87 -0
api/src/lib/queue/processors/insights/utils.ts +10 -0
api/src/lib/queue/processors/mail.ts +10 -7
api/src/lib/queue/types.ts +32 -7

api/src/lib/queue/QueueRegistry.ts

@@ -1,17 +1,17 @@
  - import { Queue, QueueOptions } from 'bullmq';
  + import { Queue, QueueOptions } from "bullmq";
    
  - export type QueueName = 'mail'; // TODO: add image here later on
  + export type QueueName = "mail" | "insights";
    
    const queues = new Map<QueueName, Queue>();
    
    const defaultOpts: QueueOptions = {
      connection: {
        host: process.env.VALKEY_HOST,
  -     port: Number(process.env.VALKEY_PORT)
  +     port: Number(process.env.VALKEY_PORT),
      },
      defaultJobOptions: {
        attempts: 3,
  -     backoff: { type: 'exponential', delay: 2000 },
  +     backoff: { type: "exponential", delay: 2000 },
        removeOnComplete: { count: 100 },
        removeOnFail: { count: 500 },
      },

@@ -26,5 +26,5 @@ return queues.get(name)!;
    }
    
    export async function closeAllQueues(): Promise<void> {
  -   await Promise.all([...queues.values()].map(q => q.close()));
  +   await Promise.all([...queues.values()].map((q) => q.close()));
    }

api/src/lib/queue/WorkerRegistry.ts

@@ -1,34 +1,40 @@
  - import { Worker, WorkerOptions, Processor } from 'bullmq';
  - import { QueueName } from '@app/lib/queue/QueueRegistry';
  - import { mailProcessor } from  '@app/lib/queue/processors/mail';
  + import { Worker, WorkerOptions, Processor } from "bullmq";
  + import { getQueue, QueueName } from "@app/lib/queue/QueueRegistry";
  + import { mailProcessor } from "@app/lib/queue/processors/mail";
  + import { insightProcessor } from "@app/lib/queue/processors/insights";
  + import { InsightJobName } from "@app/lib/queue/types";
    
    type ProcessorMap = Record<QueueName, Processor>;
    
    const processorMap: ProcessorMap = {
  -   mail: mailProcessor
  +   mail: mailProcessor,
  +   insights: insightProcessor,
    };
    
    // Per-queue concurrency
    // TODO: When we process images, we need to bump for that queue
    const concurrencyMap: Record<QueueName, number> = {
  -   mail: 10
  +   mail: 10,
  +   insights: 5,
    };
    
    const workers: Worker[] = [];
    
    export function bootWorkers(): void {
  +   console.log("Booting workers...");
      for (const [name, processor] of Object.entries(processorMap)) {
        const queueName = name as QueueName;
    
  +     console.log(`Worker ${queueName} booting...`);
        const worker = new Worker(queueName, processor, {
          connection: {
            host: process.env.VALKEY_HOST,
  -         port: Number(process.env.VALKEY_PORT)
  +         port: Number(process.env.VALKEY_PORT),
          },
          concurrency: concurrencyMap[queueName],
        } satisfies WorkerOptions);
    
  -     worker.on('failed', (job, err) => {
  +     worker.on("failed", (job, err) => {
          console.error(`[${queueName}] job ${job?.id} failed:`, err.message);
        });
    

@@ -36,6 +42,18 @@ workers.push(worker);
      }
    }
    
  + export async function scheduleInsights(): Promise<void> {
  +   const queue = getQueue("insights");
  +   await queue.add(
  +     InsightJobName.FanOut,
  +     {},
  +     {
  +       repeat: { pattern: "0 3 * * 1" }, // Mondays at 3am
  +       jobId: "insight-fan-out-weekly", // prevents duplicate repeatable jobs on restart
  +     },
  +   );
  + }
  + 
    export async function closeAllWorkers(): Promise<void> {
  -   await Promise.all(workers.map(w => w.close()));
  +   await Promise.all(workers.map((w) => w.close()));
    }

api/src/lib/queue/index.ts

@@ -1,5 +1,9 @@
  - export { getQueue, closeAllQueues } from '@app/lib/queue/QueueRegistry';
  - export { bootWorkers, closeAllWorkers } from '@app/lib/queue/WorkerRegistry';
  - export { MailJobName }    from '@app/lib/queue/types';
  + export { getQueue, closeAllQueues } from "@app/lib/queue/QueueRegistry";
  + export {
  +   bootWorkers,
  +   closeAllWorkers,
  +   scheduleInsights,
  + } from "@app/lib/queue/WorkerRegistry";
  + export { MailJobName } from "@app/lib/queue/types";
    
  - export type { MailJobData } from '@app/lib/queue/types';
  + export type { MailJobData } from "@app/lib/queue/types";

api/src/lib/queue/processors/insights/energySleep.ts

@@ -0,0 +1,100 @@
  + import { prisma } from "@app/lib/prisma";
  + import { InsightType, InsightPeriod } from "@prisma/client";
  + import { InsightPeriodPayload } from "@app/lib/queue/types";
  + 
  + // Pearson correlation coefficient, returns value in [-1, 1]
  + // TODO: Document this more and write to thesis
  + function pearson(xs: number[], ys: number[]): number {
  +   const n = xs.length;
  +   if (n < 2) return 0;
  + 
  +   const meanX = xs.reduce((a, b) => a + b, 0) / n;
  +   const meanY = ys.reduce((a, b) => a + b, 0) / n;
  + 
  +   const num = xs.reduce((sum, x, i) => sum + (x - meanX) * (ys[i] - meanY), 0);
  +   const denX = Math.sqrt(xs.reduce((sum, x) => sum + (x - meanX) ** 2, 0));
  +   const denY = Math.sqrt(ys.reduce((sum, y) => sum + (y - meanY) ** 2, 0));
  + 
  +   if (denX === 0 || denY === 0) return 0;
  +   return num / (denX * denY);
  + }
  + 
  + export async function processEnergySleep({
  +   userId,
  +   periodStart,
  +   periodEnd,
  + }: InsightPeriodPayload): Promise<void> {
  +   const start = new Date(periodStart);
  +   const end = new Date(periodEnd);
  + 
  +   const sleepRecords = await prisma.sleepRecord.findMany({
  +     where: { userId, date: { gte: start, lte: end } },
  +     orderBy: { date: "asc" },
  +   });
  + 
  +   if (sleepRecords.length < 3) return;
  + 
  +   // For each sleep record, find the closest mood entry on the following day
  +   // XXX: What if the user logs two sleep entries on the same day?
  +   const pairs: Array<{ sleep: number; energy: number }> = [];
  + 
  +   for (const record of sleepRecords) {
  +     const nextDayStart = new Date(record.date);
  +     nextDayStart.setDate(nextDayStart.getDate() + 1);
  +     const nextDayEnd = new Date(nextDayStart);
  +     nextDayEnd.setDate(nextDayEnd.getDate() + 1);
  + 
  +     const mood = await prisma.mood.findFirst({
  +       where: { userId, moment: { gte: nextDayStart, lt: nextDayEnd } },
  +       orderBy: { moment: "asc" },
  +       select: { energyLevel: true },
  +     });
  + 
  +     if (mood) {
  +       pairs.push({ sleep: record.average, energy: mood.energyLevel });
  +     }
  +   }
  + 
  +   if (pairs.length < 3) return;
  + 
  +   const sleepValues = pairs.map((p) => p.sleep);
  +   const energyValues = pairs.map((p) => p.energy);
  +   const score = parseFloat(pearson(sleepValues, energyValues).toFixed(3));
  + 
  +   await prisma.insight.upsert({
  +     where: {
  +       userId_type_periodStart: {
  +         userId,
  +         type: InsightType.ENERGY_SLEEP_CORRELATION,
  +         periodStart: start,
  +       },
  +     },
  +     update: {
  +       body: buildBody(score, pairs.length),
  +       metadata: { correlationScore: score, sampleSize: pairs.length },
  +       generatedAt: new Date(),
  +     },
  +     create: {
  +       userId,
  +       type: InsightType.ENERGY_SLEEP_CORRELATION,
  +       period: InsightPeriod.WEEKLY,
  +       title: "Sono e Energia",
  +       body: buildBody(score, pairs.length),
  +       metadata: { correlationScore: score, sampleSize: pairs.length },
  +       periodStart: start,
  +       periodEnd: end,
  +     },
  +   });
  + }
  + 
  + function buildBody(score: number, sampleSize: number): string {
  +   const base = sampleSize < 5 ? " (dados limitados, continue registrando)" : "";
  + 
  +   if (score > 0.6)
  +     return `Seu sono tem forte impacto positivo na sua energia no dia seguinte.${base}`;
  +   if (score > 0.3)
  +     return `Seu sono tem impacto moderado na sua energia no dia seguinte.${base}`;
  +   if (score < -0.3)
  +     return `Curiosamente, mais sono está associado a menos energia — vale investigar.${base}`;
  +   return `Não encontramos uma relação clara entre seu sono e energia ainda.${base}`;
  + }

api/src/lib/queue/processors/insights/index.ts

@@ -0,0 +1,62 @@
  + import { Job } from "bullmq";
  + import {
  +   InsightJobName,
  +   InsightJobData,
  +   InsightPeriodPayload,
  + } from "@app/lib/queue/types";
  + import { getQueue } from "@app/lib/queue/QueueRegistry";
  + import { prisma } from "@app/lib/prisma";
  + import { processMoodTrend } from "@app/lib/queue/processors/insights/moodTrend";
  + import { processEnergySleep } from "@app/lib/queue/processors/insights/energySleep";
  + import { processTriggerPattern } from "@app/lib/queue/processors/insights/triggerPattern";
  + 
  + export async function insightProcessor(
  +   job: Job<InsightJobData["data"]>,
  + ): Promise<void> {
  +   console.log(`Insight job ${job.name} started`);
  + 
  +   switch (job.name as InsightJobName) {
  +     case InsightJobName.FanOut:
  +       return fanOut();
  + 
  +     case InsightJobName.MoodTrend:
  +       return processMoodTrend(job.data as InsightPeriodPayload);
  + 
  +     case InsightJobName.EnergySleep:
  +       return processEnergySleep(job.data as InsightPeriodPayload);
  + 
  +     case InsightJobName.TriggerPattern:
  +       return processTriggerPattern(job.data as InsightPeriodPayload);
  + 
  +     default:
  +       throw new Error(`Unknown insight job: ${job.name}`);
  +   }
  + }
  + 
  + async function fanOut(): Promise<void> {
  +   const users = await prisma.user.findMany({
  +     where: { active: true },
  +     select: { id: true },
  +   });
  + 
  +   const queue = getQueue("insights");
  +   const periodEnd = new Date();
  +   const periodStart = new Date();
  +   periodStart.setDate(periodEnd.getDate() - 7);
  + 
  +   const payload: Omit<InsightPeriodPayload, "userId"> = {
  +     periodStart: periodStart.toISOString(),
  +     periodEnd: periodEnd.toISOString(),
  +   };
  + 
  +   await queue.addBulk(
  +     users.flatMap((u) => [
  +       { name: InsightJobName.MoodTrend, data: { userId: u.id, ...payload } },
  +       { name: InsightJobName.EnergySleep, data: { userId: u.id, ...payload } },
  +       {
  +         name: InsightJobName.TriggerPattern,
  +         data: { userId: u.id, ...payload },
  +       },
  +     ]),
  +   );
  + }

api/src/lib/queue/processors/insights/moodTrend.ts

@@ -0,0 +1,81 @@
  + import { prisma } from "@app/lib/prisma";
  + import { InsightType, InsightPeriod, BaseMoodOption } from "@prisma/client";
  + import { InsightPeriodPayload } from "@app/lib/queue/types";
  + 
  + const MOOD_SCORE: Record<BaseMoodOption, number> = {
  +   GREAT: 5,
  +   GOOD: 4,
  +   NEUTRAL: 3,
  +   SAD: 2,
  +   ANGRY: 1,
  + };
  + 
  + const avg = (values: number[]) =>
  +   values.reduce((a, b) => a + b, 0) / values.length;
  + 
  + export async function processMoodTrend({
  +   userId,
  +   periodStart,
  +   periodEnd,
  + }: InsightPeriodPayload): Promise<void> {
  +   const start = new Date(periodStart);
  +   const end = new Date(periodEnd);
  + 
  +   const moods = await prisma.mood.findMany({
  +     where: { userId, moment: { gte: start, lte: end } },
  +     orderBy: { moment: "asc" },
  +     select: { selectedMood: true, energyLevel: true, moment: true },
  +   });
  + 
  +   if (moods.length < 3) return;
  + 
  +   const scores = moods.map((m) => MOOD_SCORE[m.selectedMood]);
  +   const mid = Math.floor(scores.length / 2);
  +   const avgFirst = avg(scores.slice(0, mid));
  +   const avgSecond = avg(scores.slice(mid));
  +   const delta = Math.round(((avgSecond - avgFirst) / avgFirst) * 100);
  + 
  +   // Dominant mood in the period
  +   const moodCounts = moods.reduce<Partial<Record<BaseMoodOption, number>>>(
  +     (acc, m) => ({ ...acc, [m.selectedMood]: (acc[m.selectedMood] ?? 0) + 1 }),
  +     {},
  +   );
  +   const dominantMood = (
  +     Object.entries(moodCounts) as [BaseMoodOption, number][]
  +   ).sort((a, b) => b[1] - a[1])[0][0];
  + 
  +   const avgEnergy = avg(moods.map((m) => m.energyLevel));
  + 
  +   await prisma.insight.upsert({
  +     where: {
  +       userId_type_periodStart: {
  +         userId,
  +         type: InsightType.MOOD_TREND,
  +         periodStart: start,
  +       },
  +     },
  +     update: {
  +       body: buildBody(delta),
  +       metadata: { delta, avgFirst, avgSecond, dominantMood, avgEnergy },
  +       generatedAt: new Date(),
  +     },
  +     create: {
  +       userId,
  +       type: InsightType.MOOD_TREND,
  +       period: InsightPeriod.WEEKLY,
  +       title: "Tendência de Humor",
  +       body: buildBody(delta),
  +       metadata: { delta, avgFirst, avgSecond, dominantMood, avgEnergy },
  +       periodStart: start,
  +       periodEnd: end,
  +     },
  +   });
  + }
  + 
  + function buildBody(delta: number): string {
  +   if (delta > 10)
  +     return `Seu humor melhorou ${delta}% em relação ao início do período.`;
  +   if (delta < -10)
  +     return `Seu humor caiu ${Math.abs(delta)}% em relação ao início do período.`;
  +   return "Seu humor se manteve estável durante o período.";
  + }

api/src/lib/queue/processors/insights/triggerPattern.ts

@@ -0,0 +1,87 @@
  + import { prisma } from "@app/lib/prisma";
  + import { InsightType, InsightPeriod, TriggerType } from "@prisma/client";
  + import { InsightPeriodPayload } from "@app/lib/queue/types";
  + 
  + const TRIGGER_LABEL: Record<TriggerType, string> = {
  +   SOCIAL: "social",
  +   WORK: "trabalho",
  +   HEALTH: "saúde",
  +   PHYSICAL: "físico",
  +   FAMILY: "família",
  +   OTHER: "outros",
  + };
  + 
  + export async function processTriggerPattern({
  +   userId,
  +   periodStart,
  +   periodEnd,
  + }: InsightPeriodPayload): Promise<void> {
  +   const start = new Date(periodStart);
  +   const end = new Date(periodEnd);
  + 
  +   const triggers = await prisma.trigger.findMany({
  +     where: { userId, moment: { gte: start, lte: end } },
  +     select: { category: true },
  +   });
  + 
  +   if (triggers.length === 0) return;
  + 
  +   // Count by category
  +   const counts = triggers.reduce<Partial<Record<TriggerType, number>>>(
  +     (acc, t) => ({ ...acc, [t.category]: (acc[t.category] ?? 0) + 1 }),
  +     {},
  +   );
  + 
  +   const sorted = (Object.entries(counts) as [TriggerType, number][]).sort(
  +     (a, b) => b[1] - a[1],
  +   );
  + 
  +   const [topCategory, topCount] = sorted[0];
  +   const distribution = Object.fromEntries(sorted);
  + 
  +   await prisma.insight.upsert({
  +     where: {
  +       userId_type_periodStart: {
  +         userId,
  +         type: InsightType.TRIGGER_PATTERN,
  +         periodStart: start,
  +       },
  +     },
  +     update: {
  +       body: buildBody(topCategory, topCount, triggers.length),
  +       metadata: {
  +         topCategory,
  +         topCount,
  +         total: triggers.length,
  +         distribution,
  +       },
  +       generatedAt: new Date(),
  +     },
  +     create: {
  +       userId,
  +       type: InsightType.TRIGGER_PATTERN,
  +       period: InsightPeriod.WEEKLY,
  +       title: "Padrão de Gatilhos",
  +       body: buildBody(topCategory, topCount, triggers.length),
  +       metadata: {
  +         topCategory,
  +         topCount,
  +         total: triggers.length,
  +         distribution,
  +       },
  +       periodStart: start,
  +       periodEnd: end,
  +     },
  +   });
  + }
  + 
  + function buildBody(top: TriggerType, count: number, total: number): string {
  +   const pct = Math.round((count / total) * 100);
  +   const label = TRIGGER_LABEL[top];
  +   return `
  +     Gatilhos de ${label} foram os mais frequentes,
  +     representando ${pct}% dos registros do período.
  +   `
  +     .replace(/\s*\n\s*/g, " ")
  +     .trim();
  + }

api/src/lib/queue/processors/insights/utils.ts

@@ -0,0 +1,10 @@
  + export function rollingPeriod(days = 7) {
  +   const periodEnd = new Date();
  +   const periodStart = new Date();
  +   periodStart.setDate(periodEnd.getDate() - days);
  + 
  +   return {
  +     periodStart: periodStart.toISOString(),
  +     periodEnd: periodEnd.toISOString(),
  +   };
  + }

api/src/lib/queue/processors/mail.ts

@@ -1,18 +1,21 @@
  - import { Job } from 'bullmq';
  + import { Job } from "bullmq";
    import {
  -   MailJobName, MailJobData,
  +   MailJobName,
  +   MailJobData,
      WelcomeEmailPayload,
      PasswordResetPayload,
  -   ActivateAccountEmailPayload
  - } from '@app/lib/queue/types';
  +   ActivateAccountEmailPayload,
  + } from "@app/lib/queue/types";
    
    import {
      sendWelcomeEmail,
      sendResetPasswordEmail,
  -   sendActivateAccountEmail
  - } from '@app/services/mail'
  +   sendActivateAccountEmail,
  + } from "@app/services/mail";
    
  - export async function mailProcessor(job: Job<MailJobData['data']>): Promise<void> {
  + export async function mailProcessor(
  +   job: Job<MailJobData["data"]>,
  + ): Promise<void> {
      // job.name is the discriminant
      switch (job.name as MailJobName) {
        case MailJobName.WelcomeEmail: {

api/src/lib/queue/types.ts

@@ -1,15 +1,40 @@
    // Mail queue
    export enum MailJobName {
  -   WelcomeEmail         = 'mail:welcome',
  -   PasswordReset        = 'mail:password-reset',
  -   ActivateAccountEmail = 'mail:activate-account'
  +   WelcomeEmail = "mail:welcome",
  +   PasswordReset = "mail:password-reset",
  +   ActivateAccountEmail = "mail:activate-account",
    }
    
  - export type WelcomeEmailPayload    = { userId: number; code: string };
  - export type PasswordResetPayload   = { userId: number; token: string };
  + export type WelcomeEmailPayload = { userId: number; code: string };
  + export type PasswordResetPayload = { userId: number; token: string };
    export type ActivateAccountEmailPayload = { userId: number; code: string };
    
    export type MailJobData =
  -   | { name: MailJobName.WelcomeEmail;  data: WelcomeEmailPayload }
  +   | { name: MailJobName.WelcomeEmail; data: WelcomeEmailPayload }
      | { name: MailJobName.PasswordReset; data: PasswordResetPayload }
  -   | { name: MailJobName.ActivateAccountEmail; data: ActivateAccountEmailPayload };
  +   | {
  +       name: MailJobName.ActivateAccountEmail;
  +       data: ActivateAccountEmailPayload;
  +     };
  + 
  + // Insight queue
  + export enum InsightJobName {
  +   FanOut = "insight:fan-out",
  +   MoodTrend = "insight:mood-trend",
  +   EnergySleep = "insight:energy-sleep",
  +   TriggerPattern = "insight:trigger-pattern",
  + }
  + 
  + export type InsightPeriodPayload = {
  +   userId: number;
  +   periodStart: string;
  +   periodEnd: string;
  + };
  + 
  + export type InsightFanOutPayload = Record<string, never>; // no data, just triggers
  + 
  + export type InsightJobData =
  +   | { name: InsightJobName.FanOut; data: InsightFanOutPayload }
  +   | { name: InsightJobName.MoodTrend; data: InsightPeriodPayload }
  +   | { name: InsightJobName.EnergySleep; data: InsightPeriodPayload }
  +   | { name: InsightJobName.TriggerPattern; data: InsightPeriodPayload };