feat(WHE-22): ingest configured topic messages with idempotent persistence

packages/db/drizzle/0002_tough_sandman.sql

@@ -0,0 +1,22 @@
+CREATE TABLE "purchase_messages" (
+	"id" uuid PRIMARY KEY DEFAULT gen_random_uuid() NOT NULL,
+	"household_id" uuid NOT NULL,
+	"sender_member_id" uuid,
+	"sender_telegram_user_id" text NOT NULL,
+	"sender_display_name" text,
+	"raw_text" text NOT NULL,
+	"telegram_chat_id" text NOT NULL,
+	"telegram_message_id" text NOT NULL,
+	"telegram_thread_id" text NOT NULL,
+	"telegram_update_id" text NOT NULL,
+	"message_sent_at" timestamp with time zone,
+	"processing_status" text DEFAULT 'pending' NOT NULL,
+	"ingested_at" timestamp with time zone DEFAULT now() NOT NULL
+);
+--> statement-breakpoint
+ALTER TABLE "purchase_messages" ADD CONSTRAINT "purchase_messages_household_id_households_id_fk" FOREIGN KEY ("household_id") REFERENCES "public"."households"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
+ALTER TABLE "purchase_messages" ADD CONSTRAINT "purchase_messages_sender_member_id_members_id_fk" FOREIGN KEY ("sender_member_id") REFERENCES "public"."members"("id") ON DELETE set null ON UPDATE no action;--> statement-breakpoint
+CREATE INDEX "purchase_messages_household_thread_idx" ON "purchase_messages" USING btree ("household_id","telegram_thread_id");--> statement-breakpoint
+CREATE INDEX "purchase_messages_sender_idx" ON "purchase_messages" USING btree ("sender_telegram_user_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "purchase_messages_household_tg_message_unique" ON "purchase_messages" USING btree ("household_id","telegram_chat_id","telegram_message_id");--> statement-breakpoint
+CREATE UNIQUE INDEX "purchase_messages_household_tg_update_unique" ON "purchase_messages" USING btree ("household_id","telegram_update_id");

packages/db/drizzle/meta/_journal.json

@@ -15,6 +15,13 @@
       "when": 1772669239939,
       "tag": "0001_spicy_sersi",
       "breakpoints": true
+    },
+    {
+      "idx": 2,
+      "version": "7",
+      "when": 1772670548136,
+      "tag": "0002_tough_sandman",
+      "breakpoints": true
     }
   ]
 }

packages/db/package.json

@@ -2,6 +2,9 @@
   "name": "@household/db",
   "private": true,
   "type": "module",
+  "exports": {
+    ".": "./src/index.ts"
+  },
   "scripts": {
     "build": "bun build src/index.ts --outdir dist --target bun",
     "typecheck": "tsgo --project tsconfig.json --noEmit",
@@ -10,7 +13,6 @@
     "seed": "bun run src/seed.ts"
   },
   "dependencies": {
-    "@household/config": "workspace:*",
     "drizzle-orm": "^0.44.5",
     "postgres": "^3.4.7"
   }

packages/db/src/client.ts

@@ -1,12 +1,21 @@
 import postgres from 'postgres'
 import { drizzle } from 'drizzle-orm/postgres-js'
 
-import { env } from '@household/config'
+export interface DbClientOptions {
+  max?: number
+  prepare?: boolean
+}
 
-const queryClient = postgres(env.DATABASE_URL, {
-  prepare: false,
-  max: 5
-})
+export function createDbClient(databaseUrl: string, options: DbClientOptions = {}) {
+  const queryClient = postgres(databaseUrl, {
+    max: options.max ?? 5,
+    prepare: options.prepare ?? false
+  })
 
-export const db = drizzle(queryClient)
-export { queryClient }
+  const db = drizzle(queryClient)
+
+  return {
+    db,
+    queryClient
+  }
+}

packages/db/src/index.ts

@@ -1,2 +1,2 @@
-export { db, queryClient } from './client'
+export { createDbClient } from './client'
 export * as schema from './schema'

packages/db/src/schema.ts

@@ -180,6 +180,45 @@ export const purchaseEntries = pgTable(
   })
 )
 
+export const purchaseMessages = pgTable(
+  'purchase_messages',
+  {
+    id: uuid('id').defaultRandom().primaryKey(),
+    householdId: uuid('household_id')
+      .notNull()
+      .references(() => households.id, { onDelete: 'cascade' }),
+    senderMemberId: uuid('sender_member_id').references(() => members.id, {
+      onDelete: 'set null'
+    }),
+    senderTelegramUserId: text('sender_telegram_user_id').notNull(),
+    senderDisplayName: text('sender_display_name'),
+    rawText: text('raw_text').notNull(),
+    telegramChatId: text('telegram_chat_id').notNull(),
+    telegramMessageId: text('telegram_message_id').notNull(),
+    telegramThreadId: text('telegram_thread_id').notNull(),
+    telegramUpdateId: text('telegram_update_id').notNull(),
+    messageSentAt: timestamp('message_sent_at', { withTimezone: true }),
+    processingStatus: text('processing_status').default('pending').notNull(),
+    ingestedAt: timestamp('ingested_at', { withTimezone: true }).defaultNow().notNull()
+  },
+  (table) => ({
+    householdThreadIdx: index('purchase_messages_household_thread_idx').on(
+      table.householdId,
+      table.telegramThreadId
+    ),
+    senderIdx: index('purchase_messages_sender_idx').on(table.senderTelegramUserId),
+    tgMessageUnique: uniqueIndex('purchase_messages_household_tg_message_unique').on(
+      table.householdId,
+      table.telegramChatId,
+      table.telegramMessageId
+    ),
+    tgUpdateUnique: uniqueIndex('purchase_messages_household_tg_update_unique').on(
+      table.householdId,
+      table.telegramUpdateId
+    )
+  })
+)
+
 export const processedBotMessages = pgTable(
   'processed_bot_messages',
   {
@@ -261,4 +300,5 @@ export type Member = typeof members.$inferSelect
 export type BillingCycle = typeof billingCycles.$inferSelect
 export type UtilityBill = typeof utilityBills.$inferSelect
 export type PurchaseEntry = typeof purchaseEntries.$inferSelect
+export type PurchaseMessage = typeof purchaseMessages.$inferSelect
 export type Settlement = typeof settlements.$inferSelect

packages/db/src/seed.ts

@@ -1,7 +1,5 @@
 import { and, eq } from 'drizzle-orm'
-import { drizzle } from 'drizzle-orm/postgres-js'
-import postgres from 'postgres'
-
+import { createDbClient } from './client'
 import {
   billingCycles,
   households,
@@ -20,13 +18,11 @@ if (!databaseUrl) {
   throw new Error('DATABASE_URL is required for db seed')
 }
 
-const queryClient = postgres(databaseUrl, {
-  prepare: false,
-  max: 2
+const { db, queryClient } = createDbClient(databaseUrl, {
+  max: 2,
+  prepare: false
 })
 
-const db = drizzle(queryClient)
-
 const FIXTURE_IDS = {
   household: '11111111-1111-4111-8111-111111111111',
   cycle: '22222222-2222-4222-8222-222222222222',