[DRAFT] Natural Language Partner Content Search

apps/web/app/(ee)/api/admin/partner-content/backfill/route.ts

@@ -0,0 +1,68 @@
+import { handleAndReturnErrorResponse } from "@/lib/api/errors";
+import { parseRequestBody } from "@/lib/api/utils";
+import { withAdmin } from "@/lib/auth";
+import {
+  createPartnerContentRunStamp,
+  enqueuePartnerContentEnumerate,
+  partnerContentIngestionFilterSchema,
+} from "@/lib/partner-content-search/ingestion/enqueue";
+import { NextResponse } from "next/server";
+import * as z from "zod/v4";
+
+export const dynamic = "force-dynamic";
+export const maxDuration = 10;
+
+const backfillTriggerSchema = z.object({
+  filter: partnerContentIngestionFilterSchema,
+  runStamp: z.string().min(1).optional(),
+  dryRun: z.boolean().default(false),
+  dispatcherDryRun: z.boolean().default(false),
+});
+
+// POST /api/admin/partner-content/backfill
+export const POST = withAdmin(
+  async ({ req }) => {
+    try {
+      // Fail loud on malformed JSON instead of silently widening to a
+      // full backfill; parseRequestBody throws a 400 DubApiError.
+      const body = backfillTriggerSchema.parse(await parseRequestBody(req));
+      const runStamp = body.runStamp ?? createPartnerContentRunStamp();
+
+      const payload = {
+        mode: "backfill" as const,
+        filter: body.filter,
+        runStamp,
+        dryRun: body.dispatcherDryRun,
+      };
+
+      if (body.dryRun) {
+        return NextResponse.json({
+          success: true,
+          triggerDryRun: true,
+          dispatcherDryRun: payload.dryRun,
+          wouldEnqueue: false,
+          enumeratePayload: payload,
+        });
+      }
+
+      const qstashResponse = await enqueuePartnerContentEnumerate(payload);
+
+      return NextResponse.json({
+        success: true,
+        triggerDryRun: false,
+        dispatcherDryRun: payload.dryRun,
+        wouldEnqueue: true,
+        runStamp,
+        enumeratePayload: payload,
+        qstashResponse,
+      });
+    } catch (error) {
+      // Normalize ZodError -> 422, DubApiError -> 4xx, QStash failure -> 500
+      // (the withAdmin wrapper does not do this for us).
+      return handleAndReturnErrorResponse(error);
+    }
+  },
+  {
+    requiredRoles: ["owner"],
+  },
+);

apps/web/app/(ee)/api/admin/partner-content/search/route.ts

@@ -0,0 +1,242 @@
+import { handleAndReturnErrorResponse } from "@/lib/api/errors";
+import { parseRequestBody } from "@/lib/api/utils";
+import { withAdmin } from "@/lib/auth";
+import {
+  PARTNER_CONTENT_SEARCH_LIMITS,
+  PARTNER_CONTENT_SEARCH_MODELS,
+} from "@/lib/partner-content-search/constants";
+import {
+  embedPartnerContentTexts,
+  serializeEmbeddingForVector,
+} from "@/lib/partner-content-search/voyage";
+import { prisma } from "@dub/prisma";
+import { PlatformType, Prisma } from "@dub/prisma/client";
+import { NextResponse } from "next/server";
+import * as z from "zod/v4";
+
+export const dynamic = "force-dynamic";
+export const maxDuration = 30;
+
+const DEFAULT_PARTNER_LIMIT = 10;
+const DEFAULT_CHUNKS_PER_PARTNER = 3;
+
+const partnerContentSearchSchema = z.object({
+  query: z.string().trim().min(1).max(500),
+  limit: z.number().int().positive().max(50).default(DEFAULT_PARTNER_LIMIT),
+  chunksPerPartner: z
+    .number()
+    .int()
+    .positive()
+    .max(10)
+    .default(DEFAULT_CHUNKS_PER_PARTNER),
+  candidateChunkCount: z
+    .number()
+    .int()
+    .positive()
+    .max(PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount)
+    .optional(),
+  partnerIds: z.array(z.string()).min(1).max(100).optional(),
+  platform: z.enum(PlatformType).optional(),
+});
+
+type PartnerContentSearchRow = {
+  chunkId: string;
+  partnerContentItemId: string;
+  partnerId: string;
+  partnerName: string;
+  partnerUsername: string | null;
+  partnerImage: string | null;
+  partnerDescription: string | null;
+  platformType: string;
+  platformIdentifier: string;
+  platformContentId: string;
+  contentUrl: string;
+  contentTitle: string | null;
+  contentThumbnailUrl: string | null;
+  contentPublishedAt: Date | null;
+  chunkIndex: number;
+  chunkText: string;
+  startMs: number | null;
+  endMs: number | null;
+  distance: number | string;
+};
+
+// POST /api/admin/partner-content/search
+export const POST = withAdmin(
+  async ({ req }) => {
+    try {
+      const body = partnerContentSearchSchema.parse(
+        await parseRequestBody(req),
+      );
+      const candidateChunkCount =
+        body.candidateChunkCount ??
+        Math.min(
+          PARTNER_CONTENT_SEARCH_LIMITS.chunkCandidateCount,
+          Math.max(25, body.limit * body.chunksPerPartner * 5),
+        );
+
+      const [queryEmbedding] = await embedPartnerContentTexts({
+        input: [body.query],
+        inputType: "query",
+      });
+
+      const queryVector = serializeEmbeddingForVector(queryEmbedding);
+      const rows = await searchPartnerContentChunks({
+        queryVector,
+        limit: candidateChunkCount,
+        partnerIds: body.partnerIds,
+        platform: body.platform,
+      });
+
+      return NextResponse.json({
+        success: true,
+        query: body.query,
+        candidateChunkCount,
+        embeddingModel: PARTNER_CONTENT_SEARCH_MODELS.embedding.model,
+        resultCount: rows.length,
+        partners: groupPartnerSearchResults({
+          rows,
+          limit: body.limit,
+          chunksPerPartner: body.chunksPerPartner,
+        }),
+      });
+    } catch (error) {
+      return handleAndReturnErrorResponse(error);
+    }
+  },
+  {
+    requiredRoles: ["owner"],
+  },
+);
+
+async function searchPartnerContentChunks({
+  queryVector,
+  limit,
+  partnerIds,
+  platform,
+}: {
+  queryVector: string;
+  limit: number;
+  partnerIds?: string[];
+  platform?: PlatformType;
+}) {
+  const partnerFilter = partnerIds?.length
+    ? Prisma.sql`AND c.partnerId IN (${Prisma.join(partnerIds)})`
+    : Prisma.empty;
+  const platformFilter = platform
+    ? Prisma.sql`AND pp.type = ${platform}`
+    : Prisma.empty;
+
+  return await prisma.$queryRaw<PartnerContentSearchRow[]>(Prisma.sql`
+    SELECT
+      c.id AS chunkId,
+      c.partnerContentItemId,
+      c.partnerId,
+      p.name AS partnerName,
+      p.username AS partnerUsername,
+      p.image AS partnerImage,
+      p.description AS partnerDescription,
+      pp.type AS platformType,
+      pp.identifier AS platformIdentifier,
+      pci.platformContentId,
+      pci.url AS contentUrl,
+      pci.title AS contentTitle,
+      pci.thumbnailUrl AS contentThumbnailUrl,
+      pci.publishedAt AS contentPublishedAt,
+      c.chunkIndex,
+      c.chunkText,
+      c.startMs,
+      c.endMs,
+      DISTANCE(TO_VECTOR(${queryVector}), c.embedding, 'cosine') AS distance
+    FROM PartnerContentChunk c
+    INNER JOIN PartnerContentItem pci ON pci.id = c.partnerContentItemId
+    INNER JOIN Partner p ON p.id = c.partnerId
+    INNER JOIN PartnerPlatform pp ON pp.id = pci.partnerPlatformId
+    WHERE c.embedding IS NOT NULL
+      ${partnerFilter}
+      ${platformFilter}
+    ORDER BY distance ASC
+    LIMIT ${limit}
+  `);
+}
+
+function groupPartnerSearchResults({
+  rows,
+  limit,
+  chunksPerPartner,
+}: {
+  rows: PartnerContentSearchRow[];
+  limit: number;
+  chunksPerPartner: number;
+}) {
+  const partners = new Map<
+    string,
+    {
+      partnerId: string;
+      name: string;
+      username: string | null;
+      image: string | null;
+      description: string | null;
+      bestDistance: number;
+      score: number;
+      chunks: ReturnType<typeof toChunkResult>[];
+    }
+  >();
+
+  for (const row of rows) {
+    const distance = Number(row.distance);
+    const partner = partners.get(row.partnerId) ?? {
+      partnerId: row.partnerId,
+      name: row.partnerName,
+      username: row.partnerUsername,
+      image: row.partnerImage,
+      description: row.partnerDescription,
+      bestDistance: distance,
+      score: toScore(distance),
+      chunks: [],
+    };
+
+    partner.bestDistance = Math.min(partner.bestDistance, distance);
+    partner.score = toScore(partner.bestDistance);
+
+    if (partner.chunks.length < chunksPerPartner) {
+      partner.chunks.push(toChunkResult(row, distance));
+    }
+
+    partners.set(row.partnerId, partner);
+  }
+
+  return Array.from(partners.values())
+    .sort((a, b) => a.bestDistance - b.bestDistance)
+    .slice(0, limit);
+}
+
+function toChunkResult(row: PartnerContentSearchRow, distance: number) {
+  return {
+    chunkId: row.chunkId,
+    partnerContentItemId: row.partnerContentItemId,
+    platform: {
+      type: row.platformType,
+      identifier: row.platformIdentifier,
+    },
+    content: {
+      platformContentId: row.platformContentId,
+      url: row.contentUrl,
+      title: row.contentTitle,
+      thumbnailUrl: row.contentThumbnailUrl,
+      publishedAt: row.contentPublishedAt?.toISOString() ?? null,
+    },
+    chunk: {
+      index: row.chunkIndex,
+      chunkText: row.chunkText,
+      startMs: row.startMs,
+      endMs: row.endMs,
+    },
+    distance,
+    score: toScore(distance),
+  };
+}
+
+function toScore(distance: number) {
+  return Number((1 - distance).toFixed(6));
+}