From da8b64c93d6e130952d9ecfa7c9ad543c105ee4f Mon Sep 17 00:00:00 2001 From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com> Date: Tue, 28 Jul 2026 02:29:12 -0700 Subject: [PATCH] fix: address self-review findings --- .changeset/fix-search-tokenizers.md | 5 + docs/src/content/docs/reference/rest-api.mdx | 36 +++- packages/core/src/api/schemas/search.ts | 2 + .../src/astro/routes/api/search/enable.ts | 6 +- .../src/astro/routes/api/search/rebuild.ts | 7 +- packages/core/src/database/types.ts | 2 +- packages/core/src/index.ts | 2 + packages/core/src/schema/registry.ts | 7 +- packages/core/src/search/fts-manager.ts | 60 +++++- packages/core/src/search/index.ts | 2 + packages/core/src/search/types.ts | 6 + .../integration/search/tokenizer.test.ts | 177 ++++++++++++++++++ .../core/tests/unit/api/search-schema.test.ts | 47 +++++ 13 files changed, 344 insertions(+), 15 deletions(-) create mode 100644 .changeset/fix-search-tokenizers.md create mode 100644 packages/core/tests/integration/search/tokenizer.test.ts create mode 100644 packages/core/tests/unit/api/search-schema.test.ts diff --git a/.changeset/fix-search-tokenizers.md b/.changeset/fix-search-tokenizers.md new file mode 100644 index 0000000000..f54acd7b5f --- /dev/null +++ b/.changeset/fix-search-tokenizers.md @@ -0,0 +1,5 @@ +--- +"emdash": patch +--- + +Adds per-collection search tokenizers so multilingual content can use trigram or non-stemming Unicode search. diff --git a/docs/src/content/docs/reference/rest-api.mdx b/docs/src/content/docs/reference/rest-api.mdx index 875af32603..7cc818168e 100644 --- a/docs/src/content/docs/reference/rest-api.mdx +++ b/docs/src/content/docs/reference/rest-api.mdx @@ -750,13 +750,47 @@ GET /_emdash/api/search/suggest?q=hel&limit=5 Returns prefix-matched titles for autocomplete. +### Configure Collection Search + +```http +POST /_emdash/api/search/enable +Content-Type: application/json + +{ + "collection": "posts", + "enabled": true, + "tokenize": "trigram", + "weights": { + "title": 10, + "content": 1 + } +} +``` + +The optional `tokenize` field controls how SQLite FTS5 indexes the collection. Changing it on an +enabled collection rebuilds and repopulates that collection's search index. + +| Value | When to use | +| -------------------- | ----------- | +| `porter unicode61` | Default. English-language content that benefits from Porter stemming, such as matching related word forms. Porter stemming is English-specific. | +| `unicode61` | Languages that use word separators but should not use English stemming. | +| `trigram` | Languages with text that is not separated by spaces, including Japanese, Chinese, Thai, Khmer, Lao, and Burmese, or when substring matching is needed. Queries shorter than three Unicode characters return no matches. | + +Omitting `tokenize` on a collection without a stored tokenizer uses `porter unicode61`. Disabling +search preserves the configured tokenizer for the next enable operation. + ### Rebuild Search Index ```http POST /_emdash/api/search/rebuild +Content-Type: application/json + +{ + "collection": "posts" +} ``` -Rebuild FTS index for all or specific collections. +Rebuilds the FTS index for the specified collection using its stored tokenizer and field weights. ### Search Stats diff --git a/packages/core/src/api/schemas/search.ts b/packages/core/src/api/schemas/search.ts index 290c54010a..90c108828a 100644 --- a/packages/core/src/api/schemas/search.ts +++ b/packages/core/src/api/schemas/search.ts @@ -1,5 +1,6 @@ import { z } from "zod"; +import { SEARCH_TOKENIZERS } from "../../search/types.js"; import { localeCode } from "./common.js"; // --------------------------------------------------------------------------- @@ -37,6 +38,7 @@ export const searchEnableBody = z collection: z.string().min(1), enabled: z.boolean(), weights: z.record(z.string(), z.number()).optional(), + tokenize: z.enum(SEARCH_TOKENIZERS).optional(), }) .meta({ id: "SearchEnableBody" }); diff --git a/packages/core/src/astro/routes/api/search/enable.ts b/packages/core/src/astro/routes/api/search/enable.ts index 4cb21c6a3d..b58ba06c35 100644 --- a/packages/core/src/astro/routes/api/search/enable.ts +++ b/packages/core/src/astro/routes/api/search/enable.ts @@ -21,6 +21,7 @@ export const prerender = false; * - collection: Collection slug (required) * - enabled: boolean (required) * - weights: Optional field weights for ranking + * - tokenize: Optional FTS5 tokenizer configuration */ export const POST: APIRoute = async ({ request, locals }) => { const { emdash, user } = locals; @@ -40,7 +41,10 @@ export const POST: APIRoute = async ({ request, locals }) => { try { if (body.enabled) { // Enable search - creates FTS table, triggers, and populates index - await ftsManager.enableSearch(body.collection, { weights: body.weights }); + await ftsManager.enableSearch(body.collection, { + weights: body.weights, + tokenize: body.tokenize, + }); const stats = await ftsManager.getIndexStats(body.collection); diff --git a/packages/core/src/astro/routes/api/search/rebuild.ts b/packages/core/src/astro/routes/api/search/rebuild.ts index e317127485..1b05389eb8 100644 --- a/packages/core/src/astro/routes/api/search/rebuild.ts +++ b/packages/core/src/astro/routes/api/search/rebuild.ts @@ -57,7 +57,12 @@ export const POST: APIRoute = async ({ request, locals }) => { } // Rebuild the index - await ftsManager.rebuildIndex(body.collection, searchableFields, config.weights); + await ftsManager.rebuildIndex( + body.collection, + searchableFields, + config.weights, + config.tokenize, + ); // Get stats after rebuild const stats = await ftsManager.getIndexStats(body.collection); diff --git a/packages/core/src/database/types.ts b/packages/core/src/database/types.ts index 0f4ba13ed7..5eb6112a15 100644 --- a/packages/core/src/database/types.ts +++ b/packages/core/src/database/types.ts @@ -293,7 +293,7 @@ export interface CollectionTable { icon: string | null; supports: string | null; // JSON array source: string | null; - search_config: string | null; // JSON: { enabled: boolean, weights: Record } + search_config: string | null; // JSON: SearchConfig has_seo: number; // 0 or 1 — opt-in SEO fields for this collection url_pattern: string | null; // URL pattern with {slug} placeholder (e.g. "/blog/{slug}") comments_enabled: Generated; // 0 or 1 diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 488fe2789f..f67e85ba4d 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -509,6 +509,7 @@ export type { // Search export { + SEARCH_TOKENIZERS, FTSManager, search, searchWithDb, @@ -520,6 +521,7 @@ export { } from "./search/index.js"; export type { SearchConfig, + SearchTokenizer, SearchOptions, CollectionSearchOptions, SearchResult, diff --git a/packages/core/src/schema/registry.ts b/packages/core/src/schema/registry.ts index a2608f45a2..22c14ad238 100644 --- a/packages/core/src/schema/registry.ts +++ b/packages/core/src/schema/registry.ts @@ -822,7 +822,12 @@ export class SchemaRegistry { const ftsActive = config?.enabled === true; if (wantsSearch && searchableFields.length > 0 && ftsActive) { - await ftsManager.rebuildIndex(collectionSlug, searchableFields, config?.weights); + await ftsManager.rebuildIndex( + collectionSlug, + searchableFields, + config?.weights, + config?.tokenize, + ); } else if (ftsActive && (!wantsSearch || searchableFields.length === 0)) { await ftsManager.disableSearch(collectionSlug); } diff --git a/packages/core/src/search/fts-manager.ts b/packages/core/src/search/fts-manager.ts index c200b74125..d13dbc5de3 100644 --- a/packages/core/src/search/fts-manager.ts +++ b/packages/core/src/search/fts-manager.ts @@ -10,7 +10,22 @@ import { sql } from "kysely"; import { isSqlite, tableExists as dialectTableExists } from "../database/dialect-helpers.js"; import type { Database } from "../database/types.js"; import { validateIdentifier } from "../database/validate.js"; -import type { SearchConfig } from "./types.js"; +import { SEARCH_TOKENIZERS } from "./types.js"; +import type { SearchConfig, SearchTokenizer } from "./types.js"; + +const DEFAULT_SEARCH_TOKENIZER: SearchTokenizer = "porter unicode61"; + +function isSearchTokenizer(value: unknown): value is SearchTokenizer { + return SEARCH_TOKENIZERS.some((tokenizer) => tokenizer === value); +} + +function resolveSearchTokenizer(tokenize?: SearchTokenizer): SearchTokenizer { + if (tokenize === undefined) return DEFAULT_SEARCH_TOKENIZER; + if (!isSearchTokenizer(tokenize)) { + throw new Error(`Unsupported FTS5 tokenizer: "${String(tokenize)}"`); + } + return tokenize; +} /** * FTS5 Manager @@ -66,12 +81,15 @@ export class FTSManager { * @param collectionSlug - The collection slug * @param searchableFields - Array of field names to index * @param weights - Optional field weights for ranking + * @param tokenize - Optional FTS5 tokenizer configuration */ async createFtsTable( collectionSlug: string, searchableFields: string[], _weights?: Record, + tokenize?: SearchTokenizer, ): Promise { + const tokenizer = resolveSearchTokenizer(tokenize); if (!isSqlite(this.db)) return; this.validateInputs(collectionSlug, searchableFields); const ftsTable = this.getFtsTableName(collectionSlug); @@ -88,14 +106,13 @@ export class FTSManager { // `createTriggers` keep the index in sync; they MUST use the // external-content-safe `'delete'` command (see notes there) to // avoid `SQLITE_CORRUPT_VTAB` on UPDATE/DELETE. - // tokenize='porter unicode61' enables stemming (run matches running, ran, etc.) await sql .raw(` CREATE VIRTUAL TABLE IF NOT EXISTS "${ftsTable}" USING fts5( ${columns}, content='${contentTable}', content_rowid='rowid', - tokenize='porter unicode61' + tokenize='${tokenizer}' ) `) .execute(this.db); @@ -247,13 +264,15 @@ export class FTSManager { collectionSlug: string, searchableFields: string[], weights?: Record, + tokenize?: SearchTokenizer, ): Promise { + resolveSearchTokenizer(tokenize); if (!isSqlite(this.db)) return; // Drop existing table and triggers await this.dropFtsTable(collectionSlug); // Recreate table and triggers - await this.createFtsTable(collectionSlug, searchableFields, weights); + await this.createFtsTable(collectionSlug, searchableFields, weights, tokenize); // Populate from existing content await this.populateFromContent(collectionSlug, searchableFields); @@ -314,6 +333,12 @@ export class FTSManager { } config.weights = weights; } + if ("tokenize" in parsed) { + if (!isSearchTokenizer(parsed.tokenize)) { + return null; + } + config.tokenize = parsed.tokenize; + } return config; } catch { return null; @@ -324,6 +349,9 @@ export class FTSManager { * Update the search configuration for a collection */ async setSearchConfig(collectionSlug: string, config: SearchConfig): Promise { + if (config.tokenize !== undefined) { + resolveSearchTokenizer(config.tokenize); + } await this.db .updateTable("_emdash_collections") .set({ search_config: JSON.stringify(config) }) @@ -399,8 +427,11 @@ export class FTSManager { */ async enableSearch( collectionSlug: string, - options?: { weights?: Record }, + options?: { weights?: Record; tokenize?: SearchTokenizer }, ): Promise { + if (options?.tokenize !== undefined) { + resolveSearchTokenizer(options.tokenize); + } if (!isSqlite(this.db)) { throw new Error("Full-text search is only available with SQLite databases"); } @@ -414,13 +445,18 @@ export class FTSManager { ); } + const existing = await this.getSearchConfig(collectionSlug); + const weights = options?.weights ?? existing?.weights; + const tokenize = options?.tokenize ?? existing?.tokenize; + // Rebuild from scratch to ensure clean state (no duplicate rows) - await this.rebuildIndex(collectionSlug, searchableFields, options?.weights); + await this.rebuildIndex(collectionSlug, searchableFields, weights, tokenize); // Update search config await this.setSearchConfig(collectionSlug, { enabled: true, - weights: options?.weights, + weights, + tokenize, }); } @@ -433,7 +469,11 @@ export class FTSManager { if (!isSqlite(this.db)) return; await this.dropFtsTable(collectionSlug); const existing = await this.getSearchConfig(collectionSlug); - await this.setSearchConfig(collectionSlug, { enabled: false, weights: existing?.weights }); + await this.setSearchConfig(collectionSlug, { + enabled: false, + weights: existing?.weights, + tokenize: existing?.tokenize, + }); } /** @@ -490,7 +530,7 @@ export class FTSManager { } console.warn(`FTS index for "${collectionSlug}" is missing. Rebuilding.`); - await this.rebuildIndex(collectionSlug, fields, config.weights); + await this.rebuildIndex(collectionSlug, fields, config.weights, config.tokenize); return true; } @@ -515,7 +555,7 @@ export class FTSManager { `FTS index for "${collectionSlug}" has ${ftsRows} rows but content table has ${contentRows}. Rebuilding.`, ); if (fields.length > 0) { - await this.rebuildIndex(collectionSlug, fields, config?.weights); + await this.rebuildIndex(collectionSlug, fields, config?.weights, config?.tokenize); } return true; } diff --git a/packages/core/src/search/index.ts b/packages/core/src/search/index.ts index 2d4163c189..e9b6ee1fcb 100644 --- a/packages/core/src/search/index.ts +++ b/packages/core/src/search/index.ts @@ -7,6 +7,7 @@ // Types export type { SearchConfig, + SearchTokenizer, SearchOptions, CollectionSearchOptions, SearchResult, @@ -15,6 +16,7 @@ export type { Suggestion, SearchStats, } from "./types.js"; +export { SEARCH_TOKENIZERS } from "./types.js"; // FTS Manager export { FTSManager } from "./fts-manager.js"; diff --git a/packages/core/src/search/types.ts b/packages/core/src/search/types.ts index d6885370c7..638be8214b 100644 --- a/packages/core/src/search/types.ts +++ b/packages/core/src/search/types.ts @@ -7,11 +7,17 @@ /** * Search configuration for a collection */ +export const SEARCH_TOKENIZERS = ["porter unicode61", "unicode61", "trigram"] as const; + +export type SearchTokenizer = (typeof SEARCH_TOKENIZERS)[number]; + export interface SearchConfig { /** Whether search is enabled for this collection */ enabled: boolean; /** Field weights for ranking (higher = more important) */ weights?: Record; + /** FTS5 tokenizer configuration (defaults to English Porter stemming) */ + tokenize?: SearchTokenizer; } /** diff --git a/packages/core/tests/integration/search/tokenizer.test.ts b/packages/core/tests/integration/search/tokenizer.test.ts new file mode 100644 index 0000000000..a502022b02 --- /dev/null +++ b/packages/core/tests/integration/search/tokenizer.test.ts @@ -0,0 +1,177 @@ +import type { Kysely } from "kysely"; +import { sql } from "kysely"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import { ContentRepository } from "../../../src/database/repositories/content.js"; +import type { Database } from "../../../src/database/types.js"; +import { SchemaRegistry } from "../../../src/schema/registry.js"; +import { FTSManager } from "../../../src/search/fts-manager.js"; +import { searchWithDb } from "../../../src/search/query.js"; +import type { SearchConfig, SearchTokenizer } from "../../../src/search/types.js"; +import { setupTestDatabase, teardownTestDatabase } from "../../utils/test-db.js"; + +describe("FTS tokenizers", () => { + let db: Kysely; + let registry: SchemaRegistry; + let repo: ContentRepository; + let ftsManager: FTSManager; + + beforeEach(async () => { + db = await setupTestDatabase(); + registry = new SchemaRegistry(db); + repo = new ContentRepository(db); + ftsManager = new FTSManager(db); + + await registry.createCollection({ + slug: "articles", + label: "Articles", + supports: ["search"], + }); + await registry.createField("articles", { + slug: "title", + label: "Title", + type: "string", + searchable: true, + }); + }); + + afterEach(async () => { + await teardownTestDatabase(db); + }); + + async function createArticle(slug: string, title: string): Promise { + await repo.create({ + type: "articles", + slug, + status: "published", + publishedAt: new Date().toISOString(), + data: { title }, + }); + } + + async function search(query: string): Promise { + const result = await searchWithDb(db, query, { + collections: ["articles"], + status: "published", + }); + return result.items.map((item) => item.slug ?? ""); + } + + async function getFtsDefinition(): Promise { + const result = await sql<{ sql: string }>` + SELECT sql FROM sqlite_master + WHERE type = 'table' AND name = '_emdash_fts_articles' + `.execute(db); + return result.rows[0]?.sql ?? ""; + } + + it("uses Porter stemming by default when tokenize is omitted", async () => { + await createArticle("language", "Relations between languages"); + await ftsManager.enableSearch("articles"); + + expect(await ftsManager.getSearchConfig("articles")).toEqual({ + enabled: true, + }); + expect(await search("relational")).toEqual(["language"]); + expect(await getFtsDefinition()).toContain("tokenize='porter unicode61'"); + }); + + it("re-enables an existing collection with trigram and matches a mid-sentence Japanese term", async () => { + await createArticle("tokyo", "これは東京都の観光案内です"); + await ftsManager.enableSearch("articles"); + + expect(await search("東京都")).toEqual([]); + + await ftsManager.enableSearch("articles", { tokenize: "trigram" }); + + expect(await ftsManager.getSearchConfig("articles")).toEqual({ + enabled: true, + tokenize: "trigram", + }); + expect(await search("東京都")).toEqual(["tokyo"]); + expect(await search("東京")).toEqual([]); + expect(await getFtsDefinition()).toContain("tokenize='trigram'"); + }); + + it("preserves existing weights when changing only the tokenizer", async () => { + await ftsManager.enableSearch("articles", { weights: { title: 10 } }); + + await ftsManager.enableSearch("articles", { tokenize: "trigram" }); + + expect(await ftsManager.getSearchConfig("articles")).toEqual({ + enabled: true, + weights: { title: 10 }, + tokenize: "trigram", + }); + }); + + it("preserves trigram through disable, re-enable, schema rebuild, and missing-index repair", async () => { + await createArticle("tokyo", "これは東京都の観光案内です"); + await ftsManager.enableSearch("articles", { tokenize: "trigram" }); + + await ftsManager.disableSearch("articles"); + expect(await ftsManager.getSearchConfig("articles")).toEqual({ + enabled: false, + tokenize: "trigram", + }); + + await ftsManager.enableSearch("articles"); + expect(await search("東京都")).toEqual(["tokyo"]); + + await registry.createField("articles", { + slug: "summary", + label: "Summary", + type: "text", + searchable: true, + }); + expect(await getFtsDefinition()).toContain("tokenize='trigram'"); + expect(await search("東京都")).toEqual(["tokyo"]); + + await ftsManager.dropFtsTable("articles"); + await expect(ftsManager.verifyAndRepairIndex("articles")).resolves.toBe(true); + expect(await getFtsDefinition()).toContain("tokenize='trigram'"); + expect(await search("東京都")).toEqual(["tokyo"]); + }); + + it("rejects unsupported runtime tokenizer values without replacing config or the index", async () => { + await createArticle("tokyo", "これは東京都の観光案内です"); + await ftsManager.enableSearch("articles", { tokenize: "trigram" }); + const originalDefinition = await getFtsDefinition(); + const invalidTokenizer = "trigram'); DROP TABLE ec_articles; --" as SearchTokenizer; + + await expect( + ftsManager.createFtsTable("articles", ["title"], undefined, invalidTokenizer), + ).rejects.toThrow("Unsupported FTS5 tokenizer"); + expect(await getFtsDefinition()).toBe(originalDefinition); + + await expect( + ftsManager.rebuildIndex("articles", ["title"], undefined, invalidTokenizer), + ).rejects.toThrow("Unsupported FTS5 tokenizer"); + expect(await getFtsDefinition()).toBe(originalDefinition); + expect(await search("東京都")).toEqual(["tokyo"]); + + await expect( + ftsManager.setSearchConfig("articles", { + enabled: true, + tokenize: invalidTokenizer, + }), + ).rejects.toThrow("Unsupported FTS5 tokenizer"); + expect(await ftsManager.getSearchConfig("articles")).toEqual({ + enabled: true, + tokenize: "trigram", + }); + + await db + .updateTable("_emdash_collections") + .set({ + search_config: JSON.stringify({ + enabled: true, + tokenize: invalidTokenizer, + } satisfies SearchConfig), + }) + .where("slug", "=", "articles") + .execute(); + expect(await ftsManager.getSearchConfig("articles")).toBeNull(); + expect(await getFtsDefinition()).toBe(originalDefinition); + }); +}); diff --git a/packages/core/tests/unit/api/search-schema.test.ts b/packages/core/tests/unit/api/search-schema.test.ts new file mode 100644 index 0000000000..f8977c45b4 --- /dev/null +++ b/packages/core/tests/unit/api/search-schema.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it } from "vitest"; + +import { searchEnableBody } from "../../../src/api/schemas/search.js"; + +describe("searchEnableBody", () => { + it.each(["porter unicode61", "unicode61", "trigram"] as const)( + "accepts the %s tokenizer", + (tokenize) => { + expect( + searchEnableBody.parse({ + collection: "posts", + enabled: true, + tokenize, + }), + ).toEqual({ + collection: "posts", + enabled: true, + tokenize, + }); + }, + ); + + it("allows the tokenizer to be omitted", () => { + expect( + searchEnableBody.parse({ + collection: "posts", + enabled: true, + }), + ).toEqual({ + collection: "posts", + enabled: true, + }); + }); + + it.each(["porter", "trigram'); DROP TABLE ec_posts; --"])( + "rejects unsupported tokenizer %s", + (tokenize) => { + expect(() => + searchEnableBody.parse({ + collection: "posts", + enabled: true, + tokenize, + }), + ).toThrow(); + }, + ); +});