fix: make FTS5 tokenizer configurable per collection

.changeset/fix-search-tokenizers.md

@@ -0,0 +1,5 @@
+---
+"emdash": patch
+---
+
+Adds per-collection search tokenizers so multilingual content can use trigram or non-stemming Unicode search.

docs/src/content/docs/reference/rest-api.mdx

@@ -750,13 +750,47 @@ GET /_emdash/api/search/suggest?q=hel&limit=5
 
 Returns prefix-matched titles for autocomplete.
 
+### Configure Collection Search
+
+```http
+POST /_emdash/api/search/enable
+Content-Type: application/json
+
+{
+  "collection": "posts",
+  "enabled": true,
+  "tokenize": "trigram",
+  "weights": {
+    "title": 10,
+    "content": 1
+  }
+}
+```
+
+The optional `tokenize` field controls how SQLite FTS5 indexes the collection. Changing it on an
+enabled collection rebuilds and repopulates that collection's search index.
+
+| Value                | When to use |
+| -------------------- | ----------- |
+| `porter unicode61`   | Default. English-language content that benefits from Porter stemming, such as matching related word forms. Porter stemming is English-specific. |
+| `unicode61`          | Languages that use word separators but should not use English stemming. |
+| `trigram`            | Languages with text that is not separated by spaces, including Japanese, Chinese, Thai, Khmer, Lao, and Burmese, or when substring matching is needed. Queries shorter than three Unicode characters return no matches. |
+
+Omitting `tokenize` on a collection without a stored tokenizer uses `porter unicode61`. Disabling
+search preserves the configured tokenizer for the next enable operation.
+
 ### Rebuild Search Index
 
 ```http
 POST /_emdash/api/search/rebuild
+Content-Type: application/json
+
+{
+  "collection": "posts"
+}
 ```
 
-Rebuild FTS index for all or specific collections.
+Rebuilds the FTS index for the specified collection using its stored tokenizer and field weights.
 
 ### Search Stats
 

packages/core/src/api/schemas/search.ts

@@ -1,5 +1,6 @@
 import { z } from "zod";
 
+import { SEARCH_TOKENIZERS } from "../../search/types.js";
 import { localeCode } from "./common.js";
 
 // ---------------------------------------------------------------------------
@@ -37,6 +38,7 @@ export const searchEnableBody = z
 		collection: z.string().min(1),
 		enabled: z.boolean(),
 		weights: z.record(z.string(), z.number()).optional(),
+		tokenize: z.enum(SEARCH_TOKENIZERS).optional(),
 	})
 	.meta({ id: "SearchEnableBody" });
 

packages/core/src/astro/routes/api/search/enable.ts

@@ -21,6 +21,7 @@ export const prerender = false;
  * - collection: Collection slug (required)
  * - enabled: boolean (required)
  * - weights: Optional field weights for ranking
+ * - tokenize: Optional FTS5 tokenizer configuration
  */
 export const POST: APIRoute = async ({ request, locals }) => {
 	const { emdash, user } = locals;
@@ -40,7 +41,10 @@ export const POST: APIRoute = async ({ request, locals }) => {
 	try {
 		if (body.enabled) {
 			// Enable search - creates FTS table, triggers, and populates index
-			await ftsManager.enableSearch(body.collection, { weights: body.weights });
+			await ftsManager.enableSearch(body.collection, {
+				weights: body.weights,
+				tokenize: body.tokenize,
+			});
 
 			const stats = await ftsManager.getIndexStats(body.collection);
 

packages/core/src/astro/routes/api/search/rebuild.ts

@@ -57,7 +57,12 @@ export const POST: APIRoute = async ({ request, locals }) => {
 		}
 
 		// Rebuild the index
-		await ftsManager.rebuildIndex(body.collection, searchableFields, config.weights);
+		await ftsManager.rebuildIndex(
+			body.collection,
+			searchableFields,
+			config.weights,
+			config.tokenize,
+		);
 
 		// Get stats after rebuild
 		const stats = await ftsManager.getIndexStats(body.collection);

packages/core/src/database/types.ts

@@ -293,7 +293,7 @@ export interface CollectionTable {
 	icon: string | null;
 	supports: string | null; // JSON array
 	source: string | null;
-	search_config: string | null; // JSON: { enabled: boolean, weights: Record<string, number> }
+	search_config: string | null; // JSON: SearchConfig
 	has_seo: number; // 0 or 1 — opt-in SEO fields for this collection
 	url_pattern: string | null; // URL pattern with {slug} placeholder (e.g. "/blog/{slug}")
 	comments_enabled: Generated<number>; // 0 or 1

packages/core/src/index.ts

@@ -509,6 +509,7 @@ export type {
 
 // Search
 export {
+	SEARCH_TOKENIZERS,
 	FTSManager,
 	search,
 	searchWithDb,
@@ -520,6 +521,7 @@ export {
 } from "./search/index.js";
 export type {
 	SearchConfig,
+	SearchTokenizer,
 	SearchOptions,
 	CollectionSearchOptions,
 	SearchResult,

packages/core/src/schema/registry.ts

@@ -822,7 +822,12 @@ export class SchemaRegistry {
 		const ftsActive = config?.enabled === true;
 
 		if (wantsSearch && searchableFields.length > 0 && ftsActive) {
-			await ftsManager.rebuildIndex(collectionSlug, searchableFields, config?.weights);
+			await ftsManager.rebuildIndex(
+				collectionSlug,
+				searchableFields,
+				config?.weights,
+				config?.tokenize,
+			);
 		} else if (ftsActive && (!wantsSearch || searchableFields.length === 0)) {
 			await ftsManager.disableSearch(collectionSlug);
 		}

packages/core/src/search/fts-manager.ts

@@ -10,7 +10,22 @@ import { sql } from "kysely";
 import { isSqlite, tableExists as dialectTableExists } from "../database/dialect-helpers.js";
 import type { Database } from "../database/types.js";
 import { validateIdentifier } from "../database/validate.js";
-import type { SearchConfig } from "./types.js";
+import { SEARCH_TOKENIZERS } from "./types.js";
+import type { SearchConfig, SearchTokenizer } from "./types.js";
+
+const DEFAULT_SEARCH_TOKENIZER: SearchTokenizer = "porter unicode61";
+
+function isSearchTokenizer(value: unknown): value is SearchTokenizer {
+	return SEARCH_TOKENIZERS.some((tokenizer) => tokenizer === value);
+}
+
+function resolveSearchTokenizer(tokenize?: SearchTokenizer): SearchTokenizer {
+	if (tokenize === undefined) return DEFAULT_SEARCH_TOKENIZER;
+	if (!isSearchTokenizer(tokenize)) {
+		throw new Error(`Unsupported FTS5 tokenizer: "${String(tokenize)}"`);
+	}
+	return tokenize;
+}
 
 /**
  * FTS5 Manager
@@ -66,12 +81,15 @@ export class FTSManager {
 	 * @param collectionSlug - The collection slug
 	 * @param searchableFields - Array of field names to index
 	 * @param weights - Optional field weights for ranking
+	 * @param tokenize - Optional FTS5 tokenizer configuration
 	 */
 	async createFtsTable(
 		collectionSlug: string,
 		searchableFields: string[],
 		_weights?: Record<string, number>,
+		tokenize?: SearchTokenizer,
 	): Promise<void> {
+		const tokenizer = resolveSearchTokenizer(tokenize);
 		if (!isSqlite(this.db)) return;
 		this.validateInputs(collectionSlug, searchableFields);
 		const ftsTable = this.getFtsTableName(collectionSlug);
@@ -88,14 +106,13 @@ export class FTSManager {
 		// `createTriggers` keep the index in sync; they MUST use the
 		// external-content-safe `'delete'` command (see notes there) to
 		// avoid `SQLITE_CORRUPT_VTAB` on UPDATE/DELETE.
-		// tokenize='porter unicode61' enables stemming (run matches running, ran, etc.)
 		await sql
 			.raw(`
 			CREATE VIRTUAL TABLE IF NOT EXISTS "${ftsTable}" USING fts5(
 				${columns},
 				content='${contentTable}',
 				content_rowid='rowid',
-				tokenize='porter unicode61'
+				tokenize='${tokenizer}'
 			)
 		`)
 			.execute(this.db);
@@ -247,13 +264,15 @@ export class FTSManager {
 		collectionSlug: string,
 		searchableFields: string[],
 		weights?: Record<string, number>,
+		tokenize?: SearchTokenizer,
 	): Promise<void> {
+		resolveSearchTokenizer(tokenize);
 		if (!isSqlite(this.db)) return;
 		// Drop existing table and triggers
 		await this.dropFtsTable(collectionSlug);
 
 		// Recreate table and triggers
-		await this.createFtsTable(collectionSlug, searchableFields, weights);
+		await this.createFtsTable(collectionSlug, searchableFields, weights, tokenize);
 
 		// Populate from existing content
 		await this.populateFromContent(collectionSlug, searchableFields);
@@ -314,6 +333,12 @@ export class FTSManager {
 				}
 				config.weights = weights;
 			}
+			if ("tokenize" in parsed) {
+				if (!isSearchTokenizer(parsed.tokenize)) {
+					return null;
+				}
+				config.tokenize = parsed.tokenize;
+			}
 			return config;
 		} catch {
 			return null;
@@ -324,6 +349,9 @@ export class FTSManager {
 	 * Update the search configuration for a collection
 	 */
 	async setSearchConfig(collectionSlug: string, config: SearchConfig): Promise<void> {
+		if (config.tokenize !== undefined) {
+			resolveSearchTokenizer(config.tokenize);
+		}
 		await this.db
 			.updateTable("_emdash_collections")
 			.set({ search_config: JSON.stringify(config) })
@@ -399,8 +427,11 @@ export class FTSManager {
 	 */
 	async enableSearch(
 		collectionSlug: string,
-		options?: { weights?: Record<string, number> },
+		options?: { weights?: Record<string, number>; tokenize?: SearchTokenizer },
 	): Promise<void> {
+		if (options?.tokenize !== undefined) {
+			resolveSearchTokenizer(options.tokenize);
+		}
 		if (!isSqlite(this.db)) {
 			throw new Error("Full-text search is only available with SQLite databases");
 		}
@@ -414,13 +445,18 @@ export class FTSManager {
 			);
 		}
 
+		const existing = await this.getSearchConfig(collectionSlug);
+		const weights = options?.weights ?? existing?.weights;
+		const tokenize = options?.tokenize ?? existing?.tokenize;
+
 		// Rebuild from scratch to ensure clean state (no duplicate rows)
-		await this.rebuildIndex(collectionSlug, searchableFields, options?.weights);
+		await this.rebuildIndex(collectionSlug, searchableFields, weights, tokenize);
 
 		// Update search config
 		await this.setSearchConfig(collectionSlug, {
 			enabled: true,
-			weights: options?.weights,
+			weights,
+			tokenize,
 		});
 	}
 
@@ -433,7 +469,11 @@ export class FTSManager {
 		if (!isSqlite(this.db)) return;
 		await this.dropFtsTable(collectionSlug);
 		const existing = await this.getSearchConfig(collectionSlug);
-		await this.setSearchConfig(collectionSlug, { enabled: false, weights: existing?.weights });
+		await this.setSearchConfig(collectionSlug, {
+			enabled: false,
+			weights: existing?.weights,
+			tokenize: existing?.tokenize,
+		});
 	}
 
 	/**
@@ -490,7 +530,7 @@ export class FTSManager {
 			}
 
 			console.warn(`FTS index for "${collectionSlug}" is missing. Rebuilding.`);
-			await this.rebuildIndex(collectionSlug, fields, config.weights);
+			await this.rebuildIndex(collectionSlug, fields, config.weights, config.tokenize);
 			return true;
 		}
 
@@ -515,7 +555,7 @@ export class FTSManager {
 				`FTS index for "${collectionSlug}" has ${ftsRows} rows but content table has ${contentRows}. Rebuilding.`,
 			);
 			if (fields.length > 0) {
-				await this.rebuildIndex(collectionSlug, fields, config?.weights);
+				await this.rebuildIndex(collectionSlug, fields, config?.weights, config?.tokenize);
 			}
 			return true;
 		}

packages/core/src/search/index.ts

@@ -7,6 +7,7 @@
 // Types
 export type {
 	SearchConfig,
+	SearchTokenizer,
 	SearchOptions,
 	CollectionSearchOptions,
 	SearchResult,
@@ -15,6 +16,7 @@ export type {
 	Suggestion,
 	SearchStats,
 } from "./types.js";
+export { SEARCH_TOKENIZERS } from "./types.js";
 
 // FTS Manager
 export { FTSManager } from "./fts-manager.js";

packages/core/src/search/types.ts

@@ -7,11 +7,17 @@
 /**
  * Search configuration for a collection
  */
+export const SEARCH_TOKENIZERS = ["porter unicode61", "unicode61", "trigram"] as const;
+
+export type SearchTokenizer = (typeof SEARCH_TOKENIZERS)[number];
+
 export interface SearchConfig {
 	/** Whether search is enabled for this collection */
 	enabled: boolean;
 	/** Field weights for ranking (higher = more important) */
 	weights?: Record<string, number>;
+	/** FTS5 tokenizer configuration (defaults to English Porter stemming) */
+	tokenize?: SearchTokenizer;
 }
 
 /**