configurable language for fulltext-search

.devcontainer/.env.devcontainer

@@ -31,6 +31,8 @@ KBIN_META_KEYWORDS="mbin, content aggregator, open source, fediverse"
 KBIN_HEADER_LOGO=false
 KBIN_FEDERATION_PAGE_ENABLED=true
 MBIN_DEFAULT_THEME=default
+# Language used to parse content and search-expressions for ranking
+MBIN_SEARCH_LANGUAGE=english
 # Set the max image file size (in bytes)
 # This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file
 MBIN_MAX_IMAGE_BYTES=6000000

.env.example

@@ -31,6 +31,8 @@ KBIN_META_KEYWORDS="mbin, content aggregator, open source, fediverse"
 KBIN_HEADER_LOGO=false
 KBIN_FEDERATION_PAGE_ENABLED=true
 MBIN_DEFAULT_THEME=default
+# Language used to parse content and search-expressions for ranking
+MBIN_SEARCH_LANGUAGE=english
 # Set the max image file size (in bytes)
 # This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file
 MBIN_MAX_IMAGE_BYTES=6000000

.env.example_docker

@@ -36,6 +36,8 @@ KBIN_META_KEYWORDS="mbin, content aggregator, open source, fediverse"
 KBIN_HEADER_LOGO=false
 KBIN_FEDERATION_PAGE_ENABLED=true
 MBIN_DEFAULT_THEME=default
+# Language used to parse content and search-expressions for ranking
+MBIN_SEARCH_LANGUAGE=english
 # Set the max image file size (in bytes)
 # This should be set to <= `upload_max_filesize` and `post_max_size` in the server's php.ini file
 MBIN_MAX_IMAGE_BYTES=6000000
@@ -102,15 +104,6 @@ S3_REGION=
 S3_ENDPOINT=
 S3_VERSION=
 
-# Only let admins generate oauth clients
-KBIN_ADMIN_ONLY_OAUTH_CLIENTS=false
-
-# Manually approve every new user
-MBIN_NEW_USERS_NEED_APPROVAL=false
-
-# use an allowlist instead of a ban list
-MBIN_USE_FEDERATION_ALLOW_LIST=false
-
 # oAuth (optional)
 OAUTH_AZURE_ID=
 OAUTH_AZURE_SECRET=

config/services.yaml

@@ -126,6 +126,9 @@ parameters:
     mbin_new_users_need_approval: '%env(bool:default::MBIN_NEW_USERS_NEED_APPROVAL)%'
     mbin_use_federation_allow_list: '%env(bool:default::MBIN_USE_FEDERATION_ALLOW_LIST)%'
 
+    mbin_search_lang_default: 'english'
+    mbin_search_lang: '%env(string:default:mbin_search_lang_default:MBIN_SEARCH_LANGUAGE)%'
+
     trueVal: 'true'
     mbin_monitoring_enabled: '%env(bool:default::MBIN_MONITORING_ENABLED)%'
     mbin_monitoring_query_parameters_enabled: '%env(bool:default::MBIN_MONITORING_QUERY_PARAMETERS_ENABLED)%'
@@ -216,6 +219,7 @@ services:
             $mbinDownvotesMode: '%mbin_downvotes_mode%'
             $mbinNewUsersNeedApproval: '%mbin_new_users_need_approval%'
             $mbinUseFederationAllowList: '%mbin_use_federation_allow_list%'
+            $mbinSearchLang: '%mbin_search_lang%'
 
     # Markdown
     App\Markdown\Factory\EnvironmentFactory:

docs/02-admin/04-running-mbin/01-first_setup.md

@@ -4,7 +4,7 @@
 > If you are running docker, then you have to prefix the following commands with
 > `docker compose exec php`.
 
-Create new admin user (without email verification), please change the `username`, `email` and `password` below:
+Create new admin user (without email verification). Please change the `username`, `email` and `password` below:
 
 ```bash
 php bin/console mbin:user:create <username> <email@example.com> <password>
@@ -26,12 +26,28 @@ php bin/console mbin:magazine:create random
 
 ### Manual user activation
 
-Activate a user account (bypassing email verification), please change the `username` below:
+If you need to activate a user account manually (bypassing email verification) run the following command.
+Please change the `username` below:
 
 ```bash
 php bin/console mbin:user:verify <username> -a
 ```
 
+### Setup of search language
+
+If you want your instance to use a language different from English for indexing content for the search,
+then you need to adjust the `MBIN_SEARCH_LANGUAGE` environment variable in the `.env` file.
+
+After changing the setting, the server needs to be restarted and also the following command to be run:
+```bash
+php bin/console mbin:db:migrate-search-lang
+```
+
+To see which languages are supported by your database, run the following SQL query:
+```sql
+SELECT cfgname FROM pg_ts_config;
+```
+
 ### Mercure
 
 If you are not going to use Mercure, you have to disable it in the admin panel.

docs/02-admin/04-running-mbin/05-cli.md

@@ -527,6 +527,16 @@ Usage:
 php bin/console mbin:user:create [-r|--remove] [--admin] [--moderator] <username> <email> <password>
 ```
 
+### Migrate-Search-Lang
+
+This command recreates all ts_vector columns and indexes in the database used for the search.
+As the index is language-sensitive, it needs to be recreated whenever you change the language 
+in the `MBIN_SEARCH_LANGUAGE` environment variable or after installation with a language different from `english`.
+
+```bash
+php bin/console mbin:db:migrate-search-lang
+```
+
 ### Update-Local-Domain
 
 This command will remove all remote posts from belonging to the local domain. This command is only relevant for instances

docs/02-admin/FAQ.md

@@ -53,8 +53,6 @@ Read more below about AMQProxy.
 
 ## What is AMQProxy?
 
-AMQProxy is a proxy service for AMQP (Advanced Message Queuing Protocol) most used with message brokers like RabbitMQ. It allows for channel pooling and reusing, hence reducing the AMQP protocol (TCP packages) overhead.
-
 AMQProxy is a proxy service for AMQP (Advanced Message Queuing Protocol), most often used with message brokers like RabbitMQ. It allows for channel pooling and reuse, significantly reducing AMQP protocol overhead and TCP connection.  
 By maintaining persistent connections to the broker, AMQProxy minimizes connection setup latency and resource consumption, improving throughput and scalability for high-load applications. It also simplifies client configuration and load balancing by acting as a single entry point between multiple clients and one (or more) RabbitMQ instances.
 

src/Command/MigrateDbTsCommand.php

@@ -0,0 +1,76 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Command;
+
+use App\Service\SettingsManager;
+use Doctrine\DBAL\Connection;
+use Doctrine\ORM\EntityManagerInterface;
+use Symfony\Component\Console\Attribute\AsCommand;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Style\SymfonyStyle;
+
+#[AsCommand(
+    name: 'mbin:db:migrate-search-lang',
+    description: 'Migrates all ts_vector columns to the current language',
+)]
+class MigrateDbTsCommand extends Command
+{
+    public function __construct(
+        private readonly EntityManagerInterface $entityManager,
+        private readonly SettingsManager $settingsManager,
+    ) {
+        parent::__construct();
+    }
+
+    protected function execute(InputInterface $input, OutputInterface $output): int
+    {
+        $io = new SymfonyStyle($input, $output);
+        $lang = $this->settingsManager->getSearchLang();
+
+        if (!$this->checkLanguage($lang)) {
+            $io->error("the language '$lang' is not supported by the database");
+
+            return Command::FAILURE;
+        }
+        $io->info("migrating ts_vectors to '$lang'");
+
+        $conn = $this->entityManager->getConnection();
+        $this->recreateColumn($conn, 'entry', 'title_ts', 'title', 'entry_title_ts_idx', $lang, $io);
+        $this->recreateColumn($conn, 'entry', 'body_ts', 'body', 'entry_body_ts_idx', $lang, $io);
+        $this->recreateColumn($conn, 'post', 'body_ts', 'body', 'post_body_ts_idx', $lang, $io);
+        $this->recreateColumn($conn, 'post_comment', 'body_ts', 'body', 'post_comment_body_ts_idx', $lang, $io);
+        $this->recreateColumn($conn, 'entry_comment', 'body_ts', 'body', 'entry_comment_body_ts_idx', $lang, $io);
+        $this->recreateColumn($conn, 'magazine', 'name_ts', 'name', 'magazine_name_ts', $lang, $io);
+        $this->recreateColumn($conn, 'magazine', 'title_ts', 'title', 'magazine_title_ts', $lang, $io);
+        $this->recreateColumn($conn, 'magazine', 'description_ts', 'description', 'magazine_description_ts', $lang, $io);
+        $this->recreateColumn($conn, '"user"', 'username_ts', 'username', 'user_username_ts', $lang, $io);
+        $this->recreateColumn($conn, '"user"', 'title_ts', 'title', 'user_title_ts', $lang, $io);
+        $this->recreateColumn($conn, '"user"', 'about_ts', 'about', 'user_about_ts', $lang, $io);
+
+        $io->success('done');
+
+        return Command::SUCCESS;
+    }
+
+    private function checkLanguage(string $lang): bool
+    {
+        $conn = $this->entityManager->getConnection();
+        $supportedLanguages = $conn->executeQuery('SELECT cfgname FROM pg_ts_config;')->fetchFirstColumn();
+
+        return \in_array($lang, $supportedLanguages, true);
+    }
+
+    private function recreateColumn(Connection $conn, string $table, string $column, string $srcColumn, string $idxName, string $lang, SymfonyStyle $io): void
+    {
+        $conn->executeStatement("DROP INDEX $idxName;");
+        $conn->executeStatement("ALTER TABLE $table DROP COLUMN $column;");
+        $conn->executeStatement("ALTER TABLE $table ADD COLUMN $column tsvector GENERATED ALWAYS AS (to_tsvector('$lang', $srcColumn)) STORED;");
+        $conn->executeStatement("CREATE INDEX $idxName ON $table USING GIN ($column);");
+
+        $io->writeln("$table.$column");
+    }
+}

src/Repository/SearchRepository.php

@@ -10,6 +10,7 @@
 use App\Entity\User;
 use App\Pagination\NativeQueryAdapter;
 use App\Pagination\Transformation\ContentPopulationTransformer;
+use App\Service\SettingsManager;
 use Doctrine\DBAL\Exception;
 use Doctrine\ORM\EntityManagerInterface;
 use Pagerfanta\Adapter\AdapterInterface;
@@ -29,6 +30,7 @@ public function __construct(
         private readonly CacheInterface $cache,
         private readonly LoggerInterface $logger,
         private readonly Security $security,
+        private readonly SettingsManager $settingsManager,
     ) {
     }
 
@@ -184,10 +186,10 @@ public function search(
         $createdWhereUser = null !== $sinceDate ? 'AND u.created_at >= :since' : '';
         $blockMagazineAndUserResult = null !== $authorId || null !== $magazineId ? 'AND false' : '';
         $conn = $this->entityManager->getConnection();
-        $sqlEntry = "SELECT e.id, e.created_at, e.visibility, 2 * ts_rank_cd(e.title_ts, plainto_tsquery(:query)) + ts_rank_cd(e.body_ts, plainto_tsquery(:query)) as rank, 'entry' AS type FROM entry e
+        $sqlEntry = "SELECT e.id, e.created_at, e.visibility, 2 * ts_rank_cd(e.title_ts, plainto_tsquery(:tsLang, :query)) + ts_rank_cd(e.body_ts, plainto_tsquery(:tsLang, :query)) as rank, 'entry' AS type FROM entry e
             INNER JOIN public.user u ON u.id = user_id
             INNER JOIN magazine m ON e.magazine_id = m.id
-            WHERE (e.body_ts @@ plainto_tsquery( :query ) = true OR e.title_ts @@ plainto_tsquery( :query ) = true OR e.title LIKE :likeQuery)
+            WHERE (e.body_ts @@ plainto_tsquery( :tsLang, :query ) = true OR e.title_ts @@ plainto_tsquery( :tsLang, :query ) = true OR e.title LIKE :likeQuery)
                 AND e.visibility = :visibility
                 AND u.is_deleted = false
                 AND (u.ap_discoverable = true OR u.ap_discoverable IS NULL)
@@ -197,10 +199,10 @@ public function search(
                 AND NOT EXISTS (SELECT hl.id FROM hashtag_link hl INNER JOIN hashtag h ON h.id = hl.hashtag_id AND h.banned = true WHERE hl.entry_id = e.id)
                 $authorWhere $magazineWhere $createdWhere
         UNION ALL
-        SELECT e.id, e.created_at, e.visibility, 3 * ts_rank_cd(e.body_ts, plainto_tsquery(:query)) as rank, 'entry_comment' AS type FROM entry_comment e
+        SELECT e.id, e.created_at, e.visibility, 3 * ts_rank_cd(e.body_ts, plainto_tsquery(:tsLang, :query)) as rank, 'entry_comment' AS type FROM entry_comment e
             INNER JOIN public.user u ON u.id = user_id
             INNER JOIN magazine m ON e.magazine_id = m.id
-            WHERE (e.body_ts @@ plainto_tsquery( :query ) = true)
+            WHERE (e.body_ts @@ plainto_tsquery( :tsLang, :query ) = true)
                 AND e.visibility = :visibility
                 AND u.is_deleted = false
                 AND (u.ap_discoverable = true OR u.ap_discoverable IS NULL)
@@ -210,10 +212,10 @@ public function search(
                 AND NOT EXISTS (SELECT hl.id FROM hashtag_link hl INNER JOIN hashtag h ON h.id = hl.hashtag_id AND h.banned = true WHERE hl.entry_comment_id = e.id)
                 $authorWhere $magazineWhere $createdWhere
         ";
-        $sqlPost = "SELECT e.id, e.created_at, e.visibility, 3 * ts_rank_cd(e.body_ts, plainto_tsquery(:query)) as rank, 'post' AS type FROM post e
+        $sqlPost = "SELECT e.id, e.created_at, e.visibility, 3 * ts_rank_cd(e.body_ts, plainto_tsquery(:tsLang, :query)) as rank, 'post' AS type FROM post e
             INNER JOIN public.user u ON u.id = user_id
             INNER JOIN magazine m ON e.magazine_id = m.id
-            WHERE (e.body_ts @@ plainto_tsquery( :query ) = true)
+            WHERE (e.body_ts @@ plainto_tsquery( :tsLang, :query ) = true)
                 AND e.visibility = :visibility
                 AND u.is_deleted = false
                 AND (u.ap_discoverable = true OR u.ap_discoverable IS NULL)
@@ -223,10 +225,10 @@ public function search(
                 AND NOT EXISTS (SELECT hl.id FROM hashtag_link hl INNER JOIN hashtag h ON h.id = hl.hashtag_id AND h.banned = true WHERE hl.post_id = e.id)
                 $authorWhere $magazineWhere $createdWhere
         UNION ALL
-        SELECT e.id, e.created_at, e.visibility, 3 * ts_rank_cd(e.body_ts, plainto_tsquery(:query)) as rank, 'post_comment' AS type FROM post_comment e
+        SELECT e.id, e.created_at, e.visibility, 3 * ts_rank_cd(e.body_ts, plainto_tsquery(:tsLang, :query)) as rank, 'post_comment' AS type FROM post_comment e
             INNER JOIN public.user u ON u.id = user_id
             INNER JOIN magazine m ON e.magazine_id = m.id
-            WHERE (e.body_ts @@ plainto_tsquery( :query ) = true)
+            WHERE (e.body_ts @@ plainto_tsquery( :tsLang, :query ) = true)
                 AND e.visibility = :visibility
                 AND u.is_deleted = false
                 AND (u.ap_discoverable = true OR u.ap_discoverable IS NULL)
@@ -237,8 +239,8 @@ public function search(
                 $authorWhere $magazineWhere $createdWhere
         ";
 
-        $sqlMagazine = "SELECT m.Id, m.created_at, m.visibility, ts_rank_cd(m.name_ts, plainto_tsquery(:query)) + ts_rank_cd(m.title_ts, plainto_tsquery(:query)) + ts_rank_cd(m.description_ts, plainto_tsquery(:query)) as rank, 'magazine' AS type FROM magazine m
-            WHERE (m.name_ts @@ plainto_tsquery( :query ) = true OR m.title_ts @@ plainto_tsquery( :query ) = true OR m.description_ts @@ plainto_tsquery( :query ) = true OR m.title LIKE :likeQuery)
+        $sqlMagazine = "SELECT m.Id, m.created_at, m.visibility, ts_rank_cd(m.name_ts, plainto_tsquery(:tsLang, :query)) + ts_rank_cd(m.title_ts, plainto_tsquery(:tsLang, :query)) + ts_rank_cd(m.description_ts, plainto_tsquery(:tsLang, :query)) as rank, 'magazine' AS type FROM magazine m
+            WHERE (m.name_ts @@ plainto_tsquery( :tsLang, :query ) = true OR m.title_ts @@ plainto_tsquery( :tsLang, :query ) = true OR m.description_ts @@ plainto_tsquery( :tsLang, :query ) = true OR m.title LIKE :likeQuery)
                 AND m.visibility = :visibility
                 AND m.ap_deleted_at IS NULL
                 AND m.marked_for_deletion_at IS NULL
@@ -247,8 +249,8 @@ public function search(
                 $createdWhereMagazine $blockMagazineAndUserResult
         ";
 
-        $sqlUser = "SELECT u.Id, u.created_at, u.visibility, ts_rank_cd(u.username_ts, plainto_tsquery(:query)) + ts_rank_cd(u.title_ts, plainto_tsquery(:query)) + ts_rank_cd(u.about_ts, plainto_tsquery(:query)) as rank, 'user' AS type FROM \"user\" u
-            WHERE (u.username_ts @@ plainto_tsquery( :query ) = true OR u.title_ts @@ plainto_tsquery( :query ) = true OR u.about_ts @@ plainto_tsquery( :query ) = true OR u.username LIKE :likeQuery)
+        $sqlUser = "SELECT u.Id, u.created_at, u.visibility, ts_rank_cd(u.username_ts, plainto_tsquery(:tsLang, :query)) + ts_rank_cd(u.title_ts, plainto_tsquery(:tsLang, :query)) + ts_rank_cd(u.about_ts, plainto_tsquery(:tsLang, :query)) as rank, 'user' AS type FROM \"user\" u
+            WHERE (u.username_ts @@ plainto_tsquery( :tsLang, :query ) = true OR u.title_ts @@ plainto_tsquery( :tsLang, :query ) = true OR u.about_ts @@ plainto_tsquery( :tsLang, :query ) = true OR u.username LIKE :likeQuery)
                 AND u.visibility = :visibility
                 AND u.is_deleted = false
                 AND u.marked_for_deletion_at IS NULL
@@ -277,6 +279,7 @@ public function search(
             'likeQuery' => "%$query%",
             'visibility' => VisibilityInterface::VISIBILITY_VISIBLE,
             'queryingUser' => $searchingUser?->getId() ?? -1,
+            'tsLang' => $this->settingsManager->getSearchLang(),
         ];
 
         $this->logger->debug('Search query: {sql}', ['sql' => $sql]);

src/Service/SettingsManager.php

@@ -51,6 +51,7 @@ public function __construct(
         private readonly bool $mbinNewUsersNeedApproval,
         private readonly LoggerInterface $logger,
         private readonly bool $mbinUseFederationAllowList,
+        private readonly string $mbinSearchLang,
     ) {
         if (!self::$dto || 'test' === $this->kernel->getEnvironment()) {
             $results = $this->repository->findAll();
@@ -260,6 +261,11 @@ public function getMaxImageByteString(): string
         return $megaBytes.' MB';
     }
 
+    public function getSearchLang(): string
+    {
+        return $this->mbinSearchLang;
+    }
+
     /**
      * this should only be called in the test environment.
      */

tests/Unit/Service/SettingsManagerTest.php

@@ -67,7 +67,8 @@ public function testGetMaxImageByteStringDefault(): void
             mbinDownvotesMode: DownvotesMode::Enabled,
             mbinNewUsersNeedApproval: false,
             logger: $logger,
-            mbinUseFederationAllowList: false
+            mbinUseFederationAllowList: false,
+            mbinSearchLang: 'english',
         );
 
         // Assert
@@ -118,7 +119,8 @@ public function testGetMaxImageByteStringOverridden(): void
             mbinDownvotesMode: DownvotesMode::Enabled,
             mbinNewUsersNeedApproval: false,
             logger: $logger,
-            mbinUseFederationAllowList: false
+            mbinUseFederationAllowList: false,
+            mbinSearchLang: 'english',
         );
 
         // Assert