From 85ddad897d5bfbd85a3034e3beb9d921c08c75fa Mon Sep 17 00:00:00 2001 From: edwh Date: Wed, 3 Jun 2026 23:22:54 +0100 Subject: [PATCH 01/26] feat(moderation): auto-approve clean NULL-status posts after a delay MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Members whose group posting status is NULL ('auto-moderated' — never explicitly set) previously had their posts stuck in Pending until a mod acted or the 48-hour fallback fired. NULL is now reinterpreted as a fast, safe path: their content-check-clean posts auto-approve after a configurable delay (default 20 min), giving mods and microvolunteers a window to intervene, unless a danger signal is present, with a configurable quality-check sample held back for manual review. Backend (iznik-batch): - AutoApproveCleanService + messages:auto-approve-clean command (scheduled every minute). Eligible = NULL posting status, content-check clean (checked_at set, reasons NULL), past the per-group/site delay, group not moderated/closed/Big-Switch, not held/deleted/spam-flagged. - Danger-signal vetoes: microvolunteering rejects, mod notes, recent negative mod logs (rejections/deletions/modmails/spam classifications), known-spammer records, pending membership reviews. - Quality-check sample held deterministically per message. - Spam unaffected: only posts that already passed messages:contentcheck are eligible; suspect posts keep their reasons and stay Pending (or move to Spam) as before. - config/freegle.php autoapprove block (delay_minutes=20, quality_check_percent=0, danger_log_days=90); per-group override via settings.autoapprove.* (0/absent = site default). No schema migration and no routing change. Go API: - /modtools/messages gains a filter param: autoapproved (approvedby IS NULL), recentjoin (joined <7d), outsidecga (location outside group polygon). Frontend (modtools): - Approved page filter dropdown (defaults to All; switches to summary view for Auto-approved). - Group settings controls for the delay and quality-check percentage. Tests: 25 service + 4 command unit tests (Laravel); 3 Go endpoint tests. --- .../Message/AutoApproveCleanCommand.php | 48 +++ .../app/Services/AutoApproveCleanService.php | 306 ++++++++++++++ iznik-batch/config/freegle.php | 24 ++ iznik-batch/routes/console.php | 9 + .../Message/AutoApproveCleanCommandTest.php | 69 +++ .../Services/AutoApproveCleanServiceTest.php | 400 ++++++++++++++++++ .../modtools/components/ModSettingsGroup.vue | 16 + .../messages/approved/[[id]]/[[term]].vue | 35 ++ iznik-server-go/message/message_list.go | 26 ++ iznik-server-go/test/message_test.go | 132 ++++++ plans/active/autoapprove-delay.md | 133 ++++++ 11 files changed, 1198 insertions(+) create mode 100644 iznik-batch/app/Console/Commands/Message/AutoApproveCleanCommand.php create mode 100644 iznik-batch/app/Services/AutoApproveCleanService.php create mode 100644 iznik-batch/tests/Unit/Commands/Message/AutoApproveCleanCommandTest.php create mode 100644 iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php create mode 100644 plans/active/autoapprove-delay.md diff --git a/iznik-batch/app/Console/Commands/Message/AutoApproveCleanCommand.php b/iznik-batch/app/Console/Commands/Message/AutoApproveCleanCommand.php new file mode 100644 index 0000000000..c3ccf691c2 --- /dev/null +++ b/iznik-batch/app/Console/Commands/Message/AutoApproveCleanCommand.php @@ -0,0 +1,48 @@ +registerShutdownHandlers(); + + $dryRun = (bool) $this->option('dry-run'); + + if ($dryRun) { + $this->info('DRY RUN — no changes will be made.'); + } + + Log::info('Starting auto-approve-clean processing', ['dry_run' => $dryRun]); + $this->info('Processing content-check-clean pending messages for auto-approval...'); + + $stats = $service->process($dryRun); + + $prefix = $dryRun ? '[DRY RUN] ' : ''; + $this->info( + "{$prefix}Approved: {$stats['approved']}, Held (quality): {$stats['held_quality']}, " . + "Vetoed: {$stats['vetoed']}, Skipped: {$stats['skipped']}, Errors: {$stats['errors']}" + ); + + if ($stats['errors'] > 0) { + $this->warn("Errors: {$stats['errors']}"); + } + + Log::info('Auto-approve-clean processing complete', $stats); + + return $stats['errors'] > 0 ? Command::FAILURE : Command::SUCCESS; + } +} diff --git a/iznik-batch/app/Services/AutoApproveCleanService.php b/iznik-batch/app/Services/AutoApproveCleanService.php new file mode 100644 index 0000000000..3e3c4e600f --- /dev/null +++ b/iznik-batch/app/Services/AutoApproveCleanService.php @@ -0,0 +1,306 @@ + 0, 'held_quality' => 0, 'vetoed' => 0, 'skipped' => 0, 'errors' => 0]; + + // The delay is per-group (settings.autoapprove.delay_minutes) with a site-wide + // fallback, so it must be resolved in SQL — a single global threshold would ignore + // group overrides. A 0/absent override means "use the site default". + $candidates = DB::table('messages_groups as mg') + ->join('messages as m', 'm.id', '=', 'mg.msgid') + ->join('users as u', 'u.id', '=', 'm.fromuser') + ->join('memberships as mem', function ($j) { + $j->on('mem.userid', '=', 'm.fromuser')->on('mem.groupid', '=', 'mg.groupid'); + }) + ->join('groups as g', 'g.id', '=', 'mg.groupid') + ->select('mg.msgid', 'mg.groupid', 'm.fromuser', 'm.spamtype', 'm.subject', DB::raw('m.type as msgtype')) + ->where('mg.collection', MessageGroup::COLLECTION_PENDING) + ->whereNull('mg.heldby') + ->whereNull('m.heldby') + ->whereNull('mg.spamreason') + ->whereNull('m.spamreason') + ->where('mg.deleted', 0) + ->whereNull('m.deleted') + ->whereNull('u.deleted') + ->whereNull('mem.ourPostingStatus') // the auto-moderated tier + ->whereNotNull('mg.contentcheck_checked_at') // content check has run ... + ->whereNull('mg.contentcheck_reasons') // ... and the post was clean + ->whereRaw( + "mg.arrival <= (NOW() - INTERVAL COALESCE(NULLIF(CAST(JSON_UNQUOTE(JSON_EXTRACT(g.settings, '$.autoapprove.delay_minutes')) AS UNSIGNED), 0), ?) MINUTE)", + [$this->defaultDelayMinutes()] + ) + ->orderBy('mg.msgid') + ->orderBy('mg.groupid') + ->get(); + + foreach ($candidates as $row) { + try { + if (!$this->groupAllowsAutoApprove((int) $row->groupid)) { + $stats['skipped']++; + continue; + } + + if ($this->hasDangerSignals((int) $row->msgid, (int) $row->groupid, (int) $row->fromuser)) { + $stats['vetoed']++; + continue; + } + + if ($this->isQualitySampled((int) $row->msgid, (int) $row->groupid)) { + $stats['held_quality']++; + continue; + } + + if ($dryRun) { + $stats['approved']++; + Log::info("Dry run: would auto-approve clean message #{$row->msgid} on group #{$row->groupid}"); + continue; + } + + $this->approve($row); + $stats['approved']++; + } catch (\Exception $e) { + Log::error("AutoApproveClean: error processing message #{$row->msgid}: " . $e->getMessage()); + $stats['errors']++; + } + } + + return $stats; + } + + /** + * The group must be open, published and NOT moderated — a moderated group (or one + * under the Big Switch) deliberately wants every post reviewed by a human. + */ + protected function groupAllowsAutoApprove(int $groupid): bool + { + $group = Group::find($groupid); + if (!$group) { + return false; + } + if (!$group->getSetting('publish', true)) { + return false; + } + if ($group->isClosed()) { + return false; + } + if ($group->getAttribute('autofunctionoverride')) { + return false; + } + if ($group->getAttribute('overridemoderation') === 'ModerateAll') { + return false; + } + if (!empty($group->getSetting('moderated', 0))) { + return false; + } + $rules = $group->rules ?? []; + if (!empty($rules['fullymoderated'])) { + return false; + } + + return true; + } + + /** + * Any one of these "danger signals" keeps the post in Pending for a moderator to handle. + */ + protected function hasDangerSignals(int $msgid, int $groupid, int $fromuser): bool + { + // A microvolunteer flagged the post as not OK. + if (DB::table('microactions') + ->where('msgid', $msgid) + ->where('actiontype', 'CheckMessage') + ->where('result', 'Reject') + ->exists()) { + return true; + } + + // A moderator has left a note on this member. + if (DB::table('users_comments')->where('userid', $fromuser)->exists()) { + return true; + } + + // A recent negative moderation action against this member (rejection, deletion, + // modmail, spam classification) — not a self-initiated action. + if (DB::table('logs') + ->where('user', $fromuser) + ->where('timestamp', '>=', now()->subDays($this->dangerLogDays())) + ->where(function ($q) { + $q->whereColumn('byuser', '!=', 'user')->orWhereNull('byuser'); + }) + ->where(function ($q) { + $q->where(function ($q2) { + $q2->where('type', 'Message')->whereIn('subtype', self::DANGER_MESSAGE_SUBTYPES); + })->orWhere(function ($q2) { + $q2->where('type', 'User')->whereIn('subtype', self::DANGER_USER_SUBTYPES); + }); + }) + ->exists()) { + return true; + } + + // A known or suspected spammer. + if (DB::table('spam_users') + ->where('userid', $fromuser) + ->whereIn('collection', ['Spammer', 'PendingAdd']) + ->exists()) { + return true; + } + + // A moderation review is outstanding on this membership. + if (DB::table('memberships') + ->where('userid', $fromuser) + ->where('groupid', $groupid) + ->whereNotNull('reviewrequestedat') + ->where(function ($q) { + $q->whereNull('reviewedat')->orWhereColumn('reviewedat', '<', 'reviewrequestedat'); + }) + ->exists()) { + return true; + } + + return false; + } + + /** + * Deterministically hold a configurable percentage of otherwise-eligible posts in + * Pending so a moderator spot-checks the auto-approval quality. Deterministic on msgid + * so a held message never oscillates between runs. + */ + protected function isQualitySampled(int $msgid, int $groupid): bool + { + $percent = $this->qualityPercentForGroup($groupid); + if ($percent <= 0) { + return false; + } + if ($percent >= 100) { + return true; + } + + return (abs(crc32((string) $msgid)) % 100) < $percent; + } + + protected function qualityPercentForGroup(int $groupid): int + { + $group = Group::find($groupid); + $settings = $group?->settings ?? []; + $val = $settings['autoapprove']['quality_check_percent'] ?? null; + if ($val === null || $val === '') { + return $this->defaultQualityCheckPercent(); + } + + return (int) $val; + } + + /** + * Approve a message on a group. Mirrors AutoApproveService::approveOnGroup side effects + * (NULL approvedby, reset arrival, Autoapproved log, Ham recording) and additionally + * queues a freebie-alert task for Offers, matching the immediate contentcheck approval. + */ + protected function approve(object $row): void + { + // notSpam parity: whitelist the subject when it was flagged for cross-group reuse. + if ($row->spamtype === 'SubjectUsedForDifferentGroups' && $row->subject) { + DB::table('spam_whitelist_subjects')->insertOrIgnore([ + 'subject' => AutoApproveService::getPrunedSubject($row->subject), + 'comment' => 'Marked as not spam', + ]); + } + + // notSpam parity: record HAM when the message had been marked spam. + if ($row->spamtype) { + DB::table('messages_spamham')->upsert( + ['msgid' => $row->msgid, 'spamham' => 'Ham'], + ['msgid'], + ['spamham'] + ); + } + + DB::transaction(function () use ($row) { + DB::table('messages_groups') + ->where('msgid', $row->msgid) + ->where('groupid', $row->groupid) + ->where('collection', '!=', MessageGroup::COLLECTION_APPROVED) + ->update([ + 'collection' => MessageGroup::COLLECTION_APPROVED, + 'approvedby' => null, // NULL marks an auto-approval + 'approvedat' => now(), + 'arrival' => now(), // so the digest picks it up as new + ]); + + DB::table('logs')->insert([ + 'timestamp' => now(), + 'type' => 'Message', + 'subtype' => 'Autoapproved', + 'msgid' => $row->msgid, + 'groupid' => $row->groupid, + 'user' => $row->fromuser, + ]); + + if ($row->msgtype === Message::TYPE_OFFER) { + DB::table('background_tasks')->insert([ + 'task_type' => BackgroundTask::TASK_FREEBIE_ALERTS_ADD, + 'data' => json_encode(['msgid' => (int) $row->msgid]), + ]); + } + }); + + Log::info("AutoApproveClean: approved message #{$row->msgid} on group #{$row->groupid}"); + } +} diff --git a/iznik-batch/config/freegle.php b/iznik-batch/config/freegle.php index e89fd047d7..bd6728c1d9 100644 --- a/iznik-batch/config/freegle.php +++ b/iznik-batch/config/freegle.php @@ -601,6 +601,30 @@ 'api_key' => env('FREEBIE_ALERTS_KEY', ''), ], + /* + |-------------------------------------------------------------------------- + | Auto-approve (delayed) for NULL-status ("auto-moderated") members + |-------------------------------------------------------------------------- + | + | Site-wide defaults for messages:auto-approve-clean. A group may override + | delay_minutes / quality_check_percent via settings.autoapprove.* (an + | absent or 0 delay override falls back to the site default below). + | + | - delay_minutes: how long a content-check-clean post stays in + | Pending before auto-approval, giving mods and + | microvolunteers a chance to intervene. + | - quality_check_percent: percentage of otherwise-eligible posts held back + | in Pending for a manual mod quality check (0 = none). + | - danger_log_days: how far back to look for negative moderation log + | entries that veto auto-approval. + | + */ + 'autoapprove' => [ + 'delay_minutes' => (int) env('FREEGLE_AUTOAPPROVE_DELAY_MINUTES', 20), + 'quality_check_percent' => (int) env('FREEGLE_AUTOAPPROVE_QUALITY_CHECK_PCT', 0), + 'danger_log_days' => (int) env('FREEGLE_AUTOAPPROVE_DANGER_LOG_DAYS', 90), + ], + 'email_health' => [ 'incoming_window_hours' => env('FREEGLE_EMAIL_HEALTH_INCOMING_WINDOW_HOURS', 2), 'outgoing_min_per_hour' => env('FREEGLE_EMAIL_HEALTH_OUTGOING_MIN_PER_HOUR', 10), diff --git a/iznik-batch/routes/console.php b/iznik-batch/routes/console.php index 883deae89d..4ea0e9524a 100644 --- a/iznik-batch/routes/console.php +++ b/iznik-batch/routes/console.php @@ -163,6 +163,15 @@ function cronLog(string $command): string ->sendOutputTo(cronLog('ripple:proximity-notes')) ->runInBackground(); +// Auto-approve content-check-clean posts from NULL-status ("auto-moderated") members +// after the configured delay (default 20 min), unless danger signals are present and +// excluding a quality-check sample. Runs every minute so the window is honoured tightly. +Schedule::command('messages:auto-approve-clean') + ->everyMinute() + ->withoutOverlapping() + ->sendOutputTo(cronLog('messages:auto-approve-clean')) + ->runInBackground(); + // Update UK spatial data - runs monthly. // Downloads UK OSM PBF file and rebuilds deprivation quintile CSV for spatial server. // Signals Go spatial server to reload after update. diff --git a/iznik-batch/tests/Unit/Commands/Message/AutoApproveCleanCommandTest.php b/iznik-batch/tests/Unit/Commands/Message/AutoApproveCleanCommandTest.php new file mode 100644 index 0000000000..07ac8314b2 --- /dev/null +++ b/iznik-batch/tests/Unit/Commands/Message/AutoApproveCleanCommandTest.php @@ -0,0 +1,69 @@ + 0, 'held_quality' => 0, 'vetoed' => 0, 'skipped' => 0, 'errors' => 0, + ], $overrides); + } + + public function test_success_with_no_errors(): void + { + $service = $this->createMock(AutoApproveCleanService::class); + $service->method('process')->willReturn($this->stats([ + 'approved' => 5, 'held_quality' => 1, 'vetoed' => 2, 'skipped' => 3, + ])); + $this->app->instance(AutoApproveCleanService::class, $service); + + $this->artisan('messages:auto-approve-clean') + ->expectsOutputToContain('Approved: 5, Held (quality): 1, Vetoed: 2, Skipped: 3, Errors: 0') + ->assertExitCode(Command::SUCCESS); + } + + public function test_failure_when_errors_present(): void + { + $service = $this->createMock(AutoApproveCleanService::class); + $service->method('process')->willReturn($this->stats(['approved' => 1, 'errors' => 3])); + $this->app->instance(AutoApproveCleanService::class, $service); + + $this->artisan('messages:auto-approve-clean') + ->expectsOutputToContain('Errors: 3') + ->assertExitCode(Command::FAILURE); + } + + public function test_dry_run_announces_mode_and_passes_flag(): void + { + $service = $this->createMock(AutoApproveCleanService::class); + $service->expects($this->once()) + ->method('process') + ->with(true) + ->willReturn($this->stats()); + $this->app->instance(AutoApproveCleanService::class, $service); + + $this->artisan('messages:auto-approve-clean', ['--dry-run' => true]) + ->expectsOutputToContain('no changes will be made') + ->expectsOutputToContain('[DRY RUN] Approved: 0') + ->assertExitCode(Command::SUCCESS); + } + + public function test_passes_false_when_not_dry_run(): void + { + $service = $this->createMock(AutoApproveCleanService::class); + $service->expects($this->once()) + ->method('process') + ->with(false) + ->willReturn($this->stats()); + $this->app->instance(AutoApproveCleanService::class, $service); + + $this->artisan('messages:auto-approve-clean') + ->assertExitCode(Command::SUCCESS); + } +} diff --git a/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php b/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php new file mode 100644 index 0000000000..1d3533f365 --- /dev/null +++ b/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php @@ -0,0 +1,400 @@ +service = new AutoApproveCleanService(); + + // Deterministic site defaults for the tests. + config([ + 'freegle.autoapprove.delay_minutes' => 20, + 'freegle.autoapprove.quality_check_percent' => 0, + 'freegle.autoapprove.danger_log_days' => 90, + ]); + } + + /** + * Build a clean, content-checked, NULL-status pending post that is eligible for + * auto-approval (arrival older than the default 20-minute delay). + * + * @return array{0: User, 1: Group, 2: Message} + */ + private function makeApprovable(array $opts = []): array + { + $user = $this->createTestUser(); + $group = $this->createTestGroup($opts['group'] ?? []); + $this->createMembership($user, $group, array_merge([ + 'added' => now()->subDays(2), + // ourPostingStatus left NULL — the auto-moderated tier. + ], $opts['membership'] ?? [])); + + $message = $this->createTestMessage($user, $group, $opts['message'] ?? []); + + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update(array_merge([ + 'collection' => MessageGroup::COLLECTION_PENDING, + 'arrival' => now()->subMinutes(21), + 'contentcheck_checked_at' => now()->subMinutes(20), + 'contentcheck_reasons' => null, + ], $opts['mg'] ?? [])); + + return [$user, $group, $message]; + } + + private function assertApproved(int $msgid, int $groupid): void + { + $mg = DB::table('messages_groups')->where('msgid', $msgid)->where('groupid', $groupid)->first(); + $this->assertEquals(MessageGroup::COLLECTION_APPROVED, $mg->collection, 'message should be Approved'); + $this->assertNull($mg->approvedby, 'auto-approval leaves approvedby NULL'); + $this->assertDatabaseHas('logs', [ + 'msgid' => $msgid, 'groupid' => $groupid, 'type' => 'Message', 'subtype' => 'Autoapproved', + ]); + } + + private function assertStillPending(int $msgid, int $groupid): void + { + $this->assertDatabaseHas('messages_groups', [ + 'msgid' => $msgid, 'groupid' => $groupid, 'collection' => MessageGroup::COLLECTION_PENDING, + ]); + $this->assertDatabaseMissing('logs', [ + 'msgid' => $msgid, 'type' => 'Message', 'subtype' => 'Autoapproved', + ]); + } + + public function test_stats_structure(): void + { + $stats = $this->service->process(); + foreach (['approved', 'held_quality', 'vetoed', 'skipped', 'errors'] as $key) { + $this->assertArrayHasKey($key, $stats); + } + } + + public function test_approves_clean_null_status_post_after_delay(): void + { + [$user, $group, $message] = $this->makeApprovable(); + + $stats = $this->service->process(); + + $this->assertGreaterThanOrEqual(1, $stats['approved']); + $this->assertApproved($message->id, $group->id); + } + + public function test_does_not_approve_before_delay(): void + { + [$user, $group, $message] = $this->makeApprovable([ + 'mg' => ['arrival' => now()->subMinutes(19)], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_default_posting_status(): void + { + // DEFAULT members are trusted and approved immediately by contentcheck — not our job. + [$user, $group, $message] = $this->makeApprovable([ + 'membership' => ['ourPostingStatus' => 'DEFAULT'], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_moderated_posting_status(): void + { + // Explicit MODERATED stays moderated. + [$user, $group, $message] = $this->makeApprovable([ + 'membership' => ['ourPostingStatus' => 'MODERATED'], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_when_content_check_not_run(): void + { + [$user, $group, $message] = $this->makeApprovable([ + 'mg' => ['contentcheck_checked_at' => null], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_when_content_check_flagged_reasons(): void + { + // Suspect content (reasons present) must keep the post in Pending for a mod. + [$user, $group, $message] = $this->makeApprovable([ + 'mg' => ['contentcheck_reasons' => json_encode([['check' => 'Money', 'action' => 'flag']])], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_held_message(): void + { + [$user, $group, $message] = $this->makeApprovable(); + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update(['heldby' => $user->id]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_message_with_spamreason(): void + { + [$user, $group, $message] = $this->makeApprovable(); + DB::table('messages')->where('id', $message->id)->update(['spamreason' => 'CountryBlocked']); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_soft_deleted_message(): void + { + [$user, $group, $message] = $this->makeApprovable(); + DB::table('messages')->where('id', $message->id)->update(['deleted' => now()]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_moderated_group(): void + { + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['moderated' => 1]], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_skips_closed_group(): void + { + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['closed' => true]], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_veto_microvolunteering_reject(): void + { + [$user, $group, $message] = $this->makeApprovable(); + $reviewer = $this->createTestUser(); + DB::table('microactions')->insert([ + 'userid' => $reviewer->id, + 'msgid' => $message->id, + 'actiontype' => 'CheckMessage', + 'result' => 'Reject', + 'score_negative' => 1, + ]); + + $stats = $this->service->process(); + + $this->assertGreaterThanOrEqual(1, $stats['vetoed']); + $this->assertStillPending($message->id, $group->id); + } + + public function test_does_not_veto_microvolunteering_approve(): void + { + [$user, $group, $message] = $this->makeApprovable(); + $reviewer = $this->createTestUser(); + DB::table('microactions')->insert([ + 'userid' => $reviewer->id, + 'msgid' => $message->id, + 'actiontype' => 'CheckMessage', + 'result' => 'Approve', + 'score_negative' => 0, + ]); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + } + + public function test_veto_user_note(): void + { + [$user, $group, $message] = $this->makeApprovable(); + DB::table('users_comments')->insert([ + 'userid' => $user->id, + 'groupid' => $group->id, + 'user1' => 'Keep an eye on this member.', + ]); + + $stats = $this->service->process(); + + $this->assertGreaterThanOrEqual(1, $stats['vetoed']); + $this->assertStillPending($message->id, $group->id); + } + + public function test_veto_recent_negative_mod_log(): void + { + [$user, $group, $message] = $this->makeApprovable(); + $mod = $this->createTestUser(); + DB::table('logs')->insert([ + 'timestamp' => now()->subDays(1), + 'type' => 'User', + 'subtype' => 'Mailed', + 'user' => $user->id, + 'byuser' => $mod->id, + 'groupid' => $group->id, + ]); + + $stats = $this->service->process(); + + $this->assertGreaterThanOrEqual(1, $stats['vetoed']); + $this->assertStillPending($message->id, $group->id); + } + + public function test_does_not_veto_old_negative_log(): void + { + [$user, $group, $message] = $this->makeApprovable(); + $mod = $this->createTestUser(); + DB::table('logs')->insert([ + 'timestamp' => now()->subDays(120), // outside the 90-day danger window + 'type' => 'User', + 'subtype' => 'Mailed', + 'user' => $user->id, + 'byuser' => $mod->id, + 'groupid' => $group->id, + ]); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + } + + public function test_veto_known_spammer(): void + { + [$user, $group, $message] = $this->makeApprovable(); + DB::table('spam_users')->insert([ + 'userid' => $user->id, + 'collection' => 'Spammer', + ]); + + $stats = $this->service->process(); + + $this->assertGreaterThanOrEqual(1, $stats['vetoed']); + $this->assertStillPending($message->id, $group->id); + } + + public function test_veto_membership_review_pending(): void + { + [$user, $group, $message] = $this->makeApprovable(); + DB::table('memberships') + ->where('userid', $user->id) + ->where('groupid', $group->id) + ->update(['reviewrequestedat' => now()->subHour(), 'reviewedat' => null]); + + $stats = $this->service->process(); + + $this->assertGreaterThanOrEqual(1, $stats['vetoed']); + $this->assertStillPending($message->id, $group->id); + } + + public function test_quality_sample_100_holds_all(): void + { + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['autoapprove' => ['quality_check_percent' => 100]]], + ]); + + $stats = $this->service->process(); + + $this->assertGreaterThanOrEqual(1, $stats['held_quality']); + $this->assertStillPending($message->id, $group->id); + } + + public function test_quality_sample_zero_holds_none(): void + { + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['autoapprove' => ['quality_check_percent' => 0]]], + ]); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + } + + public function test_per_group_delay_override(): void + { + // Site default is 20 min; this group overrides to 5 min, so a 6-min-old post approves. + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['autoapprove' => ['delay_minutes' => 5]]], + 'mg' => ['arrival' => now()->subMinutes(6)], + ]); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + } + + public function test_zero_group_delay_falls_back_to_site_default(): void + { + // A 0 override means "use the site default" (20), so a 6-min-old post stays pending. + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['autoapprove' => ['delay_minutes' => 0]]], + 'mg' => ['arrival' => now()->subMinutes(6)], + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_dry_run_does_not_modify_database(): void + { + [$user, $group, $message] = $this->makeApprovable(); + + $stats = $this->service->process(dryRun: true); + + $this->assertGreaterThanOrEqual(1, $stats['approved']); + $this->assertStillPending($message->id, $group->id); + } + + public function test_records_ham_for_spam_message(): void + { + [$user, $group, $message] = $this->makeApprovable(); + DB::table('messages')->where('id', $message->id)->update(['spamtype' => 'SubjectUsedForDifferentGroups']); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + $this->assertDatabaseHas('messages_spamham', ['msgid' => $message->id, 'spamham' => 'Ham']); + } +} diff --git a/iznik-nuxt3/modtools/components/ModSettingsGroup.vue b/iznik-nuxt3/modtools/components/ModSettingsGroup.vue index 5c2e5f3b21..4225147a16 100644 --- a/iznik-nuxt3/modtools/components/ModSettingsGroup.vue +++ b/iznik-nuxt3/modtools/components/ModSettingsGroup.vue @@ -569,6 +569,22 @@ toggle-checked="Yes" toggle-unchecked="No" /> + + Select a community to search messages. +
@@ -56,11 +64,13 @@ import { ref, computed, watch, onMounted, nextTick } from 'vue' import { useRoute, useRouter } from '#imports' import { useMessageStore } from '@/stores/message' +import { useMiscStore } from '@/stores/misc' import { setupModMessages } from '@/composables/useModMessages' import { useMe } from '~/composables/useMe' // Stores const messageStore = useMessageStore() +const miscStore = useMiscStore() // Composables const modMessages = setupModMessages(true) @@ -87,6 +97,13 @@ const { // Local state (formerly data()) const chosengroupid = ref(0) const bump = ref(0) +const filter = ref('') +const filterOptions = [ + { value: '', text: 'All messages' }, + { value: 'recentjoin', text: 'Joined last 7 days' }, + { value: 'autoapproved', text: 'Auto-approved' }, + { value: 'outsidecga', text: 'Outside group area' }, +] const urlOverride = ref(false) const loaded = ref(false) const highlightMsgId = ref(null) @@ -215,6 +232,21 @@ function searchedMessage(term) { } } +watch(filter, (newVal) => { + // The auto-approved view is for scanning posts that went out without a human + // checking, so default it to the compact summary view. + if (newVal === 'autoapproved') { + miscStore.set({ key: 'modtoolsMessagesApprovedSummary', value: true }) + } + show.value = 0 + context.value = null + modMessages.listingIds.value = new Set() + modMessages.listingIdOrder.value = [] + messageStore.clear() + bump.value++ +}) + + function searchedMember(term) { show.value = 0 messageTerm.value = null @@ -289,6 +321,9 @@ async function loadMore($state) { modtools: true, summary: false, } + if (filter.value) { + params.filter = filter.value + } } params.context = context.value diff --git a/iznik-server-go/message/message_list.go b/iznik-server-go/message/message_list.go index 5a77062815..189e526d5b 100644 --- a/iznik-server-go/message/message_list.go +++ b/iznik-server-go/message/message_list.go @@ -448,6 +448,11 @@ func ListMessagesMT(c *fiber.Ctx) error { search := c.Query("search", "") fromuserStr := c.Query("fromuser", "0") fromuser, _ := strconv.ParseUint(fromuserStr, 10, 64) + // Optional moderation filter for the standard (non-search) listing: + // autoapproved — posts approved without a moderator (mg.approvedby IS NULL) + // recentjoin — posts from members who joined this group in the last 7 days + // outsidecga — posts whose location falls outside the group's area polygon + filter := c.Query("filter", "") var msgIDs []uint64 @@ -552,11 +557,32 @@ func ListMessagesMT(c *fiber.Ctx) error { branchArgs = []interface{}{utils.COLLECTION_PENDING, utils.COLLECTION_SPAM} } + // Optional moderation filter. Values are matched against a fixed set, so the + // injected SQL is constant (no user-supplied text reaches the query). + filterJoin := "" + filterWhere := "" + switch filter { + case "autoapproved": + filterWhere = "AND mg.approvedby IS NULL " + case "recentjoin": + filterJoin = "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid " + filterWhere = "AND mem.added >= NOW() - INTERVAL 7 DAY " + case "outsidecga": + // Use the canonical spatial point (messages_spatial), matching how + // location containment is checked elsewhere. Unmappable posts (no + // spatial row) can't be "outside" and are excluded by the INNER JOIN. + filterJoin = "INNER JOIN `groups` g ON g.id = mg.groupid " + + "INNER JOIN messages_spatial ms ON ms.msgid = mg.msgid AND ms.groupid = mg.groupid " + filterWhere = "AND g.polyindex IS NOT NULL AND NOT ST_Contains(g.polyindex, ms.point) " + } + branchSQL := "SELECT mg.msgid, mg.arrival FROM messages_groups mg " + "INNER JOIN messages m ON m.id = mg.msgid " + "INNER JOIN users u ON u.id = m.fromuser " + + filterJoin + "WHERE mg.groupid = %GID% AND " + collectionFilter + " AND mg.deleted = 0 " + "AND m.deleted IS NULL AND m.fromuser IS NOT NULL AND u.deleted IS NULL " + + filterWhere + contentcheckFilter + " " if fromuser > 0 { diff --git a/iznik-server-go/test/message_test.go b/iznik-server-go/test/message_test.go index 7688519631..2f20e5d3eb 100644 --- a/iznik-server-go/test/message_test.go +++ b/iznik-server-go/test/message_test.go @@ -5552,6 +5552,138 @@ func TestListMessagesMT_LimboUserMessageNotReturned(t *testing.T) { db.Exec("UPDATE users SET deleted = NULL WHERE id = ?", posterID) } +func TestListMessagesMT_FilterAutoApproved(t *testing.T) { + // filter=autoapproved returns only posts approved without a moderator + // (messages_groups.approvedby IS NULL). + prefix := uniquePrefix("lstmt_autoapp") + db := database.DBConn + + groupID := CreateTestGroup(t, prefix) + posterID := CreateTestUser(t, prefix+"_poster", "User") + modID := CreateTestUser(t, prefix+"_mod", "User") + CreateTestMembership(t, posterID, groupID, "Member") + CreateTestMembership(t, modID, groupID, "Moderator") + _, modToken := CreateTestSession(t, modID) + + autoMsg := CreateTestMessage(t, posterID, groupID, prefix+" auto approved", 52.0, -1.0) + manualMsg := CreateTestMessage(t, posterID, groupID, prefix+" mod approved", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid=?", autoMsg) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=? WHERE msgid=?", modID, manualMsg) + + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=autoapproved&jwt=%s", groupID, modToken), nil)) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + var body map[string]interface{} + json.NewDecoder(resp.Body).Decode(&body) + msgs, _ := body["messages"].([]interface{}) + foundAuto, foundManual := false, false + for _, id := range msgs { + if id == float64(autoMsg) { + foundAuto = true + } + if id == float64(manualMsg) { + foundManual = true + } + } + assert.True(t, foundAuto, "auto-approved message should appear under autoapproved filter") + assert.False(t, foundManual, "mod-approved message should not appear under autoapproved filter") + + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", autoMsg, manualMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?)", autoMsg, manualMsg) +} + +func TestListMessagesMT_FilterRecentJoin(t *testing.T) { + // filter=recentjoin returns only posts from members who joined this group + // within the last 7 days. + prefix := uniquePrefix("lstmt_recentjoin") + db := database.DBConn + + groupID := CreateTestGroup(t, prefix) + recentID := CreateTestUser(t, prefix+"_recent", "User") + oldID := CreateTestUser(t, prefix+"_old", "User") + modID := CreateTestUser(t, prefix+"_mod", "User") + CreateTestMembership(t, recentID, groupID, "Member") + CreateTestMembership(t, oldID, groupID, "Member") + CreateTestMembership(t, modID, groupID, "Moderator") + _, modToken := CreateTestSession(t, modID) + + // The old member joined 30 days ago. + db.Exec("UPDATE memberships SET added = NOW() - INTERVAL 30 DAY WHERE userid = ? AND groupid = ?", oldID, groupID) + + recentMsg := CreateTestMessage(t, recentID, groupID, prefix+" recent joiner post", 52.0, -1.0) + oldMsg := CreateTestMessage(t, oldID, groupID, prefix+" old joiner post", 52.0, -1.0) + + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=recentjoin&jwt=%s", groupID, modToken), nil)) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + var body map[string]interface{} + json.NewDecoder(resp.Body).Decode(&body) + msgs, _ := body["messages"].([]interface{}) + foundRecent, foundOld := false, false + for _, id := range msgs { + if id == float64(recentMsg) { + foundRecent = true + } + if id == float64(oldMsg) { + foundOld = true + } + } + assert.True(t, foundRecent, "recent joiner's post should appear under recentjoin filter") + assert.False(t, foundOld, "long-standing member's post should not appear under recentjoin filter") + + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", recentMsg, oldMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?)", recentMsg, oldMsg) +} + +func TestListMessagesMT_FilterOutsideCGA(t *testing.T) { + // filter=outsidecga returns only posts whose location falls outside the + // group's area polygon. + prefix := uniquePrefix("lstmt_outcga") + db := database.DBConn + + groupID := CreateTestGroup(t, prefix) + posterID := CreateTestUser(t, prefix+"_poster", "User") + modID := CreateTestUser(t, prefix+"_mod", "User") + CreateTestMembership(t, posterID, groupID, "Member") + CreateTestMembership(t, modID, groupID, "Moderator") + _, modToken := CreateTestSession(t, modID) + + // Define the group's area: a small box around (lng -1.0, lat 52.0). Freegle + // stores degrees labelled SRID 3857, matching messages_spatial.point. + db.Exec("UPDATE `groups` SET polyindex = ST_GeomFromText('POLYGON((-1.1 51.9, -0.9 51.9, -0.9 52.1, -1.1 52.1, -1.1 51.9))', 3857) WHERE id = ?", groupID) + + insideMsg := CreateTestMessage(t, posterID, groupID, prefix+" inside area", 52.0, -1.0) + outsideMsg := CreateTestMessage(t, posterID, groupID, prefix+" outside area", 52.0, -5.0) + + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=outsidecga&jwt=%s", groupID, modToken), nil)) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + var body map[string]interface{} + json.NewDecoder(resp.Body).Decode(&body) + msgs, _ := body["messages"].([]interface{}) + foundInside, foundOutside := false, false + for _, id := range msgs { + if id == float64(insideMsg) { + foundInside = true + } + if id == float64(outsideMsg) { + foundOutside = true + } + } + assert.True(t, foundOutside, "out-of-area post should appear under outsidecga filter") + assert.False(t, foundInside, "in-area post should not appear under outsidecga filter") + + db.Exec("DELETE FROM messages_spatial WHERE msgid IN (?, ?)", insideMsg, outsideMsg) + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", insideMsg, outsideMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?)", insideMsg, outsideMsg) +} + func TestListMessagesPendingUnauthorized(t *testing.T) { prefix := uniquePrefix("lstmsg_pend_unauth") diff --git a/plans/active/autoapprove-delay.md b/plans/active/autoapprove-delay.md new file mode 100644 index 0000000000..85611c9a97 --- /dev/null +++ b/plans/active/autoapprove-delay.md @@ -0,0 +1,133 @@ +# Auto-approve NULL-status posts after a configurable delay + +Branch: `feature/autoapprove-delay` (worktree `autoapprove`) + +## Goal + +Members whose group posting status is **NULL** ("auto-moderated" tier — never explicitly +set) currently have their posts stuck in Pending until a mod acts or the 48-hour fallback +fires. Introduce a faster, safer path: their **content-check-clean** posts auto-approve +after a configurable delay (default **20 minutes**), giving mods and microvolunteers a +window to intervene, **unless** danger signals are present, with a configurable +**quality-check sample** held back for manual review. + +Plus a modtools Approved-page filter dropdown (auto-approved / recently-joined / outside +CGA) that defaults to summary view for the auto-approved variant. + +## Production sizing (live, last 30 days) + +- 66,074 posts; 11.3% from members who joined that group in the last 30 days. +- Of recent-joiner posts: **NULL 76.5%**, DEFAULT 22.9%, MODERATED 0.3%, UNMODERATED 0.3%. +- The 20-minute path carries ~5,700 posts/30d (~190/day) — today they wait for a mod or 48h. + +## Key architecture (from code exploration) + +- **All posts already land in Pending.** Go `handleJoinAndPost` always inserts Pending; + PHP `JoinAndPost`/email (MailRouter) route NULL → Pending. **No routing change needed.** +- `messages:contentcheck` (every minute, `ContentCheckService::processUnprocessed`) + runs spam/worry/PII/language checks **once** per fresh pending row: + - clean + user not-moderated (DEFAULT/UNMODERATED) + group not moderated → promote now. + - clean + user **NULL**/MODERATED **or** group moderated → kept Pending, sets + `contentcheck_checked_at=now`, `contentcheck_reasons=NULL`. + - flagged (`action=flag`) → kept Pending with `contentcheck_reasons` JSON. + - blocked (`action=block`) → collection Spam. +- `messages:auto-approve` (hourly, `AutoApproveService`, 48h) is the existing safety net. +- Approve = `UPDATE messages_groups SET collection='Approved', approvedby=NULL, + approvedat=NOW(), arrival=NOW()` + log `Autoapproved` + spatial index + freebie alerts + for OFFERs. Digest cron mails it (reads collection='Approved', arrival>lastsent). + +## Design — new batch service `AutoApproveCleanService` + +`messages:auto-approve-clean`, scheduled **every minute** (`withoutOverlapping`, +`runInBackground`). Picks up Pending rows that: + +1. poster's `memberships.ourPostingStatus IS NULL` — the auto-moderated tier (the + "reinterpret NULL as group settings" change; explicit MODERATED stays moderated, + DEFAULT/UNMODERATED already approved immediately by contentcheck). +2. `contentcheck_checked_at IS NOT NULL AND contentcheck_reasons IS NULL` — content + check ran and was **clean**. (This is how "all posts still go through spam checks; + suspect → Pending" is honoured: suspect rows have reasons and are excluded.) +3. `mg.arrival <= NOW() - INTERVAL {delay} MINUTE`, where `{delay}` is the per-group + `settings.autoapprove.delay_minutes` (JSON_EXTRACT) falling back to + `config('freegle.autoapprove.delay_minutes', 20)`. +4. `mg.heldby IS NULL`, `m.heldby IS NULL`, `mg.spamreason IS NULL`, `m.spamreason IS NULL`, + `mg.deleted=0`, `m.deleted IS NULL`, `u.deleted IS NULL`. +5. Group is **not moderated**: `settings.moderated` falsy AND `rules.fullymoderated` falsy + AND `overridemoderation != 'ModerateAll'` AND publish AND not closed AND not + `autofunctionoverride` (mirrors AutoApproveService gating, minus the 48h membership age). +6. **No danger signals** (per msgid/groupid/fromuser) — see below. +7. **Not** in the quality-check sample (deterministic per msgid; percent from + `settings.autoapprove.quality_check_percent` ?? `config(...quality_check_percent, 0)`). + +Then approve (same side effects as `AutoApproveService::approveOnGroup`) + log Autoapproved. + +### Danger signals (veto → leave Pending for a mod) + +- **Microvolunteering reject**: `microactions` where `msgid=`, `actiontype='CheckMessage'`, + `result='Reject'` → count ≥ 1. +- **User notes**: `users_comments` where `userid=fromuser` → any row. +- **Recent negative mod action**: `logs` where `user=fromuser` AND `byuser<>user` AND + `timestamp >= NOW()-{danger_log_days}` AND + ((type='Message' AND subtype IN ('Rejected','Deleted','Replied')) OR + (type='User' AND subtype IN ('Mailed','Rejected','Deleted','Suspect','ClassifiedSpam'))). +- **Known spammer**: `spam_users` where `userid=fromuser` AND collection IN ('Spammer','PendingAdd'). +- **Membership review pending**: `memberships.reviewrequestedat IS NOT NULL` AND + (`reviewedat IS NULL` OR `reviewedat < reviewrequestedat`). + +(Worry-words / concern keywords already excluded via `contentcheck_reasons IS NULL`.) + +### Quality-check sample + +`abs(crc32((string)$msgid)) % 100 < percent` → **held** (skip; mod reviews, or 48h fallback +catches it). Deterministic so a message never oscillates. + +## Frontend / Go API filter (secondary) + +- Go `ListMessagesMT` (`/api/modtools/messages`): new `filter` param — `autoapproved` + (`mg.approvedby IS NULL`), `recentjoin` (join memberships added ≥ NOW()-7d), + `outsidecga` (spatial: message point not within group polygon). Go tests for each. +- Approved page `[[id]]/[[term]].vue`: `` dropdown (default "All"); pass + `filter` to `fetchMessagesMT`; when `autoapproved` selected set the summary miscStore key. +- `ModSettingsGroup.vue`: two `` controls (delay minutes, quality %). + +## Config / settings + +- `config/freegle.php` → `autoapprove` block (delay_minutes=20, quality_check_percent=0, + danger_log_days=90). +- `iznik-server/include/group/Group.php` `defaultSettings['autoapprove']` gains + `delay_minutes`/`quality_check_percent` (for the modtools UI + PHP parity). + +## Gotchas (thought through) + +1. **Re-evaluation**: contentcheck only touches each row once (`contentcheck_checked_at IS + NULL`). This service must query already-checked rows → keys off `checked_at NOT NULL AND + reasons NULL`, NOT `checked_at NULL`. +2. **Per-group delay** must be in the SQL (JSON_EXTRACT COALESCE default) so a group's + shorter/longer override is honoured; a single global threshold would be wrong. +3. **Moderated groups**: contentcheck keeps clean posts Pending with reasons=NULL even on + moderated groups, so this service must explicitly exclude moderated/Big-Switch groups. +4. **Races**: query filters `collection='Pending'` and the UPDATE uses `collection<>'Approved'` + so a mod approve/reject in the window can't double-fire; held rows excluded. +5. **48h service overlap**: both set approvedby=NULL and guard on collection; harmless. +6. **Deleted user/message**: filtered out (no Autoapproved log on invisible rows). +7. **No membership-age gate** (unlike 48h service): the brief wants NULL members to post + after the delay regardless of join age; danger signals + content checks + QA sample + carry the risk. New-member visibility is also covered by the recent-join filter. +8. **arrival reset on approve**: digest keys off arrival>lastsent; per-(msgid,groupid) + update only resets the row being approved. Correct for multi-group posts. +9. **No schema migration / no routing change** → minimal blast radius. + +## Status + +| # | Task | Status | Notes | +|---|------|--------|-------| +| 1 | Design doc | ✅ | this file | +| 2 | AutoApproveCleanService + tests (TDD) | ✅ | 25 unit tests; 29/29 green after held-FK fix | +| 3 | AutoApproveCleanCommand + test | ✅ | 4 command tests | +| 4 | Schedule entry (console.php) | ✅ | everyMinute, withoutOverlapping | +| 5 | config/freegle.php defaults | ✅ | autoapprove block (no Group.php change — absent=site default) | +| 6 | Go ListMessagesMT filter + tests | ✅ | autoapproved/recentjoin/outsidecga + 3 Go tests; suite running | +| 7 | Frontend approved-page dropdown + summary | ✅ | b-form-select; summary on autoapproved | +| 8 | ModSettingsGroup.vue settings controls | ✅ | delay_minutes + quality_check_percent | +| 9 | Run all suites via worktree status API | 🔄 | Laravel green; Go running; Vitest pending | +| 10 | PR | ⬜ | never merge | From 9c4ae84982a3829da235ff0d7d6429c2915d526c Mon Sep 17 00:00:00 2001 From: edwh Date: Wed, 3 Jun 2026 23:34:09 +0100 Subject: [PATCH 02/26] docs(autoapprove): record green test runs and PR link in plan --- plans/active/autoapprove-delay.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plans/active/autoapprove-delay.md b/plans/active/autoapprove-delay.md index 85611c9a97..9792f96946 100644 --- a/plans/active/autoapprove-delay.md +++ b/plans/active/autoapprove-delay.md @@ -129,5 +129,5 @@ catches it). Deterministic so a message never oscillates. | 6 | Go ListMessagesMT filter + tests | ✅ | autoapproved/recentjoin/outsidecga + 3 Go tests; suite running | | 7 | Frontend approved-page dropdown + summary | ✅ | b-form-select; summary on autoapproved | | 8 | ModSettingsGroup.vue settings controls | ✅ | delay_minutes + quality_check_percent | -| 9 | Run all suites via worktree status API | 🔄 | Laravel green; Go running; Vitest pending | -| 10 | PR | ⬜ | never merge | +| 9 | Run all suites via worktree status API | ✅ | full Laravel 3962/3962 ✓; Go 3004/3004 ✓; Vitest modtools 4475/4475 ✓ | +| 10 | Push + PR (Freegle/Iznik) | ✅ | PR #639 — https://github.com/Freegle/Iznik/pull/639 (awaiting CI; never merge) | From dfcd70953d2bcc9f7774baa765c64e7b742cb856 Mon Sep 17 00:00:00 2001 From: edwh Date: Thu, 4 Jun 2026 08:33:27 +0100 Subject: [PATCH 03/26] feat(modtools): restructure moderation views into Pending/Checked/Trusted/Approved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the Approved-page filter dropdown with four nav views: - Pending — unchanged: full pending queue (incl. posts still in their auto-approve hold window, where a mod can stop them before they go live). - Checked — live posts auto-approved via the automated checks, from auto-moderated (NULL posting status) members. Oversight only. - Trusted — live posts that went live without moderation from trusted (group-settings) members. Oversight only. - Approved — unchanged: all live messages (Checked/Trusted are focused subsets). - Go /modtools/messages: replace the autoapproved/recentjoin/outsidecga filters with view filters checked (approvedby NULL + poster status NULL) and trusted (approvedby NULL + poster status DEFAULT/UNMODERATED). - Session work counts: add checked + trusted counts (last 24h, informational/blue, not added to the actionable total). Nav shows Pending red, Checked/Trusted blue, Approved none. - New Checked/Trusted pages default to summary view; revert the Approved-page dropdown. - Tests: replace the Go filter tests with checked/trusted; add Vitest specs for both pages. --- iznik-nuxt3/modtools/layouts/default.vue | 16 ++ .../messages/approved/[[id]]/[[term]].vue | 35 ---- .../messages/checked/[[id]]/[[term]].vue | 170 ++++++++++++++++++ .../messages/trusted/[[id]]/[[term]].vue | 168 +++++++++++++++++ .../pages/modtools/messages/checked.spec.js | 158 ++++++++++++++++ .../pages/modtools/messages/trusted.spec.js | 158 ++++++++++++++++ iznik-server-go/message/message_list.go | 26 ++- iznik-server-go/session/session.go | 39 ++++ iznik-server-go/test/message_test.go | 150 +++++++--------- 9 files changed, 780 insertions(+), 140 deletions(-) create mode 100644 iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue create mode 100644 iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue create mode 100644 iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js create mode 100644 iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js diff --git a/iznik-nuxt3/modtools/layouts/default.vue b/iznik-nuxt3/modtools/layouts/default.vue index aac47517d2..df1b47d38e 100644 --- a/iznik-nuxt3/modtools/layouts/default.vue +++ b/iznik-nuxt3/modtools/layouts/default.vue @@ -92,6 +92,22 @@ indent @mobilehidemenu="mobilehidemenu" /> + + Select a community to search messages. -
@@ -64,13 +56,11 @@ import { ref, computed, watch, onMounted, nextTick } from 'vue' import { useRoute, useRouter } from '#imports' import { useMessageStore } from '@/stores/message' -import { useMiscStore } from '@/stores/misc' import { setupModMessages } from '@/composables/useModMessages' import { useMe } from '~/composables/useMe' // Stores const messageStore = useMessageStore() -const miscStore = useMiscStore() // Composables const modMessages = setupModMessages(true) @@ -97,13 +87,6 @@ const { // Local state (formerly data()) const chosengroupid = ref(0) const bump = ref(0) -const filter = ref('') -const filterOptions = [ - { value: '', text: 'All messages' }, - { value: 'recentjoin', text: 'Joined last 7 days' }, - { value: 'autoapproved', text: 'Auto-approved' }, - { value: 'outsidecga', text: 'Outside group area' }, -] const urlOverride = ref(false) const loaded = ref(false) const highlightMsgId = ref(null) @@ -232,21 +215,6 @@ function searchedMessage(term) { } } -watch(filter, (newVal) => { - // The auto-approved view is for scanning posts that went out without a human - // checking, so default it to the compact summary view. - if (newVal === 'autoapproved') { - miscStore.set({ key: 'modtoolsMessagesApprovedSummary', value: true }) - } - show.value = 0 - context.value = null - modMessages.listingIds.value = new Set() - modMessages.listingIdOrder.value = [] - messageStore.clear() - bump.value++ -}) - - function searchedMember(term) { show.value = 0 messageTerm.value = null @@ -321,9 +289,6 @@ async function loadMore($state) { modtools: true, summary: false, } - if (filter.value) { - params.filter = filter.value - } } params.context = context.value diff --git a/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue b/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue new file mode 100644 index 0000000000..22e0f303e0 --- /dev/null +++ b/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue @@ -0,0 +1,170 @@ + + + diff --git a/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue b/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue new file mode 100644 index 0000000000..8e01084e12 --- /dev/null +++ b/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue @@ -0,0 +1,168 @@ + + + diff --git a/iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js b/iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js new file mode 100644 index 0000000000..40ca473488 --- /dev/null +++ b/iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js @@ -0,0 +1,158 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { mount } from '@vue/test-utils' +import { createPinia, setActivePinia } from 'pinia' +import { ref, computed } from 'vue' +import CheckedPage from '~/modtools/pages/messages/checked/[[id]]/[[term]].vue' + +const mockBusy = ref(false) +const mockContext = ref(null) +const mockGroup = ref(null) +const mockGroupid = ref(0) +const mockShow = ref(0) +const mockCollection = ref(null) +const mockDistance = ref(10) +const mockSummarykey = ref(false) +const mockMessages = ref([]) +const mockListingIds = ref(new Set()) + +vi.mock('~/composables/useModMessages', () => ({ + setupModMessages: () => ({ + busy: mockBusy, + context: mockContext, + group: mockGroup, + groupid: mockGroupid, + show: mockShow, + collection: mockCollection, + distance: mockDistance, + summarykey: mockSummarykey, + messages: mockMessages, + listingIds: mockListingIds, + }), +})) + +const mockMessageStore = { + list: {}, + context: null, + fetchMessagesMT: vi.fn().mockResolvedValue([]), + clear: vi.fn(), +} +vi.mock('@/stores/message', () => ({ + useMessageStore: () => mockMessageStore, +})) + +const mockMiscStore = { get: vi.fn(), set: vi.fn() } +vi.mock('@/stores/misc', () => ({ + useMiscStore: () => mockMiscStore, +})) + +const mockModGroupStore = { + received: true, + get: vi.fn(), + fetchIfNeedBeMT: vi.fn().mockResolvedValue(undefined), +} +vi.mock('@/stores/modgroup', () => ({ + useModGroupStore: () => mockModGroupStore, +})) + +vi.mock('~/composables/useMe', () => ({ + useMe: () => ({ me: ref({ id: 1, displayname: 'Test User' }) }), +})) + +const mockRouteParams = ref({ id: undefined, term: undefined }) +const mockRouterPush = vi.fn() + +vi.mock('#imports', async () => { + const actual = await vi.importActual('#imports') + return { + ...actual, + useRoute: () => ({ params: mockRouteParams.value, query: {} }), + useRouter: () => ({ push: mockRouterPush }), + } +}) + +globalThis.__testUseRoute = () => ({ params: mockRouteParams.value, query: {} }) +globalThis.__testUseRouter = () => ({ + push: mockRouterPush, + replace: mockRouterPush, + currentRoute: { value: { path: '/' } }, +}) + +describe('messages/checked/[[id]]/[[term]].vue page', () => { + function mountComponent() { + return mount(CheckedPage, { + global: { + plugins: [createPinia()], + stubs: { + 'client-only': { template: '
' }, + ScrollToTop: { template: '
' }, + ModGroupSelect: { + template: '
', + props: ['modelValue', 'all', 'modonly', 'remember', 'urlOverride'], + }, + ModtoolsViewControl: { + template: '
', + props: ['misckey'], + }, + NoticeMessage: { + template: '
', + }, + ModMessages: { template: '
' }, + Spinner: { template: '
', props: ['size'] }, + 'infinite-loading': { + template: + '
', + props: ['direction', 'distance', 'identifier'], + emits: ['infinite'], + }, + }, + }, + }) + } + + beforeEach(() => { + vi.clearAllMocks() + setActivePinia(createPinia()) + mockBusy.value = false + mockContext.value = null + mockGroupid.value = 0 + mockShow.value = 0 + mockCollection.value = null + mockMessages.value = [] + mockListingIds.value = new Set() + mockMessageStore.list = {} + mockMessageStore.context = null + mockMiscStore.get.mockReturnValue(undefined) + mockRouteParams.value = { id: undefined, term: undefined } + }) + + it('mounts and targets the Approved collection', () => { + const wrapper = mountComponent() + expect(wrapper.exists()).toBe(true) + expect(mockCollection.value).toBe('Approved') + }) + + it('defaults the oversight list to summary view', () => { + mountComponent() + expect(mockMiscStore.set).toHaveBeenCalledWith({ + key: 'modtoolsMessagesCheckedSummary', + value: true, + }) + }) + + it('loadMore fetches with the checked filter on the Approved collection', async () => { + const wrapper = mountComponent() + const $state = { loaded: vi.fn(), complete: vi.fn(), error: vi.fn() } + await wrapper.vm.loadMore($state) + expect(mockMessageStore.fetchMessagesMT).toHaveBeenCalled() + const params = mockMessageStore.fetchMessagesMT.mock.calls[0][0] + expect(params.filter).toBe('checked') + expect(params.collection).toBe('Approved') + }) + + it('reads the group id from the route param', () => { + mockRouteParams.value = { id: '42', term: undefined } + const wrapper = mountComponent() + expect(wrapper.vm.id).toBe(42) + expect(mockGroupid.value).toBe(42) + }) +}) diff --git a/iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js b/iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js new file mode 100644 index 0000000000..952a2aef2d --- /dev/null +++ b/iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js @@ -0,0 +1,158 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { mount } from '@vue/test-utils' +import { createPinia, setActivePinia } from 'pinia' +import { ref } from 'vue' +import TrustedPage from '~/modtools/pages/messages/trusted/[[id]]/[[term]].vue' + +const mockBusy = ref(false) +const mockContext = ref(null) +const mockGroup = ref(null) +const mockGroupid = ref(0) +const mockShow = ref(0) +const mockCollection = ref(null) +const mockDistance = ref(10) +const mockSummarykey = ref(false) +const mockMessages = ref([]) +const mockListingIds = ref(new Set()) + +vi.mock('~/composables/useModMessages', () => ({ + setupModMessages: () => ({ + busy: mockBusy, + context: mockContext, + group: mockGroup, + groupid: mockGroupid, + show: mockShow, + collection: mockCollection, + distance: mockDistance, + summarykey: mockSummarykey, + messages: mockMessages, + listingIds: mockListingIds, + }), +})) + +const mockMessageStore = { + list: {}, + context: null, + fetchMessagesMT: vi.fn().mockResolvedValue([]), + clear: vi.fn(), +} +vi.mock('@/stores/message', () => ({ + useMessageStore: () => mockMessageStore, +})) + +const mockMiscStore = { get: vi.fn(), set: vi.fn() } +vi.mock('@/stores/misc', () => ({ + useMiscStore: () => mockMiscStore, +})) + +const mockModGroupStore = { + received: true, + get: vi.fn(), + fetchIfNeedBeMT: vi.fn().mockResolvedValue(undefined), +} +vi.mock('@/stores/modgroup', () => ({ + useModGroupStore: () => mockModGroupStore, +})) + +vi.mock('~/composables/useMe', () => ({ + useMe: () => ({ me: ref({ id: 1, displayname: 'Test User' }) }), +})) + +const mockRouteParams = ref({ id: undefined, term: undefined }) +const mockRouterPush = vi.fn() + +vi.mock('#imports', async () => { + const actual = await vi.importActual('#imports') + return { + ...actual, + useRoute: () => ({ params: mockRouteParams.value, query: {} }), + useRouter: () => ({ push: mockRouterPush }), + } +}) + +globalThis.__testUseRoute = () => ({ params: mockRouteParams.value, query: {} }) +globalThis.__testUseRouter = () => ({ + push: mockRouterPush, + replace: mockRouterPush, + currentRoute: { value: { path: '/' } }, +}) + +describe('messages/trusted/[[id]]/[[term]].vue page', () => { + function mountComponent() { + return mount(TrustedPage, { + global: { + plugins: [createPinia()], + stubs: { + 'client-only': { template: '
' }, + ScrollToTop: { template: '
' }, + ModGroupSelect: { + template: '
', + props: ['modelValue', 'all', 'modonly', 'remember', 'urlOverride'], + }, + ModtoolsViewControl: { + template: '
', + props: ['misckey'], + }, + NoticeMessage: { + template: '
', + }, + ModMessages: { template: '
' }, + Spinner: { template: '
', props: ['size'] }, + 'infinite-loading': { + template: + '
', + props: ['direction', 'distance', 'identifier'], + emits: ['infinite'], + }, + }, + }, + }) + } + + beforeEach(() => { + vi.clearAllMocks() + setActivePinia(createPinia()) + mockBusy.value = false + mockContext.value = null + mockGroupid.value = 0 + mockShow.value = 0 + mockCollection.value = null + mockMessages.value = [] + mockListingIds.value = new Set() + mockMessageStore.list = {} + mockMessageStore.context = null + mockMiscStore.get.mockReturnValue(undefined) + mockRouteParams.value = { id: undefined, term: undefined } + }) + + it('mounts and targets the Approved collection', () => { + const wrapper = mountComponent() + expect(wrapper.exists()).toBe(true) + expect(mockCollection.value).toBe('Approved') + }) + + it('defaults the oversight list to summary view', () => { + mountComponent() + expect(mockMiscStore.set).toHaveBeenCalledWith({ + key: 'modtoolsMessagesTrustedSummary', + value: true, + }) + }) + + it('loadMore fetches with the trusted filter on the Approved collection', async () => { + const wrapper = mountComponent() + const $state = { loaded: vi.fn(), complete: vi.fn(), error: vi.fn() } + await wrapper.vm.loadMore($state) + expect(mockMessageStore.fetchMessagesMT).toHaveBeenCalled() + const params = mockMessageStore.fetchMessagesMT.mock.calls[0][0] + expect(params.filter).toBe('trusted') + expect(params.collection).toBe('Approved') + }) + + it('reads the group id from the route param', () => { + mockRouteParams.value = { id: '42', term: undefined } + const wrapper = mountComponent() + expect(wrapper.vm.id).toBe(42) + expect(mockGroupid.value).toBe(42) + }) +}) diff --git a/iznik-server-go/message/message_list.go b/iznik-server-go/message/message_list.go index 189e526d5b..f10d7c2388 100644 --- a/iznik-server-go/message/message_list.go +++ b/iznik-server-go/message/message_list.go @@ -448,10 +448,10 @@ func ListMessagesMT(c *fiber.Ctx) error { search := c.Query("search", "") fromuserStr := c.Query("fromuser", "0") fromuser, _ := strconv.ParseUint(fromuserStr, 10, 64) - // Optional moderation filter for the standard (non-search) listing: - // autoapproved — posts approved without a moderator (mg.approvedby IS NULL) - // recentjoin — posts from members who joined this group in the last 7 days - // outsidecga — posts whose location falls outside the group's area polygon + // Optional oversight view of the Approved collection (non-search listing): + // checked — live posts auto-approved via the automated checks, from + // auto-moderated (NULL posting status) members + // trusted — live posts from trusted (group-settings) members filter := c.Query("filter", "") var msgIDs []uint64 @@ -562,18 +562,14 @@ func ListMessagesMT(c *fiber.Ctx) error { filterJoin := "" filterWhere := "" switch filter { - case "autoapproved": - filterWhere = "AND mg.approvedby IS NULL " - case "recentjoin": + case "checked": + // Auto-approved (approvedby NULL) from auto-moderated (NULL) members. filterJoin = "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid " - filterWhere = "AND mem.added >= NOW() - INTERVAL 7 DAY " - case "outsidecga": - // Use the canonical spatial point (messages_spatial), matching how - // location containment is checked elsewhere. Unmappable posts (no - // spatial row) can't be "outside" and are excluded by the INNER JOIN. - filterJoin = "INNER JOIN `groups` g ON g.id = mg.groupid " + - "INNER JOIN messages_spatial ms ON ms.msgid = mg.msgid AND ms.groupid = mg.groupid " - filterWhere = "AND g.polyindex IS NOT NULL AND NOT ST_Contains(g.polyindex, ms.point) " + filterWhere = "AND mg.approvedby IS NULL AND mem.ourPostingStatus IS NULL " + case "trusted": + // Went live without moderation from trusted (group-settings) members. + filterJoin = "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid " + filterWhere = "AND mg.approvedby IS NULL AND (mem.ourPostingStatus = 'DEFAULT' OR mem.ourPostingStatus = 'UNMODERATED') " } branchSQL := "SELECT mg.msgid, mg.arrival FROM messages_groups mg " + diff --git a/iznik-server-go/session/session.go b/iznik-server-go/session/session.go index 0c3a5aea7d..ebdec5f7a4 100644 --- a/iznik-server-go/session/session.go +++ b/iznik-server-go/session/session.go @@ -1030,6 +1030,10 @@ func GetSession(c *fiber.Ctx) error { var housekeeping, cronjobs int64 var emailin, emailout int64 var helperEscalated int64 + // Informational (blue) counts of recently-live posts for the oversight + // views: checked = auto-approved posts from auto-moderated (NULL) members; + // trusted = posts that went live from trusted (group-settings) members. + var checked, trusted int64 var wg2 sync.WaitGroup @@ -1076,6 +1080,39 @@ func GetSession(c *fiber.Ctx) error { } }() + // --- Checked: recently auto-approved posts from auto-moderated (NULL) members. + // Informational oversight count (blue), bounded to the last 24h so it stays + // useful rather than the unbounded all-time approved total. --- + wg2.Add(1) + go func() { + defer wg2.Done() + db.Raw("SELECT COUNT(*) FROM messages_groups mg "+ + "INNER JOIN messages m ON m.id = mg.msgid "+ + "INNER JOIN users u ON u.id = m.fromuser "+ + "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid "+ + "WHERE mg.groupid IN ? AND mg.collection = ? AND mg.deleted = 0 "+ + "AND m.deleted IS NULL AND u.deleted IS NULL "+ + "AND mg.approvedby IS NULL AND mem.ourPostingStatus IS NULL "+ + "AND mg.arrival >= NOW() - INTERVAL 1 DAY", + modGroupIDs, utils.COLLECTION_APPROVED).Scan(&checked) + }() + + // --- Trusted: recently-live posts from trusted (group-settings) members. + // Informational oversight count (blue), bounded to the last 24h. --- + wg2.Add(1) + go func() { + defer wg2.Done() + db.Raw("SELECT COUNT(*) FROM messages_groups mg "+ + "INNER JOIN messages m ON m.id = mg.msgid "+ + "INNER JOIN users u ON u.id = m.fromuser "+ + "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid "+ + "WHERE mg.groupid IN ? AND mg.collection = ? AND mg.deleted = 0 "+ + "AND m.deleted IS NULL AND u.deleted IS NULL "+ + "AND mg.approvedby IS NULL AND mem.ourPostingStatus IN ('DEFAULT', 'UNMODERATED') "+ + "AND mg.arrival >= NOW() - INTERVAL 1 DAY", + modGroupIDs, utils.COLLECTION_APPROVED).Scan(&trusted) + }() + // --- Spam messages (only for active groups) --- wg2.Add(1) go func() { @@ -1452,6 +1489,8 @@ func GetSession(c *fiber.Ctx) error { work = fiber.Map{ "pending": pending, "pendingother": pendingother, + "checked": checked, + "trusted": trusted, "spam": spam, "pendingmembers": pendingmembers, "spammembers": spammembers, diff --git a/iznik-server-go/test/message_test.go b/iznik-server-go/test/message_test.go index 2f20e5d3eb..51d74d02c4 100644 --- a/iznik-server-go/test/message_test.go +++ b/iznik-server-go/test/message_test.go @@ -5552,136 +5552,106 @@ func TestListMessagesMT_LimboUserMessageNotReturned(t *testing.T) { db.Exec("UPDATE users SET deleted = NULL WHERE id = ?", posterID) } -func TestListMessagesMT_FilterAutoApproved(t *testing.T) { - // filter=autoapproved returns only posts approved without a moderator - // (messages_groups.approvedby IS NULL). - prefix := uniquePrefix("lstmt_autoapp") +func TestListMessagesMT_FilterChecked(t *testing.T) { + // filter=checked returns live posts auto-approved (approvedby IS NULL) from + // auto-moderated (NULL posting status) members — not trusted-member posts and + // not mod-approved posts. + prefix := uniquePrefix("lstmt_checked") db := database.DBConn groupID := CreateTestGroup(t, prefix) - posterID := CreateTestUser(t, prefix+"_poster", "User") + nullPoster := CreateTestUser(t, prefix+"_null", "User") + defaultPoster := CreateTestUser(t, prefix+"_default", "User") modID := CreateTestUser(t, prefix+"_mod", "User") - CreateTestMembership(t, posterID, groupID, "Member") + CreateTestMembership(t, nullPoster, groupID, "Member") + CreateTestMembership(t, defaultPoster, groupID, "Member") CreateTestMembership(t, modID, groupID, "Moderator") + db.Exec("UPDATE memberships SET ourPostingStatus = NULL WHERE userid = ? AND groupid = ?", nullPoster, groupID) + db.Exec("UPDATE memberships SET ourPostingStatus = 'DEFAULT' WHERE userid = ? AND groupid = ?", defaultPoster, groupID) _, modToken := CreateTestSession(t, modID) - autoMsg := CreateTestMessage(t, posterID, groupID, prefix+" auto approved", 52.0, -1.0) - manualMsg := CreateTestMessage(t, posterID, groupID, prefix+" mod approved", 52.0, -1.0) - db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid=?", autoMsg) - db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=? WHERE msgid=?", modID, manualMsg) + checkedMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" auto checked", 52.0, -1.0) + trustedMsg := CreateTestMessage(t, defaultPoster, groupID, prefix+" trusted live", 52.0, -1.0) + modApprovedMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" mod approved", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid IN (?, ?)", checkedMsg, trustedMsg) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=? WHERE msgid=?", modID, modApprovedMsg) resp, err := getApp().Test(httptest.NewRequest("GET", - fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=autoapproved&jwt=%s", groupID, modToken), nil)) + fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=checked&jwt=%s", groupID, modToken), nil)) assert.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) var body map[string]interface{} json.NewDecoder(resp.Body).Decode(&body) msgs, _ := body["messages"].([]interface{}) - foundAuto, foundManual := false, false + foundChecked, foundTrusted, foundMod := false, false, false for _, id := range msgs { - if id == float64(autoMsg) { - foundAuto = true - } - if id == float64(manualMsg) { - foundManual = true + switch id { + case float64(checkedMsg): + foundChecked = true + case float64(trustedMsg): + foundTrusted = true + case float64(modApprovedMsg): + foundMod = true } } - assert.True(t, foundAuto, "auto-approved message should appear under autoapproved filter") - assert.False(t, foundManual, "mod-approved message should not appear under autoapproved filter") + assert.True(t, foundChecked, "auto-approved post from NULL member should appear under checked") + assert.False(t, foundTrusted, "trusted-member post should not appear under checked") + assert.False(t, foundMod, "mod-approved post should not appear under checked") - db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", autoMsg, manualMsg) - db.Exec("DELETE FROM messages WHERE id IN (?, ?)", autoMsg, manualMsg) + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg) } -func TestListMessagesMT_FilterRecentJoin(t *testing.T) { - // filter=recentjoin returns only posts from members who joined this group - // within the last 7 days. - prefix := uniquePrefix("lstmt_recentjoin") +func TestListMessagesMT_FilterTrusted(t *testing.T) { + // filter=trusted returns live posts (approvedby IS NULL) from trusted + // (DEFAULT/UNMODERATED posting status) members — not auto-moderated NULL + // members and not mod-approved posts. + prefix := uniquePrefix("lstmt_trusted") db := database.DBConn groupID := CreateTestGroup(t, prefix) - recentID := CreateTestUser(t, prefix+"_recent", "User") - oldID := CreateTestUser(t, prefix+"_old", "User") + nullPoster := CreateTestUser(t, prefix+"_null", "User") + defaultPoster := CreateTestUser(t, prefix+"_default", "User") modID := CreateTestUser(t, prefix+"_mod", "User") - CreateTestMembership(t, recentID, groupID, "Member") - CreateTestMembership(t, oldID, groupID, "Member") + CreateTestMembership(t, nullPoster, groupID, "Member") + CreateTestMembership(t, defaultPoster, groupID, "Member") CreateTestMembership(t, modID, groupID, "Moderator") + db.Exec("UPDATE memberships SET ourPostingStatus = NULL WHERE userid = ? AND groupid = ?", nullPoster, groupID) + db.Exec("UPDATE memberships SET ourPostingStatus = 'DEFAULT' WHERE userid = ? AND groupid = ?", defaultPoster, groupID) _, modToken := CreateTestSession(t, modID) - // The old member joined 30 days ago. - db.Exec("UPDATE memberships SET added = NOW() - INTERVAL 30 DAY WHERE userid = ? AND groupid = ?", oldID, groupID) - - recentMsg := CreateTestMessage(t, recentID, groupID, prefix+" recent joiner post", 52.0, -1.0) - oldMsg := CreateTestMessage(t, oldID, groupID, prefix+" old joiner post", 52.0, -1.0) + checkedMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" auto checked", 52.0, -1.0) + trustedMsg := CreateTestMessage(t, defaultPoster, groupID, prefix+" trusted live", 52.0, -1.0) + modApprovedMsg := CreateTestMessage(t, defaultPoster, groupID, prefix+" mod approved", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid IN (?, ?)", checkedMsg, trustedMsg) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=? WHERE msgid=?", modID, modApprovedMsg) resp, err := getApp().Test(httptest.NewRequest("GET", - fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=recentjoin&jwt=%s", groupID, modToken), nil)) + fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=trusted&jwt=%s", groupID, modToken), nil)) assert.NoError(t, err) assert.Equal(t, 200, resp.StatusCode) var body map[string]interface{} json.NewDecoder(resp.Body).Decode(&body) msgs, _ := body["messages"].([]interface{}) - foundRecent, foundOld := false, false + foundChecked, foundTrusted, foundMod := false, false, false for _, id := range msgs { - if id == float64(recentMsg) { - foundRecent = true - } - if id == float64(oldMsg) { - foundOld = true - } - } - assert.True(t, foundRecent, "recent joiner's post should appear under recentjoin filter") - assert.False(t, foundOld, "long-standing member's post should not appear under recentjoin filter") - - db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", recentMsg, oldMsg) - db.Exec("DELETE FROM messages WHERE id IN (?, ?)", recentMsg, oldMsg) -} - -func TestListMessagesMT_FilterOutsideCGA(t *testing.T) { - // filter=outsidecga returns only posts whose location falls outside the - // group's area polygon. - prefix := uniquePrefix("lstmt_outcga") - db := database.DBConn - - groupID := CreateTestGroup(t, prefix) - posterID := CreateTestUser(t, prefix+"_poster", "User") - modID := CreateTestUser(t, prefix+"_mod", "User") - CreateTestMembership(t, posterID, groupID, "Member") - CreateTestMembership(t, modID, groupID, "Moderator") - _, modToken := CreateTestSession(t, modID) - - // Define the group's area: a small box around (lng -1.0, lat 52.0). Freegle - // stores degrees labelled SRID 3857, matching messages_spatial.point. - db.Exec("UPDATE `groups` SET polyindex = ST_GeomFromText('POLYGON((-1.1 51.9, -0.9 51.9, -0.9 52.1, -1.1 52.1, -1.1 51.9))', 3857) WHERE id = ?", groupID) - - insideMsg := CreateTestMessage(t, posterID, groupID, prefix+" inside area", 52.0, -1.0) - outsideMsg := CreateTestMessage(t, posterID, groupID, prefix+" outside area", 52.0, -5.0) - - resp, err := getApp().Test(httptest.NewRequest("GET", - fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=outsidecga&jwt=%s", groupID, modToken), nil)) - assert.NoError(t, err) - assert.Equal(t, 200, resp.StatusCode) - - var body map[string]interface{} - json.NewDecoder(resp.Body).Decode(&body) - msgs, _ := body["messages"].([]interface{}) - foundInside, foundOutside := false, false - for _, id := range msgs { - if id == float64(insideMsg) { - foundInside = true - } - if id == float64(outsideMsg) { - foundOutside = true + switch id { + case float64(checkedMsg): + foundChecked = true + case float64(trustedMsg): + foundTrusted = true + case float64(modApprovedMsg): + foundMod = true } } - assert.True(t, foundOutside, "out-of-area post should appear under outsidecga filter") - assert.False(t, foundInside, "in-area post should not appear under outsidecga filter") + assert.True(t, foundTrusted, "trusted-member post should appear under trusted") + assert.False(t, foundChecked, "auto-moderated NULL-member post should not appear under trusted") + assert.False(t, foundMod, "mod-approved post should not appear under trusted") - db.Exec("DELETE FROM messages_spatial WHERE msgid IN (?, ?)", insideMsg, outsideMsg) - db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", insideMsg, outsideMsg) - db.Exec("DELETE FROM messages WHERE id IN (?, ?)", insideMsg, outsideMsg) + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg) } func TestListMessagesPendingUnauthorized(t *testing.T) { From 26c7fd8b2ea300f8d822fb05537c7da6ab06c880 Mon Sep 17 00:00:00 2001 From: edwh Date: Thu, 4 Jun 2026 09:08:50 +0100 Subject: [PATCH 04/26] feat(modtools): track moderator review of auto-published posts (checkedby/checkedat) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Auto-published posts (auto-moderated Checked + trusted Trusted members) have no field recording that a mod has reviewed them — approvedby is NULL, contentcheck_* is only the automated pass. Add checkedat/checkedby to messages_groups so the Checked/Trusted oversight queues become real, clearable 'unchecked' queues. - Migration: messages_groups.checkedat + checkedby (FK users), composite index. - session.go: Checked/Trusted counts now = UNCHECKED within the auto-check window (was 24h volume), so the badge is outstanding work that drops as mods check. - message_list.go: checked/trusted views show only unchecked-within-window posts. - POST /modtools/messages/markchecked: mark posts (or a whole bucket) checked. - Window is a named constant (utils.MESSAGE_CHECK_WINDOW_DAYS=7), not hardcoded in SQL strings; posts auto-check after the window so the queue can't pile up. --- ...add_checked_columns_to_messages_groups.php | 39 +++++++++ iznik-server-go/message/markchecked.go | 86 +++++++++++++++++++ iznik-server-go/message/message_list.go | 12 ++- iznik-server-go/router/routes.go | 1 + iznik-server-go/session/session.go | 24 ++++-- iznik-server-go/utils/utils.go | 5 ++ 6 files changed, 157 insertions(+), 10 deletions(-) create mode 100644 iznik-batch/database/migrations/2026_06_04_000001_add_checked_columns_to_messages_groups.php create mode 100644 iznik-server-go/message/markchecked.go diff --git a/iznik-batch/database/migrations/2026_06_04_000001_add_checked_columns_to_messages_groups.php b/iznik-batch/database/migrations/2026_06_04_000001_add_checked_columns_to_messages_groups.php new file mode 100644 index 0000000000..5ffd009837 --- /dev/null +++ b/iznik-batch/database/migrations/2026_06_04_000001_add_checked_columns_to_messages_groups.php @@ -0,0 +1,39 @@ +timestamp('checkedat')->nullable()->after('rejectedat'); + $table->unsignedBigInteger('checkedby')->nullable()->after('checkedat'); + + $table->foreign('checkedby')->references('id')->on('users')->onDelete('set null'); + // Composite index supports the "unchecked in this group" oversight query. + $table->index(['groupid', 'checkedat'], 'messages_groups_groupid_checkedat_idx'); + }); + } + + public function down(): void + { + Schema::table('messages_groups', function (Blueprint $table) { + $table->dropForeign(['checkedby']); + $table->dropIndex('messages_groups_groupid_checkedat_idx'); + $table->dropColumn(['checkedat', 'checkedby']); + }); + } +}; diff --git a/iznik-server-go/message/markchecked.go b/iznik-server-go/message/markchecked.go new file mode 100644 index 0000000000..39b0f28257 --- /dev/null +++ b/iznik-server-go/message/markchecked.go @@ -0,0 +1,86 @@ +package message + +import ( + "fmt" + + "github.com/freegle/iznik-server-go/database" + "github.com/freegle/iznik-server-go/user" + "github.com/freegle/iznik-server-go/utils" + "github.com/gofiber/fiber/v2" +) + +// MarkCheckedRequest is the request body for marking auto-published posts as +// checked by a moderator (clearing them from the Checked/Trusted oversight queues). +type MarkCheckedRequest struct { + Groupid uint64 `json:"groupid"` // 0 = all of the mod's groups + Filter string `json:"filter"` // "checked" or "trusted" (used when no ids given) + IDs []uint64 `json:"ids,omitempty"` // specific messages, else mark the whole bucket +} + +// MarkChecked records that a moderator has reviewed auto-published posts. These +// are posts that went live without a manual approval (auto-moderated and trusted +// members), so there is no approvedby/heldby to key off — this sets a dedicated +// checkedat/checkedby marker that drives the Checked/Trusted counts. +// +// @Summary Mark auto-published posts as checked by a moderator +// @Tags message +// @Accept json +// @Produce json +// @Success 200 {object} map[string]interface{} +// @Router /modtools/messages/markchecked [post] +func MarkChecked(c *fiber.Ctx) error { + myid := user.WhoAmI(c) + if myid == 0 { + return fiber.NewError(fiber.StatusUnauthorized, "Not logged in") + } + + var req MarkCheckedRequest + if err := c.BodyParser(&req); err != nil { + return fiber.NewError(fiber.StatusBadRequest, "Invalid request body") + } + + db := database.DBConn + + // Resolve the groups the mod may act on. + var groupIDs []uint64 + if req.Groupid == 0 { + groupIDs = user.GetActiveModGroupIDs(myid) + if len(groupIDs) == 0 { + return c.JSON(fiber.Map{"success": true, "checked": 0}) + } + } else { + if !user.IsModOfGroup(myid, req.Groupid) { + return fiber.NewError(fiber.StatusForbidden, "Not a moderator for this group") + } + groupIDs = []uint64{req.Groupid} + } + + var rowsAffected int64 + if len(req.IDs) > 0 { + // Mark the specified posts checked — only ones still unchecked, and only + // on groups the mod moderates. + r := db.Exec("UPDATE messages_groups SET checkedat = NOW(), checkedby = ? "+ + "WHERE msgid IN ? AND groupid IN ? AND checkedat IS NULL", + myid, req.IDs, groupIDs) + rowsAffected = r.RowsAffected + } else { + // Mark the whole bucket checked (the "mark all as checked" action). The + // bucket condition mirrors the Checked/Trusted list filter so exactly the + // posts a mod is looking at get cleared. + statusWhere := "mem.ourPostingStatus IS NULL" + if req.Filter == "trusted" { + statusWhere = "(mem.ourPostingStatus = 'DEFAULT' OR mem.ourPostingStatus = 'UNMODERATED')" + } + r := db.Exec("UPDATE messages_groups mg "+ + "INNER JOIN messages m ON m.id = mg.msgid "+ + "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid "+ + "SET mg.checkedat = NOW(), mg.checkedby = ? "+ + "WHERE mg.groupid IN ? AND mg.collection = ? AND mg.deleted = 0 "+ + "AND mg.approvedby IS NULL AND "+statusWhere+" "+ + fmt.Sprintf("AND mg.checkedat IS NULL AND mg.arrival >= NOW() - INTERVAL %d DAY", utils.MESSAGE_CHECK_WINDOW_DAYS), + myid, groupIDs, utils.COLLECTION_APPROVED) + rowsAffected = r.RowsAffected + } + + return c.JSON(fiber.Map{"success": true, "checked": rowsAffected}) +} diff --git a/iznik-server-go/message/message_list.go b/iznik-server-go/message/message_list.go index f10d7c2388..25b685dc38 100644 --- a/iznik-server-go/message/message_list.go +++ b/iznik-server-go/message/message_list.go @@ -2,6 +2,7 @@ package message import ( "encoding/json" + "fmt" "os" "strconv" "strings" @@ -561,15 +562,22 @@ func ListMessagesMT(c *fiber.Ctx) error { // injected SQL is constant (no user-supplied text reaches the query). filterJoin := "" filterWhere := "" + // The Checked/Trusted oversight queues show only posts a mod has NOT yet + // marked checked, within the auto-check window — so the list matches the + // session work-count badge and clears as posts are checked. + checkedWindow := fmt.Sprintf( + "AND mg.checkedat IS NULL AND mg.arrival >= NOW() - INTERVAL %d DAY ", + utils.MESSAGE_CHECK_WINDOW_DAYS, + ) switch filter { case "checked": // Auto-approved (approvedby NULL) from auto-moderated (NULL) members. filterJoin = "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid " - filterWhere = "AND mg.approvedby IS NULL AND mem.ourPostingStatus IS NULL " + filterWhere = "AND mg.approvedby IS NULL AND mem.ourPostingStatus IS NULL " + checkedWindow case "trusted": // Went live without moderation from trusted (group-settings) members. filterJoin = "INNER JOIN memberships mem ON mem.userid = m.fromuser AND mem.groupid = mg.groupid " - filterWhere = "AND mg.approvedby IS NULL AND (mem.ourPostingStatus = 'DEFAULT' OR mem.ourPostingStatus = 'UNMODERATED') " + filterWhere = "AND mg.approvedby IS NULL AND (mem.ourPostingStatus = 'DEFAULT' OR mem.ourPostingStatus = 'UNMODERATED') " + checkedWindow } branchSQL := "SELECT mg.msgid, mg.arrival FROM messages_groups mg " + diff --git a/iznik-server-go/router/routes.go b/iznik-server-go/router/routes.go index 3cfb4dd53c..6cfd423999 100644 --- a/iznik-server-go/router/routes.go +++ b/iznik-server-go/router/routes.go @@ -868,6 +868,7 @@ func SetupRoutes(app *fiber.App) { // @Tags message rg.Get("/messages", deprecation.Marker("GET /messages", "2026-08-01"), message.ListMessages) rg.Get("/modtools/messages", message.ListMessagesMT) + rg.Post("/modtools/messages/markchecked", message.MarkChecked) // Message Count // @Router /message/count [get] diff --git a/iznik-server-go/session/session.go b/iznik-server-go/session/session.go index ebdec5f7a4..260231ae8c 100644 --- a/iznik-server-go/session/session.go +++ b/iznik-server-go/session/session.go @@ -1030,13 +1030,21 @@ func GetSession(c *fiber.Ctx) error { var housekeeping, cronjobs int64 var emailin, emailout int64 var helperEscalated int64 - // Informational (blue) counts of recently-live posts for the oversight + // Informational (blue) counts of UNCHECKED live posts for the oversight // views: checked = auto-approved posts from auto-moderated (NULL) members; // trusted = posts that went live from trusted (group-settings) members. + // A post leaves the count when a mod marks it checked (checkedat set) or + // after 7 days (auto-checked, so the queue can't pile up indefinitely). var checked, trusted int64 var wg2 sync.WaitGroup + // Oversight queues count only unchecked posts within the auto-check window. + checkedWindowSQL := fmt.Sprintf( + "AND mg.checkedat IS NULL AND mg.arrival >= NOW() - INTERVAL %d DAY", + utils.MESSAGE_CHECK_WINDOW_DAYS, + ) + // --- Pending messages: active groups split by held, inactive all → pendingother --- // Only count messages where contentcheck_checked_at IS NOT NULL: the content // check has run and left the message pending (moderated user/group or flagged @@ -1080,9 +1088,9 @@ func GetSession(c *fiber.Ctx) error { } }() - // --- Checked: recently auto-approved posts from auto-moderated (NULL) members. - // Informational oversight count (blue), bounded to the last 24h so it stays - // useful rather than the unbounded all-time approved total. --- + // --- Checked: UNCHECKED auto-approved posts from auto-moderated (NULL) members. + // Outstanding oversight work (blue): a mod hasn't marked it checked and it + // is within the 7-day auto-check window. --- wg2.Add(1) go func() { defer wg2.Done() @@ -1093,12 +1101,12 @@ func GetSession(c *fiber.Ctx) error { "WHERE mg.groupid IN ? AND mg.collection = ? AND mg.deleted = 0 "+ "AND m.deleted IS NULL AND u.deleted IS NULL "+ "AND mg.approvedby IS NULL AND mem.ourPostingStatus IS NULL "+ - "AND mg.arrival >= NOW() - INTERVAL 1 DAY", + checkedWindowSQL, modGroupIDs, utils.COLLECTION_APPROVED).Scan(&checked) }() - // --- Trusted: recently-live posts from trusted (group-settings) members. - // Informational oversight count (blue), bounded to the last 24h. --- + // --- Trusted: UNCHECKED live posts from trusted (group-settings) members. + // Outstanding oversight work (blue), same 7-day auto-check window. --- wg2.Add(1) go func() { defer wg2.Done() @@ -1109,7 +1117,7 @@ func GetSession(c *fiber.Ctx) error { "WHERE mg.groupid IN ? AND mg.collection = ? AND mg.deleted = 0 "+ "AND m.deleted IS NULL AND u.deleted IS NULL "+ "AND mg.approvedby IS NULL AND mem.ourPostingStatus IN ('DEFAULT', 'UNMODERATED') "+ - "AND mg.arrival >= NOW() - INTERVAL 1 DAY", + checkedWindowSQL, modGroupIDs, utils.COLLECTION_APPROVED).Scan(&trusted) }() diff --git a/iznik-server-go/utils/utils.go b/iznik-server-go/utils/utils.go index a86abd08af..b34cd26314 100644 --- a/iznik-server-go/utils/utils.go +++ b/iznik-server-go/utils/utils.go @@ -100,6 +100,11 @@ const COLLECTION_REJECTED = "Rejected" const COLLECTION_BANNED = "Banned" const COLLECTION_DRAFT = "Draft" +// MESSAGE_CHECK_WINDOW_DAYS is how long an auto-published post stays in the +// ModTools Checked/Trusted oversight queues (and their work-count badges) before +// it is treated as auto-checked, so the queue cannot pile up indefinitely. +const MESSAGE_CHECK_WINDOW_DAYS = 7 + const POSTING_STATUS_MODERATED = "MODERATED" const POSTING_STATUS_PROHIBITED = "PROHIBITED" const POSTING_STATUS_DEFAULT = "DEFAULT" From b44a0e78ead11f9e288660ba8ac3e9355b4b4697 Mon Sep 17 00:00:00 2001 From: edwh Date: Thu, 4 Jun 2026 09:15:27 +0100 Subject: [PATCH 05/26] feat(modtools): mark-all-as-checked action + tests for the oversight queues - Checked/Trusted pages get a 'Mark all as checked' button that clears the bucket (POST /modtools/messages/markchecked) and refreshes the nav badge. - MessageAPI.markChecked + message store action. - Go tests: checked filter excludes already-checked posts; markchecked endpoint sets checkedat/checkedby and removes posts from the queue; non-mod gets 403. - Vitest: markAllChecked clears the bucket and refreshes work counts. --- iznik-nuxt3/api/MessageAPI.js | 6 ++ .../messages/checked/[[id]]/[[term]].vue | 39 ++++++++- .../messages/trusted/[[id]]/[[term]].vue | 39 ++++++++- iznik-nuxt3/stores/message.js | 7 ++ .../pages/modtools/messages/checked.spec.js | 18 ++++ .../pages/modtools/messages/trusted.spec.js | 18 ++++ iznik-server-go/test/message_test.go | 85 ++++++++++++++++++- 7 files changed, 202 insertions(+), 10 deletions(-) diff --git a/iznik-nuxt3/api/MessageAPI.js b/iznik-nuxt3/api/MessageAPI.js index 5d150533f6..f97197649a 100644 --- a/iznik-nuxt3/api/MessageAPI.js +++ b/iznik-nuxt3/api/MessageAPI.js @@ -70,6 +70,12 @@ export default class MessageAPI extends BaseAPI { return this.$getv2('/modtools/messages', params) } + // Mark auto-published posts (Checked/Trusted oversight queues) as reviewed by a + // moderator. Pass {groupid, filter} to clear a whole bucket, or {groupid, ids}. + markChecked(params) { + return this.$postv2('/modtools/messages/markchecked', params) + } + update(event) { return this.$postv2('/message', event) } diff --git a/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue b/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue index 22e0f303e0..e26932be4c 100644 --- a/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue +++ b/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue @@ -10,14 +10,26 @@ remember="checked" :url-override="urlOverride" /> - +
+ + + {{ marking ? 'Marking…' : 'Mark all as checked' }} + +
- No auto-checked messages have gone live yet. These are posts from - auto-moderated members that passed the automated checks. + Nothing to check. These are posts that went live automatically from + auto-moderated members — once you've checked them they drop off here, and + anything older than a week is treated as checked.
@@ -48,11 +60,13 @@ import { useMessageStore } from '@/stores/message' import { useMiscStore } from '@/stores/misc' import { useModGroupStore } from '@/stores/modgroup' import { useMe } from '~/composables/useMe' +import { useModMe } from '~/composables/useModMe' const messageStore = useMessageStore() const miscStore = useMiscStore() const modGroupStore = useModGroupStore() const route = useRoute() +const { checkWork } = useModMe() const FILTER = 'checked' const summaryKey = 'modtoolsMessagesCheckedSummary' @@ -86,6 +100,23 @@ const groupsreceived = computed(() => modGroupStore.received) const bump = ref(0) const urlOverride = ref(false) +const marking = ref(false) + +// Mark every post currently in this oversight queue as checked, then refresh. +async function markAllChecked() { + marking.value = true + try { + await messageStore.markChecked({ groupid: groupid.value, filter: FILTER }) + show.value = 0 + context.value = null + listingIds.value = new Set() + messageStore.clear() + bump.value++ + await checkWork(true) + } finally { + marking.value = false + } +} watch(groupid, async (newVal) => { const router = useRouter() @@ -165,6 +196,8 @@ defineExpose({ busy, messages, groupid, + marking, + markAllChecked, loadMore, }) diff --git a/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue b/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue index 8e01084e12..ac3b204e70 100644 --- a/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue +++ b/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue @@ -10,14 +10,26 @@ remember="trusted" :url-override="urlOverride" /> - +
+ + + {{ marking ? 'Marking…' : 'Mark all as checked' }} + +
- No posts from trusted members have gone live recently. These are posts - from members on Group Settings that publish without moderation. + Nothing to check. These are posts that went live without moderation from + trusted (Group Settings) members — once you've checked them they drop off + here, and anything older than a week is treated as checked.
@@ -46,11 +58,13 @@ import { useMessageStore } from '@/stores/message' import { useMiscStore } from '@/stores/misc' import { useModGroupStore } from '@/stores/modgroup' import { useMe } from '~/composables/useMe' +import { useModMe } from '~/composables/useModMe' const messageStore = useMessageStore() const miscStore = useMiscStore() const modGroupStore = useModGroupStore() const route = useRoute() +const { checkWork } = useModMe() const FILTER = 'trusted' const summaryKey = 'modtoolsMessagesTrustedSummary' @@ -84,6 +98,23 @@ const groupsreceived = computed(() => modGroupStore.received) const bump = ref(0) const urlOverride = ref(false) +const marking = ref(false) + +// Mark every post currently in this oversight queue as checked, then refresh. +async function markAllChecked() { + marking.value = true + try { + await messageStore.markChecked({ groupid: groupid.value, filter: FILTER }) + show.value = 0 + context.value = null + listingIds.value = new Set() + messageStore.clear() + bump.value++ + await checkWork(true) + } finally { + marking.value = false + } +} watch(groupid, async (newVal) => { const router = useRouter() @@ -163,6 +194,8 @@ defineExpose({ busy, messages, groupid, + marking, + markAllChecked, loadMore, }) diff --git a/iznik-nuxt3/stores/message.js b/iznik-nuxt3/stores/message.js index 09aeeb727a..4b40c5036f 100644 --- a/iznik-nuxt3/stores/message.js +++ b/iznik-nuxt3/stores/message.js @@ -656,6 +656,13 @@ export const useMessageStore = defineStore({ ) return fetched.filter((id) => id !== null) }, + // Mark auto-published posts as reviewed by a moderator, clearing them from + // the Checked/Trusted oversight queues. Returns the number marked. + async markChecked(params) { + const data = await api(this.config).message.markChecked(params) + return data?.checked ?? 0 + }, + async fetchMessagesMT(params) { if (params.context) { // Server expects context as a JSON-encoded string; URLSearchParams diff --git a/iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js b/iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js index 40ca473488..8aa082eab3 100644 --- a/iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js +++ b/iznik-nuxt3/tests/unit/pages/modtools/messages/checked.spec.js @@ -34,12 +34,18 @@ const mockMessageStore = { list: {}, context: null, fetchMessagesMT: vi.fn().mockResolvedValue([]), + markChecked: vi.fn().mockResolvedValue(2), clear: vi.fn(), } vi.mock('@/stores/message', () => ({ useMessageStore: () => mockMessageStore, })) +const mockCheckWork = vi.fn().mockResolvedValue(undefined) +vi.mock('~/composables/useModMe', () => ({ + useModMe: () => ({ checkWork: mockCheckWork }), +})) + const mockMiscStore = { get: vi.fn(), set: vi.fn() } vi.mock('@/stores/misc', () => ({ useMiscStore: () => mockMiscStore, @@ -155,4 +161,16 @@ describe('messages/checked/[[id]]/[[term]].vue page', () => { expect(wrapper.vm.id).toBe(42) expect(mockGroupid.value).toBe(42) }) + + it('markAllChecked clears the checked bucket and refreshes work counts', async () => { + mockGroupid.value = 7 + const wrapper = mountComponent() + await wrapper.vm.markAllChecked() + expect(mockMessageStore.markChecked).toHaveBeenCalledWith({ + groupid: 7, + filter: 'checked', + }) + expect(mockMessageStore.clear).toHaveBeenCalled() + expect(mockCheckWork).toHaveBeenCalled() + }) }) diff --git a/iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js b/iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js index 952a2aef2d..6e7c5d60f5 100644 --- a/iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js +++ b/iznik-nuxt3/tests/unit/pages/modtools/messages/trusted.spec.js @@ -34,12 +34,18 @@ const mockMessageStore = { list: {}, context: null, fetchMessagesMT: vi.fn().mockResolvedValue([]), + markChecked: vi.fn().mockResolvedValue(2), clear: vi.fn(), } vi.mock('@/stores/message', () => ({ useMessageStore: () => mockMessageStore, })) +const mockCheckWork = vi.fn().mockResolvedValue(undefined) +vi.mock('~/composables/useModMe', () => ({ + useModMe: () => ({ checkWork: mockCheckWork }), +})) + const mockMiscStore = { get: vi.fn(), set: vi.fn() } vi.mock('@/stores/misc', () => ({ useMiscStore: () => mockMiscStore, @@ -155,4 +161,16 @@ describe('messages/trusted/[[id]]/[[term]].vue page', () => { expect(wrapper.vm.id).toBe(42) expect(mockGroupid.value).toBe(42) }) + + it('markAllChecked clears the trusted bucket and refreshes work counts', async () => { + mockGroupid.value = 7 + const wrapper = mountComponent() + await wrapper.vm.markAllChecked() + expect(mockMessageStore.markChecked).toHaveBeenCalledWith({ + groupid: 7, + filter: 'trusted', + }) + expect(mockMessageStore.clear).toHaveBeenCalled() + expect(mockCheckWork).toHaveBeenCalled() + }) }) diff --git a/iznik-server-go/test/message_test.go b/iznik-server-go/test/message_test.go index 51d74d02c4..7a90f9e0fc 100644 --- a/iznik-server-go/test/message_test.go +++ b/iznik-server-go/test/message_test.go @@ -5573,8 +5573,11 @@ func TestListMessagesMT_FilterChecked(t *testing.T) { checkedMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" auto checked", 52.0, -1.0) trustedMsg := CreateTestMessage(t, defaultPoster, groupID, prefix+" trusted live", 52.0, -1.0) modApprovedMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" mod approved", 52.0, -1.0) - db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid IN (?, ?)", checkedMsg, trustedMsg) + // A post a mod has already marked checked must drop off the queue. + alreadyCheckedMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" already checked", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid IN (?, ?, ?)", checkedMsg, trustedMsg, alreadyCheckedMsg) db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=? WHERE msgid=?", modID, modApprovedMsg) + db.Exec("UPDATE messages_groups SET checkedat=NOW(), checkedby=? WHERE msgid=?", modID, alreadyCheckedMsg) resp, err := getApp().Test(httptest.NewRequest("GET", fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=checked&jwt=%s", groupID, modToken), nil)) @@ -5584,7 +5587,7 @@ func TestListMessagesMT_FilterChecked(t *testing.T) { var body map[string]interface{} json.NewDecoder(resp.Body).Decode(&body) msgs, _ := body["messages"].([]interface{}) - foundChecked, foundTrusted, foundMod := false, false, false + foundChecked, foundTrusted, foundMod, foundAlready := false, false, false, false for _, id := range msgs { switch id { case float64(checkedMsg): @@ -5593,14 +5596,17 @@ func TestListMessagesMT_FilterChecked(t *testing.T) { foundTrusted = true case float64(modApprovedMsg): foundMod = true + case float64(alreadyCheckedMsg): + foundAlready = true } } assert.True(t, foundChecked, "auto-approved post from NULL member should appear under checked") assert.False(t, foundTrusted, "trusted-member post should not appear under checked") assert.False(t, foundMod, "mod-approved post should not appear under checked") + assert.False(t, foundAlready, "a post already marked checked should not appear under checked") - db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg) - db.Exec("DELETE FROM messages WHERE id IN (?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg) + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg, alreadyCheckedMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg, alreadyCheckedMsg) } func TestListMessagesMT_FilterTrusted(t *testing.T) { @@ -5654,6 +5660,77 @@ func TestListMessagesMT_FilterTrusted(t *testing.T) { db.Exec("DELETE FROM messages WHERE id IN (?, ?, ?)", checkedMsg, trustedMsg, modApprovedMsg) } +func TestListMessagesMT_MarkChecked(t *testing.T) { + // Marking the checked bucket sets checkedat/checkedby and removes the posts + // from the checked oversight queue. + prefix := uniquePrefix("lstmt_markchk") + db := database.DBConn + + groupID := CreateTestGroup(t, prefix) + nullPoster := CreateTestUser(t, prefix+"_null", "User") + modID := CreateTestUser(t, prefix+"_mod", "User") + CreateTestMembership(t, nullPoster, groupID, "Member") + CreateTestMembership(t, modID, groupID, "Moderator") + db.Exec("UPDATE memberships SET ourPostingStatus = NULL WHERE userid = ? AND groupid = ?", nullPoster, groupID) + _, modToken := CreateTestSession(t, modID) + + msg1 := CreateTestMessage(t, nullPoster, groupID, prefix+" auto checked 1", 52.0, -1.0) + msg2 := CreateTestMessage(t, nullPoster, groupID, prefix+" auto checked 2", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid IN (?, ?)", msg1, msg2) + + listURL := fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Approved&filter=checked&jwt=%s", groupID, modToken) + resp, _ := getApp().Test(httptest.NewRequest("GET", listURL, nil)) + var before map[string]interface{} + json.NewDecoder(resp.Body).Decode(&before) + beforeMsgs, _ := before["messages"].([]interface{}) + assert.GreaterOrEqual(t, len(beforeMsgs), 2, "both posts should be in the checked queue before marking") + + // Mark the whole checked bucket for this group checked. + markBody := fmt.Sprintf(`{"groupid": %d, "filter": "checked"}`, groupID) + markReq := httptest.NewRequest("POST", + fmt.Sprintf("/api/modtools/messages/markchecked?jwt=%s", modToken), strings.NewReader(markBody)) + markReq.Header.Set("Content-Type", "application/json") + markResp, err := getApp().Test(markReq) + assert.NoError(t, err) + assert.Equal(t, 200, markResp.StatusCode) + + // checkedat/checkedby are now set. + var checkedCount int64 + db.Raw("SELECT COUNT(*) FROM messages_groups WHERE msgid IN (?, ?) AND checkedat IS NOT NULL AND checkedby = ?", + msg1, msg2, modID).Scan(&checkedCount) + assert.Equal(t, int64(2), checkedCount, "both posts should be marked checked") + + // They no longer appear under the checked queue. + resp2, _ := getApp().Test(httptest.NewRequest("GET", listURL, nil)) + var after map[string]interface{} + json.NewDecoder(resp2.Body).Decode(&after) + afterMsgs, _ := after["messages"].([]interface{}) + for _, id := range afterMsgs { + assert.NotEqual(t, float64(msg1), id, "checked post should leave the queue") + assert.NotEqual(t, float64(msg2), id, "checked post should leave the queue") + } + + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", msg1, msg2) + db.Exec("DELETE FROM messages WHERE id IN (?, ?)", msg1, msg2) +} + +func TestMarkCheckedUnauthorized(t *testing.T) { + // Non-mods can't mark posts checked. + prefix := uniquePrefix("markchk_unauth") + groupID := CreateTestGroup(t, prefix) + regularID := CreateTestUser(t, prefix+"_regular", "User") + CreateTestMembership(t, regularID, groupID, "Member") + _, regularToken := CreateTestSession(t, regularID) + + markBody := fmt.Sprintf(`{"groupid": %d, "filter": "checked"}`, groupID) + markReq := httptest.NewRequest("POST", + fmt.Sprintf("/api/modtools/messages/markchecked?jwt=%s", regularToken), strings.NewReader(markBody)) + markReq.Header.Set("Content-Type", "application/json") + resp, err := getApp().Test(markReq) + assert.NoError(t, err) + assert.Equal(t, 403, resp.StatusCode) +} + func TestListMessagesPendingUnauthorized(t *testing.T) { prefix := uniquePrefix("lstmsg_pend_unauth") From 797e0a39baa1bddf843162076ebc500c72d369a8 Mon Sep 17 00:00:00 2001 From: edwh Date: Thu, 4 Jun 2026 09:34:39 +0100 Subject: [PATCH 06/26] feat(modtools): SysAdmin moderation analytics + quality-sample tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a 'Moderation' tab to the SysAdmin section (Admin/Support) to judge whether auto-approving is working over a selectable period: - Where posts went: arrived, manually approved/rejected, auto-approved (Checked delay path vs 48h fallback), trusted (immediate). - Did auto-approving go wrong: of auto-approved posts, how many a mod later marked checked, and how many later needed intervention (rejected/deleted/edited/held) — the auto-publish error rate. - Quality-check sample: posts held back for manual review, and how many a mod then rejected — cross-checked against the auto-publish error rate to give a verdict. - Migration: messages_groups.quality_sample marks posts the auto-approve service held back (AutoApproveCleanService sets it), so the sample is measurable. - Go GET /modtools/moderationstats (Admin/Support) computes the metrics from the logs table + messages_groups, parallel per-metric queries. - MessageAPI.moderationStats + message store action; ModSysAdminModerationStats.vue. - Tests: Go endpoint (admin/forbidden/dates/counts), Laravel quality_sample marker, Vitest component (fetch on mount, percentages, re-fetch). --- .../app/Services/AutoApproveCleanService.php | 10 + ..._add_quality_sample_to_messages_groups.php | 33 +++ .../Services/AutoApproveCleanServiceTest.php | 6 + iznik-nuxt3/api/MessageAPI.js | 5 + .../components/ModSysAdminModerationStats.vue | 206 ++++++++++++++++++ iznik-nuxt3/modtools/pages/sysadmin/index.vue | 20 ++ iznik-nuxt3/stores/message.js | 5 + .../ModSysAdminModerationStats.spec.js | 83 +++++++ iznik-server-go/moderation/moderationstats.go | 143 ++++++++++++ iznik-server-go/router/routes.go | 4 + iznik-server-go/test/moderationstats_test.go | 79 +++++++ 11 files changed, 594 insertions(+) create mode 100644 iznik-batch/database/migrations/2026_06_04_000002_add_quality_sample_to_messages_groups.php create mode 100644 iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue create mode 100644 iznik-nuxt3/tests/unit/components/modtools/ModSysAdminModerationStats.spec.js create mode 100644 iznik-server-go/moderation/moderationstats.go create mode 100644 iznik-server-go/test/moderationstats_test.go diff --git a/iznik-batch/app/Services/AutoApproveCleanService.php b/iznik-batch/app/Services/AutoApproveCleanService.php index 3e3c4e600f..d7dc374bc6 100644 --- a/iznik-batch/app/Services/AutoApproveCleanService.php +++ b/iznik-batch/app/Services/AutoApproveCleanService.php @@ -103,6 +103,16 @@ public function process(bool $dryRun = false): array } if ($this->isQualitySampled((int) $row->msgid, (int) $row->groupid)) { + if (!$dryRun) { + // Mark it as a quality-check sample so the moderation-stats + // dashboard can compare the mod's verdict on the sample + // against the auto-approved population's later error rate. + DB::table('messages_groups') + ->where('msgid', $row->msgid) + ->where('groupid', $row->groupid) + ->where('quality_sample', 0) + ->update(['quality_sample' => 1]); + } $stats['held_quality']++; continue; } diff --git a/iznik-batch/database/migrations/2026_06_04_000002_add_quality_sample_to_messages_groups.php b/iznik-batch/database/migrations/2026_06_04_000002_add_quality_sample_to_messages_groups.php new file mode 100644 index 0000000000..b20c1fbbde --- /dev/null +++ b/iznik-batch/database/migrations/2026_06_04_000002_add_quality_sample_to_messages_groups.php @@ -0,0 +1,33 @@ +boolean('quality_sample')->default(false)->after('checkedby'); + $table->index(['groupid', 'quality_sample'], 'messages_groups_groupid_qsample_idx'); + }); + } + + public function down(): void + { + Schema::table('messages_groups', function (Blueprint $table) { + $table->dropIndex('messages_groups_groupid_qsample_idx'); + $table->dropColumn('quality_sample'); + }); + } +}; diff --git a/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php b/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php index 1d3533f365..6c62b4906e 100644 --- a/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php +++ b/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php @@ -338,6 +338,12 @@ public function test_quality_sample_100_holds_all(): void $this->assertGreaterThanOrEqual(1, $stats['held_quality']); $this->assertStillPending($message->id, $group->id); + // Held posts are marked as a quality-check sample for the stats dashboard. + $this->assertDatabaseHas('messages_groups', [ + 'msgid' => $message->id, + 'groupid' => $group->id, + 'quality_sample' => 1, + ]); } public function test_quality_sample_zero_holds_none(): void diff --git a/iznik-nuxt3/api/MessageAPI.js b/iznik-nuxt3/api/MessageAPI.js index f97197649a..121c229659 100644 --- a/iznik-nuxt3/api/MessageAPI.js +++ b/iznik-nuxt3/api/MessageAPI.js @@ -76,6 +76,11 @@ export default class MessageAPI extends BaseAPI { return this.$postv2('/modtools/messages/markchecked', params) } + // SysAdmin moderation analytics for a date range ({start, end}). + moderationStats(params) { + return this.$getv2('/modtools/moderationstats', params) + } + update(event) { return this.$postv2('/message', event) } diff --git a/iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue b/iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue new file mode 100644 index 0000000000..017e6bf29b --- /dev/null +++ b/iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue @@ -0,0 +1,206 @@ + + + diff --git a/iznik-nuxt3/modtools/pages/sysadmin/index.vue b/iznik-nuxt3/modtools/pages/sysadmin/index.vue index 73bf37dd14..1804ac0f6f 100644 --- a/iznik-nuxt3/modtools/pages/sysadmin/index.vue +++ b/iznik-nuxt3/modtools/pages/sysadmin/index.vue @@ -109,6 +109,17 @@ :key="'recommendations-' + recommendationsBump" /> + + + + + +
@@ -142,6 +153,8 @@ const showRippling = ref(false) const ripplingBump = ref(0) const showRecommendations = ref(false) const recommendationsBump = ref(0) +const showModeration = ref(false) +const moderationBump = ref(0) const topTabMap = { housekeeping: 0, @@ -151,6 +164,7 @@ const topTabMap = { incoming: 4, rippling: 5, recommendations: 6, + moderation: 7, } function onHousekeepingTab() { @@ -188,6 +202,11 @@ function onRecommendationsTab() { recommendationsBump.value = Date.now() } +function onModerationTab() { + showModeration.value = true + moderationBump.value = Date.now() +} + onMounted(() => { const tab = route.query.tab if (tab && topTabMap[tab] !== undefined) { @@ -200,6 +219,7 @@ onMounted(() => { else if (tab === 'incoming') onIncomingEmailTab() else if (tab === 'rippling') onRipplingTab() else if (tab === 'recommendations') onRecommendationsTab() + else if (tab === 'moderation') onModerationTab() } else { // Default to showing housekeeping onHousekeepingTab() diff --git a/iznik-nuxt3/stores/message.js b/iznik-nuxt3/stores/message.js index 4b40c5036f..4ffd95c168 100644 --- a/iznik-nuxt3/stores/message.js +++ b/iznik-nuxt3/stores/message.js @@ -663,6 +663,11 @@ export const useMessageStore = defineStore({ return data?.checked ?? 0 }, + // Fetch SysAdmin moderation analytics for a date range ({start, end}). + async fetchModerationStats(params) { + return await api(this.config).message.moderationStats(params) + }, + async fetchMessagesMT(params) { if (params.context) { // Server expects context as a JSON-encoded string; URLSearchParams diff --git a/iznik-nuxt3/tests/unit/components/modtools/ModSysAdminModerationStats.spec.js b/iznik-nuxt3/tests/unit/components/modtools/ModSysAdminModerationStats.spec.js new file mode 100644 index 0000000000..0ce85e2ac5 --- /dev/null +++ b/iznik-nuxt3/tests/unit/components/modtools/ModSysAdminModerationStats.spec.js @@ -0,0 +1,83 @@ +import { describe, it, expect, vi, beforeEach } from 'vitest' +import { shallowMount, flushPromises } from '@vue/test-utils' +import { createPinia, setActivePinia } from 'pinia' +import ModSysAdminModerationStats from '~/modtools/components/ModSysAdminModerationStats.vue' + +const sampleStats = { + arrived: 100, + manualApproved: 20, + manualRejected: 5, + autoChecked: 60, + autoFallback: 5, + trusted: 10, + autoApproved: 65, + autoModChecked: 30, + autoLaterActioned: 2, + qualitySampled: 40, + qualitySampleBad: 1, +} + +const mockMessageStore = { + fetchModerationStats: vi.fn().mockResolvedValue(sampleStats), +} +vi.mock('@/stores/message', () => ({ + useMessageStore: () => mockMessageStore, +})) + +describe('ModSysAdminModerationStats.vue', () => { + function mountComponent() { + return shallowMount(ModSysAdminModerationStats, { + global: { + plugins: [createPinia()], + stubs: { + 'b-form-group': true, + 'b-form-input': true, + 'b-button': true, + 'b-card': true, + 'b-table-simple': true, + 'b-tbody': true, + 'b-tr': true, + 'b-td': true, + NoticeMessage: true, + }, + }, + }) + } + + beforeEach(() => { + vi.clearAllMocks() + setActivePinia(createPinia()) + mockMessageStore.fetchModerationStats.mockResolvedValue(sampleStats) + }) + + it('fetches stats for the default 30-day range on mount', async () => { + const wrapper = mountComponent() + await flushPromises() + expect(mockMessageStore.fetchModerationStats).toHaveBeenCalledTimes(1) + const params = mockMessageStore.fetchModerationStats.mock.calls[0][0] + expect(params.start).toMatch(/^\d{4}-\d{2}-\d{2}$/) + expect(params.end).toMatch(/^\d{4}-\d{2}-\d{2}$/) + expect(new Date(params.start) <= new Date(params.end)).toBe(true) + expect(wrapper.vm.stats).toEqual(sampleStats) + }) + + it('formats percentages and guards divide-by-zero', () => { + const wrapper = mountComponent() + expect(wrapper.vm.pct(1, 2)).toBe('50.0%') + expect(wrapper.vm.pct(2, 65)).toBe('3.1%') + expect(wrapper.vm.pct(5, 0)).toBe('—') + }) + + it('re-fetches when Update is triggered', async () => { + const wrapper = mountComponent() + await flushPromises() + mockMessageStore.fetchModerationStats.mockClear() + wrapper.vm.startDate = '2026-01-01' + wrapper.vm.endDate = '2026-01-31' + await wrapper.vm.fetchStats() + expect(mockMessageStore.fetchModerationStats).toHaveBeenCalledWith({ + start: '2026-01-01', + end: '2026-01-31', + }) + }) +}) diff --git a/iznik-server-go/moderation/moderationstats.go b/iznik-server-go/moderation/moderationstats.go new file mode 100644 index 0000000000..8d76301a63 --- /dev/null +++ b/iznik-server-go/moderation/moderationstats.go @@ -0,0 +1,143 @@ +package moderation + +import ( + "strings" + "sync" + + "github.com/freegle/iznik-server-go/database" + "github.com/freegle/iznik-server-go/user" + "github.com/gofiber/fiber/v2" +) + +// ModerationStats summarises how posts were moderated over a period, so admins +// can judge whether auto-approving is working: the volume down each path, the +// rate at which auto-published posts later needed intervention, and the +// moderator's verdict on the quality-check sample for comparison. +type ModerationStats struct { + Start string `json:"start"` + End string `json:"end"` + + // How posts were handled in the period. + Arrived int64 `json:"arrived"` // posts that arrived (Received log) + ManualApproved int64 `json:"manualApproved"` // approved by a moderator + ManualRejected int64 `json:"manualRejected"` // rejected/deleted by a moderator + AutoApproved int64 `json:"autoApproved"` // auto-approved (Checked) — Autoapproved log + Trusted int64 `json:"trusted"` // went live immediately (trusted members) + + // Quality of the auto-published population (posts auto-approved in the period). + // NB: 'auto-approved' (Autoapproved log) covers both the Checked delay path and + // the 48h fallback — they aren't distinguished retrospectively. Trusted posts + // (no log) are excluded from this error-rate denominator. + AutoModChecked int64 `json:"autoModChecked"` // a moderator later marked it checked + AutoLaterActioned int64 `json:"autoLaterActioned"` // later rejected/deleted/edited/held after going live + + // Quality-check sample held back for manual review. + QualitySampled int64 `json:"qualitySampled"` // posts held back for a manual quality check + QualitySampleBad int64 `json:"qualitySampleBad"` // of those, ones a moderator then rejected/deleted +} + +// Stats handles GET /modtools/moderationstats?start=&end= (Admin/Support only). +// +// @Summary Moderation analytics for the auto-approve approach +// @Tags moderation +// @Produce json +// @Param start query string true "Start date (YYYY-MM-DD)" +// @Param end query string true "End date (YYYY-MM-DD)" +// @Success 200 {object} ModerationStats +// @Router /modtools/moderationstats [get] +func Stats(c *fiber.Ctx) error { + myid := user.WhoAmI(c) + if myid == 0 { + return fiber.NewError(fiber.StatusUnauthorized, "Not logged in") + } + if !user.IsAdminOrSupport(myid) { + return fiber.NewError(fiber.StatusForbidden, "Support or Admin role required") + } + + start := c.Query("start", "") + end := c.Query("end", "") + if start == "" || end == "" { + return fiber.NewError(fiber.StatusBadRequest, "start and end dates are required") + } + // Make a bare end date inclusive of the whole day. + if !strings.Contains(end, " ") && !strings.Contains(end, "T") { + end += " 23:59:59" + } + + db := database.DBConn + stats := ModerationStats{Start: c.Query("start", ""), End: c.Query("end", "")} + + var wg sync.WaitGroup + run := func(fn func()) { + wg.Add(1) + go func() { + defer wg.Done() + fn() + }() + } + + // --- Flow in the period --- + run(func() { + db.Raw("SELECT COUNT(*) FROM logs l WHERE l.type='Message' AND l.subtype='Received' "+ + "AND l.timestamp BETWEEN ? AND ?", start, end).Scan(&stats.Arrived) + }) + run(func() { + db.Raw("SELECT COUNT(*) FROM logs l WHERE l.type='Message' AND l.subtype='Approved' "+ + "AND l.byuser IS NOT NULL AND l.timestamp BETWEEN ? AND ?", start, end).Scan(&stats.ManualApproved) + }) + run(func() { + db.Raw("SELECT COUNT(*) FROM logs l WHERE l.type='Message' AND l.subtype IN ('Rejected','Deleted') "+ + "AND l.byuser IS NOT NULL AND l.byuser <> l.user AND l.timestamp BETWEEN ? AND ?", start, end).Scan(&stats.ManualRejected) + }) + run(func() { + db.Raw("SELECT COUNT(*) FROM logs l WHERE l.type='Message' AND l.subtype='Autoapproved' "+ + "AND l.timestamp BETWEEN ? AND ?", start, end).Scan(&stats.AutoApproved) + }) + run(func() { + // Trusted: went live without moderation (approvedby NULL, content-checked, + // trusted member) and with no Autoapproved log. + db.Raw("SELECT COUNT(*) FROM messages_groups mg "+ + "JOIN messages m ON m.id=mg.msgid "+ + "JOIN memberships mem ON mem.userid=m.fromuser AND mem.groupid=mg.groupid "+ + "WHERE mg.approvedat BETWEEN ? AND ? AND mg.approvedby IS NULL "+ + "AND mg.contentcheck_checked_at IS NOT NULL AND mg.deleted=0 "+ + "AND (mem.ourPostingStatus='DEFAULT' OR mem.ourPostingStatus='UNMODERATED') "+ + "AND NOT EXISTS (SELECT 1 FROM logs l WHERE l.msgid=mg.msgid AND l.type='Message' AND l.subtype='Autoapproved')", + start, end).Scan(&stats.Trusted) + }) + + // --- Quality of the auto-published population (AutoApproved counted above) --- + run(func() { + db.Raw("SELECT COUNT(DISTINCT l.msgid) FROM logs l "+ + "JOIN messages_groups mg ON mg.msgid=l.msgid AND mg.groupid=l.groupid "+ + "WHERE l.type='Message' AND l.subtype='Autoapproved' AND l.timestamp BETWEEN ? AND ? "+ + "AND mg.checkedat IS NOT NULL", start, end).Scan(&stats.AutoModChecked) + }) + run(func() { + // Auto-approved posts that later needed intervention: a rejection, deletion, + // edit or hold by someone other than the author, after the auto-approval. + db.Raw("SELECT COUNT(DISTINCT a.msgid) FROM logs a "+ + "JOIN logs x ON x.msgid=a.msgid AND x.groupid=a.groupid "+ + "WHERE a.type='Message' AND a.subtype='Autoapproved' AND a.timestamp BETWEEN ? AND ? "+ + "AND x.type='Message' AND x.subtype IN ('Rejected','Deleted','Edit','Hold') "+ + "AND x.byuser IS NOT NULL AND x.byuser <> a.user AND x.timestamp > a.timestamp", + start, end).Scan(&stats.AutoLaterActioned) + }) + + // --- Quality-check sample --- + run(func() { + db.Raw("SELECT COUNT(*) FROM messages_groups mg WHERE mg.quality_sample=1 "+ + "AND mg.arrival BETWEEN ? AND ?", start, end).Scan(&stats.QualitySampled) + }) + run(func() { + db.Raw("SELECT COUNT(DISTINCT mg.msgid) FROM messages_groups mg "+ + "JOIN logs l ON l.msgid=mg.msgid AND l.groupid=mg.groupid "+ + "WHERE mg.quality_sample=1 AND mg.arrival BETWEEN ? AND ? "+ + "AND l.type='Message' AND l.subtype IN ('Rejected','Deleted') "+ + "AND l.byuser IS NOT NULL AND l.byuser <> l.user", start, end).Scan(&stats.QualitySampleBad) + }) + + wg.Wait() + + return c.JSON(stats) +} diff --git a/iznik-server-go/router/routes.go b/iznik-server-go/router/routes.go index 6cfd423999..1fae7fec78 100644 --- a/iznik-server-go/router/routes.go +++ b/iznik-server-go/router/routes.go @@ -47,6 +47,7 @@ import ( "github.com/freegle/iznik-server-go/group" "github.com/freegle/iznik-server-go/housekeeper" "github.com/freegle/iznik-server-go/image" + "github.com/freegle/iznik-server-go/moderation" "github.com/freegle/iznik-server-go/isochrone" "github.com/freegle/iznik-server-go/job" "github.com/freegle/iznik-server-go/location" @@ -1370,6 +1371,9 @@ func SetupRoutes(app *fiber.App) { // @Failure 403 {object} fiber.Error "Forbidden" rg.Get("/modtools/email/stats", emailtracking.Stats) + // Moderation analytics for the auto-approve approach (Admin/Support only). + rg.Get("/modtools/moderationstats", moderation.Stats) + // Email Statistics Time Series (authenticated, admin only) // @Router /email/stats/timeseries [get] // @Summary Get daily email statistics for charting diff --git a/iznik-server-go/test/moderationstats_test.go b/iznik-server-go/test/moderationstats_test.go new file mode 100644 index 0000000000..292a349b24 --- /dev/null +++ b/iznik-server-go/test/moderationstats_test.go @@ -0,0 +1,79 @@ +package test + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "testing" + "time" + + "github.com/freegle/iznik-server-go/database" + "github.com/stretchr/testify/assert" +) + +func TestModerationStats_NonAdminForbidden(t *testing.T) { + prefix := uniquePrefix("modstats_unauth") + userID := CreateTestUser(t, prefix, "User") + _, token := CreateTestSession(t, userID) + + today := time.Now().Format("2006-01-02") + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/modtools/moderationstats?start=%s&end=%s&jwt=%s", today, today, token), nil)) + assert.NoError(t, err) + assert.Equal(t, 403, resp.StatusCode) +} + +func TestModerationStats_RequiresDates(t *testing.T) { + prefix := uniquePrefix("modstats_nodate") + adminID := CreateTestUser(t, prefix, "Admin") + _, token := CreateTestSession(t, adminID) + + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/modtools/moderationstats?jwt=%s", token), nil)) + assert.NoError(t, err) + assert.Equal(t, 400, resp.StatusCode) +} + +func TestModerationStats_CountsAutoApprovedAndSample(t *testing.T) { + prefix := uniquePrefix("modstats") + db := database.DBConn + + groupID := CreateTestGroup(t, prefix) + nullPoster := CreateTestUser(t, prefix+"_null", "User") + adminID := CreateTestUser(t, prefix+"_admin", "Admin") + CreateTestMembership(t, nullPoster, groupID, "Member") + db.Exec("UPDATE memberships SET ourPostingStatus = NULL WHERE userid = ? AND groupid = ?", nullPoster, groupID) + _, adminToken := CreateTestSession(t, adminID) + + // An auto-approved (Checked path) post: Autoapproved log + content-checked. + autoMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" auto checked", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL, approvedat=NOW(), contentcheck_checked_at=NOW() WHERE msgid=?", autoMsg) + db.Exec("INSERT INTO logs (timestamp, type, subtype, msgid, groupid, user) VALUES (NOW(), 'Message', 'Autoapproved', ?, ?, ?)", + autoMsg, groupID, nullPoster) + + // A quality-check sample post (held back for manual review). + sampleMsg := CreateTestMessage(t, nullPoster, groupID, prefix+" quality sample", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Pending', quality_sample=1, arrival=NOW() WHERE msgid=?", sampleMsg) + + today := time.Now().Format("2006-01-02") + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/modtools/moderationstats?start=%s&end=%s&jwt=%s", today, today, adminToken), nil)) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + var stats map[string]interface{} + json.NewDecoder(resp.Body).Decode(&stats) + + asFloat := func(k string) float64 { + v, _ := stats[k].(float64) + return v + } + // The auto-approved post is counted in the auto-approved (Checked) total. + assert.GreaterOrEqual(t, asFloat("autoApproved"), float64(1)) + // The quality-check sample is counted. + assert.GreaterOrEqual(t, asFloat("qualitySampled"), float64(1)) + + db.Exec("DELETE FROM logs WHERE msgid IN (?, ?)", autoMsg, sampleMsg) + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", autoMsg, sampleMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?)", autoMsg, sampleMsg) +} From 859515ab2511a5d373a8970deff7fd8d6235f444 Mon Sep 17 00:00:00 2001 From: edwh Date: Mon, 15 Jun 2026 18:57:44 +0100 Subject: [PATCH 07/26] feat(batch): add messages_groups.autoapprove_hold_until column + index (A1) Nullable timestamp with a groupid composite index. Both auto-approve services respect a hold predicate (next commit); NULL = no hold (behaviour unchanged for existing rows). Co-Authored-By: Claude Opus 4.8 --- ...oapprove_hold_until_to_messages_groups.php | 38 +++++++++++++++++++ .../AutoApproveHoldUntilMigrationTest.php | 26 +++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 iznik-batch/database/migrations/2026_06_15_000001_add_autoapprove_hold_until_to_messages_groups.php create mode 100644 iznik-batch/tests/Unit/Services/AutoApproveHoldUntilMigrationTest.php diff --git a/iznik-batch/database/migrations/2026_06_15_000001_add_autoapprove_hold_until_to_messages_groups.php b/iznik-batch/database/migrations/2026_06_15_000001_add_autoapprove_hold_until_to_messages_groups.php new file mode 100644 index 0000000000..89d0360b41 --- /dev/null +++ b/iznik-batch/database/migrations/2026_06_15_000001_add_autoapprove_hold_until_to_messages_groups.php @@ -0,0 +1,38 @@ +timestamp('autoapprove_hold_until')->nullable()->after('quality_sample'); + // Keeps the candidate-query predicate fast: both auto-approvers gain + // AND (mg.autoapprove_hold_until IS NULL OR mg.autoapprove_hold_until <= NOW()) + $table->index(['groupid', 'autoapprove_hold_until'], 'messages_groups_groupid_hold_until_idx'); + }); + } + + public function down(): void + { + Schema::table('messages_groups', function (Blueprint $table) { + $table->dropIndex('messages_groups_groupid_hold_until_idx'); + $table->dropColumn('autoapprove_hold_until'); + }); + } +}; diff --git a/iznik-batch/tests/Unit/Services/AutoApproveHoldUntilMigrationTest.php b/iznik-batch/tests/Unit/Services/AutoApproveHoldUntilMigrationTest.php new file mode 100644 index 0000000000..14b5a4ade0 --- /dev/null +++ b/iznik-batch/tests/Unit/Services/AutoApproveHoldUntilMigrationTest.php @@ -0,0 +1,26 @@ +assertTrue( + Schema::hasColumn('messages_groups', 'autoapprove_hold_until'), + 'messages_groups.autoapprove_hold_until column must exist after migration' + ); + } + + public function test_autoapprove_hold_until_index_exists(): void + { + $indexes = collect( + DB::select("SHOW INDEX FROM messages_groups WHERE Key_name = 'messages_groups_groupid_hold_until_idx'") + ); + $this->assertNotEmpty($indexes, 'messages_groups_groupid_hold_until_idx index must exist'); + } +} From 3dc96341dbf97a0c48f552ba93f37c25da161e60 Mon Sep 17 00:00:00 2001 From: edwh Date: Mon, 15 Jun 2026 18:57:44 +0100 Subject: [PATCH 08/26] fix(batch): hold predicate, cross-group Spam guard, quality-sample filter (A2,D1,D3,D6) - A2: both AutoApproveCleanService and AutoApproveService skip rows whose autoapprove_hold_until is set and in the future. - D1: add the cross-group Spam-collection whereNotExists guard to the clean service (Discourse #9654 parity) so a message in Spam on another group is never auto-approved. Test included. - D3: exclude already-sampled rows from the candidate query and only count held_quality outside dry-run, fixing per-minute rescans and stat inflation. - D6: assert an approved OFFER queues TASK_FREEBIE_ALERTS_ADD (WANTED does not). Co-Authored-By: Claude Opus 4.8 --- .../app/Services/AutoApproveCleanService.php | 17 ++- .../app/Services/AutoApproveService.php | 6 + .../Services/AutoApproveCleanServiceTest.php | 133 ++++++++++++++++++ .../Services/AutoApproveServiceHoldTest.php | 70 +++++++++ 4 files changed, 225 insertions(+), 1 deletion(-) create mode 100644 iznik-batch/tests/Unit/Services/AutoApproveServiceHoldTest.php diff --git a/iznik-batch/app/Services/AutoApproveCleanService.php b/iznik-batch/app/Services/AutoApproveCleanService.php index d7dc374bc6..1e340f3326 100644 --- a/iznik-batch/app/Services/AutoApproveCleanService.php +++ b/iznik-batch/app/Services/AutoApproveCleanService.php @@ -76,16 +76,31 @@ public function process(bool $dryRun = false): array ->whereNull('m.heldby') ->whereNull('mg.spamreason') ->whereNull('m.spamreason') + // Never auto-approve a message that is in the Spam collection on ANY + // group (Discourse #9654). Spam-collection messages surface in the + // Pending review queue but must be actioned by a human — mirroring + // the identical guard in AutoApproveService. + ->whereNotExists(function ($q) { + $q->select(DB::raw(1)) + ->from('messages_groups as spam_mg') + ->whereColumn('spam_mg.msgid', 'mg.msgid') + ->where('spam_mg.collection', MessageGroup::COLLECTION_SPAM) + ->where('spam_mg.deleted', 0); + }) ->where('mg.deleted', 0) ->whereNull('m.deleted') ->whereNull('u.deleted') ->whereNull('mem.ourPostingStatus') // the auto-moderated tier ->whereNotNull('mg.contentcheck_checked_at') // content check has run ... ->whereNull('mg.contentcheck_reasons') // ... and the post was clean + ->where('mg.quality_sample', 0) // already-sampled rows are excluded entirely ->whereRaw( "mg.arrival <= (NOW() - INTERVAL COALESCE(NULLIF(CAST(JSON_UNQUOTE(JSON_EXTRACT(g.settings, '$.autoapprove.delay_minutes')) AS UNSIGNED), 0), ?) MINUTE)", [$this->defaultDelayMinutes()] ) + ->whereRaw( + '(mg.autoapprove_hold_until IS NULL OR mg.autoapprove_hold_until <= NOW())' + ) ->orderBy('mg.msgid') ->orderBy('mg.groupid') ->get(); @@ -112,8 +127,8 @@ public function process(bool $dryRun = false): array ->where('groupid', $row->groupid) ->where('quality_sample', 0) ->update(['quality_sample' => 1]); + $stats['held_quality']++; } - $stats['held_quality']++; continue; } diff --git a/iznik-batch/app/Services/AutoApproveService.php b/iznik-batch/app/Services/AutoApproveService.php index a631b1e209..e152a245d7 100644 --- a/iznik-batch/app/Services/AutoApproveService.php +++ b/iznik-batch/app/Services/AutoApproveService.php @@ -103,6 +103,12 @@ public function process(bool $dryRun = false): array ->whereColumn('messages_outcomes.msgid', 'messages_groups.msgid') ->whereIn('messages_outcomes.outcome', ['Taken', 'Received']); }) + // Respect a moderator-set hold window (set by the Go Pending list fetch, + // extend-only to NOW()+10m) so a post a mod is actively reviewing is not + // auto-approved out from under them. + ->whereRaw( + '(messages_groups.autoapprove_hold_until IS NULL OR messages_groups.autoapprove_hold_until <= NOW())' + ) ->where(function ($q) { // Normal posts: the 48h fallback (unchanged). $q->where(function ($q2) { diff --git a/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php b/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php index 6c62b4906e..4eb1df2b5e 100644 --- a/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php +++ b/iznik-batch/tests/Unit/Services/AutoApproveCleanServiceTest.php @@ -403,4 +403,137 @@ public function test_records_ham_for_spam_message(): void $this->assertApproved($message->id, $group->id); $this->assertDatabaseHas('messages_spamham', ['msgid' => $message->id, 'spamham' => 'Ham']); } + + // --- A2: autoapprove_hold_until predicate --- + + public function test_hold_until_future_keeps_post_pending(): void + { + // A post whose autoapprove_hold_until is 5 minutes in the future must + // be skipped by the auto-approver even though the delay has elapsed. + [$user, $group, $message] = $this->makeApprovable(); + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update(['autoapprove_hold_until' => now()->addMinutes(5)]); + + $this->service->process(); + + $this->assertStillPending($message->id, $group->id); + } + + public function test_hold_until_past_allows_approval(): void + { + // A post whose autoapprove_hold_until has already expired must still + // be auto-approved normally. + [$user, $group, $message] = $this->makeApprovable(); + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update(['autoapprove_hold_until' => now()->subMinutes(1)]); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + } + + // --- D1: cross-group Spam-collection guard --- + + public function test_spam_on_another_group_blocks_approval_on_this_group(): void + { + // Message is clean and pending on group A (eligible for auto-approval), + // BUT also in the Spam collection on group B. + // AutoApproveCleanService must NOT auto-approve it on group A. + [$user, $groupA, $message] = $this->makeApprovable(); + + $groupB = $this->createTestGroup(); + // Insert a Spam-collection row for the same message on group B. + DB::table('messages_groups')->insert([ + 'msgid' => $message->id, + 'groupid' => $groupB->id, + 'collection' => MessageGroup::COLLECTION_SPAM, + 'arrival' => now(), + 'deleted' => 0, + ]); + + $this->service->process(); + + $this->assertStillPending($message->id, $groupA->id); + } + + // --- D3: quality-sample filter + dry-run stat placement --- + + public function test_already_quality_sampled_row_is_excluded_from_candidate_query(): void + { + // A row already marked quality_sample=1 must not appear in the candidate + // query at all. This verifies the ->where('mg.quality_sample', 0) guard. + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['autoapprove' => ['quality_check_percent' => 0]]], + ]); + // Manually pre-mark the row as already sampled (e.g. by a previous run). + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update(['quality_sample' => 1]); + + $stats = $this->service->process(); + + // The row is excluded from the candidate set entirely — held_quality is 0 + // because the service never even sees it. + $this->assertEquals(0, $stats['held_quality'], 'already-sampled row must not increment held_quality again'); + $this->assertStillPending($message->id, $group->id); + } + + public function test_held_quality_stat_not_incremented_in_dry_run(): void + { + // $stats['held_quality']++ must be inside the if(!$dryRun) block so a + // dry-run pass doesn't falsely inflate the cron stat. + [$user, $group, $message] = $this->makeApprovable([ + 'group' => ['settings' => ['autoapprove' => ['quality_check_percent' => 100]]], + ]); + + $stats = $this->service->process(dryRun: true); + + $this->assertEquals(0, $stats['held_quality'], 'held_quality must be 0 in a dry run'); + $this->assertStillPending($message->id, $group->id); + } + + // --- D6: OFFER freebie-alert background task --- + + public function test_approved_offer_queues_freebie_alert_background_task(): void + { + // An approved OFFER must insert a background_tasks row with + // task_type=TASK_FREEBIE_ALERTS_ADD containing the msgid, so the Go + // background-task processor can fan out push notifications to nearby users. + [$user, $group, $message] = $this->makeApprovable([ + 'message' => ['type' => \App\Models\Message::TYPE_OFFER], + ]); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + $this->assertDatabaseHas('background_tasks', [ + 'task_type' => \App\Models\BackgroundTask::TASK_FREEBIE_ALERTS_ADD, + ]); + // Verify the data payload contains the correct msgid. + $task = DB::table('background_tasks') + ->where('task_type', \App\Models\BackgroundTask::TASK_FREEBIE_ALERTS_ADD) + ->first(); + $data = json_decode($task->data, true); + $this->assertEquals((int) $message->id, $data['msgid']); + } + + public function test_approved_wanted_does_not_queue_freebie_alert(): void + { + // Only OFFERs trigger freebie alerts; WANTEDs must not insert a row. + [$user, $group, $message] = $this->makeApprovable([ + 'message' => ['type' => \App\Models\Message::TYPE_WANTED], + ]); + + $this->service->process(); + + $this->assertApproved($message->id, $group->id); + $this->assertDatabaseMissing('background_tasks', [ + 'task_type' => \App\Models\BackgroundTask::TASK_FREEBIE_ALERTS_ADD, + ]); + } } diff --git a/iznik-batch/tests/Unit/Services/AutoApproveServiceHoldTest.php b/iznik-batch/tests/Unit/Services/AutoApproveServiceHoldTest.php new file mode 100644 index 0000000000..a8207c1d2b --- /dev/null +++ b/iznik-batch/tests/Unit/Services/AutoApproveServiceHoldTest.php @@ -0,0 +1,70 @@ +service = new AutoApproveService(); + } + + private function makeApprovable48h(): array + { + $user = $this->createTestUser(); + $group = $this->createTestGroup(); + $this->createMembership($user, $group, ['added' => now()->subDays(5)]); + $message = $this->createTestMessage($user, $group); + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update([ + 'collection' => MessageGroup::COLLECTION_PENDING, + 'arrival' => now()->subHours(50), + ]); + + return [$user, $group, $message]; + } + + public function test_48h_hold_until_future_keeps_post_pending(): void + { + [$user, $group, $message] = $this->makeApprovable48h(); + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update(['autoapprove_hold_until' => now()->addMinutes(10)]); + + $this->service->process(); + + $this->assertDatabaseHas('messages_groups', [ + 'msgid' => $message->id, + 'groupid' => $group->id, + 'collection' => MessageGroup::COLLECTION_PENDING, + ]); + } + + public function test_48h_hold_until_past_allows_approval(): void + { + [$user, $group, $message] = $this->makeApprovable48h(); + DB::table('messages_groups') + ->where('msgid', $message->id) + ->where('groupid', $group->id) + ->update(['autoapprove_hold_until' => now()->subMinutes(1)]); + + $this->service->process(); + + $this->assertDatabaseHas('messages_groups', [ + 'msgid' => $message->id, + 'groupid' => $group->id, + 'collection' => MessageGroup::COLLECTION_APPROVED, + ]); + } +} From 78b12ce275c30050355a559f3cdb8bd0bcd74898 Mon Sep 17 00:00:00 2001 From: edwh Date: Mon, 15 Jun 2026 19:10:59 +0100 Subject: [PATCH 09/26] feat(go): Pending hold-bump, accurate autoapproveat, + review fixes (A3,A4,D2,D4,D9,D10,D11) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A3: ListMessagesMT bumps autoapprove_hold_until >=NOW()+10m (extend-only) for fetched Pending rows, so loading the page guarantees a review window. - A4: GET /message/:id exposes per-group autoapproveat for Pending posts viewed by a mod (new message/autoapproveat.go). Accurate: a concurrent danger-signal query (mirrors AutoApproveCleanService::hasDangerSignals) + group-allows + quality-sample => clean-path delay, 48h fallback, or nil (danger/spam/held). - D2: reword session.go/utils.go comments — older posts drop off the queue (no checkedat write), so AutoModChecked keeps measuring real mod checks. - D4: scope moderationstats Trusted NOT EXISTS by groupid (cross-post miscount). - D9/D10/D11: markchecked guards collection=Approved on the ids path, 400 on an unknown filter, 403 for a non-mod with groupid=0. Co-Authored-By: Claude Opus 4.8 --- .../{[[id]]/[[term]].vue => [[id]].vue} | 0 .../{[[id]]/[[term]].vue => [[id]].vue} | 0 iznik-server-go/message/autoapproveat.go | 173 ++++++++++++++++++ iznik-server-go/message/markchecked.go | 10 +- iznik-server-go/message/message.go | 11 +- iznik-server-go/message/messageGroup.go | 14 ++ iznik-server-go/message/message_list.go | 16 ++ iznik-server-go/moderation/moderationstats.go | 2 +- iznik-server-go/session/session.go | 9 +- iznik-server-go/utils/utils.go | 6 +- 10 files changed, 230 insertions(+), 11 deletions(-) rename iznik-nuxt3/modtools/pages/messages/checked/{[[id]]/[[term]].vue => [[id]].vue} (100%) rename iznik-nuxt3/modtools/pages/messages/trusted/{[[id]]/[[term]].vue => [[id]].vue} (100%) create mode 100644 iznik-server-go/message/autoapproveat.go diff --git a/iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue b/iznik-nuxt3/modtools/pages/messages/checked/[[id]].vue similarity index 100% rename from iznik-nuxt3/modtools/pages/messages/checked/[[id]]/[[term]].vue rename to iznik-nuxt3/modtools/pages/messages/checked/[[id]].vue diff --git a/iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue b/iznik-nuxt3/modtools/pages/messages/trusted/[[id]].vue similarity index 100% rename from iznik-nuxt3/modtools/pages/messages/trusted/[[id]]/[[term]].vue rename to iznik-nuxt3/modtools/pages/messages/trusted/[[id]].vue diff --git a/iznik-server-go/message/autoapproveat.go b/iznik-server-go/message/autoapproveat.go new file mode 100644 index 0000000000..0778e88b5c --- /dev/null +++ b/iznik-server-go/message/autoapproveat.go @@ -0,0 +1,173 @@ +package message + +import ( + "encoding/json" + "hash/crc32" + "strconv" + "time" + + "github.com/freegle/iznik-server-go/utils" + "gorm.io/gorm" +) + +// phpTruthy mirrors PHP's !empty(): nil, false, 0, "" and "0" are falsy; everything +// else is truthy. Used so the Go group-allows check matches AutoApproveCleanService's +// PHP getSetting()/empty() semantics exactly. +func phpTruthy(v interface{}) bool { + switch x := v.(type) { + case nil: + return false + case bool: + return x + case float64: + return x != 0 + case string: + return x != "" && x != "0" + default: + return true + } +} + +// computeAutoapproveat fills MessageGroup.Autoapproveat for the Pending group entries of +// a message a moderator is viewing. It is an ACCURATE estimate of when the post will be +// auto-approved: +// +// - danger-signalled / Spam-on-any-group / held -> nil (no auto-approval expected) +// - clean path (NULL posting status, content-check clean, group allows, not sampled) +// -> arrival + group delay (site default 20m) +// - otherwise (not clean, but not held/spam) -> arrival + 48h (the broad fallback) +// +// then capped below by autoapprove_hold_until (the extend-only hold set on Pending load). +// +// PARITY: the clean-path eligibility and danger signals mirror +// app/Services/AutoApproveCleanService.php (hasDangerSignals, groupAllowsAutoApprove, +// isQualitySampled). Keep the two in sync. +func computeAutoapproveat(db *gorm.DB, message *Message, groups []MessageGroup, idStr string) { + // Spam on ANY group blocks auto-approval everywhere (Discourse #9654 parity). + for _, mg := range groups { + if mg.Collection == utils.COLLECTION_SPAM { + return + } + } + + var pendingIdx []int + var gids []uint64 + for i := range groups { + if groups[i].Collection == utils.COLLECTION_PENDING && groups[i].Heldby == nil { + pendingIdx = append(pendingIdx, i) + gids = append(gids, groups[i].Groupid) + } + } + if len(pendingIdx) == 0 { + return + } + + idNum, _ := strconv.ParseUint(idStr, 10, 64) + + // Danger signals — mirror AutoApproveCleanService::hasDangerSignals exactly. One + // combined query: danger=1 if ANY signal fires for this poster/message/groups. + var danger bool + db.Raw(`SELECT ( + EXISTS (SELECT 1 FROM microactions WHERE msgid = ? AND actiontype = 'CheckMessage' AND result = 'Reject') + OR EXISTS (SELECT 1 FROM users_comments WHERE userid = ?) + OR EXISTS (SELECT 1 FROM logs WHERE user = ? AND timestamp >= NOW() - INTERVAL 90 DAY + AND (byuser != user OR byuser IS NULL) + AND ((type = 'Message' AND subtype IN ('Rejected','Deleted','Replied')) + OR (type = 'User' AND subtype IN ('Mailed','Rejected','Deleted','Suspect','ClassifiedSpam')))) + OR EXISTS (SELECT 1 FROM spam_users WHERE userid = ? AND collection IN ('Spammer','PendingAdd')) + OR EXISTS (SELECT 1 FROM memberships WHERE userid = ? AND groupid IN ? + AND reviewrequestedat IS NOT NULL + AND (reviewedat IS NULL OR reviewedat < reviewrequestedat)) + ) AS danger`, + idNum, message.Fromuser, message.Fromuser, message.Fromuser, message.Fromuser, gids, + ).Scan(&danger) + if danger { + return + } + + for _, i := range pendingIdx { + mg := &groups[i] + + var row struct { + OurPostingStatus *string `gorm:"column:ourpostingstatus"` + Settings *string `gorm:"column:settings"` + Rules *string `gorm:"column:rules"` + Autofunctionoverride *string `gorm:"column:autofunctionoverride"` + Overridemoderation *string `gorm:"column:overridemoderation"` + } + db.Raw("SELECT mem.ourPostingStatus AS ourpostingstatus, g.settings AS settings, g.rules AS rules, "+ + "g.autofunctionoverride AS autofunctionoverride, g.overridemoderation AS overridemoderation "+ + "FROM `groups` g LEFT JOIN memberships mem ON mem.groupid = g.id AND mem.userid = ? "+ + "WHERE g.id = ? LIMIT 1", message.Fromuser, mg.Groupid).Scan(&row) + + var settings map[string]interface{} + if row.Settings != nil && *row.Settings != "" { + _ = json.Unmarshal([]byte(*row.Settings), &settings) + } + + // groupAllowsAutoApprove parity (PHP AutoApproveCleanService::groupAllowsAutoApprove). + groupAllows := true + if settings != nil { + if v, ok := settings["publish"]; ok && !phpTruthy(v) { // getSetting('publish', true) + groupAllows = false + } + } + if groupAllows && settings != nil && phpTruthy(settings["closed"]) { // isClosed() + groupAllows = false + } + if groupAllows && row.Autofunctionoverride != nil && phpTruthy(*row.Autofunctionoverride) { + groupAllows = false + } + if groupAllows && row.Overridemoderation != nil && *row.Overridemoderation == "ModerateAll" { + groupAllows = false + } + if groupAllows && settings != nil && phpTruthy(settings["moderated"]) { // getSetting('moderated', 0) + groupAllows = false + } + if groupAllows && row.Rules != nil && *row.Rules != "" { + var rules map[string]interface{} + if json.Unmarshal([]byte(*row.Rules), &rules) == nil && phpTruthy(rules["fullymoderated"]) { + groupAllows = false + } + } + + onCleanPath := groupAllows && + row.OurPostingStatus == nil && + mg.ContentcheckCheckedAt != nil && + mg.ContentcheckReasons == nil && + mg.QualitySample == 0 + + // delay_minutes / quality_check_percent from settings.autoapprove (0/absent => default). + delayMinutes := 20 + qualityPercent := 0 + if settings != nil { + if aa, ok := settings["autoapprove"].(map[string]interface{}); ok { + if dm, ok := aa["delay_minutes"].(float64); ok && dm > 0 { + delayMinutes = int(dm) + } + if qp, ok := aa["quality_check_percent"].(float64); ok && qp > 0 { + qualityPercent = int(qp) + } + } + } + // Deterministic quality sample (mirror PHP isQualitySampled: crc32(msgid) % 100 < percent). + qualitySampled := qualityPercent > 0 && int(crc32.ChecksumIEEE([]byte(idStr))%100) < qualityPercent + + var base time.Time + if onCleanPath && !qualitySampled { + base = mg.Arrival.Add(time.Duration(delayMinutes) * time.Minute) + } else if mg.Spamtype == nil { + // Not on the clean path (or quality-sampled), but not spam/held: 48h fallback. + // (Estimate nuance: the fallback also needs >=48h membership, which we ignore.) + base = mg.Arrival.Add(48 * time.Hour) + } + + if !base.IsZero() { + t := base + if mg.AutoapproveHoldUntil != nil && mg.AutoapproveHoldUntil.After(t) { + t = *mg.AutoapproveHoldUntil + } + groups[i].Autoapproveat = &t + } + } +} diff --git a/iznik-server-go/message/markchecked.go b/iznik-server-go/message/markchecked.go index 39b0f28257..c6fd73f480 100644 --- a/iznik-server-go/message/markchecked.go +++ b/iznik-server-go/message/markchecked.go @@ -46,7 +46,8 @@ func MarkChecked(c *fiber.Ctx) error { if req.Groupid == 0 { groupIDs = user.GetActiveModGroupIDs(myid) if len(groupIDs) == 0 { - return c.JSON(fiber.Map{"success": true, "checked": 0}) + // Not a moderator of any group — cannot mark anything checked (D11). + return fiber.NewError(fiber.StatusForbidden, "Not a moderator") } } else { if !user.IsModOfGroup(myid, req.Groupid) { @@ -60,13 +61,16 @@ func MarkChecked(c *fiber.Ctx) error { // Mark the specified posts checked — only ones still unchecked, and only // on groups the mod moderates. r := db.Exec("UPDATE messages_groups SET checkedat = NOW(), checkedby = ? "+ - "WHERE msgid IN ? AND groupid IN ? AND checkedat IS NULL", - myid, req.IDs, groupIDs) + "WHERE msgid IN ? AND groupid IN ? AND collection = ? AND deleted = 0 AND checkedat IS NULL", + myid, req.IDs, groupIDs, utils.COLLECTION_APPROVED) rowsAffected = r.RowsAffected } else { // Mark the whole bucket checked (the "mark all as checked" action). The // bucket condition mirrors the Checked/Trusted list filter so exactly the // posts a mod is looking at get cleared. + if req.Filter != "checked" && req.Filter != "trusted" { + return fiber.NewError(fiber.StatusBadRequest, "filter must be 'checked' or 'trusted' (D10)") + } statusWhere := "mem.ourPostingStatus IS NULL" if req.Filter == "trusted" { statusWhere = "(mem.ourPostingStatus = 'DEFAULT' OR mem.ourPostingStatus = 'UNMODERATED')" diff --git a/iznik-server-go/message/message.go b/iznik-server-go/message/message.go index 205e89a565..2f1a19dbed 100644 --- a/iznik-server-go/message/message.go +++ b/iznik-server-go/message/message.go @@ -485,7 +485,7 @@ func GetMessagesByIds(myid uint64, ids []string, isPartner bool) []Message { // Both APPROVED and PENDING messages are visible to all users. This is not a privacy // issue because these messages were posted with the intention of being public. It also // allows shared links to work even before moderation approval. - db.Raw("SELECT groupid, msgid, arrival, collection, autoreposts, approvedby, heldby, spamtype, spamreason, contentcheck_checked_at, contentcheck_reasons, rippled_in FROM messages_groups WHERE msgid = ? AND deleted = 0", id).Scan(&messageGroups) + db.Raw("SELECT groupid, msgid, arrival, collection, autoreposts, approvedby, heldby, spamtype, spamreason, contentcheck_checked_at, contentcheck_reasons, rippled_in, quality_sample, autoapprove_hold_until FROM messages_groups WHERE msgid = ? AND deleted = 0", id).Scan(&messageGroups) // Moderator-only "quicker to get to" P/Q note, kept in its own rippling_proximity // table (off the hot messages_groups path). Best-effort: only for mods, and a @@ -607,6 +607,15 @@ func GetMessagesByIds(myid uint64, ids []string, isPartner bool) []Message { isGroupMod = isModForMessage(db, myid, idNum) } + // A4. Auto-approve countdown: for Pending posts viewed by a moderator, + // compute the per-group autoapproveat estimate (clean-path delay or 48h + // fallback, capped by autoapprove_hold_until; nil if danger-signalled, + // spam, held, or not on an auto-approve path). Mirrors the eligibility of + // AutoApproveCleanService/AutoApproveService — see message/autoapproveat.go. + if isGroupMod && myid > 0 { + computeAutoapproveat(db, &message, messageGroups, id) + } + // Postings (history of which groups this message was on) are public information, // returned to all callers — matching V1 behaviour. var messagePostings []MessagePosting diff --git a/iznik-server-go/message/messageGroup.go b/iznik-server-go/message/messageGroup.go index eeb62f6f5e..24b1ed8221 100644 --- a/iznik-server-go/message/messageGroup.go +++ b/iznik-server-go/message/messageGroup.go @@ -42,4 +42,18 @@ type MessageGroup struct { // routing/KNN calls failed at ripple-in time — the frontend shows nothing in all three cases. RippleProximityP *string `json:"ripple_proximity_p,omitempty"` RippleProximityQ *string `json:"ripple_proximity_q,omitempty"` + + // QualitySample is set by AutoApproveCleanService when a clean post is held back + // for a manual quality check. Scanned for the autoapproveat estimate; not serialised. + QualitySample int `json:"-" gorm:"column:quality_sample"` + + // AutoapproveHoldUntil is the server-side extend-only hold set when the Pending + // queue is viewed (see ListMessagesMT). Scanned but not serialised — the frontend + // uses the computed Autoapproveat below. + AutoapproveHoldUntil *time.Time `json:"-" gorm:"column:autoapprove_hold_until"` + + // Autoapproveat is the earliest time this post may be auto-approved, exposed only + // on Pending messages viewed by a group moderator. nil = no auto-approval expected + // (held / spam / danger-signalled, or not on any auto-approve path). + Autoapproveat *time.Time `json:"autoapproveat,omitempty" gorm:"-"` } diff --git a/iznik-server-go/message/message_list.go b/iznik-server-go/message/message_list.go index 25b685dc38..6a9fec4e7d 100644 --- a/iznik-server-go/message/message_list.go +++ b/iznik-server-go/message/message_list.go @@ -608,6 +608,22 @@ func ListMessagesMT(c *fiber.Ctx) error { return c.JSON(fiber.Map{"messages": []uint64{}}) } + // A3. Reset-to-minimum hold (extend-only) for the messages just loaded. When a + // moderator views the Pending queue, every fetched post is given at least 10 more + // minutes before it becomes eligible for auto-approval, so nothing vanishes while + // the page is open. GREATEST(COALESCE(autoapprove_hold_until, NOW()), NOW()+10m) is + // extend-only: an existing hold further out is never shortened. Scope: Pending, + // heldby IS NULL, deleted=0, only the mod's groups. + if collection == utils.COLLECTION_PENDING && len(groupIDs) > 0 { + db.Exec( + "UPDATE messages_groups "+ + "SET autoapprove_hold_until = GREATEST(COALESCE(autoapprove_hold_until, NOW()), NOW() + INTERVAL 10 MINUTE) "+ + "WHERE msgid IN ? AND groupid IN ? "+ + "AND collection = 'Pending' AND heldby IS NULL AND deleted = 0", + msgIDs, groupIDs, + ) + } + // Build pagination context from last ID. var respCtx *PaginationContext if len(msgIDs) == limit { diff --git a/iznik-server-go/moderation/moderationstats.go b/iznik-server-go/moderation/moderationstats.go index 8d76301a63..9232cef1fb 100644 --- a/iznik-server-go/moderation/moderationstats.go +++ b/iznik-server-go/moderation/moderationstats.go @@ -102,7 +102,7 @@ func Stats(c *fiber.Ctx) error { "WHERE mg.approvedat BETWEEN ? AND ? AND mg.approvedby IS NULL "+ "AND mg.contentcheck_checked_at IS NOT NULL AND mg.deleted=0 "+ "AND (mem.ourPostingStatus='DEFAULT' OR mem.ourPostingStatus='UNMODERATED') "+ - "AND NOT EXISTS (SELECT 1 FROM logs l WHERE l.msgid=mg.msgid AND l.type='Message' AND l.subtype='Autoapproved')", + "AND NOT EXISTS (SELECT 1 FROM logs l WHERE l.msgid=mg.msgid AND l.groupid=mg.groupid AND l.type='Message' AND l.subtype='Autoapproved')", start, end).Scan(&stats.Trusted) }) diff --git a/iznik-server-go/session/session.go b/iznik-server-go/session/session.go index 260231ae8c..7a476cff62 100644 --- a/iznik-server-go/session/session.go +++ b/iznik-server-go/session/session.go @@ -1034,12 +1034,13 @@ func GetSession(c *fiber.Ctx) error { // views: checked = auto-approved posts from auto-moderated (NULL) members; // trusted = posts that went live from trusted (group-settings) members. // A post leaves the count when a mod marks it checked (checkedat set) or - // after 7 days (auto-checked, so the queue can't pile up indefinitely). + // once it is older than the window — older posts simply drop off the queue + // (no checkedat is written), so the queue can't pile up indefinitely. var checked, trusted int64 var wg2 sync.WaitGroup - // Oversight queues count only unchecked posts within the auto-check window. + // Oversight queues count only unchecked posts within the check window. checkedWindowSQL := fmt.Sprintf( "AND mg.checkedat IS NULL AND mg.arrival >= NOW() - INTERVAL %d DAY", utils.MESSAGE_CHECK_WINDOW_DAYS, @@ -1090,7 +1091,7 @@ func GetSession(c *fiber.Ctx) error { // --- Checked: UNCHECKED auto-approved posts from auto-moderated (NULL) members. // Outstanding oversight work (blue): a mod hasn't marked it checked and it - // is within the 7-day auto-check window. --- + // is within the 7-day check window. --- wg2.Add(1) go func() { defer wg2.Done() @@ -1106,7 +1107,7 @@ func GetSession(c *fiber.Ctx) error { }() // --- Trusted: UNCHECKED live posts from trusted (group-settings) members. - // Outstanding oversight work (blue), same 7-day auto-check window. --- + // Outstanding oversight work (blue), same 7-day check window. --- wg2.Add(1) go func() { defer wg2.Done() diff --git a/iznik-server-go/utils/utils.go b/iznik-server-go/utils/utils.go index b34cd26314..eef0a32dd3 100644 --- a/iznik-server-go/utils/utils.go +++ b/iznik-server-go/utils/utils.go @@ -101,8 +101,10 @@ const COLLECTION_BANNED = "Banned" const COLLECTION_DRAFT = "Draft" // MESSAGE_CHECK_WINDOW_DAYS is how long an auto-published post stays in the -// ModTools Checked/Trusted oversight queues (and their work-count badges) before -// it is treated as auto-checked, so the queue cannot pile up indefinitely. +// ModTools Checked/Trusted oversight queues (and their work-count badges). Once a +// post is older than this it simply drops off the queue — no checkedat is written, +// so it is NOT counted as moderator-checked — which bounds the queue without +// inflating the "a mod checked this" moderation-stats metric. const MESSAGE_CHECK_WINDOW_DAYS = 7 const POSTING_STATUS_MODERATED = "MODERATED" From 3a9dd8695e01f05f348420429c03a5ed294f097c Mon Sep 17 00:00:00 2001 From: edwh Date: Mon, 15 Jun 2026 19:10:59 +0100 Subject: [PATCH 10/26] test(go): countdown hold-bump, autoapproveat gating, markchecked guards (A3,A4,D9,D11) TestListMessagesMTPendingBumpsHold (extend-only), TestAutoapproveatPendingModGating (clean=>time, danger=>nil, non-mod=>nil), TestMarkCheckedSpecificIDs (collection guard + group scoping), TestMarkCheckedCrossGroupAndNonMod (403 paths). All green. Co-Authored-By: Claude Opus 4.8 --- .../test/autoapprove_countdown_test.go | 181 ++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100644 iznik-server-go/test/autoapprove_countdown_test.go diff --git a/iznik-server-go/test/autoapprove_countdown_test.go b/iznik-server-go/test/autoapprove_countdown_test.go new file mode 100644 index 0000000000..4e7e234838 --- /dev/null +++ b/iznik-server-go/test/autoapprove_countdown_test.go @@ -0,0 +1,181 @@ +package test + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/freegle/iznik-server-go/database" + "github.com/stretchr/testify/assert" +) + +// getAutoapproveatField fetches a message as the given user and returns the +// autoapproveat value for the named group (nil if absent / not a mod / not pending). +func getAutoapproveatField(t *testing.T, msgid uint64, groupid uint64, token string) interface{} { + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/message/%d?jwt=%s", msgid, token), nil)) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + var body map[string]interface{} + json.NewDecoder(resp.Body).Decode(&body) + groups, _ := body["groups"].([]interface{}) + for _, g := range groups { + gm, _ := g.(map[string]interface{}) + if gid, ok := gm["groupid"].(float64); ok && uint64(gid) == groupid { + return gm["autoapproveat"] + } + } + return nil +} + +// A3: loading the Pending queue bumps autoapprove_hold_until to >= NOW()+10m +// (extend-only — an existing longer hold is never shortened). +func TestListMessagesMTPendingBumpsHold(t *testing.T) { + prefix := uniquePrefix("hold_bump") + db := database.DBConn + + groupID := CreateTestGroup(t, prefix) + poster := CreateTestUser(t, prefix+"_poster", "User") + modID := CreateTestUser(t, prefix+"_mod", "User") + CreateTestMembership(t, poster, groupID, "Member") + CreateTestMembership(t, modID, groupID, "Moderator") + db.Exec("UPDATE memberships SET ourPostingStatus = NULL WHERE userid = ? AND groupid = ?", poster, groupID) + _, modToken := CreateTestSession(t, modID) + + // Fresh pending post with no hold. contentcheck_checked_at must be set so the + // post is visible in the Pending list (else the content-check filter hides it). + freshMsg := CreateTestMessage(t, poster, groupID, prefix+" fresh pending", 52.0, -1.0) + // Pending post already held 60 min out — extend-only must not shorten it. + heldMsg := CreateTestMessage(t, poster, groupID, prefix+" held pending", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Pending', arrival=NOW(), contentcheck_checked_at=NOW(), autoapprove_hold_until=NULL WHERE msgid=?", freshMsg) + db.Exec("UPDATE messages_groups SET collection='Pending', arrival=NOW(), contentcheck_checked_at=NOW(), autoapprove_hold_until=NOW() + INTERVAL 60 MINUTE WHERE msgid=?", heldMsg) + + resp, err := getApp().Test(httptest.NewRequest("GET", + fmt.Sprintf("/api/modtools/messages?groupid=%d&collection=Pending&jwt=%s", groupID, modToken), nil)) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + var freshSecs int64 + db.Raw("SELECT TIMESTAMPDIFF(SECOND, NOW(), autoapprove_hold_until) FROM messages_groups WHERE msgid=? AND groupid=?", freshMsg, groupID).Scan(&freshSecs) + assert.GreaterOrEqual(t, freshSecs, int64(9*60), "fresh pending post should be held >= ~10 min after load") + + var heldSecs int64 + db.Raw("SELECT TIMESTAMPDIFF(SECOND, NOW(), autoapprove_hold_until) FROM messages_groups WHERE msgid=? AND groupid=?", heldMsg, groupID).Scan(&heldSecs) + assert.Greater(t, heldSecs, int64(30*60), "existing longer hold must not be shortened (extend-only)") + + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", freshMsg, heldMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?)", freshMsg, heldMsg) +} + +// A4: GET /message/:id exposes autoapproveat only for Pending posts viewed by a +// moderator; clean-path posts get a time, danger-signalled posts get nil, and +// non-mods never see it. +func TestAutoapproveatPendingModGating(t *testing.T) { + prefix := uniquePrefix("autoapproveat") + db := database.DBConn + + groupID := CreateTestGroup(t, prefix) + poster := CreateTestUser(t, prefix+"_poster", "User") + modID := CreateTestUser(t, prefix+"_mod", "User") + regularID := CreateTestUser(t, prefix+"_reg", "User") + CreateTestMembership(t, poster, groupID, "Member") + CreateTestMembership(t, modID, groupID, "Moderator") + CreateTestMembership(t, regularID, groupID, "Member") + db.Exec("UPDATE memberships SET ourPostingStatus = NULL WHERE userid = ? AND groupid = ?", poster, groupID) + _, modToken := CreateTestSession(t, modID) + _, regToken := CreateTestSession(t, regularID) + + // Clean-path pending post: NULL poster, content-check clean, arrival 5 min ago. + cleanMsg := CreateTestMessage(t, poster, groupID, prefix+" clean pending", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Pending', arrival=NOW() - INTERVAL 5 MINUTE, contentcheck_checked_at=NOW() - INTERVAL 4 MINUTE, contentcheck_reasons=NULL, autoapprove_hold_until=NULL WHERE msgid=?", cleanMsg) + + // Mod sees autoapproveat. + assert.NotNil(t, getAutoapproveatField(t, cleanMsg, groupID, modToken), + "mod should see autoapproveat on a clean pending post") + // Non-mod does not. + assert.Nil(t, getAutoapproveatField(t, cleanMsg, groupID, regToken), + "non-mod must not see autoapproveat") + + // Danger-signalled post (poster is a known spammer) → no countdown. + dangerMsg := CreateTestMessage(t, poster, groupID, prefix+" danger pending", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Pending', arrival=NOW() - INTERVAL 5 MINUTE, contentcheck_checked_at=NOW() - INTERVAL 4 MINUTE WHERE msgid=?", dangerMsg) + db.Exec("INSERT INTO spam_users (userid, collection, added) VALUES (?, 'Spammer', NOW())", poster) + assert.Nil(t, getAutoapproveatField(t, dangerMsg, groupID, modToken), + "danger-signalled post must not show a countdown") + + db.Exec("DELETE FROM spam_users WHERE userid=?", poster) + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?)", cleanMsg, dangerMsg) + db.Exec("DELETE FROM messages WHERE id IN (?, ?)", cleanMsg, dangerMsg) +} + +// D9: markchecked with explicit ids marks only Approved rows on the mod's groups, +// leaving Pending rows and other-group rows untouched. +func TestMarkCheckedSpecificIDs(t *testing.T) { + prefix := uniquePrefix("markchk_ids") + db := database.DBConn + + groupA := CreateTestGroup(t, prefix+"_a") + groupB := CreateTestGroup(t, prefix+"_b") + poster := CreateTestUser(t, prefix+"_poster", "User") + modID := CreateTestUser(t, prefix+"_mod", "User") + CreateTestMembership(t, poster, groupA, "Member") + CreateTestMembership(t, poster, groupB, "Member") + CreateTestMembership(t, modID, groupA, "Moderator") // mod of A only + _, modToken := CreateTestSession(t, modID) + + approvedA := CreateTestMessage(t, poster, groupA, prefix+" approved a", 52.0, -1.0) + pendingA := CreateTestMessage(t, poster, groupA, prefix+" pending a", 52.0, -1.0) + approvedB := CreateTestMessage(t, poster, groupB, prefix+" approved b", 52.0, -1.0) + db.Exec("UPDATE messages_groups SET collection='Approved', approvedby=NULL WHERE msgid IN (?, ?)", approvedA, approvedB) + db.Exec("UPDATE messages_groups SET collection='Pending' WHERE msgid=?", pendingA) + + body := fmt.Sprintf(`{"groupid": %d, "ids": [%d, %d, %d]}`, groupA, approvedA, pendingA, approvedB) + req := httptest.NewRequest("POST", fmt.Sprintf("/api/modtools/messages/markchecked?jwt=%s", modToken), strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + resp, err := getApp().Test(req) + assert.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + + var aChecked, pendChecked, bChecked int64 + db.Raw("SELECT COUNT(*) FROM messages_groups WHERE msgid=? AND groupid=? AND checkedat IS NOT NULL", approvedA, groupA).Scan(&aChecked) + db.Raw("SELECT COUNT(*) FROM messages_groups WHERE msgid=? AND groupid=? AND checkedat IS NOT NULL", pendingA, groupA).Scan(&pendChecked) + db.Raw("SELECT COUNT(*) FROM messages_groups WHERE msgid=? AND groupid=? AND checkedat IS NOT NULL", approvedB, groupB).Scan(&bChecked) + assert.Equal(t, int64(1), aChecked, "Approved post on the mod's group should be checked") + assert.Equal(t, int64(0), pendChecked, "Pending post must not be marked checked (collection guard, D9)") + assert.Equal(t, int64(0), bChecked, "post on a non-moderated group must not be checked") + + db.Exec("DELETE FROM messages_groups WHERE msgid IN (?, ?, ?)", approvedA, pendingA, approvedB) + db.Exec("DELETE FROM messages WHERE id IN (?, ?, ?)", approvedA, pendingA, approvedB) +} + +// D11: markchecked returns 403 for a mod acting on a group they don't moderate, +// and for a non-mod using groupid=0. +func TestMarkCheckedCrossGroupAndNonMod(t *testing.T) { + prefix := uniquePrefix("markchk_403") + + groupA := CreateTestGroup(t, prefix+"_a") + groupB := CreateTestGroup(t, prefix+"_b") + modB := CreateTestUser(t, prefix+"_modb", "User") + CreateTestMembership(t, modB, groupB, "Moderator") // mod of B only + _, modBToken := CreateTestSession(t, modB) + + // Mod of B targeting group A → 403. + bodyA := fmt.Sprintf(`{"groupid": %d, "filter": "checked"}`, groupA) + reqA := httptest.NewRequest("POST", fmt.Sprintf("/api/modtools/messages/markchecked?jwt=%s", modBToken), strings.NewReader(bodyA)) + reqA.Header.Set("Content-Type", "application/json") + respA, err := getApp().Test(reqA) + assert.NoError(t, err) + assert.Equal(t, 403, respA.StatusCode, "mod of B must not mark group A") + + // Non-mod with groupid=0 → 403. + regID := CreateTestUser(t, prefix+"_reg", "User") + _, regToken := CreateTestSession(t, regID) + body0 := `{"groupid": 0, "filter": "checked"}` + req0 := httptest.NewRequest("POST", fmt.Sprintf("/api/modtools/messages/markchecked?jwt=%s", regToken), strings.NewReader(body0)) + req0.Header.Set("Content-Type", "application/json") + resp0, err := getApp().Test(req0) + assert.NoError(t, err) + assert.Equal(t, 403, resp0.StatusCode, "non-mod with groupid=0 must get 403") +} From f28588ba7998df4257d105a5717b5bef806a2fd6 Mon Sep 17 00:00:00 2001 From: edwh Date: Mon, 15 Jun 2026 19:17:27 +0100 Subject: [PATCH 11/26] feat(modtools): Pending auto-approve countdown + oversight help boxes (A5,C,D2,D5,D8,D12) - A5: live countdown badge on Pending messages (ModMessage.vue) from per-group autoapproveat: >1h muted, <=30m prominent, <=0 Auto-approving..., null=hidden. - C: ModHelpPending/Checked/Trusted/Approved help boxes (useHelpBox), wired into the four queue pages, explaining what goes where. - D2: oversight empty-state copy now says posts "drop off this queue" (not the inaccurate "treated as checked"). - D5: drop the unused [[term]] route segment from the checked/trusted pages. - D8: align the moderation-stats auto-approved row label (Checked delay + 48h fallback). - D12: remove orphaned autoChecked/autoFallback from the stats spec fixture. Vitest: countdown (5 cases), four ModHelp specs, and the stats spec all green. Co-Authored-By: Claude Opus 4.8 --- .../modtools/components/ModHelpApproved.vue | 19 +++++++ .../modtools/components/ModHelpChecked.vue | 20 +++++++ .../modtools/components/ModHelpPending.vue | 23 ++++++++ .../modtools/components/ModHelpTrusted.vue | 20 +++++++ .../modtools/components/ModMessage.vue | 53 ++++++++++++++++++ .../components/ModSysAdminModerationStats.vue | 2 +- .../messages/approved/[[id]]/[[term]].vue | 1 + .../pages/messages/checked/[[id]].vue | 3 +- .../messages/pending/[[id]]/[[term]].vue | 1 + .../pages/messages/trusted/[[id]].vue | 3 +- .../modtools/ModHelpApproved.spec.js | 52 ++++++++++++++++++ .../modtools/ModHelpChecked.spec.js | 53 ++++++++++++++++++ .../modtools/ModHelpPending.spec.js | 51 +++++++++++++++++ .../modtools/ModHelpTrusted.spec.js | 52 ++++++++++++++++++ .../components/modtools/ModMessage.spec.js | 55 +++++++++++++++++++ .../ModSysAdminModerationStats.spec.js | 4 +- 16 files changed, 407 insertions(+), 5 deletions(-) create mode 100644 iznik-nuxt3/modtools/components/ModHelpApproved.vue create mode 100644 iznik-nuxt3/modtools/components/ModHelpChecked.vue create mode 100644 iznik-nuxt3/modtools/components/ModHelpPending.vue create mode 100644 iznik-nuxt3/modtools/components/ModHelpTrusted.vue create mode 100644 iznik-nuxt3/tests/unit/components/modtools/ModHelpApproved.spec.js create mode 100644 iznik-nuxt3/tests/unit/components/modtools/ModHelpChecked.spec.js create mode 100644 iznik-nuxt3/tests/unit/components/modtools/ModHelpPending.spec.js create mode 100644 iznik-nuxt3/tests/unit/components/modtools/ModHelpTrusted.spec.js diff --git a/iznik-nuxt3/modtools/components/ModHelpApproved.vue b/iznik-nuxt3/modtools/components/ModHelpApproved.vue new file mode 100644 index 0000000000..b21837ac16 --- /dev/null +++ b/iznik-nuxt3/modtools/components/ModHelpApproved.vue @@ -0,0 +1,19 @@ + + diff --git a/iznik-nuxt3/modtools/components/ModHelpChecked.vue b/iznik-nuxt3/modtools/components/ModHelpChecked.vue new file mode 100644 index 0000000000..fbc2804f19 --- /dev/null +++ b/iznik-nuxt3/modtools/components/ModHelpChecked.vue @@ -0,0 +1,20 @@ + + diff --git a/iznik-nuxt3/modtools/components/ModHelpPending.vue b/iznik-nuxt3/modtools/components/ModHelpPending.vue new file mode 100644 index 0000000000..21d56fb222 --- /dev/null +++ b/iznik-nuxt3/modtools/components/ModHelpPending.vue @@ -0,0 +1,23 @@ + + diff --git a/iznik-nuxt3/modtools/components/ModHelpTrusted.vue b/iznik-nuxt3/modtools/components/ModHelpTrusted.vue new file mode 100644 index 0000000000..80959763ba --- /dev/null +++ b/iznik-nuxt3/modtools/components/ModHelpTrusted.vue @@ -0,0 +1,20 @@ + + diff --git a/iznik-nuxt3/modtools/components/ModMessage.vue b/iznik-nuxt3/modtools/components/ModMessage.vue index 15c93b46b4..b983ed0fac 100644 --- a/iznik-nuxt3/modtools/components/ModMessage.vue +++ b/iznik-nuxt3/modtools/components/ModMessage.vue @@ -133,6 +133,14 @@ > Pending + + {{ countdownLabel.text }} +
Deadline: end {{ dateonly(message.deadline) }} @@ -901,6 +909,11 @@ const userStore = useUserStore() const message = computed(() => messageStore.byId(props.messageid)) +// --- Auto-approve countdown (A5) --- +// `now` ticks every second (see onMounted) so the countdown label recomputes live. +const now = ref(Date.now()) +let countdownInterval = null + watch( () => props.messageid, async (id) => { @@ -1184,6 +1197,37 @@ const pending = computed(() => { return hasCollection('Pending') }) +// The soonest non-null autoapproveat across all Pending groups for this message. +const soonestAutoapproveat = computed(() => { + if (!message.value?.groups) return null + let best = null + for (const g of message.value.groups) { + if (g.collection === 'Pending' && g.autoapproveat) { + const t = new Date(g.autoapproveat).getTime() + if (best === null || t < best) best = t + } + } + return best +}) + +// Live countdown label + CSS class, recomputed every second via `now`. +const countdownLabel = computed(() => { + const target = soonestAutoapproveat.value + if (target === null) return null + const secsLeft = Math.floor((target - now.value) / 1000) + if (secsLeft <= 0) return { text: 'Auto-approving…', cls: 'text-info' } + const totalMins = Math.floor(secsLeft / 60) + const secs = secsLeft % 60 + if (totalMins >= 60) { + const hrs = Math.floor(totalMins / 60) + return { text: `Auto-approves in ~${hrs}h`, cls: 'text-muted' } + } + return { + text: `Auto-approves in ${totalMins}m ${String(secs).padStart(2, '0')}s`, + cls: 'text-warning fw-bold', + } +}) + const eSubject = computed(() => { if (!message.value) return '' return twem(message.value.subject) @@ -1471,12 +1515,21 @@ onMounted(() => { userStore.fetch(message.value.heldby) } } + + // Per-second ticker for the auto-approve countdown badge. + countdownInterval = setInterval(() => { + now.value = Date.now() + }, 1000) }) onBeforeUnmount(() => { if (message.value) { emit('destroy', message.value.id, props.next) } + if (countdownInterval) { + clearInterval(countdownInterval) + countdownInterval = null + } }) function updateComments() { diff --git a/iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue b/iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue index 017e6bf29b..c08212ff43 100644 --- a/iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue +++ b/iznik-nuxt3/modtools/components/ModSysAdminModerationStats.vue @@ -44,7 +44,7 @@ {{ pct(stats.manualRejected, stats.arrived) }} - Auto-approved (Checked) + Auto-approved (Checked delay + 48h fallback) {{ stats.autoApproved }} {{ pct(stats.autoApproved, stats.arrived) }} diff --git a/iznik-nuxt3/modtools/pages/messages/approved/[[id]]/[[term]].vue b/iznik-nuxt3/modtools/pages/messages/approved/[[id]]/[[term]].vue index 41236251f7..39c75cffd4 100644 --- a/iznik-nuxt3/modtools/pages/messages/approved/[[id]]/[[term]].vue +++ b/iznik-nuxt3/modtools/pages/messages/approved/[[id]]/[[term]].vue @@ -1,6 +1,7 @@