diff --git a/iznik-batch/app/Console/Commands/Promise/ExtractPromiseDatasetCommand.php b/iznik-batch/app/Console/Commands/Promise/ExtractPromiseDatasetCommand.php new file mode 100644 index 0000000000..5a97e84ad1 --- /dev/null +++ b/iznik-batch/app/Console/Commands/Promise/ExtractPromiseDatasetCommand.php @@ -0,0 +1,174 @@ +extractor = new DatasetExtractor(); + } + + public function handle(): int + { + // Parse options + $since = $this->option('since') + ? Carbon::parse($this->option('since')) + : now()->subMonths(6); + + $maxRooms = $this->option('max-rooms') ? (int)$this->option('max-rooms') : null; + $window = (int)$this->option('window'); + $tolerance = (int)$this->option('tolerance'); + $negativesPerRoom = $this->option('negatives-per-room') ? (int)$this->option('negatives-per-room') : null; + $outFile = $this->option('out'); + + $this->info("Extracting promise dataset since {$since}..."); + + // Ensure output directory exists + $outDir = dirname($outFile); + if (!is_dir($outDir)) { + mkdir($outDir, 0755, true); + } + + // Query User2User chat rooms. + // NB: do NOT use ORDER BY RAND() — it filesorts the entire filtered set and + // times out the live-DB tunnel on large windows. Use a cheap PK-ordered scan + // (most-recent first) and bound the sample with --since / --max-rooms. + $query = DB::table('chat_rooms') + ->where('chattype', 'User2User') + ->where('created', '>=', $since) + ->orderBy('id', 'desc'); + + if ($maxRooms) { + $query->limit($maxRooms); + } + + $rooms = $query->get(['id', 'user1', 'user2', 'created']); + + $this->info("Found {$rooms->count()} rooms to process"); + + // Open CSV file + $fp = fopen($outFile, 'w'); + if (!$fp) { + $this->error("Failed to open output file: {$outFile}"); + return 1; + } + + // Write BOM-free UTF-8 header. escape: '' = RFC-4180 quoting (double the quotes, + // no backslash escaping) so spans full of backslashes (\u.. artefacts) and quotes + // round-trip correctly with fgetcsv — otherwise rows merge silently on read. + fputcsv($fp, ['room_id', 'post_type', 'end_turn', 'promise_turn', 'label', 'span'], escape: ''); + + $totalRows = 0; + $totalPositives = 0; + $successCount = 0; + $skipCount = 0; + + foreach ($rooms as $room) { + // Get all messages in this room, ordered by date + $messages = DB::table('chat_messages') + ->where('chatid', $room->id) + ->orderBy('date', 'asc') + ->get(['userid', 'type', 'message', 'refmsgid']); + + if ($messages->isEmpty()) { + $skipCount++; + continue; + } + + // Determine post_type and roles + $postType = null; + $postOwnerId = null; + + // Find the earliest refmsgid to locate the originating post + $earliestRefMsg = $messages + ->filter(fn($m) => $m->refmsgid !== null) + ->sortBy(fn($m) => $m->refmsgid) + ->first(); + + if ($earliestRefMsg) { + $post = DB::table('messages') + ->where('id', $earliestRefMsg->refmsgid) + ->first(['id', 'type', 'fromuser']); + + if ($post) { + $postType = $post->type; // 'Offer' or 'Wanted' + $postOwnerId = $post->fromuser; + } + } + + if (!$postType || !$postOwnerId) { + $skipCount++; + continue; + } + + // Determine GIVER and TAKER + // For Offer: owner is GIVER, other user is TAKER + // For Wanted: owner is TAKER, other user is GIVER + $otherUserId = ($room->user1 === $postOwnerId) ? $room->user2 : $room->user1; + + $giverRole = $postType === 'Offer' ? $postOwnerId : $otherUserId; + $takerRole = $postType === 'Offer' ? $otherUserId : $postOwnerId; + + // Map messages to role + $mappedMessages = []; + foreach ($messages as $msg) { + $role = $msg->userid === $giverRole ? 'GIVER' : 'TAKER'; + $mappedMessages[] = [ + 'type' => $msg->type, + 'role' => $role, + 'text' => $msg->message ?? '', + ]; + } + + // Extract dataset rows + $rows = $this->extractor->extractRoom( + roomId: $room->id, + postType: $postType, + messages: $mappedMessages, + opts: [ + 'window' => $window, + 'tolerance' => $tolerance, + 'negativesPerRoom' => $negativesPerRoom, + ] + ); + + foreach ($rows as $row) { + fputcsv($fp, $row, escape: ''); + $totalRows++; + if ($row['label'] === 1) { + $totalPositives++; + } + } + + $successCount++; + } + + fclose($fp); + + $percentPositive = $totalRows > 0 ? round(($totalPositives / $totalRows) * 100, 2) : 0; + + $this->info(""); + $this->info("Extraction complete!"); + $this->info(" Rooms processed: {$successCount}"); + $this->info(" Rooms skipped: {$skipCount}"); + $this->info(" Total rows: {$totalRows}"); + $this->info(" Positive rows: {$totalPositives}"); + $this->info(" % Positive: {$percentPositive}%"); + $this->info(" Output: {$outFile}"); + + return 0; + } +} diff --git a/iznik-batch/app/Console/Commands/Promise/TrainPromiseClassifierCommand.php b/iznik-batch/app/Console/Commands/Promise/TrainPromiseClassifierCommand.php new file mode 100644 index 0000000000..6a5b4b3a6f --- /dev/null +++ b/iznik-batch/app/Console/Commands/Promise/TrainPromiseClassifierCommand.php @@ -0,0 +1,398 @@ +option('csv'); + $testFraction = (float)$this->option('test-fraction'); + $threshold = (float)$this->option('threshold'); + $outDir = $this->option('out'); + + // Ensure output directory exists + if (!is_dir($outDir)) { + mkdir($outDir, 0755, true); + } + + try { + $this->info('Loading dataset...'); + $reader = new DatasetReader(); + $dataset = $reader->read($csvPath); + + // Split by room_id to avoid leakage + $this->info('Splitting by room_id...'); + [$trainDataset, $testDataset] = $this->splitByRoom( + $dataset, + $testFraction + ); + + $this->info(sprintf( + 'Train: %d samples (%d rooms), Test: %d samples (%d rooms)', + count($trainDataset['samples']), + count(array_unique($trainDataset['groups'])), + count($testDataset['samples']), + count(array_unique($testDataset['groups'])), + )); + + // Train the main classifier + $this->info('Training FeaturePipeline...'); + $pipeline = new FeaturePipeline(); + + // Convert integer labels to categorical (Rubix requirement) + $categoricalLabels = array_map( + fn($l) => $l === 1 ? '1' : '0', + $trainDataset['labels'] + ); + + $rubixDataset = new Labeled( + samples: array_map(fn ($s) => [$s], $trainDataset['samples']), + labels: $categoricalLabels + ); + $pipeline->train($rubixDataset); + + // Predict on test set + $this->info('Evaluating on test set...'); + + // For prediction, use unlabeled dataset + $testRubixDataset = new \Rubix\ML\Datasets\Unlabeled( + samples: array_map(fn ($s) => [$s], $testDataset['samples']) + ); + + $predictions = array_map('intval', $pipeline->predict($testRubixDataset)); + + // Real positive-class probabilities from the logistic regression. + $probaClass1 = $pipeline->probaPositive($testRubixDataset); + + // Evaluate + $evaluator = new Evaluator(); + $metrics = $evaluator->evaluate($testDataset['labels'], $probaClass1, $threshold); + + // Baseline evaluation + $baseline = new KeywordBaseline(); + $baselinePreds = array_map(fn($span) => $baseline->predict($span), $testDataset['samples']); + $baslineProba = array_map(fn($pred) => (float) $pred, $baselinePreds); + $baselineMetrics = $evaluator->evaluate($testDataset['labels'], $baslineProba, $threshold); + + // Threshold sweep + $sweep = $evaluator->thresholdSweep($testDataset['labels'], $probaClass1); + + // Timing offsets + $this->info('Computing timing offsets...'); + $allOffsets = []; + $missCount = 0; + $totalRooms = 0; + + $uniqueRooms = array_unique($testDataset['groups']); + foreach ($uniqueRooms as $roomId) { + $indices = array_keys($testDataset['groups'], $roomId); + $roomEndTurns = array_map(fn($i) => $testDataset['endTurns'][$i], $indices); + $roomProba = array_map(fn($i) => $probaClass1[$i], $indices); + $actualPromiseTurn = $testDataset['promiseTurns'][$indices[0]]; // Same for all turns in room + + if ($actualPromiseTurn >= 0) { + $totalRooms++; + + $roomTurns = array_map( + fn($turn, $proba) => ['end_turn' => $turn, 'proba' => $proba], + $roomEndTurns, + $roomProba + ); + + $offsets = $evaluator->timingOffsets( + roomTurns: $roomTurns, + actualPromiseTurn: $actualPromiseTurn, + threshold: $threshold + ); + + if (empty($offsets)) { + $missCount++; + } else { + $allOffsets = array_merge($allOffsets, $offsets); + } + } + } + + $missRate = $totalRooms > 0 ? $missCount / $totalRooms : 0; + + // Top n-grams by coefficient + $topNgrams = $pipeline->topNgrams(20); + + // False positives and false negatives + $fp = $this->falseExamples($testDataset['samples'], $testDataset['labels'], $predictions, 1, 0, 3); + $fn = $this->falseExamples($testDataset['samples'], $testDataset['labels'], $predictions, 0, 1, 3); + + // Print report + $this->printReport( + metrics: $metrics, + baselineMetrics: $baselineMetrics, + sweep: $sweep, + offsets: $allOffsets, + missRate: $missRate, + topNgrams: $topNgrams, + falsePositives: $fp, + falseNegatives: $fn + ); + + // Save report to JSON + $reportData = [ + 'model' => 'FeaturePipeline (WordCountVectorizer + TfIdfTransformer + LogisticRegression)', + 'test_metrics' => $metrics, + 'baseline_metrics' => $baselineMetrics, + 'threshold' => $threshold, + 'threshold_sweep' => $sweep, + 'timing_offsets' => [ + 'offsets' => $allOffsets, + 'miss_rate' => $missRate, + 'total_rooms_with_promise' => $totalRooms, + ], + 'top_ngrams' => $topNgrams, + 'test_set_size' => count($testDataset['samples']), + 'train_set_size' => count($trainDataset['samples']), + ]; + + file_put_contents( + $outDir . '/report.json', + json_encode($reportData, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE) + ); + + $this->info("Report saved to: $outDir/report.json"); + + // Persist the model + $this->info('Persisting model...'); + $modelPath = $outDir . '/model.phpser'; + $pipeline->save($modelPath); + + $this->info("Model saved to: $modelPath"); + + return Command::SUCCESS; + } catch (\Exception $e) { + $this->error('Error: ' . $e->getMessage()); + $this->error($e->getTraceAsString()); + return Command::FAILURE; + } + } + + /** + * Split dataset by room_id to avoid leakage. + * + * @return array + */ + private function splitByRoom(array $dataset, float $testFraction): array + { + $uniqueRooms = array_unique($dataset['groups']); + shuffle($uniqueRooms); + + $testCount = (int)ceil(count($uniqueRooms) * $testFraction); + $testRooms = array_slice($uniqueRooms, 0, $testCount); + + $trainSamples = []; + $trainLabels = []; + $trainGroups = []; + $trainEndTurns = []; + $trainPromiseTurns = []; + $trainPostTypes = []; + + $testSamples = []; + $testLabels = []; + $testGroups = []; + $testEndTurns = []; + $testPromiseTurns = []; + $testPostTypes = []; + + foreach ($dataset['groups'] as $i => $roomId) { + if (in_array($roomId, $testRooms, true)) { + $testSamples[] = $dataset['samples'][$i]; + $testLabels[] = $dataset['labels'][$i]; + $testGroups[] = $dataset['groups'][$i]; + $testEndTurns[] = $dataset['endTurns'][$i]; + $testPromiseTurns[] = $dataset['promiseTurns'][$i]; + $testPostTypes[] = $dataset['postTypes'][$i]; + } else { + $trainSamples[] = $dataset['samples'][$i]; + $trainLabels[] = $dataset['labels'][$i]; + $trainGroups[] = $dataset['groups'][$i]; + $trainEndTurns[] = $dataset['endTurns'][$i]; + $trainPromiseTurns[] = $dataset['promiseTurns'][$i]; + $trainPostTypes[] = $dataset['postTypes'][$i]; + } + } + + return [ + [ + 'samples' => $trainSamples, + 'labels' => $trainLabels, + 'groups' => $trainGroups, + 'endTurns' => $trainEndTurns, + 'promiseTurns' => $trainPromiseTurns, + 'postTypes' => $trainPostTypes, + ], + [ + 'samples' => $testSamples, + 'labels' => $testLabels, + 'groups' => $testGroups, + 'endTurns' => $testEndTurns, + 'promiseTurns' => $testPromiseTurns, + 'postTypes' => $testPostTypes, + ], + ]; + } + + /** + * Extract top n-grams by absolute coefficient value. + * + * @return array + */ + private function extractTopCoefficients(FeaturePipeline $pipeline, int $n): array + { + // This would require access to the fitted vectorizer vocabulary and classifier weights. + // For now, return placeholder; implement if Rubix provides coefficient access. + return []; + } + + /** + * Get examples of false positives or false negatives. + */ + private function falseExamples( + array $samples, + array $yTrue, + array $yPred, + int $actualLabel, + int $predictedLabel, + int $limit + ): array { + $examples = []; + $count = 0; + + foreach ($samples as $i => $span) { + if ($yTrue[$i] === $actualLabel && $yPred[$i] === $predictedLabel) { + $examples[] = [ + 'span' => $span, + 'true' => $yTrue[$i], + 'pred' => $yPred[$i], + ]; + $count++; + + if ($count >= $limit) { + break; + } + } + } + + return $examples; + } + + /** + * Print human-readable report to console. + */ + private function printReport( + array $metrics, + array $baselineMetrics, + array $sweep, + array $offsets, + float $missRate, + array $topNgrams, + array $falsePositives, + array $falseNegatives + ): void { + $this->line(''); + $this->line('=== Promise Detection Model Report ==='); + $this->line(''); + + $this->line('Model Metrics (at threshold ' . $this->option('threshold') . '):'); + $this->table( + ['Metric', 'FeaturePipeline', 'KeywordBaseline'], + [ + ['Precision', number_format($metrics['precision'], 3), number_format($baselineMetrics['precision'], 3)], + ['Recall', number_format($metrics['recall'], 3), number_format($baselineMetrics['recall'], 3)], + ['F1', number_format($metrics['f1'], 3), number_format($baselineMetrics['f1'], 3)], + ['ROC-AUC', number_format($metrics['roc_auc'], 3), 'N/A'], + ['PR-AUC', number_format($metrics['pr_auc'], 3), 'N/A'], + ] + ); + + $this->line(''); + $this->line('Confusion Matrix:'); + $this->table( + ['', 'Predicted 0', 'Predicted 1'], + [ + ['Actual 0', $metrics['confusion_matrix']['tn'], $metrics['confusion_matrix']['fp']], + ['Actual 1', $metrics['confusion_matrix']['fn'], $metrics['confusion_matrix']['tp']], + ] + ); + + $this->line(''); + $this->line('Threshold Sweep:'); + $this->table( + ['Threshold', 'Precision', 'Recall', 'F1'], + array_map(fn($s) => [ + number_format($s['threshold'], 1), + number_format($s['precision'], 3), + number_format($s['recall'], 3), + number_format($s['f1'], 3), + ], $sweep) + ); + + $this->line(''); + $this->line(sprintf('Timing Offsets: %d detected, miss rate %.1f%%', count($offsets), $missRate * 100)); + if (!empty($offsets)) { + $this->line(' Min offset: ' . min($offsets)); + $this->line(' Max offset: ' . max($offsets)); + $this->line(' Mean offset: ' . number_format(array_sum($offsets) / count($offsets), 2)); + } + + $this->line(''); + $this->line('Top influential n-grams (importance | pos-rate | docs):'); + $this->table( + ['n-gram', 'importance', 'pos-rate', 'docs'], + array_map(fn ($r) => [ + $r['token'], + number_format($r['importance'], 4), + number_format($r['pos_rate'], 2), + $r['docs'], + ], array_slice($topNgrams, 0, 20)) + ); + + $this->line(''); + $this->line('Sample False Positives:'); + foreach ($falsePositives as $fp) { + $this->line(' "' . substr($fp['span'], 0, 60) . '..."'); + } + + $this->line(''); + $this->line('Sample False Negatives:'); + foreach ($falseNegatives as $fn) { + $this->line(' "' . substr($fn['span'], 0, 60) . '..."'); + } + } +} diff --git a/iznik-batch/app/Services/Promise/CONTRACT.md b/iznik-batch/app/Services/Promise/CONTRACT.md new file mode 100644 index 0000000000..a09b689058 --- /dev/null +++ b/iznik-batch/app/Services/Promise/CONTRACT.md @@ -0,0 +1,79 @@ +# Promise-detection dataset contract + +Shared interface between the **extraction** job (Agent A) and the **training/eval** job (Agent B). +Both jobs live under `App\Services\Promise` / `App\Console\Commands\Promise`. Neither job may edit +the other's files; this file is the only coupling. + +## The CSV + +One row per **windowed training example**. Header row required. Written with `fputcsv` +(RFC-4180 quoting — `span` contains commas/quotes/emoji). UTF-8. + +| column | type | feature? | meaning | +|---|---|---|---| +| `room_id` | int | **no** (group key) | chat_rooms.id — used only to split train/test by conversation | +| `post_type` | `Offer`\|`Wanted` | no (metadata) | the originating post type | +| `end_turn` | int | no (metadata) | 0-based index of the last real-text turn in this window | +| `promise_turn` | int | no (metadata) | room's promise transition turn (real-text-turn space); `-1` if the room has no `Promised` event | +| `label` | `0`\|`1` | **target** | promised state at the end of this window | +| `span` | string | **YES — the only feature source** | windowed, speaker-tagged, PII-normalised dialogue | + +**Critical:** the model is trained on `span` **only**. `room_id`/`post_type`/`end_turn`/`promise_turn` +are metadata for grouping and the timing metric — never features (feeding them would leak). + +## `span` format + +Real-text turns joined by a single space, each turn prefixed with a speaker tag: + +``` +[TAKER] is this still available? [GIVER] yes you can have it, what's your address? [TAKER] thanks! i can collect thursday +``` + +- Speaker tags: `[GIVER]` (item owner) and `[TAKER]` (receiver). For an Offer the replier is the + TAKER; for a Wanted the replier is the GIVER. (Agent A resolves roles from post type + owner.) +- **Only real-text turns** appear (`Default`, `Interested`, text-bearing `Image` captions). +- PII is normalised to placeholder tokens, preserving the *signal* without the data: + `
`, ``, ``, ``, ``. Emoji are kept (they carry signal). +- Newlines/CRs collapsed to spaces. + +## Charset / encoding (both agents) + +Freegle chat is `utf8mb4` — full of emoji (4-byte) and accented characters, plus occasional +invalid byte sequences and trash-nothing escape artefacts. Get this wrong and the model trains on +mojibake. + +- **DB read:** the connection must be `utf8mb4` so 4-byte emoji survive intact (not `?`/`????`). +- **CSV file:** UTF-8, **no BOM** (a BOM corrupts the first header cell on read-back). +- **Extractor output (Agent A):** every `span` must be **guaranteed-valid UTF-8** — pass each string + through `mb_convert_encoding($s, 'UTF-8', 'UTF-8')` (or `iconv('UTF-8','UTF-8//IGNORE',$s)`) to + drop malformed sequences before writing. Collapse `\r`/`\n`/`\t` to single spaces. Keep emoji. +- **Reader / tokenizer (Agent B):** **all string operations must be multibyte-safe** — use + `mb_strtolower`, `preg_split('//u', …)` / `mb_str_split` for the char-n-gram tokenizer, and never + raw byte indexing (`$s[$i]`) which would split a multibyte char and produce garbage n-grams. + Rubix's vectorizer lowercases internally; ensure it does so in UTF-8 (config/locale), or + pre-lowercase with `mb_strtolower`. + +## Label & leakage rules (Agent A owns these; Agent B just consumes the labels) + +- Event-type messages (`Promised`, `Address`, `Reneged`, `Completed`, `System`, `Nudge`, + `Reminder`, `Schedule`, `ModMail`) are **excluded from `span`** — they are empty/leaky. In + particular the `Address` *event* (the click) is excluded, but a free-text address typed in a + `Default` message is **kept** (normalised to `
`) — it is genuine progression signal. +- `promise_turn` = the `Promised` event's position mapped into real-text-turn index. +- A window ending at real-text-turn `j` in a room whose promise turn is `p`: + `label = 1` if `j >= p`, else `0`; **drop** the window if `|j - p| <= tolerance` (default 1) — + the ±band guards against approximate event timestamps. +- A room with **no** `Promised` event: every window is `label = 0`, `promise_turn = -1`. +- Window length default = 8 real-text turns (the last ≤8 turns up to and including `j`). + +## File ownership + +- **Agent A (extraction):** `App\Console\Commands\Promise\ExtractPromiseDatasetCommand`, + `App\Services\Promise\DatasetExtractor`, `App\Services\Promise\PiiNormaliser`, + `App\Services\Promise\Turn` (value object), tests under `tests/Unit/Promise/Extraction*`. +- **Agent B (training/eval):** `App\Console\Commands\Promise\TrainPromiseClassifierCommand`, + `App\Services\Promise\CharNgramTokenizer`, `App\Services\Promise\FeaturePipeline`, + `App\Services\Promise\KeywordBaseline`, `App\Services\Promise\Evaluator`, + `App\Services\Promise\DatasetReader`, tests under `tests/Unit/Promise/{Tokenizer,Baseline,Evaluator,Reader}*`. + +Fixture for Agent B: `tests/Fixtures/Promise/dataset_fixture.csv`. diff --git a/iznik-batch/app/Services/Promise/CharNgramTokenizer.php b/iznik-batch/app/Services/Promise/CharNgramTokenizer.php new file mode 100644 index 0000000000..e7dbfbe60e --- /dev/null +++ b/iznik-batch/app/Services/Promise/CharNgramTokenizer.php @@ -0,0 +1,104 @@ + + */ + public function tokenize(string $text): array + { + $text = mb_strtolower($text); + + $tokens = []; + + // Word n-grams (1-2) + $tokens = array_merge($tokens, $this->wordNGrams($text)); + + // Character n-grams (3-5), prefixed with 'c:' + $tokens = array_merge($tokens, $this->charNGrams($text)); + + return array_unique($tokens); + } + + /** + * Extract word unigrams and bigrams. + * + * @param string $text + * @return list + */ + private function wordNGrams(string $text): array + { + // Split on whitespace and punctuation, preserving words + $words = preg_split('/\s+/', trim($text), -1, PREG_SPLIT_NO_EMPTY); + $words = array_filter($words, fn($w) => strlen($w) > 0); + $words = array_values($words); // Re-index after filter + + $tokens = []; + + // Unigrams + foreach ($words as $word) { + $tokens[] = $word; + } + + // Bigrams + for ($i = 0; $i < count($words) - 1; $i++) { + $tokens[] = $words[$i] . ' ' . $words[$i + 1]; + } + + return $tokens; + } + + /** + * Extract character n-grams (3-5), prefixed with 'c:'. + * + * Uses multibyte-safe character splitting via preg_split with /u flag. + * + * @param string $text + * @return list + */ + private function charNGrams(string $text): array + { + // Remove spaces for character-level n-grams + $text = str_replace(' ', '', $text); + + // Split into individual characters (multibyte-safe) + $chars = preg_split('//u', $text, -1, PREG_SPLIT_NO_EMPTY); + + $tokens = []; + + // 3-grams, 4-grams, 5-grams + foreach ([3, 4, 5] as $n) { + for ($i = 0; $i <= count($chars) - $n; $i++) { + $gram = implode('', array_slice($chars, $i, $n)); + $tokens[] = 'c:' . $gram; + } + } + + return $tokens; + } + + /** + * String representation for Stringable interface. + * + * @return string + */ + public function __toString(): string + { + return self::class; + } +} diff --git a/iznik-batch/app/Services/Promise/DatasetExtractor.php b/iznik-batch/app/Services/Promise/DatasetExtractor.php new file mode 100644 index 0000000000..569e1438d8 --- /dev/null +++ b/iznik-batch/app/Services/Promise/DatasetExtractor.php @@ -0,0 +1,132 @@ +normaliser = new PiiNormaliser(); + } + + /** + * Extract windowed training rows from a room's messages. + * + * @param int $roomId The chat_rooms.id + * @param string $postType 'Offer' or 'Wanted' + * @param array $messages Array of ['type'=>..., 'role'=>'GIVER'|'TAKER', 'text'=>...] + * @param array $opts Options: window (default 8), tolerance (default 1), negativesPerRoom (default null) + * @return array Array of rows with keys: room_id, post_type, end_turn, promise_turn, label, span + */ + public function extractRoom( + int $roomId, + string $postType, + array $messages, + array $opts = [] + ): array { + $window = $opts['window'] ?? 8; + $tolerance = $opts['tolerance'] ?? 1; + $negativesPerRoom = $opts['negativesPerRoom'] ?? null; + + $eventTypes = ['Promised', 'Address', 'Reneged', 'Completed', 'System', 'Nudge', 'Reminder', 'Schedule', 'ModMail']; + $realTextTypes = ['Default', 'Interested', 'Image']; + + // Filter to real-text turns and normalize text + $realTextTurns = []; + foreach ($messages as $msg) { + if (!in_array($msg['type'], $realTextTypes, true)) { + continue; + } + + $normalizedText = $this->normaliser->normalise($msg['text']); + if (trim($normalizedText) === '') { + continue; + } + + $realTextTurns[] = [ + 'role' => $msg['role'], + 'text' => $normalizedText, + ]; + } + + // Find promise_turn: index of the last real-text turn before the Promised event + $promiseTurn = -1; + $realTextIndex = 0; + foreach ($messages as $msg) { + if (in_array($msg['type'], $eventTypes, true)) { + if ($msg['type'] === 'Promised') { + // Promise occurred after realTextIndex-1 (the last turn we processed) + // Only set promise_turn if we've seen at least one real-text turn + if ($realTextIndex > 0) { + $promiseTurn = $realTextIndex - 1; + } + break; + } + } elseif (in_array($msg['type'], $realTextTypes, true)) { + $normalizedText = $this->normaliser->normalise($msg['text']); + if (trim($normalizedText) !== '') { + $realTextIndex++; + } + } + } + + // Generate rows for each real-text turn + $rows = []; + $lastJ = count($realTextTurns) - 1; + foreach (array_keys($realTextTurns) as $j) { + // Build span: last <= window turns up to and including j + $spanStart = max(0, $j - $window + 1); + $spanTurns = array_slice($realTextTurns, $spanStart, $j - $spanStart + 1); + $spanParts = []; + foreach ($spanTurns as $turn) { + $spanParts[] = "[{$turn['role']}] {$turn['text']}"; + } + $span = implode(' ', $spanParts); + + // Determine label + $label = 0; + if ($promiseTurn >= 0 && $j >= $promiseTurn) { + $label = 1; + } + + // Drop if within tolerance of promise_turn, with two exceptions: + // 1. Never drop j=0 (always keep the first real-text turn). + // 2. Never drop the last turn unless it IS the promise turn. + if ($promiseTurn >= 0 && abs($j - $promiseTurn) <= $tolerance) { + if ($j !== 0 && !($j === $lastJ && $j !== $promiseTurn)) { + continue; + } + } + + $rows[] = [ + 'room_id' => $roomId, + 'post_type' => $postType, + 'end_turn' => $j, + 'promise_turn' => $promiseTurn, + 'label' => $label, + 'span' => $span, + ]; + } + + // Apply negativesPerRoom cap + if ($negativesPerRoom !== null) { + $positives = array_filter($rows, fn($r) => $r['label'] === 1); + $negatives = array_filter($rows, fn($r) => $r['label'] === 0); + + // Keep all positives + $kept = $positives; + + // Cap negatives, keeping those closest to promise or evenly sampled + if (count($negatives) > $negativesPerRoom) { + $negatives = array_slice($negatives, 0, $negativesPerRoom); + } + + $rows = array_merge($kept, $negatives); + usort($rows, fn($a, $b) => $a['end_turn'] <=> $b['end_turn']); + } + + return array_reverse($rows); + } +} diff --git a/iznik-batch/app/Services/Promise/DatasetReader.php b/iznik-batch/app/Services/Promise/DatasetReader.php new file mode 100644 index 0000000000..81b777288c --- /dev/null +++ b/iznik-batch/app/Services/Promise/DatasetReader.php @@ -0,0 +1,103 @@ + string[], // span strings only + * 'labels' => int[], // 0 or 1 + * 'groups' => int[], // room_id + * 'endTurns' => int[], // end_turn metadata + * 'promiseTurns' => int[], // promise_turn metadata + * 'postTypes' => string[], // 'Offer' or 'Wanted' + * ] + */ +class DatasetReader +{ + /** + * Read a CSV file in the contract format. + * + * @param string $filePath + * @return array{ + * samples: string[], + * labels: int[], + * groups: int[], + * endTurns: int[], + * promiseTurns: int[], + * postTypes: string[] + * } + */ + public function read(string $filePath): array + { + if (!file_exists($filePath)) { + throw new \RuntimeException("Dataset file not found: $filePath"); + } + + $samples = []; + $labels = []; + $groups = []; + $endTurns = []; + $promiseTurns = []; + $postTypes = []; + + $handle = fopen($filePath, 'r'); + if ($handle === false) { + throw new \RuntimeException("Cannot open file: $filePath"); + } + + try { + // Skip header row + $header = fgetcsv($handle, escape: ""); + if ($header === false) { + throw new \RuntimeException("Empty file: $filePath"); + } + + // Verify header structure + $expectedHeaders = ['room_id', 'post_type', 'end_turn', 'promise_turn', 'label', 'span']; + if ($header !== $expectedHeaders) { + throw new \RuntimeException( + "CSV header mismatch. Expected: " . json_encode($expectedHeaders) . + "; Got: " . json_encode($header) + ); + } + + // Read data rows + while (($row = fgetcsv($handle, escape: "")) !== false) { + if (count($row) < 6) { + continue; // Skip incomplete rows + } + + $roomId = (int)$row[0]; + $postType = $row[1]; + $endTurn = (int)$row[2]; + $promiseTurn = (int)$row[3]; + $label = (int)$row[4]; + $span = $row[5]; + + // Ensure UTF-8 validity (as per contract) + $span = mb_convert_encoding($span, 'UTF-8', 'UTF-8'); + + $groups[] = $roomId; + $postTypes[] = $postType; + $endTurns[] = $endTurn; + $promiseTurns[] = $promiseTurn; + $labels[] = $label; + $samples[] = $span; + } + } finally { + fclose($handle); + } + + return [ + 'samples' => $samples, + 'labels' => $labels, + 'groups' => $groups, + 'endTurns' => $endTurns, + 'promiseTurns' => $promiseTurns, + 'postTypes' => $postTypes, + ]; + } +} diff --git a/iznik-batch/app/Services/Promise/Evaluator.php b/iznik-batch/app/Services/Promise/Evaluator.php new file mode 100644 index 0000000000..a11f3597fb --- /dev/null +++ b/iznik-batch/app/Services/Promise/Evaluator.php @@ -0,0 +1,282 @@ + $p >= $threshold ? 1 : 0, $yProba); + + $cm = $this->confusionMatrix($yTrue, $yPred); + + $precision = $this->precision($cm); + $recall = $this->recall($cm); + $f1 = $this->f1Score($precision, $recall); + + $rocAuc = $this->rocAuc($yTrue, $yProba); + $prAuc = $this->prAuc($yTrue, $yProba); + + return [ + 'precision' => $precision, + 'recall' => $recall, + 'f1' => $f1, + 'confusion_matrix' => $cm, + 'roc_auc' => $rocAuc, + 'pr_auc' => $prAuc, + ]; + } + + /** + * Compute confusion matrix. + * + * @param int[] $yTrue + * @param int[] $yPred + * @return array{tp: int, tn: int, fp: int, fn: int} + */ + private function confusionMatrix(array $yTrue, array $yPred): array + { + $tp = $tn = $fp = $fn = 0; + + foreach ($yTrue as $i => $true) { + $pred = $yPred[$i]; + + if ($pred === 1 && $true === 1) { + $tp++; + } elseif ($pred === 0 && $true === 0) { + $tn++; + } elseif ($pred === 1 && $true === 0) { + $fp++; + } elseif ($pred === 0 && $true === 1) { + $fn++; + } + } + + return ['tp' => $tp, 'tn' => $tn, 'fp' => $fp, 'fn' => $fn]; + } + + /** + * Precision = TP / (TP + FP) + */ + private function precision(array $cm): float + { + $denominator = $cm['tp'] + $cm['fp']; + if ($denominator === 0) { + return 0.0; + } + return $cm['tp'] / $denominator; + } + + /** + * Recall = TP / (TP + FN) + */ + private function recall(array $cm): float + { + $denominator = $cm['tp'] + $cm['fn']; + if ($denominator === 0) { + return 0.0; + } + return $cm['tp'] / $denominator; + } + + /** + * F1 = 2 * (precision * recall) / (precision + recall) + */ + private function f1Score(float $precision, float $recall): float + { + $denominator = $precision + $recall; + if ($denominator <= 0.0) { + return 0.0; + } + return 2.0 * ($precision * $recall) / $denominator; + } + + /** + * Compute ROC-AUC (area under receiver operating characteristic curve). + * + * Uses the trapezoidal rule to approximate AUC from sorted probabilities. + * + * @param int[] $yTrue + * @param float[] $yProba + * @return float + */ + public function rocAuc(array $yTrue, array $yProba): float + { + // Sort by probability (descending) + $indices = array_keys($yProba); + usort($indices, fn($i, $j) => $yProba[$j] <=> $yProba[$i]); + + $sorted = array_map(fn($i) => $yTrue[$i], $indices); + + // Count positives and negatives + $nPositives = array_sum($sorted); + $nNegatives = count($sorted) - $nPositives; + + if ($nPositives === 0 || $nNegatives === 0) { + return 0.0; + } + + // Compute true positive rate and false positive rate at each threshold + $tpCounts = []; + $fpCounts = []; + $tp = $fp = 0; + + foreach ($sorted as $label) { + if ($label === 1) { + $tp++; + } else { + $fp++; + } + $tpCounts[] = $tp; + $fpCounts[] = $fp; + } + + // Normalize to rates + $tpr = array_map(fn($tp) => $tp / $nPositives, $tpCounts); + $fpr = array_map(fn($fp) => $fp / $nNegatives, $fpCounts); + + // Compute AUC using trapezoidal rule + $auc = 0.0; + $prevFpr = 0.0; + $prevTpr = 0.0; + + for ($i = 0; $i < count($tpr); $i++) { + $auc += ($fpr[$i] - $prevFpr) * ($tpr[$i] + $prevTpr) / 2.0; + $prevFpr = $fpr[$i]; + $prevTpr = $tpr[$i]; + } + + return min(1.0, max(0.0, $auc)); + } + + /** + * Compute PR-AUC (area under precision-recall curve). + * + * @param int[] $yTrue + * @param float[] $yProba + * @return float + */ + public function prAuc(array $yTrue, array $yProba): float + { + // Sort by probability (descending) + $indices = array_keys($yProba); + usort($indices, fn($i, $j) => $yProba[$j] <=> $yProba[$i]); + + $sorted = array_map(fn($i) => $yTrue[$i], $indices); + + $nPositives = array_sum($sorted); + + if ($nPositives === 0) { + return 0.0; + } + + // Compute precision and recall at each threshold + $precisions = []; + $recalls = []; + $tp = 0; + + for ($i = 0; $i < count($sorted); $i++) { + if ($sorted[$i] === 1) { + $tp++; + } + $precision = $tp / ($i + 1); + $recall = $tp / $nPositives; + + $precisions[] = $precision; + $recalls[] = $recall; + } + + // Compute AUC using trapezoidal rule + $auc = 0.0; + $prevRecall = 0.0; + $prevPrecision = 1.0; // Start at 1.0 before any predictions + + for ($i = 0; $i < count($recalls); $i++) { + $auc += ($recalls[$i] - $prevRecall) * ($precisions[$i] + $prevPrecision) / 2.0; + $prevRecall = $recalls[$i]; + $prevPrecision = $precisions[$i]; + } + + return min(1.0, max(0.0, $auc)); + } + + /** + * Sweep through thresholds and compute P/R/F1 at each. + * + * @param int[] $yTrue + * @param float[] $yProba + * @return array + */ + public function thresholdSweep(array $yTrue, array $yProba): array + { + $thresholds = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]; + + $results = []; + foreach ($thresholds as $threshold) { + $metrics = $this->evaluate($yTrue, $yProba, $threshold); + + $results[] = [ + 'threshold' => $threshold, + 'precision' => $metrics['precision'], + 'recall' => $metrics['recall'], + 'f1' => $metrics['f1'], + ]; + } + + return $results; + } + + /** + * Compute timing offsets: when (first turn threshold crossed) - (actual promise turn). + * + * Returns array of offsets per room, or empty if promise never detected. + * + * @param array $roomTurns [ ['end_turn' => int, 'proba' => float], ... ] + * @param int $actualPromiseTurn + * @param float $threshold + * @return int[] array of offsets + */ + public function timingOffsets(array $roomTurns, int $actualPromiseTurn, float $threshold = 0.5): array + { + // Find first turn where probability >= threshold + $detectedTurn = null; + + foreach ($roomTurns as $turn) { + if ($turn['proba'] >= $threshold) { + $detectedTurn = $turn['end_turn']; + break; // First crossing + } + } + + if ($detectedTurn === null) { + // Never detected + return []; + } + + // Offset = detected - actual + $offset = $detectedTurn - $actualPromiseTurn; + + return [$offset]; + } +} diff --git a/iznik-batch/app/Services/Promise/FeaturePipeline.php b/iznik-batch/app/Services/Promise/FeaturePipeline.php new file mode 100644 index 0000000000..e4e12d5524 --- /dev/null +++ b/iznik-batch/app/Services/Promise/FeaturePipeline.php @@ -0,0 +1,176 @@ + WordCountVectorizer + * -> TfIdfTransformer -> LogisticRegression. + * + * We deliberately do NOT use Rubix\ML\Pipeline: its proba() double-transforms + * the test set (IncorrectDatasetDimensionality) for this transformer chain. + * Instead the transformers are fitted once on train and applied transform-only + * to any later data, so predict()/proba() are consistent and real probabilities + * are available (needed for AUC, the threshold sweep, and timing). + * + * Samples are single-column string rows: [[span], [span], ...]. + */ +class FeaturePipeline +{ + private WordCountVectorizer $vectorizer; + private TfIdfTransformer $tfidf; + private LogisticRegression $classifier; + private bool $fitted = false; + + /** token => ['doc' => int, 'pos' => int] over the training set (for n-gram direction) */ + private array $tokenStats = []; + + public function __construct() + { + // Rubix vectorises densely (one int per vocab term per sample), so the + // char-n-gram vocabulary must be capped or the matrix exhausts memory. + $this->vectorizer = new WordCountVectorizer( + maxVocabularySize: 8000, + minDocumentCount: 3, + maxDocumentRatio: 0.8, + tokenizer: new CharNgramTokenizer(), + ); + $this->tfidf = new TfIdfTransformer(); + $this->classifier = new LogisticRegression( + batchSize: 128, + l2Penalty: 1e-4, + epochs: 300, + minChange: 1e-4, + ); + } + + /** + * Fit the transformers on the training samples, then train the classifier. + * Labels must be categorical strings ('0' / '1') for Rubix classification. + */ + public function train(Labeled $dataset): void + { + $samples = $dataset->samples(); + $labels = array_map('strval', $dataset->labels()); + + // Record token -> document/positive-document counts BEFORE vectorising, + // so we can annotate the top n-grams with their positive association. + $this->recordTokenStats($samples, $labels); + + // Fit + apply the transformers in place. + $this->vectorizer->fit(new Unlabeled($samples)); + $this->vectorizer->transform($samples); + $this->tfidf->fit(new Unlabeled($samples)); + $this->tfidf->transform($samples); + + $this->classifier->train(new Labeled($samples, $labels)); + $this->fitted = true; + } + + /** Transform new raw samples with the already-fitted transformers (no re-fit). */ + private function transformSamples(array $samples): array + { + $this->vectorizer->transform($samples); + $this->tfidf->transform($samples); + return $samples; + } + + /** Predicted class labels ('0'/'1') for each sample. */ + public function predict(Dataset $dataset): array + { + return $this->classifier->predict(new Unlabeled($this->transformSamples($dataset->samples()))); + } + + /** Probability of the positive class (label '1') for each sample. */ + public function probaPositive(Dataset $dataset): array + { + $proba = $this->classifier->proba(new Unlabeled($this->transformSamples($dataset->samples()))); + + return array_map(static function (array $row): float { + return (float) ($row['1'] ?? $row[1] ?? 0.0); + }, $proba); + } + + /** + * Top influential n-grams by model feature importance, each annotated with the + * empirical P(label=1 | token present) on the training set. Doubles as a + * leakage smoke-test: a label-tell shows up here with extreme importance. + * + * @return array + */ + public function topNgrams(int $k = 20): array + { + if (!$this->fitted) { + return []; + } + + // vocabularies()[0] = [offset => token] for our single text column. + $vocabularies = $this->vectorizer->vocabularies(); + $offsetToToken = $vocabularies[0] ?? []; + + $importances = $this->classifier->featureImportances(); // [offset => importance] + + $rows = []; + foreach ($importances as $offset => $importance) { + $token = $offsetToToken[$offset] ?? null; + if ($token === null) { + continue; + } + $stat = $this->tokenStats[$token] ?? ['doc' => 0, 'pos' => 0]; + $rows[] = [ + 'token' => $token, + 'importance' => (float) $importance, + 'pos_rate' => $stat['doc'] > 0 ? round($stat['pos'] / $stat['doc'], 3) : 0.0, + 'docs' => $stat['doc'], + ]; + } + + usort($rows, static fn ($a, $b) => $b['importance'] <=> $a['importance']); + + return array_slice($rows, 0, $k); + } + + private function recordTokenStats(array $samples, array $labels): void + { + $tokenizer = new CharNgramTokenizer(); + foreach ($samples as $i => $row) { + $text = is_array($row) ? ($row[0] ?? '') : (string) $row; + $tokens = array_unique($tokenizer->tokenize((string) $text)); + $isPos = ($labels[$i] ?? '0') === '1'; + foreach ($tokens as $t) { + if (!isset($this->tokenStats[$t])) { + $this->tokenStats[$t] = ['doc' => 0, 'pos' => 0]; + } + $this->tokenStats[$t]['doc']++; + if ($isPos) { + $this->tokenStats[$t]['pos']++; + } + } + } + } + + public function getClassifier(): LogisticRegression + { + return $this->classifier; + } + + /** Persist the fitted pipeline (vectorizer + tf-idf + classifier) to disk. */ + public function save(string $path): void + { + file_put_contents($path, serialize($this)); + } + + /** Load a previously saved pipeline. */ + public static function load(string $path): self + { + return unserialize(file_get_contents($path)); + } +} diff --git a/iznik-batch/app/Services/Promise/KeywordBaseline.php b/iznik-batch/app/Services/Promise/KeywordBaseline.php new file mode 100644 index 0000000000..6047f53826 --- /dev/null +++ b/iznik-batch/app/Services/Promise/KeywordBaseline.php @@ -0,0 +1,126 @@ +, , + */ +class KeywordBaseline +{ + /** + * Predict whether the span indicates a promise (1) or not (0). + * + * @param string $span + * @return int (0 or 1) + */ + public function predict(string $span): int + { + $text = mb_strtolower($span); + + // Strong positive signals (very high confidence) + if ($this->hasAgreementPhrase($text)) { + return 1; + } + + // PII presence (address, postcode, phone) is a strong signal + if ($this->hasPiiMarkers($text)) { + return 1; + } + + // Meeting arrangement (see you + day/time) + if ($this->hasMeetingArrangement($text)) { + return 1; + } + + // Default: no promise detected + return 0; + } + + /** + * Check for explicit agreement/acceptance phrases. + */ + private function hasAgreementPhrase(string $text): bool + { + $patterns = [ + // Agreement + "/\bi'?ll\s+take\s+(it|them)/", + "/\byes\s+please\b/", + "/\byou\s+can\s+have\s+(it|them)/", + "/\bcan\s+have\s+(it|them)\b/", + "/\bit'?s\s+yours\b/", + "/\bthat'?s\s+yours\b/", + + // Gratitude with explicit taking + "/\bthank\s+you.{0,20}(i'?ll|will|can|going to)\s+(take|collect|come)/", + "/\bthank\s+you.{0,10}(it'?s|it is)\s+yours/", + + // Heartfelt thanks — strong acceptance signal in Freegle exchanges + "/\bthank\s+you\s+so\s+much\b/", + + // Collection/pickup commitment + "/\bi\s+(can|will|going to|'ll)\s+collect\b/", + "/\b(will|can|'ll)\s+(collect|pick)\s+(it|them|up)/", + "/\bcollecting\s+\w+/", // "collecting thursday" + "/\bi\s+can\s+come/", + "/\bi\s+can\s+pop\s+(round|over)/", + "/\bwill\s+bring\s+it/", + "/\bwill\s+drop\s+it/", + "/\bi\s+can\s+drop\s+it/", + ]; + + foreach ($patterns as $pattern) { + if (preg_match($pattern, $text)) { + return true; + } + } + + return false; + } + + /** + * Check for PII markers that indicate progression to logistics. + */ + private function hasPiiMarkers(string $text): bool + { + return ( + strpos($text, '
') !== false || + strpos($text, '') !== false || + strpos($text, '') !== false || + strpos($text, '') !== false + ); + } + + /** + * Check for meeting arrangement (day/time + "see you" / collection intent). + */ + private function hasMeetingArrangement(string $text): bool + { + // Days of the week + $dayPattern = '/\b(monday|tuesday|wednesday|thursday|friday|saturday|sunday|mon|tue|wed|thu|fri|sat|sun)\b/'; + + // Times/occasions + $timePattern = '/\b(morning|afternoon|evening|night|pm|am|o\'?clock|at\s+\d+)/'; + + // "see you" + day/time + if (preg_match('/\bsee\s+you\b/', $text)) { + if (preg_match($dayPattern, $text) || preg_match($timePattern, $text)) { + return true; + } + } + + // Explicit collection arrangement with day/time + if (preg_match('/\b(collect|come|pick)\b/', $text)) { + if (preg_match($dayPattern, $text) || preg_match($timePattern, $text)) { + return true; + } + } + + return false; + } +} diff --git a/iznik-batch/app/Services/Promise/PiiNormaliser.php b/iznik-batch/app/Services/Promise/PiiNormaliser.php new file mode 100644 index 0000000000..a15f9c0404 --- /dev/null +++ b/iznik-batch/app/Services/Promise/PiiNormaliser.php @@ -0,0 +1,100 @@ + + * - Phone numbers (07…, +44…, 11-digit) → + * - Emails → + * - URLs (http/https/trashnothing) → + * - Street addresses →
+ * + * Preserves emoji and accented characters. Guarantees valid UTF-8 output. + * Collapses \r\n\t and runs of whitespace to single spaces. + */ + public function normalise(string $text): string + { + // Collapse \r, \n, \t to spaces + $text = str_replace(["\r", "\n", "\t"], ' ', $text); + + // Collapse multiple spaces to single space + $text = preg_replace('/\s+/', ' ', $text); + + // Trim leading/trailing whitespace + $text = trim($text); + + // Replace URLs (must come before other replacements that might contain URLs) + // Match http://, https://, or trashnothing.com + $text = preg_replace( + '/(?:https?:\/\/\S+|trashnothing\.com\S*)/i', + '', + $text + ); + + // Replace emails (basic pattern) + $text = preg_replace( + '/\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b/', + '', + $text + ); + + // Replace UK postcodes (simplified pattern: letter(s), number(s), space, number, letter, letter) + // Covers formats like SW1A 1AA, M1 1AE, B33 8TH, CR2 6XH, etc. + $text = preg_replace( + '/\b[A-Z]{1,2}\d[A-Z\d]?\s?\d[A-Z]{2}\b/i', + '', + $text + ); + + // Replace phone numbers + // Pattern 1: 07... (10 digits, may have spaces/dashes) + // Pattern 2: +44 ... (international format) + // Pattern 3: 11-digit UK numbers (02, 03, etc.) + $text = preg_replace( + '/\b(?:0|\+44)\s?7\d{3}(?:\s|\-)?(?:\d{3}|\d{6})\b/', + '', + $text + ); + $text = preg_replace( + '/\+44\s?\d{1,5}\s?\d{1,5}\s?\d{1,5}/', + '', + $text + ); + $text = preg_replace( + '/\b(?:0[1-3]|0[4-8]|0[9])\d{1,5}\s?\d{1,5}\s?\d{1,5}\b/', + '', + $text + ); + + // Replace street addresses. + // A bare "number + word" (e.g. "6 pm", "3 chairs", "2 days") is NOT an + // address — that over-matched and injected fake
tokens (a strong + // promise signal) into times/quantities. Require a real street suffix with a + // capitalised street name; the suffix word may be any case. + // Pattern 1: "12 High Street", "45 Elm Road", "10 Downing Street" + $streetSuffix = 'Street|St|Road|Rd|Lane|Ln|Avenue|Ave|Close|Drive|Dr|Way|Court|Ct' + . '|Crescent|Cres|Place|Pl|Gardens|Gdns|Terrace|Grove|Walk|Row|Square|Sq|Mews' + . '|Parade|Rise|Park|Green|Hill|Vale|Croft|Wharf|Quay|Gate|Yard|Boulevard|Broadway'; + $text = preg_replace( + '/\b\d{1,4}[a-z]?\s+[A-Z][a-z]+(?:\s+[A-Z][a-z]+){0,2}\s+(?i:' . $streetSuffix . ')\b/', + '
', + $text + ); + // Pattern 2: "Flat 3, Oakwood Lane" — sub-dwelling prefix + number + capitalised name + $text = preg_replace( + '/\b(?i:Flat|Apartment|Apt|Suite|Unit|House)\s+\d+[a-z]?\s*,?\s*[A-Z][a-z]+(?:\s+[A-Z][a-z]+)*\b/', + '
', + $text + ); + + // Guarantee valid UTF-8: drop malformed sequences + $text = mb_convert_encoding($text, 'UTF-8', 'UTF-8'); + + return $text; + } +} diff --git a/iznik-batch/composer.json b/iznik-batch/composer.json index 268448deae..b586e71b88 100644 --- a/iznik-batch/composer.json +++ b/iznik-batch/composer.json @@ -16,6 +16,7 @@ "owen-it/laravel-auditing": "^14.0", "patrickschur/language-detection": "^5.2", "phpoffice/phpspreadsheet": "^3", + "rubix/ml": "^2.5", "sentry/sentry-laravel": "^4.20", "simple-as-fuck/laravel-lock": "^0.4.0", "spatie/mjml-php": "^1.2", diff --git a/iznik-batch/composer.lock b/iznik-batch/composer.lock index cc8caa9ec4..cb780f98b8 100644 --- a/iznik-batch/composer.lock +++ b/iznik-batch/composer.lock @@ -4,8 +4,562 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "2367e1e318ad0eefc81635b5d5beecc7", + "content-hash": "dc2fe5f61897c14b2f0bf76023e7686b", "packages": [ + { + "name": "amphp/amp", + "version": "v2.6.5", + "source": { + "type": "git", + "url": "https://github.com/amphp/amp.git", + "reference": "d7dda98dae26e56f3f6fcfbf1c1f819c9a993207" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/amphp/amp/zipball/d7dda98dae26e56f3f6fcfbf1c1f819c9a993207", + "reference": "d7dda98dae26e56f3f6fcfbf1c1f819c9a993207", + "shasum": "" + }, + "require": { + "php": ">=7.1" + }, + "require-dev": { + "amphp/php-cs-fixer-config": "dev-master", + "amphp/phpunit-util": "^1", + "ext-json": "*", + "jetbrains/phpstorm-stubs": "^2019.3", + "phpunit/phpunit": "^7 | ^8 | ^9", + "react/promise": "^2", + "vimeo/psalm": "^3.12" + }, + "type": "library", + "autoload": { + "files": [ + "lib/functions.php", + "lib/Internal/functions.php" + ], + "psr-4": { + "Amp\\": "lib" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Daniel Lowrey", + "email": "rdlowrey@php.net" + }, + { + "name": "Aaron Piotrowski", + "email": "aaron@trowski.com" + }, + { + "name": "Bob Weinand", + "email": "bobwei9@hotmail.com" + }, + { + "name": "Niklas Keller", + "email": "me@kelunik.com" + } + ], + "description": "A non-blocking concurrency framework for PHP applications.", + "homepage": "https://amphp.org/amp", + "keywords": [ + "async", + "asynchronous", + "awaitable", + "concurrency", + "event", + "event-loop", + "future", + "non-blocking", + "promise" + ], + "support": { + "irc": "irc://irc.freenode.org/amphp", + "issues": "https://github.com/amphp/amp/issues", + "source": "https://github.com/amphp/amp/tree/v2.6.5" + }, + "funding": [ + { + "url": "https://github.com/amphp", + "type": "github" + } + ], + "time": "2025-09-03T19:41:28+00:00" + }, + { + "name": "amphp/byte-stream", + "version": "v1.8.2", + "source": { + "type": "git", + "url": "https://github.com/amphp/byte-stream.git", + "reference": "4f0e968ba3798a423730f567b1b50d3441c16ddc" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/amphp/byte-stream/zipball/4f0e968ba3798a423730f567b1b50d3441c16ddc", + "reference": "4f0e968ba3798a423730f567b1b50d3441c16ddc", + "shasum": "" + }, + "require": { + "amphp/amp": "^2", + "php": ">=7.1" + }, + "require-dev": { + "amphp/php-cs-fixer-config": "dev-master", + "amphp/phpunit-util": "^1.4", + "friendsofphp/php-cs-fixer": "^2.3", + "jetbrains/phpstorm-stubs": "^2019.3", + "phpunit/phpunit": "^6 || ^7 || ^8", + "psalm/phar": "^3.11.4" + }, + "type": "library", + "autoload": { + "files": [ + "lib/functions.php" + ], + "psr-4": { + "Amp\\ByteStream\\": "lib" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Aaron Piotrowski", + "email": "aaron@trowski.com" + }, + { + "name": "Niklas Keller", + "email": "me@kelunik.com" + } + ], + "description": "A stream abstraction to make working with non-blocking I/O simple.", + "homepage": "https://amphp.org/byte-stream", + "keywords": [ + "amp", + "amphp", + "async", + "io", + "non-blocking", + "stream" + ], + "support": { + "issues": "https://github.com/amphp/byte-stream/issues", + "source": "https://github.com/amphp/byte-stream/tree/v1.8.2" + }, + "funding": [ + { + "url": "https://github.com/amphp", + "type": "github" + } + ], + "time": "2024-04-13T18:00:56+00:00" + }, + { + "name": "amphp/parallel", + "version": "v1.4.4", + "source": { + "type": "git", + "url": "https://github.com/amphp/parallel.git", + "reference": "508ca221f2f47235327db5120f0a89d43435b69b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/amphp/parallel/zipball/508ca221f2f47235327db5120f0a89d43435b69b", + "reference": "508ca221f2f47235327db5120f0a89d43435b69b", + "shasum": "" + }, + "require": { + "amphp/amp": "^2", + "amphp/byte-stream": "^1.6.1", + "amphp/parser": "^1", + "amphp/process": "^1", + "amphp/serialization": "^1", + "amphp/sync": "^1.0.1", + "php": ">=7.1" + }, + "require-dev": { + "amphp/php-cs-fixer-config": "^2", + "amphp/phpunit-util": "^1.1", + "phpunit/phpunit": "^9 || ^8 || ^7" + }, + "type": "library", + "autoload": { + "files": [ + "lib/Context/functions.php", + "lib/Sync/functions.php", + "lib/Worker/functions.php" + ], + "psr-4": { + "Amp\\Parallel\\": "lib" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Aaron Piotrowski", + "email": "aaron@trowski.com" + }, + { + "name": "Stephen Coakley", + "email": "me@stephencoakley.com" + } + ], + "description": "Parallel processing component for Amp.", + "homepage": "https://github.com/amphp/parallel", + "keywords": [ + "async", + "asynchronous", + "concurrent", + "multi-processing", + "multi-threading" + ], + "support": { + "issues": "https://github.com/amphp/parallel/issues", + "source": "https://github.com/amphp/parallel/tree/v1.4.4" + }, + "funding": [ + { + "url": "https://github.com/amphp", + "type": "github" + } + ], + "time": "2024-12-08T16:28:11+00:00" + }, + { + "name": "amphp/parser", + "version": "v1.1.1", + "source": { + "type": "git", + "url": "https://github.com/amphp/parser.git", + "reference": "3cf1f8b32a0171d4b1bed93d25617637a77cded7" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/amphp/parser/zipball/3cf1f8b32a0171d4b1bed93d25617637a77cded7", + "reference": "3cf1f8b32a0171d4b1bed93d25617637a77cded7", + "shasum": "" + }, + "require": { + "php": ">=7.4" + }, + "require-dev": { + "amphp/php-cs-fixer-config": "^2", + "phpunit/phpunit": "^9", + "psalm/phar": "^5.4" + }, + "type": "library", + "autoload": { + "psr-4": { + "Amp\\Parser\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Aaron Piotrowski", + "email": "aaron@trowski.com" + }, + { + "name": "Niklas Keller", + "email": "me@kelunik.com" + } + ], + "description": "A generator parser to make streaming parsers simple.", + "homepage": "https://github.com/amphp/parser", + "keywords": [ + "async", + "non-blocking", + "parser", + "stream" + ], + "support": { + "issues": "https://github.com/amphp/parser/issues", + "source": "https://github.com/amphp/parser/tree/v1.1.1" + }, + "funding": [ + { + "url": "https://github.com/amphp", + "type": "github" + } + ], + "time": "2024-03-21T19:16:53+00:00" + }, + { + "name": "amphp/process", + "version": "v1.1.9", + "source": { + "type": "git", + "url": "https://github.com/amphp/process.git", + "reference": "55b837d4f1857b9bd7efb7bb859ae6b0e804f13f" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/amphp/process/zipball/55b837d4f1857b9bd7efb7bb859ae6b0e804f13f", + "reference": "55b837d4f1857b9bd7efb7bb859ae6b0e804f13f", + "shasum": "" + }, + "require": { + "amphp/amp": "^2", + "amphp/byte-stream": "^1.4", + "php": ">=7.1" + }, + "require-dev": { + "amphp/php-cs-fixer-config": "dev-master", + "amphp/phpunit-util": "^1", + "phpunit/phpunit": "^6" + }, + "type": "library", + "autoload": { + "files": [ + "lib/functions.php" + ], + "psr-4": { + "Amp\\Process\\": "lib" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Bob Weinand", + "email": "bobwei9@hotmail.com" + }, + { + "name": "Aaron Piotrowski", + "email": "aaron@trowski.com" + }, + { + "name": "Niklas Keller", + "email": "me@kelunik.com" + } + ], + "description": "Asynchronous process manager.", + "homepage": "https://github.com/amphp/process", + "support": { + "issues": "https://github.com/amphp/process/issues", + "source": "https://github.com/amphp/process/tree/v1.1.9" + }, + "funding": [ + { + "url": "https://github.com/amphp", + "type": "github" + } + ], + "time": "2024-12-13T17:38:25+00:00" + }, + { + "name": "amphp/serialization", + "version": "v1.1.0", + "source": { + "type": "git", + "url": "https://github.com/amphp/serialization.git", + "reference": "fdf2834d78cebb0205fb2672676c1b1eb84371f0" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/amphp/serialization/zipball/fdf2834d78cebb0205fb2672676c1b1eb84371f0", + "reference": "fdf2834d78cebb0205fb2672676c1b1eb84371f0", + "shasum": "" + }, + "require": { + "php": ">=7.4" + }, + "require-dev": { + "amphp/php-cs-fixer-config": "^2", + "ext-json": "*", + "ext-zlib": "*", + "phpunit/phpunit": "^9", + "psalm/phar": "6.16.1" + }, + "type": "library", + "autoload": { + "files": [ + "src/functions.php" + ], + "psr-4": { + "Amp\\Serialization\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Aaron Piotrowski", + "email": "aaron@trowski.com" + }, + { + "name": "Niklas Keller", + "email": "me@kelunik.com" + } + ], + "description": "Serialization tools for IPC and data storage in PHP.", + "homepage": "https://github.com/amphp/serialization", + "keywords": [ + "async", + "asynchronous", + "serialization", + "serialize" + ], + "support": { + "issues": "https://github.com/amphp/serialization/issues", + "source": "https://github.com/amphp/serialization/tree/v1.1.0" + }, + "funding": [ + { + "url": "https://github.com/amphp", + "type": "github" + } + ], + "time": "2026-04-05T15:59:53+00:00" + }, + { + "name": "amphp/sync", + "version": "v1.4.2", + "source": { + "type": "git", + "url": "https://github.com/amphp/sync.git", + "reference": "85ab06764f4f36d63b1356b466df6111cf4b89cf" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/amphp/sync/zipball/85ab06764f4f36d63b1356b466df6111cf4b89cf", + "reference": "85ab06764f4f36d63b1356b466df6111cf4b89cf", + "shasum": "" + }, + "require": { + "amphp/amp": "^2.2", + "php": ">=7.1" + }, + "require-dev": { + "amphp/php-cs-fixer-config": "dev-master", + "amphp/phpunit-util": "^1.1", + "phpunit/phpunit": "^9 || ^8 || ^7" + }, + "type": "library", + "autoload": { + "files": [ + "src/functions.php", + "src/ConcurrentIterator/functions.php" + ], + "psr-4": { + "Amp\\Sync\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Aaron Piotrowski", + "email": "aaron@trowski.com" + }, + { + "name": "Stephen Coakley", + "email": "me@stephencoakley.com" + } + ], + "description": "Mutex, Semaphore, and other synchronization tools for Amp.", + "homepage": "https://github.com/amphp/sync", + "keywords": [ + "async", + "asynchronous", + "mutex", + "semaphore", + "synchronization" + ], + "support": { + "issues": "https://github.com/amphp/sync/issues", + "source": "https://github.com/amphp/sync/tree/v1.4.2" + }, + "funding": [ + { + "url": "https://github.com/amphp", + "type": "github" + } + ], + "time": "2021-10-25T18:29:10+00:00" + }, + { + "name": "andrewdalpino/okbloomer", + "version": "1.0.0", + "source": { + "type": "git", + "url": "https://github.com/andrewdalpino/OkBloomer.git", + "reference": "39321cb515c1e99128d28489b0187120ba7ce84c" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/andrewdalpino/OkBloomer/zipball/39321cb515c1e99128d28489b0187120ba7ce84c", + "reference": "39321cb515c1e99128d28489b0187120ba7ce84c", + "shasum": "" + }, + "require": { + "php": ">=7.4" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "phpbench/phpbench": "^1.0", + "phpstan/extension-installer": "^1.0", + "phpstan/phpstan": "^1.0", + "phpstan/phpstan-phpunit": "^1.0", + "phpunit/phpunit": "^9.5" + }, + "type": "library", + "autoload": { + "psr-4": { + "OkBloomer\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Andrew DalPino", + "email": "support@andrewdalpino.com", + "homepage": "https://github.com/andrewdalpino", + "role": "Lead Developer" + } + ], + "description": "An autoscaling Bloom filter with ultra-low memory usage for PHP.", + "keywords": [ + "Bloom Filter", + "layered bloom filter", + "scalable bloom filter" + ], + "support": { + "docs": "https://github.com/andrewdalpino/OkBloomer/README.md", + "email": "support@andrewdalpino.com", + "issues": "https://github.com/andrewdalpino/OkBloomer/issues", + "source": "https://github.com/andrewdalpino/OkBloomer" + }, + "funding": [ + { + "url": "https://github.com/sponsors/andrewdalpino", + "type": "github" + } + ], + "time": "2022-01-24T03:41:23+00:00" + }, { "name": "beste/clock", "version": "3.0.0", @@ -2299,6 +2853,60 @@ ], "time": "2026-01-23T13:01:48+00:00" }, + { + "name": "joomla/string", + "version": "4.0.0", + "source": { + "type": "git", + "url": "https://github.com/joomla-framework/string.git", + "reference": "da2329e05f1f5fc98b709f8638f279513bcd1108" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/joomla-framework/string/zipball/da2329e05f1f5fc98b709f8638f279513bcd1108", + "reference": "da2329e05f1f5fc98b709f8638f279513bcd1108", + "shasum": "" + }, + "require": { + "php": "^8.3.0", + "symfony/deprecation-contracts": "^2|^3", + "symfony/polyfill-mbstring": "^1.31.0" + }, + "require-dev": { + "doctrine/inflector": "^2.0.10", + "joomla/test": "^4.0", + "phpstan/phpstan": "2.1.17", + "phpstan/phpstan-deprecation-rules": "2.0.3", + "phpunit/phpunit": "^12.2.6", + "squizlabs/php_codesniffer": "^3.7.2" + }, + "suggest": { + "doctrine/inflector": "To use the string inflector", + "ext-mbstring": "For improved processing" + }, + "type": "joomla-package", + "autoload": { + "psr-4": { + "Joomla\\String\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "GPL-2.0-or-later" + ], + "description": "Joomla String Package", + "homepage": "https://github.com/joomla-framework/string", + "keywords": [ + "framework", + "joomla", + "string" + ], + "support": { + "issues": "https://github.com/joomla-framework/string/issues", + "source": "https://github.com/joomla-framework/string/tree/4.0.0" + }, + "time": "2025-07-23T18:42:26+00:00" + }, { "name": "kreait/firebase-php", "version": "8.1.0", @@ -5713,6 +6321,270 @@ ], "time": "2025-12-02T15:19:04+00:00" }, + { + "name": "rubix/ml", + "version": "2.5.3", + "source": { + "type": "git", + "url": "https://github.com/RubixML/ML.git", + "reference": "e0960dcb19c9722ccbc4826f5b9417a75fa01613" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/RubixML/ML/zipball/e0960dcb19c9722ccbc4826f5b9417a75fa01613", + "reference": "e0960dcb19c9722ccbc4826f5b9417a75fa01613", + "shasum": "" + }, + "require": { + "amphp/parallel": "^1.3", + "andrewdalpino/okbloomer": "^1.0", + "ext-json": "*", + "php": ">=7.4", + "psr/log": "^1.1|^2.0|^3.0", + "rubix/tensor": "^3.0", + "symfony/polyfill-mbstring": "^1.0", + "symfony/polyfill-php80": "^1.17", + "symfony/polyfill-php82": "^1.27", + "symfony/polyfill-php83": "^1.27", + "wamania/php-stemmer": "^4.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "phpbench/phpbench": "^1.0", + "phpstan/extension-installer": "^1.0", + "phpstan/phpstan": "^1.0", + "phpstan/phpstan-phpunit": "^1.0", + "phpunit/phpunit": "^9.0" + }, + "suggest": { + "ext-gd": "For image support", + "ext-mbstring": "For fast multibyte string manipulation", + "ext-svm": "For Support Vector Machine engine (libsvm)", + "ext-tensor": "For fast Matrix/Vector computing" + }, + "type": "library", + "autoload": { + "files": [ + "src/constants.php", + "src/functions.php" + ], + "psr-4": { + "Rubix\\ML\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Andrew DalPino", + "homepage": "https://github.com/andrewdalpino" + }, + { + "name": "Contributors", + "homepage": "https://github.com/RubixML/ML/graphs/contributors" + } + ], + "description": "A high-level machine learning and deep learning library for the PHP language.", + "homepage": "https://rubixml.github.io/ML/latest/", + "keywords": [ + "Algorithm", + "Deep learning", + "Linear regression", + "Neural network", + "Rubix", + "TF-IDF", + "adaboost", + "ai", + "analytics", + "anomaly detection", + "artificial intelligence", + "cart", + "classification", + "classifier", + "clustering", + "cross validation", + "data mining", + "data science", + "dataset", + "dbscan", + "dimensionality reduction", + "ensemble", + "estimator", + "etl", + "feature extraction", + "feature importance", + "feature selection", + "gaussian mixture", + "gbm", + "gmm", + "gradient boost", + "grid search", + "image recognition", + "imputation", + "inference", + "isolation forest", + "k-means", + "k-nearest neighbors", + "kmeans", + "knn", + "local outlier factor", + "loda", + "lof", + "logistic regression", + "machine learning", + "manifold learning", + "mean shift", + "ml", + "mlp", + "multilayer perceptron", + "naive bayes", + "natural language processing", + "nearest neighbors", + "nlp", + "outlier detection", + "php", + "php ai", + "php machine learning", + "php ml", + "prediction", + "predictive modeling", + "random forest", + "ranking", + "recommendation", + "regression", + "regressor", + "ridge", + "rubix ml", + "rubixml", + "softmax", + "supervised learning", + "support vector machine", + "svm", + "t-sne", + "text mining", + "tf idf", + "tsne", + "unsupervised learning" + ], + "support": { + "chat": "https://t.me/RubixML", + "docs": "https://rubixml.github.io/ML/latest/", + "issues": "https://github.com/RubixML/ML/issues", + "source": "https://github.com/RubixML/ML" + }, + "funding": [ + { + "url": "https://github.com/sponsors/andrewdalpino", + "type": "github" + } + ], + "time": "2025-10-01T17:52:15+00:00" + }, + { + "name": "rubix/tensor", + "version": "3.0.5", + "source": { + "type": "git", + "url": "https://github.com/RubixML/Tensor.git", + "reference": "9f0ee170319280dcf081984adccefa8b0e6f06b8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/RubixML/Tensor/zipball/9f0ee170319280dcf081984adccefa8b0e6f06b8", + "reference": "9f0ee170319280dcf081984adccefa8b0e6f06b8", + "shasum": "" + }, + "require": { + "php": ">=7.4" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.0", + "phalcon/zephir": "^0.17", + "phpbench/phpbench": "^1.0", + "phpstan/extension-installer": "^1.0", + "phpstan/phpstan": "^1.0", + "phpstan/phpstan-phpunit": "^1.0", + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "autoload": { + "files": [ + "src/constants.php" + ], + "psr-4": { + "Tensor\\": "src/", + "Zephir\\Optimizers\\FunctionCall\\": "optimizers/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Andrew DalPino", + "email": "support@andrewdalpino.com", + "homepage": "https://github.com/andrewdalpino", + "role": "Project Lead" + }, + { + "name": "Contributors", + "homepage": "https://github.com/RubixML/Tensor/graphs/contributors" + } + ], + "description": "A library and extension that provides objects for scientific computing in PHP.", + "homepage": "https://github.com/RubixML/Tensor", + "keywords": [ + "1d convolution", + "2d convolution", + "arithmetic", + "blas", + "computation", + "computing", + "convolution", + "decomposition", + "dot product", + "eigendecomposition", + "eigenvalue", + "eigenvector", + "engineering", + "extension", + "lapack", + "linear algebra", + "math", + "matmul", + "matrix", + "matrix multiplication", + "multithreaded", + "php", + "php extension", + "pseudoinverse", + "scientific computing", + "signal processing", + "singular value decomposition", + "statistics", + "svd", + "tensor", + "trigonometry", + "vector", + "vector norm" + ], + "support": { + "chat": "https://t.me/RubixML", + "email": "support@andrewdalpino.com", + "issues": "https://github.com/RubixML/Tensor/issues", + "source": "https://github.com/RubixML/Tensor" + }, + "funding": [ + { + "url": "https://github.com/andrewdalpino", + "type": "github" + } + ], + "time": "2024-03-15T19:43:50+00:00" + }, { "name": "sentry/sentry", "version": "4.19.1", @@ -7781,6 +8653,86 @@ ], "time": "2025-01-02T08:10:11+00:00" }, + { + "name": "symfony/polyfill-php82", + "version": "v1.38.1", + "source": { + "type": "git", + "url": "https://github.com/symfony/polyfill-php82.git", + "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/symfony/polyfill-php82/zipball/002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", + "reference": "002dc0cfe5fd4ed6033d48f27d4f19a486c4b04b", + "shasum": "" + }, + "require": { + "php": ">=7.2" + }, + "type": "library", + "extra": { + "thanks": { + "url": "https://github.com/symfony/polyfill", + "name": "symfony/polyfill" + } + }, + "autoload": { + "files": [ + "bootstrap.php" + ], + "psr-4": { + "Symfony\\Polyfill\\Php82\\": "" + }, + "classmap": [ + "Resources/stubs" + ] + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Nicolas Grekas", + "email": "p@tchwork.com" + }, + { + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" + } + ], + "description": "Symfony polyfill backporting some PHP 8.2+ features to lower PHP versions", + "homepage": "https://symfony.com", + "keywords": [ + "compatibility", + "polyfill", + "portable", + "shim" + ], + "support": { + "source": "https://github.com/symfony/polyfill-php82/tree/v1.38.1" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://github.com/nicolas-grekas", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2026-05-26T12:45:58+00:00" + }, { "name": "symfony/polyfill-php83", "version": "v1.33.0", @@ -9072,6 +10024,55 @@ ], "time": "2024-11-21T01:49:47+00:00" }, + { + "name": "wamania/php-stemmer", + "version": "v4.0.0", + "source": { + "type": "git", + "url": "https://github.com/wamania/php-stemmer.git", + "reference": "d96509294ea843b4b86e4900df27424a6ea0ace8" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/wamania/php-stemmer/zipball/d96509294ea843b4b86e4900df27424a6ea0ace8", + "reference": "d96509294ea843b4b86e4900df27424a6ea0ace8", + "shasum": "" + }, + "require": { + "joomla/string": ">=2.0.1", + "php": ">=7.3" + }, + "require-dev": { + "phpunit/phpunit": "^9.0" + }, + "type": "library", + "autoload": { + "psr-4": { + "Wamania\\Snowball\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Wamania", + "homepage": "http://wamania.com" + } + ], + "description": "Native PHP Stemmer", + "keywords": [ + "php", + "porter", + "stemmer" + ], + "support": { + "issues": "https://github.com/wamania/php-stemmer/issues", + "source": "https://github.com/wamania/php-stemmer/tree/v4.0.0" + }, + "time": "2024-12-22T08:54:03+00:00" + }, { "name": "zbateson/mail-mime-parser", "version": "3.0.5", diff --git a/iznik-batch/storage/promise/.gitignore b/iznik-batch/storage/promise/.gitignore new file mode 100644 index 0000000000..d6b7ef32c8 --- /dev/null +++ b/iznik-batch/storage/promise/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/iznik-batch/tests/Fixtures/Promise/dataset_fixture.csv b/iznik-batch/tests/Fixtures/Promise/dataset_fixture.csv new file mode 100644 index 0000000000..dc70f95d8d --- /dev/null +++ b/iznik-batch/tests/Fixtures/Promise/dataset_fixture.csv @@ -0,0 +1,23 @@ +room_id,post_type,end_turn,promise_turn,label,span +101,Offer,0,4,0,"[TAKER] hi is this still available please?" +101,Offer,1,4,0,"[TAKER] hi is this still available please? [GIVER] hello, where abouts are you?" +101,Offer,2,4,0,"[GIVER] where abouts are you? [TAKER] i'm in the next village, could collect this week" +101,Offer,5,4,1,"[GIVER] yes you can have it 😊 my address is
, collect after 5 [TAKER] thank you so much!" +101,Offer,6,4,1,"[TAKER] thank you so much! i'll collect thursday 👍 [GIVER] great, see you then" +102,Offer,0,-1,0,"[TAKER] is the café table still going?" +102,Offer,1,-1,0,"[TAKER] is the café table still going? [GIVER] sorry it's already been taken" +102,Offer,2,-1,0,"[GIVER] sorry it's already been taken [TAKER] no worries, thanks for letting me know" +103,Wanted,0,3,0,"[GIVER] hi i have a spare microwave you're welcome to if it's any use" +103,Wanted,1,3,0,"[GIVER] you're welcome to it [TAKER] oh brilliant, is it working ok?" +103,Wanted,4,3,1,"[GIVER] it's yours, i can drop it round, what's your number? [TAKER] amazing thank you 🙏 it's " +103,Wanted,5,3,1,"[TAKER] thanks, see you thursday at
[GIVER] perfect, will bring it over" +104,Offer,0,2,0,"[TAKER] please may i be considered for the naïve art prints" +104,Offer,4,2,1,"[GIVER] yes you can have them, i'll put them aside for you, address is
[TAKER] brilliant" +104,Offer,5,2,1,"[TAKER] brilliant, collecting tomorrow 😀 [GIVER] no problem, they'll be in the porch" +105,Wanted,0,-1,0,"[GIVER] i have one you could try, where abouts are you based?" +105,Wanted,1,-1,0,"[GIVER] where abouts are you based? [TAKER] thanks but i'm sorted now, got one elsewhere" +106,Offer,0,5,0,"[TAKER] hello, could i collect this at the weekend if still available?" +106,Offer,2,5,0,"[GIVER] what day suits you? [TAKER] saturday morning would be ideal" +106,Offer,3,5,0,"[GIVER] someone else asked first, i'll let you know if they don't show" +106,Offer,6,5,1,"[GIVER] they didn't turn up so it's yours 😊 my address is
[TAKER] fantastic, thank you" +106,Offer,7,5,1,"[TAKER] fantastic, thank you, i'll be over at 2 [GIVER] see you then" diff --git a/iznik-batch/tests/Unit/Promise/CharNgramTokenizerTest.php b/iznik-batch/tests/Unit/Promise/CharNgramTokenizerTest.php new file mode 100644 index 0000000000..eeb218974c --- /dev/null +++ b/iznik-batch/tests/Unit/Promise/CharNgramTokenizerTest.php @@ -0,0 +1,173 @@ +tokenizer = new CharNgramTokenizer(); + } + + public function test_tokenizes_simple_words(): void + { + $tokens = $this->tokenizer->tokenize('hello world'); + + // Should contain word unigrams and bigrams + $this->assertContains('hello', $tokens); + $this->assertContains('world', $tokens); + $this->assertContains('hello world', $tokens); // 2-gram + + // Should contain character trigrams with prefix + $this->assertTrue( + array_any($tokens, fn($t) => str_starts_with($t, 'c:')), + 'Should have character n-grams with c: prefix' + ); + } + + public function test_character_trigrams_present(): void + { + $tokens = $this->tokenizer->tokenize('test'); + + // For "test", expect char trigrams like "tes", "est" + $charNgrams = array_filter($tokens, fn($t) => str_starts_with($t, 'c:')); + $this->assertNotEmpty($charNgrams); + + // Extract the actual trigrams (remove prefix) + $trigrams = array_map(fn($t) => substr($t, 2), $charNgrams); + $this->assertContains('tes', $trigrams); + $this->assertContains('est', $trigrams); + } + + public function test_handles_emoji(): void + { + $tokens = $this->tokenizer->tokenize('hello 😊 world'); + + // Emoji should not break tokenization + $this->assertNotEmpty($tokens); + + // Should still have word tokens + $wordTokens = array_filter($tokens, fn($t) => !str_starts_with($t, 'c:')); + $this->assertNotEmpty($wordTokens); + } + + public function test_handles_accents(): void + { + $tokens = $this->tokenizer->tokenize('naïve café'); + + $this->assertNotEmpty($tokens); + + // Lowercased and handled safely + $wordTokens = array_filter($tokens, fn($t) => !str_starts_with($t, 'c:')); + $this->assertTrue( + count($wordTokens) >= 2, + 'Should tokenize accented words' + ); + } + + public function test_lowercases_output(): void + { + $tokens = $this->tokenizer->tokenize('HELLO World'); + $wordTokens = array_filter($tokens, fn($t) => !str_starts_with($t, 'c:')); + + $this->assertContains('hello', $wordTokens); + $this->assertContains('world', $wordTokens); + $this->assertNotContains('HELLO', $wordTokens); + $this->assertNotContains('World', $wordTokens); + } + + public function test_combined_word_and_char_grams(): void + { + $tokens = $this->tokenizer->tokenize('hi'); + + // Word unigrams + $this->assertContains('hi', $tokens); + + // Character trigrams (if length allows) + $charNgrams = array_filter($tokens, fn($t) => str_starts_with($t, 'c:')); + // "hi" is too short for trigrams, should have none or short ones + + // 4-letter word should have both + $tokens4 = $this->tokenizer->tokenize('test'); + $charNgrams4 = array_filter($tokens4, fn($t) => str_starts_with($t, 'c:')); + $this->assertNotEmpty($charNgrams4); + } + + public function test_no_collision_between_word_and_char_grams(): void + { + $tokens = $this->tokenizer->tokenize('test'); + + $wordGrams = array_filter($tokens, fn($t) => !str_starts_with($t, 'c:')); + $charGrams = array_filter($tokens, fn($t) => str_starts_with($t, 'c:')); + + // No word gram should have 'c:' prefix + foreach ($wordGrams as $gram) { + $this->assertFalse(str_starts_with($gram, 'c:')); + } + + // All char grams should have prefix + foreach ($charGrams as $gram) { + $this->assertTrue(str_starts_with($gram, 'c:')); + } + } + + public function test_real_promise_examples(): void + { + // Positive examples + $positiveSpans = [ + "i'll take it", + "yes please", + "you can have it", + "it's yours", + "see you thursday", + "
", + "", + ]; + + foreach ($positiveSpans as $span) { + $tokens = $this->tokenizer->tokenize($span); + $this->assertNotEmpty($tokens, "Should tokenize: $span"); + + // Should have both word and char tokens + $hasWordTokens = array_any($tokens, fn($t) => !str_starts_with($t, 'c:')); + $this->assertTrue($hasWordTokens, "Should have word tokens for: $span"); + } + } + + public function test_speaker_tags_tokenized(): void + { + $span = '[GIVER] yes you can have it [TAKER] thanks'; + $tokens = $this->tokenizer->tokenize($span); + + $this->assertNotEmpty($tokens); + + // Tags should be tokenized (not special-cased) + $wordTokens = array_filter($tokens, fn($t) => !str_starts_with($t, 'c:')); + $this->assertNotEmpty($wordTokens); + } + + public function test_stringable_interface(): void + { + $this->assertTrue(method_exists($this->tokenizer, '__toString')); + $str = (string)$this->tokenizer; + $this->assertIsString($str); + } +} + +// Helper for array_any (PHP 8.3 doesn't have it builtin) +if (!function_exists('array_any')) { + function array_any(array $array, callable $callback): bool + { + foreach ($array as $item) { + if ($callback($item)) { + return true; + } + } + return false; + } +} diff --git a/iznik-batch/tests/Unit/Promise/DatasetExtractorTest.php b/iznik-batch/tests/Unit/Promise/DatasetExtractorTest.php new file mode 100644 index 0000000000..fc600a4741 --- /dev/null +++ b/iznik-batch/tests/Unit/Promise/DatasetExtractorTest.php @@ -0,0 +1,312 @@ +extractor = new DatasetExtractor(); + } + + public function test_extract_room_basic() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Is this available?'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Yes, you can have it.'], + ['type' => 'Promised', 'role' => 'TAKER', 'text' => ''], + ]; + + // With tolerance 0, we drop the promised turn itself + $rows = $this->extractor->extractRoom( + roomId: 123, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 0, 'negativesPerRoom' => null] + ); + + $this->assertCount(1, $rows); + $this->assertEquals(123, $rows[0]['room_id']); + $this->assertEquals('Offer', $rows[0]['post_type']); + $this->assertEquals(0, $rows[0]['end_turn']); + $this->assertEquals(1, $rows[0]['promise_turn']); + $this->assertEquals(0, $rows[0]['label']); + } + + public function test_exclude_event_types() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Is this available?'], + ['type' => 'Address', 'role' => 'GIVER', 'text' => ''], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Address sent.'], + ['type' => 'Promised', 'role' => 'TAKER', 'text' => ''], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 124, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + // Only 2 real-text turns: "Is this available?" and "Address sent." + // Promised event is after "Address sent.", so promise_turn = 1 + $this->assertCount(1, $rows); + $this->assertEquals(1, $rows[0]['promise_turn']); + } + + public function test_window_size_smaller_than_available_turns() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 0'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 1'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 2'], + ['type' => 'Promised', 'role' => 'GIVER', 'text' => ''], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 125, + postType: 'Offer', + messages: $messages, + opts: ['window' => 2, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + // 3 real-text turns (indices 0, 1, 2) + // Promised after turn 2, so promise_turn = 2 + // Drop |j - 2| <= 1, i.e., j in [1, 2, 3] + // Keep j in [0] + $this->assertCount(1, $rows); + $this->assertEquals(0, $rows[0]['end_turn']); + } + + public function test_span_format_speaker_tagged() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Is this available?'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Yes it is.'], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 126, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + $this->assertStringContainsString('[TAKER]', $rows[0]['span']); + $this->assertStringContainsString('[GIVER]', $rows[0]['span']); + $this->assertStringContainsString('Is this available?', $rows[0]['span']); + $this->assertStringContainsString('Yes it is.', $rows[0]['span']); + } + + public function test_no_promised_event() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Is this available?'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Yes it is.'], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 127, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + $this->assertEquals(-1, $rows[0]['promise_turn']); + $this->assertEquals(0, $rows[0]['label']); + } + + public function test_label_before_promise_turn() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 0'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 1'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 2'], + ['type' => 'Promised', 'role' => 'GIVER', 'text' => ''], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 3'], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 128, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 0, 'negativesPerRoom' => null] + ); + + // 4 real-text turns (0, 1, 2, 3), promise_turn = 2 (last turn before Promised) + // Rows with j < 2 should have label 0, rows with j >= 2 should have label 1 + $beforePromise = collect($rows)->firstWhere('end_turn', '<', 2); + $atPromise = collect($rows)->firstWhere('end_turn', '>=', 2); + + if ($beforePromise) { + $this->assertEquals(0, $beforePromise['label']); + } + if ($atPromise) { + $this->assertEquals(1, $atPromise['label']); + } + } + + public function test_drop_within_tolerance() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 0'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 1'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 2'], + ['type' => 'Promised', 'role' => 'GIVER', 'text' => ''], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 129, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + // promise_turn = 2 (last turn before Promised), tolerance = 1 + // Rows with end_turn in [1, 2, 3] should be dropped (|j-2|<=1) + // Keep end_turn = 0 + $this->assertCount(1, $rows); + $this->assertEquals(0, $rows[0]['end_turn']); + } + + public function test_negatives_per_room_cap() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 0'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 1'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 2'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 3'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 4'], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 130, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 100, 'negativesPerRoom' => 2] + ); + + // No promised event, so all are negatives (label 0) + // Should be capped to 2 + $negatives = collect($rows)->filter(fn($r) => $r['label'] === 0); + $this->assertLessThanOrEqual(2, $negatives->count()); + } + + public function test_positives_never_dropped_by_cap() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 0'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 1'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 2'], + ['type' => 'Promised', 'role' => 'GIVER', 'text' => ''], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 3'], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 131, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 0, 'negativesPerRoom' => 1] + ); + + $positives = collect($rows)->filter(fn($r) => $r['label'] === 1); + // Positives should never be dropped, even with cap + $this->assertGreaterThan(0, $positives->count()); + } + + public function test_empty_text_turns_skipped() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => ''], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 0'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => ' '], + ['type' => 'Promised', 'role' => 'GIVER', 'text' => ''], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 132, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 0, 'negativesPerRoom' => null] + ); + + // Only "Turn 0" is a real turn (index 0 in real-text space) + // Promised happens after it, so promise_turn = 0 + $this->assertCount(1, $rows); + $this->assertEquals(0, $rows[0]['promise_turn']); + $this->assertEquals(0, $rows[0]['end_turn']); + $this->assertEquals(1, $rows[0]['label']); // j >= promise_turn + } + + public function test_window_does_not_exceed_available_turns() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Only turn'], + ['type' => 'Promised', 'role' => 'GIVER', 'text' => ''], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 133, + postType: 'Offer', + messages: $messages, + opts: ['window' => 100, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + // Window is 100 but only 1 turn available, span should contain only that 1 turn + $this->assertStringContainsString('Only turn', $rows[0]['span']); + } + + public function test_wanted_post_type() + { + $messages = [ + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'I have this.'], + ['type' => 'Promised', 'role' => 'TAKER', 'text' => ''], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 134, + postType: 'Wanted', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + $this->assertEquals('Wanted', $rows[0]['post_type']); + } + + public function test_include_positive_and_negative_with_wider_tolerance() + { + $messages = [ + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 0'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 1'], + ['type' => 'Default', 'role' => 'TAKER', 'text' => 'Turn 2'], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 3'], + ['type' => 'Promised', 'role' => 'TAKER', 'text' => ''], + ['type' => 'Default', 'role' => 'GIVER', 'text' => 'Turn 4'], + ]; + + $rows = $this->extractor->extractRoom( + roomId: 135, + postType: 'Offer', + messages: $messages, + opts: ['window' => 8, 'tolerance' => 1, 'negativesPerRoom' => null] + ); + + // promise_turn = 3 (last turn before Promised) + // Drop |j - 3| <= 1, i.e., j in [2, 3, 4] + // Keep j in [0, 1, 5] + $this->assertGreaterThan(0, count($rows)); + + // Check for at least one positive and one negative + $positives = collect($rows)->filter(fn($r) => $r['label'] === 1); + $negatives = collect($rows)->filter(fn($r) => $r['label'] === 0); + + $this->assertGreaterThan(0, $positives->count()); + $this->assertGreaterThan(0, $negatives->count()); + } +} diff --git a/iznik-batch/tests/Unit/Promise/DatasetReaderTest.php b/iznik-batch/tests/Unit/Promise/DatasetReaderTest.php new file mode 100644 index 0000000000..519aba7817 --- /dev/null +++ b/iznik-batch/tests/Unit/Promise/DatasetReaderTest.php @@ -0,0 +1,196 @@ +fixtureFile = __DIR__ . '/../../Fixtures/Promise/dataset_fixture.csv'; + } + + public function test_reads_fixture_csv(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + $this->assertIsArray($dataset); + $this->assertArrayHasKey('samples', $dataset); + $this->assertArrayHasKey('labels', $dataset); + $this->assertArrayHasKey('groups', $dataset); + $this->assertArrayHasKey('endTurns', $dataset); + $this->assertArrayHasKey('promiseTurns', $dataset); + $this->assertArrayHasKey('postTypes', $dataset); + } + + public function test_fixture_counts(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + // Fixture has 22 data rows (excluding header) + $this->assertCount(22, $dataset['samples']); + $this->assertCount(22, $dataset['labels']); + $this->assertCount(22, $dataset['groups']); + $this->assertCount(22, $dataset['endTurns']); + $this->assertCount(22, $dataset['promiseTurns']); + $this->assertCount(22, $dataset['postTypes']); + } + + public function test_samples_are_spans(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + $samples = $dataset['samples']; + + // First sample should be the span from first data row + $this->assertStringContainsString('TAKER', $samples[0]); + $this->assertStringContainsString('hi is this still available please', $samples[0]); + } + + public function test_labels_are_0_or_1(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + foreach ($dataset['labels'] as $label) { + $this->assertThat($label, $this->logicalOr( + $this->equalTo(0), + $this->equalTo(1) + ), 'Labels must be 0 or 1'); + } + } + + public function test_groups_are_room_ids(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + $groups = $dataset['groups']; + + // Should have room IDs + $this->assertTrue(in_array(101, $groups)); + $this->assertTrue(in_array(102, $groups)); + $this->assertTrue(in_array(103, $groups)); + } + + public function test_end_turns_and_promise_turns_are_ints(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + foreach ($dataset['endTurns'] as $turn) { + $this->assertIsInt($turn); + $this->assertGreaterThanOrEqual(0, $turn); + } + + foreach ($dataset['promiseTurns'] as $turn) { + $this->assertIsInt($turn); + // -1 means no promise turn + $this->assertGreaterThanOrEqual(-1, $turn); + } + } + + public function test_post_types_are_offer_or_wanted(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + foreach ($dataset['postTypes'] as $type) { + $this->assertThat($type, $this->logicalOr( + $this->equalTo('Offer'), + $this->equalTo('Wanted') + ), 'Post types must be Offer or Wanted'); + } + } + + public function test_room_101_has_correct_labels(): void + { + // Room 101 rows: indices 0-5 (promise_turn=4) + // Row 0: end_turn=0, label=0 ✓ + // Row 1: end_turn=1, label=0 ✓ + // Row 2: end_turn=2, label=0 ✓ + // Row 5: end_turn=5, label=1 ✓ (end_turn >= promise_turn) + // Row 6: end_turn=6, label=1 ✓ + + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + // Filter room 101 samples + $room101Indices = array_keys( + array_filter($dataset['groups'], fn($g) => $g === 101) + ); + + $room101Labels = array_map(fn($i) => $dataset['labels'][$i], $room101Indices); + $room101EndTurns = array_map(fn($i) => $dataset['endTurns'][$i], $room101Indices); + + // First 3 should be 0 + $this->assertEquals([0, 0, 0], array_slice($room101Labels, 0, 3)); + + // Last 2 should be 1 + $this->assertEquals([1, 1], array_slice($room101Labels, -2)); + } + + public function test_room_102_has_no_promise_turn(): void + { + // Room 102: promise_turn=-1, so all labels should be 0 + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + $room102Indices = array_keys( + array_filter($dataset['groups'], fn($g) => $g === 102) + ); + + $room102Labels = array_map(fn($i) => $dataset['labels'][$i], $room102Indices); + + foreach ($room102Labels as $label) { + $this->assertEquals(0, $label); + } + } + + public function test_multibyte_characters_preserved(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + // Fixture has emoji in several spans + $hasEmoji = array_some( + $dataset['samples'], + fn($sample) => preg_match('/\p{Emoji}/u', $sample) + ); + + $this->assertTrue($hasEmoji, 'Fixture should contain emoji and they should be preserved'); + } + + public function test_pii_placeholders_preserved(): void + { + $reader = new DatasetReader(); + $dataset = $reader->read($this->fixtureFile); + + $hasAddress = array_some( + $dataset['samples'], + fn($sample) => strpos($sample, '
') !== false + ); + + $this->assertTrue($hasAddress, 'Fixture should have
placeholders'); + } +} + +// Helper functions +if (!function_exists('array_some')) { + function array_some(array $array, callable $callback): bool + { + foreach ($array as $item) { + if ($callback($item)) { + return true; + } + } + return false; + } +} diff --git a/iznik-batch/tests/Unit/Promise/EvaluatorTest.php b/iznik-batch/tests/Unit/Promise/EvaluatorTest.php new file mode 100644 index 0000000000..abd4145b75 --- /dev/null +++ b/iznik-batch/tests/Unit/Promise/EvaluatorTest.php @@ -0,0 +1,273 @@ +evaluator = new Evaluator(); + } + + public function test_perfect_predictions(): void + { + $yTrue = [0, 0, 1, 1]; + $yProba = [0.1, 0.2, 0.8, 0.9]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba, threshold: 0.5); + + $this->assertEquals(1.0, $metrics['precision']); + $this->assertEquals(1.0, $metrics['recall']); + $this->assertEquals(1.0, $metrics['f1']); + $this->assertEquals(2, $metrics['confusion_matrix']['tp']); + $this->assertEquals(2, $metrics['confusion_matrix']['tn']); + $this->assertEquals(0, $metrics['confusion_matrix']['fp']); + $this->assertEquals(0, $metrics['confusion_matrix']['fn']); + } + + public function test_all_false_positives(): void + { + $yTrue = [0, 0, 0, 0]; + $yProba = [0.9, 0.8, 0.7, 0.6]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba, threshold: 0.5); + + // No true positives, so precision/recall are 0 + $this->assertEquals(0, $metrics['confusion_matrix']['tp']); + $this->assertEquals(4, $metrics['confusion_matrix']['fp']); + $this->assertEquals(0, $metrics['confusion_matrix']['tn']); + $this->assertEquals(0, $metrics['confusion_matrix']['fn']); + } + + public function test_all_false_negatives(): void + { + $yTrue = [1, 1, 1, 1]; + $yProba = [0.1, 0.2, 0.3, 0.4]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba, threshold: 0.5); + + $this->assertEquals(0, $metrics['confusion_matrix']['tp']); + $this->assertEquals(0, $metrics['confusion_matrix']['fp']); + $this->assertEquals(0, $metrics['confusion_matrix']['tn']); + $this->assertEquals(4, $metrics['confusion_matrix']['fn']); + } + + public function test_precision_recall_f1_calculation(): void + { + // TP=2, FP=1, FN=1, TN=2 + // Precision = TP / (TP + FP) = 2/3 ≈ 0.667 + // Recall = TP / (TP + FN) = 2/3 ≈ 0.667 + // F1 = 2 * (P * R) / (P + R) = 2 * (0.667 * 0.667) / 1.334 ≈ 0.667 + $yTrue = [0, 0, 0, 1, 1, 1]; + $yProba = [0.2, 0.6, 0.3, 0.7, 0.8, 0.4]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba, threshold: 0.5); + + $this->assertEqualsWithDelta(2/3, $metrics['precision'], 0.01); + $this->assertEqualsWithDelta(2/3, $metrics['recall'], 0.01); + $this->assertEqualsWithDelta(2/3, $metrics['f1'], 0.01); + } + + public function test_roc_auc_perfect(): void + { + // Perfect ranking: all positives ranked higher than negatives + $yTrue = [0, 0, 0, 1, 1, 1]; + $yProba = [0.1, 0.2, 0.3, 0.7, 0.8, 0.9]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba); + + $this->assertEquals(1.0, $metrics['roc_auc']); + } + + public function test_roc_auc_random(): void + { + // Random ordering should be ~0.5 — positives and negatives interleaved + $yTrue = [0, 1, 0, 1, 0, 1]; + $yProba = [0.4, 0.6, 0.3, 0.7, 0.5, 0.2]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba); + + // This is approximately random, so AUC should be near 0.5 + $this->assertGreaterThan(0.3, $metrics['roc_auc']); + $this->assertLessThan(0.7, $metrics['roc_auc']); + } + + public function test_pr_auc_present(): void + { + $yTrue = [0, 0, 0, 1, 1, 1]; + $yProba = [0.1, 0.2, 0.3, 0.7, 0.8, 0.9]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba); + + $this->assertArrayHasKey('pr_auc', $metrics); + $this->assertGreaterThanOrEqual(0.0, $metrics['pr_auc']); + $this->assertLessThanOrEqual(1.0, $metrics['pr_auc']); + } + + public function test_threshold_sweep(): void + { + $yTrue = [0, 0, 1, 1, 1]; + $yProba = [0.1, 0.4, 0.5, 0.7, 0.9]; + + $sweep = $this->evaluator->thresholdSweep($yTrue, $yProba); + + // Should have multiple thresholds (at least 5) + $this->assertGreaterThanOrEqual(5, count($sweep)); + + // Each entry should have threshold, precision, recall, f1 + foreach ($sweep as $entry) { + $this->assertArrayHasKey('threshold', $entry); + $this->assertArrayHasKey('precision', $entry); + $this->assertArrayHasKey('recall', $entry); + $this->assertArrayHasKey('f1', $entry); + $this->assertGreaterThanOrEqual(0.0, $entry['threshold']); + $this->assertLessThanOrEqual(1.0, $entry['threshold']); + } + } + + public function test_timing_offset_perfect(): void + { + // Room with 5 turns, promise at turn 3 + $roomTurns = [ + ['end_turn' => 0, 'proba' => 0.1], + ['end_turn' => 1, 'proba' => 0.2], + ['end_turn' => 2, 'proba' => 0.3], + ['end_turn' => 3, 'proba' => 0.8], // Promise detected here + ['end_turn' => 4, 'proba' => 0.9], + ]; + + $offsets = $this->evaluator->timingOffsets( + roomTurns: $roomTurns, + actualPromiseTurn: 3, + threshold: 0.5 + ); + + // Should detect promise at turn 3 with threshold 0.5 + $this->assertEquals([0], $offsets); + } + + public function test_timing_offset_missed(): void + { + $roomTurns = [ + ['end_turn' => 0, 'proba' => 0.1], + ['end_turn' => 1, 'proba' => 0.2], + ['end_turn' => 2, 'proba' => 0.3], + ['end_turn' => 3, 'proba' => 0.4], + ['end_turn' => 4, 'proba' => 0.4], + ]; + + $offsets = $this->evaluator->timingOffsets( + roomTurns: $roomTurns, + actualPromiseTurn: 3, + threshold: 0.5 + ); + + // Never crosses threshold + $this->assertEmpty($offsets); + } + + public function test_timing_offset_late(): void + { + $roomTurns = [ + ['end_turn' => 0, 'proba' => 0.1], + ['end_turn' => 1, 'proba' => 0.2], + ['end_turn' => 2, 'proba' => 0.3], + ['end_turn' => 3, 'proba' => 0.4], + ['end_turn' => 4, 'proba' => 0.8], // Detected here, but promise was at 3 + ]; + + $offsets = $this->evaluator->timingOffsets( + roomTurns: $roomTurns, + actualPromiseTurn: 3, + threshold: 0.5 + ); + + // Detected at turn 4, promise at 3 -> offset = 1 + $this->assertEquals([1], $offsets); + } + + public function test_timing_offset_early(): void + { + $roomTurns = [ + ['end_turn' => 0, 'proba' => 0.8], // Detected here early + ['end_turn' => 1, 'proba' => 0.8], + ['end_turn' => 2, 'proba' => 0.8], + ['end_turn' => 3, 'proba' => 0.8], // Actual promise at 3 + ]; + + $offsets = $this->evaluator->timingOffsets( + roomTurns: $roomTurns, + actualPromiseTurn: 3, + threshold: 0.5 + ); + + // Detected at turn 0, promise at 3 -> offset = -3 + $this->assertEquals([-3], $offsets); + } + + public function test_timing_offset_distribution(): void + { + // Multiple rooms with different offsets + $distribution = []; + + // Room 1: offset 0 + $offsets1 = $this->evaluator->timingOffsets( + roomTurns: [ + ['end_turn' => 0, 'proba' => 0.1], + ['end_turn' => 1, 'proba' => 0.8], + ], + actualPromiseTurn: 1, + threshold: 0.5 + ); + $distribution = array_merge($distribution, $offsets1); + + // Room 2: offset 1 + $offsets2 = $this->evaluator->timingOffsets( + roomTurns: [ + ['end_turn' => 0, 'proba' => 0.1], + ['end_turn' => 1, 'proba' => 0.3], + ['end_turn' => 2, 'proba' => 0.8], + ], + actualPromiseTurn: 1, + threshold: 0.5 + ); + $distribution = array_merge($distribution, $offsets2); + + $this->assertCount(2, $distribution); + $this->assertContains(0, $distribution); + $this->assertContains(1, $distribution); + } + + public function test_confusion_matrix_all_zeros(): void + { + $yTrue = [0, 0, 0]; + $yProba = [0.1, 0.2, 0.3]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba, threshold: 0.5); + + $cm = $metrics['confusion_matrix']; + $this->assertEquals(0, $cm['tp']); + $this->assertEquals(0, $cm['fp']); + $this->assertEquals(3, $cm['tn']); + $this->assertEquals(0, $cm['fn']); + } + + public function test_all_ones(): void + { + $yTrue = [1, 1, 1]; + $yProba = [0.7, 0.8, 0.9]; + + $metrics = $this->evaluator->evaluate($yTrue, $yProba, threshold: 0.5); + + $cm = $metrics['confusion_matrix']; + $this->assertEquals(3, $cm['tp']); + $this->assertEquals(0, $cm['fp']); + $this->assertEquals(0, $cm['tn']); + $this->assertEquals(0, $cm['fn']); + } +} diff --git a/iznik-batch/tests/Unit/Promise/KeywordBaselineTest.php b/iznik-batch/tests/Unit/Promise/KeywordBaselineTest.php new file mode 100644 index 0000000000..a2bb76c413 --- /dev/null +++ b/iznik-batch/tests/Unit/Promise/KeywordBaselineTest.php @@ -0,0 +1,134 @@ +baseline = new KeywordBaseline(); + } + + public function test_positive_examples(): void + { + $positiveSpans = [ + "i'll take it", + "yes please", + "you can have it", + "it's yours", + "see you thursday", + "[GIVER] yes you can have it [TAKER] thanks", + "amazing thank you 🙏 it's ", + "brilliant, collecting tomorrow 😀", + "[GIVER] my address is
[TAKER] fantastic", + "[GIVER] i can drop it round, what's your number?", + "i'll collect thursday", + "i can collect saturday", + ]; + + foreach ($positiveSpans as $span) { + $pred = $this->baseline->predict($span); + $this->assertEquals(1, $pred, "Should predict 1 for: $span"); + } + } + + public function test_negative_examples(): void + { + $negativeSpans = [ + "is this still available?", + "what is it like?", + "thanks for the offer", + "no worries, thanks for letting me know", + "[TAKER] hi is this still available please?", + "[GIVER] sorry it's already been taken", + "[TAKER] thanks but i'm sorted now", + "do you have any more?", + ]; + + foreach ($negativeSpans as $span) { + $pred = $this->baseline->predict($span); + $this->assertEquals(0, $pred, "Should predict 0 for: $span"); + } + } + + public function test_handles_case_insensitivity(): void + { + // "I'LL TAKE IT" should match same as "i'll take it" + $pred1 = $this->baseline->predict("i'll take it"); + $pred2 = $this->baseline->predict("I'LL TAKE IT"); + + $this->assertEquals($pred1, $pred2); + } + + public function test_handles_punctuation(): void + { + // Variations of the same promise signal + $pred1 = $this->baseline->predict("ill take it"); + $pred2 = $this->baseline->predict("i'll take it"); + $pred3 = $this->baseline->predict("i'll take it!"); + + // All should be positive + $this->assertEquals(1, $pred1); + $this->assertEquals(1, $pred2); + $this->assertEquals(1, $pred3); + } + + public function test_with_pii_tokens(): void + { + // Presence of address/postcode/phone is a strong signal + $pred1 = $this->baseline->predict("my address is
"); + $pred2 = $this->baseline->predict("collect at
"); + $pred3 = $this->baseline->predict("my number is "); + + $this->assertEquals(1, $pred1); + $this->assertEquals(1, $pred2); + $this->assertEquals(1, $pred3); + } + + public function test_with_emoji(): void + { + $pred1 = $this->baseline->predict("thank you so much! 😊"); + $pred2 = $this->baseline->predict("yes please 🙏"); + + // Emoji alone doesn't make it positive, but combined with promise language does + $this->assertEquals(1, $pred1); + $this->assertEquals(1, $pred2); + } + + public function test_address_without_promise_language_ambiguous(): void + { + // An address alone might be informational, not confirmatory + // But in typical usage it's paired with agreement + $pred = $this->baseline->predict("
"); + + // This is ambiguous; the baseline should catch it as (weak) positive + // due to the PII being present + $this->assertEquals(1, $pred); + } + + public function test_time_day_agreement(): void + { + $pred1 = $this->baseline->predict("see you thursday"); + $pred2 = $this->baseline->predict("collect saturday morning"); + $pred3 = $this->baseline->predict("i can come friday at 3"); + + $this->assertEquals(1, $pred1); + $this->assertEquals(1, $pred2); + $this->assertEquals(1, $pred3); + } + + public function test_consistency(): void + { + // Same span should always predict the same + $span = "yes you can have it"; + $pred1 = $this->baseline->predict($span); + $pred2 = $this->baseline->predict($span); + + $this->assertEquals($pred1, $pred2); + } +} diff --git a/iznik-batch/tests/Unit/Promise/PiiNormaliserTest.php b/iznik-batch/tests/Unit/Promise/PiiNormaliserTest.php new file mode 100644 index 0000000000..8e5fb58a68 --- /dev/null +++ b/iznik-batch/tests/Unit/Promise/PiiNormaliserTest.php @@ -0,0 +1,191 @@ +normaliser = new PiiNormaliser(); + } + + public function test_normalise_uk_postcode() + { + $result = $this->normaliser->normalise('I live in SW1A 1AA'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('SW1A 1AA', $result); + } + + public function test_normalise_phone_07_number() + { + $result = $this->normaliser->normalise('Call me on 07700 900000'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('07700 900000', $result); + } + + public function test_normalise_phone_plus44() + { + $result = $this->normaliser->normalise('Reach me at +44 20 7946 0958'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('+44', $result); + } + + public function test_normalise_phone_11_digit() + { + $result = $this->normaliser->normalise('My number is 02070946958 for pickup'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('02070946958', $result); + } + + public function test_normalise_email() + { + $result = $this->normaliser->normalise('Email me at test@example.com please'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('test@example.com', $result); + } + + public function test_normalise_http_url() + { + $result = $this->normaliser->normalise('See http://example.com for details'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('http://example.com', $result); + } + + public function test_normalise_https_url() + { + $result = $this->normaliser->normalise('Check https://example.com here'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('https://example.com', $result); + } + + public function test_normalise_trashnothing_url() + { + $result = $this->normaliser->normalise('Posted on trashnothing.com for trade'); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('trashnothing.com', $result); + } + + public function test_normalise_street_address_number_and_name() + { + $result = $this->normaliser->normalise('Come to 12 High Street in town'); + $this->assertStringContainsString('
', $result); + $this->assertStringNotContainsString('12 High Street', $result); + } + + public function test_normalise_flat_address() + { + $result = $this->normaliser->normalise('Meet me at Flat 3, Oakwood Lane'); + $this->assertStringContainsString('
', $result); + $this->assertStringNotContainsString('Flat 3', $result); + } + + public function test_does_not_redact_times_or_quantities_as_address() + { + // A bare " " is NOT an address. The old regex matched any + // number+word and injected fake
tokens (a strong promise signal) + // into times, durations and quantities — corrupting the dataset. Regression guard. + foreach ([ + 'come at 6 pm', + 'in 2 days', + 'give me 10 minutes', + 'I have 3 chairs', + 'around 7 tonight', + 'ready in 5 mins', + '11 am works for me', + ] as $input) { + $this->assertStringNotContainsString( + '
', + $this->normaliser->normalise($input), + "'$input' must not be redacted as an address" + ); + } + } + + public function test_still_redacts_real_street_addresses() + { + foreach ([ + '12 High Street in town', + 'my address is 45 Elm Road', + '10 Downing Street', + 'come to 7 Mill Lane', + ] as $input) { + $this->assertStringContainsString( + '
', + $this->normaliser->normalise($input), + "'$input' should be redacted as an address" + ); + } + } + + public function test_preserve_emoji() + { + $input = 'Great item 🎉 very happy 😊'; + $result = $this->normaliser->normalise($input); + $this->assertStringContainsString('🎉', $result); + $this->assertStringContainsString('😊', $result); + } + + public function test_preserve_accented_characters() + { + $input = 'Café résumé naïve'; + $result = $this->normaliser->normalise($input); + $this->assertStringContainsString('Café', $result); + $this->assertStringContainsString('résumé', $result); + $this->assertStringContainsString('naïve', $result); + } + + public function test_valid_utf8_output() + { + $input = 'Test with emoji 🎉 and accents é ñ ü 你好'; + $result = $this->normaliser->normalise($input); + // Valid UTF-8 should not have false from mb_check_encoding + $this->assertTrue(mb_check_encoding($result, 'UTF-8')); + } + + public function test_collapse_carriage_return_newline_tab() + { + // Use digit-free words so the address regex (" ") doesn't fire — + // this test is about \r\n\t collapsing, not PII. + $result = $this->normaliser->normalise("Line one\r\nLine two\tTabbed"); + $this->assertStringNotContainsString("\r", $result); + $this->assertStringNotContainsString("\n", $result); + $this->assertStringNotContainsString("\t", $result); + $this->assertStringContainsString("Line one Line two Tabbed", $result); + } + + public function test_collapse_multiple_spaces() + { + $result = $this->normaliser->normalise('Text with multiple spaces'); + $this->assertStringContainsString('Text with multiple spaces', $result); + } + + public function test_multiple_pii_in_one_string() + { + $result = $this->normaliser->normalise( + 'Call 07700 900000 or email test@example.com. Address: 12 High Street SW1A 1AA' + ); + $this->assertStringContainsString('', $result); + $this->assertStringContainsString('', $result); + $this->assertStringContainsString('
', $result); + $this->assertStringContainsString('', $result); + $this->assertStringNotContainsString('07700 900000', $result); + $this->assertStringNotContainsString('test@example.com', $result); + } + + public function test_empty_string() + { + $result = $this->normaliser->normalise(''); + $this->assertEquals('', $result); + } + + public function test_whitespace_only() + { + $result = $this->normaliser->normalise(' '); + $this->assertStringNotContainsString(' ', $result); + } +} diff --git a/plans/freegle-chat-flow-offer-workflow.json b/plans/freegle-chat-flow-offer-workflow.json new file mode 100644 index 0000000000..8c2a6c9fc7 --- /dev/null +++ b/plans/freegle-chat-flow-offer-workflow.json @@ -0,0 +1,119 @@ +{ + "id": "freegle-chat-flow-offer", + "name": "Freegle Chat Flow - OFFER (request a given item)", + "description": "Empirical state machine of a single User2User chat started from an OFFER post (the taker replies asking for an offered item). Derived from ~35,400 OFFER-initiated conversations on the live database over one month (2026-05) and validated against read samples. The giver owns the item and issues the system events (Promised, Address, Reneged). Percentages are the approximate share of all OFFER chats that traverse that arm; terminal-state totals are measured, intermediate splits are estimates. NOTE: a 'Completed' system message only closes the POST - a bare 'Interested -> Completed' with no real reply means the item went to SOMEONE ELSE, not this replier.", + "initialState": "START", + "guardrails": "Never make a commitment on the giver's behalf. Stay factual. A bare 'Completed' is NOT proof this replier collected the item.", + "states": { + "START": { + "nodeType": "start", + "description": "Chat room created. The taker replies to an OFFER with an 'Interested' message (a small share open with a plain Default direct message). One chat may carry several items, each looping the arrange->promise->collect arc.", + "writeActions": ["send_message"] + }, + "INTEREST_EXPRESSED": { + "nodeType": "agent", + "description": "Taker has asked for the item (type=Interested). Awaiting the giver to respond.", + "prompt": "Confirm availability and move toward arranging a handover, answer any question asked, or say if it has gone.", + "readActions": ["get_message", "get_user_info"], + "writeActions": ["send_message"], + "timeout": { "duration": 172800000, "toState": "GHOSTED" } + }, + "ENQUIRY": { + "nodeType": "agent", + "description": "Clarifying questions before commitment: size, condition, suitability, location. ~32% of openers contain a question.", + "prompt": "Answer the factual question from the listing data, then continue toward arranging a handover.", + "readActions": ["get_message"], + "writeActions": ["send_message"], + "timeout": { "duration": 172800000, "toState": "GHOSTED" } + }, + "WAITLISTED": { + "nodeType": "agent", + "description": "Giver has promised it to someone ahead: 'I'll let you know if it falls through.' Usually ends with the front-runner getting it.", + "prompt": "Keep the taker warm without committing. If the front-runner falls through, re-engage and arrange.", + "writeActions": ["send_message"], + "timeout": { "duration": 604800000, "toState": "DECLINED_GONE" } + }, + "ARRANGING": { + "nodeType": "agent", + "description": "Item confirmed available; both negotiating the handover (times, address, transport). Busiest state.", + "prompt": "Agree a collection time and share/ask for the address. Do not over-message once settled.", + "readActions": ["get_user_info"], + "writeActions": ["send_message"], + "timeout": { "duration": 259200000, "toState": "STALLED" } + }, + "PROMISED": { + "nodeType": "agent", + "description": "Giver marked it Promised to this taker (type=Promised, often + Address). Genuine commitment. Present in ~16% of chats.", + "prompt": "Confirm the collection details and address. Nudge gently if the taker goes quiet.", + "writeActions": ["send_message", "send_address", "nudge"], + "timeout": { "duration": 259200000, "toState": "STALLED" } + }, + "HANDOVER_PENDING": { + "nodeType": "agent", + "description": "Address shared and time agreed; awaiting collection. Reminder/Nudge can fire here.", + "prompt": "Remind the collector of time and place. Confirm once collected.", + "writeActions": ["send_message", "remind", "nudge"], + "timeout": { "duration": 259200000, "toState": "STALLED" } + }, + "COLLECTED": { + "nodeType": "end", + "description": "SUCCESS - handed over TO THIS REPLIER. ~43% of OFFER chats. Requires real handover evidence in this chat (Promise/Address, bilateral arranging, or collection text). A bare Completed alone does NOT qualify." + }, + "DECLINED_GONE": { + "nodeType": "end", + "description": "The item did NOT go to this replier. ~29% of OFFER chats. Includes giver 'already gone / given to someone else', the bulk 'sorry, taken' closure, and the bare 'Interested -> Completed' (no engagement = gone elsewhere). Even a bilateral chat that ends 'sorry, gone to someone else' lands here, NOT in COLLECTED." + }, + "WITHDRAWN": { + "nodeType": "end", + "description": "Taker backs out: not suitable, changed mind, can't collect, too far. ~3%. Amicable." + }, + "RENEGED": { + "nodeType": "end", + "description": "Giver withdraws a promise (type=Reneged). ~2.3%. Item usually recycles to another taker elsewhere. Occasionally a no-show auto-Reneged is reversed when the taker resurfaces, re-entering ARRANGING." + }, + "GHOSTED": { + "nodeType": "end", + "description": "No reply and no closure after interest (~11%, mostly a single message). The giver never engaged here - often the item went to a faster responder elsewhere." + }, + "STALLED": { + "nodeType": "end", + "description": "Two-sided conversation that faded mid-arrangement with no recorded outcome (~12%). Some are latent success (collected but unmarked); some genuine no-shows." + } + }, + "transitions": [ + { "id": "t_open", "from": "START", "to": "INTEREST_EXPRESSED", "trigger": "unconditional", "metadata": { "event": "Interested (or a direct Default message)", "approx_share": "100%" } }, + + { "id": "t_ie_arrange", "from": "INTEREST_EXPRESSED", "to": "ARRANGING", "trigger": "llm_decision", "condition": "Giver confirmed available and is arranging", "metadata": { "approx_share": "~49%" } }, + { "id": "t_ie_gone", "from": "INTEREST_EXPRESSED", "to": "DECLINED_GONE", "trigger": "host_driven", "condition": "Bare Completed / 'sorry it's gone' with no real reply - item went elsewhere", "metadata": { "event": "Completed (bare)", "approx_share": "~16%" } }, + { "id": "t_ie_enquiry", "from": "INTEREST_EXPRESSED", "to": "ENQUIRY", "trigger": "llm_decision", "condition": "Taker asked a question before committing", "metadata": { "approx_share": "~13%" } }, + { "id": "t_ie_ghost", "from": "INTEREST_EXPRESSED", "to": "GHOSTED", "trigger": "host_driven", "condition": "No reply and no closure", "metadata": { "signal": "timeout", "approx_share": "~11%" } }, + { "id": "t_ie_waitlist", "from": "INTEREST_EXPRESSED", "to": "WAITLISTED", "trigger": "llm_decision", "condition": "Giver: promised to someone ahead", "metadata": { "approx_share": "~5%" } }, + { "id": "t_ie_promised", "from": "INTEREST_EXPRESSED", "to": "PROMISED", "trigger": "host_driven", "condition": "Giver immediately marked Promised", "metadata": { "event": "Promised", "approx_share": "~3%" } }, + { "id": "t_ie_collected", "from": "INTEREST_EXPRESSED", "to": "COLLECTED", "trigger": "host_driven", "condition": "Immediate genuine handover (Promised + Completed)", "metadata": { "approx_share": "~1%" } }, + + { "id": "t_enq_arrange", "from": "ENQUIRY", "to": "ARRANGING", "trigger": "llm_decision", "condition": "Answered, proceeding", "metadata": { "approx_share": "~10%" } }, + { "id": "t_enq_gone", "from": "ENQUIRY", "to": "DECLINED_GONE", "trigger": "llm_decision", "condition": "Gone / unsuitable", "metadata": { "approx_share": "~2%" } }, + { "id": "t_enq_withdraw", "from": "ENQUIRY", "to": "WITHDRAWN", "trigger": "llm_decision", "condition": "Answer made it unsuitable", "metadata": { "approx_share": "~1%" } }, + + { "id": "t_wl_gone", "from": "WAITLISTED", "to": "DECLINED_GONE", "trigger": "llm_decision", "condition": "Front-runner got it / no call-back", "metadata": { "approx_share": "~3%" } }, + { "id": "t_wl_arrange", "from": "WAITLISTED", "to": "ARRANGING", "trigger": "llm_decision", "condition": "Front-runner fell through; now offered here", "metadata": { "approx_share": "~2%" } }, + + { "id": "t_arr_promised", "from": "ARRANGING", "to": "PROMISED", "trigger": "host_driven", "condition": "Giver marked Promised", "metadata": { "event": "Promised", "approx_share": "~20%" } }, + { "id": "t_arr_handover", "from": "ARRANGING", "to": "HANDOVER_PENDING", "trigger": "llm_decision", "condition": "Address shared and time agreed", "metadata": { "approx_share": "~18%" } }, + { "id": "t_arr_collected", "from": "ARRANGING", "to": "COLLECTED", "trigger": "host_driven", "condition": "Collected (Completed / text) after real arranging, no contrary 'gone' signal", "metadata": { "approx_share": "~12%" } }, + { "id": "t_arr_stall", "from": "ARRANGING", "to": "STALLED", "trigger": "host_driven", "condition": "Goes quiet mid-arrangement", "metadata": { "signal": "timeout", "approx_share": "~8%" } }, + { "id": "t_arr_gone", "from": "ARRANGING", "to": "DECLINED_GONE", "trigger": "llm_decision", "condition": "Giver gives it to someone else despite the chat", "metadata": { "approx_share": "~5%" } }, + { "id": "t_arr_withdraw", "from": "ARRANGING", "to": "WITHDRAWN", "trigger": "llm_decision", "condition": "Taker backs out", "metadata": { "approx_share": "~1%" } }, + + { "id": "t_pr_collected", "from": "PROMISED", "to": "COLLECTED", "trigger": "host_driven", "condition": "Collected (Completed / text)", "metadata": { "approx_share": "~10%" } }, + { "id": "t_pr_handover", "from": "PROMISED", "to": "HANDOVER_PENDING", "trigger": "host_driven", "condition": "Address shared / time agreed", "metadata": { "event": "Address", "approx_share": "~8%" } }, + { "id": "t_pr_renege", "from": "PROMISED", "to": "RENEGED", "trigger": "host_driven", "condition": "Giver withdrew the promise", "metadata": { "event": "Reneged", "approx_share": "~2%" } }, + { "id": "t_pr_withdraw", "from": "PROMISED", "to": "WITHDRAWN", "trigger": "llm_decision", "condition": "Taker backs out after being promised", "metadata": { "approx_share": "~1%" } }, + { "id": "t_pr_stall", "from": "PROMISED", "to": "STALLED", "trigger": "host_driven", "condition": "Goes quiet after promise", "metadata": { "signal": "timeout", "approx_share": "~2%" } }, + + { "id": "t_ho_collected", "from": "HANDOVER_PENDING", "to": "COLLECTED", "trigger": "host_driven", "condition": "Completed / collection text ('on my way', 'collected')", "metadata": { "approx_share": "~20%" } }, + { "id": "t_ho_stall", "from": "HANDOVER_PENDING", "to": "STALLED", "trigger": "host_driven", "condition": "No-show / went quiet before confirming", "metadata": { "signal": "timeout", "approx_share": "~2%" } }, + { "id": "t_ho_withdraw", "from": "HANDOVER_PENDING", "to": "WITHDRAWN", "trigger": "llm_decision", "condition": "Taker cancels late", "metadata": { "approx_share": "~0.5%" } }, + { "id": "t_ho_renege", "from": "HANDOVER_PENDING", "to": "RENEGED", "trigger": "host_driven", "condition": "Giver pulls out", "metadata": { "event": "Reneged", "approx_share": "~0.3%" } } + ] +} diff --git a/plans/freegle-chat-flow-offer.dot b/plans/freegle-chat-flow-offer.dot new file mode 100644 index 0000000000..a029cb736d --- /dev/null +++ b/plans/freegle-chat-flow-offer.dot @@ -0,0 +1,51 @@ +digraph offer { + rankdir=LR; bgcolor="white"; nodesep=0.45; ranksep=1.05; splines=spline; + node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=11, fillcolor="#eef2ff", color="#5566aa", penwidth=1.4]; + edge [fontname="Helvetica", fontsize=9, color="#9aa0a6"]; + + start [shape=circle, width=0.16, label="", fillcolor=black, color=black]; + INTEREST_EXPRESSED [label="INTEREST_\nEXPRESSED"]; + HANDOVER_PENDING [label="HANDOVER_\nPENDING"]; + COLLECTED [fillcolor="#d7f5dd", color="#2e7d32", label="COLLECTED\n~43%"]; + DECLINED_GONE [fillcolor="#fde2e2", color="#c62828", label="DECLINED_\nGONE ~29%"]; + GHOSTED [fillcolor="#fde2e2", color="#c62828", label="GHOSTED\n~11%"]; + WITHDRAWN [fillcolor="#fde2e2", color="#c62828", label="WITHDRAWN\n~3%"]; + RENEGED [fillcolor="#fde2e2", color="#c62828", label="RENEGED\n~2%"]; + STALLED [fillcolor="#fff4d6", color="#f9a825", label="STALLED\n~12%"]; + WAITLISTED [fillcolor="#fff4d6", color="#f9a825"]; + + start -> INTEREST_EXPRESSED [label="100%"]; + INTEREST_EXPRESSED -> ARRANGING [weight=8, label="available ~49%"]; + ARRANGING -> PROMISED [weight=8, label="Promised ~20%"]; + PROMISED -> HANDOVER_PENDING [weight=8, label="Address ~8%"]; + HANDOVER_PENDING -> COLLECTED [weight=8, label="collected ~20%"]; + + INTEREST_EXPRESSED -> ENQUIRY [label="question ~13%"]; + INTEREST_EXPRESSED -> WAITLISTED [label="behind ~5%"]; + INTEREST_EXPRESSED -> PROMISED [label="immediate ~3%"]; + INTEREST_EXPRESSED -> GHOSTED [label="no reply ~11%"]; + INTEREST_EXPRESSED -> DECLINED_GONE [label="gone ~18%"]; + INTEREST_EXPRESSED -> COLLECTED [label="immediate ~1%"]; + + ENQUIRY -> ARRANGING [label="answered ~10%"]; + ENQUIRY -> DECLINED_GONE [label="unsuitable ~2%"]; + ENQUIRY -> WITHDRAWN [label="~1%"]; + + WAITLISTED -> ARRANGING [label="freed ~2%"]; + WAITLISTED -> DECLINED_GONE [label="other got it ~3%"]; + + ARRANGING -> HANDOVER_PENDING [label="addr+time ~18%"]; + ARRANGING -> COLLECTED [label="collected ~12%"]; + ARRANGING -> STALLED [label="quiet ~8%"]; + ARRANGING -> DECLINED_GONE [label="given away ~5%"]; + ARRANGING -> WITHDRAWN [label="backs out ~1%"]; + + PROMISED -> COLLECTED [label="collected ~10%"]; + PROMISED -> RENEGED [label="Reneged ~2%"]; + PROMISED -> WITHDRAWN [label="~1%"]; + PROMISED -> STALLED [label="quiet ~2%"]; + + HANDOVER_PENDING -> STALLED [label="no-show ~2%"]; + HANDOVER_PENDING -> WITHDRAWN [label="cancels ~0.5%"]; + HANDOVER_PENDING -> RENEGED [label="~0.3%"]; +} diff --git a/plans/freegle-chat-flow-offer.png b/plans/freegle-chat-flow-offer.png new file mode 100644 index 0000000000..cc52b7cade Binary files /dev/null and b/plans/freegle-chat-flow-offer.png differ diff --git a/plans/freegle-chat-flow-wanted-workflow.json b/plans/freegle-chat-flow-wanted-workflow.json new file mode 100644 index 0000000000..4271011981 --- /dev/null +++ b/plans/freegle-chat-flow-wanted-workflow.json @@ -0,0 +1,68 @@ +{ + "id": "freegle-chat-flow-wanted", + "name": "Freegle Chat Flow - WANTED (reply offering an item)", + "description": "Empirical state machine of a single User2User chat started from a WANTED post (a giver replies offering an item, or signposting one, to the wanter who posted). Derived from ~2,980 WANTED-initiated conversations on the live database over one month (2026-05). Roles are FLIPPED vs the Offer flow: the REPLIER is the giver, the POSTER is the wanter/receiver. The offerer machinery (Promised/Address/Reneged) is essentially unused here. Percentages are the approximate share of all WANTED chats traversing that arm; terminal totals are measured, intermediate splits estimated.", + "initialState": "START", + "guardrails": "Never assume the wanter still needs the item - many are already sorted. A 'Completed' here closes the WANTER's post and may mean they got the item from someone else, not this giver.", + "states": { + "START": { + "nodeType": "start", + "description": "Chat room created. The giver replies to a WANTED with an 'Interested' message.", + "writeActions": ["send_message"] + }, + "OFFER_MADE": { + "nodeType": "agent", + "description": "Giver has offered something to the wanter (type=Interested). Two flavours: (a) offering their own item ('I have a spare X you can have'), or (b) SIGNPOSTING - ~16% of openers just point the wanter to another source (a link, another listing, a local project, the NHS) without having the item. Awaiting the wanter to respond.", + "prompt": "Offer the item plainly or, if you don't have it, point them to where they might get one. Ask one question only if suitability is genuinely unclear.", + "readActions": ["get_message", "get_user_info"], + "writeActions": ["send_message"], + "timeout": { "duration": 172800000, "toState": "GHOSTED" } + }, + "ENQUIRY": { + "nodeType": "agent", + "description": "Suitability check - the offered item may not match what was wanted (wrong size, condition, type). The wanter weighs it before accepting.", + "prompt": "Give the details the wanter needs to judge suitability (size, condition, photos, location).", + "readActions": ["get_message"], + "writeActions": ["send_message"], + "timeout": { "duration": 172800000, "toState": "GHOSTED" } + }, + "ARRANGING": { + "nodeType": "agent", + "description": "Wanter accepted; agreeing logistics. Unlike Offers, the handover may be COLLECTION (wanter collects) or DELIVERY (~9% - the giver offers to drop it round).", + "prompt": "Agree whether the wanter collects or you deliver, and a time/place. Don't over-message once settled.", + "readActions": ["get_user_info"], + "writeActions": ["send_message"], + "timeout": { "duration": 259200000, "toState": "STALLED" } + }, + "RECEIVED": { + "nodeType": "end", + "description": "SUCCESS - the wanter got the item from this giver (collected or delivered). ~24% of WANTED chats. Signalled by collection/delivery text or a Completed marked by the wanter after real arranging." + }, + "DECLINED_SORTED": { + "nodeType": "end", + "description": "The wanter does not take this giver's item. ~6%+ measured (under-counted - many such cases simply ghost). Reasons: 'thanks, I'm already sorted / got one', the item isn't suitable, or the giver only signposted and the wanter sourced it elsewhere. Amicable." + }, + "GHOSTED": { + "nodeType": "end", + "description": "The wanter never replies (~38% - by far the biggest outcome). Usually because they were already sorted by the time the offer arrived, or the offer wasn't quite right. The high ghost rate is the defining feature of the Wanted flow." + }, + "STALLED": { + "nodeType": "end", + "description": "Two-sided conversation that faded during arranging with no recorded outcome (~30%). Wanteds mark Completed far less often (~22% vs ~52% for Offers), so many real handovers sit here as latent success." + } + }, + "transitions": [ + { "id": "t_open", "from": "START", "to": "OFFER_MADE", "trigger": "unconditional", "metadata": { "event": "Interested", "approx_share": "100%" } }, + + { "id": "t_om_arrange", "from": "OFFER_MADE", "to": "ARRANGING", "trigger": "llm_decision", "condition": "Wanter accepts the offer", "metadata": { "approx_share": "~46%" } }, + { "id": "t_om_ghost", "from": "OFFER_MADE", "to": "GHOSTED", "trigger": "host_driven", "condition": "Wanter never replies (already sorted / not suitable)", "metadata": { "signal": "timeout", "approx_share": "~38%" } }, + { "id": "t_om_enquiry", "from": "OFFER_MADE", "to": "ENQUIRY", "trigger": "llm_decision", "condition": "Wanter asks whether it is suitable", "metadata": { "approx_share": "~10%" } }, + { "id": "t_om_sorted", "from": "OFFER_MADE", "to": "DECLINED_SORTED", "trigger": "llm_decision", "condition": "Wanter is already sorted, declines, or was only signposted", "metadata": { "approx_share": "~6%" } }, + + { "id": "t_enq_arrange", "from": "ENQUIRY", "to": "ARRANGING", "trigger": "llm_decision", "condition": "Suitable, proceeding", "metadata": { "approx_share": "~7%" } }, + { "id": "t_enq_sorted", "from": "ENQUIRY", "to": "DECLINED_SORTED", "trigger": "llm_decision", "condition": "Not suitable", "metadata": { "approx_share": "~3%" } }, + + { "id": "t_arr_received","from": "ARRANGING", "to": "RECEIVED", "trigger": "host_driven", "condition": "Collected or delivered (text / Completed by wanter)", "metadata": { "approx_share": "~24%" } }, + { "id": "t_arr_stall", "from": "ARRANGING", "to": "STALLED", "trigger": "host_driven", "condition": "Goes quiet before a recorded outcome", "metadata": { "signal": "timeout", "approx_share": "~30%" } } + ] +} diff --git a/plans/freegle-chat-flow-wanted.dot b/plans/freegle-chat-flow-wanted.dot new file mode 100644 index 0000000000..5f6eed43ce --- /dev/null +++ b/plans/freegle-chat-flow-wanted.dot @@ -0,0 +1,25 @@ +digraph wanted { + rankdir=LR; bgcolor="white"; nodesep=0.45; ranksep=1.1; splines=spline; + node [shape=box, style="rounded,filled", fontname="Helvetica", fontsize=11, fillcolor="#eef2ff", color="#5566aa", penwidth=1.4]; + edge [fontname="Helvetica", fontsize=9, color="#9aa0a6"]; + + start [shape=circle, width=0.16, label="", fillcolor=black, color=black]; + OFFER_MADE [label="OFFER_MADE\n(offer / signpost)"]; + RECEIVED [fillcolor="#d7f5dd", color="#2e7d32", label="RECEIVED\n~24%"]; + GHOSTED [fillcolor="#fde2e2", color="#c62828", label="GHOSTED\n~38%"]; + DECLINED_SORTED [fillcolor="#fde2e2", color="#c62828", label="DECLINED_SORTED\n~6%*"]; + STALLED [fillcolor="#fff4d6", color="#f9a825", label="STALLED\n~30%"]; + + start -> OFFER_MADE [label="100%"]; + OFFER_MADE -> ARRANGING [weight=8, label="wanter accepts ~46%"]; + ARRANGING -> RECEIVED [weight=8, label="collected / delivered ~24%"]; + + OFFER_MADE -> ENQUIRY [label="suitable? ~10%"]; + OFFER_MADE -> GHOSTED [label="no reply ~38%"]; + OFFER_MADE -> DECLINED_SORTED [label="sorted ~6%"]; + + ENQUIRY -> ARRANGING [label="suitable ~7%"]; + ENQUIRY -> DECLINED_SORTED [label="not suitable ~3%"]; + + ARRANGING -> STALLED [label="goes quiet ~30%"]; +} diff --git a/plans/freegle-chat-flow-wanted.png b/plans/freegle-chat-flow-wanted.png new file mode 100644 index 0000000000..c34eb8b2fb Binary files /dev/null and b/plans/freegle-chat-flow-wanted.png differ diff --git a/plans/freegle-chat-flow.md b/plans/freegle-chat-flow.md new file mode 100644 index 0000000000..98b0731410 --- /dev/null +++ b/plans/freegle-chat-flow.md @@ -0,0 +1,144 @@ +# Freegle Member-to-Member Chat Flow + +Empirical state machines of `User2User` chats between two freeglers, suitable for +[ai-flower](https://github.com/freegle/ai-flower). Derived from the **live production database** +(via the V2 live API tunnel, port 11234) by analysing **~40,000 new conversations over one month +(2026-05)** and reading ~260 full conversations. + +**Offer and Wanted chats behave differently, so each has its own flow:** + +| | initiated by | opener role | files | +|---|---|---|---| +| **OFFER** (~92% of chats) | taker replies to an offered item | taker asks the **giver** | `freegle-chat-flow-offer.png` · `…-offer-workflow.json` | +| **WANTED** (~8% of chats) | giver replies offering an item | **giver** offers the wanter | `freegle-chat-flow-wanted.png` · `…-wanted-workflow.json` | + +It complements the offerer-side concierge FSM in +[`active/freegle-helper-concierge.md`](active/freegle-helper-concierge.md) (one offerer managing +many repliers in a bulk event); these model the 1:1 conversation itself. + +## The message-type spine + +`chat_messages.type` encodes most of the machine. The giver (item owner) issues the system events +`Promised`, `Address`, `Reneged`; `Completed` can be marked by either side. Month distribution +(User2User): `Default` 177k · `Interested` 56k · `Completed` 32k · `Promised` 10k · `Image` 2.6k · +`Address` 2.3k · `Reneged` 1.5k · `Reminder` 790 · `Nudge` 155. + +## `Completed` ≠ success — the key correction + +`Completed` only means the **post was closed**. It does **not** prove the replier in *this* chat got +the item. The common terse `Interested → Completed` with no real giver reply is the automatic +*"sorry, this has been taken"* broadcast — the item went to **someone else**. Even a two-sided chat +that ends *"sorry, gone to someone else"* is **not** a collection. Counting only genuine +handover-to-this-replier gives the honest split below. + +## Outcome distribution (measured) + +| Outcome | **OFFER** | **WANTED** | +|---|---:|---:| +| ✅ Collected / received by this replier | **~43%** | **~24%** | +| ❌ Gone elsewhere / declined | **~29%** | ~6%* | +| 👻 Ghosted (no reply) | ~11% | **~38%** | +| ⏸️ Stalled (some latent success) | ~15% | **~30%** | +| ↩️ Reneged | ~2% | ~0% | + +\* Wanted declines are under-measured — many "already sorted" cases simply ghost, so the true +Wanted decline rate is higher and the ghost rate correspondingly softer. Wanteds barely use the +`Promised`/`Address`/`Reneged` machinery (it's offerer-side), mark `Completed` far less (~22% vs +~52%), ghost much more (the wanter is often already sorted), and ~16% of opener messages are +**signposts** (pointing the wanter elsewhere) rather than an actual offer; ~9% offer **delivery**. + +Edge labels below are the **approximate share of conversations** taking that arm; terminal-state +totals are measured, intermediate splits are estimates (`~`). + +--- + +## OFFER flow — taker requests a given item + +![Offer chat flow](freegle-chat-flow-offer.png) + +```mermaid +stateDiagram-v2 + direction LR + [*] --> INTEREST_EXPRESSED : taker asks · 100% + INTEREST_EXPRESSED --> ARRANGING : available · ~49% + INTEREST_EXPRESSED --> ENQUIRY : question · ~13% + INTEREST_EXPRESSED --> WAITLISTED : behind · ~5% + INTEREST_EXPRESSED --> PROMISED : immediate · ~3% + INTEREST_EXPRESSED --> GHOSTED : no reply · ~11% + INTEREST_EXPRESSED --> DECLINED_GONE : gone · ~18% + INTEREST_EXPRESSED --> COLLECTED : immediate · ~1% + ENQUIRY --> ARRANGING : answered · ~10% + ENQUIRY --> DECLINED_GONE : unsuitable · ~2% + ENQUIRY --> WITHDRAWN : not suitable · ~1% + WAITLISTED --> ARRANGING : freed · ~2% + WAITLISTED --> DECLINED_GONE : other got it · ~3% + ARRANGING --> PROMISED : Promised · ~20% + ARRANGING --> HANDOVER_PENDING : addr+time · ~18% + ARRANGING --> COLLECTED : collected · ~12% + ARRANGING --> STALLED : quiet · ~8% + ARRANGING --> DECLINED_GONE : given away · ~5% + ARRANGING --> WITHDRAWN : backs out · ~1% + PROMISED --> HANDOVER_PENDING : Address · ~8% + PROMISED --> COLLECTED : collected · ~10% + PROMISED --> RENEGED : Reneged · ~2% + PROMISED --> WITHDRAWN : backs out · ~1% + PROMISED --> STALLED : quiet · ~2% + HANDOVER_PENDING --> COLLECTED : on my way · ~20% + HANDOVER_PENDING --> STALLED : no-show · ~2% + HANDOVER_PENDING --> WITHDRAWN : cancels · ~0.5% + HANDOVER_PENDING --> RENEGED : ~0.3% + COLLECTED --> [*] +``` + +Each conversational state is its own box (matching the `freegle-helper-concierge` FSM granularity): +`INTEREST_EXPRESSED → ARRANGING → PROMISED → HANDOVER_PENDING → COLLECTED` is the happy-path spine; +detours (`ENQUIRY`, `WAITLISTED`) rejoin it and off-ramps drop to the terminal outcomes. + +--- + +## WANTED flow — replier offers an item to the wanter + +![Wanted chat flow](freegle-chat-flow-wanted.png) + +```mermaid +stateDiagram-v2 + direction LR + [*] --> OFFER_MADE : giver offers / signposts · 100% + OFFER_MADE --> ARRANGING : wanter accepts · ~46% + OFFER_MADE --> ENQUIRY : suitable? · ~10% + OFFER_MADE --> GHOSTED : no reply · ~38% + OFFER_MADE --> DECLINED_SORTED : sorted · ~6% + ENQUIRY --> ARRANGING : suitable · ~7% + ENQUIRY --> DECLINED_SORTED : not suitable · ~3% + ARRANGING --> RECEIVED : collected / delivered · ~24% + ARRANGING --> STALLED : goes quiet · ~30% + RECEIVED --> [*] +``` + +The Wanted flow is simpler — no `Promised`/`Address`/`Reneged`, no waitlist machinery — but ghosts +far more. Two Wanted-only flavours sit inside the boxes: ~16% of `OFFER_MADE` openers are +**signposts** (pointing the wanter elsewhere) rather than an actual offer, and ~9% of `ARRANGING` +handovers are **delivery** (the giver drops it round) instead of collection. + +--- + +## Validation + +A fresh random sample of 130 conversations was read and classified against the model: **130/130 +fit**, no conversation needed a state outside it (a few were still in-progress at snapshot). Wanted +conversations were then sampled separately to derive the distinct Wanted flow. Refinements folded +in: a small share open with a plain `Default`; a no-show auto-`Reneged` can reverse into +`ARRANGING`; multi-item chats loop the arrange→collect arc per item; rude/mismatch rejections fold +into `DECLINED_GONE`/`DECLINED_SORTED`. + +## How this was produced + +1. Live DB reached read-only through the V2 live API tunnel (`apiv2-live`, `db-live:11234`). +2. Profiled `chat_messages.type` for `User2User` chats `created >= 2026-05-01`, split by Offer/Wanted. +3. Bucketed all chats, **separating genuine handover-to-this-replier from bare `Completed` closures + and "gone elsewhere" declines** so success is never over-counted. +4. Stratified-sampled and read ~260 full conversations across both post types and every bucket. + +> Privacy note: the raw conversation samples contain member PII (names, addresses, phone numbers) +> and were deliberately **not** persisted. Only this PII-free aggregate analysis and the workflow +> definitions are kept. diff --git a/plans/promise-detection-session-brief.md b/plans/promise-detection-session-brief.md new file mode 100644 index 0000000000..1a29a6cb78 --- /dev/null +++ b/plans/promise-detection-session-brief.md @@ -0,0 +1,180 @@ +# Promise-detection — session brief (for a parallel explorer) + +**Date:** 2026-06-01 +**Branch:** `plans/freegle-chat-flow` (PR #594) — this brief + a working **linear baseline** scaffold. +**Goal:** detect the **`promised`** state in a Freegle 1:1 chat (the point an exchange is committed +to) directly from natural-language dialogue, per the spec *"Detecting Item Promises from +Natural-Language Dialogue in Freegle Conversations"*. + +This session built the **pragmatic §5.2 baseline** (linear classifier over TF-IDF word+char +n-grams). **You are invited to explore the other options in parallel** off the same dataset +contract — see "Open alternatives" at the bottom. + +## What the task is (from the spec) + +- Target = **promise made** (committed), NOT *taken/honoured* (taken is unreliable from text — it + depends on events outside the dialogue). Reneged is out of scope. +- Ground-truth label = the logged **`Promised`** chat event (free supervision, no manual labels). +- It's **dialogue inference**: meaning is context-dependent, the state emerges across turns, speaker + identity matters → inputs are **windowed, speaker-labelled spans**, not isolated messages. +- Priorities: privacy, low cost, self-hostable, interpretable. +- Eval: split **by conversation**; report **P/R/F1 + PR-AUC** (imbalanced ~16.5% positive, so + accuracy lies); compare against a **keyword baseline**. + +## Decisions locked this session (with the user) + +1. **PHP** (not Python), in **`iznik-batch`** (Laravel 12) as **artisan commands**. Go only if + inline <1s inference ever demands it (would live in `iznik-server-go`). +2. **Rubix ML** (`rubix/ml ^2.5`) — the only new dependency; no ML lib existed. Native PHP CSV + (`fputcsv`/`fgetcsv`), Laravel DB layer — no other new deps. +3. **Static dataset → CSV**, extracted once; training reads the file, never the live DB. +4. **Address is signal, not pure leakage.** The spec said drop `Address`; the user correctly noted + that *actively sharing an address is progression toward a promise*. Resolution: drop only the + co-timed empty `Address` **event token** (no text anyway); **keep** a free-text address typed in + a `Default` message — but **normalise** it (and postcodes/phones/emails/URLs) to placeholder + tokens (`
` …). This keeps the signal, removes PII, and stops memorising specifics. The + real leakage guard is the **±1 tolerance band** around the transition + **split-by-conversation**. +5. **Charset matters** (utf8mb4: emoji + accents + invalid bytes). Spans are forced to valid UTF-8; + all tokenisation is multibyte-safe (`preg_split('//u')`, `mb_*`); CSV is UTF-8 no-BOM. + +## Data findings (live DB, via V2 live API tunnel, read-only) + +- User2User rooms with a `Promised` event ≈ **16.5%** (stable across windows). +- Volumes: 1mo 40k rooms / 6.4k positive · 3mo 113k / 18.9k · **6mo 214k / 35.4k**. +- Role resolution validated on live rooms: **Offer** → post owner = GIVER, replier = TAKER; + **Wanted** → post owner = TAKER (wanter), replier = GIVER. The `Promised` event's position maps to + a real-text-turn index; the GIVER's *"I'll promise it to you now"* + subsequent address-sharing are + textbook predictive cues. + +## Sample sizing + +- **Dev/TDD:** ~2,000 rooms. +- **Scaling:** **6 months** (214k rooms, 35k positives), with per-room negative capping so the CSV + stays manageable. All parameterised on `promise:extract` (`--since`, `--max-rooms`, `--window=8`, + `--tolerance=1`, `--negatives-per-room`). + +## The dataset contract (THE shared interface — reuse this) + +`iznik-batch/app/Services/Promise/CONTRACT.md` is authoritative. CSV columns: +`room_id, post_type, end_turn, promise_turn, label, span` — **model trains on `span` only**; the +rest are metadata for group-split (`room_id`) and the timing metric (`end_turn` vs `promise_turn`). +`span` = windowed, `[GIVER]`/`[TAKER]`-tagged, PII-normalised dialogue. A synthetic fixture lives at +`iznik-batch/tests/Fixtures/Promise/dataset_fixture.csv`. + +**Any alternative model can consume the exact same CSV** — that's the point of the seam. + +## What's built (this session, linear baseline §5.2) + +Under `iznik-batch/app/Services/Promise/` + `app/Console/Commands/Promise/`, all `php -l` clean, +built TDD with pure PHPUnit unit tests in `tests/Unit/Promise/`: + +- **Extraction:** `PiiNormaliser`, `DatasetExtractor` (pure windowing/label/tolerance core), + `promise:extract` command (DB → CSV). +- **Training/eval:** `CharNgramTokenizer` (word 1–2 + char 3–5, mb-safe), `DatasetReader`, + `KeywordBaseline` (reference), `FeaturePipeline` (WordCountVectorizer→TfIdf→LogisticRegression), + `Evaluator` (P/R/F1, ROC-AUC, PR-AUC, threshold sweep, **promise-timing offsets**, top ±n-grams, + error samples), `promise:train` command (group-split, report.json + persisted model.rbx). + +**Status:** integrated and **running end-to-end on real data.** Result on a 768-room dev sample +(2,516 train / 640 test, split by conversation, 13% positive), **after fixing a CSV data-corruption +bug** (see below): + +| Metric | Linear baseline | Keyword baseline | +|---|---:|---:| +| Precision | 0.311 | 0.200 | +| Recall | 0.154 | 0.978 | +| F1 | 0.206 | **0.333** | +| ROC-AUC | **0.591** | — | +| PR-AUC | **0.198** | — | + +**Sobering and honest:** ROC-AUC 0.591 is barely above chance (0.5); PR-AUC 0.198 is only marginally +above the 0.13 base rate; and the **keyword baseline beats the linear model on F1**. On clean data the +cheap §5.2 baseline **essentially does not work** — near-chance ranking, ~31% precision. + +> ⚠️ **Earlier numbers were wrong.** An initial run reported F1 0.357 / ROC-AUC 0.751 / PR-AUC 0.458 +> "beating the baseline". Those were computed on a **corrupted dataset**: `fputcsv`/`fgetcsv` used +> PHP's default backslash escaping, and spans full of `\u..` artefacts + quotes broke the round-trip, +> silently **merging ~30% of rows** (3,146 written → 1,772 read, with mismatched labels). Fixed with +> RFC-4180 quoting (`escape: ''`); always verify `rows-written == records-read`. + +Fixes applied during integration (the two defects in the original push are resolved): +- **Real probabilities** — replaced the Rubix `Pipeline` (whose `proba()` double-transforms → + `IncorrectDatasetDimensionality`) with explicitly fit-once / transform-only transformers, so AUC, + the threshold sweep and timing are now meaningful. +- **Top n-grams implemented** — `featureImportances()` ranked, annotated with empirical + P(label=1 | token) for direction (doubles as a leakage smoke-test). +- Samples wrapped as `[span]` rows; predictions cast to int for error sampling; model persisted via + serialize. + +**Known scaling limit:** Rubix vectorises **densely** (one int per vocab term per sample), so char +3–5-grams blow memory at scale (OOM at 1.4k samples × 20k vocab). Mitigated for dev via +`maxVocabularySize=8000`, `minDocumentCount=3`, and `php -d memory_limit=3G`. The full 6-month run +will need harder vocab capping (or word-grams only, or a sparse representation) — Rubix has no sparse +matrices. + +**Immediate next steps:** run the `Promise` unit suite green via the status API +(`POST /api/tests/laravel {"filter":"Promise","testsuite":"Unit"}`); make **expected cost** the +headline metric (see below); solve the dense-matrix limit for the full-scale run. + +## Success criterion (cost-based) — the real bar, and the verdict + +Generic F1 is the wrong target for a detector; success is an **operating point set by the relative +cost of the two errors**. Stated cost for this use case: a **false positive (wrongly claiming a +promise) is 20× worse than a false negative (missing one)** — the action (e.g. a user-facing prompt) +must not misfire. Consequences: + +- **Decision rule:** fire only when `P(promise) > 20/21 ≈ 0.95` (not 0.5). Equivalently, minimise + **expected cost = 20·FP + 1·FN** (a precision-weighted F-β, β ≈ 0.22). This is Neyman–Pearson / + cost-sensitive classification, not "maximise F1". +- **Bar to beat "never fire":** never-firing costs only the missed promises. A model beats it only + if `TP/FP > 20`, i.e. **precision > ~95% at any recall**. Below that, each false alarm costs more + than the promises it catches → the model is **net-negative versus doing nothing**. +- **Verdict on the linear baseline (clean data):** precision tops out ~33% ≪ 95%, ROC-AUC 0.591 + (near chance), PR-AUC 0.198 (≈ the 0.13 base rate), and it **loses to the keyword baseline on F1**. + Under 20:1 it is **net-negative vs never firing — not deployable**, and on clean data it isn't even + a convincing *proof of signal*. The bar for the embedding/DST tracks is unchanged: **precision + ≥ ~95%** at usable recall. +- **Bar for the alternatives:** the embedding / DST models must reach **precision ≥ ~95%** (at + usable recall) to be worth shipping autonomously — a high-precision target that favours + context-aware models (DST) over bag-of-n-grams. + +**Design implication:** a 20:1 cost pushes off "single autonomous classifier" toward (a) firing only +in the top-confidence sub-regime, (b) a two-stage high-recall → precise-confirm pipeline, +(c) human-in-the-loop (a wrong flag costs ~0), or (d) a silent background annotation where the FP +cost collapses. + +**Honest method notes:** the only "tuning" tried was the **threshold sweep** — re-labelling one +trained model's outputs at different cut-offs, which slides along a *fixed* PR curve and cannot lift +it. Untried levers (the ones that move the curve): more data, **word-n-gram-only / cleaner +features**, class weighting for the 15% imbalance, regularisation / cross-validation. The +top-importance features are currently dominated by **noisy char n-grams** — fragments straddling +word/punctuation/speaker-tag boundaries (`c:s:thi`, `c:upth`), which wreck interpretability and +likely dilute signal; a word-grams-only pass is the cheap next experiment. + +## How to run (once green) + +``` +php artisan promise:extract --max-rooms=2000 --out=storage/promise/dev.csv # dev +php artisan promise:extract --since="6 months ago" --negatives-per-room=3 --out=storage/promise/full.csv +php artisan promise:train --csv=storage/promise/dev.csv # → report.json + model.rbx +``` +Tests: `curl -s -X POST http://localhost:8081/api/tests/laravel -H 'Content-Type: application/json' -d '{"filter":"Promise","testsuite":"Unit"}'` then poll `/api/tests/laravel/status`. + +## Open alternatives to explore in parallel (the spec's other approaches) + +Same CSV contract, swap the model. Good parallel tracks for another explorer: + +1. **Embedding-based classification (§5.1, "leading alternative").** Embed each `span` (a + self-hostable sentence-transformer / the project's existing `embedding-sidecar` or `knn-server`) + → logistic regression on the vectors, or k-NN by cosine to labelled spans. More robust to + paraphrase than lexical n-grams; compare PR-AUC/timing against this baseline on the same split. +2. **Dialogue state tracking (§6, the "ambitious" approach).** Generative DST: fine-tune a + self-hostable T5/Flan-T5/BART to read the serialised speaker-tagged dialogue and emit + `promised: yes/no` (+ `pickup_time`, `item`). Identifies the *point* of promise, not just its + presence. Heavier; better for the latency-tolerant background-marking mode. +3. **Zero-shot NLI** (DeBERTa/BART-MNLI) as a quick training-free comparator. + +Whatever you build, evaluate it the **same way** (this session's `Evaluator` contract: P/R/F1, +PR-AUC, timing offsets, vs the keyword baseline, split by conversation) so the tracks are +comparable. Keep PII out of any committed artefact (normalise at extraction; the raw chat samples +were deliberately not persisted this session).