From f0de6d9e5c94fc190aebdb8301b8fd259cce939f Mon Sep 17 00:00:00 2001 From: Daniel Ecer Date: Tue, 16 Jun 2026 19:13:52 +0100 Subject: [PATCH] Fix is_http for segmentation model to use token-based detection MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit part of https://github.com/eLifePathways/ScienceBeam2.0/issues/88 The segmentation model's FeatureDef called get_str_is_http(), which reads self.is_http. However, SegmentationLineFeaturesProvider reuses a single SegmentationLineFeatures instance and never sets is_http in its line loop, so the feature always returned '0'. Switched to get_str_is_http_token_based() (token_text.lower().startswith ('http')), which matches GROBID's Pattern.compile("http(s)?").find() check on the first token of each line. This affects 11 docs with 287 total Δlabels in the segmentation model, 142 of which cascade into header detection and downstream affiliation extraction. --- sciencebeam_parser/models/segmentation/data.py | 2 +- tests/models/segmentation/data_test.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/sciencebeam_parser/models/segmentation/data.py b/sciencebeam_parser/models/segmentation/data.py index 276e94cc..556564fb 100644 --- a/sciencebeam_parser/models/segmentation/data.py +++ b/sciencebeam_parser/models/segmentation/data.py @@ -444,7 +444,7 @@ def __init__( FeatureDef('is_year', lambda f: f.get_str_is_year()), FeatureDef('is_month', lambda f: f.get_str_is_month()), FeatureDef('is_email', lambda f: f.get_str_is_email()), - FeatureDef('is_http', lambda f: f.get_str_is_http()), + FeatureDef('is_http', lambda f: f.get_str_is_http_token_based()), FeatureDef('relative_document_position', lambda f: f.get_str_relative_document_position()), FeatureDef('relative_page_position', diff --git a/tests/models/segmentation/data_test.py b/tests/models/segmentation/data_test.py index 44cebee1..1e9f4aaa 100644 --- a/tests/models/segmentation/data_test.py +++ b/tests/models/segmentation/data_test.py @@ -512,6 +512,22 @@ def test_should_provide_repetitive_pattern_feature( }, ] + def test_should_set_is_http_for_line_starting_with_http_url( + self, + features_provider: SegmentationLineFeaturesProvider + ): + layout_document = LayoutDocument(pages=[ + LayoutPage(blocks=[ + LayoutBlock.for_text('https://doi.org/10.1101/043794'), + LayoutBlock.for_text('regular text line') + ]) + ]) + feature_values = [ + features.get_str_is_http_token_based() + for features in _iter_line_features(features_provider, layout_document) + ] + assert feature_values == ['1', '0'] + class TestSegmentationLineFeaturesProviderVectorAround: def _make_doc_with_block_and_graphics(