diff --git a/enterprise_catalog/apps/ai_curation/utils/algolia_utils.py b/enterprise_catalog/apps/ai_curation/utils/algolia_utils.py index 14cb46df..10f171f9 100644 --- a/enterprise_catalog/apps/ai_curation/utils/algolia_utils.py +++ b/enterprise_catalog/apps/ai_curation/utils/algolia_utils.py @@ -5,9 +5,7 @@ from django.utils.html import strip_tags -from enterprise_catalog.apps.catalog.algolia_utils import ( - get_initialized_algolia_client, -) +from enterprise_catalog.apps.api_client.algolia import AlgoliaSearchClient @dataclass @@ -60,6 +58,15 @@ def extract_program_data(hit: dict): } +def get_initialized_algolia_client(): + """ + Initializes and returns an Algolia client for updating search indices + """ + algolia_client = AlgoliaSearchClient() + algolia_client.init_index() + return algolia_client + + def fetch_catalog_metadata_from_algolia(enterprise_catalog_query_title: str): """ Returns the ocm_courses, exec_ed_courses, programs, subjects from the diff --git a/enterprise_catalog/apps/ai_curation/utils/open_ai_utils.py b/enterprise_catalog/apps/ai_curation/utils/open_ai_utils.py index 83d7521a..914386ff 100644 --- a/enterprise_catalog/apps/ai_curation/utils/open_ai_utils.py +++ b/enterprise_catalog/apps/ai_curation/utils/open_ai_utils.py @@ -81,3 +81,51 @@ def get_keywords_to_prose(query): return keywords_to_prose[0] return '' + + +def translate_to_spanish(text): + """ + Translate the given text to Spanish using OpenAI. + + Args: + text (str): The text to translate. + + Returns: + str: The translated text in Spanish. + """ + if not text: + return '' + + content = ( + f"Translate the following text to Spanish. " + f"Return ONLY the translated text without any comments, " + f"explanations, or conversational responses:\n\n{text}" + ) + messages = [ + { + 'content': content + } + ] + translated_text = chat_completions(messages=messages) + + if translated_text: + return translated_text[0] + return '' + + +def translate_object_fields(data, fields): + """ + Translate specified fields in a dictionary to Spanish. + + Args: + data (dict): The dictionary containing fields to translate. + fields (list): A list of field names to translate. + + Returns: + dict: A new dictionary with translated fields. + """ + translated_data = data.copy() + for field in fields: + if field in data and isinstance(data[field], str): + translated_data[field] = translate_to_spanish(data[field]) + return translated_data diff --git a/enterprise_catalog/apps/api/tasks.py b/enterprise_catalog/apps/api/tasks.py index e380d57c..78e9dc4f 100644 --- a/enterprise_catalog/apps/api/tasks.py +++ b/enterprise_catalog/apps/api/tasks.py @@ -27,6 +27,7 @@ _algolia_object_from_product, configure_algolia_index, create_algolia_objects, + create_spanish_algolia_object, get_algolia_object_id, get_initialized_algolia_client, get_pathway_course_keys, @@ -867,6 +868,32 @@ def add_video_to_algolia_objects( batched_metadata = _batched_metadata_with_queries(json_metadata, queries) _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata) + # Create and index Spanish version + json_metadata_es = create_spanish_algolia_object(json_metadata, video) + + if json_metadata_es: + # enterprise customer uuids for Spanish + batched_metadata_es = _batched_metadata( + json_metadata_es, + customer_uuids, + 'enterprise_customer_uuids', + '{}-customer-uuids-{}', + ) + _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata_es) + + # enterprise catalog uuids for Spanish + batched_metadata_es = _batched_metadata( + json_metadata_es, + catalog_uuids, + 'enterprise_catalog_uuids', + '{}-catalog-uuids-{}', + ) + _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata_es) + + # enterprise catalog queries for Spanish + batched_metadata_es = _batched_metadata_with_queries(json_metadata_es, queries) + _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata_es) + def add_metadata_to_algolia_objects( metadata, @@ -954,6 +981,32 @@ def add_metadata_to_algolia_objects( batched_metadata = _batched_metadata_with_queries(json_metadata, queries) _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata) + # Create and index Spanish version + json_metadata_es = create_spanish_algolia_object(json_metadata, metadata) + + if json_metadata_es: + # enterprise catalog uuids for Spanish + batched_metadata_es = _batched_metadata( + json_metadata_es, + catalog_uuids, + 'enterprise_catalog_uuids', + '{}-catalog-uuids-{}', + ) + _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata_es) + + # enterprise customer uuids for Spanish + batched_metadata_es = _batched_metadata( + json_metadata_es, + customer_uuids, + 'enterprise_customer_uuids', + '{}-customer-uuids-{}', + ) + _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata_es) + + # enterprise catalog queries for Spanish + batched_metadata_es = _batched_metadata_with_queries(json_metadata_es, queries) + _add_in_algolia_products_by_object_id(algolia_products_by_object_id, batched_metadata_es) + def get_algolia_objects_from_course_content_metadata(content_metadata): content_key = content_metadata.content_key @@ -1059,6 +1112,7 @@ def _get_algolia_products_for_batch( ) ).prefetch_related( Prefetch('catalog_queries', queryset=all_catalog_queries), + 'translations', ) if getattr(settings, 'SHOULD_INDEX_COURSES_WITH_RESTRICTED_RUNS', False): # Make the courses that we index actually contain restricted runs in the payload. @@ -1081,6 +1135,7 @@ def _get_algolia_products_for_batch( parent_content_key__in=course_content_keys ).prefetch_related( Prefetch('catalog_queries', queryset=all_catalog_queries), + 'translations', ) course_run_content_keys = [cm.content_key for cm in content_metadata_courseruns] videos = Video.objects.filter( diff --git a/enterprise_catalog/apps/api/tests/test_algolia_integration.py b/enterprise_catalog/apps/api/tests/test_algolia_integration.py new file mode 100644 index 00000000..3ea3ce46 --- /dev/null +++ b/enterprise_catalog/apps/api/tests/test_algolia_integration.py @@ -0,0 +1,150 @@ +""" +Integration tests for Algolia indexing. +""" +from unittest import mock + +from django.db import connection +from django.test import TestCase +from django.test.utils import CaptureQueriesContext + +from enterprise_catalog.apps.api.tasks import _reindex_algolia +from enterprise_catalog.apps.catalog.constants import COURSE +from enterprise_catalog.apps.catalog.models import ContentTranslation +from enterprise_catalog.apps.catalog.tests.factories import ( + CatalogQueryFactory, + ContentMetadataFactory, + EnterpriseCatalogFactory, +) + + +class AlgoliaIntegrationTests(TestCase): + """ + Integration tests for Algolia indexing logic. + """ + + @mock.patch('enterprise_catalog.apps.api.tasks._retrieve_inactive_tmp_indices') + @mock.patch('enterprise_catalog.apps.api.tasks._delete_indices') + @mock.patch('enterprise_catalog.apps.api.tasks.configure_algolia_index') + @mock.patch('enterprise_catalog.apps.api.tasks.get_initialized_algolia_client') + def test_algolia_indexing_with_spanish_translation( + self, + mock_get_client, + _mock_configure, + _mock_delete, + _mock_retrieve + ): + """ + Test that _reindex_algolia pushes double the objects when Spanish translation exists. + """ + # Setup + mock_client = mock.Mock() + mock_get_client.return_value = mock_client + + # Create a course + course = ContentMetadataFactory(content_type=COURSE, content_key='course-v1:Test+Course') + + # Create catalog query and associate with course + catalog_query = CatalogQueryFactory() + course.catalog_queries.add(catalog_query) + + # Create enterprise catalog associated with query + EnterpriseCatalogFactory(catalog_query=catalog_query) + + content_keys = [course.content_key] + + # --- Execution 1: No Translation --- + _reindex_algolia( + indexable_content_keys=content_keys, + nonindexable_content_keys=[], + dry_run=False + ) + + # Verify 3 objects pushed (catalog_uuids, customer_uuids, catalog_queries) + self.assertTrue(mock_client.replace_all_objects.called) + # Get the generator passed to replace_all_objects + args, _ = mock_client.replace_all_objects.call_args + generator = args[0] + # Consume generator to count objects + objects = list(generator) + self.assertEqual(len(objects), 3, "Should have 3 objects (English only)") + self.assertFalse(any(obj['objectID'].endswith('-es') for obj in objects)) + + # Reset mock + mock_client.reset_mock() + + # --- Execution 2: With Translation --- + ContentTranslation.objects.create( + content_metadata=course, + language_code='es', + title='Curso de Prueba' + ) + + _reindex_algolia( + indexable_content_keys=content_keys, + nonindexable_content_keys=[], + dry_run=False + ) + + # Verify 6 objects pushed (3 English + 3 Spanish) + self.assertTrue(mock_client.replace_all_objects.called) + args, _ = mock_client.replace_all_objects.call_args + generator = args[0] + objects = list(generator) + self.assertEqual(len(objects), 6, "Should have 6 objects (English + Spanish)") + + # Verify IDs + english_obj = next(obj for obj in objects if '-es-' not in obj['objectID']) + spanish_obj = next(obj for obj in objects if '-es-' in obj['objectID']) + + self.assertIsNotNone(english_obj) + self.assertIsNotNone(spanish_obj) + self.assertEqual(spanish_obj['title'], 'Curso de Prueba') + + @mock.patch('enterprise_catalog.apps.api.tasks._retrieve_inactive_tmp_indices') + @mock.patch('enterprise_catalog.apps.api.tasks._delete_indices') + @mock.patch('enterprise_catalog.apps.api.tasks.configure_algolia_index') + @mock.patch('enterprise_catalog.apps.api.tasks.get_initialized_algolia_client') + def test_algolia_indexing_performance( + self, + mock_get_client, + _mock_configure, + _mock_delete, + _mock_retrieve + ): + """ + Test that indexing performance (DB queries) does not scale linearly with number of items. + """ + mock_client = mock.Mock() + mock_get_client.return_value = mock_client + + # Helper to create content + def create_content(key_suffix): + course = ContentMetadataFactory(content_type=COURSE, content_key=f'course-v1:Test+Course+{key_suffix}') + catalog_query = CatalogQueryFactory() + course.catalog_queries.add(catalog_query) + EnterpriseCatalogFactory(catalog_query=catalog_query) + return course.content_key + + # Warmup + key1 = create_content('1') + _reindex_algolia([key1], [], dry_run=False) + + # Measure for 1 item + key2 = create_content('2') + with CaptureQueriesContext(connection) as ctx1: + _reindex_algolia([key2], [], dry_run=False) + count_1_item = len(ctx1) + + # Measure for 2 items + key3 = create_content('3') + key4 = create_content('4') + with CaptureQueriesContext(connection) as ctx2: + _reindex_algolia([key3, key4], [], dry_run=False) + count_2_items = len(ctx2) + + # The difference should be small (constant overhead), not double. + # Ideally count_2_items should be close to count_1_item. + diff = count_2_items - count_1_item + + # We expect the difference to be 0 or very small (not proportional to N) + self.assertLess(diff, 5, f"Query count increased significantly: {count_1_item} vs {count_2_items}") diff --git a/enterprise_catalog/apps/api/tests/test_spanish_algolia_helpers.py b/enterprise_catalog/apps/api/tests/test_spanish_algolia_helpers.py new file mode 100644 index 00000000..ba76a58b --- /dev/null +++ b/enterprise_catalog/apps/api/tests/test_spanish_algolia_helpers.py @@ -0,0 +1,143 @@ +""" +Unit tests for Spanish translation in Algolia helper functions. +""" +import uuid + +from django.test import TestCase + +from enterprise_catalog.apps.api import tasks +from enterprise_catalog.apps.catalog.constants import COURSE +from enterprise_catalog.apps.catalog.models import ContentTranslation +from enterprise_catalog.apps.catalog.tests.factories import ( + ContentMetadataFactory, +) +from enterprise_catalog.apps.video_catalog.tests.factories import VideoFactory + + +class TestSpanishTranslationInAlgoliaHelpers(TestCase): + """ + Tests for Spanish translation in add_metadata_to_algolia_objects and add_video_to_algolia_objects. + """ + + def test_add_metadata_to_algolia_objects_creates_spanish_objects(self): + """ + Test that add_metadata_to_algolia_objects creates Spanish versions of all batched objects + when translation exists. + """ + # Setup + course = ContentMetadataFactory(content_type=COURSE, content_key='test-course') + # Create translation so Spanish objects are generated + ContentTranslation.objects.create( + content_metadata=course, + language_code='es', + title='Curso de Prueba' + ) + + algolia_products_by_object_id = {} + catalog_uuid = str(uuid.uuid4()) + customer_uuid = str(uuid.uuid4()) + query_uuid = str(uuid.uuid4()) + + # Execute + tasks.add_metadata_to_algolia_objects( + metadata=course, + algolia_products_by_object_id=algolia_products_by_object_id, + catalog_uuids=[catalog_uuid], + customer_uuids=[customer_uuid], + catalog_queries=[(query_uuid, "Test Query")], + academy_uuids=[], + academy_tags=[], + video_ids=[], + ) + + # Verify both English and Spanish objects were created + object_ids = list(algolia_products_by_object_id.keys()) + english_objects = [oid for oid in object_ids if '-es' not in oid] + spanish_objects = [oid for oid in object_ids if '-es' in oid] + + # Should have 3 English objects (catalog, customer, query) + assert len(english_objects) == 3 + # Should have 3 Spanish objects (catalog, customer, query) + assert len(spanish_objects) == 3 + + # Verify Spanish objects have correct format + for spanish_oid in spanish_objects: + assert '-es' in spanish_oid + assert str(course.content_uuid) in spanish_oid + assert algolia_products_by_object_id[spanish_oid]['language'] == 'es' + + def test_add_video_to_algolia_objects_skips_spanish_objects(self): + """ + Test that add_video_to_algolia_objects skips Spanish versions (Video not supported yet). + """ + # Setup + video = VideoFactory() + algolia_products_by_object_id = {} + catalog_uuid = str(uuid.uuid4()) + customer_uuid = str(uuid.uuid4()) + query_uuid = str(uuid.uuid4()) + + # Execute + tasks.add_video_to_algolia_objects( + video=video, + algolia_products_by_object_id=algolia_products_by_object_id, + customer_uuids=[customer_uuid], + catalog_uuids=[catalog_uuid], + catalog_queries=[(query_uuid, "Test Query")], + ) + + # Verify only English objects were created + object_ids = list(algolia_products_by_object_id.keys()) + english_objects = [oid for oid in object_ids if '-es' not in oid] + spanish_objects = [oid for oid in object_ids if '-es' in oid] + + # Should have 3 English objects (customer, catalog, query) + assert len(english_objects) == 3 + # Should have 0 Spanish objects (Video not supported) + assert len(spanish_objects) == 0 + + def test_spanish_objects_include_same_uuids_as_english(self): + """ + Test that Spanish objects contain the same UUID lists as their English counterparts. + """ + # Setup + course = ContentMetadataFactory(content_type=COURSE, content_key='test-course') + # Create translation + ContentTranslation.objects.create( + content_metadata=course, + language_code='es', + title='Curso de Prueba' + ) + + algolia_products_by_object_id = {} + catalog_uuids = [str(uuid.uuid4()), str(uuid.uuid4())] + customer_uuids = [str(uuid.uuid4()), str(uuid.uuid4())] + + # Execute + tasks.add_metadata_to_algolia_objects( + metadata=course, + algolia_products_by_object_id=algolia_products_by_object_id, + catalog_uuids=catalog_uuids, + customer_uuids=customer_uuids, + catalog_queries=[], + academy_uuids=[], + academy_tags=[], + video_ids=[], + ) + + # Find English and Spanish catalog objects + english_catalog_obj = None + spanish_catalog_obj = None + + for oid, obj in algolia_products_by_object_id.items(): + if 'catalog-uuids' in oid and '-es' not in oid: + english_catalog_obj = obj + elif 'catalog-uuids' in oid and '-es' in oid: + spanish_catalog_obj = obj + + # Verify both objects exist and have same UUIDs + assert english_catalog_obj is not None + assert spanish_catalog_obj is not None + english_uuids = english_catalog_obj.get('enterprise_catalog_uuids') + spanish_uuids = spanish_catalog_obj.get('enterprise_catalog_uuids') + assert english_uuids == spanish_uuids diff --git a/enterprise_catalog/apps/api/tests/test_tasks.py b/enterprise_catalog/apps/api/tests/test_tasks.py index b5486a66..c6b16091 100644 --- a/enterprise_catalog/apps/api/tests/test_tasks.py +++ b/enterprise_catalog/apps/api/tests/test_tasks.py @@ -1,6 +1,7 @@ """ Tests for the enterprise_catalog API celery tasks """ + import json import uuid from datetime import datetime, timedelta @@ -27,7 +28,11 @@ PROGRAM, QUERY_FOR_RESTRICTED_RUNS, ) -from enterprise_catalog.apps.catalog.models import CatalogQuery, ContentMetadata +from enterprise_catalog.apps.catalog.models import ( + CatalogQuery, + ContentMetadata, + ContentTranslation, +) from enterprise_catalog.apps.catalog.serializers import ( DEFAULT_NORMALIZED_PRICE, NormalizedContentMetadataSerializer, @@ -828,6 +833,10 @@ class IndexEnterpriseCatalogCoursesInAlgoliaTaskTests(TestCase): def setUp(self): super().setUp() + # Create translations for the courses to ensure Spanish objects are generated + # This replaces the previous mock that forced Spanish object creation + # We'll create translations later after objects are created + # Set up a catalog, query, and metadata for a course and course associated program self.academy = AcademyFactory() self.tag1 = self.academy.tags.all()[0] @@ -867,6 +876,72 @@ def setUp(self): _hydrate_course_normalized_metadata() + # Create translations + ContentTranslation.objects.create( + content_metadata=self.course_metadata_published, + language_code='es', + title="Spanish Course Title" + ) + ContentTranslation.objects.create( + content_metadata=self.course_metadata_unpublished, + language_code='es', + title="Spanish Unpublished Course Title" + ) + ContentTranslation.objects.create( + content_metadata=self.course_run_metadata_published, + language_code='es', + title="Spanish Course Run Title" + ) + ContentTranslation.objects.create( + content_metadata=self.course_run_metadata_unpublished, + language_code='es', + title="Spanish Unpublished Course Run Title" + ) + + def _create_expected_spanish_object(self, english_object): + """ + Create an expected Spanish object from an English object. + This should mirror the mock behavior of create_spanish_algolia_object. + + The Spanish objectID has '-es-' inserted before the batch suffix. + For example: + - English: 'course-{uuid}-catalog-uuids-0' + - Spanish: 'course-{uuid}-es-catalog-uuids-0' + + Args: + english_object (dict): The English Algolia object + + Returns: + dict: The expected Spanish Algolia object + """ + spanish_object = english_object.copy() + object_id = english_object['objectID'] + + # Insert '-es-' before the batch type marker + # The objectID pattern is: {content-type}-{uuid}-{batch-type}-{batch-subtype}-{batch-number} + # Where batch-type is one of: catalog, customer + # And batch-subtype is one of: uuids, query-uuids + # Result: {content-type}-{uuid}-es-{batch-type}-{batch-subtype}-{batch-number} + + # Find the position to insert '-es-' by looking for batch type markers + batch_markers = ['-catalog-', '-customer-'] + insert_pos = -1 + + for marker in batch_markers: + pos = object_id.find(marker) + if pos != -1: + insert_pos = pos + break + + if insert_pos != -1: + # Insert '-es' before the batch marker + spanish_object['objectID'] = object_id[:insert_pos] + '-es' + object_id[insert_pos:] + else: + # Fallback: just append -es if structure is unexpected + spanish_object['objectID'] = f"{object_id}-es" + + return spanish_object + def _set_up_factory_data_for_algolia(self): expected_catalog_uuids = sorted([ str(self.enterprise_catalog_courses.uuid), @@ -959,6 +1034,15 @@ def test_get_algolia_objects_from_course_metadata(self): test_course.catalog_queries.set(catalog_queries[0:3]) + test_course.catalog_queries.set(catalog_queries[0:3]) + + # Create translation + ContentTranslation.objects.create( + content_metadata=test_course, + language_code='es', + title="Spanish Test Course Title" + ) + algolia_objects = tasks.get_algolia_objects_from_course_content_metadata(test_course) expected_transformed_advertised_course_run = { @@ -1089,8 +1173,21 @@ def test_index_algolia_program_common_uuids_only(self, mock_search_client): test_course_1.save() test_course_2.save() test_course_3.save() + test_course_3.save() _hydrate_course_normalized_metadata() + # Create translation for the program to ensure Spanish objects are generated + ContentTranslation.objects.create( + content_metadata=program_1, + language_code='es', + title="Spanish Program Title" + ) + + # Create translations for courses + ContentTranslation.objects.create(content_metadata=test_course_1, language_code='es', title="Spanish Course 1") + ContentTranslation.objects.create(content_metadata=test_course_2, language_code='es', title="Spanish Course 2") + ContentTranslation.objects.create(content_metadata=test_course_3, language_code='es', title="Spanish Course 3") + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the @@ -1105,7 +1202,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 15 products found.' in products_found_log_records[0] + assert ' 30 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_program_1_objects_to_index = [] @@ -1130,10 +1227,17 @@ def mock_replace_all_objects(products_iterable): 'academy_uuids': [], }) - # verify replace_all_objects is called with the correct Algolia object data. - expected_program_call_args = sorted(expected_program_1_objects_to_index, key=itemgetter('objectID')) + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_program_1_objects_to_index + ] + expected_all_objects = expected_program_1_objects_to_index + expected_spanish_objects + + expected_program_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_program_call_args = sorted( - [product for product in actual_algolia_products_sent if program_uuid in product['objectID']], + [ + product for product in actual_algolia_products_sent + if program_uuid in product['objectID'] + ], key=itemgetter('objectID'), ) assert expected_program_call_args == actual_program_call_args @@ -1191,6 +1295,11 @@ def test_index_algolia_program_unindexable_content(self, mock_search_client): test_course_3.save() _hydrate_course_normalized_metadata() + # Create translations for courses + ContentTranslation.objects.create(content_metadata=test_course_1, language_code='es', title="Spanish Course 1") + ContentTranslation.objects.create(content_metadata=test_course_2, language_code='es', title="Spanish Course 2") + ContentTranslation.objects.create(content_metadata=test_course_3, language_code='es', title="Spanish Course 3") + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the @@ -1205,8 +1314,8 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - # count should be "9 products found", 5 additional products are from the test course in self.setUp() - assert ' 12 products found.' in products_found_log_records[0] + # count should be "24 products found", 5 additional products are from the test course in self.setUp() + assert ' 24 products found.' in products_found_log_records[0] # assert the program was not indexed. program_uuid = program_1.json_metadata.get('uuid') @@ -1555,7 +1664,14 @@ def test_index_algolia_published_course_to_program(self, mock_search_client): # Associate published course with a published program and also an unpublished program. self.course_metadata_published.associated_content_metadata.set([program_1, program_2]) - actual_algolia_products_sent = None + # Create translation for the program + ContentTranslation.objects.create( + content_metadata=program_1, + language_code='es', + title="Spanish Program Title" + ) + + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -1569,7 +1685,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 6 products found.' in products_found_log_records[0] + assert ' 12 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_course_1_objects_to_index = [] @@ -1625,7 +1741,13 @@ def mock_replace_all_objects(products_iterable): ) # verify replace_all_objects is called with the correct Algolia object data. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) @@ -1664,7 +1786,14 @@ def test_index_algolia_unpublished_course_to_program(self, mock_search_client): # Associate unpublished course with a published program and also an unpublished program. self.course_metadata_unpublished.associated_content_metadata.set([program_1, program_2]) - actual_algolia_products_sent = None + # Create translation for the program + ContentTranslation.objects.create( + content_metadata=program_1, + language_code='es', + title="Spanish Program Title" + ) + + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -1678,7 +1807,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 6 products found.' in products_found_log_records[0] + assert ' 12 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_course_1_objects_to_index = [] @@ -1734,7 +1863,13 @@ def mock_replace_all_objects(products_iterable): ) # verify replace_all_objects is called with the correct Algolia object data. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) @@ -1761,7 +1896,14 @@ def test_index_algolia_published_course_to_pathway(self, mock_search_client,): # Associate published course with a pathway. self.course_metadata_published.associated_content_metadata.set([pathway_1]) - actual_algolia_products_sent = None + # Create translation for the pathway + ContentTranslation.objects.create( + content_metadata=pathway_1, + language_code='es', + title="Spanish Pathway Title" + ) + + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -1775,7 +1917,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 6 products found.' in products_found_log_records[0] + assert ' 12 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_course_1_objects_to_index = [] @@ -1834,7 +1976,13 @@ def mock_replace_all_objects(products_iterable): ) # verify replace_all_objects is called with the correct Algolia object data. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) @@ -1865,7 +2013,14 @@ def test_index_algolia_unpublished_course_to_pathway(self, mock_search_client): # Associate unpublished course with a pathway. self.course_metadata_unpublished.associated_content_metadata.set([pathway_1]) - actual_algolia_products_sent = None + # Create translation for the pathway + ContentTranslation.objects.create( + content_metadata=pathway_1, + language_code='es', + title="Spanish Pathway Title" + ) + + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -1879,7 +2034,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 6 products found.' in products_found_log_records[0] + assert ' 12 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_course_1_objects_to_index = [] @@ -1938,7 +2093,13 @@ def mock_replace_all_objects(products_iterable): ) # verify replace_all_objects is called with the correct Algolia object data. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) @@ -1979,7 +2140,19 @@ def test_index_algolia_program_to_pathway(self, mock_search_client): program_1.associated_content_metadata.set([pathway_1]) program_2.associated_content_metadata.set([pathway_1]) - actual_algolia_products_sent = None + # Create translations + ContentTranslation.objects.create( + content_metadata=program_1, + language_code='es', + title="Spanish Program 1 Title" + ) + ContentTranslation.objects.create( + content_metadata=pathway_1, + language_code='es', + title="Spanish Pathway Title" + ) + + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -1993,7 +2166,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 9 products found.' in products_found_log_records[0] + assert ' 18 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_course_1_objects_to_index = [] @@ -2075,7 +2248,13 @@ def mock_replace_all_objects(products_iterable): ) # verify replace_all_objects is called with the correct Algolia object data. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) @@ -2212,7 +2391,7 @@ def test_index_algolia_restricted_runs_mixed_course(self, mock_search_client): run=courserun_restricted_for_catalog_B, ) - actual_algolia_products_sent = None + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -2226,7 +2405,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 3 products found.' in products_found_log_records[0] + assert ' 6 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_algolia_objects_to_index = [] @@ -2274,7 +2453,13 @@ def mock_replace_all_objects(products_iterable): }) # Verify replace_all_objects is called with the correct Algolia object data. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) @@ -2422,7 +2607,7 @@ def test_index_algolia_restricted_runs_unicorn_course(self, mock_search_client): run=courserun_restricted_for_catalog_B, ) - actual_algolia_products_sent = None + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -2436,7 +2621,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 3 products found.' in products_found_log_records[0] + assert ' 6 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_algolia_objects_to_index = [] @@ -2480,7 +2665,13 @@ def mock_replace_all_objects(products_iterable): }) # Verify replace_all_objects is called with the correct Algolia object data. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) @@ -2492,7 +2683,7 @@ def test_index_algolia_with_batched_uuids(self, mock_search_client): """ algolia_data = self._set_up_factory_data_for_algolia() - actual_algolia_products_sent = None + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -2506,7 +2697,7 @@ def mock_replace_all_objects(products_iterable): with self.assertLogs(level='INFO') as info_logs: tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter - assert ' 6 products found.' in info_logs.output[-1] + assert ' 12 products found.' in info_logs.output[-1] # create expected data to be added/updated in the Algolia index. expected_algolia_objects_to_index = [] @@ -2556,7 +2747,15 @@ def mock_replace_all_objects(products_iterable): 'academy_tags': algolia_data['academy_tags'], }) # verify replace_all_objects is called with the correct Algolia object data - self.assertEqual(expected_algolia_objects_to_index, actual_algolia_products_sent) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + actual_all_products = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) + expected_all_sorted = sorted(expected_all_objects, key=itemgetter('objectID')) + self.assertEqual(expected_all_sorted, actual_all_products) mock_search_client().replace_all_objects.assert_called_once() @mock.patch('enterprise_catalog.apps.api.tasks.get_initialized_algolia_client', return_value=mock.MagicMock()) @@ -2568,7 +2767,7 @@ def test_index_algolia_with_important_catalog_titles(self, mock_search_client): # override the explore UI titles with test data to show every batch contains them explore_titles = [algolia_data['query_titles'][0]] - actual_algolia_products_sent = None + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -2583,7 +2782,7 @@ def mock_replace_all_objects(products_iterable): with self.assertLogs(level='INFO') as info_logs: tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter - assert ' 6 products found.' in info_logs.output[-1] + assert ' 12 products found.' in info_logs.output[-1] # create expected data to be added/updated in the Algolia index. expected_algolia_objects_to_index = [] @@ -2636,7 +2835,15 @@ def mock_replace_all_objects(products_iterable): }) # verify replace_all_objects is called with the correct Algolia object data - self.assertEqual(expected_algolia_objects_to_index, actual_algolia_products_sent) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + actual_all_products = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) + expected_all_sorted = sorted(expected_all_objects, key=itemgetter('objectID')) + self.assertEqual(expected_all_sorted, actual_all_products) mock_search_client().replace_all_objects.assert_called_once() @mock.patch('enterprise_catalog.apps.api.tasks.get_initialized_algolia_client', return_value=mock.MagicMock()) @@ -2692,7 +2899,7 @@ def test_index_algolia_dry_run(self, mock_search_client): tasks.index_enterprise_catalog_in_algolia_task(force, dry_run) mock_search_client().replace_all_objects.assert_not_called() - assert '[ENTERPRISE_CATALOG_ALGOLIA_REINDEX] [DRY RUN] 6 products found.' in info_logs.output[-1] + assert '[ENTERPRISE_CATALOG_ALGOLIA_REINDEX] [DRY RUN] 12 products found.' in info_logs.output[-1] assert any( '[ENTERPRISE_CATALOG_ALGOLIA_REINDEX] [DRY RUN] skipping algolia_client.replace_all_objects().' in record for record in info_logs.output @@ -3073,7 +3280,39 @@ def test_index_algolia_all_uuids(self, mock_search_client): [program_for_unpublished_course, pathway_for_unpublished_course] ) - actual_algolia_products_sent = None + # Create translations + ContentTranslation.objects.create( + content_metadata=program_for_main_course, + language_code='es', + title="Spanish Program 1" + ) + ContentTranslation.objects.create( + content_metadata=program_for_pathway, + language_code='es', + title="Spanish Program 2" + ) + ContentTranslation.objects.create( + content_metadata=pathway_for_course, + language_code='es', + title="Spanish Pathway 1" + ) + ContentTranslation.objects.create( + content_metadata=pathway_for_courserun, + language_code='es', + title="Spanish Pathway 2" + ) + ContentTranslation.objects.create( + content_metadata=program_for_unpublished_course, + language_code='es', + title="Spanish Program 3" + ) + ContentTranslation.objects.create( + content_metadata=pathway_for_unpublished_course, + language_code='es', + title="Spanish Pathway 3" + ) + + actual_algolia_products_sent = [] # `replace_all_objects` is swapped out for a mock implementation that forces generator evaluation and saves the # result into `actual_algolia_products_sent` for unit testing. @@ -3087,7 +3326,7 @@ def mock_replace_all_objects(products_iterable): tasks.index_enterprise_catalog_in_algolia_task() # pylint: disable=no-value-for-parameter products_found_log_records = [record for record in info_logs.output if ' products found.' in record] - assert ' 15 products found.' in products_found_log_records[0] + assert ' 30 products found.' in products_found_log_records[0] # create expected data to be added/updated in the Algolia index. expected_algolia_objects_to_index = [] @@ -3223,6 +3462,12 @@ def mock_replace_all_objects(products_iterable): # verify replace_all_objects is called with the correct Algolia object data # on the first invocation and with programs/pathways only on the second invocation. - expected_call_args = sorted(expected_algolia_objects_to_index, key=itemgetter('objectID')) + # Create Spanish versions of expected objects + expected_spanish_objects = [ + self._create_expected_spanish_object(obj) for obj in expected_algolia_objects_to_index + ] + expected_all_objects = expected_algolia_objects_to_index + expected_spanish_objects + + expected_call_args = sorted(expected_all_objects, key=itemgetter('objectID')) actual_call_args = sorted(actual_algolia_products_sent, key=itemgetter('objectID')) assert expected_call_args == self._sort_tags_in_algolia_object_list(actual_call_args) diff --git a/enterprise_catalog/apps/catalog/admin.py b/enterprise_catalog/apps/catalog/admin.py index 20623bee..4301e0f9 100644 --- a/enterprise_catalog/apps/catalog/admin.py +++ b/enterprise_catalog/apps/catalog/admin.py @@ -14,6 +14,7 @@ from enterprise_catalog.apps.catalog.models import ( CatalogQuery, ContentMetadata, + ContentTranslation, EnterpriseCatalog, EnterpriseCatalogRoleAssignment, RestrictedCourseMetadata, @@ -307,3 +308,28 @@ class Meta: fields = ('user', 'role', 'enterprise_id', 'applies_to_all_contexts') form = EnterpriseCatalogRoleAssignmentAdminForm + + +@admin.register(ContentTranslation) +class ContentTranslationAdmin(admin.ModelAdmin): + """ + Admin configuration for the ContentTranslation model. + """ + list_display = ('content_metadata', 'language_code', 'modified', 'source_hash') + list_filter = ('language_code', 'modified') + search_fields = ('content_metadata__content_key', 'title') + readonly_fields = ('created', 'modified', 'source_hash') + raw_id_fields = ('content_metadata',) + fields = ( + 'content_metadata', + 'language_code', + 'title', + 'short_description', + 'full_description', + 'outcome', + 'prerequisites', + 'subtitle', + 'source_hash', + 'created', + 'modified', + ) diff --git a/enterprise_catalog/apps/catalog/algolia_utils.py b/enterprise_catalog/apps/catalog/algolia_utils.py index 5dcb62b2..46d889b2 100644 --- a/enterprise_catalog/apps/catalog/algolia_utils.py +++ b/enterprise_catalog/apps/catalog/algolia_utils.py @@ -36,7 +36,10 @@ get_course_first_paid_enrollable_seat_price, is_course_run_active, ) -from enterprise_catalog.apps.catalog.models import ContentMetadata +from enterprise_catalog.apps.catalog.models import ( + ContentMetadata, + ContentTranslation, +) from enterprise_catalog.apps.catalog.serializers import ( NormalizedContentMetadataSerializer, ) @@ -52,6 +55,9 @@ ) +LOGGER = logging.getLogger(__name__) + + logger = logging.getLogger(__name__) ALGOLIA_UUID_BATCH_SIZE = 100 @@ -229,6 +235,7 @@ def _should_index_course(course_metadata): Returns: bool: Whether or not the course should be indexed by algolia. """ + course_json_metadata = course_metadata.json_metadata advertised_course_run = get_advertised_course_run(course_json_metadata) @@ -1614,3 +1621,68 @@ def create_algolia_objects(products, algolia_fields): ] return algolia_objects + + +def create_spanish_algolia_object(algolia_object, content_metadata=None): + """ + Creates a Spanish version of the Algolia object. + + Args: + algolia_object (dict): The original English Algolia object. + content_metadata (ContentMetadata or Video, optional): The metadata instance + to fetch pre-computed translations from. If None or if it's a Video object, + falls back to inline translation. + + Returns: + dict: A new Algolia object with translated fields and updated objectID. + """ + spanish_object = copy.deepcopy(algolia_object) + translation = None + + # Only attempt to fetch pre-computed translation for ContentMetadata objects + # Videos don't have ContentTranslation support yet + if content_metadata and isinstance(content_metadata, ContentMetadata): + try: + translation = content_metadata.translations.get(language_code='es') + except ContentTranslation.DoesNotExist: + LOGGER.warning( + '[SPANISH_TRANSLATION] No pre-computed translation found for %s, ' + 'falling back to inline translation', + content_metadata.content_key + ) + except Exception as exc: # pylint: disable=broad-except + LOGGER.error( + '[SPANISH_TRANSLATION] Error fetching translation for %s: %s', + content_metadata.content_key, + exc, + exc_info=True + ) + + # Use pre-computed translation if available + if translation: + # Apply translated fields + if translation.title: + spanish_object['title'] = translation.title + if translation.short_description: + spanish_object['short_description'] = translation.short_description + if translation.full_description: + spanish_object['full_description'] = translation.full_description + if translation.subtitle: + spanish_object['subtitle'] = translation.subtitle + + LOGGER.debug( + '[SPANISH_TRANSLATION] Using pre-computed translation for %s', + content_metadata.content_key + ) + else: + LOGGER.debug( + '[SPANISH_TRANSLATION] No pre-computed translation available for %s, skipping Spanish object', + getattr(content_metadata, 'content_key', 'unknown') if content_metadata else 'unknown' + ) + return None + + # Update objectID to indicate Spanish version + spanish_object['objectID'] = f"{spanish_object['objectID']}-es" + spanish_object['language'] = 'es' + + return spanish_object diff --git a/enterprise_catalog/apps/catalog/management/commands/populate_spanish_translations.py b/enterprise_catalog/apps/catalog/management/commands/populate_spanish_translations.py new file mode 100644 index 00000000..11234609 --- /dev/null +++ b/enterprise_catalog/apps/catalog/management/commands/populate_spanish_translations.py @@ -0,0 +1,220 @@ +""" +Management command to pre-populate Spanish translations for content metadata. +""" +import logging + +from django.core.management.base import BaseCommand + +from enterprise_catalog.apps.ai_curation.utils.open_ai_utils import ( + translate_object_fields, +) +from enterprise_catalog.apps.catalog.models import ( + ContentMetadata, + ContentTranslation, +) +from enterprise_catalog.apps.catalog.utils import compute_source_hash + + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + """ + Management command to pre-populate Spanish translations for content metadata. + + This command translates content metadata fields to Spanish and stores them + in the ContentTranslation model for faster Algolia indexing. + + Example usage: + # Populate all translations + ./manage.py populate_spanish_translations + + # Populate specific content + ./manage.py populate_spanish_translations --content-keys "course-v1:edX+DemoX" + + # Force re-translation + ./manage.py populate_spanish_translations --force + + # Process in smaller batches + ./manage.py populate_spanish_translations --batch-size 50 + """ + help = 'Pre-populate Spanish translations for content metadata' + + def add_arguments(self, parser): + parser.add_argument( + '--content-keys', + nargs='+', + help='Specific content keys to translate' + ) + parser.add_argument( + '--force', + action='store_true', + help='Force re-translation even if translation exists and hash matches' + ) + parser.add_argument( + '--batch-size', + type=int, + default=100, + help='Number of items to process in each batch (default: 100)' + ) + parser.add_argument( + '--language', + default='es', + help='Target language code (default: es for Spanish)' + ) + parser.add_argument( + '--dry-run', + action='store_true', + help='Run without actually saving translations' + ) + + def handle(self, *args, **options): + """ + Main command handler. + """ + content_keys = options.get('content_keys') + force = options.get('force') + batch_size = options.get('batch_size') + language_code = options.get('language') + dry_run = options.get('dry_run') + + # Build queryset + queryset = ContentMetadata.objects.all() + if content_keys: + queryset = queryset.filter(content_key__in=content_keys) + + total_count = queryset.count() + logger.info( + 'Starting translation for %s content items to %s', + total_count, + language_code + ) + + if dry_run: + logger.warning('DRY RUN MODE - No changes will be saved') + + processed_count = 0 + created_count = 0 + updated_count = 0 + skipped_count = 0 + error_count = 0 + + # Process in batches + for batch_start in range(0, total_count, batch_size): + batch_end = min(batch_start + batch_size, total_count) + batch = queryset[batch_start:batch_end] + + logger.info('Processing batch %s-%s...', batch_start, batch_end) + + for content in batch: + try: + result = self._process_content( + content, + language_code, + force, + dry_run + ) + + if result == 'created': + created_count += 1 + elif result == 'updated': + updated_count += 1 + elif result == 'skipped': + skipped_count += 1 + + processed_count += 1 + + if processed_count % 10 == 0: + logger.info( + 'Progress: %s/%s (Created: %s, Updated: %s, Skipped: %s, Errors: %s)', + processed_count, + total_count, + created_count, + updated_count, + skipped_count, + error_count + ) + + except Exception as exc: # pylint: disable=broad-except + error_count += 1 + logger.error( + 'Error processing content %s: %s', + content.content_key, + exc, + exc_info=True + ) + + # Final summary + logger.info( + 'Translation Complete! Processed: %s | Created: %s | Updated: %s | Skipped: %s | Errors: %s', + processed_count, + created_count, + updated_count, + skipped_count, + error_count + ) + + def _process_content(self, content, language_code, force, dry_run): + """ + Process a single content metadata item. + + Args: + content: ContentMetadata instance + language_code: Target language code + force: Whether to force re-translation + dry_run: Whether to skip saving + + Returns: + str: 'created', 'updated', or 'skipped' + """ + # Compute source hash + source_hash = compute_source_hash(content) + + # Check if translation exists + try: + translation = ContentTranslation.objects.get( + content_metadata=content, + language_code=language_code + ) + exists = True + except ContentTranslation.DoesNotExist: + translation = ContentTranslation( + content_metadata=content, + language_code=language_code + ) + exists = False + + # Skip if translation exists, hash matches, and not forcing + if exists and translation.source_hash == source_hash and not force: + logger.debug( + f'Skipping {content.content_key} - translation up to date' + ) + return 'skipped' + + # Translate fields + fields_to_translate = [ + 'title', 'short_description', 'full_description', 'subtitle' + ] + + translated_data = translate_object_fields( + content.json_metadata, + fields_to_translate + ) + + # Update translation fields + translation.title = translated_data.get('title') + translation.short_description = translated_data.get('short_description') + translation.full_description = translated_data.get('full_description') + translation.subtitle = translated_data.get('subtitle') + translation.source_hash = source_hash + + # Save if not dry run + if not dry_run: + translation.save() + + result = 'created' if not exists else 'updated' + logger.info( + f'{result.capitalize()} translation for {content.content_key} ({language_code})' + ) + + return result diff --git a/enterprise_catalog/apps/catalog/migrations/0045_add_content_translation.py b/enterprise_catalog/apps/catalog/migrations/0045_add_content_translation.py new file mode 100644 index 00000000..7e223939 --- /dev/null +++ b/enterprise_catalog/apps/catalog/migrations/0045_add_content_translation.py @@ -0,0 +1,44 @@ +# Generated by Django 5.2.8 on 2025-12-05 02:20 + +import django.db.models.deletion +import django.utils.timezone +import model_utils.fields +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('catalog', '0044_mariadb_uuid_conversion'), + ] + + operations = [ + migrations.AlterField( + model_name='restrictedcoursemetadata', + name='restricted_run_allowed_for_restricted_course', + field=models.ManyToManyField(through='catalog.RestrictedRunAllowedForRestrictedCourse', through_fields=('course', 'run'), to='catalog.contentmetadata'), + ), + migrations.CreateModel( + name='ContentTranslation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), + ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), + ('language_code', models.CharField(help_text="ISO 639-1 language code (e.g., 'es' for Spanish)", max_length=10)), + ('title', models.CharField(blank=True, help_text='Translated title', max_length=255, null=True)), + ('short_description', models.TextField(blank=True, help_text='Translated short description', null=True)), + ('full_description', models.TextField(blank=True, help_text='Translated full description', null=True)), + ('outcome', models.TextField(blank=True, help_text='Translated learning outcomes', null=True)), + ('prerequisites', models.TextField(blank=True, help_text='Translated prerequisites', null=True)), + ('subtitle', models.CharField(blank=True, help_text='Translated subtitle', max_length=255, null=True)), + ('source_hash', models.CharField(help_text='SHA256 hash of source content to detect changes', max_length=64)), + ('content_metadata', models.ForeignKey(help_text='The content metadata this translation belongs to', on_delete=django.db.models.deletion.CASCADE, related_name='translations', to='catalog.contentmetadata')), + ], + options={ + 'verbose_name': 'Content Translation', + 'verbose_name_plural': 'Content Translations', + 'indexes': [models.Index(fields=['content_metadata', 'language_code'], name='catalog_con_content_bb6ff8_idx')], + 'unique_together': {('content_metadata', 'language_code')}, + }, + ), + ] diff --git a/enterprise_catalog/apps/catalog/models.py b/enterprise_catalog/apps/catalog/models.py index b3f4a51a..c636271d 100644 --- a/enterprise_catalog/apps/catalog/models.py +++ b/enterprise_catalog/apps/catalog/models.py @@ -1741,3 +1741,79 @@ def current_options(cls): 'no_async': current_config.no_async, } return {} + + +class ContentTranslation(TimeStampedModel): + """ + Stores pre-computed translations for content metadata. + + This model caches translations to improve Algolia indexing performance + by avoiding inline API calls during the indexing process. + + .. no_pii: + """ + content_metadata = models.ForeignKey( + ContentMetadata, + on_delete=models.CASCADE, + related_name='translations', + help_text=_("The content metadata this translation belongs to") + ) + language_code = models.CharField( + max_length=10, + help_text=_("ISO 639-1 language code (e.g., 'es' for Spanish)") + ) + + # Translated fields + title = models.CharField( + max_length=255, + blank=True, + null=True, + help_text=_("Translated title") + ) + short_description = models.TextField( + blank=True, + null=True, + help_text=_("Translated short description") + ) + full_description = models.TextField( + blank=True, + null=True, + help_text=_("Translated full description") + ) + outcome = models.TextField( + blank=True, + null=True, + help_text=_("Translated learning outcomes") + ) + prerequisites = models.TextField( + blank=True, + null=True, + help_text=_("Translated prerequisites") + ) + subtitle = models.CharField( + max_length=255, + blank=True, + null=True, + help_text=_("Translated subtitle") + ) + + # Metadata for change detection + source_hash = models.CharField( + max_length=64, + help_text=_("SHA256 hash of source content to detect changes") + ) + + class Meta: + verbose_name = _("Content Translation") + verbose_name_plural = _("Content Translations") + app_label = 'catalog' + unique_together = ('content_metadata', 'language_code') + indexes = [ + models.Index(fields=['content_metadata', 'language_code']), + ] + + def __str__(self): + """ + Return human-readable string representation. + """ + return f"" diff --git a/enterprise_catalog/apps/catalog/tests/test_algolia_translation.py b/enterprise_catalog/apps/catalog/tests/test_algolia_translation.py new file mode 100644 index 00000000..9acdeca4 --- /dev/null +++ b/enterprise_catalog/apps/catalog/tests/test_algolia_translation.py @@ -0,0 +1,126 @@ +from django.test import TestCase + +from enterprise_catalog.apps.api.tasks import add_metadata_to_algolia_objects +from enterprise_catalog.apps.catalog.algolia_utils import ( + create_spanish_algolia_object, +) +from enterprise_catalog.apps.catalog.models import ContentTranslation +from enterprise_catalog.apps.catalog.tests.factories import ( + ContentMetadataFactory, +) + + +class AlgoliaTranslationTests(TestCase): + def test_create_spanish_algolia_object_no_translation(self): + """ + Test that create_spanish_algolia_object returns None when no translation exists. + """ + original_object = {'objectID': 'course-123'} + content_metadata = ContentMetadataFactory(content_type='course', content_key='course-123') + + result = create_spanish_algolia_object(original_object, content_metadata) + self.assertIsNone(result) + + def test_create_spanish_algolia_object_with_translation(self): + """ + Test that create_spanish_algolia_object returns translated object when translation exists. + """ + original_object = { + 'objectID': 'course-123', + 'title': 'Original Title', + 'short_description': 'Original Description', + 'full_description': 'Original Full', + 'subtitle': 'Original Subtitle' + } + content_metadata = ContentMetadataFactory(content_type='course', content_key='course-123') + ContentTranslation.objects.create( + content_metadata=content_metadata, + language_code='es', + title='Título Español', + short_description='Descripción Español', + full_description='Descripción Completa Español', + subtitle='Subtítulo Español' + ) + + result = create_spanish_algolia_object(original_object, content_metadata) + + self.assertIsNotNone(result) + self.assertEqual(result['objectID'], 'course-123-es') + self.assertEqual(result['title'], 'Título Español') + self.assertEqual(result['short_description'], 'Descripción Español') + self.assertEqual(result['full_description'], 'Descripción Completa Español') + self.assertEqual(result['subtitle'], 'Subtítulo Español') + self.assertEqual(result['language'], 'es') + + def test_add_metadata_to_algolia_objects_creates_spanish_version(self): + """ + Test that add_metadata_to_algolia_objects creates Spanish objects when translation exists. + """ + metadata = ContentMetadataFactory(content_type='course') + ContentTranslation.objects.create( + content_metadata=metadata, + language_code='es', + title='Título Español' + ) + + algolia_products = {} + catalog_uuids = ['cat-1'] + customer_uuids = ['cust-1'] + catalog_queries = [('query-1', 'Query Title')] + academy_uuids = [] + academy_tags = [] + video_ids = [] + + add_metadata_to_algolia_objects( + metadata, + algolia_products, + catalog_uuids, + customer_uuids, + catalog_queries, + academy_uuids, + academy_tags, + video_ids + ) + + # Check for Spanish objects + spanish_keys = [k for k in algolia_products.keys() if '-es' in k] + self.assertGreater(len(spanish_keys), 0) + + # Verify one of the Spanish objects + spanish_obj = algolia_products[spanish_keys[0]] + self.assertEqual(spanish_obj['title'], 'Título Español') + self.assertIn('-es', spanish_obj['objectID']) + + def test_add_metadata_to_algolia_objects_skips_spanish_version(self): + """ + Test that add_metadata_to_algolia_objects skips Spanish objects when no translation exists. + """ + metadata = ContentMetadataFactory(content_type='course') + # No translation created + + algolia_products = {} + catalog_uuids = ['cat-1'] + customer_uuids = ['cust-1'] + catalog_queries = [('query-1', 'Query Title')] + academy_uuids = [] + academy_tags = [] + video_ids = [] + + add_metadata_to_algolia_objects( + metadata, + algolia_products, + catalog_uuids, + customer_uuids, + catalog_queries, + academy_uuids, + academy_tags, + video_ids + ) + + # Check for Spanish objects - should be none + spanish_keys = [k for k in algolia_products.keys() if '-es' in k] + self.assertEqual(len(spanish_keys), 0) + + # Check for English objects - should exist + english_keys = [k for k in algolia_products.keys() if not k.endswith('-es') and 'catalog-uuids' in k] + self.assertGreater(len(english_keys), 0) diff --git a/enterprise_catalog/apps/catalog/tests/test_content_translation.py b/enterprise_catalog/apps/catalog/tests/test_content_translation.py new file mode 100644 index 00000000..f9d38397 --- /dev/null +++ b/enterprise_catalog/apps/catalog/tests/test_content_translation.py @@ -0,0 +1,240 @@ +""" +Tests for ContentTranslation model and translation utilities. +""" +from django.test import TestCase + +from enterprise_catalog.apps.catalog.models import ContentTranslation +from enterprise_catalog.apps.catalog.tests.factories import ( + ContentMetadataFactory, +) +from enterprise_catalog.apps.catalog.utils import compute_source_hash + + +class ContentTranslationModelTests(TestCase): + """ + Tests for the ContentTranslation model. + """ + + def setUp(self): + """Set up test data.""" + self.content_metadata = ContentMetadataFactory( + content_type='course', + content_key='test-course-key', + json_metadata={ + 'title': 'Test Course', + 'short_description': 'A test course description', + 'full_description': 'Full description of the test course', + 'outcome': 'Learning outcomes', + 'prerequisites': 'Prerequisites for the course', + 'subtitle': 'Course subtitle' + } + ) + + def test_create_translation(self): + """Test creating a translation.""" + translation = ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='es', + title='Curso de Prueba', + short_description='Una descripción del curso de prueba', + full_description='Descripción completa del curso de prueba', + outcome='Resultados de aprendizaje', + prerequisites='Requisitos previos para el curso', + subtitle='Subtítulo del curso', + source_hash='test_hash_123' + ) + + self.assertEqual(translation.content_metadata, self.content_metadata) + self.assertEqual(translation.language_code, 'es') + self.assertEqual(translation.title, 'Curso de Prueba') + self.assertEqual(translation.source_hash, 'test_hash_123') + + def test_translation_unique_together(self): + """Test that content_metadata + language_code combination is unique.""" + ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='es', + title='First Translation', + source_hash='hash1' + ) + + # Creating another translation for the same content + language should fail + with self.assertRaises(Exception): + ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='es', + title='Second Translation', + source_hash='hash2' + ) + + def test_translation_related_name(self): + """Test accessing translations via related name.""" + translation = ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='es', + title='Spanish Title', + source_hash='hash' + ) + + # Access via related name + self.assertEqual(self.content_metadata.translations.count(), 1) + self.assertEqual(self.content_metadata.translations.first(), translation) + + def test_translation_str_representation(self): + """Test string representation of translation.""" + translation = ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='es', + title='Test', + source_hash='hash' + ) + + expected_str = f"" + self.assertEqual(str(translation), expected_str) + + def test_translation_cascade_delete(self): + """Test that translations are deleted when content_metadata is deleted.""" + ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='es', + title='Test', + source_hash='hash' + ) + + self.assertEqual(ContentTranslation.objects.count(), 1) + + # Delete the content metadata + self.content_metadata.delete() + + # Translation should also be deleted + self.assertEqual(ContentTranslation.objects.count(), 0) + + def test_multiple_languages(self): + """Test creating translations for multiple languages.""" + ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='es', + title='Spanish Title', + source_hash='hash1' + ) + + ContentTranslation.objects.create( + content_metadata=self.content_metadata, + language_code='fr', + title='French Title', + source_hash='hash2' + ) + + self.assertEqual(self.content_metadata.translations.count(), 2) + self.assertTrue( + self.content_metadata.translations.filter(language_code='es').exists() + ) + self.assertTrue( + self.content_metadata.translations.filter(language_code='fr').exists() + ) + + +class ComputeSourceHashTests(TestCase): + """ + Tests for the compute_source_hash utility function. + """ + + def test_compute_source_hash_basic(self): + """Test computing hash with basic content.""" + content = ContentMetadataFactory( + json_metadata={ + 'title': 'Test Course', + 'short_description': 'Description', + 'full_description': 'Full desc', + 'outcome': 'Outcomes', + 'prerequisites': 'Prereqs', + 'subtitle': 'Subtitle' + } + ) + + hash_value = compute_source_hash(content) + + # Hash should be a 64-character hex string (SHA256) + self.assertEqual(len(hash_value), 64) + self.assertTrue(all(c in '0123456789abcdef' for c in hash_value)) + + def test_compute_source_hash_consistency(self): + """Test that same content produces same hash.""" + json_metadata = { + 'title': 'Test Course', + 'short_description': 'Description', + 'full_description': 'Full desc', + 'outcome': 'Outcomes', + 'prerequisites': 'Prereqs', + 'subtitle': 'Subtitle' + } + + content1 = ContentMetadataFactory(json_metadata=json_metadata.copy()) + content2 = ContentMetadataFactory(json_metadata=json_metadata.copy()) + + hash1 = compute_source_hash(content1) + hash2 = compute_source_hash(content2) + + self.assertEqual(hash1, hash2) + + def test_compute_source_hash_different_content(self): + """Test that different content produces different hashes.""" + content1 = ContentMetadataFactory( + json_metadata={'title': 'Course 1', 'short_description': 'Desc 1'} + ) + content2 = ContentMetadataFactory( + json_metadata={'title': 'Course 2', 'short_description': 'Desc 2'} + ) + + hash1 = compute_source_hash(content1) + hash2 = compute_source_hash(content2) + + self.assertNotEqual(hash1, hash2) + + def test_compute_source_hash_missing_fields(self): + """Test hash computation with missing translatable fields.""" + content = ContentMetadataFactory( + json_metadata={ + 'title': 'Test Course', + # Missing other fields + } + ) + + hash_value = compute_source_hash(content) + + # Should still produce a valid hash + self.assertEqual(len(hash_value), 64) + + def test_compute_source_hash_custom_fields(self): + """Test hash computation with custom field list.""" + content = ContentMetadataFactory( + json_metadata={ + 'title': 'Test Course', + 'short_description': 'Description', + 'custom_field': 'Custom value' + } + ) + + # Compute hash with only title + hash1 = compute_source_hash(content, fields=['title']) + + # Compute hash with title and short_description + hash2 = compute_source_hash(content, fields=['title', 'short_description']) + + # Hashes should be different + self.assertNotEqual(hash1, hash2) + + def test_compute_source_hash_empty_values(self): + """Test hash computation with empty string values.""" + content = ContentMetadataFactory( + json_metadata={ + 'title': '', + 'short_description': '', + 'full_description': 'Some content' + } + ) + + hash_value = compute_source_hash(content) + + # Should produce a valid hash even with empty values + self.assertEqual(len(hash_value), 64) diff --git a/enterprise_catalog/apps/catalog/tests/test_populate_spanish_translations.py b/enterprise_catalog/apps/catalog/tests/test_populate_spanish_translations.py new file mode 100644 index 00000000..f90abfa5 --- /dev/null +++ b/enterprise_catalog/apps/catalog/tests/test_populate_spanish_translations.py @@ -0,0 +1,251 @@ +""" +Tests for populate_spanish_translations management command. +""" +from unittest import mock + +from django.core.management import call_command +from django.test import TestCase + +from enterprise_catalog.apps.catalog.models import ContentTranslation +from enterprise_catalog.apps.catalog.tests.factories import ( + ContentMetadataFactory, +) +from enterprise_catalog.apps.catalog.utils import compute_source_hash + + +class PopulateSpanishTranslationsCommandTests(TestCase): + """ + Tests for the populate_spanish_translations management command. + """ + + def setUp(self): + """Set up test data.""" + self.content1 = ContentMetadataFactory( + content_key='course-1', + content_type='course', + json_metadata={ + 'title': 'Introduction to Python', + 'short_description': 'Learn Python basics', + 'full_description': 'A comprehensive course on Python fundamentals', + } + ) + self.content2 = ContentMetadataFactory( + content_key='course-2', + content_type='course', + json_metadata={ + 'title': 'Advanced JavaScript', + 'short_description': 'Master JavaScript', + } + ) + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_creates_translations(self, mock_translate): + """Test that command creates new translations.""" + mock_translate.return_value = { + 'title': 'Título traducido', + 'short_description': 'Descripción corta', + } + + # Run command + call_command('populate_spanish_translations') + + # Check translations were created + self.assertEqual(ContentTranslation.objects.count(), 2) + + translation1 = ContentTranslation.objects.get(content_metadata=self.content1) + self.assertEqual(translation1.language_code, 'es') + self.assertEqual(translation1.title, 'Título traducido') + self.assertIsNotNone(translation1.source_hash) + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_skips_existing_translations(self, mock_translate): + """Test that command skips translations with matching hash.""" + mock_translate.return_value = {'title': 'Translated'} + + # Create existing translation with correct hash + source_hash = compute_source_hash(self.content1) + ContentTranslation.objects.create( + content_metadata=self.content1, + language_code='es', + title='Existing Translation', + source_hash=source_hash + ) + + # Run command + call_command('populate_spanish_translations') + + # Should not update existing translation + translation = ContentTranslation.objects.get(content_metadata=self.content1) + self.assertEqual(translation.title, 'Existing Translation') + + # Should create translation for content2 + self.assertEqual(ContentTranslation.objects.count(), 2) + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_force_retranslate(self, mock_translate): + """Test that --force flag re-translates existing content.""" + mock_translate.return_value = {'title': 'New Translation'} + + # Create existing translation + ContentTranslation.objects.create( + content_metadata=self.content1, + language_code='es', + title='Old Translation', + source_hash='old_hash' + ) + + # Run command with force + call_command('populate_spanish_translations', force=True) + + # Should update existing translation + translation = ContentTranslation.objects.get(content_metadata=self.content1) + self.assertEqual(translation.title, 'New Translation') + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_content_keys_filter(self, mock_translate): + """Test filtering by content keys.""" + mock_translate.return_value = {'title': 'Translated'} + + # Run command for only content1 + call_command('populate_spanish_translations', content_keys=['course-1']) + + # Should only create translation for content1 + self.assertEqual(ContentTranslation.objects.count(), 1) + self.assertTrue( + ContentTranslation.objects.filter(content_metadata=self.content1).exists() + ) + self.assertFalse( + ContentTranslation.objects.filter(content_metadata=self.content2).exists() + ) + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_dry_run(self, mock_translate): + """Test that--dry-run doesn't save translations.""" + mock_translate.return_value = {'title': 'Translated'} + + # Run command in dry-run mode + call_command('populate_spanish_translations', dry_run=True) + + # No translations should be saved + self.assertEqual(ContentTranslation.objects.count(), 0) + + # But translate should still be called + self.assertTrue(mock_translate.called) + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_updates_stale_translations(self, mock_translate): + """Test that command updates translations when content changes.""" + mock_translate.return_value = {'title': 'Updated Translation'} + + # Create translation with old hash + ContentTranslation.objects.create( + content_metadata=self.content1, + language_code='es', + title='Old Translation', + source_hash='outdated_hash' # Different from current content + ) + + # Run command (without force) + call_command('populate_spanish_translations') + + # Should update the translation because hash doesn't match + translation = ContentTranslation.objects.get(content_metadata=self.content1) + self.assertEqual(translation.title, 'Updated Translation') + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_batch_processing(self, mock_translate): + """Test command processes in batches.""" + # Create more content + for i in range(5): + ContentMetadataFactory( + content_key=f'course-{i + 3}', + json_metadata={'title': f'Course {i + 3}'} + ) + + mock_translate.return_value = {'title': 'Translated'} + + # Run with small batch size + call_command('populate_spanish_translations', batch_size=2) + + # All content should be translated + self.assertEqual(ContentTranslation.objects.count(), 7) # 2 original + 5 new + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_handles_translation_errors(self, mock_translate): + """Test command continues after translation errors.""" + # First call succeeds, second fails, third succeeds + mock_translate.side_effect = [ + {'title': 'Success 1'}, + Exception('Translation API error'), + {'title': 'Success 2'}, + ] + + # Create third content + ContentMetadataFactory(content_key='course-3', json_metadata={'title': 'Course 3'}) + + # Run command - should not crash + call_command('populate_spanish_translations') + + # Should have created 2 translations (skipped the one that errored) + self.assertEqual(ContentTranslation.objects.count(), 2) + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_custom_language(self, mock_translate): + """Test command supports custom language codes.""" + mock_translate.return_value = {'title': 'Titre français'} + + # Run command for French + call_command('populate_spanish_translations', language='fr') + + # Should create French translation + translation = ContentTranslation.objects.first() + self.assertEqual(translation.language_code, 'fr') + + @mock.patch( + 'enterprise_catalog.apps.catalog.management.commands.' + 'populate_spanish_translations.translate_object_fields' + ) + def test_command_translates_all_fields(self, mock_translate): + """Test that command translates all relevant fields.""" + mock_translate.return_value = { + 'title': 'Título', + 'short_description': 'Descripción corta', + 'full_description': 'Descripción completa', + 'subtitle': 'Subtítulo', + } + + # Run command + call_command('populate_spanish_translations') + + translation = ContentTranslation.objects.first() + self.assertEqual(translation.title, 'Título') + self.assertEqual(translation.short_description, 'Descripción corta') + self.assertEqual(translation.full_description, 'Descripción completa') + self.assertEqual(translation.subtitle, 'Subtítulo') diff --git a/enterprise_catalog/apps/catalog/utils.py b/enterprise_catalog/apps/catalog/utils.py index f2b6f7eb..a471b28a 100644 --- a/enterprise_catalog/apps/catalog/utils.py +++ b/enterprise_catalog/apps/catalog/utils.py @@ -157,3 +157,30 @@ def to_timestamp(datetime_str): except (ValueError, TypeError) as exc: LOGGER.error(f"[to_timestamp][{exc}] Could not parse date string: {datetime_str}") return None + + +def compute_source_hash(content_metadata, fields=None): + """ + Compute a SHA256 hash of source content fields for change detection. + + Args: + content_metadata: ContentMetadata instance + fields: List of field names to include in hash (default: translatable fields) + + Returns: + str: SHA256 hash of concatenated field values + """ + if fields is None: + # Only hash fields that are actually translated + fields = [ + 'title', 'short_description', 'full_description', 'subtitle' + ] + + content_parts = [] + for field in fields: + value = content_metadata.json_metadata.get(field, '') + if value: + content_parts.append(str(value)) + + combined = '|'.join(content_parts) + return hashlib.sha256(combined.encode('utf-8')).hexdigest()