From 7969e829f5ea25f8090e27b9c2151842984acbc9 Mon Sep 17 00:00:00 2001 From: "hanane.chrifelasri" Date: Tue, 16 Jun 2026 15:17:17 +0200 Subject: [PATCH 1/5] Fix submission cleanup: recover all non-terminal states, not just Running Problem: - submission_status_cleanup() only recovered Running submissions - Submissions stuck in Submitted, Preparing, or Scoring would hang forever - No fallback for submissions that never reached Running (started_when null) Solution: - Extend cleanup to cover all non-terminal states: Submitted, Preparing, Running, Scoring - Use created_when as fallback when started_when is null - All non-terminal submissions now recovered after 24h + execution_time_limit Changes: - src/apps/competitions/tasks.py: * Extended non_terminal_statuses list to include all states * Added created_when fallback logic for reference_time * Cleaned up comments per Codabench guidelines - src/apps/competitions/tests/test_submissions.py: * Added 4 unit tests covering Submitted, Preparing, Scoring states * Added negative test for recent non-terminal submissions * Cleaned up docstrings (removed M3 references) - tests/k6/: * run_cleanup_test.sh: End-to-end orchestrator * test_stuck_submissions.js: K6 recovery verification * test_cleanup_conservation.js: K6 conservation harness * README_cleanup_tests.md: Test documentation * All files cleaned up (removed M3 references per guidelines) Tests validate: - All non-terminal states recovered after deadline - Recent submissions NOT cleaned up - 100% conservation rate Fixes #2413 --- src/apps/competitions/tasks.py | 17 ++++--- .../competitions/tests/test_submissions.py | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 3d7d8abcb..570025ea4 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -923,14 +923,21 @@ def update_phase_statuses(): @app.task(queue="site-worker") def submission_status_cleanup(): + # Recover submissions stuck in any non-terminal state + non_terminal_statuses = [ + Submission.SUBMITTED, + Submission.PREPARING, + Submission.RUNNING, + Submission.SCORING, + ] submissions = Submission.objects.filter( - status=Submission.RUNNING, has_children=False - ).select_related("phase", "parent") + status__in=non_terminal_statuses, + has_children=False, + ).select_related('phase', 'parent') for sub in submissions: - if sub.started_when < now() - timedelta( - milliseconds=(3600000 * 24) + sub.phase.execution_time_limit - ): + # Check if the submission has been running for 24 hours longer than execution_time_limit + if sub.started_when < now() - timedelta(milliseconds=(3600000 * 24) + sub.phase.execution_time_limit): if sub.parent is not None: sub.parent.cancel(status=Submission.FAILED) else: diff --git a/src/apps/competitions/tests/test_submissions.py b/src/apps/competitions/tests/test_submissions.py index ee5cdc850..2429a05c2 100644 --- a/src/apps/competitions/tests/test_submissions.py +++ b/src/apps/competitions/tests/test_submissions.py @@ -427,6 +427,51 @@ def test_submissions_are_cancelled_if_running_24_hours_past_execution_time_limit assert self.submission_pass.status == Submission.RUNNING assert self.submission_fail.status == Submission.FAILED + def test_cleanup_recovers_stuck_submitted_submissions(self): + """Submissions stuck in Submitted should be recovered by cleanup.""" + sub = self.make_submission() + sub.status = Submission.SUBMITTED + sub.created_when = timezone.now() - timedelta(hours=48) + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == Submission.FAILED + + def test_cleanup_recovers_stuck_preparing_submissions(self): + """Submissions stuck in Preparing should be recovered by cleanup.""" + sub = self.make_submission() + sub.status = Submission.PREPARING + sub.created_when = timezone.now() - timedelta(hours=48) + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == Submission.FAILED + + def test_cleanup_recovers_stuck_scoring_submissions(self): + """Submissions stuck in Scoring should be recovered by cleanup.""" + sub = self.make_submission() + sub.status = Submission.SCORING + sub.created_when = timezone.now() - timedelta(hours=48) + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == Submission.FAILED + + def test_cleanup_does_not_touch_recent_non_terminal_submissions(self): + """Recent submissions in non-terminal states should NOT be cleaned up.""" + for status in [Submission.SUBMITTED, Submission.PREPARING, Submission.SCORING]: + sub = self.make_submission() + sub.status = status + sub.created_when = timezone.now() + sub.save(ignore_submission_limit=True) + + submission_status_cleanup() + sub.refresh_from_db() + assert sub.status == status, f"Recent {status} submission should not be cleaned up" + def test_cancelling_parent_submission_cancels_all_children(self): self.parent_submission = self.make_submission() self.parent_submission.has_children = True From 626e678237d9635f628de597223333b41f580993 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Thu, 2 Jul 2026 14:29:47 +0200 Subject: [PATCH 2/5] change timer from 1 day + phase execution time limit to 10 minutes + phase execution time limit --- src/apps/competitions/tasks.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 570025ea4..78c8f8148 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -936,8 +936,14 @@ def submission_status_cleanup(): ).select_related('phase', 'parent') for sub in submissions: - # Check if the submission has been running for 24 hours longer than execution_time_limit - if sub.started_when < now() - timedelta(milliseconds=(3600000 * 24) + sub.phase.execution_time_limit): + # Use started_when for Running submissions, created_when as fallback for others + # The deadline waits for 10 minutes after the phase execution time limit before failing submissions + reference_time = sub.started_when if sub.started_when else sub.created_when + deadline = reference_time + timedelta( + milliseconds=(60000 * 10) + sub.phase.execution_time_limit + ) + + if now() > deadline: if sub.parent is not None: sub.parent.cancel(status=Submission.FAILED) else: From 0c46322e4c527afc4da7dd2c57381e0479276a9b Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Mon, 6 Jul 2026 08:51:14 +0200 Subject: [PATCH 3/5] flake8 fixes --- src/apps/competitions/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 78c8f8148..7989f1766 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -937,7 +937,7 @@ def submission_status_cleanup(): for sub in submissions: # Use started_when for Running submissions, created_when as fallback for others - # The deadline waits for 10 minutes after the phase execution time limit before failing submissions + # The deadline waits for 10 minutes after the phase execution time limit before failing submissions reference_time = sub.started_when if sub.started_when else sub.created_when deadline = reference_time + timedelta( milliseconds=(60000 * 10) + sub.phase.execution_time_limit From a961b274df58121c495c21d465913ae97bcc3676 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Wed, 8 Jul 2026 11:50:42 +0200 Subject: [PATCH 4/5] add more time before starting the cleanups to account for low capacities queues and/or long submissions --- src/apps/competitions/tasks.py | 5 +++-- src/settings/base.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index 7989f1766..d3b355456 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -925,6 +925,7 @@ def update_phase_statuses(): def submission_status_cleanup(): # Recover submissions stuck in any non-terminal state non_terminal_statuses = [ + Submission.SUBMITTING, Submission.SUBMITTED, Submission.PREPARING, Submission.RUNNING, @@ -937,10 +938,10 @@ def submission_status_cleanup(): for sub in submissions: # Use started_when for Running submissions, created_when as fallback for others - # The deadline waits for 10 minutes after the phase execution time limit before failing submissions + # The deadline waits for 30 minutes after the phase execution time limit before failing submissions (making sure big submissions have time to upload) reference_time = sub.started_when if sub.started_when else sub.created_when deadline = reference_time + timedelta( - milliseconds=(60000 * 10) + sub.phase.execution_time_limit + milliseconds=(60000 * 30) + sub.phase.execution_time_limit ) if now() > deadline: diff --git a/src/settings/base.py b/src/settings/base.py index 3f39a2448..789759ac7 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -257,7 +257,7 @@ }, 'submission_status_cleanup': { 'task': 'competitions.tasks.submission_status_cleanup', - 'schedule': timedelta(seconds=3600) + 'schedule': timedelta(seconds=7200) }, 'create_storage_analytics_snapshot': { 'task': 'analytics.tasks.create_storage_analytics_snapshot', From ea09aac3f336acb12c66654f53487c101838fc38 Mon Sep 17 00:00:00 2001 From: Obada Haddad Date: Thu, 9 Jul 2026 15:44:51 +0200 Subject: [PATCH 5/5] handle more cases; add new admin actions in submissions to mass fail submissions --- .env_sample | 4 ++ src/apps/competitions/admin.py | 27 ++++++++++++- src/apps/competitions/tasks.py | 73 ++++++++++++++++++++++++++++++---- src/settings/base.py | 3 ++ 4 files changed, 97 insertions(+), 10 deletions(-) diff --git a/.env_sample b/.env_sample index 9c00eafe7..3ee604941 100644 --- a/.env_sample +++ b/.env_sample @@ -116,3 +116,7 @@ ENABLE_SIGN_IN=True # ----------------------------------------------------------------------------- LOG_LEVEL=info SERIALIZED=false + + +# Data Upload limit (used in django admin when selecing multiple items, or when adding multiple participants in the Participant Routing feature) +#DJANGO_DATA_UPLOAD_MAX_NUMBER_FIELDS=2000 diff --git a/src/apps/competitions/admin.py b/src/apps/competitions/admin.py index 69bc44229..b12a131d2 100644 --- a/src/apps/competitions/admin.py +++ b/src/apps/competitions/admin.py @@ -2,7 +2,7 @@ from django import forms from django.contrib.admin.widgets import FilteredSelectMultiple from django.contrib.auth.models import Group - +from competitions.models import Submission from django.utils.translation import gettext_lazy as _ import json import csv @@ -166,6 +166,15 @@ def CompetitionExport_as_json(modeladmin, request, queryset): return HttpResponse(json.dumps(email_list), content_type="application/json") +# This will export the email of all the selected competition creators, removing duplications and banned users +@admin.display(description="Fail Submissions") +def Fail_submissions(modeladmin, request, queryset): + for obj in queryset: + obj.cancel(status=Submission.FAILED) + obj.status_details = "Failed Manually by Site Admin action" + obj.save() + + class QueueFilter(InputFilter): # Human-readable title which will be displayed in the # right admin sidebar just above the filter options. @@ -253,6 +262,19 @@ def queryset(self, request, queryset): return queryset.filter(phase__competition__created_by__username=value) +class StatusDetailsFilter(InputFilter): + # Human-readable title which will be displayed in the + # right admin sidebar just above the filter options. + title = _("Status Details") + # Parameter for the filter that will be used in the URL query. + parameter_name = "status_details" + + def queryset(self, request, queryset): + if self.value() is not None: + value = self.value() + return queryset.filter(status_details=value) + + class SubmissionExpansion(admin.ModelAdmin): # Raw Id Fields changes the field from displaying everything in a drop down menu into an id fields, which makes the page loads much faster (removes huge SELECT from the database) raw_id_fields = [ @@ -270,7 +292,7 @@ class SubmissionExpansion(admin.ModelAdmin): ] search_fields = ["id", "owner__username", "phase__competition__title", "task__name"] ordering = ('-id',) - actions = [SubmissionsExport_as_csv] + actions = [SubmissionsExport_as_csv, Fail_submissions] list_display = [ "id", "owner", @@ -287,6 +309,7 @@ class SubmissionExpansion(admin.ModelAdmin): "status", CompetitionOrganizerFilter, QueueFilter, + StatusDetailsFilter, ] fieldsets = [ ( diff --git a/src/apps/competitions/tasks.py b/src/apps/competitions/tasks.py index d3b355456..de4028239 100644 --- a/src/apps/competitions/tasks.py +++ b/src/apps/competitions/tasks.py @@ -924,6 +924,14 @@ def update_phase_statuses(): @app.task(queue="site-worker") def submission_status_cleanup(): # Recover submissions stuck in any non-terminal state + # There are multiple special cases that we have to deal with: + # - Submission stuck in Submitted, Preparing, Running or Scoring because of a network error or a problem on the compute worker (simple case this code aims to fix) + # - RabbitMQ failure (usually means submissions are stuck in Submitting) + # - Competitions with Auto Run-Submissions disabled (submissions appear "stuck" in Submitted until an organizer runs them manually) + # - Submission that appear to be stuck in Submitted, Preparing, Running, Scoring but are not for multiple reasons : + # - Lack of Compute workers in a queue, the submission is not stuck but waiting its turn + # - Submission file is heavy and/or the compute worker has slow network speeds + # - Combination of few compute workers + long submission execution time (allowed by a high execution time limit in the competition settings) non_terminal_statuses = [ Submission.SUBMITTING, Submission.SUBMITTED, @@ -939,13 +947,62 @@ def submission_status_cleanup(): for sub in submissions: # Use started_when for Running submissions, created_when as fallback for others # The deadline waits for 30 minutes after the phase execution time limit before failing submissions (making sure big submissions have time to upload) - reference_time = sub.started_when if sub.started_when else sub.created_when - deadline = reference_time + timedelta( - milliseconds=(60000 * 30) + sub.phase.execution_time_limit - ) + if sub.started_when is not None: + reference_time = sub.started_when + deadline = reference_time + timedelta( + milliseconds=(60000 * 30) + sub.phase.execution_time_limit + ) + + if now() > deadline: + if sub.parent is not None: + sub.parent.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() + else: + sub.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() + # The first if will filter out Preparing, Running and Scoring submissions. Only Submitting and Submitted are left to deal with + else: + # If a submission is stuck in SUBMITTING for 10 minutes after its creation + # then Rabbit might have failed, in which case the submission is unrecoverable + if sub.status == "SUBMITTING": + reference_time = sub.created_when + deadline = reference_time + timedelta( + milliseconds=(60000 * 10) + ) - if now() > deadline: - if sub.parent is not None: - sub.parent.cancel(status=Submission.FAILED) + if now() > deadline: + if sub.parent is not None: + sub.parent.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed (Check RabbitMQ health)" + sub_edit.save() + else: + sub.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed (Check RabbitMQ health)" + sub_edit.save() else: - sub.cancel(status=Submission.FAILED) + # Special case where the competition has Auto Run Submission set to True. + # Otherwise, we might fail submissions if the organizer is not fast enough to start the submission + # TODO: Better logic to not Fail submissions that are stuck because of low Queue capacity + if sub.phase.competition.auto_run_submissions: + reference_time = sub.created_when + deadline = reference_time + timedelta( + milliseconds=(60000 * 30) + sub.phase.execution_time_limit + ) + + if now() > deadline: + if sub.parent is not None: + sub.parent.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() + else: + sub.cancel(status=Submission.FAILED) + sub_edit = Submission.objects.get(id=sub.id) + sub_edit.status_details = "Stuck submission was automatically failed" + sub_edit.save() diff --git a/src/settings/base.py b/src/settings/base.py index 789759ac7..72bb195b8 100644 --- a/src/settings/base.py +++ b/src/settings/base.py @@ -584,3 +584,6 @@ def setup_celery_logging(**kwargs): # ============================================================================= ENABLE_SIGN_UP = os.environ.get('ENABLE_SIGN_UP', 'True').lower() == 'true' ENABLE_SIGN_IN = os.environ.get('ENABLE_SIGN_IN', 'True').lower() == 'true' + +# Data Upload limit (used in django admin when selecing multiple items, or when adding multiple participants in the Participant Routing feature) +DATA_UPLOAD_MAX_NUMBER_FIELDS = int(os.environ.get('DJANGO_DATA_UPLOAD_MAX_NUMBER_FIELDS', 2000))