Skip to content

build: rebalance unit test shards to reduce CI critical path (~29m --> ~23m) and use fewer workers (16 --> 10) #38287

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
feanil merged 4 commits into from
Apr 14, 2026
Merged

build: rebalance unit test shards to reduce CI critical path (~29m --> ~23m) and use fewer workers (16 --> 10) #38287

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d2e1cef
build: collect per-test timing data.
feanil Apr 4, 2026
00c5c5a
build: rebalance unit test shards to reduce critical path
feanil Apr 5, 2026
e9f093a
build: split cms-1 shard — move contentstore/ to new cms-2
feanil Apr 10, 2026
4f1e0c8
build: only collect per-test timing data on master pushes
feanil Apr 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
187 changes: 80 additions & 107 deletions .github/workflows/unit-test-shards.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,168 +2,169 @@
"lms-1": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/branding/",
"lms/djangoapps/bulk_email/",
"lms/djangoapps/bulk_enroll/",
"lms/djangoapps/bulk_user_retirement/",
"lms/djangoapps/ccx/",
"lms/djangoapps/certificates/",
"lms/djangoapps/commerce/"
"lms/djangoapps/discussion/rest_api/"
]
},
"lms-2": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/ccx/",
"lms/djangoapps/commerce/",
"lms/djangoapps/course_api/",
"lms/djangoapps/course_blocks/",
"lms/djangoapps/course_goals/",
"lms/djangoapps/course_home_api/",
"lms/djangoapps/course_wiki/",
"lms/djangoapps/coursewarehistoryextended/",
"lms/djangoapps/debug/"
"lms/djangoapps/discussion/notification_prefs/",
"lms/djangoapps/instructor_task/",
"lms/djangoapps/ora_staff_grader/",
"lms/djangoapps/survey/",
"lms/djangoapps/teams/",
"lms/djangoapps/verify_student/"
]
},
"lms-3": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/courseware/"
"lms/djangoapps/branding/",
"lms/djangoapps/bulk_enroll/",
"lms/djangoapps/courseware/",
"lms/djangoapps/instructor_analytics/",
"lms/djangoapps/learner_dashboard/",
"lms/djangoapps/lti_provider/",
"lms/djangoapps/program_enrollments/"
]
},
"lms-4": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/discussion/",
"lms/djangoapps/edxnotes/",
"lms/djangoapps/experiments/"
]
},
"lms-5": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/gating/",
"lms/djangoapps/bulk_user_retirement/",
"lms/djangoapps/course_blocks/",
"lms/djangoapps/course_goals/",
"lms/djangoapps/course_home_api/",
"lms/djangoapps/coursewarehistoryextended/",
"lms/djangoapps/discussion/tests/",
"lms/djangoapps/experiments/",
"lms/djangoapps/grades/",
"lms/djangoapps/instructor/",
"lms/djangoapps/instructor_analytics/"
"lms/djangoapps/mfe_config_api/",
"lms/djangoapps/staticbook/",
"lms/lib/"
]
},
"lms-6": {
"lms-5": {
"settings": "lms.envs.test",
"paths": [
"lms/djangoapps/instructor_task/",
"lms/djangoapps/learner_dashboard/",
"lms/djangoapps/bulk_email/",
"lms/djangoapps/certificates/",
"lms/djangoapps/debug/",
"lms/djangoapps/discussion/django_comment_client/",
"lms/djangoapps/edxnotes/",
"lms/djangoapps/gating/",
"lms/djangoapps/learner_home/",
"lms/djangoapps/lms_initialization/",
"lms/djangoapps/lms_xblock/",
"lms/djangoapps/lti_provider/",
"lms/djangoapps/mailing/",
"lms/djangoapps/mobile_api/",
"lms/djangoapps/monitoring/",
"lms/djangoapps/ora_staff_grader/",
"lms/djangoapps/program_enrollments/",
"lms/djangoapps/rss_proxy/",
"lms/djangoapps/static_template_view/",
"lms/djangoapps/staticbook/",
"lms/djangoapps/support/",
"lms/djangoapps/survey/",
"lms/djangoapps/teams/",
"lms/djangoapps/tests/",
"lms/djangoapps/user_tours/",
"lms/djangoapps/verify_student/",
"lms/djangoapps/mfe_config_api/",
"lms/envs/",
"lms/lib/",
"lms/tests.py"
]
},
"openedx-1-with-lms": {
"shared-with-lms-1": {
"settings": "lms.envs.test",
"paths": [
"openedx/core/djangoapps/ace_common/",
"openedx/core/djangoapps/cors_csrf/",
"common/djangoapps/",
"openedx/core/djangoapps/agreements/",
"openedx/core/djangoapps/api_admin/",
"openedx/core/djangoapps/authz/",
"openedx/core/djangoapps/cache_toolbox/",
"openedx/core/djangoapps/ccxcon/",
"openedx/core/djangoapps/content/",
"openedx/core/djangoapps/content_libraries/",
"openedx/core/djangoapps/course_apps/",
"openedx/core/djangoapps/credentials/",
"openedx/core/djangoapps/credit/",
"openedx/core/djangoapps/dark_lang/",
"openedx/core/djangoapps/django_comment_common/",
"openedx/core/djangoapps/embargo/",
"openedx/core/djangoapps/header_control/",
"openedx/core/djangoapps/heartbeat/",
"openedx/core/djangoapps/models/",
"openedx/core/djangoapps/notifications/",
"openedx/core/djangoapps/oauth_dispatch/",
"openedx/core/djangoapps/safe_sessions/",
"openedx/core/djangoapps/schedules/",
"openedx/core/djangoapps/user_api/",
"openedx/core/djangoapps/util/",
"openedx/core/djangoapps/video_pipeline/",
"openedx/core/djangoapps/waffle_utils/",
"openedx/core/djangoapps/xblock/",
"openedx/core/tests/",
"openedx/features/",
"openedx/tests/"
]
},
"shared-with-lms-2": {
"settings": "lms.envs.test",
"paths": [
"openedx/core/djangoapps/ace_common/",
"openedx/core/djangoapps/auth_exchange/",
"openedx/core/djangoapps/bookmarks/",
"openedx/core/djangoapps/cache_toolbox/",
"openedx/core/djangoapps/catalog/",
"openedx/core/djangoapps/ccxcon/",
"openedx/core/djangoapps/commerce/",
"openedx/core/djangoapps/common_initialization/",
"openedx/core/djangoapps/common_views/",
"openedx/core/djangoapps/config_model_utils/",
"openedx/core/djangoapps/content/",
"openedx/core/djangoapps/content_libraries/",
"openedx/core/djangoapps/contentserver/",
"openedx/core/djangoapps/cookie_metadata/",
"openedx/core/djangoapps/course_apps/",
"openedx/core/djangoapps/cors_csrf/",
"openedx/core/djangoapps/course_date_signals/",
"openedx/core/djangoapps/course_groups/",
"openedx/core/djangoapps/course_live/",
"openedx/core/djangoapps/courseware_api/",
"openedx/core/djangoapps/crawlers/",
"openedx/core/djangoapps/credentials/",
"openedx/core/djangoapps/credit/",
"openedx/core/djangoapps/course_live/",
"openedx/core/djangoapps/dark_lang/",
"openedx/core/djangoapps/debug/",
"openedx/core/djangoapps/discussions/",
"openedx/core/djangoapps/django_comment_common/",
"openedx/core/djangoapps/embargo/",
"openedx/core/djangoapps/enrollments/",
"openedx/core/djangoapps/external_user_ids/"
]
},
"openedx-2-with-lms": {
"settings": "lms.envs.test",
"paths": [
"openedx/core/djangoapps/external_user_ids/",
"openedx/core/djangoapps/geoinfo/",
"openedx/core/djangoapps/header_control/",
"openedx/core/djangoapps/heartbeat/",
"openedx/core/djangoapps/lang_pref/",
"openedx/core/djangoapps/models/",
"openedx/core/djangoapps/monkey_patch/",
"openedx/core/djangoapps/notifications/",
"openedx/core/djangoapps/oauth_dispatch/",
"openedx/core/djangoapps/olx_rest_api/",
"openedx/core/djangoapps/password_policy/",
"openedx/core/djangoapps/plugin_api/",
"openedx/core/djangoapps/plugins/",
"openedx/core/djangoapps/profile_images/",
"openedx/core/djangoapps/programs/",
"openedx/core/djangoapps/safe_sessions/",
"openedx/core/djangoapps/schedules/",
"openedx/core/djangoapps/service_status/",
"openedx/core/djangoapps/session_inactivity_timeout/",
"openedx/core/djangoapps/signals/",
"openedx/core/djangoapps/site_configuration/",
"openedx/core/djangoapps/system_wide_roles/",
"openedx/core/djangoapps/theming/",
"openedx/core/djangoapps/user_api/",
"openedx/core/djangoapps/user_authn/",
"openedx/core/djangoapps/util/",
"openedx/core/djangoapps/verified_track_content/",
"openedx/core/djangoapps/video_config/",
"openedx/core/djangoapps/video_pipeline/",
"openedx/core/djangoapps/waffle_utils/",
"openedx/core/djangoapps/xblock/",
"openedx/core/djangoapps/xmodule_django/",
"openedx/core/djangoapps/zendesk_proxy/",
"openedx/core/djangoapps/authz/",
"openedx/core/djangolib/",
"openedx/core/lib/",
"openedx/core/tests/",
"openedx/features/",
"openedx/testing/",
"openedx/tests/"
"xmodule/"
]
},
"openedx-1-with-cms": {
"shared-with-cms-1": {
Copy link
Copy Markdown
Member

@kdmccormick kdmccormick Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It was nice for debugging that for each X-with-lms shard there was a corresponding X-with-cms, especially for those tests which would pass in one system and fail in the other. Would you be willing to change it so that we have parallel shared-with-lms-[1,2] and shared-with-cms-[1,2] shards? It would only add one additional shard and I don't think it'd increase the critical test time.

If not, then could you simplify shared-with-cms-1 definition into just the paths xmodule/, common/, and openedx/?

Copy link
Copy Markdown
Contributor Author

@feanil feanil Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I split them for convenience for now. I didn't want to collapse the tests because that will make it harder to re-balance them in the future since it would require more lookups to do the rebalancing.

Copy link
Copy Markdown
Member

@kdmccormick kdmccormick Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did you catch this feedback?

It was nice for debugging that for each X-with-lms shard there was a corresponding X-with-cms, especially for those tests which would pass in one system and fail in the other. Would you be willing to change it so that we have parallel shared-with-lms-[1,2] and shared-with-cms-[1,2] shards? It would only add one additional shard and I don't think it'd increase the critical test time.

Copy link
Copy Markdown
Member

@kdmccormick kdmccormick Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nevermind, just saw your commit

Copy link
Copy Markdown
Contributor Author

@feanil feanil Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL, that apparently we don't run all of the openedx apps under CMS, just some of them and if we try to run all of them there are issues:

These folders are run under LMS but not CMS:

openedx/core/djangoapps/course_live/
openedx/core/djangoapps/notifications/
openedx/core/djangolib/
openedx/core/tests/
openedx/features/
openedx/testing/

What do you think about landing this as is? I think it could be further improved and there's more to investigate but I don't want this to be blocked on existing issues.

Copy link
Copy Markdown
Member

@kdmccormick kdmccormick Apr 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wow that's crazy

Yeah, in that case, totally OK with there being just one CMS shard

I wrote a followup issue, do you mind linking it here with a TODO comment? #38355

Otherwise, LGTM ✅

Copy link
Copy Markdown
Member

@kdmccormick kdmccormick Apr 14, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, I guess you can't add a comment here's, it's JSON

"settings": "cms.envs.test",
"paths": [
"common/djangoapps/",
"openedx/core/djangoapps/ace_common/",
"openedx/core/djangoapps/cors_csrf/",
"openedx/core/djangoapps/agreements/",
"openedx/core/djangoapps/api_admin/",
"openedx/core/djangoapps/auth_exchange/",
"openedx/core/djangoapps/authz/",
"openedx/core/djangoapps/bookmarks/",
"openedx/core/djangoapps/cache_toolbox/",
"openedx/core/djangoapps/catalog/",
Expand All @@ -175,8 +176,10 @@
"openedx/core/djangoapps/content/",
"openedx/core/djangoapps/content_libraries/",
"openedx/core/djangoapps/content_staging/",
"openedx/core/djangoapps/content_tagging/",
"openedx/core/djangoapps/contentserver/",
"openedx/core/djangoapps/cookie_metadata/",
"openedx/core/djangoapps/cors_csrf/",
"openedx/core/djangoapps/course_apps/",
"openedx/core/djangoapps/course_date_signals/",
"openedx/core/djangoapps/course_groups/",
Expand All @@ -190,13 +193,7 @@
"openedx/core/djangoapps/django_comment_common/",
"openedx/core/djangoapps/embargo/",
"openedx/core/djangoapps/enrollments/",
"openedx/core/djangoapps/external_user_ids/"
]
},
"openedx-2-with-cms": {
"settings": "cms.envs.test",
"paths": [
"openedx/core/djangoapps/content_tagging/",
"openedx/core/djangoapps/external_user_ids/",
"openedx/core/djangoapps/geoinfo/",
"openedx/core/djangoapps/header_control/",
"openedx/core/djangoapps/heartbeat/",
Expand Down Expand Up @@ -228,9 +225,9 @@
"openedx/core/djangoapps/xblock/",
"openedx/core/djangoapps/xmodule_django/",
"openedx/core/djangoapps/zendesk_proxy/",
"openedx/core/djangoapps/authz/",
"openedx/core/lib/",
"openedx/tests/"
"openedx/tests/",
"xmodule/"
]
},
"cms-1": {
Expand All @@ -240,8 +237,8 @@
"cms/djangoapps/cms_user_tasks/",
"cms/djangoapps/course_creators/",
"cms/djangoapps/export_course_metadata/",
"cms/djangoapps/modulestore_migrator/",
"cms/djangoapps/models/",
"cms/djangoapps/modulestore_migrator/",
"cms/djangoapps/pipeline_js/",
"cms/djangoapps/xblock_config/",
"cms/envs/",
Expand All @@ -253,29 +250,5 @@
"paths": [
"cms/djangoapps/contentstore/"
]
},
"common-with-lms": {
"settings": "lms.envs.test",
"paths": [
"common/djangoapps/"
]
},
"common-with-cms": {
"settings": "cms.envs.test",
"paths": [
"common/djangoapps/"
]
},
"xmodule-with-lms": {
"settings": "lms.envs.test",
"paths": [
"xmodule/"
]
},
"xmodule-with-cms": {
"settings": "cms.envs.test",
"paths": [
"xmodule/"
]
}
}
34 changes: 22 additions & 12 deletions .github/workflows/unit-tests.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,25 +25,21 @@ jobs:
- "3.12"
django-version:
- "pinned"
# When updating the shards, remember to make the same changes in
# .github/workflows/unit-tests-gh-hosted.yml
shard_name:
- "lms-1"
- "lms-2"
- "lms-3"
- "lms-4"
- "lms-5"
- "lms-6"
- "openedx-1-with-lms"
- "openedx-2-with-lms"
- "openedx-1-with-cms"
- "openedx-2-with-cms"
- "shared-with-lms-1"
- "shared-with-lms-2"
# Note: The shared-with-cms-1 shard is currently a subset of both
# shared-with-lms-1 and shared-with-lms-2. Some shared tests are
# not run -with-cms at all.
# https://github.com/openedx/openedx-platform/issues/38355
- "shared-with-cms-1"
Copy link
Copy Markdown
Member

@kdmccormick kdmccormick Apr 14, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
- "shared-with-cms-1"
# Note: The shared-with-cms-1 shard is currently a subset of both
# shared-with-lms-1 and shared-with-lms-2. Some shared tests are
# not run -with-cms at all.
# https://github.com/openedx/openedx-platform/issues/38355
- "shared-with-cms-1"

- "cms-1"
- "cms-2"
- "common-with-lms"
- "common-with-cms"
- "xmodule-with-lms"
- "xmodule-with-cms"
mongo-version:
- "7.0"
os-version:
Expand Down Expand Up @@ -115,11 +111,25 @@ jobs:
shell: bash
run: |
echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV
if [[ "${{ github.ref }}" == "refs/heads/master" ]]; then
echo "report_log_arg=--report-log=reports/pytest-report-${{ matrix.shard_name }}.jsonl" >> $GITHUB_ENV
else
echo "report_log_arg=" >> $GITHUB_ENV
fi

- name: run tests
shell: bash
run: |
python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=.
python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=. \
${{ env.report_log_arg }}

- name: Upload pytest timing report
if: github.ref == 'refs/heads/master'
uses: actions/upload-artifact@v7
with:
name: pytest-report-${{ matrix.shard_name }}-${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.mongo-version }}-${{ matrix.os-version }}
path: reports/pytest-report-${{ matrix.shard_name }}.jsonl
overwrite: true

- name: rename warnings json file
if: success()
Expand Down
3 changes: 3 additions & 0 deletions requirements/edx/development.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1724,6 +1724,7 @@ pytest==8.2.0
# pytest-json-report
# pytest-metadata
# pytest-randomly
# pytest-reportlog
# pytest-xdist
pytest-attrib==0.1.3
# via -r requirements/edx/testing.txt
Expand All @@ -1739,6 +1740,8 @@ pytest-metadata==3.1.1
# pytest-json-report
pytest-randomly==4.0.1
# via -r requirements/edx/testing.txt
pytest-reportlog==1.0.0
# via -r requirements/edx/testing.txt
pytest-xdist[psutil]==3.8.0
# via -r requirements/edx/testing.txt
python-dateutil==2.9.0.post0
Expand Down
Loading
Loading