diff --git a/.github/workflows/migrations-check.yml b/.github/workflows/migrations-check.yml deleted file mode 100644 index 84e334d68872..000000000000 --- a/.github/workflows/migrations-check.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: Check Django Migrations - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -jobs: - check_migrations: - name: check migrations - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-24.04] - python-version: - - "3.11" - # 'pinned' is used to install the latest patch version of Django - # within the global constraint i.e. Django==4.2.8 in current case - # because we have global constraint of Django<4.2 - django-version: ["pinned"] - mongo-version: - - "7" - mysql-version: ["8"] - services: - mongo: - image: mongo:${{ matrix.mongo-version }} - ports: - - 27017:27017 - # Note: Calling mongo here only works with mongo 4, in newer versions of mongo - # we'll have to use `mongosh`, hence the 'which mongosh mongo'. - options: >- - --health-cmd "$(which mongosh mongo) --quiet --eval 'db.runCommand(\"ping\")'" - --health-interval 10s - --health-timeout 5s - --health-retries 3 - mysql: - image: mysql:${{ matrix.mysql-version }} - ports: - - 3306:3306 - env: - MYSQL_DATABASE: "edxapp" - MYSQL_USER: "edxapp001" - MYSQL_PASSWORD: "password" - MYSQL_RANDOM_ROOT_PASSWORD: true - options: >- - --health-cmd "mysqladmin ping" - --health-interval 10s - --health-timeout 5s - --health-retries 3 - steps: - - name: Setup mongodb user - run: | - docker exec ${{ job.services.mongo.id }} mongosh edxapp --eval ' - db.createUser( - { - user: "edxapp", - pwd: "password", - roles: [ - { role: "readWrite", db: "edxapp" }, - ] - } - ); - ' - - - name: Verify mongo and mysql db credentials - run: | - mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp - docker exec ${{ job.services.mongo.id }} mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp - - - name: Checkout repo - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system Packages - run: | - sudo apt-get update - make ubuntu-requirements - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v4 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Python dependencies - run: | - make dev-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: list installed package versions - run: | - sudo pip freeze - - - name: Run Tests - env: - LMS_CFG: lms/envs/minimal.yml - # This is from the LMS dir on purpose since we don't need anything different for the CMS yet. - STUDIO_CFG: lms/envs/minimal.yml - run: | - echo "Running the LMS migrations." - ./manage.py lms migrate - echo "Running the CMS migrations." - ./manage.py cms migrate - - # This job aggregates test results. It's the required check for branch protection. - # https://github.com/marketplace/actions/alls-green#why - # https://github.com/orgs/community/discussions/33579 - success: - name: Migrations checks successful - if: always() - needs: - - check_migrations - runs-on: ubuntu-24.04 - steps: - - name: Decide whether the needed jobs succeeded or failed - # uses: re-actors/alls-green@v1.2.1 - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/pylint-checks.yml b/.github/workflows/pylint-checks.yml deleted file mode 100644 index abece08a5933..000000000000 --- a/.github/workflows/pylint-checks.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Pylint Checks - -on: - pull_request: - push: - branches: - - master - -jobs: - run-pylint: - runs-on: ubuntu-24.04 - strategy: - fail-fast: false - matrix: - include: - - module-name: lms-1 - path: "lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/ lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/ lms/djangoapps/courseware/ lms/djangoapps/course_goals/ lms/djangoapps/rss_proxy/" - - module-name: lms-2 - path: "lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/ lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/experiments/ lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/learner_home/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/ora_staff_grader/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/djangoapps/mfe_config_api/ lms/envs/ lms/lib/ lms/tests.py" - - module-name: openedx-1 - path: "openedx/core/types/ openedx/core/djangoapps/ace_common/ openedx/core/djangoapps/agreements/ openedx/core/djangoapps/api_admin/ openedx/core/djangoapps/auth_exchange/ openedx/core/djangoapps/bookmarks/ openedx/core/djangoapps/cache_toolbox/ openedx/core/djangoapps/catalog/ openedx/core/djangoapps/ccxcon/ openedx/core/djangoapps/commerce/ openedx/core/djangoapps/common_initialization/ openedx/core/djangoapps/common_views/ openedx/core/djangoapps/config_model_utils/ openedx/core/djangoapps/content/ openedx/core/djangoapps/content_libraries/ openedx/core/djangoapps/content_staging/ openedx/core/djangoapps/contentserver/ openedx/core/djangoapps/cookie_metadata/ openedx/core/djangoapps/cors_csrf/ openedx/core/djangoapps/course_apps/ openedx/core/djangoapps/course_date_signals/ openedx/core/djangoapps/course_groups/ openedx/core/djangoapps/courseware_api/ openedx/core/djangoapps/crawlers/ openedx/core/djangoapps/credentials/ openedx/core/djangoapps/credit/ openedx/core/djangoapps/dark_lang/ openedx/core/djangoapps/debug/ openedx/core/djangoapps/discussions/ openedx/core/djangoapps/django_comment_common/ openedx/core/djangoapps/embargo/ openedx/core/djangoapps/enrollments/ openedx/core/djangoapps/external_user_ids/ openedx/core/djangoapps/zendesk_proxy/ openedx/core/djangolib/ openedx/core/lib/ openedx/core/tests/ openedx/core/djangoapps/course_live/" - - module-name: openedx-2 - path: "openedx/core/djangoapps/geoinfo/ openedx/core/djangoapps/header_control/ openedx/core/djangoapps/heartbeat/ openedx/core/djangoapps/lang_pref/ openedx/core/djangoapps/models/ openedx/core/djangoapps/monkey_patch/ openedx/core/djangoapps/oauth_dispatch/ openedx/core/djangoapps/olx_rest_api/ openedx/core/djangoapps/password_policy/ openedx/core/djangoapps/plugin_api/ openedx/core/djangoapps/plugins/ openedx/core/djangoapps/profile_images/ openedx/core/djangoapps/programs/ openedx/core/djangoapps/safe_sessions/ openedx/core/djangoapps/schedules/ openedx/core/djangoapps/service_status/ openedx/core/djangoapps/session_inactivity_timeout/ openedx/core/djangoapps/signals/ openedx/core/djangoapps/site_configuration/ openedx/core/djangoapps/system_wide_roles/ openedx/core/djangoapps/theming/ openedx/core/djangoapps/user_api/ openedx/core/djangoapps/user_authn/ openedx/core/djangoapps/util/ openedx/core/djangoapps/verified_track_content/ openedx/core/djangoapps/video_config/ openedx/core/djangoapps/video_pipeline/ openedx/core/djangoapps/waffle_utils/ openedx/core/djangoapps/xblock/ openedx/core/djangoapps/xmodule_django/ openedx/core/tests/ openedx/features/ openedx/testing/ openedx/tests/ openedx/core/djangoapps/notifications/ openedx/core/djangoapps/staticfiles/ openedx/core/djangoapps/content_tagging/" - - module-name: common - path: "common" - - module-name: cms - path: "cms" - - module-name: xmodule - path: "xmodule" - - name: pylint ${{ matrix.module-name }} - steps: - - name: Check out repo - uses: actions/checkout@v4 - - - name: Install required system packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v4 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install required Python dependencies - run: | - # dev-requirements is needed because the linter will otherwise - # trip over some dev-only things like django-debug-toolbar - # (import debug_toolbar) that aren't in testing.txt. - make dev-requirements - # After all requirements are installed, check that they're consistent with each other - pip check - - - name: Run quality tests - run: | - pylint ${{ matrix.path }} - - # This job aggregates test results. It's the required check for branch protection. - # https://github.com/marketplace/actions/alls-green#why - # https://github.com/orgs/community/discussions/33579 - success: - name: Pylint checks successful - if: always() - needs: - - run-pylint - runs-on: ubuntu-24.04 - steps: - - name: Decide whether the needed jobs succeeded or failed - # uses: re-actors/alls-green@v1.2.1 - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe - with: - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/quality-checks.yml b/.github/workflows/quality-checks.yml index 3c80c1fac82f..db89aa1f0f6e 100644 --- a/.github/workflows/quality-checks.yml +++ b/.github/workflows/quality-checks.yml @@ -1,94 +1,16 @@ -name: Quality checks +name: CI - Quality Checks +### TODO: Update this flow from edx-platfrom branch on: pull_request: - push: + types: [ opened, synchronize, reopened ] branches: + - develop - master - - open-release/lilac.master - jobs: - run_tests: - name: Quality Others - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-24.04] - python-version: - - "3.11" - node-version: [20] - + quality-checks: + name: Quality Checks + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 2 - - - name: Fetch base branch for comparison - run: git fetch --depth=1 origin ${{ github.base_ref }} - - - name: Install Required System Packages - run: sudo apt-get update && sudo apt-get install libxmlsec1-dev - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: ${{ matrix.node-version }} - - - name: Setup npm - run: npm i -g npm@8.5.x - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v4 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/testing.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Install Required Python Dependencies - env: - PIP_SRC: ${{ runner.temp }} - run: | - make test-requirements - - - name: Install npm - env: - PIP_SRC: ${{ runner.temp }} - run: npm ci - - - name: Install python packages - env: - PIP_SRC: ${{ runner.temp }} - run: | - pip install -e . - - - name: Run Quality Tests - env: - PIP_SRC: ${{ runner.temp }} - TARGET_BRANCH: ${{ github.base_ref }} - run: | - make pycodestyle - make xsslint - make pii_check - make check_keywords - - - name: Save Job Artifacts - if: always() - uses: actions/upload-artifact@v4 - with: - name: Build-Artifacts - path: | - **/reports/**/* - test_root/log/**/*.log - *.log - overwrite: true + - name: Dummy Test + run: echo 'Add quality checks here' diff --git a/.github/workflows/unit-test-shards.json b/.github/workflows/unit-test-shards.json deleted file mode 100644 index 7184bf917ecb..000000000000 --- a/.github/workflows/unit-test-shards.json +++ /dev/null @@ -1,280 +0,0 @@ -{ - "lms-1": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/branding/", - "lms/djangoapps/bulk_email/", - "lms/djangoapps/bulk_enroll/", - "lms/djangoapps/bulk_user_retirement/", - "lms/djangoapps/ccx/", - "lms/djangoapps/certificates/", - "lms/djangoapps/commerce/" - ] - }, - "lms-2": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/course_api/", - "lms/djangoapps/course_blocks/", - "lms/djangoapps/course_goals/", - "lms/djangoapps/course_home_api/", - "lms/djangoapps/course_wiki/", - "lms/djangoapps/coursewarehistoryextended/", - "lms/djangoapps/debug/" - ] - }, - "lms-3": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/courseware/" - ] - }, - "lms-4": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/discussion/", - "lms/djangoapps/edxnotes/", - "lms/djangoapps/experiments/" - ] - }, - "lms-5": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/gating/", - "lms/djangoapps/grades/", - "lms/djangoapps/instructor/", - "lms/djangoapps/instructor_analytics/" - ] - }, - "lms-6": { - "settings": "lms.envs.test", - "paths": [ - "lms/djangoapps/instructor_task/", - "lms/djangoapps/learner_dashboard/", - "lms/djangoapps/learner_home/", - "lms/djangoapps/lms_initialization/", - "lms/djangoapps/lms_xblock/", - "lms/djangoapps/lti_provider/", - "lms/djangoapps/mailing/", - "lms/djangoapps/mobile_api/", - "lms/djangoapps/monitoring/", - "lms/djangoapps/ora_staff_grader/", - "lms/djangoapps/program_enrollments/", - "lms/djangoapps/rss_proxy/", - "lms/djangoapps/static_template_view/", - "lms/djangoapps/staticbook/", - "lms/djangoapps/support/", - "lms/djangoapps/survey/", - "lms/djangoapps/teams/", - "lms/djangoapps/tests/", - "lms/djangoapps/user_tours/", - "lms/djangoapps/verify_student/", - "lms/djangoapps/mfe_config_api/", - "lms/envs/", - "lms/lib/", - "lms/tests.py" - ] - }, - "openedx-1-with-lms": { - "settings": "lms.envs.test", - "paths": [ - "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", - "openedx/core/djangoapps/agreements/", - "openedx/core/djangoapps/api_admin/", - "openedx/core/djangoapps/auth_exchange/", - "openedx/core/djangoapps/bookmarks/", - "openedx/core/djangoapps/cache_toolbox/", - "openedx/core/djangoapps/catalog/", - "openedx/core/djangoapps/ccxcon/", - "openedx/core/djangoapps/commerce/", - "openedx/core/djangoapps/common_initialization/", - "openedx/core/djangoapps/common_views/", - "openedx/core/djangoapps/config_model_utils/", - "openedx/core/djangoapps/content/", - "openedx/core/djangoapps/content_libraries/", - "openedx/core/djangoapps/contentserver/", - "openedx/core/djangoapps/cookie_metadata/", - "openedx/core/djangoapps/course_apps/", - "openedx/core/djangoapps/course_date_signals/", - "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/courseware_api/", - "openedx/core/djangoapps/crawlers/", - "openedx/core/djangoapps/credentials/", - "openedx/core/djangoapps/credit/", - "openedx/core/djangoapps/course_live/", - "openedx/core/djangoapps/dark_lang/", - "openedx/core/djangoapps/debug/", - "openedx/core/djangoapps/discussions/", - "openedx/core/djangoapps/django_comment_common/", - "openedx/core/djangoapps/embargo/", - "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-2-with-lms": { - "settings": "lms.envs.test", - "paths": [ - "openedx/core/djangoapps/geoinfo/", - "openedx/core/djangoapps/header_control/", - "openedx/core/djangoapps/heartbeat/", - "openedx/core/djangoapps/lang_pref/", - "openedx/core/djangoapps/models/", - "openedx/core/djangoapps/monkey_patch/", - "openedx/core/djangoapps/notifications/", - "openedx/core/djangoapps/oauth_dispatch/", - "openedx/core/djangoapps/olx_rest_api/", - "openedx/core/djangoapps/password_policy/", - "openedx/core/djangoapps/plugin_api/", - "openedx/core/djangoapps/plugins/", - "openedx/core/djangoapps/profile_images/", - "openedx/core/djangoapps/programs/", - "openedx/core/djangoapps/safe_sessions/", - "openedx/core/djangoapps/schedules/", - "openedx/core/djangoapps/service_status/", - "openedx/core/djangoapps/session_inactivity_timeout/", - "openedx/core/djangoapps/signals/", - "openedx/core/djangoapps/site_configuration/", - "openedx/core/djangoapps/system_wide_roles/", - "openedx/core/djangoapps/theming/", - "openedx/core/djangoapps/user_api/", - "openedx/core/djangoapps/user_authn/", - "openedx/core/djangoapps/util/", - "openedx/core/djangoapps/verified_track_content/", - "openedx/core/djangoapps/video_config/", - "openedx/core/djangoapps/video_pipeline/", - "openedx/core/djangoapps/waffle_utils/", - "openedx/core/djangoapps/xblock/", - "openedx/core/djangoapps/xmodule_django/", - "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/djangolib/", - "openedx/core/lib/", - "openedx/core/tests/", - "openedx/features/", - "openedx/testing/", - "openedx/tests/" - ] - }, - "openedx-1-with-cms": { - "settings": "cms.envs.test", - "paths": [ - "openedx/core/djangoapps/ace_common/", - "openedx/core/djangoapps/cors_csrf/", - "openedx/core/djangoapps/agreements/", - "openedx/core/djangoapps/api_admin/", - "openedx/core/djangoapps/auth_exchange/", - "openedx/core/djangoapps/bookmarks/", - "openedx/core/djangoapps/cache_toolbox/", - "openedx/core/djangoapps/catalog/", - "openedx/core/djangoapps/ccxcon/", - "openedx/core/djangoapps/commerce/", - "openedx/core/djangoapps/common_initialization/", - "openedx/core/djangoapps/common_views/", - "openedx/core/djangoapps/config_model_utils/", - "openedx/core/djangoapps/content/", - "openedx/core/djangoapps/content_libraries/", - "openedx/core/djangoapps/content_staging/", - "openedx/core/djangoapps/contentserver/", - "openedx/core/djangoapps/cookie_metadata/", - "openedx/core/djangoapps/course_apps/", - "openedx/core/djangoapps/course_date_signals/", - "openedx/core/djangoapps/course_groups/", - "openedx/core/djangoapps/courseware_api/", - "openedx/core/djangoapps/crawlers/", - "openedx/core/djangoapps/credentials/", - "openedx/core/djangoapps/credit/", - "openedx/core/djangoapps/dark_lang/", - "openedx/core/djangoapps/debug/", - "openedx/core/djangoapps/discussions/", - "openedx/core/djangoapps/django_comment_common/", - "openedx/core/djangoapps/embargo/", - "openedx/core/djangoapps/enrollments/", - "openedx/core/djangoapps/external_user_ids/" - ] - }, - "openedx-2-with-cms": { - "settings": "cms.envs.test", - "paths": [ - "openedx/core/djangoapps/content_tagging/", - "openedx/core/djangoapps/geoinfo/", - "openedx/core/djangoapps/header_control/", - "openedx/core/djangoapps/heartbeat/", - "openedx/core/djangoapps/lang_pref/", - "openedx/core/djangoapps/models/", - "openedx/core/djangoapps/monkey_patch/", - "openedx/core/djangoapps/oauth_dispatch/", - "openedx/core/djangoapps/olx_rest_api/", - "openedx/core/djangoapps/password_policy/", - "openedx/core/djangoapps/plugin_api/", - "openedx/core/djangoapps/plugins/", - "openedx/core/djangoapps/profile_images/", - "openedx/core/djangoapps/programs/", - "openedx/core/djangoapps/safe_sessions/", - "openedx/core/djangoapps/schedules/", - "openedx/core/djangoapps/service_status/", - "openedx/core/djangoapps/session_inactivity_timeout/", - "openedx/core/djangoapps/signals/", - "openedx/core/djangoapps/site_configuration/", - "openedx/core/djangoapps/system_wide_roles/", - "openedx/core/djangoapps/theming/", - "openedx/core/djangoapps/user_api/", - "openedx/core/djangoapps/user_authn/", - "openedx/core/djangoapps/util/", - "openedx/core/djangoapps/verified_track_content/", - "openedx/core/djangoapps/video_config/", - "openedx/core/djangoapps/video_pipeline/", - "openedx/core/djangoapps/waffle_utils/", - "openedx/core/djangoapps/xblock/", - "openedx/core/djangoapps/xmodule_django/", - "openedx/core/djangoapps/zendesk_proxy/", - "openedx/core/lib/", - "openedx/tests/" - ] - }, - "cms-1": { - "settings": "cms.envs.test", - "paths": [ - "cms/djangoapps/api/", - "cms/djangoapps/cms_user_tasks/", - "cms/djangoapps/course_creators/", - "cms/djangoapps/export_course_metadata/", - "cms/djangoapps/import_from_modulestore/", - "cms/djangoapps/maintenance/", - "cms/djangoapps/models/", - "cms/djangoapps/pipeline_js/", - "cms/djangoapps/xblock_config/", - "cms/envs/", - "cms/lib/" - ] - }, - "cms-2": { - "settings": "cms.envs.test", - "paths": [ - "cms/djangoapps/contentstore/" - ] - }, - "common-with-lms": { - "settings": "lms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "common-with-cms": { - "settings": "cms.envs.test", - "paths": [ - "common/djangoapps/" - ] - }, - "xmodule-with-lms": { - "settings": "lms.envs.test", - "paths": [ - "xmodule/" - ] - }, - "xmodule-with-cms": { - "settings": "cms.envs.test", - "paths": [ - "xmodule/" - ] - } -} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index 9867ac72f273..000000000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,303 +0,0 @@ -name: unit-tests - -on: - pull_request: - push: - branches: - - master - -concurrency: - # We only need to be running tests for the latest commit on each PR - # (however, we fully test every commit on master, even as new ones land). - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/master' }} - -jobs: - run-tests: - name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }}) - runs-on: ${{ matrix.os-version }} - strategy: - matrix: - python-version: - - "3.11" - django-version: - - "pinned" - # When updating the shards, remember to make the same changes in - # .github/workflows/unit-tests-gh-hosted.yml - shard_name: - - "lms-1" - - "lms-2" - - "lms-3" - - "lms-4" - - "lms-5" - - "lms-6" - - "openedx-1-with-lms" - - "openedx-2-with-lms" - - "openedx-1-with-cms" - - "openedx-2-with-cms" - - "cms-1" - - "cms-2" - - "common-with-lms" - - "common-with-cms" - - "xmodule-with-lms" - - "xmodule-with-cms" - mongo-version: - - "7.0" - os-version: - - ubuntu-24.04 - - # It's useful to run some subset of the tests on the older version of Ubuntu - # so that we don't spend too many resources on this but can find major issues quickly - # while we're in a situation where we support two versions. This section may be commented - # out when not in use to easily add/drop future support for any given major dependency. - # - # We're testing the older version of Ubuntu and running the xmodule tests since those rely on many - # dependent complex libraries and will hopefully catch most issues quickly. - include: - - shard_name: "xmodule-with-cms" - python-version: "3.11" - django-version: "pinned" - mongo-version: "7.0" - os-version: "ubuntu-22.04" - - shard_name: "xmodule-with-lms" - python-version: "3.11" - django-version: "pinned" - mongo-version: "7.0" - os-version: "ubuntu-22.04" - - steps: - - name: checkout repo - uses: actions/checkout@v4 - - - name: install system requirements - run: | - sudo apt-get update && sudo apt-get install libmysqlclient-dev libxmlsec1-dev lynx - - # We pull this image a lot, and Dockerhub will rate limit us if we pull too often. - # This is an attempt to cache the image for better performance and to work around that. - # It will cache all pulled images, so if we add new images to this we'll need to update the key. - - name: Cache Docker images - uses: ScribeMD/docker-cache@0.5.0 - with: - key: docker-${{ runner.os }}-mongo-${{ matrix.mongo-version }} - - - name: Start MongoDB - uses: supercharge/mongodb-github-action@1.11.0 - with: - mongodb-version: ${{ matrix.mongo-version }} - - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: install requirements - run: | - make test-requirements - if [[ "${{ matrix.django-version }}" != "pinned" ]]; then - pip install "django~=${{ matrix.django-version }}.0" - pip check # fail if this test-reqs/Django combination is broken - fi - - - name: list installed package versions - run: | - pip freeze - - - name: set settings path - shell: bash - run: | - echo "settings_path=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} --output settings )" >> $GITHUB_ENV - - - name: get unit tests for shard - shell: bash - run: | - echo "unit_test_paths=$(python scripts/unit_test_shards_parser.py --shard-name=${{ matrix.shard_name }} )" >> $GITHUB_ENV - - - name: run tests - shell: bash - run: | - python -Wd -m pytest -p no:randomly --ds=${{ env.settings_path }} ${{ env.unit_test_paths }} --cov=. - - - name: rename warnings json file - if: success() - shell: bash - run: | - cd test_root/log - mv pytest_warnings.json pytest_warnings_${{ matrix.shard_name }}.json - - - name: save pytest warnings json file - if: success() - uses: actions/upload-artifact@v4 - with: - name: pytest-warnings-json-${{ matrix.shard_name }} - path: | - test_root/log/pytest_warnings*.json - overwrite: true - - - name: Renaming coverage data file - run: | - mv reports/.coverage reports/${{ matrix.shard_name }}.coverage - - - name: Upload coverage - uses: actions/upload-artifact@v4 - with: - name: coverage-${{ matrix.shard_name }} - path: reports/${{ matrix.shard_name }}.coverage - overwrite: true - - collect-and-verify: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - - name: Setup Python - uses: actions/setup-python@v5 - with: - python-version: 3.11 - - - name: install system requirements - run: | - sudo apt-get update && sudo apt-get install libxmlsec1-dev - - - name: install requirements - run: | - make test-requirements - - - name: collect tests from all modules - shell: bash - run: | - echo "root_cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test cms/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "root_lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test lms/ openedx/ common/djangoapps/ xmodule/ -q | head -n -2 | wc -l)" >> $GITHUB_ENV - - - name: get GHA unit test paths - shell: bash - run: | - echo "cms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --cms-only)" >> $GITHUB_ENV - echo "lms_unit_test_paths=$(python scripts/gha_unit_tests_collector.py --lms-only)" >> $GITHUB_ENV - - - name: collect tests from GHA unit test shards - shell: bash - run: | - echo "cms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=cms.envs.test ${{ env.cms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - echo "lms_unit_tests_count=$(pytest --disable-warnings --collect-only --ds=lms.envs.test ${{ env.lms_unit_test_paths }} -q | head -n -2 | wc -l)" >> $GITHUB_ENV - - - name: add unit tests count - shell: bash - run: | - echo "root_all_unit_tests_count=$((${{ env.root_cms_unit_tests_count }}+${{ env.root_lms_unit_tests_count }}))" >> $GITHUB_ENV - echo "shards_all_unit_tests_count=$((${{ env.cms_unit_tests_count }}+${{ env.lms_unit_tests_count }}))" >> $GITHUB_ENV - - - name: print unit tests count - shell: bash - run: | - echo CMS unit tests from root: ${{ env.root_cms_unit_tests_count }} - echo LMS unit tests from root: ${{ env.root_lms_unit_tests_count }} - echo CMS unit tests from shards: ${{ env.cms_unit_tests_count }} - echo LMS unit tests from shards: ${{ env.lms_unit_tests_count }} - echo All root unit tests count: ${{ env.root_all_unit_tests_count }} - echo All shards unit tests count: ${{ env.shards_all_unit_tests_count }} - - - name: fail the check - shell: bash - if: ${{ env.root_all_unit_tests_count != env.shards_all_unit_tests_count }} - run: | - echo "::error title='Unit test modules in unit-test-shards.json (unit-tests.yml workflow) are outdated'::unit tests running in unit-tests - workflow don't match the count for unit tests for entire edx-platform suite, please update the unit-test-shards.json under .github/workflows - to add any missing apps and match the count. for more details please take a look at scripts/gha-shards-readme.md" - exit 1 - - # This job aggregates test results. It's the required check for branch protection. - # https://github.com/marketplace/actions/alls-green#why - # https://github.com/orgs/community/discussions/33579 - success: - name: Unit tests successful - runs-on: ubuntu-24.04 - if: always() - needs: [run-tests] - steps: - - name: Decide whether the needed jobs succeeded or failed - # uses: re-actors/alls-green@v1.2.1 - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe - with: - jobs: ${{ toJSON(needs) }} - - compile-warnings-report: - runs-on: ubuntu-24.04 - needs: [run-tests] - steps: - - uses: actions/checkout@v4 - - name: collect pytest warnings files - uses: actions/download-artifact@v4 - with: - pattern: pytest-warnings-json-* - merge-multiple: true - path: test_root/log - - - name: display structure of downloaded files - run: ls -la test_root/log - - - name: compile warnings report - run: | - python openedx/core/process_warnings.py --dir-path test_root/log --html-path reports/pytest_warnings/warning_report_all.html - - - name: save warning report - if: success() - uses: actions/upload-artifact@v4 - with: - name: pytest-warning-report-html - path: | - reports/pytest_warnings/warning_report_all.html - overwrite: true - - merge-artifacts: - runs-on: ubuntu-24.04 - needs: [compile-warnings-report] - steps: - - name: Merge Pytest Warnings JSON Artifacts - uses: actions/upload-artifact/merge@v4 - with: - name: pytest-warnings-json - pattern: pytest-warnings-json-* - delete-merged: true - - - name: Merge Coverage Artifacts - uses: actions/upload-artifact/merge@v4 - with: - name: coverage - pattern: coverage-* - delete-merged: true - - # Combine and upload coverage reports. - coverage: - if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false)) - runs-on: ubuntu-24.04 - needs: [run-tests] - strategy: - matrix: - python-version: - - 3.11 - steps: - - name: Checkout repo - uses: actions/checkout@v4 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Download all artifacts - uses: actions/download-artifact@v4 - with: - pattern: coverage-* - merge-multiple: true - path: reports - - - name: Install Python dependencies - run: | - pip install -r requirements/edx/coverage.txt - - - name: Run coverage - run: | - coverage combine reports/* - coverage report - coverage xml - - uses: codecov/codecov-action@v4 diff --git a/.github/workflows/units-test-scripts-structures-pruning.yml b/.github/workflows/units-test-scripts-structures-pruning.yml deleted file mode 100644 index cbf9da8f5c9f..000000000000 --- a/.github/workflows/units-test-scripts-structures-pruning.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: units-test-scripts-common - -on: - pull_request: - push: - branches: - - master - -jobs: - test: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: - - "3.12" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/structures_pruning/requirements/testing.txt - - - name: Run pytest - run: | - pytest scripts/structures_pruning diff --git a/.github/workflows/units-test-scripts-user-retirement.yml b/.github/workflows/units-test-scripts-user-retirement.yml deleted file mode 100644 index f1b2b2c539f6..000000000000 --- a/.github/workflows/units-test-scripts-user-retirement.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: units-test-scripts-user-retirement - -on: - pull_request: - push: - branches: - - master - -jobs: - test: - runs-on: ubuntu-latest - - strategy: - matrix: - python-version: - - "3.12" - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r scripts/user_retirement/requirements/testing.txt - - - name: Run pytest - run: | - pytest scripts/user_retirement diff --git a/lms/djangoapps/instructor_task/migrations/0007_alter_instructortask_task_output.py b/lms/djangoapps/instructor_task/migrations/0007_alter_instructortask_task_output.py new file mode 100644 index 000000000000..175ebc3a44f2 --- /dev/null +++ b/lms/djangoapps/instructor_task/migrations/0007_alter_instructortask_task_output.py @@ -0,0 +1,18 @@ +# Generated by Django 4.2.20 on 2025-10-17 15:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('instructor_task', '0006_alter_historicalinstructortaskschedule_options'), + ] + + operations = [ + migrations.AlterField( + model_name='instructortask', + name='task_output', + field=models.TextField(null=True), + ), + ] diff --git a/lms/djangoapps/instructor_task/models.py b/lms/djangoapps/instructor_task/models.py index 913e4775a946..3f28eb5c6306 100644 --- a/lms/djangoapps/instructor_task/models.py +++ b/lms/djangoapps/instructor_task/models.py @@ -76,7 +76,7 @@ class Meta: task_input = models.TextField() task_id = models.CharField(max_length=255, db_index=True) # max_length from celery_taskmeta task_state = models.CharField(max_length=50, null=True, db_index=True) # max_length from celery_taskmeta - task_output = models.CharField(max_length=1024, null=True) + task_output = models.TextField(null=True) requester = models.ForeignKey(User, db_index=True, on_delete=models.CASCADE) created = models.DateTimeField(auto_now_add=True, null=True) updated = models.DateTimeField(auto_now=True) diff --git a/lms/djangoapps/instructor_task/subtasks.py b/lms/djangoapps/instructor_task/subtasks.py index db0d2380ea78..3fc478e4e1c4 100644 --- a/lms/djangoapps/instructor_task/subtasks.py +++ b/lms/djangoapps/instructor_task/subtasks.py @@ -17,6 +17,7 @@ from common.djangoapps.util.db import outer_atomic +from openedx_wikilearn_features.wikimedia_general.djangoapps_patches.instructor_task.utils import update_task_dict from .exceptions import DuplicateTaskException from .models import PROGRESS, QUEUING, InstructorTask @@ -249,7 +250,7 @@ def initialize_subtask_info(entry, action_name, total_num, subtask_id_list): 'duration_ms': int(0), 'start_time': time() } - entry.task_output = InstructorTask.create_output_for_success(task_progress) + entry.task_output = InstructorTask.create_output_for_success(update_task_dict(task_progress)) entry.task_state = PROGRESS # Write out the subtasks information. @@ -263,7 +264,7 @@ def initialize_subtask_info(entry, action_name, total_num, subtask_id_list): 'failed': 0, 'status': subtask_status } - entry.subtasks = json.dumps(subtask_dict) + entry.subtasks = json.dumps(update_task_dict(subtask_dict)) # and save the entry immediately, before any subtasks actually start work: entry.save_now() @@ -549,7 +550,7 @@ def _update_subtask_status(entry_id, current_task_id, new_subtask_status): # retry. new_state = new_subtask_status.state if new_subtask_status is not None and new_state in READY_STATES: - for statname in ['attempted', 'succeeded', 'failed', 'skipped']: + for statname in ['attempted', 'succeeded', 'failed', 'skipped', 'skip_details', 'failure_details']: task_progress[statname] += getattr(new_subtask_status, statname) # Figure out if we're actually done (i.e. this is the last task to complete). diff --git a/lms/djangoapps/instructor_task/views.py b/lms/djangoapps/instructor_task/views.py index 51e2812577fe..f8c1434153ff 100644 --- a/lms/djangoapps/instructor_task/views.py +++ b/lms/djangoapps/instructor_task/views.py @@ -9,6 +9,7 @@ from lms.djangoapps.instructor_task.api_helper import get_status_from_instructor_task, get_updated_instructor_task from lms.djangoapps.instructor_task.models import PROGRESS +from openedx_wikilearn_features.wikimedia_general.djangoapps_patches.instructor_task.utils import get_detailed_message log = logging.getLogger(__name__) @@ -228,5 +229,5 @@ def get_task_completion_info(instructor_task): # lint-amnesty, pylint: disable= total=num_total, skipped=num_skipped, student=student - ) + ) + "\n" + get_detailed_message(task_output) return (succeeded, message)