Merge pull request openedx#35713 from openedx/feanil/ubuntu-24.04

feanil/ubuntu 24.04

(cherry picked from commit 642dae6)

Author: feanil

.github/workflows/check-consistent-dependencies.yml

@@ -15,7 +15,7 @@ defaults:
 jobs:
   check-requirements:
     name: Compile requirements
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
 
     steps:
       # Only run remaining steps if there are changes to requirements/**

.github/workflows/ci-static-analysis.yml

@@ -10,7 +10,7 @@ jobs:
       matrix:
         python-version:
           - "3.11"
-        os: ["ubuntu-22.04"]
+        os: ["ubuntu-24.04"]
 
     steps:
       - uses: actions/checkout@v4

.github/workflows/migrations-check.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python-version:
           - "3.11"
         # 'pinned' is used to install the latest patch version of Django
@@ -126,7 +126,7 @@ jobs:
     if: always()
     needs:
       - check_migrations
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         # uses: re-actors/[email protected]

.github/workflows/pylint-checks.yml

@@ -8,7 +8,7 @@ on:
 
 jobs:
   run-pylint:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
@@ -75,7 +75,7 @@ jobs:
     if: always()
     needs:
       - run-pylint
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - name: Decide whether the needed jobs succeeded or failed
         # uses: re-actors/[email protected]

.github/workflows/quality-checks.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python-version:
           - "3.11"
         node-version: [20]

.github/workflows/static-assets-check.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        os: [ubuntu-22.04]
+        os: [ubuntu-24.04]
         python-version:
           - "3.11"
         node-version: [18, 20]

.github/workflows/unit-tests.yml

@@ -15,7 +15,7 @@ concurrency:
 jobs:
   run-tests:
     name: ${{ matrix.shard_name }}(py=${{ matrix.python-version }},dj=${{ matrix.django-version }},mongo=${{ matrix.mongo-version }})
-    runs-on: ubuntu-22.04
+    runs-on: ${{ matrix.os-version }}
     strategy:
       matrix:
         python-version:
@@ -43,22 +43,27 @@ jobs:
           - "xmodule-with-cms"
         mongo-version:
           - "7.0"
+        os-version:
+          - ubuntu-24.04
 
-        # We only need to test older versions of Mongo with modules that directly interface with Mongo (that is: xmodule.modulestore)
-        # This code is left here as an example for future refernce in case we need to reduce the number of shards we're
-        # testing but still have good confidence with older versions of mongo.  We use Mongo 4.4 in the example.
+        # It's useful to run some subset of the tests on the older version of Ubuntu
+        # so that we don't spend too many resources on this but can find major issues quickly
+        # while we're in a situation where we support two versions.  This section may be commented
+        # out when not in use to easily add/drop future support for any given major dependency.
         #
-        # exclude:
-        #   - mongo-version: "4.4"
-        # include:
-        #   - shard_name: "xmodule-with-cms"
-        #     python-version: "3.11"
-        #     django-version: "pinned"
-        #     mongo-version: "4.4"
-        #   - shard_name: "xmodule-with-lms"
-        #     python-version: "3.11"
-        #     django-version: "pinned"
-        #     mongo-version: "4.4"
+        # We're testing the older version of Ubuntu and running the xmodule tests since those rely on many
+        # dependent complex libraries and will hopefully catch most issues quickly.
+        include:
+          - shard_name: "xmodule-with-cms"
+            python-version: "3.11"
+            django-version: "pinned"
+            mongo-version: "7.0"
+            os-version: "ubuntu-22.04"
+          - shard_name: "xmodule-with-lms"
+            python-version: "3.11"
+            django-version: "pinned"
+            mongo-version: "7.0"
+            os-version: "ubuntu-22.04"
 
     steps:
       - name: checkout repo
@@ -90,19 +95,10 @@ jobs:
           activate = 1
           EOF
 
-      - name: install mongo version
-        run: |
-          if [[ "${{ matrix.mongo-version }}" != "4.4" ]]; then
-            wget -qO - https://www.mongodb.org/static/pgp/server-${{ matrix.mongo-version }}.asc | sudo apt-key add -
-            echo "deb https://repo.mongodb.org/apt/ubuntu focal/mongodb-org/${{ matrix.mongo-version }} multiverse" | sudo tee /etc/apt/sources.list.d/mongodb-org-${{ matrix.mongo-version }}.list
-            sudo apt-get update && sudo apt-get install -y mongodb-org="${{ matrix.mongo-version }}.*"
-          fi
-
-      - name: start mongod server for tests
-        run: |
-          sudo mkdir -p /data/db
-          sudo chmod -R a+rw /data/db
-          mongod &
+      - name: Start MongoDB
+        uses: supercharge/[email protected]
+        with:
+          mongodb-version: ${{ matrix.mongo-version }}
 
       - name: Setup Python
         uses: actions/setup-python@v5
@@ -164,7 +160,7 @@ jobs:
           overwrite: true
 
   collect-and-verify:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python
@@ -229,7 +225,7 @@ jobs:
   # https://github.com/orgs/community/discussions/33579
   success:
     name: Unit tests successful
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     if: always()
     needs: [run-tests]
     steps:
@@ -240,7 +236,7 @@ jobs:
           jobs: ${{ toJSON(needs) }}
 
   compile-warnings-report:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [run-tests]
     steps:
       - uses: actions/checkout@v4
@@ -268,7 +264,7 @@ jobs:
           overwrite: true
 
   merge-artifacts:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [compile-warnings-report]
     steps:
       - name: Merge Pytest Warnings JSON Artifacts
@@ -288,7 +284,7 @@ jobs:
   # Combine and upload coverage reports.
   coverage:
     if: (github.repository == 'edx/edx-platform-private') || (github.repository == 'openedx/edx-platform' && (startsWith(github.base_ref, 'open-release') == false))
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     needs: [run-tests]
     strategy:
       matrix:

requirements/constraints.txt

@@ -105,13 +105,6 @@ event-tracking==3.0.0
 # https://github.com/openedx/edx-platform/issues/31616
 libsass==0.10.0
 
-# Date: 2024-04-30
-# lxml>=5.0 introduced breaking changes related to system dependencies
-# lxml==5.2.1 introduced new extra so we'll nee to rename lxml --> lxml[html-clean]
-# This constraint can be removed once we upgrade to Python 3.11
-# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35272
-lxml<5.0
-
 # Date: 2018-12-14
 # markdown>=3.4.0 has failures due to internal refactorings which causes the tests to fail
 # pinning the version untill the issue gets resolved in the package itself
@@ -192,8 +185,3 @@ social-auth-app-django<=5.4.1
 # # Date: 2024-10-14
 # # The edx-enterprise is currently using edx-rest-api-client==5.7.1, which needs to be updated first.
 # edx-rest-api-client==5.7.1
-
-# Date: 2024-04-24
-# xmlsec==1.3.14 breaking tests or all builds, can be removed once a fix is available
-# Issue for unpinning: https://github.com/openedx/edx-platform/issues/35264
-xmlsec<1.3.14

requirements/edx-sandbox/base.in

@@ -2,7 +2,7 @@
 
 chem                                # A helper library for chemistry calculations
 cryptography                        # Implementations of assorted cryptography algorithms
-lxml                                # XML parser
+lxml[html_clean]                    # XML parser
 matplotlib                          # 2D plotting library
 networkx                            # Utilities for creating, manipulating, and studying network graphs
 nltk                                # Natural language processing; used by the chem package

requirements/edx-sandbox/base.txt

@@ -26,11 +26,13 @@ joblib==1.4.2
     # via nltk
 kiwisolver==1.4.7
     # via matplotlib
-lxml==4.9.4
+lxml[html-clean,html_clean]==5.3.0
     # via
-    #   -c requirements/edx-sandbox/../constraints.txt
     #   -r requirements/edx-sandbox/base.in
+    #   lxml-html-clean
     #   openedx-calc
+lxml-html-clean==0.3.1
+    # via lxml
 markupsafe==3.0.2
     # via
     #   chem

requirements/edx/base.txt

@@ -715,19 +715,21 @@ loremipsum==1.0.5
     # via ora2
 lti-consumer-xblock==9.11.3
     # via -r requirements/edx/kernel.in
-lxml==4.9.4
+lxml[html-clean,html_clean]==5.3.0
     # via
-    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/kernel.in
     #   edx-i18n-tools
     #   edxval
     #   lti-consumer-xblock
+    #   lxml-html-clean
     #   olxcleaner
     #   openedx-calc
     #   ora2
     #   python3-saml
     #   xblock
     #   xmlsec
+lxml-html-clean==0.3.1
+    # via lxml
 mailsnake==1.6.4
     # via -r requirements/edx/bundled.in
 mako==1.3.6
@@ -1301,10 +1303,8 @@ xblock-utils==4.0.0
     # via
     #   edx-sga
     #   xblock-poll
-xmlsec==1.3.13
-    # via
-    #   -c requirements/edx/../constraints.txt
-    #   python3-saml
+xmlsec==1.3.14
+    # via python3-saml
 xss-utils==0.6.0
     # via -r requirements/edx/kernel.in
 yarl==1.16.0

requirements/edx/development.txt

@@ -1193,21 +1193,26 @@ lti-consumer-xblock==9.11.3
     # via
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
-lxml==4.9.4
+lxml[html-clean]==5.3.0
     # via
-    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
     #   edx-i18n-tools
     #   edxval
     #   lti-consumer-xblock
+    #   lxml-html-clean
     #   olxcleaner
     #   openedx-calc
     #   ora2
     #   pyquery
     #   python3-saml
     #   xblock
     #   xmlsec
+lxml-html-clean==0.3.1
+    # via
+    #   -r requirements/edx/doc.txt
+    #   -r requirements/edx/testing.txt
+    #   lxml
 mailsnake==1.6.4
     # via
     #   -r requirements/edx/doc.txt
@@ -2293,9 +2298,8 @@ xblock-utils==4.0.0
     #   -r requirements/edx/testing.txt
     #   edx-sga
     #   xblock-poll
-xmlsec==1.3.13
+xmlsec==1.3.14
     # via
-    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/doc.txt
     #   -r requirements/edx/testing.txt
     #   python3-saml

requirements/edx/doc.txt

@@ -863,19 +863,23 @@ loremipsum==1.0.5
     #   ora2
 lti-consumer-xblock==9.11.3
     # via -r requirements/edx/base.txt
-lxml==4.9.4
+lxml[html-clean]==5.3.0
     # via
-    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/base.txt
     #   edx-i18n-tools
     #   edxval
     #   lti-consumer-xblock
+    #   lxml-html-clean
     #   olxcleaner
     #   openedx-calc
     #   ora2
     #   python3-saml
     #   xblock
     #   xmlsec
+lxml-html-clean==0.3.1
+    # via
+    #   -r requirements/edx/base.txt
+    #   lxml
 mailsnake==1.6.4
     # via -r requirements/edx/base.txt
 mako==1.3.6
@@ -1607,9 +1611,8 @@ xblock-utils==4.0.0
     #   -r requirements/edx/base.txt
     #   edx-sga
     #   xblock-poll
-xmlsec==1.3.13
+xmlsec==1.3.14
     # via
-    #   -c requirements/edx/../constraints.txt
     #   -r requirements/edx/base.txt
     #   python3-saml
 xss-utils==0.6.0

requirements/edx/kernel.in

@@ -103,7 +103,7 @@ importlib_metadata                  # Used to access entry_points in i18n_api pl
 jsonfield                           # Django model field for validated JSON; used in several apps
 laboratory                          # Library for testing that code refactors/infrastructure changes produce identical results
 importlib_metadata                  # Used to access entry_points in i18n_api plugin
-lxml                                # XML parser
+lxml[html_clean]                    # XML parser
 lti-consumer-xblock>=7.3.0
 mako                                # Primary template language used for server-side page rendering
 Markdown                            # Convert text markup to HTML; used in capa problems, forums, and course wikis