Upgrade to Ubuntu 24.04 for our application (#12002)

- Use Ubuntu 24.04 for our Docker images
- The application runs with Python 3.12 now
- Run tests with Python 3.12
- Update APT package names
- Install `uv` in the default directory `~/.local/bin`
- Add `uv` to the PATH
- Create a virtualenv inside the Docker image for our application.
`--system` option does not work anymore with the newer `pip` versions.
It fails saying that we should install Python dependencies using system
package.
- Upgrade Celery dependency to be compatible with Python 3.12
- The `pickle` error we used to see with Celery does not happen anymore
with the newer version + py3.12
- Remove `tzdata` because it's installed automatically now

## ToDo

- [x] tests pass locally
- [x] build a failed build that raises `ConfigError` without breaking
- [x] build a success build
- [x] tests pass in CircleCI
- [x] update -ops to use Ubuntu 24.04
- [x] build an AMI
- [x] deploy _one_ instance and run a build there
- [ ] do the same upgrade on .com

## Notes

- All the Python commands inside the Docker container needs to be
prefixed with `uv run` now. Example: `uv run tox -e 312` or `uv run
django-admin` and similars.

Related readthedocs/readthedocs-ops#1496
Closes #12000

Author: humitos

.circleci/config.yml

@@ -7,7 +7,7 @@ orbs:
 jobs:
   tests:
     docker:
-      - image: 'cimg/python:3.10'
+      - image: 'cimg/python:3.12'
         environment:
           # Don't skip search tests.
           TOX_POSARGS: ''
@@ -27,12 +27,12 @@ jobs:
       - run: sudo apt update
       - run: sudo apt install -y rclone
       - run: pip install --user tox
-      - run: tox -e py310
+      - run: tox -e py312
       - codecov/upload
 
   tests-ext-theme:
     docker:
-      - image: 'cimg/python:3.10'
+      - image: 'cimg/python:3.12'
         environment:
           # Don't skip search tests.
           TOX_POSARGS: ''
@@ -55,7 +55,7 @@ jobs:
 
   tests-embedapi:
     docker:
-      - image: 'cimg/python:3.10'
+      - image: 'cimg/python:3.12'
     steps:
       - checkout
       - run: git submodule sync
@@ -66,7 +66,7 @@ jobs:
 
   checks:
     docker:
-      - image: 'cimg/python:3.10'
+      - image: 'cimg/python:3.12'
     steps:
       - checkout
       - run: git submodule sync

common

@@ -1,4 +1,4 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ARG GITHUB_USER
 ARG GITHUB_TOKEN
@@ -26,8 +26,8 @@ RUN apt-get -y install \
         libmysqlclient-dev \
         libfreetype6 \
         libjpeg-dev \
-        sqlite \
-        netcat \
+        sqlite3 \
+        netcat-traditional \
         telnet \
         lsb-release \
         npm \
@@ -57,25 +57,30 @@ RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
     locale-gen
 
 # Install ``uv`` package manager
-RUN curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="/usr/bin/" sh
+RUN curl -LsSf https://astral.sh/uv/install.sh | sh
+ENV PATH="/root/.local/bin/:$PATH"
+RUN uv venv
 
 # Ensure that ``python`` is in the PATH so that ``./manage.py`` works
 RUN ln -s /usr/bin/python3 /usr/bin/python
 
 WORKDIR /tmp
 
 COPY requirements/docker.txt docker.txt
-RUN uv pip sync --system docker.txt
+RUN uv pip sync docker.txt
 
 # Install readthedocs-ext only if GITHUB_TOKEN is provided
 WORKDIR /usr/src/app/checkouts/
 RUN if [ -n "$GITHUB_TOKEN" ] ; \
         then \
         git clone --depth 1 https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/readthedocs/readthedocs-ext ; \
-        uv pip install --system --no-cache-dir -e readthedocs-ext ; \
+        uv pip install --no-cache-dir -e readthedocs-ext ; \
         fi
 
 RUN git clone --depth 1 https://github.com/readthedocs/ext-theme ; \
-    uv pip install --system --no-cache-dir -e ext-theme
+    uv pip install --no-cache-dir -e ext-theme
+
+# Required by polymorphic
+RUN uv pip install --no-cache-dir -U setuptools
 
 WORKDIR /usr/src/app/checkouts/readthedocs.org

dockerfiles/Dockerfile

@@ -180,7 +180,7 @@ save some work while typing docker compose commands. This section explains these
 ``inv docker.test``
     Runs all the test suites inside the container.
 
-    * ``--arguments`` will pass arguments to Tox command (e.g. ``--arguments "-e py310 -- -k test_api"``)
+    * ``--arguments`` will pass arguments to Tox command (e.g. ``--arguments "-e py312 -- -k test_api"``)
 
 ``inv docker.pull``
     Downloads and tags all the Docker images required for builders.

docs/dev/install.rst

@@ -46,22 +46,22 @@ To target a specific environment:
 
 .. prompt:: bash
 
-   tox -e py310
+   tox -e py312
 
 To run a subset of tests:
 
 .. prompt:: bash
 
-   tox -e py310 -- -k test_celery
+   tox -e py312 -- -k test_celery
 
 The ``tox`` configuration has the following environments configured. You can
 target a single environment to limit the test suite:
 
-py310
+py312
     Run our test suite using Python 3.10
 
-py310-debug
-    Same as ``py310``, but there are some useful debugging tools available in the environment.
+py312-debug
+    Same as ``py312``, but there are some useful debugging tools available in the environment.
 
 lint
     Run code linting using `Prospector`_. This currently runs `pylint`_,

docs/dev/tests.rst

@@ -24,22 +24,22 @@ async-timeout==5.0.1
     # via
     #   -r requirements/pip.txt
     #   redis
-billiard==3.6.4.0
+billiard==4.2.1
     # via
     #   -r requirements/pip.txt
     #   celery
-boto3==1.36.21
+boto3==1.36.22
     # via
     #   -r requirements/pip.txt
     #   django-storages
-botocore==1.36.21
+botocore==1.36.22
     # via
     #   -r requirements/pip.txt
     #   boto3
     #   s3transfer
 bumpver==2024.1130
     # via -r requirements/pip.txt
-celery==5.2.7
+celery==5.4.0
     # via
     #   -r requirements/pip.txt
     #   django-celery-beat
@@ -339,16 +339,15 @@ python-dateutil==2.9.0.post0
     # via
     #   -r requirements/pip.txt
     #   botocore
+    #   celery
     #   elasticsearch-dsl
     #   python-crontab
 python3-saml==1.16.0
     # via
     #   -r requirements/pip.txt
     #   django-allauth
 pytz==2025.1
-    # via
-    #   -r requirements/pip.txt
-    #   celery
+    # via -r requirements/pip.txt
 pyyaml==6.0.2
     # via -r requirements/pip.txt
 qrcode==8.0
@@ -382,9 +381,9 @@ s3transfer==0.11.2
     # via
     #   -r requirements/pip.txt
     #   boto3
-selectolax==0.3.27
+selectolax==0.3.28
     # via -r requirements/pip.txt
-sentry-sdk==2.21.0
+sentry-sdk==2.22.0
     # via structlog-sentry
 six==1.17.0
     # via
@@ -438,6 +437,7 @@ typing-extensions==4.12.2
 tzdata==2025.1
     # via
     #   -r requirements/pip.txt
+    #   celery
     #   django-celery-beat
     #   kombu
 ua-parser==1.0.1

requirements/deploy.txt

@@ -26,15 +26,15 @@ async-timeout==5.0.1
     #   redis
 attrs==25.1.0
     # via wmctrl
-billiard==3.6.4.0
+billiard==4.2.1
     # via
     #   -r requirements/pip.txt
     #   celery
-boto3==1.36.21
+boto3==1.36.22
     # via
     #   -r requirements/pip.txt
     #   django-storages
-botocore==1.36.21
+botocore==1.36.22
     # via
     #   -r requirements/pip.txt
     #   boto3
@@ -43,7 +43,7 @@ bumpver==2024.1130
     # via -r requirements/pip.txt
 cachetools==5.5.1
     # via tox
-celery==5.2.7
+celery==5.4.0
     # via
     #   -r requirements/pip.txt
     #   django-celery-beat
@@ -369,16 +369,15 @@ python-dateutil==2.9.0.post0
     # via
     #   -r requirements/pip.txt
     #   botocore
+    #   celery
     #   elasticsearch-dsl
     #   python-crontab
 python3-saml==1.16.0
     # via
     #   -r requirements/pip.txt
     #   django-allauth
 pytz==2025.1
-    # via
-    #   -r requirements/pip.txt
-    #   celery
+    # via -r requirements/pip.txt
 pyyaml==6.0.2
     # via -r requirements/pip.txt
 qrcode==8.0
@@ -414,7 +413,7 @@ s3transfer==0.11.2
     # via
     #   -r requirements/pip.txt
     #   boto3
-selectolax==0.3.27
+selectolax==0.3.28
     # via -r requirements/pip.txt
 six==1.17.0
     # via
@@ -472,6 +471,7 @@ typing-extensions==4.12.2
 tzdata==2025.1
     # via
     #   -r requirements/pip.txt
+    #   celery
     #   django-celery-beat
     #   kombu
 ua-parser==1.0.1

requirements/docker.txt

@@ -175,7 +175,7 @@ uvicorn==0.34.0
     # via sphinx-autobuild
 watchfiles==1.0.4
     # via sphinx-autobuild
-websockets==14.2
+websockets==15.0
     # via sphinx-autobuild
 
 # The following packages are considered to be unsafe in a requirements file:

requirements/docs.txt

@@ -50,41 +50,9 @@ dnspython
 # Used for Redis cache Django backend (`django.core.cache.backends.redis.RedisCache`)
 redis
 
-# NOTE: we have to pin to this version because Celery>=5.3.x detects our ``ConfigError`` exception is not picklable
-# and creates a ``UnpickleableExceptionWrapper`` which is not what our code expects.
-# This may be due to a Python bug which can't pickle/unpickle the exception properly.
-# We could consider either:
-#  - handle ``UnpickleableExceptionWrapper`` in our code and inspect what's the inner exception
-#  - find a way to disable pickling the exceptions
-#  - make our custom exception picklable
-#
-# References:
-#  - https://github.com/celery/celery/pull/8149
-#  - https://github.com/celery/celery/blob/2b4b500ca1212016824a5fa2996cfb752f0763a7/celery/utils/serialization.py#L154
-#  - https://github.com/python/cpython/issues/76877
-#
-# Reproducible example:
-# >>> import pickle
-# >>> class MyException(Exception):
-# ...   def __init__(self, a, b):
-# ...     self.a = a
-# ...     self.b = b
-# ...     super().__init__(a)
-# ...
-# >>> pickle.loads(pickle.dumps(MyException("a", "b")))
-# Traceback (most recent call last):
-#   File "<stdin>", line 1, in <module>
-# TypeError: MyException.__init__() missing 1 required positional argument: 'b'
-# >>>
-celery==5.2.7
+celery
 django-celery-beat
 
-# Docker images need a dependency explicity marked for tzdata in order to have
-# necessary timezone data.
-# https://github.com/readthedocs/readthedocs.org/issues/10453
-# TODO: remove this dependency once we upgrade Celery. It should auto-install it.
-tzdata
-
 django-allauth[socialaccount,saml,mfa]
 
 requests-oauthlib

requirements/pip.in

@@ -15,17 +15,17 @@ asgiref==3.8.1
     #   django-cors-headers
 async-timeout==5.0.1
     # via redis
-billiard==3.6.4.0
+billiard==4.2.1
     # via celery
-boto3==1.36.21
+boto3==1.36.22
     # via django-storages
-botocore==1.36.21
+botocore==1.36.22
     # via
     #   boto3
     #   s3transfer
 bumpver==2024.1130
     # via -r requirements/pip.in
-celery==5.2.7
+celery==5.4.0
     # via
     #   -r requirements/pip.in
     #   django-celery-beat
@@ -241,14 +241,13 @@ python-dateutil==2.9.0.post0
     # via
     #   -r requirements/pip.in
     #   botocore
+    #   celery
     #   elasticsearch-dsl
     #   python-crontab
 python3-saml==1.16.0
     # via django-allauth
 pytz==2025.1
-    # via
-    #   -r requirements/pip.in
-    #   celery
+    # via -r requirements/pip.in
 pyyaml==6.0.2
     # via -r requirements/pip.in
 qrcode==8.0
@@ -278,7 +277,7 @@ rest-framework-generic-relations==2.2.0
     # via -r requirements/pip.in
 s3transfer==0.11.2
     # via boto3
-selectolax==0.3.27
+selectolax==0.3.28
     # via -r requirements/pip.in
 six==1.17.0
     # via
@@ -315,6 +314,7 @@ typing-extensions==4.12.2
 tzdata==2025.1
     # via
     #   -r requirements/pip.in
+    #   celery
     #   django-celery-beat
     #   kombu
 ua-parser==1.0.1