diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py
index 65d07f9d1f7..2adf047b5c2 100644
--- a/readthedocs/doc_builder/environments.py
+++ b/readthedocs/doc_builder/environments.py
@@ -493,6 +493,7 @@ def __init__(
             config=None,
             environment=None,
             record=True,
+            **kwargs,
     ):
         super().__init__(project, environment)
         self.version = version
@@ -552,6 +553,8 @@ class DockerBuildEnvironment(BuildEnvironment):
     container_time_limit = DOCKER_LIMITS.get('time')
 
     def __init__(self, *args, **kwargs):
+        container_image = kwargs.pop('container_image', None)
+
         super().__init__(*args, **kwargs)
         self.client = None
         self.container = None
@@ -568,6 +571,16 @@ def __init__(self, *args, **kwargs):
         if self.project.container_image:
             self.container_image = self.project.container_image
 
+        # Override the ``container_image`` if we pass it via argument.
+        #
+        # FIXME: This is a temporal fix while we explore how to make
+        # ``ubuntu-20.04`` the default build image without breaking lot of
+        # builds. For now, we are passing
+        # ``container_image='readthedocs/build:ubuntu-20.04'`` for the setup
+        # VCS step.
+        if container_image:
+            self.container_image = container_image
+
         if self.project.container_mem_limit:
             self.container_mem_limit = self.project.container_mem_limit
         if self.project.container_time_limit:
diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py
index 4e41ea072f1..214d9d00d6c 100644
--- a/readthedocs/projects/models.py
+++ b/readthedocs/projects/models.py
@@ -1766,7 +1766,6 @@ def add_features(sender, **kwargs):
     ALL_VERSIONS_IN_HTML_CONTEXT = 'all_versions_in_html_context'
     CACHED_ENVIRONMENT = 'cached_environment'
     LIMIT_CONCURRENT_BUILDS = 'limit_concurrent_builds'
-    UPDATE_CA_CERTIFICATES = 'update_ca_certificates'
     CDN_ENABLED = 'cdn_enabled'
 
     # Versions sync related features
@@ -1851,10 +1850,6 @@ def add_features(sender, **kwargs):
             LIMIT_CONCURRENT_BUILDS,
             _('Limit the amount of concurrent builds'),
         ),
-        (
-            UPDATE_CA_CERTIFICATES,
-            _('Update ca-certificates Ubuntu package before VCS clone'),
-        ),
         (
             CDN_ENABLED,
             _('CDN support for a project\'s public versions when privacy levels are enabled.'),
diff --git a/readthedocs/projects/tasks/builds.py b/readthedocs/projects/tasks/builds.py
index 23f7a63c415..e2bd7ab0fdb 100644
--- a/readthedocs/projects/tasks/builds.py
+++ b/readthedocs/projects/tasks/builds.py
@@ -561,6 +561,9 @@ def run_setup(self):
             version=self.data.version,
             build=self.data.build,
             environment=self.get_vcs_env_vars(),
+            # Force the ``container_image`` to use one that has the latest
+            # ca-certificate package which is compatible with Lets Encrypt
+            container_image=settings.RTD_DOCKER_BUILD_SETTINGS['os']['ubuntu-20.04']
         )
 
         # Environment used for code checkout & initial configuration reading
@@ -656,25 +659,6 @@ def setup_vcs(self, environment):
         This also syncs versions in the DB.
         """
         self.update_build(state=BUILD_STATE_CLONING)
-
-        # Install a newer version of ca-certificates packages because it's
-        # required for Let's Encrypt certificates
-        # https://github.com/readthedocs/readthedocs.org/issues/8555
-        # https://community.letsencrypt.org/t/openssl-client-compatibility-changes-for-let-s-encrypt-certificates/143816
-        # TODO: remove this when a newer version of ``ca-certificates`` gets
-        # pre-installed in the Docker images
-        if self.data.project.has_feature(Feature.UPDATE_CA_CERTIFICATES):
-            environment.run(
-                'apt-get', 'update', '--assume-yes', '--quiet',
-                user=settings.RTD_DOCKER_SUPER_USER,
-                record=False,
-            )
-            environment.run(
-                'apt-get', 'install', '--assume-yes', '--quiet', 'ca-certificates',
-                user=settings.RTD_DOCKER_SUPER_USER,
-                record=False,
-            )
-
         self.sync_repo(environment)
 
         commit = self.data.build_commit or self.get_vcs_repo(environment).commit