From fd380a5239edb8b076dceb9a118b5471d9422b31 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Tue, 11 Oct 2022 16:24:37 +0200 Subject: [PATCH 01/70] WIP: Adds context to 404 page --- readthedocs/core/views/__init__.py | 18 ++++++++++++++++++ readthedocs/urls.py | 2 ++ 2 files changed, 20 insertions(+) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 7caa3b18cb6..4d2e34ea114 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -48,6 +48,24 @@ def get_context_data(self, **kwargs): return context +def server_error_404(request, template_name="404.html"): + """A simple 404 handler.""" + project_slug = getattr(request, "host_project_slug", None) + log.debug( + "404 page detected a project slug in request.", + project_slug=project_slug, + ) + project = None + if project_slug: + try: + project = Project.objects.get(slug=project_slug) + except Project.DoesNotExist: + pass + r = render(request, template_name, context={"project": project}) + r.status_code = 404 + return r + + def server_error_500(request, template_name='500.html'): """A simple 500 handler so we get media.""" r = render(request, template_name) diff --git a/readthedocs/urls.py b/readthedocs/urls.py index 579c059ca96..1df4981624e 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -13,12 +13,14 @@ HomepageView, SupportView, do_not_track, + server_error_404, server_error_500, ) from readthedocs.search.views import GlobalSearchView admin.autodiscover() +handler404 = server_error_404 handler500 = server_error_500 basic_urls = [ From aca278d7141a0af4c1f8d950cdd49b545b043302 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Wed, 12 Oct 2022 16:08:59 +0200 Subject: [PATCH 02/70] WIP: Adds a test 404 page --- readthedocs/urls.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/urls.py b/readthedocs/urls.py index 1df4981624e..63198333aa3 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -25,6 +25,7 @@ basic_urls = [ path("", HomepageView.as_view(), name="homepage"), + path("404/", server_error_404), re_path(r"^security/", TemplateView.as_view(template_name="security.html")), re_path( r'^\.well-known/security.txt$', From 7f86767c843164432818d5be6a7a5c605dbe1249 Mon Sep 17 00:00:00 2001 From: Benjamin Bach Date: Wed, 12 Oct 2022 17:31:26 +0200 Subject: [PATCH 03/70] Contextualization gives us 3 interesting scenarios for the 404 page --- readthedocs/templates/404.html | 12 ++++++++++++ readthedocs/urls.py | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/readthedocs/templates/404.html b/readthedocs/templates/404.html index a364ec864c5..9393e9b0ba1 100644 --- a/readthedocs/templates/404.html +++ b/readthedocs/templates/404.html @@ -16,6 +16,18 @@ {% block language-select-form %}{% endblock %} {% block content %} + +{% if project %} +{# project exists #} +Project {{ project }} was found but the page from the project wasn't +{% elif "_proxito_404_" in request.path %} +{# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #} +Project does not exist +{% else %} +{# we are on readthedocs.org itself #} +readthedocs.org should not have 404s, so you must have followed a broken link. +{% endif %} + 
 
         \          SORRY            /
diff --git a/readthedocs/urls.py b/readthedocs/urls.py
index 63198333aa3..6bc61c4ca20 100644
--- a/readthedocs/urls.py
+++ b/readthedocs/urls.py
@@ -25,7 +25,6 @@
 
 basic_urls = [
     path("", HomepageView.as_view(), name="homepage"),
-    path("404/", server_error_404),
     re_path(r"^security/", TemplateView.as_view(template_name="security.html")),
     re_path(
         r'^\.well-known/security.txt$',
@@ -131,6 +130,7 @@
         document_root=os.path.join(settings.MEDIA_ROOT, build_format),
     )
 debug_urls += [
+    path("404/", server_error_404),
     path(
         "style-catalog/",
         TemplateView.as_view(template_name="style_catalog.html"),

From 31ec5f555fa137b70432079677de65443ed11684 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Oct 2022 17:51:02 +0200
Subject: [PATCH 04/70] 404 handler: Receive exception keyword

---
 readthedocs/core/views/__init__.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 4d2e34ea114..bfdde2bc216 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -48,7 +48,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-def server_error_404(request, template_name="404.html"):
+def server_error_404(request, template_name="404.html", exception=exception):
     """A simple 404 handler."""
     project_slug = getattr(request, "host_project_slug", None)
     log.debug(

From 8cbff084845a9a7ad0fd660f96be8bab245ec171 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 27 Oct 2022 19:33:34 +0200
Subject: [PATCH 05/70] Improve contextualized messages, not ready for review,
 needs further testing, and search for isn't safe to assume

---
 readthedocs/core/views/__init__.py |  8 +++--
 readthedocs/settings/base.py       |  4 +--
 readthedocs/templates/404.html     | 47 +++++++++++++++++++++++++-----
 3 files changed, 48 insertions(+), 11 deletions(-)

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index bfdde2bc216..398c5812ece 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -48,7 +48,7 @@ def get_context_data(self, **kwargs):
         return context
 
 
-def server_error_404(request, template_name="404.html", exception=exception):
+def server_error_404(request, template_name="404.html", exception=None):
     """A simple 404 handler."""
     project_slug = getattr(request, "host_project_slug", None)
     log.debug(
@@ -61,7 +61,11 @@ def server_error_404(request, template_name="404.html", exception=exception):
             project = Project.objects.get(slug=project_slug)
         except Project.DoesNotExist:
             pass
-    r = render(request, template_name, context={"project": project})
+    r = render(
+        request,
+        template_name,
+        context={"project": project, "project_slug": project_slug},
+    )
     r.status_code = 404
     return r
 
diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
index b1366a3003e..752ee342137 100644
--- a/readthedocs/settings/base.py
+++ b/readthedocs/settings/base.py
@@ -44,7 +44,7 @@ class CommunityBaseSettings(Settings):
     DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
     # Debug settings
-    DEBUG = True
+    DEBUG = False
     RTD_FORCE_SHOW_DEBUG_TOOLBAR = False
 
     @property
@@ -73,7 +73,7 @@ def SHOW_DEBUG_TOOLBAR(self):
     PRODUCTION_DOMAIN = 'readthedocs.org'
     PUBLIC_DOMAIN = None
     PUBLIC_DOMAIN_USES_HTTPS = False
-    USE_SUBDOMAIN = False
+    USE_SUBDOMAIN = True
     PUBLIC_API_URL = 'https://{}'.format(PRODUCTION_DOMAIN)
     RTD_INTERSPHINX_URL = 'https://{}'.format(PRODUCTION_DOMAIN)
     RTD_EXTERNAL_VERSION_DOMAIN = 'external-builds.readthedocs.io'
diff --git a/readthedocs/templates/404.html b/readthedocs/templates/404.html
index 9393e9b0ba1..082e74210fa 100644
--- a/readthedocs/templates/404.html
+++ b/readthedocs/templates/404.html
@@ -3,7 +3,7 @@
 {% load i18n %}
 
 {% block title %}
-  {% trans "Maze Found" %}
+  {% trans "404 - Maze Found" %}
 {% endblock %}
 
 {% block header-wrapper %}
@@ -17,15 +17,48 @@
 
 {% block content %}
 
+
{% trans "404 - page not found" %}

+
 {% if project %}
-{# project exists #}
-Project {{ project }} was found but the page from the project wasn't
+
+  {# project exists #}
+
+  
You are browsing the documentation of "{{ project.slug }}". The page you are looking for at {{ request.path }} was not found.

+
+  
{% trans "Documentation changes over time. Pages are moved around. You can try to navigate to the front page of the project and finding the page through its navigation." %}

+
+  
{% trans "You can try searching for a keyword:" %}

+
+  

+    
+
+    
+  

+
+  
{% trans "Are you the project owner?" %}

+
+  
{% trans "Here are some tips to address 404 errors:" %}

+
+  
+
 {% elif "_proxito_404_" in request.path %}
-{# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #}
-Project does not exist
+
+  {# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #}
+
+  

+    {% blocktrans %}The project "{{ project_slug }}" that you are looking for either does not exist, has been deleted or does not yet have any valid builds.{% endblocktrans %}
+  

+
 {% else %}
-{# we are on readthedocs.org itself #}
-readthedocs.org should not have 404s, so you must have followed a broken link.
+
+  
{% trans "You have encountered a 404 on Read the Docs. You are either looking for a page that does not exist or a project that has been removed." %}

+
+  
{% trans "If you believe this is a technical error, please contact our support:" %} {% trans "Go to support" %}»

+
 {% endif %}
 
 

From bc73398ce8e62f1a01fd34c8249f38edeb008a40 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Tue, 1 Nov 2022 15:01:22 +0100
Subject: [PATCH 06/70] Remove search form, that was a bad idea, since it will
 only work on standard Sphinx projects.

---
 readthedocs/templates/404.html | 17 ++++++-----------
 1 file changed, 6 insertions(+), 11 deletions(-)

diff --git a/readthedocs/templates/404.html b/readthedocs/templates/404.html
index 082e74210fa..5f57d22c304 100644
--- a/readthedocs/templates/404.html
+++ b/readthedocs/templates/404.html
@@ -23,17 +23,12 @@ 
{% trans "404 - page not found" %}

 
   {# project exists #}
 
-  
You are browsing the documentation of "{{ project.slug }}". The page you are looking for at {{ request.path }} was not found.

-
-  
{% trans "Documentation changes over time. Pages are moved around. You can try to navigate to the front page of the project and finding the page through its navigation." %}

-
-  
{% trans "You can try searching for a keyword:" %}

-
-  

-    
+  

+    {% blocktrans with project_slug=project.slug request_path=request.path %}You are browsing the documentation of "{{ project_slug }}". The page you are looking for at {{ request.path }} was not found.{% endblocktrans %}
+  

 
-    
-  

+  

+    {% blocktrans %}Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.{% endblocktrans %}

 
   
{% trans "Are you the project owner?" %}

 
@@ -50,7 +45,7 @@ 
{% trans "Are you the project owner?" %}

   {# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #}
 
   

-    {% blocktrans %}The project "{{ project_slug }}" that you are looking for either does not exist, has been deleted or does not yet have any valid builds.{% endblocktrans %}
+    {% blocktrans with project_slug=project_slug %}The project "{{ project_slug }}" that you are looking for either does not exist, has been deleted or does not yet have any valid builds.{% endblocktrans %}
   

 
 {% else %}

From a8573f9bf922fd16a6d467856e35261c2f164419 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 2 Nov 2022 18:39:44 +0100
Subject: [PATCH 07/70] Early work on an approach that uses a custom Http404
 exception to save relevant 404 context

---
 readthedocs/core/views/__init__.py      | 12 ++++----
 readthedocs/proxito/exceptions.py       |  9 ++++++
 readthedocs/proxito/views/decorators.py | 16 +++++++---
 readthedocs/proxito/views/serve.py      | 29 +++++++++++++-----
 readthedocs/proxito/views/utils.py      | 40 ++++++++++++++++++++-----
 5 files changed, 81 insertions(+), 25 deletions(-)
 create mode 100644 readthedocs/proxito/exceptions.py

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 398c5812ece..16592ed7474 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -50,17 +50,15 @@ def get_context_data(self, **kwargs):
 
 def server_error_404(request, template_name="404.html", exception=None):
     """A simple 404 handler."""
-    project_slug = getattr(request, "host_project_slug", None)
+    # This property is set by ProxitoHttp404. We could also have a look at the
+    # subproject_slug
+    project_slug = getattr(exception, "project_slug", None)
+    log.debug(exception)
+    project = getattr(exception, "project", None)
     log.debug(
         "404 page detected a project slug in request.",
         project_slug=project_slug,
     )
-    project = None
-    if project_slug:
-        try:
-            project = Project.objects.get(slug=project_slug)
-        except Project.DoesNotExist:
-            pass
     r = render(
         request,
         template_name,
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
new file mode 100644
index 00000000000..233385f1f1b
--- /dev/null
+++ b/readthedocs/proxito/exceptions.py
@@ -0,0 +1,9 @@
+from django.http.response import Http404
+
+
+class ProxitoHttp404(Http404):
+    def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
+        self.project_slug = project_slug
+        self.project = project
+        self.subproject_slug = subproject_slug
+        super().__init__(message)
diff --git a/readthedocs/proxito/views/decorators.py b/readthedocs/proxito/views/decorators.py
index 11a744e72b6..1e422523e9e 100644
--- a/readthedocs/proxito/views/decorators.py
+++ b/readthedocs/proxito/views/decorators.py
@@ -1,9 +1,9 @@
-import structlog
 from functools import wraps
 
-from django.http import Http404
+import structlog
 
 from readthedocs.projects.models import Project, ProjectRelationship
+from readthedocs.proxito.exceptions import ProxitoHttp404
 
 log = structlog.get_logger(__name__)  # noqa
 
@@ -44,7 +44,9 @@ def inner_view(  # noqa
                         subproject_slug=subproject_slug,
                         project_slug=kwargs['project'].slug,
                     )
-                    raise Http404('Invalid subproject slug')
+                    raise ProxitoHttp404(
+                        "Invalid subproject slug", subproject_slug=subproject_slug
+                    )
         return view_func(request, subproject=subproject, *args, **kwargs)
 
     return inner_view
@@ -74,7 +76,13 @@ def inner_view(  # noqa
             try:
                 project = Project.objects.get(slug=project_slug)
             except Project.DoesNotExist:
-                raise Http404('Project does not exist.')
+                log.debug(
+                    "Project not found.",
+                    project_slug=project_slug,
+                )
+                raise ProxitoHttp404(
+                    "Project does not exist.", project_slug=project_slug
+                )
         return view_func(request, project=project, *args, **kwargs)
 
     return inner_view
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 1e08401cb3f..a98dc2beda6 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -4,7 +4,7 @@
 
 import structlog
 from django.conf import settings
-from django.http import Http404, HttpResponse, HttpResponseRedirect
+from django.http import HttpResponse, HttpResponseRedirect
 from django.shortcuts import render
 from django.urls import resolve as url_resolve
 from django.utils.decorators import method_decorator
@@ -22,6 +22,7 @@
 from readthedocs.projects.constants import SPHINX_HTMLDIR
 from readthedocs.projects.models import Feature
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
+from readthedocs.proxito.exceptions import ProxitoHttp404
 from readthedocs.redirects.exceptions import InfiniteRedirectException
 from readthedocs.storage import build_media_storage, staticfiles_storage
 
@@ -109,7 +110,7 @@ def get(self,
         # version on the database we want to return 404.
         if (version and not version.active) or (version_slug and not version):
             log.warning("Version does not exist or is not active.")
-            raise Http404("Version does not exist or is not active.")
+            raise ProxitoHttp404("Version does not exist or is not active.")
 
         if self._is_cache_enabled(final_project) and version and not version.is_private:
             # All public versions can be cached.
@@ -174,7 +175,7 @@ def get(self,
                 'Invalid URL for project with versions.',
                 filename=filename,
             )
-            raise Http404('Invalid URL for project with versions')
+            raise ProxitoHttp404("Invalid URL for project with versions")
 
         redirect_path, http_status = self.get_redirect(
             project=final_project,
@@ -261,10 +262,17 @@ def get(self, request, proxito_path, template_name='404.html'):
             proxito_path,
             urlconf='readthedocs.proxito.urls',
         )
+        log.debug("Resolved a URL.")
 
         version_slug = kwargs.get('version_slug')
         version_slug = self.get_version_from_host(request, version_slug)
-        final_project, lang_slug, version_slug, filename = _get_project_data_from_request(  # noqa
+        log.debug("Getting _get_project_data_from_request.")
+        (
+            final_project,
+            lang_slug,
+            version_slug,
+            filename,
+        ) = _get_project_data_from_request(  # noqa
             request,
             project_slug=kwargs.get('project_slug'),
             subproject_slug=kwargs.get('subproject_slug'),
@@ -272,6 +280,7 @@ def get(self, request, proxito_path, template_name='404.html'):
             version_slug=version_slug,
             filename=kwargs.get('filename', ''),
         )
+        log.debug("Finished _get_project_data_from_request.")
 
         log.bind(
             project_slug=final_project.slug,
@@ -390,7 +399,13 @@ def get(self, request, proxito_path, template_name='404.html'):
             path=filename,
             full_path=proxito_path,
         )
-        raise Http404('No custom 404 page found.')
+        log.debug("Raising ProxitoHttp404")
+        raise ProxitoHttp404(
+            "No custom 404 page found.",
+            project=final_project,
+            project_slug=kwargs.get("project_slug"),
+            subproject_slug=kwargs.get("subproject_slug"),
+        )
 
     def _register_broken_link(self, project, version, path, full_path):
         try:
@@ -472,7 +487,7 @@ def get(self, request, project):
 
         if no_serve_robots_txt:
             # ... we do return a 404
-            raise Http404()
+            raise ProxitoHttp404()
 
         storage_path = project.get_storage_path(
             type_='html',
@@ -595,7 +610,7 @@ def changefreqs_generator():
             only_active=True,
         )
         if not public_versions.exists():
-            raise Http404
+            raise ProxitoHttp404()
 
         sorted_versions = sort_version_aware(public_versions)
 
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 77cb58d7fdb..84b14435686 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -2,7 +2,10 @@
 
 import structlog
 from django.http import HttpResponse
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import render
+
+from readthedocs.projects.models import Project
+from readthedocs.proxito.exceptions import ProxitoHttp404
 
 from .decorators import map_project_slug, map_subproject_slug
 
@@ -30,11 +33,23 @@ def proxito_404_page_handler(request, exception=None, template_name='404.html'):
     """
 
     if request.resolver_match and request.resolver_match.url_name != 'proxito_404_handler':
+        log.debug("Displaying a 'fast 404'")
         return fast_404(request, exception, template_name)
 
-    resp = render(request, template_name)
-    resp.status_code = 404
-    return resp
+    project_slug = getattr(exception, "project_slug", None)
+    log.debug(exception)
+    project = getattr(exception, "project", None)
+    log.debug(
+        "404 page detected a project slug in request.",
+        project_slug=project_slug,
+    )
+    r = render(
+        request,
+        template_name,
+        context={"project": project, "project_slug": project_slug},
+    )
+    r.status_code = 404
+    return r
 
 
 @map_project_slug
@@ -70,9 +85,20 @@ def _get_project_data_from_request(
     if not lang_slug or lang_slug == current_project.language:
         final_project = current_project
     else:
-        final_project = get_object_or_404(
-            current_project.translations.all(), language=lang_slug
-        )
+        try:
+            final_project = current_project.translations.get(language=lang_slug)
+        except Project.DoesNotExist:
+            log.debug(
+                "Raising 404 for language slug",
+                lang_slug=lang_slug,
+                project_slug=current_project.slug,
+            )
+            raise ProxitoHttp404(
+                "Did not find translation",
+                project=current_project,
+                project_slug=current_project.slug,
+                subproject_slug=lang_slug,
+            )
 
     # Handle single version by grabbing the default version
     # We might have version_slug when we're serving a PR

From 8bd53b6faeb88546837276f964bf986f29319ee8 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 3 Nov 2022 18:13:56 +0100
Subject: [PATCH 08/70] Refactor error templates to separate contextualized
 handling + some template DRY stuff.

---
 readthedocs/core/views/__init__.py            | 18 ++--
 readthedocs/proxito/exceptions.py             | 47 ++++++++++
 readthedocs/templates/401.html                | 11 +--
 readthedocs/templates/404.html                | 86 +------------------
 readthedocs/templates/429.html                | 11 +--
 readthedocs/templates/500.html                | 11 +--
 readthedocs/templates/error_header.html       |  1 +
 readthedocs/templates/errors/404/base.html    | 50 +++++++++++
 .../templates/errors/404/community.html       | 12 +++
 .../templates/errors/404/include_tips.html    | 11 +++
 .../templates/errors/404/no_project.html      | 16 ++++
 .../templates/errors/404/no_project_page.html | 22 +++++
 .../templates/errors/404/no_subproject.html   | 22 +++++
 readthedocs/templates/errors/base.html        | 28 ++++++
 14 files changed, 227 insertions(+), 119 deletions(-)
 create mode 100644 readthedocs/templates/errors/404/base.html
 create mode 100644 readthedocs/templates/errors/404/community.html
 create mode 100644 readthedocs/templates/errors/404/include_tips.html
 create mode 100644 readthedocs/templates/errors/404/no_project.html
 create mode 100644 readthedocs/templates/errors/404/no_project_page.html
 create mode 100644 readthedocs/templates/errors/404/no_subproject.html
 create mode 100644 readthedocs/templates/errors/base.html

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 16592ed7474..3eb85bdd1b4 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -48,17 +48,25 @@ def get_context_data(self, **kwargs):
         return context
 
 
-def server_error_404(request, template_name="404.html", exception=None):
+def server_error_404(
+    request, template_name="errors/404/community.html", exception=None
+):
     """A simple 404 handler."""
     # This property is set by ProxitoHttp404. We could also have a look at the
     # subproject_slug
+
     project_slug = getattr(exception, "project_slug", None)
     log.debug(exception)
     project = getattr(exception, "project", None)
-    log.debug(
-        "404 page detected a project slug in request.",
-        project_slug=project_slug,
-    )
+    subproject_slug = getattr(exception, "subproject_slug", None)
+
+    if subproject_slug:
+        template_name = "errors/404/no_subproject.html"
+    elif project:
+        template_name = "errors/404/no_project_page.html"
+    elif project_slug:
+        template_name = "errors/404/no_project.html"
+
     r = render(
         request,
         template_name,
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 233385f1f1b..7f65abb7f92 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -1,7 +1,54 @@
+"""
+This module contains exceptions that may be raised in the proxito application
+but are handled elsewhere in the Django project.
+
+--------------
+404 exceptions
+--------------
+
+Because an Http404 can be raised anywhere in the code,
+we want to add more context for targeted error handling and user-facing communication
+"""
 from django.http.response import Http404
 
 
 class ProxitoHttp404(Http404):
+    """
+    General error class for proxity 404s
+    """
+
+    def __init__(self, message):
+        super().__init__(message)
+
+
+class ProxitoProjectHttp404(ProxitoHttp404):
+    """
+    Raised if a project was not found
+    """
+
+    def __init__(self, message, project_slug=None):
+        self.project_slug = project_slug
+        super().__init__(message)
+
+
+class ProxitoSubProjectHttp404(ProxitoProjectHttp404):
+    """
+    Raised if a sub-project was not found
+    """
+    def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
+        self.project_slug = project_slug
+        self.project = project
+        self.subproject_slug = subproject_slug
+        super().__init__(message)
+
+
+class ProxitoProjectPageHttp404(ProxitoHttp404):
+    """
+    Raised if a page inside an existing project was not found
+
+    Note: The containing project can be both a project or a subproject inside another project
+    """
+
     def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
         self.project_slug = project_slug
         self.project = project
diff --git a/readthedocs/templates/401.html b/readthedocs/templates/401.html
index d9d818be0da..b5db786a6df 100644
--- a/readthedocs/templates/401.html
+++ b/readthedocs/templates/401.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "errors/base.html" %}
 {% load core_tags %}
 {% load i18n %}
 
@@ -6,15 +6,6 @@
   {% trans "Maze Found" %}
 {% endblock %}
 
-{% block header-wrapper %}
-  {% include "error_header.html" %}
-{% endblock %}
-
-{% block notify %}{% endblock %}
-
-{# Hide the language select form so we don't set a CSRF cookie #}
-{% block language-select-form %}{% endblock %}
-
 {% block content %}
 
Permission Denied

 

diff --git a/readthedocs/templates/404.html b/readthedocs/templates/404.html
index 5f57d22c304..790202fa6a0 100644
--- a/readthedocs/templates/404.html
+++ b/readthedocs/templates/404.html
@@ -1,84 +1,2 @@
-{% extends "base.html" %}
-{% load core_tags %}
-{% load i18n %}
-
-{% block title %}
-  {% trans "404 - Maze Found" %}
-{% endblock %}
-
-{% block header-wrapper %}
-  {% include "error_header.html" %}
-{% endblock %}
-
-{% block notify %}{% endblock %}
-
-{# Hide the language select form so we don't set a CSRF cookie #}
-{% block language-select-form %}{% endblock %}
-
-{% block content %}
-
-
{% trans "404 - page not found" %}

-
-{% if project %}
-
-  {# project exists #}
-
-  

-    {% blocktrans with project_slug=project.slug request_path=request.path %}You are browsing the documentation of "{{ project_slug }}". The page you are looking for at {{ request.path }} was not found.{% endblocktrans %}
-  

-
-  

-    {% blocktrans %}Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.{% endblocktrans %}

-
-  
{% trans "Are you the project owner?" %}

-
-  
{% trans "Here are some tips to address 404 errors:" %}

-
-  
-
-{% elif "_proxito_404_" in request.path %}
-
-  {# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #}
-
-  

-    {% blocktrans with project_slug=project_slug %}The project "{{ project_slug }}" that you are looking for either does not exist, has been deleted or does not yet have any valid builds.{% endblocktrans %}
-  

-
-{% else %}
-
-  
{% trans "You have encountered a 404 on Read the Docs. You are either looking for a page that does not exist or a project that has been removed." %}

-
-  
{% trans "If you believe this is a technical error, please contact our support:" %} {% trans "Go to support" %}»

-
-{% endif %}
-
-
-
-        \          SORRY            /
-         \                         /
-          \    This page does     /
-           ]   not exist yet.    [    ,'|
-           ]                     [   /  |
-           ]___               ___[ ,'   |
-           ]  ]\             /[  [ |:   |
-           ]  ] \           / [  [ |:   |
-           ]  ]  ]         [  [  [ |:   |
-           ]  ]  ]__     __[  [  [ |:   |
-           ]  ]  ] ]\ _ /[ [  [  [ |:   |
-           ]  ]  ] ] (#) [ [  [  [ :===='
-           ]  ]  ]_].nHn.[_[  [  [
-           ]  ]  ]  HHHHH. [  [  [
-           ]  ] /   `HH("N  \ [  [
-           ]__]/     HHH  "  \[__[
-           ]         NNN         [
-           ]         N/"         [
-           ]         N H         [
-          /          N            \
-         /           q,            \
-        /                           \
-

-{% endblock %}
+{% extends "errors/404/base.html" %}
+{# this is a deprecated template, will be removed. Please use one of the targeted 404s in templates/404/ #}
diff --git a/readthedocs/templates/429.html b/readthedocs/templates/429.html
index b7c79fd8c16..600a2b3ff7a 100644
--- a/readthedocs/templates/429.html
+++ b/readthedocs/templates/429.html
@@ -1,4 +1,4 @@
-{% extends "base.html" %}
+{% extends "errors/base.html" %}
 {% load core_tags %}
 {% load i18n %}
 
@@ -6,15 +6,6 @@
   {% trans "Too many requests" %}
 {% endblock %}
 
-{% block header-wrapper %}
-  {% include "error_header.html" %}
-{% endblock %}
-
-{% block notify %}{% endblock %}
-
-{# Hide the language select form so we don't set a CSRF cookie #}
-{% block language-select-form %}{% endblock %}
-
 {% block content %}
 
               .--~~,__
diff --git a/readthedocs/templates/500.html b/readthedocs/templates/500.html
index 70693b5ea3a..381ca625572 100644
--- a/readthedocs/templates/500.html
+++ b/readthedocs/templates/500.html
@@ -1,19 +1,10 @@
-{% extends "base.html" %}
+{% extends "errors/base.html" %}
 {% load i18n %}
 
 {% block title %}
   {% trans "Server Error" %}
 {% endblock %}
 
-{% block header-wrapper %}
-  {% include "error_header.html" %}
-{% endblock %}
-
-{% block notify %}{% endblock %}
-
-{# Hide the language select form so we don't set a CSRF cookie #}
-{% block language-select-form %}{% endblock %}
-
 {% block content %}
 
           .
diff --git a/readthedocs/templates/error_header.html b/readthedocs/templates/error_header.html
index e6cd5b2234f..ecebdbc745b 100644
--- a/readthedocs/templates/error_header.html
+++ b/readthedocs/templates/error_header.html
@@ -1,4 +1,5 @@
 {% load i18n %}
+{# this template is deprecated, please extend from errors/base.html #}
 
     

       

diff --git a/readthedocs/templates/errors/404/base.html b/readthedocs/templates/errors/404/base.html
new file mode 100644
index 00000000000..b82c0e9ff37
--- /dev/null
+++ b/readthedocs/templates/errors/404/base.html
@@ -0,0 +1,50 @@
+{% extends "errors/base.html" %}
+{% load core_tags %}
+{% load i18n %}
+
+{% block title %}
+  {% trans "404 - Maze Found" %}
+{% endblock %}
+
+{% block content %}
+
+
{% trans "404 - page not found" %}

+
+{% block 404_error_message %}
+
+{% endblock %}
+
+{% if project %}
+
+{% elif "_proxito_404_" in request.path %}
+
+{% else %}
+
+{% endif %}
+
+
+
+        \          SORRY            /
+         \                         /
+          \    This page does     /
+           ]   not exist yet.    [    ,'|
+           ]                     [   /  |
+           ]___               ___[ ,'   |
+           ]  ]\             /[  [ |:   |
+           ]  ] \           / [  [ |:   |
+           ]  ]  ]         [  [  [ |:   |
+           ]  ]  ]__     __[  [  [ |:   |
+           ]  ]  ] ]\ _ /[ [  [  [ |:   |
+           ]  ]  ] ] (#) [ [  [  [ :===='
+           ]  ]  ]_].nHn.[_[  [  [
+           ]  ]  ]  HHHHH. [  [  [
+           ]  ] /   `HH("N  \ [  [
+           ]__]/     HHH  "  \[__[
+           ]         NNN         [
+           ]         N/"         [
+           ]         N H         [
+          /          N            \
+         /           q,            \
+        /                           \
+

+{% endblock %}
diff --git a/readthedocs/templates/errors/404/community.html b/readthedocs/templates/errors/404/community.html
new file mode 100644
index 00000000000..81774812422
--- /dev/null
+++ b/readthedocs/templates/errors/404/community.html
@@ -0,0 +1,12 @@
+{% extends "errors/404/base.html" %}
+{% load i18n %}
+
+{# something on readthedocs.org does not exist - this 404 is not intended for projects #}
+
+{% block 404_error_message %}
+
+  
{% trans "You have encountered a 404 on Read the Docs. You are either looking for a page that does not exist or a project that has been removed." %}

+
+  
{% trans "If you believe this is a technical error, please contact our support:" %} {% trans "Go to support" %}»

+
+{% endblock %}
diff --git a/readthedocs/templates/errors/404/include_tips.html b/readthedocs/templates/errors/404/include_tips.html
new file mode 100644
index 00000000000..ee42ee8b74f
--- /dev/null
+++ b/readthedocs/templates/errors/404/include_tips.html
@@ -0,0 +1,11 @@
+{% load i18n %}
+
+
{% trans "Are you the project owner?" %}

+
+
{% trans "Here are some tips to address 404 errors:" %}

+
+
diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
new file mode 100644
index 00000000000..f3ff62966ed
--- /dev/null
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -0,0 +1,16 @@
+{% extends "errors/404/base.html" %}
+{% load i18n %}
+
+{# visiting a slug or subdomain that doesn't match the intended project #}
+
+{% block 404_error_message %}
+
+  {# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #}
+
+  

+    {% blocktrans trimmed with project_slug=project_slug %}
+      The project "{{ project_slug }}" that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
+    {% endblocktrans %}
+  

+
+{% endblock %}
diff --git a/readthedocs/templates/errors/404/no_project_page.html b/readthedocs/templates/errors/404/no_project_page.html
new file mode 100644
index 00000000000..bce6d1042bd
--- /dev/null
+++ b/readthedocs/templates/errors/404/no_project_page.html
@@ -0,0 +1,22 @@
+{% extends "errors/404/base.html" %}
+{% load i18n %}
+
+{# project exists but page doesn't #}
+
+{% block 404_error_message %}
+
+  

+    {% blocktrans trimmed with project_slug=project.slug request_path=request.path %}
+      You are browsing the documentation of "{{ project_slug }}". The page you are looking for at {{ request.path }} was not found.
+    {% endblocktrans %}
+  

+
+  

+    {% blocktrans trimmed %}
+      Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
+    {% endblocktrans %}
+  

+
+  {% include "errors/404/include_tips.html" %}
+
+{% endblock %}
diff --git a/readthedocs/templates/errors/404/no_subproject.html b/readthedocs/templates/errors/404/no_subproject.html
new file mode 100644
index 00000000000..bce6d1042bd
--- /dev/null
+++ b/readthedocs/templates/errors/404/no_subproject.html
@@ -0,0 +1,22 @@
+{% extends "errors/404/base.html" %}
+{% load i18n %}
+
+{# project exists but page doesn't #}
+
+{% block 404_error_message %}
+
+  

+    {% blocktrans trimmed with project_slug=project.slug request_path=request.path %}
+      You are browsing the documentation of "{{ project_slug }}". The page you are looking for at {{ request.path }} was not found.
+    {% endblocktrans %}
+  

+
+  

+    {% blocktrans trimmed %}
+      Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
+    {% endblocktrans %}
+  

+
+  {% include "errors/404/include_tips.html" %}
+
+{% endblock %}
diff --git a/readthedocs/templates/errors/base.html b/readthedocs/templates/errors/base.html
new file mode 100644
index 00000000000..6bc2a137ab4
--- /dev/null
+++ b/readthedocs/templates/errors/base.html
@@ -0,0 +1,28 @@
+{% extends "base.html" %}
+{% load core_tags %}
+{% load i18n %}
+
+{% block header-wrapper %}
+    

+      

+    
+
+        
+        

+            

+                {% comment %}Translators: Name of the website{% endcomment %}
+                
+                  Read the Docs
+                
+            

+        

+        
+      

+    

+    
+{% endblock %}
+
+{% block notify %}{% endblock %}
+
+{# Hide the language select form so we don't set a CSRF cookie #}
+{% block language-select-form %}{% endblock %}

From 7d1c69ad06a0d31cfaf293c6287d832059a91831 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Fri, 4 Nov 2022 01:20:23 +0100
Subject: [PATCH 09/70] Apply new exceptions in some of the cases

---
 readthedocs/proxito/views/decorators.py | 9 ++++++---
 readthedocs/proxito/views/serve.py      | 4 ++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/readthedocs/proxito/views/decorators.py b/readthedocs/proxito/views/decorators.py
index 1e422523e9e..924a62e2e35 100644
--- a/readthedocs/proxito/views/decorators.py
+++ b/readthedocs/proxito/views/decorators.py
@@ -3,7 +3,10 @@
 import structlog
 
 from readthedocs.projects.models import Project, ProjectRelationship
-from readthedocs.proxito.exceptions import ProxitoHttp404
+from readthedocs.proxito.exceptions import (
+    ProxitoProjectHttp404,
+    ProxitoSubProjectHttp404,
+)
 
 log = structlog.get_logger(__name__)  # noqa
 
@@ -44,7 +47,7 @@ def inner_view(  # noqa
                         subproject_slug=subproject_slug,
                         project_slug=kwargs['project'].slug,
                     )
-                    raise ProxitoHttp404(
+                    raise ProxitoSubProjectHttp404(
                         "Invalid subproject slug", subproject_slug=subproject_slug
                     )
         return view_func(request, subproject=subproject, *args, **kwargs)
@@ -80,7 +83,7 @@ def inner_view(  # noqa
                     "Project not found.",
                     project_slug=project_slug,
                 )
-                raise ProxitoHttp404(
+                raise ProxitoProjectHttp404(
                     "Project does not exist.", project_slug=project_slug
                 )
         return view_func(request, project=project, *args, **kwargs)
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index a98dc2beda6..596eec5ef16 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -22,7 +22,7 @@
 from readthedocs.projects.constants import SPHINX_HTMLDIR
 from readthedocs.projects.models import Feature
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
-from readthedocs.proxito.exceptions import ProxitoHttp404
+from readthedocs.proxito.exceptions import ProxitoHttp404, ProxitoProjectPageHttp404
 from readthedocs.redirects.exceptions import InfiniteRedirectException
 from readthedocs.storage import build_media_storage, staticfiles_storage
 
@@ -400,7 +400,7 @@ def get(self, request, proxito_path, template_name='404.html'):
             full_path=proxito_path,
         )
         log.debug("Raising ProxitoHttp404")
-        raise ProxitoHttp404(
+        raise ProxitoProjectPageHttp404(
             "No custom 404 page found.",
             project=final_project,
             project_slug=kwargs.get("project_slug"),

From a30fa3cc1538a3eca33fa91f3a344d41bca84622 Mon Sep 17 00:00:00 2001
From: Benjamin Balder Bach 
Date: Mon, 7 Nov 2022 13:43:45 +0100
Subject: [PATCH 10/70] Update readthedocs/proxito/exceptions.py

Co-authored-by: Manuel Kaufmann 
---
 readthedocs/proxito/exceptions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 7f65abb7f92..c66c342f4e5 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -1,5 +1,5 @@
 """
-This module contains exceptions that may be raised in the proxito application
+This module contains exceptions that may be raised in El Proxito application
 but are handled elsewhere in the Django project.
 
 --------------

From c50554400120e11c533e3b494dc1361f34225360 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Wed, 15 Feb 2023 19:15:19 -0500
Subject: [PATCH 11/70] Refactor unresolver

---
 readthedocs/core/unresolver.py                | 170 +++++++++++++-----
 .../rtd_tests/tests/test_unresolver.py        | 134 +++++++++-----
 2 files changed, 212 insertions(+), 92 deletions(-)

diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py
index d12c27a0818..f1361e32ef6 100644
--- a/readthedocs/core/unresolver.py
+++ b/readthedocs/core/unresolver.py
@@ -44,6 +44,33 @@ class InvalidCustomDomainError(DomainError):
     pass
 
 
+class VersionNotFoundError(UnresolverError):
+    def __init__(self, project, version_slug, filename):
+        self.project = project
+        self.version_slug = version_slug
+        self.filename = filename
+
+
+class TranslationNotFoundError(UnresolverError):
+    def __init__(self, project, language, filename):
+        self.project = project
+        self.language = language
+        self.filename = filename
+
+
+class UnresolvedPathError(UnresolverError):
+    def __init__(self, project, path):
+        self.project = project
+        self.path = path
+
+
+class InvalidExternalVersionError(UnresolverError):
+    def __init__(self, project, version, external_version_slug):
+        self.project = project
+        self.version = version
+        self.external_version_slug = external_version_slug
+
+
 @dataclass(slots=True)
 class UnresolvedURL:
 
@@ -57,9 +84,9 @@ class UnresolvedURL:
     # It can be the same as parent_project.
     project: Project
 
-    version: Version = None
-    filename: str = None
-    parsed_url: ParseResult = None
+    version: Version
+    filename: str
+    parsed_url: ParseResult
     domain: Domain = None
     external: bool = False
 
@@ -76,6 +103,8 @@ class DomainSourceType(Enum):
 
 @dataclass(slots=True)
 class UnresolvedDomain:
+    # The domain that was used to extract the information from.
+    source_domain: str
     source: DomainSourceType
     project: Project
     domain: Domain = None
@@ -141,46 +170,88 @@ def unresolve(self, url, append_indexhtml=True):
         :param append_indexhtml: If `True` directories will be normalized
          to end with ``/index.html``.
         """
-        parsed = urlparse(url)
-        domain = self.get_domain_from_host(parsed.netloc)
+        parsed_url = urlparse(url)
+        domain = self.get_domain_from_host(parsed_url.netloc)
         unresolved_domain = self.unresolve_domain(domain)
+        return self._unresolve(
+            unresolved_domain=unresolved_domain,
+            parsed_url=parsed_url,
+            append_indexhtml=append_indexhtml,
+        )
+
+    def unresolve_path(self, unresolved_domain, path, append_indexhtml=True):
+        """
+        Unresolve a path given a unresolved domain.
+
+        This is the same as the unresolve method,
+        but this method takes an unresolved domain
+        from unresolve_domain as input.
+
+        :param unresolved_domain: An UnresolvedDomain object.
+        :param path: Path to unresolve (this shouldn't include the protocol or querystrings).
+        :param append_indexhtml: If `True` directories will be normalized
+         to end with ``/index.html``.
+        """
+        # We don't call unparse() on the path,
+        # since it could be parsed as a full URL if it starts with a protocol.
+        parsed_url = ParseResult(
+            scheme="", netloc="", path=path, params="", query="", fragment=""
+        )
+        return self._unresolve(
+            unresolved_domain=unresolved_domain,
+            parsed_url=parsed_url,
+            append_indexhtml=append_indexhtml,
+        )
 
+    def _unresolve(self, unresolved_domain, parsed_url, append_indexhtml):
+        """
+        The actual unresolver.
+
+        Extracted into a separate method so it can be re-used by
+        the unresolve and unresolve_path methods.
+        """
         current_project, version, filename = self._unresolve_path(
             parent_project=unresolved_domain.project,
-            path=parsed.path,
+            path=parsed_url.path,
             external_version_slug=unresolved_domain.external_version_slug,
         )
 
         # Make sure we are serving the external version from the subdomain.
-        if unresolved_domain.is_from_external_domain and version:
+        if unresolved_domain.is_from_external_domain:
+            domain = unresolved_domain.source_domain
             if unresolved_domain.external_version_slug != version.slug:
                 log.warning(
                     "Invalid version for external domain.",
                     domain=domain,
                     version_slug=version.slug,
                 )
-                version = None
-                filename = None
-            elif not version.is_external:
+                raise InvalidExternalVersionError(
+                    project=current_project,
+                    version=version,
+                    external_version_slug=unresolved_domain.external_version_slug,
+                )
+            if not version.is_external:
                 log.warning(
                     "Attempt of serving a non-external version from RTD_EXTERNAL_VERSION_DOMAIN.",
                     domain=domain,
                     version_slug=version.slug,
                     version_type=version.type,
-                    url=url,
                 )
-                version = None
-                filename = None
+                raise InvalidExternalVersionError(
+                    project=current_project,
+                    version=version,
+                    external_version_slug=unresolved_domain.external_version_slug,
+                )
 
-        if append_indexhtml and filename and filename.endswith("/"):
+        if append_indexhtml and filename.endswith("/"):
             filename += "index.html"
 
         return UnresolvedURL(
             parent_project=unresolved_domain.project,
-            project=current_project or unresolved_domain.project,
+            project=current_project,
             version=version,
             filename=filename,
-            parsed_url=parsed,
+            parsed_url=parsed_url,
             domain=unresolved_domain.domain,
             external=unresolved_domain.is_from_external_domain,
         )
@@ -197,12 +268,10 @@ def _match_multiversion_project(self, parent_project, path):
         """
         Try to match a multiversion project.
 
-        If the translation exists, we return a result even if the version doesn't,
-        so the translation is taken as the current project (useful for 404 pages).
+        An exception is raised if we weren't able to find a matching version or language,
+        this exception has the current project (useful for 404 pages).
 
-        :returns: None or a tuple with the current project, version and file.
-         A tuple with only the project means we weren't able to find a version,
-         but the translation was correct.
+        :returns: A tuple with the current project, version and file.
         """
         match = self.multiversion_pattern.match(path)
         if not match:
@@ -216,14 +285,18 @@ def _match_multiversion_project(self, parent_project, path):
             project = parent_project
         else:
             project = parent_project.translations.filter(language=language).first()
+            if not project:
+                raise TranslationNotFoundError(
+                    project=parent_project, language=language, filename=file
+                )
 
-        if project:
-            version = project.versions.filter(slug=version_slug).first()
-            if version:
-                return project, version, file
-            return project, None, None
+        version = project.versions.filter(slug=version_slug).first()
+        if not version:
+            raise VersionNotFoundError(
+                project=project, version_slug=version_slug, filename=file
+            )
 
-        return None
+        return project, version, file
 
     def _match_subproject(self, parent_project, path, external_version_slug=None):
         """
@@ -232,12 +305,7 @@ def _match_subproject(self, parent_project, path, external_version_slug=None):
         If the subproject exists, we try to resolve the rest of the path
         with the subproject as the canonical project.
 
-        If the subproject exists, we return a result even if version doesn't,
-        so the subproject is taken as the current project (useful for 404 pages).
-
-        :returns: None or a tuple with the current project, version and file.
-         A tuple with only the project means we were able to find the subproject,
-         but we weren't able to resolve the rest of the path.
+        :returns: A tuple with the current project, version and file.
         """
         match = self.subproject_pattern.match(path)
         if not match:
@@ -260,12 +328,7 @@ def _match_subproject(self, parent_project, path, external_version_slug=None):
                 check_subprojects=False,
                 external_version_slug=external_version_slug,
             )
-            # If we got a valid response, return that,
-            # otherwise return the current subproject
-            # as the current project without a valid version or path.
-            if response:
-                return response
-            return subproject, None, None
+            return response
         return None
 
     def _match_single_version_project(
@@ -276,21 +339,30 @@ def _match_single_version_project(
 
         By default any path will match. If `external_version_slug` is given,
         that version is used instead of the project's default version.
+
+        An exception is raised if we weren't able to find a matching version,
+        this exception has the current project (useful for 404 pages).
+
+        :returns: A tuple with the current project, version and file.
         """
         file = self._normalize_filename(path)
         if external_version_slug:
+            version_slug = external_version_slug
             version = (
                 parent_project.versions(manager=EXTERNAL)
-                .filter(slug=external_version_slug)
+                .filter(slug=version_slug)
                 .first()
             )
         else:
-            version = parent_project.versions.filter(
-                slug=parent_project.default_version
-            ).first()
-        if version:
-            return parent_project, version, file
-        return parent_project, None, None
+            version_slug = parent_project.default_version
+            version = parent_project.versions.filter(slug=version_slug).first()
+
+        if not version:
+            raise VersionNotFoundError(
+                project=parent_project, version_slug=version_slug, filename=file
+            )
+
+        return parent_project, version, file
 
     def _unresolve_path(
         self, parent_project, path, check_subprojects=True, external_version_slug=None
@@ -352,7 +424,7 @@ def _unresolve_path(
             if response:
                 return response
 
-        return parent_project, None, None
+        raise UnresolvedPathError(project=parent_project, path=path)
 
     @staticmethod
     def get_domain_from_host(host):
@@ -384,6 +456,7 @@ def unresolve_domain(self, domain):
                 project_slug = subdomain
                 log.debug("Public domain.", domain=domain)
                 return UnresolvedDomain(
+                    source_domain=domain,
                     source=DomainSourceType.public_domain,
                     project=self._resolve_project_slug(project_slug, domain),
                 )
@@ -400,6 +473,7 @@ def unresolve_domain(self, domain):
                     project_slug, version_slug = subdomain.rsplit("--", maxsplit=1)
                     log.debug("External versions domain.", domain=domain)
                     return UnresolvedDomain(
+                        source_domain=domain,
                         source=DomainSourceType.external_domain,
                         project=self._resolve_project_slug(project_slug, domain),
                         external_version_slug=version_slug,
@@ -425,6 +499,7 @@ def unresolve_domain(self, domain):
 
         log.debug("Custom domain.", domain=domain)
         return UnresolvedDomain(
+            source_domain=domain,
             source=DomainSourceType.custom_domain,
             project=domain_object.project,
             domain=domain_object,
@@ -464,6 +539,7 @@ def unresolve_domain_from_request(self, request):
                     project_slug=project.slug,
                 )
                 return UnresolvedDomain(
+                    source_domain=host,
                     source=DomainSourceType.http_header,
                     project=project,
                 )
diff --git a/readthedocs/rtd_tests/tests/test_unresolver.py b/readthedocs/rtd_tests/tests/test_unresolver.py
index c6bdc44e8b7..f95d72cb731 100644
--- a/readthedocs/rtd_tests/tests/test_unresolver.py
+++ b/readthedocs/rtd_tests/tests/test_unresolver.py
@@ -8,7 +8,11 @@
 from readthedocs.core.unresolver import (
     InvalidCustomDomainError,
     InvalidExternalDomainError,
+    InvalidExternalVersionError,
     SuspiciousHostnameError,
+    TranslationNotFoundError,
+    UnresolvedPathError,
+    VersionNotFoundError,
     unresolve,
 )
 from readthedocs.projects.models import Domain, Project
@@ -63,17 +67,44 @@ def test_file_with_trailing_slash(self):
         self.assertEqual(parts.version, self.version)
         self.assertEqual(parts.filename, "/foo/index.html")
 
+    def test_project_no_version_and_language(self):
+        with pytest.raises(UnresolvedPathError) as excinfo:
+            unresolve("https://pip.readthedocs.io/")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.path, "/")
+
+        with pytest.raises(UnresolvedPathError) as excinfo:
+            unresolve("https://pip.readthedocs.io/foo/bar")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.path, "/foo/bar")
+
+    def test_subproject_no_version_and_language(self):
+        with pytest.raises(UnresolvedPathError) as excinfo:
+            unresolve("https://pip.readthedocs.io/projects/sub/")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.subproject)
+        self.assertEqual(exc.path, "/")
+
+        with pytest.raises(UnresolvedPathError) as excinfo:
+            unresolve("https://pip.readthedocs.io/projects/sub/foo/bar")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.subproject)
+        self.assertEqual(exc.path, "/foo/bar")
+
     def test_path_no_version(self):
         urls = [
             "https://pip.readthedocs.io/en",
             "https://pip.readthedocs.io/en/",
         ]
         for url in urls:
-            parts = unresolve(url)
-            self.assertEqual(parts.parent_project, self.pip)
-            self.assertEqual(parts.project, self.pip)
-            self.assertEqual(parts.version, None)
-            self.assertEqual(parts.filename, None)
+            with pytest.raises(VersionNotFoundError) as excinfo:
+                unresolve(url)
+            exc = excinfo.value
+            self.assertEqual(exc.project, self.pip)
+            self.assertEqual(exc.version_slug, None)
+            self.assertEqual(exc.filename, "/")
 
     def test_unresolver_subproject(self):
         parts = unresolve("https://pip.readthedocs.io/projects/sub/ja/latest/foo.html")
@@ -116,18 +147,22 @@ def test_unresolve_subproject_alias(self):
         self.assertEqual(parts.filename, "/index.html")
 
     def test_unresolve_subproject_invalid_version(self):
-        parts = unresolve("https://pip.readthedocs.io/projects/sub/ja/nothing/foo.html")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.subproject)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+        with pytest.raises(VersionNotFoundError) as excinfo:
+            unresolve("https://pip.readthedocs.io/projects/sub/ja/nothing/foo.html")
+        exc = excinfo.value
+
+        self.assertEqual(exc.project, self.subproject)
+        self.assertEqual(exc.version_slug, "nothing")
+        self.assertEqual(exc.filename, "/foo.html")
 
     def test_unresolve_subproject_invalid_translation(self):
-        parts = unresolve("https://pip.readthedocs.io/projects/sub/es/latest/foo.html")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.subproject)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+        with pytest.raises(TranslationNotFoundError) as excinfo:
+            unresolve("https://pip.readthedocs.io/projects/sub/es/latest/foo.html")
+
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.subproject)
+        self.assertEqual(exc.language, "es")
+        self.assertEqual(exc.filename, "/foo.html")
 
     def test_unresolver_translation(self):
         parts = unresolve("https://pip.readthedocs.io/ja/latest/foo.html")
@@ -137,11 +172,12 @@ def test_unresolver_translation(self):
         self.assertEqual(parts.filename, "/foo.html")
 
     def test_unresolve_no_existing_translation(self):
-        parts = unresolve("https://pip.readthedocs.io/es/latest/")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.pip)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+        with pytest.raises(TranslationNotFoundError) as excinfo:
+            unresolve("https://pip.readthedocs.io/es/latest/")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.language, "es")
+        self.assertEqual(exc.filename, "/")
 
     def test_unresolver_custom_domain(self):
         self.domain = fixture.get(
@@ -229,41 +265,49 @@ def test_external_version_single_version_project(self):
     def test_unexisting_external_version_single_version_project(self):
         self.pip.single_version = True
         self.pip.save()
-        parts = unresolve("https://pip--10.dev.readthedocs.build/")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.pip)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+        with pytest.raises(VersionNotFoundError) as excinfo:
+            unresolve("https://pip--10.dev.readthedocs.build/")
+
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.version_slug, "10")
+        self.assertEqual(exc.filename, "/")
 
     def test_non_external_version_single_version_project(self):
         self.pip.single_version = True
         self.pip.save()
-        parts = unresolve("https://pip--latest.dev.readthedocs.build/")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.pip)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+
+        with pytest.raises(VersionNotFoundError) as excinfo:
+            unresolve("https://pip--latest.dev.readthedocs.build/")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.version_slug, "latest")
+        self.assertEqual(exc.filename, "/")
 
     def test_unresolve_external_version_no_existing_version(self):
-        parts = unresolve("https://pip--10.dev.readthedocs.build/en/10/")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.pip)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+        with pytest.raises(VersionNotFoundError) as excinfo:
+            unresolve("https://pip--10.dev.readthedocs.build/en/10/")
+
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.version_slug, "10")
+        self.assertEqual(exc.filename, "/")
 
     def test_external_version_not_matching_final_version(self):
-        parts = unresolve("https://pip--10.dev.readthedocs.build/en/latest/")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.pip)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+        with pytest.raises(InvalidExternalVersionError) as excinfo:
+            unresolve("https://pip--10.dev.readthedocs.build/en/latest/")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.version, self.version)
+        self.assertEqual(exc.external_version_slug, "10")
 
     def test_normal_version_in_external_version_subdomain(self):
-        parts = unresolve("https://pip--latest.dev.readthedocs.build/en/latest/")
-        self.assertEqual(parts.parent_project, self.pip)
-        self.assertEqual(parts.project, self.pip)
-        self.assertEqual(parts.version, None)
-        self.assertEqual(parts.filename, None)
+        with pytest.raises(InvalidExternalVersionError) as excinfo:
+            unresolve("https://pip--latest.dev.readthedocs.build/en/latest/")
+        exc = excinfo.value
+        self.assertEqual(exc.project, self.pip)
+        self.assertEqual(exc.external_version_slug, "latest")
+        self.assertEqual(exc.version, self.version)
 
     def test_malformed_external_version(self):
         with pytest.raises(InvalidExternalDomainError):

From 14bdbd25144f1349c3b7f46c315368c40b3d7a77 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Tue, 21 Feb 2023 14:45:45 -0500
Subject: [PATCH 12/70] Move external version checks inside unresolver

---
 readthedocs/core/unresolver.py                | 59 +++++++------------
 .../rtd_tests/tests/test_unresolver.py        |  8 +--
 2 files changed, 25 insertions(+), 42 deletions(-)

diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py
index f1361e32ef6..7e909d8ead3 100644
--- a/readthedocs/core/unresolver.py
+++ b/readthedocs/core/unresolver.py
@@ -6,7 +6,7 @@
 import structlog
 from django.conf import settings
 
-from readthedocs.builds.constants import EXTERNAL
+from readthedocs.builds.constants import EXTERNAL, INTERNAL
 from readthedocs.builds.models import Version
 from readthedocs.constants import pattern_opts
 from readthedocs.projects.models import Domain, Feature, Project
@@ -65,9 +65,9 @@ def __init__(self, project, path):
 
 
 class InvalidExternalVersionError(UnresolverError):
-    def __init__(self, project, version, external_version_slug):
+    def __init__(self, project, version_slug, external_version_slug):
         self.project = project
-        self.version = version
+        self.version_slug = version_slug
         self.external_version_slug = external_version_slug
 
 
@@ -216,33 +216,6 @@ def _unresolve(self, unresolved_domain, parsed_url, append_indexhtml):
             external_version_slug=unresolved_domain.external_version_slug,
         )
 
-        # Make sure we are serving the external version from the subdomain.
-        if unresolved_domain.is_from_external_domain:
-            domain = unresolved_domain.source_domain
-            if unresolved_domain.external_version_slug != version.slug:
-                log.warning(
-                    "Invalid version for external domain.",
-                    domain=domain,
-                    version_slug=version.slug,
-                )
-                raise InvalidExternalVersionError(
-                    project=current_project,
-                    version=version,
-                    external_version_slug=unresolved_domain.external_version_slug,
-                )
-            if not version.is_external:
-                log.warning(
-                    "Attempt of serving a non-external version from RTD_EXTERNAL_VERSION_DOMAIN.",
-                    domain=domain,
-                    version_slug=version.slug,
-                    version_type=version.type,
-                )
-                raise InvalidExternalVersionError(
-                    project=current_project,
-                    version=version,
-                    external_version_slug=unresolved_domain.external_version_slug,
-                )
-
         if append_indexhtml and filename.endswith("/"):
             filename += "index.html"
 
@@ -264,7 +237,9 @@ def _normalize_filename(filename):
             filename = "/" + filename
         return filename
 
-    def _match_multiversion_project(self, parent_project, path):
+    def _match_multiversion_project(
+        self, parent_project, path, external_version_slug=None
+    ):
         """
         Try to match a multiversion project.
 
@@ -290,7 +265,15 @@ def _match_multiversion_project(self, parent_project, path):
                     project=parent_project, language=language, filename=file
                 )
 
-        version = project.versions.filter(slug=version_slug).first()
+        if external_version_slug and external_version_slug != version_slug:
+            raise InvalidExternalVersionError(
+                project=project,
+                version_slug=version_slug,
+                external_version_slug=external_version_slug,
+            )
+
+        manager = EXTERNAL if external_version_slug else INTERNAL
+        version = project.versions(manager=manager).filter(slug=version_slug).first()
         if not version:
             raise VersionNotFoundError(
                 project=project, version_slug=version_slug, filename=file
@@ -348,15 +331,14 @@ def _match_single_version_project(
         file = self._normalize_filename(path)
         if external_version_slug:
             version_slug = external_version_slug
-            version = (
-                parent_project.versions(manager=EXTERNAL)
-                .filter(slug=version_slug)
-                .first()
-            )
+            manager = EXTERNAL
         else:
             version_slug = parent_project.default_version
-            version = parent_project.versions.filter(slug=version_slug).first()
+            manager = INTERNAL
 
+        version = (
+            parent_project.versions(manager=manager).filter(slug=version_slug).first()
+        )
         if not version:
             raise VersionNotFoundError(
                 project=parent_project, version_slug=version_slug, filename=file
@@ -400,6 +382,7 @@ def _unresolve_path(
             response = self._match_multiversion_project(
                 parent_project=parent_project,
                 path=path,
+                external_version_slug=external_version_slug,
             )
             if response:
                 return response
diff --git a/readthedocs/rtd_tests/tests/test_unresolver.py b/readthedocs/rtd_tests/tests/test_unresolver.py
index f95d72cb731..482806fdf01 100644
--- a/readthedocs/rtd_tests/tests/test_unresolver.py
+++ b/readthedocs/rtd_tests/tests/test_unresolver.py
@@ -298,16 +298,16 @@ def test_external_version_not_matching_final_version(self):
             unresolve("https://pip--10.dev.readthedocs.build/en/latest/")
         exc = excinfo.value
         self.assertEqual(exc.project, self.pip)
-        self.assertEqual(exc.version, self.version)
+        self.assertEqual(exc.version_slug, "latest")
         self.assertEqual(exc.external_version_slug, "10")
 
     def test_normal_version_in_external_version_subdomain(self):
-        with pytest.raises(InvalidExternalVersionError) as excinfo:
+        with pytest.raises(VersionNotFoundError) as excinfo:
             unresolve("https://pip--latest.dev.readthedocs.build/en/latest/")
         exc = excinfo.value
         self.assertEqual(exc.project, self.pip)
-        self.assertEqual(exc.external_version_slug, "latest")
-        self.assertEqual(exc.version, self.version)
+        self.assertEqual(exc.version_slug, "latest")
+        self.assertEqual(exc.filename, "/")
 
     def test_malformed_external_version(self):
         with pytest.raises(InvalidExternalDomainError):

From dd4363153c8f2b67a3f0032478a368223eeccc27 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 22 Feb 2023 14:13:17 +0100
Subject: [PATCH 13/70] !fixup 8cbff084845a9a7ad0fd660f96be8bab245ec171

---
 readthedocs/settings/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
index 97202bb6d2f..99827f9f687 100644
--- a/readthedocs/settings/base.py
+++ b/readthedocs/settings/base.py
@@ -75,7 +75,7 @@ def SHOW_DEBUG_TOOLBAR(self):
     PRODUCTION_DOMAIN = 'readthedocs.org'
     PUBLIC_DOMAIN = None
     PUBLIC_DOMAIN_USES_HTTPS = False
-    USE_SUBDOMAIN = True
+    USE_SUBDOMAIN = False
     PUBLIC_API_URL = 'https://{}'.format(PRODUCTION_DOMAIN)
     RTD_INTERSPHINX_URL = 'https://{}'.format(PRODUCTION_DOMAIN)
     RTD_EXTERNAL_VERSION_DOMAIN = 'external-builds.readthedocs.io'

From 0c07025aaeb6fd4b6ef590272c9693b60c9c6f0f Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 22 Feb 2023 14:51:43 +0100
Subject: [PATCH 14/70] Update link to Custom 404 docs, remove rediraffe from
 tips

---
 readthedocs/templates/errors/404/include_tips.html | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/readthedocs/templates/errors/404/include_tips.html b/readthedocs/templates/errors/404/include_tips.html
index ee42ee8b74f..e6d1b3809f9 100644
--- a/readthedocs/templates/errors/404/include_tips.html
+++ b/readthedocs/templates/errors/404/include_tips.html
@@ -5,7 +5,6 @@ 
{% trans "Are you the project owner?" %}

 
{% trans "Here are some tips to address 404 errors:" %}

 
 

From 9367e9edc2dcc04f81d9c89c893e046b8a48d56b Mon Sep 17 00:00:00 2001
From: Benjamin Balder Bach 
Date: Wed, 22 Feb 2023 14:57:23 +0100
Subject: [PATCH 15/70] Apply suggestions from code review

Co-authored-by: Manuel Kaufmann 
---
 readthedocs/templates/errors/404/community.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/templates/errors/404/community.html b/readthedocs/templates/errors/404/community.html
index 81774812422..22c08f3ddf0 100644
--- a/readthedocs/templates/errors/404/community.html
+++ b/readthedocs/templates/errors/404/community.html
@@ -1,7 +1,7 @@
 {% extends "errors/404/base.html" %}
 {% load i18n %}
 
-{# something on readthedocs.org does not exist - this 404 is not intended for projects #}
+{# This page on the application site does not exist. This 404 is not intended for user hosted projects. #}
 
 {% block 404_error_message %}
 

From d38491e4e0bfc581c35b4b7364f31213f3757cc7 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 22 Feb 2023 15:05:13 +0100
Subject: [PATCH 16/70] Raise custom 404 if subproject isn't found

---
 readthedocs/proxito/views/serve.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index cc546350d0f..29280ebbcc0 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -5,7 +5,7 @@
 import structlog
 from django.conf import settings
 from django.http import Http404, HttpResponse, HttpResponseRedirect
-from django.shortcuts import get_object_or_404, render
+from django.shortcuts import render
 from django.urls import resolve as url_resolve
 from django.utils.decorators import method_decorator
 from django.views import View
@@ -19,10 +19,14 @@
 from readthedocs.core.resolver import resolve_path
 from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.projects import constants
-from readthedocs.projects.models import Domain, Feature
+from readthedocs.projects.models import Domain, Feature, ProjectRelationship
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 from readthedocs.proxito.constants import RedirectType
-from readthedocs.proxito.exceptions import ProxitoHttp404, ProxitoProjectPageHttp404
+from readthedocs.proxito.exceptions import (
+    ProxitoHttp404,
+    ProxitoProjectPageHttp404,
+    ProxitoSubProjectHttp404,
+)
 from readthedocs.redirects.exceptions import InfiniteRedirectException
 from readthedocs.storage import build_media_storage
 
@@ -44,11 +48,13 @@ def get(self, request, subproject_slug=None, filename=""):
         project = unresolved_domain.project
 
         # Use the project from the domain, or use the subproject slug.
-        # TODO: Do not raise a normal Http404 here, use custom...
         if subproject_slug:
-            project = get_object_or_404(
-                project.subprojects, alias=subproject_slug
-            ).child
+            try:
+                project = project.subprojects.get(alias=subproject_slug).child
+            except ProjectRelationship.DoesNotExist:
+                raise ProxitoSubProjectHttp404(
+                    f"Did not find subproject slug {subproject_slug} for project {project.slug}"
+                )
 
         # Get the default version from the current project,
         # or the version from the external domain.

From 7f6f1b70f03b2b175643c613f29b8abf555219ef Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 22 Feb 2023 16:55:24 +0100
Subject: [PATCH 17/70] WIP: Conclude structure of exception classes and
 templates, needs more content work

---
 media/css/core.css                            |  2 +
 readthedocs/core/views/__init__.py            | 68 ++++++++++++-------
 readthedocs/proxito/exceptions.py             | 40 +++++++----
 readthedocs/proxito/views/decorators.py       | 12 ++--
 readthedocs/proxito/views/serve.py            |  6 +-
 readthedocs/proxito/views/utils.py            | 32 +++++++--
 readthedocs/templates/errors/404/base.html    | 21 +++---
 .../templates/errors/404/community.html       | 12 ----
 .../templates/errors/404/include_tips.html    |  6 +-
 .../templates/errors/404/no_project.html      |  4 +-
 .../templates/errors/404/no_project_page.html |  5 +-
 .../templates/errors/404/no_subproject.html   |  7 +-
 .../templates/errors/404/no_version.html      | 23 +++++++
 13 files changed, 155 insertions(+), 83 deletions(-)
 delete mode 100644 readthedocs/templates/errors/404/community.html
 create mode 100644 readthedocs/templates/errors/404/no_version.html

diff --git a/media/css/core.css b/media/css/core.css
index 7d40d19b3dc..a41dd9c67c3 100644
--- a/media/css/core.css
+++ b/media/css/core.css
@@ -1331,6 +1331,8 @@ div.module.project-subprojects li.subproject a.subproject-edit:before {
     content: "\f044";
 }
 
+#content ul.normal_list {list-style: disc; margin-left: 20px;}
+
 /* Pygments */
 div.highlight pre .hll { background-color: #ffffcc }
 div.highlight pre .c { color: #60a0b0; font-style: italic } /* Comment */
diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 3eb85bdd1b4..1f70cd70696 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -13,6 +13,12 @@
 
 from readthedocs.core.mixins import PrivateViewMixin
 from readthedocs.projects.models import Project
+from readthedocs.proxito.exceptions import (
+    ProxitoProjectHttp404,
+    ProxitoProjectPageHttp404,
+    ProxitoProjectVersionHttp404,
+    ProxitoSubProjectHttp404,
+)
 
 log = structlog.get_logger(__name__)
 
@@ -48,32 +54,42 @@ def get_context_data(self, **kwargs):
         return context
 
 
-def server_error_404(
-    request, template_name="errors/404/community.html", exception=None
-):
-    """A simple 404 handler."""
-    # This property is set by ProxitoHttp404. We could also have a look at the
-    # subproject_slug
-
-    project_slug = getattr(exception, "project_slug", None)
-    log.debug(exception)
-    project = getattr(exception, "project", None)
-    subproject_slug = getattr(exception, "subproject_slug", None)
-
-    if subproject_slug:
-        template_name = "errors/404/no_subproject.html"
-    elif project:
-        template_name = "errors/404/no_project_page.html"
-    elif project_slug:
-        template_name = "errors/404/no_project.html"
-
-    r = render(
-        request,
-        template_name,
-        context={"project": project, "project_slug": project_slug},
-    )
-    r.status_code = 404
-    return r
+def server_error_404(request, template_name="errors/404/base.html", exception=None):
+    """Generic 404 handler, handling 404 exception types raised throughout the app"""
+
+    try:
+        log.debug("404 view handler active")
+        log.debug(exception)
+        # Properties are set by ProxitoHttp404. We could also have a look at the
+        # subproject_slug
+        project_slug = getattr(exception, "project_slug", None)
+        version_slug = getattr(exception, "version_slug", None)
+        project = getattr(exception, "project", None)
+        subproject_slug = getattr(exception, "subproject_slug", None)
+
+        if isinstance(exception, ProxitoProjectVersionHttp404):
+            template_name = "errors/404/no_version.html"
+        elif isinstance(exception, ProxitoSubProjectHttp404):
+            template_name = "errors/404/no_subproject.html"
+        elif isinstance(exception, ProxitoProjectPageHttp404):
+            template_name = "errors/404/no_project_page.html"
+        elif isinstance(exception, ProxitoProjectHttp404):
+            template_name = "errors/404/no_project.html"
+
+        r = render(
+            request,
+            template_name,
+            context={
+                "project": project,
+                "project_slug": project_slug,
+                "subproject_slug": subproject_slug,
+                "version_slug": version_slug,
+            },
+        )
+        r.status_code = 404
+        return r
+    except Exception as e:
+        log.debug(e)
 
 
 def server_error_500(request, template_name='500.html'):
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index c66c342f4e5..d095ec6208d 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -14,16 +14,13 @@
 
 class ProxitoHttp404(Http404):
     """
-    General error class for proxity 404s
+    General error class for proxity 404s.
     """
 
-    def __init__(self, message):
-        super().__init__(message)
-
 
 class ProxitoProjectHttp404(ProxitoHttp404):
     """
-    Raised if a project was not found
+    Raised if a project was not found.
     """
 
     def __init__(self, message, project_slug=None):
@@ -33,24 +30,41 @@ def __init__(self, message, project_slug=None):
 
 class ProxitoSubProjectHttp404(ProxitoProjectHttp404):
     """
-    Raised if a sub-project was not found
+    Raised if a sub-project was not found.
     """
     def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
-        self.project_slug = project_slug
+        super().__init__(message, project_slug=project_slug)
         self.project = project
         self.subproject_slug = subproject_slug
-        super().__init__(message)
 
 
 class ProxitoProjectPageHttp404(ProxitoHttp404):
     """
-    Raised if a page inside an existing project was not found
-
-    Note: The containing project can be both a project or a subproject inside another project
+    Raised if a page inside an existing project was not found.
     """
 
     def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
-        self.project_slug = project_slug
+        super().__init__(message, project_slug=project_slug)
         self.project = project
         self.subproject_slug = subproject_slug
-        super().__init__(message)
+
+
+class ProxitoProjectVersionHttp404(ProxitoHttp404):
+    """
+    Raised if a version was not found.
+
+    Note: The containing project can be both a project or a subproject inside another project.
+    """
+
+    def __init__(
+        self,
+        message,
+        project_slug=None,
+        project=None,
+        subproject_slug=None,
+        version_slug=None,
+    ):
+        super().__init__(message, project_slug=project_slug)
+        self.project = project
+        self.subproject_slug = subproject_slug
+        self.version_slug = version_slug
diff --git a/readthedocs/proxito/views/decorators.py b/readthedocs/proxito/views/decorators.py
index 59ca4030b55..73741882e94 100644
--- a/readthedocs/proxito/views/decorators.py
+++ b/readthedocs/proxito/views/decorators.py
@@ -23,14 +23,14 @@ def map_subproject_slug(view_func):
 
     @wraps(view_func)
     def inner_view(  # noqa
-            request, subproject=None, subproject_slug=None, *args, **kwargs
+        request, project=None, subproject=None, subproject_slug=None, *args, **kwargs
     ):
         if subproject is None and subproject_slug:
             # Try to fetch by subproject alias first, otherwise we might end up
             # redirected to an unrelated project.
             # Depends on a project passed into kwargs
             rel = (
-                ProjectRelationship.objects.filter(parent=kwargs["project"])
+                ProjectRelationship.objects.filter(parent=project)
                 .filter(Q(alias=subproject_slug) | Q(child__slug=subproject_slug))
                 .first()
             )
@@ -40,10 +40,14 @@ def inner_view(  # noqa
                 log.warning(
                     "The slug is not subproject of project.",
                     subproject_slug=subproject_slug,
-                    project_slug=kwargs["project"].slug,
+                    project_slug=project.slug,
+                    project=project,
                 )
                 raise ProxitoSubProjectHttp404(
-                    "Invalid subproject slug", subproject_slug=subproject_slug
+                    "Invalid subproject slug",
+                    project_slug=project.slug,
+                    project=project,
+                    subproject_slug=subproject_slug,
                 )
         return view_func(request, subproject=subproject, *args, **kwargs)
 
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 29280ebbcc0..da4ce1f4124 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -46,6 +46,7 @@ def get(self, request, subproject_slug=None, filename=""):
 
         unresolved_domain = request.unresolved_domain
         project = unresolved_domain.project
+        parent_project = project
 
         # Use the project from the domain, or use the subproject slug.
         if subproject_slug:
@@ -53,7 +54,10 @@ def get(self, request, subproject_slug=None, filename=""):
                 project = project.subprojects.get(alias=subproject_slug).child
             except ProjectRelationship.DoesNotExist:
                 raise ProxitoSubProjectHttp404(
-                    f"Did not find subproject slug {subproject_slug} for project {project.slug}"
+                    f"Did not find subproject slug {subproject_slug} for project {project.slug}",
+                    project=parent_project,
+                    project_slug=parent_project.slug,
+                    subproject_slug=subproject_slug,
                 )
 
         # Get the default version from the current project,
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 84b14435686..39abed59954 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -5,7 +5,13 @@
 from django.shortcuts import render
 
 from readthedocs.projects.models import Project
-from readthedocs.proxito.exceptions import ProxitoHttp404
+from readthedocs.proxito.exceptions import (
+    ProxitoHttp404,
+    ProxitoProjectHttp404,
+    ProxitoProjectPageHttp404,
+    ProxitoProjectVersionHttp404,
+    ProxitoSubProjectHttp404,
+)
 
 from .decorators import map_project_slug, map_subproject_slug
 
@@ -22,7 +28,9 @@ def fast_404(request, *args, **kwargs):
     return HttpResponse("Not Found.", status=404)
 
 
-def proxito_404_page_handler(request, exception=None, template_name='404.html'):
+def proxito_404_page_handler(
+    request, exception=None, template_name="errors/404/base.html"
+):
     """
     Decide what 404 page return depending if it's an internal NGINX redirect.
 
@@ -36,9 +44,20 @@ def proxito_404_page_handler(request, exception=None, template_name='404.html'):
         log.debug("Displaying a 'fast 404'")
         return fast_404(request, exception, template_name)
 
+    if isinstance(exception, ProxitoProjectVersionHttp404):
+        template_name = "errors/404/no_version.html"
+    elif isinstance(exception, ProxitoSubProjectHttp404):
+        template_name = "errors/404/no_subproject.html"
+    elif isinstance(exception, ProxitoProjectPageHttp404):
+        template_name = "errors/404/no_project_page.html"
+    elif isinstance(exception, ProxitoProjectHttp404):
+        template_name = "errors/404/no_project.html"
+
     project_slug = getattr(exception, "project_slug", None)
-    log.debug(exception)
+    version_slug = getattr(exception, "version_slug", None)
     project = getattr(exception, "project", None)
+    subproject_slug = getattr(exception, "subproject_slug", None)
+
     log.debug(
         "404 page detected a project slug in request.",
         project_slug=project_slug,
@@ -46,7 +65,12 @@ def proxito_404_page_handler(request, exception=None, template_name='404.html'):
     r = render(
         request,
         template_name,
-        context={"project": project, "project_slug": project_slug},
+        context={
+            "project": project,
+            "project_slug": project_slug,
+            "subproject_slug": subproject_slug,
+            "version_slug": version_slug,
+        },
     )
     r.status_code = 404
     return r
diff --git a/readthedocs/templates/errors/404/base.html b/readthedocs/templates/errors/404/base.html
index b82c0e9ff37..a82af8670ae 100644
--- a/readthedocs/templates/errors/404/base.html
+++ b/readthedocs/templates/errors/404/base.html
@@ -3,31 +3,23 @@
 {% load i18n %}
 
 {% block title %}
-  {% trans "404 - Maze Found" %}
+  {% trans "404 Not Found" %}
 {% endblock %}
 
 {% block content %}
 
-
{% trans "404 - page not found" %}

+
{% trans "404 Not Found" %}

 
 {% block 404_error_message %}
 
-{% endblock %}
-
-{% if project %}
-
-{% elif "_proxito_404_" in request.path %}
-
-{% else %}
-
-{% endif %}
+  
{% trans "You have encountered a 404 on Read the Docs. You are either looking for a page that does not exist or a project that has been removed." %}

 
 
 
-        \          SORRY            /
+        \       What a maze!         /
          \                         /
           \    This page does     /
-           ]   not exist yet.    [    ,'|
+           ]     not exist.      [    ,'|
            ]                     [   /  |
            ]___               ___[ ,'   |
            ]  ]\             /[  [ |:   |
@@ -47,4 +39,7 @@ 
{% trans "404 - page not found" %}

          /           q,            \
         /                           \
 

+
+{% endblock %}
+
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/community.html b/readthedocs/templates/errors/404/community.html
deleted file mode 100644
index 22c08f3ddf0..00000000000
--- a/readthedocs/templates/errors/404/community.html
+++ /dev/null
@@ -1,12 +0,0 @@
-{% extends "errors/404/base.html" %}
-{% load i18n %}
-
-{# This page on the application site does not exist. This 404 is not intended for user hosted projects. #}
-
-{% block 404_error_message %}
-
-  
{% trans "You have encountered a 404 on Read the Docs. You are either looking for a page that does not exist or a project that has been removed." %}

-
-  
{% trans "If you believe this is a technical error, please contact our support:" %} {% trans "Go to support" %}»

-
-{% endblock %}
diff --git a/readthedocs/templates/errors/404/include_tips.html b/readthedocs/templates/errors/404/include_tips.html
index e6d1b3809f9..78017e3a3c4 100644
--- a/readthedocs/templates/errors/404/include_tips.html
+++ b/readthedocs/templates/errors/404/include_tips.html
@@ -4,7 +4,7 @@ 
{% trans "Are you the project owner?" %}

 
 
{% trans "Here are some tips to address 404 errors:" %}

 
-
    
-  
  • {% trans "Use custom 404 pages:" %} {% trans "Read more" %}»
    • 
-  
  • {% trans "Create redirects when you move contents:" %} {% trans "Read more" %}»
    • 
+
diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
index f3ff62966ed..31f627142cd 100644
--- a/readthedocs/templates/errors/404/no_project.html
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -8,8 +8,8 @@
   {# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #}
 
   
    
-    {% blocktrans trimmed with project_slug=project_slug %}
-      The project "{{ project_slug }}" that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
+    {% blocktrans trimmed %}
+      The project {{ project_slug }} that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
     {% endblocktrans %}
   
    
 
diff --git a/readthedocs/templates/errors/404/no_project_page.html b/readthedocs/templates/errors/404/no_project_page.html
index bce6d1042bd..77da321ceb2 100644
--- a/readthedocs/templates/errors/404/no_project_page.html
+++ b/readthedocs/templates/errors/404/no_project_page.html
@@ -6,8 +6,9 @@
 {% block 404_error_message %}
 
   
    
-    {% blocktrans trimmed with project_slug=project.slug request_path=request.path %}
-      You are browsing the documentation of "{{ project_slug }}". The page you are looking for at {{ request.path }} was not found.
+    {% blocktrans trimmed with request_path=request.path %}
+      You are browsing the documentation of {{ project_slug }}.
+      The page you are looking for at {{ request_path }} was not found.
     {% endblocktrans %}
   
    
 
diff --git a/readthedocs/templates/errors/404/no_subproject.html b/readthedocs/templates/errors/404/no_subproject.html
index bce6d1042bd..1bd2dae4f9b 100644
--- a/readthedocs/templates/errors/404/no_subproject.html
+++ b/readthedocs/templates/errors/404/no_subproject.html
@@ -1,13 +1,14 @@
 {% extends "errors/404/base.html" %}
 {% load i18n %}
 
-{# project exists but page doesn't #}
+{# project exists but subproject doesn't #}
 
 {% block 404_error_message %}
 
   
    
-    {% blocktrans trimmed with project_slug=project.slug request_path=request.path %}
-      You are browsing the documentation of "{{ project_slug }}". The page you are looking for at {{ request.path }} was not found.
+    {% blocktrans trimmed with request_path=request.path %}
+      You are browsing the documentation of project {{ project_slug }}.
+      The subproject {{ subproject_slug }} you are looking for at {{ request_path }} was not found.
     {% endblocktrans %}
   
    
 
diff --git a/readthedocs/templates/errors/404/no_version.html b/readthedocs/templates/errors/404/no_version.html
new file mode 100644
index 00000000000..7e5696ff8dd
--- /dev/null
+++ b/readthedocs/templates/errors/404/no_version.html
@@ -0,0 +1,23 @@
+{% extends "errors/404/base.html" %}
+{% load i18n %}
+
+{# project exists but version doesn't #}
+
+{% block 404_error_message %}
+
+  
    
+    {% blocktrans trimmed with request_path=request.path %}
+      You are browsing the documentation of project {{ project_slug }}.
+      The version you are looking for at {{ request_path }} was not found.
+    {% endblocktrans %}
+  
    
+
+  
    
+    {% blocktrans trimmed %}
+      Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
+    {% endblocktrans %}
+  
    
+
+  {% include "errors/404/include_tips.html" %}
+
+{% endblock %}

From 36b9f02820ba79380b516acb7a28a3d87723670b Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Wed, 22 Feb 2023 17:11:42 -0500
Subject: [PATCH 18/70] Updates from review

---
 readthedocs/core/unresolver.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py
index 7e909d8ead3..52bea1a902e 100644
--- a/readthedocs/core/unresolver.py
+++ b/readthedocs/core/unresolver.py
@@ -107,6 +107,7 @@ class UnresolvedDomain:
     source_domain: str
     source: DomainSourceType
     project: Project
+    # Domain object for custom domains.
     domain: Domain = None
     external_version_slug: str = None
 
@@ -247,6 +248,7 @@ def _match_multiversion_project(
         this exception has the current project (useful for 404 pages).
 
         :returns: A tuple with the current project, version and file.
+         Returns `None` if there isn't a total or partial match.
         """
         match = self.multiversion_pattern.match(path)
         if not match:
@@ -289,6 +291,7 @@ def _match_subproject(self, parent_project, path, external_version_slug=None):
         with the subproject as the canonical project.
 
         :returns: A tuple with the current project, version and file.
+         Returns `None` if there isn't a total or partial match.
         """
         match = self.subproject_pattern.match(path)
         if not match:
@@ -327,6 +330,7 @@ def _match_single_version_project(
         this exception has the current project (useful for 404 pages).
 
         :returns: A tuple with the current project, version and file.
+         Returns `None` if there isn't a total or partial match.
         """
         file = self._normalize_filename(path)
         if external_version_slug:

From a4dee70d0b1d83df41190a08d46b7242b8708818 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 12:52:22 +0100
Subject: [PATCH 19/70] Reuse 404 view between proxito and general app, remove
 debug logging

---
 readthedocs/core/views/__init__.py            | 69 +++++++++----------
 readthedocs/proxito/views/utils.py            | 40 ++---------
 readthedocs/templates/errors/404/base.html    |  6 +-
 .../templates/errors/404/no_project.html      |  2 +-
 4 files changed, 42 insertions(+), 75 deletions(-)

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 1f70cd70696..87429afb4ac 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -55,41 +55,40 @@ def get_context_data(self, **kwargs):
 
 
 def server_error_404(request, template_name="errors/404/base.html", exception=None):
-    """Generic 404 handler, handling 404 exception types raised throughout the app"""
-
-    try:
-        log.debug("404 view handler active")
-        log.debug(exception)
-        # Properties are set by ProxitoHttp404. We could also have a look at the
-        # subproject_slug
-        project_slug = getattr(exception, "project_slug", None)
-        version_slug = getattr(exception, "version_slug", None)
-        project = getattr(exception, "project", None)
-        subproject_slug = getattr(exception, "subproject_slug", None)
-
-        if isinstance(exception, ProxitoProjectVersionHttp404):
-            template_name = "errors/404/no_version.html"
-        elif isinstance(exception, ProxitoSubProjectHttp404):
-            template_name = "errors/404/no_subproject.html"
-        elif isinstance(exception, ProxitoProjectPageHttp404):
-            template_name = "errors/404/no_project_page.html"
-        elif isinstance(exception, ProxitoProjectHttp404):
-            template_name = "errors/404/no_project.html"
-
-        r = render(
-            request,
-            template_name,
-            context={
-                "project": project,
-                "project_slug": project_slug,
-                "subproject_slug": subproject_slug,
-                "version_slug": version_slug,
-            },
-        )
-        r.status_code = 404
-        return r
-    except Exception as e:
-        log.debug(e)
+    """
+    Serves a 404 error message, handling 404 exception types raised throughout the app.
+    Notice that handling of 404 errors happens elsewhere in views and middleware,
+    this view is expected to serve an actual 404 message.
+    """
+
+    # Properties are set by ProxitoHttp404. We could also have a look at the
+    # subproject_slug
+    project_slug = getattr(exception, "project_slug", None)
+    version_slug = getattr(exception, "version_slug", None)
+    project = getattr(exception, "project", None)
+    subproject_slug = getattr(exception, "subproject_slug", None)
+
+    if isinstance(exception, ProxitoProjectVersionHttp404):
+        template_name = "errors/404/no_version.html"
+    elif isinstance(exception, ProxitoSubProjectHttp404):
+        template_name = "errors/404/no_subproject.html"
+    elif isinstance(exception, ProxitoProjectPageHttp404):
+        template_name = "errors/404/no_project_page.html"
+    elif isinstance(exception, ProxitoProjectHttp404):
+        template_name = "errors/404/no_project.html"
+
+    r = render(
+        request,
+        template_name,
+        context={
+            "project": project,
+            "project_slug": project_slug,
+            "subproject_slug": subproject_slug,
+            "version_slug": version_slug,
+        },
+    )
+    r.status_code = 404
+    return r
 
 
 def server_error_500(request, template_name='500.html'):
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 39abed59954..7a31199a621 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -2,7 +2,6 @@
 
 import structlog
 from django.http import HttpResponse
-from django.shortcuts import render
 
 from readthedocs.projects.models import Project
 from readthedocs.proxito.exceptions import (
@@ -13,6 +12,7 @@
     ProxitoSubProjectHttp404,
 )
 
+from ...core.views import server_error_404
 from .decorators import map_project_slug, map_subproject_slug
 
 log = structlog.get_logger(__name__)  # noqa
@@ -41,39 +41,12 @@ def proxito_404_page_handler(
     """
 
     if request.resolver_match and request.resolver_match.url_name != 'proxito_404_handler':
-        log.debug("Displaying a 'fast 404'")
         return fast_404(request, exception, template_name)
 
-    if isinstance(exception, ProxitoProjectVersionHttp404):
-        template_name = "errors/404/no_version.html"
-    elif isinstance(exception, ProxitoSubProjectHttp404):
-        template_name = "errors/404/no_subproject.html"
-    elif isinstance(exception, ProxitoProjectPageHttp404):
-        template_name = "errors/404/no_project_page.html"
-    elif isinstance(exception, ProxitoProjectHttp404):
-        template_name = "errors/404/no_project.html"
-
-    project_slug = getattr(exception, "project_slug", None)
-    version_slug = getattr(exception, "version_slug", None)
-    project = getattr(exception, "project", None)
-    subproject_slug = getattr(exception, "subproject_slug", None)
-
-    log.debug(
-        "404 page detected a project slug in request.",
-        project_slug=project_slug,
+    # Serve the general 404 error message
+    return server_error_404(
+        request, exception=None, template_name="errors/404/base.html"
     )
-    r = render(
-        request,
-        template_name,
-        context={
-            "project": project,
-            "project_slug": project_slug,
-            "subproject_slug": subproject_slug,
-            "version_slug": version_slug,
-        },
-    )
-    r.status_code = 404
-    return r
 
 
 @map_project_slug
@@ -112,11 +85,6 @@ def _get_project_data_from_request(
         try:
             final_project = current_project.translations.get(language=lang_slug)
         except Project.DoesNotExist:
-            log.debug(
-                "Raising 404 for language slug",
-                lang_slug=lang_slug,
-                project_slug=current_project.slug,
-            )
             raise ProxitoHttp404(
                 "Did not find translation",
                 project=current_project,
diff --git a/readthedocs/templates/errors/404/base.html b/readthedocs/templates/errors/404/base.html
index a82af8670ae..71dbb95b19b 100644
--- a/readthedocs/templates/errors/404/base.html
+++ b/readthedocs/templates/errors/404/base.html
@@ -10,13 +10,13 @@
 
 
    {% trans "404 Not Found" %}
    
 
-{% block 404_error_message %}
+  {% block 404_error_message %}
 
   
    {% trans "You have encountered a 404 on Read the Docs. You are either looking for a page that does not exist or a project that has been removed." %}
    
 
 
     
-        \       What a maze!         /
+        \       What a maze!        /
          \                         /
           \    This page does     /
            ]     not exist.      [    ,'|
@@ -40,6 +40,6 @@ 
    {% trans "404 Not Found" %}
    
         /                           \
 
    
 
-{% endblock %}
+  {% endblock %}
 
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
index 31f627142cd..6cadf24f80d 100644
--- a/readthedocs/templates/errors/404/no_project.html
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -1,7 +1,7 @@
 {% extends "errors/404/base.html" %}
 {% load i18n %}
 
-{# visiting a slug or subdomain that doesn't match the intended project #}
+{# visiting a slug or subdomain that doesn't match a project #}
 
 {% block 404_error_message %}
 

From 797e96b8cbf2c0e708c2144ff84bbad1d163ab3c Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 13:15:41 +0100
Subject: [PATCH 20/70] Removes log.debug in views.serve

---
 readthedocs/proxito/views/serve.py | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index da4ce1f4124..50e98db452a 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -316,11 +316,9 @@ def get(self, request, proxito_path, template_name='404.html'):
             proxito_path,
             urlconf='readthedocs.proxito.urls',
         )
-        log.debug("Resolved a URL.")
 
         version_slug = kwargs.get('version_slug')
         version_slug = self.get_version_from_host(request, version_slug)
-        log.debug("Getting _get_project_data_from_request.")
         (
             final_project,
             lang_slug,
@@ -334,7 +332,6 @@ def get(self, request, proxito_path, template_name='404.html'):
             version_slug=version_slug,
             filename=kwargs.get('filename', ''),
         )
-        log.debug("Finished _get_project_data_from_request.")
 
         log.bind(
             project_slug=final_project.slug,

From 8d38b8f4bfc1b5687351da34e898801f4d13465f Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 15:47:31 +0100
Subject: [PATCH 21/70] More work on text copy, fix bugs, raise
 ProxitoProjectVersionHttp404 in an additional case

---
 media/css/core.css                            |  1 +
 readthedocs/proxito/exceptions.py             |  4 +--
 readthedocs/proxito/middleware.py             | 12 +++++--
 readthedocs/proxito/views/decorators.py       |  5 ++-
 readthedocs/proxito/views/serve.py            | 32 ++++++++++++-------
 readthedocs/proxito/views/utils.py            |  4 +--
 readthedocs/templates/core/widesearchbar.html |  2 +-
 readthedocs/templates/errors/404/base.html    |  3 +-
 .../errors/404/include_error_message.html     | 16 ++++++++++
 .../templates/errors/404/include_search.html  |  7 ++++
 .../templates/errors/404/include_tips.html    |  3 +-
 .../templates/errors/404/no_project.html      | 22 ++++++++-----
 .../templates/errors/404/no_project_page.html | 16 ++--------
 .../templates/errors/404/no_subproject.html   |  6 ++++
 .../templates/errors/404/no_version.html      | 17 ++--------
 readthedocs/templates/homepage.html           |  3 +-
 16 files changed, 92 insertions(+), 61 deletions(-)
 create mode 100644 readthedocs/templates/errors/404/include_error_message.html
 create mode 100644 readthedocs/templates/errors/404/include_search.html

diff --git a/media/css/core.css b/media/css/core.css
index a41dd9c67c3..fa58300a696 100644
--- a/media/css/core.css
+++ b/media/css/core.css
@@ -1332,6 +1332,7 @@ div.module.project-subprojects li.subproject a.subproject-edit:before {
 }
 
 #content ul.normal_list {list-style: disc; margin-left: 20px;}
+#content code {background: #eee; border: 1px solid #ccc; padding: 3px; display: inline-block;}
 
 /* Pygments */
 div.highlight pre .hll { background-color: #ffffcc }
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index d095ec6208d..d51b0fa6469 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -38,7 +38,7 @@ def __init__(self, message, project_slug=None, project=None, subproject_slug=Non
         self.subproject_slug = subproject_slug
 
 
-class ProxitoProjectPageHttp404(ProxitoHttp404):
+class ProxitoProjectPageHttp404(ProxitoProjectHttp404):
     """
     Raised if a page inside an existing project was not found.
     """
@@ -49,7 +49,7 @@ def __init__(self, message, project_slug=None, project=None, subproject_slug=Non
         self.subproject_slug = subproject_slug
 
 
-class ProxitoProjectVersionHttp404(ProxitoHttp404):
+class ProxitoProjectVersionHttp404(ProxitoProjectHttp404):
     """
     Raised if a version was not found.
 
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index c6d882a564c..7daf86cbea6 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -12,7 +12,6 @@
 import structlog
 from django.conf import settings
 from django.core.exceptions import SuspiciousOperation
-from django.http import Http404
 from django.shortcuts import redirect, render
 from django.urls import reverse
 from django.utils.deprecation import MiddlewareMixin
@@ -26,6 +25,7 @@
     unresolver,
 )
 from readthedocs.core.utils import get_cache_tag
+from readthedocs.proxito.exceptions import ProxitoProjectHttp404
 
 log = structlog.get_logger(__name__)
 
@@ -213,9 +213,15 @@ def process_request(self, request):  # noqa
                 context={"host": exc.domain},
                 status=400,
             )
-        except (InvalidSubdomainError, InvalidExternalDomainError):
+        except (InvalidSubdomainError, InvalidExternalDomainError) as e:
             log.debug("Invalid project set on the subdomain.")
-            raise Http404
+            # We pass e.domain as 'project_slug' because we cannot unpack a slug from the
+            # domain at this stage. The domain can be a custom domain docs.foobar.tld,
+            # in which case 'docs' would be an incorrect slug.
+            raise ProxitoProjectHttp404(
+                f"No project found for requested domain {e.domain}.",
+                project_slug=e.domain,
+            )
         except InvalidCustomDomainError as exc:
             # Some person is CNAMEing to us without configuring a domain - 404.
             log.debug("CNAME 404.", domain=exc.domain)
diff --git a/readthedocs/proxito/views/decorators.py b/readthedocs/proxito/views/decorators.py
index 73741882e94..00f0f4044e7 100644
--- a/readthedocs/proxito/views/decorators.py
+++ b/readthedocs/proxito/views/decorators.py
@@ -23,8 +23,11 @@ def map_subproject_slug(view_func):
 
     @wraps(view_func)
     def inner_view(  # noqa
-        request, project=None, subproject=None, subproject_slug=None, *args, **kwargs
+        request, subproject=None, subproject_slug=None, *args, **kwargs
     ):
+        # Something not entirely clear is happening when unpacking args and kwargs
+        # so the project is fetched from kwarg dictionary.
+        project = kwargs["project"]
         if subproject is None and subproject_slug:
             # Try to fetch by subproject alias first, otherwise we might end up
             # redirected to an unrelated project.
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 50e98db452a..219609b4bc0 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -25,6 +25,7 @@
 from readthedocs.proxito.exceptions import (
     ProxitoHttp404,
     ProxitoProjectPageHttp404,
+    ProxitoProjectVersionHttp404,
     ProxitoSubProjectHttp404,
 )
 from readthedocs.redirects.exceptions import InfiniteRedirectException
@@ -104,8 +105,8 @@ def get(self,
         version_slug = self.get_version_from_host(request, version_slug)
         final_project, lang_slug, version_slug, filename = _get_project_data_from_request(  # noqa
             request,
-            project_slug=project_slug,
-            subproject_slug=subproject_slug,
+            project_slug,
+            subproject_slug,
             lang_slug=lang_slug,
             version_slug=version_slug,
             filename=filename,
@@ -326,9 +327,9 @@ def get(self, request, proxito_path, template_name='404.html'):
             filename,
         ) = _get_project_data_from_request(  # noqa
             request,
-            project_slug=kwargs.get('project_slug'),
-            subproject_slug=kwargs.get('subproject_slug'),
-            lang_slug=kwargs.get('lang_slug'),
+            kwargs.get("project_slug"),
+            kwargs.get("subproject_slug"),
+            lang_slug=kwargs.get("lang_slug"),
             version_slug=version_slug,
             filename=kwargs.get('filename', ''),
         )
@@ -373,6 +374,8 @@ def get(self, request, proxito_path, template_name='404.html'):
                     # TODO: decide if we need to check for infinite redirect here
                     # (from URL == to URL)
                     return HttpResponseRedirect(redirect_url)
+        else:
+            log.debug("No version in request")
 
         # Check and perform redirects on 404 handler
         # NOTE: this redirect check must be done after trying files like
@@ -395,6 +398,7 @@ def get(self, request, proxito_path, template_name='404.html'):
         version = Version.objects.filter(
             project=final_project, slug=version_slug
         ).first()
+        version_found = bool(version)
 
         # If there are no redirect,
         # try to serve the custom 404 of the current version (version_slug)
@@ -438,12 +442,18 @@ def get(self, request, proxito_path, template_name='404.html'):
                     resp.status_code = 404
                     return resp
 
-        raise ProxitoProjectPageHttp404(
-            "No custom 404 page found.",
-            project=final_project,
-            project_slug=kwargs.get("project_slug"),
-            subproject_slug=kwargs.get("subproject_slug"),
-        )
+        if version_found:
+            raise ProxitoProjectPageHttp404(
+                "Page not found and no custom 404",
+                project=final_project,
+                project_slug=final_project.slug,
+            )
+        else:
+            raise ProxitoProjectVersionHttp404(
+                "Version not found and no custom 404",
+                project=final_project,
+                project_slug=final_project.slug,
+            )
 
     def _register_broken_link(self, project, version, path, full_path):
         try:
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 7a31199a621..d3b3aa61270 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -44,9 +44,7 @@ def proxito_404_page_handler(
         return fast_404(request, exception, template_name)
 
     # Serve the general 404 error message
-    return server_error_404(
-        request, exception=None, template_name="errors/404/base.html"
-    )
+    return server_error_404(request, exception=exception, template_name=template_name)
 
 
 @map_project_slug
diff --git a/readthedocs/templates/core/widesearchbar.html b/readthedocs/templates/core/widesearchbar.html
index d310e94b5e9..25cbf44a4e3 100644
--- a/readthedocs/templates/core/widesearchbar.html
+++ b/readthedocs/templates/core/widesearchbar.html
@@ -7,7 +7,7 @@ 
    {% trans "Search all the docs" %}
    
           
    
 
 
-            
    
+            
               
    
                 
               
    
diff --git a/readthedocs/templates/errors/404/base.html b/readthedocs/templates/errors/404/base.html
index 71dbb95b19b..524fde40346 100644
--- a/readthedocs/templates/errors/404/base.html
+++ b/readthedocs/templates/errors/404/base.html
@@ -11,8 +11,9 @@
 
    {% trans "404 Not Found" %}
    
 
   {% block 404_error_message %}
-
+
    
   
    {% trans "You have encountered a 404 on Read the Docs. You are either looking for a page that does not exist or a project that has been removed." %}
    
+
    
 
 
     
diff --git a/readthedocs/templates/errors/404/include_error_message.html b/readthedocs/templates/errors/404/include_error_message.html
new file mode 100644
index 00000000000..3ec8de3482b
--- /dev/null
+++ b/readthedocs/templates/errors/404/include_error_message.html
@@ -0,0 +1,16 @@
+{% load i18n %}
+
    
+  
    
+    {% blocktrans trimmed with request_path=request.path %}
+      You are browsing the documentation of {{ project_slug }}.
+      The {{ not_found_subject }} you are looking for at {{ request_path }} was not found.
+    {% endblocktrans %}
+  
    
+
+  
    
+    {% blocktrans trimmed %}
+      Documentation changes over time and pages are moved around.
+      You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
+    {% endblocktrans %}
+  
    
+
    
diff --git a/readthedocs/templates/errors/404/include_search.html b/readthedocs/templates/errors/404/include_search.html
new file mode 100644
index 00000000000..c8986f6ee00
--- /dev/null
+++ b/readthedocs/templates/errors/404/include_search.html
@@ -0,0 +1,7 @@
+{% load i18n %}
+
+
    {% trans "Try searching?" %}
    
+
+
    
+  {% include "core/widesearchbar.html" with form_action="https://readthedocs.org/search/" %}
+
    
diff --git a/readthedocs/templates/errors/404/include_tips.html b/readthedocs/templates/errors/404/include_tips.html
index 78017e3a3c4..d507a918d33 100644
--- a/readthedocs/templates/errors/404/include_tips.html
+++ b/readthedocs/templates/errors/404/include_tips.html
@@ -1,5 +1,5 @@
 {% load i18n %}
-
+
    
 
    {% trans "Are you the project owner?" %}
    
 
 
    {% trans "Here are some tips to address 404 errors:" %}
    
@@ -8,3 +8,4 @@ 
    {% trans "Are you the project owner?" %}
    
   
  • {% trans "Use custom 404 pages:" %} {% trans "Read more" %} »
    • 
   
  • {% trans "Create redirects when you move contents:" %} {% trans "Read more" %} »
    • 
 

+
diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
index 6cadf24f80d..c4ddc904217 100644
--- a/readthedocs/templates/errors/404/no_project.html
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -4,13 +4,19 @@
 {# visiting a slug or subdomain that doesn't match a project #}
 
 {% block 404_error_message %}
+  

+    

+      {% blocktrans trimmed %}
+        The project {{ project_slug }} that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
+      {% endblocktrans %}
+    

+    

+      {% blocktrans trimmed with request_path=request.path %}
+        You are browsing the documentation of {{ project_slug }}.
+        The page you are looking for at {{ request_path }} was not found.
+      {% endblocktrans %}
+    

+  

 
-  {# we are looking at a 404 generated by proxito, we then know that the project itself doesn't exist or hasn't been built #}
-
-  

-    {% blocktrans trimmed %}
-      The project {{ project_slug }} that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
-    {% endblocktrans %}
-  

-
+  {% include "errors/404/include_search.html" %}
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/no_project_page.html b/readthedocs/templates/errors/404/no_project_page.html
index 77da321ceb2..3146523f3f8 100644
--- a/readthedocs/templates/errors/404/no_project_page.html
+++ b/readthedocs/templates/errors/404/no_project_page.html
@@ -4,20 +4,8 @@
 {# project exists but page doesn't #}
 
 {% block 404_error_message %}
-
-  

-    {% blocktrans trimmed with request_path=request.path %}
-      You are browsing the documentation of {{ project_slug }}.
-      The page you are looking for at {{ request_path }} was not found.
-    {% endblocktrans %}
-  

-
-  

-    {% blocktrans trimmed %}
-      Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
-    {% endblocktrans %}
-  

-
+  {% include "errors/404/include_error_message.html" with not_found_subject="page" %}
+  {% include "errors/404/include_search.html" %}
   {% include "errors/404/include_tips.html" %}
 
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/no_subproject.html b/readthedocs/templates/errors/404/no_subproject.html
index 1bd2dae4f9b..16cfff8caa2 100644
--- a/readthedocs/templates/errors/404/no_subproject.html
+++ b/readthedocs/templates/errors/404/no_subproject.html
@@ -5,6 +5,9 @@
 
 {% block 404_error_message %}
 
+  {% include "errors/404/include_error_message.html" with not_found_subject="subproject" %}
+
+

   

     {% blocktrans trimmed with request_path=request.path %}
       You are browsing the documentation of project {{ project_slug }}.
@@ -17,6 +20,9 @@
       Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
     {% endblocktrans %}
   

+

+
+  {% include "errors/404/include_search.html" %}
 
   {% include "errors/404/include_tips.html" %}
 
diff --git a/readthedocs/templates/errors/404/no_version.html b/readthedocs/templates/errors/404/no_version.html
index 7e5696ff8dd..6405d3e80fa 100644
--- a/readthedocs/templates/errors/404/no_version.html
+++ b/readthedocs/templates/errors/404/no_version.html
@@ -4,20 +4,7 @@
 {# project exists but version doesn't #}
 
 {% block 404_error_message %}
-
-  

-    {% blocktrans trimmed with request_path=request.path %}
-      You are browsing the documentation of project {{ project_slug }}.
-      The version you are looking for at {{ request_path }} was not found.
-    {% endblocktrans %}
-  

-
-  

-    {% blocktrans trimmed %}
-      Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
-    {% endblocktrans %}
-  

-
+  {% include "errors/404/include_error_message.html" with not_found_subject="project version" %}
+  {% include "errors/404/include_search.html" %}
   {% include "errors/404/include_tips.html" %}
-
 {% endblock %}
diff --git a/readthedocs/templates/homepage.html b/readthedocs/templates/homepage.html
index 96757dd222a..7832c09bdbb 100644
--- a/readthedocs/templates/homepage.html
+++ b/readthedocs/templates/homepage.html
@@ -112,7 +112,8 @@ 
{% trans "Multiple versions" %}

 
   
   

-    {% include "core/widesearchbar.html" %}
+    {% url "search" as search_form_action %}
+    {% include "core/widesearchbar.html" with form_action=search_form_action %}
   

 
   {% get_current_language as language %}

From b60a1093f432485cc5b8900ac32913fa1ad0d142 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 16:24:32 +0100
Subject: [PATCH 22/70] Contextualize when subprojects aren't found. The
 disadvantages of the current decorator structure is worth paying attention to
 here.

---
 readthedocs/proxito/views/decorators.py       |  5 ++
 readthedocs/proxito/views/serve.py            | 81 ++++++++++++++-----
 .../templates/errors/404/no_subproject.html   | 15 ----
 3 files changed, 65 insertions(+), 36 deletions(-)

diff --git a/readthedocs/proxito/views/decorators.py b/readthedocs/proxito/views/decorators.py
index 00f0f4044e7..14664eea0fe 100644
--- a/readthedocs/proxito/views/decorators.py
+++ b/readthedocs/proxito/views/decorators.py
@@ -25,10 +25,14 @@ def map_subproject_slug(view_func):
     def inner_view(  # noqa
         request, subproject=None, subproject_slug=None, *args, **kwargs
     ):
+        log.debug(
+            f"Checking if there is a subproject to be resolved. Got subproject_slug {subproject_slug}"
+        )
         # Something not entirely clear is happening when unpacking args and kwargs
         # so the project is fetched from kwarg dictionary.
         project = kwargs["project"]
         if subproject is None and subproject_slug:
+            log.debug(f"Trying to find subproject slug {subproject_slug}")
             # Try to fetch by subproject alias first, otherwise we might end up
             # redirected to an unrelated project.
             # Depends on a project passed into kwargs
@@ -83,6 +87,7 @@ def inner_view(  # noqa
                 try:
                     project = Project.objects.get(slug=project_slug)
                 except Project.DoesNotExist:
+                    log.debug(f"No project found with slug {project_slug}")
                     raise ProxitoProjectHttp404(
                         "Project does not exist.", project_slug=project_slug
                     )
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 219609b4bc0..fe690362c2f 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -24,6 +24,7 @@
 from readthedocs.proxito.constants import RedirectType
 from readthedocs.proxito.exceptions import (
     ProxitoHttp404,
+    ProxitoProjectHttp404,
     ProxitoProjectPageHttp404,
     ProxitoProjectVersionHttp404,
     ProxitoSubProjectHttp404,
@@ -103,14 +104,31 @@ def get(self,
         so that we can decide if we need to serve docs or add a /.
         """
         version_slug = self.get_version_from_host(request, version_slug)
-        final_project, lang_slug, version_slug, filename = _get_project_data_from_request(  # noqa
-            request,
-            project_slug,
-            subproject_slug,
-            lang_slug=lang_slug,
-            version_slug=version_slug,
-            filename=filename,
-        )
+        try:
+            (
+                final_project,
+                lang_slug,
+                version_slug,
+                filename,
+            ) = _get_project_data_from_request(  # noqa
+                request,
+                project_slug,
+                subproject_slug,
+                lang_slug=lang_slug,
+                version_slug=version_slug,
+                filename=filename,
+            )
+        # This special treatment of ProxitoProjectHttp404 happens because the decorator that
+        # resolves a project doesn't know if it's resolving a subproject or a normal project
+        except ProxitoProjectHttp404 as e:
+            if subproject_slug:
+                log.debug("This is actually a subproject")
+                raise ProxitoSubProjectHttp404(
+                    f"Could not find subproject for {subproject_slug}",
+                    project_slug=e.project_slug,
+                    subproject_slug=subproject_slug,
+                )
+            raise
         version = final_project.versions.filter(slug=version_slug).first()
 
         is_external = request.unresolved_domain.is_from_external_domain
@@ -320,19 +338,33 @@ def get(self, request, proxito_path, template_name='404.html'):
 
         version_slug = kwargs.get('version_slug')
         version_slug = self.get_version_from_host(request, version_slug)
-        (
-            final_project,
-            lang_slug,
-            version_slug,
-            filename,
-        ) = _get_project_data_from_request(  # noqa
-            request,
-            kwargs.get("project_slug"),
-            kwargs.get("subproject_slug"),
-            lang_slug=kwargs.get("lang_slug"),
-            version_slug=version_slug,
-            filename=kwargs.get('filename', ''),
-        )
+        # This special treatment of ProxitoProjectHttp404 happens because the decorator that
+        # resolves a project doesn't know if it's resolving a subproject or a normal project
+        subproject_slug = kwargs.get("subproject_slug")
+        project_slug = kwargs.get("project_slug")
+        try:
+            (
+                final_project,
+                lang_slug,
+                version_slug,
+                filename,
+            ) = _get_project_data_from_request(  # noqa
+                request,
+                project_slug,
+                subproject_slug,
+                lang_slug=kwargs.get("lang_slug"),
+                version_slug=version_slug,
+                filename=kwargs.get("filename", ""),
+            )
+        except ProxitoProjectHttp404 as e:
+            if subproject_slug:
+                log.debug("This is actually a subproject")
+                raise ProxitoSubProjectHttp404(
+                    f"Could not find subproject for {subproject_slug}",
+                    project_slug=e.project_slug,
+                    subproject_slug=subproject_slug,
+                )
+            raise
 
         log.bind(
             project_slug=final_project.slug,
@@ -448,6 +480,13 @@ def get(self, request, proxito_path, template_name='404.html'):
                 project=final_project,
                 project_slug=final_project.slug,
             )
+        elif kwargs.get("subproject_slug"):
+            raise ProxitoSubProjectHttp404(
+                "Subproject not found and no custom 404",
+                project=final_project,
+                project_slug=final_project.slug,
+                subproject_slug=kwargs.get("subproject_slug"),
+            )
         else:
             raise ProxitoProjectVersionHttp404(
                 "Version not found and no custom 404",
diff --git a/readthedocs/templates/errors/404/no_subproject.html b/readthedocs/templates/errors/404/no_subproject.html
index 16cfff8caa2..72ec359a7be 100644
--- a/readthedocs/templates/errors/404/no_subproject.html
+++ b/readthedocs/templates/errors/404/no_subproject.html
@@ -7,21 +7,6 @@
 
   {% include "errors/404/include_error_message.html" with not_found_subject="subproject" %}
 
-

-  

-    {% blocktrans trimmed with request_path=request.path %}
-      You are browsing the documentation of project {{ project_slug }}.
-      The subproject {{ subproject_slug }} you are looking for at {{ request_path }} was not found.
-    {% endblocktrans %}
-  

-
-  

-    {% blocktrans trimmed %}
-      Documentation changes over time. Pages are moved around. You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
-    {% endblocktrans %}
-  

-

-
   {% include "errors/404/include_search.html" %}
 
   {% include "errors/404/include_tips.html" %}

From b557590780958312060fc6b851a26f3f66c488e0 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 16:28:53 +0100
Subject: [PATCH 23/70] Clean up

---
 readthedocs/proxito/views/decorators.py | 4 ----
 readthedocs/proxito/views/serve.py      | 4 ++--
 readthedocs/settings/base.py            | 2 +-
 3 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/readthedocs/proxito/views/decorators.py b/readthedocs/proxito/views/decorators.py
index 14664eea0fe..bffad18e684 100644
--- a/readthedocs/proxito/views/decorators.py
+++ b/readthedocs/proxito/views/decorators.py
@@ -25,14 +25,10 @@ def map_subproject_slug(view_func):
     def inner_view(  # noqa
         request, subproject=None, subproject_slug=None, *args, **kwargs
     ):
-        log.debug(
-            f"Checking if there is a subproject to be resolved. Got subproject_slug {subproject_slug}"
-        )
         # Something not entirely clear is happening when unpacking args and kwargs
         # so the project is fetched from kwarg dictionary.
         project = kwargs["project"]
         if subproject is None and subproject_slug:
-            log.debug(f"Trying to find subproject slug {subproject_slug}")
             # Try to fetch by subproject alias first, otherwise we might end up
             # redirected to an unrelated project.
             # Depends on a project passed into kwargs
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index fe690362c2f..ea4b52a8d82 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -122,7 +122,7 @@ def get(self,
         # resolves a project doesn't know if it's resolving a subproject or a normal project
         except ProxitoProjectHttp404 as e:
             if subproject_slug:
-                log.debug("This is actually a subproject")
+                log.debug("Project expected to be a subproject was not found")
                 raise ProxitoSubProjectHttp404(
                     f"Could not find subproject for {subproject_slug}",
                     project_slug=e.project_slug,
@@ -358,7 +358,7 @@ def get(self, request, proxito_path, template_name='404.html'):
             )
         except ProxitoProjectHttp404 as e:
             if subproject_slug:
-                log.debug("This is actually a subproject")
+                log.debug("Project expected to be a subproject was not found")
                 raise ProxitoSubProjectHttp404(
                     f"Could not find subproject for {subproject_slug}",
                     project_slug=e.project_slug,
diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
index 99827f9f687..4b2e76fa7b7 100644
--- a/readthedocs/settings/base.py
+++ b/readthedocs/settings/base.py
@@ -46,7 +46,7 @@ class CommunityBaseSettings(Settings):
     DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
     # Debug settings
-    DEBUG = False
+    DEBUG = True
     RTD_FORCE_SHOW_DEBUG_TOOLBAR = False
 
     @property

From 3f7f2fd1cbacd0fe19dc2bf03b905d425a539e6d Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 16:40:01 +0100
Subject: [PATCH 24/70] Add project:slug search scope when pages aren't found

---
 readthedocs/templates/core/widesearchbar.html         | 5 ++++-
 readthedocs/templates/errors/404/include_search.html  | 8 +++++++-
 readthedocs/templates/errors/404/no_project_page.html | 2 +-
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/readthedocs/templates/core/widesearchbar.html b/readthedocs/templates/core/widesearchbar.html
index 25cbf44a4e3..bcb91283106 100644
--- a/readthedocs/templates/core/widesearchbar.html
+++ b/readthedocs/templates/core/widesearchbar.html
@@ -1,8 +1,11 @@
 {% load i18n %}
 
         
+        {% if term|default_if_none:False %}
+        
{% trans "Search the docs" %}

+        {% else %}
         
{% trans "Search all the docs" %}

-
+        {% endif %}
         

           

 
diff --git a/readthedocs/templates/errors/404/include_search.html b/readthedocs/templates/errors/404/include_search.html
index c8986f6ee00..e95956d7042 100644
--- a/readthedocs/templates/errors/404/include_search.html
+++ b/readthedocs/templates/errors/404/include_search.html
@@ -3,5 +3,11 @@
 
{% trans "Try searching?" %}

 
 

-  {% include "core/widesearchbar.html" with form_action="https://readthedocs.org/search/" %}
+  {% if search_project %}
+    {% with "project:"|add:search_project as search_term %}
+      {% include "core/widesearchbar.html" with form_action="https://readthedocs.org/search/" term=search_term %}
+    {% endwith %}
+  {% else %}
+    {% include "core/widesearchbar.html" with form_action="https://readthedocs.org/search/" %}
+  {% endif %}
 

diff --git a/readthedocs/templates/errors/404/no_project_page.html b/readthedocs/templates/errors/404/no_project_page.html
index 3146523f3f8..4e5dc687c43 100644
--- a/readthedocs/templates/errors/404/no_project_page.html
+++ b/readthedocs/templates/errors/404/no_project_page.html
@@ -5,7 +5,7 @@
 
 {% block 404_error_message %}
   {% include "errors/404/include_error_message.html" with not_found_subject="page" %}
-  {% include "errors/404/include_search.html" %}
+  {% include "errors/404/include_search.html" with search_project=project_slug %}
   {% include "errors/404/include_tips.html" %}
 
 {% endblock %}

From 63e64c2f01219c6c1a0eb3fcb79c7c2a2ded79a0 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 16:51:24 +0100
Subject: [PATCH 25/70] clean up

---
 readthedocs/templates/errors/404/no_project_page.html | 1 -
 readthedocs/templates/errors/404/no_subproject.html   | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/readthedocs/templates/errors/404/no_project_page.html b/readthedocs/templates/errors/404/no_project_page.html
index 4e5dc687c43..7cf480eaca0 100644
--- a/readthedocs/templates/errors/404/no_project_page.html
+++ b/readthedocs/templates/errors/404/no_project_page.html
@@ -7,5 +7,4 @@
   {% include "errors/404/include_error_message.html" with not_found_subject="page" %}
   {% include "errors/404/include_search.html" with search_project=project_slug %}
   {% include "errors/404/include_tips.html" %}
-
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/no_subproject.html b/readthedocs/templates/errors/404/no_subproject.html
index 72ec359a7be..e85ce7d448f 100644
--- a/readthedocs/templates/errors/404/no_subproject.html
+++ b/readthedocs/templates/errors/404/no_subproject.html
@@ -4,11 +4,7 @@
 {# project exists but subproject doesn't #}
 
 {% block 404_error_message %}
-
   {% include "errors/404/include_error_message.html" with not_found_subject="subproject" %}
-
   {% include "errors/404/include_search.html" %}
-
   {% include "errors/404/include_tips.html" %}
-
 {% endblock %}

From 18823ba314090f44b9afbfc5bfe2bd5c70373825 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 16:55:14 +0100
Subject: [PATCH 26/70] Missed some linting there

---
 readthedocs/proxito/exceptions.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index d51b0fa6469..363ae46b02e 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -32,6 +32,7 @@ class ProxitoSubProjectHttp404(ProxitoProjectHttp404):
     """
     Raised if a sub-project was not found.
     """
+
     def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
         super().__init__(message, project_slug=project_slug)
         self.project = project

From 6dfde4ce4b850e8b7e64cc7441be7ea199484c84 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 23 Feb 2023 18:37:30 +0100
Subject: [PATCH 27/70] fix pylint errors

---
 readthedocs/core/views/__init__.py |  1 +
 readthedocs/proxito/exceptions.py  | 23 +++++++++++------------
 readthedocs/proxito/middleware.py  |  6 +++---
 readthedocs/proxito/views/utils.py |  8 +-------
 4 files changed, 16 insertions(+), 22 deletions(-)

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 87429afb4ac..d7827291fa0 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -57,6 +57,7 @@ def get_context_data(self, **kwargs):
 def server_error_404(request, template_name="errors/404/base.html", exception=None):
     """
     Serves a 404 error message, handling 404 exception types raised throughout the app.
+
     Notice that handling of 404 errors happens elsewhere in views and middleware,
     this view is expected to serve an actual 404 message.
     """
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 363ae46b02e..fb8921faacf 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -1,4 +1,6 @@
 """
+Exceptions for El Proxito.
+
 This module contains exceptions that may be raised in El Proxito application
 but are handled elsewhere in the Django project.
 
@@ -13,15 +15,13 @@
 
 
 class ProxitoHttp404(Http404):
-    """
-    General error class for proxity 404s.
-    """
+
+    """General error class for proxity 404s."""
 
 
 class ProxitoProjectHttp404(ProxitoHttp404):
-    """
-    Raised if a project was not found.
-    """
+
+    """Raised if a project was not found."""
 
     def __init__(self, message, project_slug=None):
         self.project_slug = project_slug
@@ -29,9 +29,8 @@ def __init__(self, message, project_slug=None):
 
 
 class ProxitoSubProjectHttp404(ProxitoProjectHttp404):
-    """
-    Raised if a sub-project was not found.
-    """
+
+    """Raised if a subproject was not found."""
 
     def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
         super().__init__(message, project_slug=project_slug)
@@ -40,9 +39,8 @@ def __init__(self, message, project_slug=None, project=None, subproject_slug=Non
 
 
 class ProxitoProjectPageHttp404(ProxitoProjectHttp404):
-    """
-    Raised if a page inside an existing project was not found.
-    """
+
+    """Raised if a page inside an existing project was not found."""
 
     def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
         super().__init__(message, project_slug=project_slug)
@@ -51,6 +49,7 @@ def __init__(self, message, project_slug=None, project=None, subproject_slug=Non
 
 
 class ProxitoProjectVersionHttp404(ProxitoProjectHttp404):
+
     """
     Raised if a version was not found.
 
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index 7daf86cbea6..93bb55f161d 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -213,14 +213,14 @@ def process_request(self, request):  # noqa
                 context={"host": exc.domain},
                 status=400,
             )
-        except (InvalidSubdomainError, InvalidExternalDomainError) as e:
+        except (InvalidSubdomainError, InvalidExternalDomainError) as exc:
             log.debug("Invalid project set on the subdomain.")
             # We pass e.domain as 'project_slug' because we cannot unpack a slug from the
             # domain at this stage. The domain can be a custom domain docs.foobar.tld,
             # in which case 'docs' would be an incorrect slug.
             raise ProxitoProjectHttp404(
-                f"No project found for requested domain {e.domain}.",
-                project_slug=e.domain,
+                f"No project found for requested domain {exc.domain}.",
+                project_slug=exc.domain,
             )
         except InvalidCustomDomainError as exc:
             # Some person is CNAMEing to us without configuring a domain - 404.
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index d3b3aa61270..baa406d9ea9 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -4,13 +4,7 @@
 from django.http import HttpResponse
 
 from readthedocs.projects.models import Project
-from readthedocs.proxito.exceptions import (
-    ProxitoHttp404,
-    ProxitoProjectHttp404,
-    ProxitoProjectPageHttp404,
-    ProxitoProjectVersionHttp404,
-    ProxitoSubProjectHttp404,
-)
+from readthedocs.proxito.exceptions import ProxitoHttp404
 
 from ...core.views import server_error_404
 from .decorators import map_project_slug, map_subproject_slug

From 38496bc12431312bced90d1a5c48010eab64a7fd Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Mon, 27 Feb 2023 17:28:30 -0500
Subject: [PATCH 28/70] Proxito: use unresolved in 404 handler

---
 readthedocs/core/unresolver.py     |   8 +-
 readthedocs/proxito/views/serve.py | 286 +++++++++++++++++++++--------
 2 files changed, 220 insertions(+), 74 deletions(-)

diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py
index 52bea1a902e..b040e4adb51 100644
--- a/readthedocs/core/unresolver.py
+++ b/readthedocs/core/unresolver.py
@@ -52,10 +52,11 @@ def __init__(self, project, version_slug, filename):
 
 
 class TranslationNotFoundError(UnresolverError):
-    def __init__(self, project, language, filename):
+    def __init__(self, project, language, filename, version_slug):
         self.project = project
         self.language = language
         self.filename = filename
+        self.version_slug = version_slug
 
 
 class UnresolvedPathError(UnresolverError):
@@ -264,7 +265,10 @@ def _match_multiversion_project(
             project = parent_project.translations.filter(language=language).first()
             if not project:
                 raise TranslationNotFoundError(
-                    project=parent_project, language=language, filename=file
+                    project=parent_project,
+                    language=language,
+                    filename=file,
+                    version_slug=version_slug,
                 )
 
         if external_version_slug and external_version_slug != version_slug:
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 2dccf3b813f..26eb4045ff3 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -289,16 +289,25 @@ def get(self, request, proxito_path, template_name='404.html'):
 
         * Handles directory indexing for URLs that don't end in a slash
         * Handles directory indexing for README.html (for now)
+        * Check for user redirects
+        * Record the broken link for analytics
         * Handles custom 404 serving
 
         For 404's, first search for a 404 page in the current version, then continues
         with the default version and finally, if none of them are found, the Read
         the Docs default page (Maze Found) is rendered by Django and served.
         """
-        # pylint: disable=too-many-locals
         log.bind(proxito_path=proxito_path)
         log.debug('Executing 404 handler.')
 
+        unresolved_domain = request.unresolved_domain
+
+        if unresolved_domain.is_from_external_domain:
+            self.version_type = EXTERNAL
+        if unresolved_domain.project.has_feature(Feature.USE_UNRESOLVER_WITH_PROXITO):
+            return self.get_using_unresolver(request, proxito_path)
+
+
         # Parse the URL using the normal urlconf, so we get proper subdomain/translation data
         _, __, kwargs = url_resolve(
             proxito_path,
@@ -321,41 +330,21 @@ def get(self, request, proxito_path, template_name='404.html'):
             version_slug=version_slug,
         )
 
-        if version_slug:
-            storage_root_path = final_project.get_storage_path(
-                type_="html",
-                version_slug=version_slug,
-                include_file=False,
-                version_type=self.version_type,
-            )
-
-            # First, check for dirhtml with slash
-            for tryfile in ("index.html", "README.html"):
-                storage_filename_path = build_media_storage.join(
-                    storage_root_path,
-                    f"{filename}/{tryfile}".lstrip("/"),
-                )
-                log.debug("Trying index filename.")
-                if build_media_storage.exists(storage_filename_path):
-                    log.info("Redirecting to index file.", tryfile=tryfile)
-                    # Use urlparse so that we maintain GET args in our redirect
-                    parts = urlparse(proxito_path)
-                    if tryfile == "README.html":
-                        new_path = parts.path.rstrip("/") + f"/{tryfile}"
-                    else:
-                        new_path = parts.path.rstrip("/") + "/"
-
-                    # `proxito_path` doesn't include query params.`
-                    query = urlparse(request.get_full_path()).query
-                    new_parts = parts._replace(
-                        path=new_path,
-                        query=query,
-                    )
-                    redirect_url = new_parts.geturl()
+        version = Version.objects.filter(
+            project=final_project, slug=version_slug
+        ).first()
 
-                    # TODO: decide if we need to check for infinite redirect here
-                    # (from URL == to URL)
-                    return HttpResponseRedirect(redirect_url)
+        # Check if an index file exists, redirect to it if so.
+        if version:
+            response = self._get_index_file_redirect(
+                request=request,
+                project=final_project,
+                version=version,
+                filename=filename,
+                full_path=proxito_path,
+            )
+            if response:
+                return response
 
         # Check and perform redirects on 404 handler
         # NOTE: this redirect check must be done after trying files like
@@ -375,21 +364,6 @@ def get(self, request, proxito_path, template_name='404.html'):
                 # Continue with our normal 404 handling in this case
                 pass
 
-        version = Version.objects.filter(
-            project=final_project, slug=version_slug
-        ).first()
-
-        # If there are no redirect,
-        # try to serve the custom 404 of the current version (version_slug)
-        # Then, try to serve the custom 404 page for the default version
-        # (project.get_default_version())
-        versions = []
-        if version:
-            versions.append(version_slug)
-        default_version_slug = final_project.get_default_version()
-        if default_version_slug != version_slug:
-            versions.append(default_version_slug)
-
         # Register 404 pages into our database for user's analytics
         self._register_broken_link(
             project=final_project,
@@ -398,28 +372,13 @@ def get(self, request, proxito_path, template_name='404.html'):
             full_path=proxito_path,
         )
 
-        for version_slug_404 in versions:
-            if not self.allowed_user(request, final_project, version_slug_404):
-                continue
-
-            storage_root_path = final_project.get_storage_path(
-                type_='html',
-                version_slug=version_slug_404,
-                include_file=False,
-                version_type=self.version_type,
-            )
-            tryfiles = ["404.html", "404/index.html"]
-            for tryfile in tryfiles:
-                storage_filename_path = build_media_storage.join(storage_root_path, tryfile)
-                if build_media_storage.exists(storage_filename_path):
-                    log.info(
-                        'Serving custom 404.html page.',
-                        version_slug_404=version_slug_404,
-                        storage_filename_path=storage_filename_path,
-                    )
-                    resp = HttpResponse(build_media_storage.open(storage_filename_path).read())
-                    resp.status_code = 404
-                    return resp
+        response = self._serve_custom_404_page(
+            request=request,
+            project=final_project,
+            version=version,
+        )
+        if response:
+            return response
 
         raise Http404('No custom 404 page found.')
 
@@ -461,6 +420,189 @@ def _register_broken_link(self, project, version, path, full_path):
                 full_path=full_path,
             )
 
+    def _serve_custom_404_page(self, request, project, version=None):
+        """
+        Try to serve a custom 404 page from this project.
+
+        If a version is given, try to serve the 404 page from that version first,
+        if it doesn't exist, try to serve the 404 page from the default version.
+
+        We check for a 404.html or 404/index.html file.
+
+        If a 404 page is found, we return a response with the content of that file,
+        `None` otherwise.
+        """
+        current_version_slug = version.slug if version else None
+        versions_slug = []
+        if current_version_slug:
+            versions_slug.append(current_version_slug)
+
+        default_version_slug = project.get_default_version()
+        if default_version_slug != current_version_slug:
+            versions_slug.append(default_version_slug)
+
+        for version_slug_404 in versions_slug:
+            if not self.allowed_user(request, project, version_slug_404):
+                continue
+
+            storage_root_path = project.get_storage_path(
+                type_="html",
+                version_slug=version_slug_404,
+                include_file=False,
+                version_type=self.version_type,
+            )
+            tryfiles = ["404.html", "404/index.html"]
+            for tryfile in tryfiles:
+                storage_filename_path = build_media_storage.join(
+                    storage_root_path, tryfile
+                )
+                if build_media_storage.exists(storage_filename_path):
+                    log.info(
+                        "Serving custom 404.html page.",
+                        version_slug_404=version_slug_404,
+                        storage_filename_path=storage_filename_path,
+                    )
+                    resp = HttpResponse(
+                        build_media_storage.open(storage_filename_path).read()
+                    )
+                    resp.status_code = 404
+                    return resp
+        return None
+
+    def _get_index_file_redirect(self, request, project, version, filename, full_path):
+        """Check if a file is a directory and redirect to its index/README file."""
+        storage_root_path = project.get_storage_path(
+            type_="html",
+            version_slug=version.slug,
+            include_file=False,
+            version_type=self.version_type,
+        )
+
+        # First, check for dirhtml with slash
+        for tryfile in ("index.html", "README.html"):
+            storage_filename_path = build_media_storage.join(
+                storage_root_path,
+                f"{filename}/{tryfile}".lstrip("/"),
+            )
+            log.debug("Trying index filename.")
+            if build_media_storage.exists(storage_filename_path):
+                log.info("Redirecting to index file.", tryfile=tryfile)
+                # Use urlparse so that we maintain GET args in our redirect
+                parts = urlparse(full_path)
+                if tryfile == "README.html":
+                    new_path = parts.path.rstrip("/") + f"/{tryfile}"
+                else:
+                    new_path = parts.path.rstrip("/") + "/"
+
+                # `full_path` doesn't include query params.`
+                query = urlparse(request.get_full_path()).query
+                redirect_url = parts._replace(
+                    path=new_path,
+                    query=query,
+                ).geturl()
+
+                # TODO: decide if we need to check for infinite redirect here
+                # (from URL == to URL)
+                return HttpResponseRedirect(redirect_url)
+
+        return None
+
+    def get_using_unresolver(self, request, path):
+        """
+        404 handler using the new proxito implementation.
+
+        This is basically a copy of the get() method, but adapted to make use
+        of the unresolver to extract the current project, version, and file.
+        """
+        unresolved_domain = request.unresolved_domain
+        project = None
+        version = None
+        filename = None
+        lang_slug = None
+        version_slug = None
+        try:
+            unresolved = unresolver.unresolve_path(
+                unresolved_domain=unresolved_domain,
+                path=path,
+                append_indexhtml=False,
+            )
+            project = unresolved.project
+            version = unresolved.version
+            filename = unresolved.filename
+            lang_slug = project.language
+            version_slug = version.slug
+        except VersionNotFoundError as exc:
+            project = exc.project
+            lang_slug = project.language
+            version_slug = exc.version_slug
+            filename = exc.filename
+        except TranslationNotFoundError as exc:
+            project = exc.project
+            lang_slug = exc.language
+            version_slug = exc.version_slug
+            filename = exc.filename
+        except InvalidExternalVersionError as exc:
+            project = exc.project
+        except UnresolvedPathError as exc:
+            project = exc.project
+            filename = exc.path
+
+        log.bind(
+            project_slug=project.slug,
+            version_slug=version_slug,
+        )
+
+        request.path_project_slug = project.slug
+        request.path_version_slug = version_slug
+
+        if version:
+            response = self._get_index_file_redirect(
+                request=request,
+                project=project,
+                version=version,
+                filename=filename,
+                full_path=path,
+            )
+            if response:
+                return response
+
+        # Check and perform redirects on 404 handler
+        # NOTE: this redirect check must be done after trying files like
+        # ``index.html`` and ``README.html`` to emulate the behavior we had when
+        # serving directly from NGINX without passing through Python.
+        redirect_path, http_status = self.get_redirect(
+            project=project,
+            lang_slug=lang_slug,
+            version_slug=version_slug,
+            filename=filename,
+            full_path=path,
+        )
+        if redirect_path and http_status:
+            try:
+                return self.get_redirect_response(
+                    request, redirect_path, path, http_status
+                )
+            except InfiniteRedirectException:
+                # Continue with our normal 404 handling in this case
+                pass
+
+        # Register 404 pages into our database for user's analytics
+        self._register_broken_link(
+            project=project,
+            version=version,
+            path=filename,
+            full_path=path,
+        )
+
+        response = self._serve_custom_404_page(
+            request=request,
+            project=project,
+            version=version,
+        )
+        if response:
+            return response
+        raise Http404("No custom 404 page found.")
+
 
 class ServeError404(SettingsOverrideObject):
     _default_class = ServeError404Base

From bece4666dc4cd862acb535c5f3ec295aba51188f Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Tue, 28 Feb 2023 15:20:57 +0100
Subject: [PATCH 29/70] Don't use request.path as the "not found" path for
 proxito 404 exceptions

---
 readthedocs/core/views/__init__.py            | 26 ++++++++++-------
 readthedocs/proxito/exceptions.py             | 28 +++++++++++++++----
 readthedocs/proxito/middleware.py             |  1 +
 readthedocs/proxito/views/serve.py            |  3 ++
 readthedocs/settings/base.py                  |  2 +-
 .../errors/404/include_error_message.html     |  4 +--
 .../templates/errors/404/no_project.html      |  4 +--
 7 files changed, 47 insertions(+), 21 deletions(-)

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index d7827291fa0..db6ed020852 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -14,6 +14,7 @@
 from readthedocs.core.mixins import PrivateViewMixin
 from readthedocs.projects.models import Project
 from readthedocs.proxito.exceptions import (
+    ProxitoHttp404,
     ProxitoProjectHttp404,
     ProxitoProjectPageHttp404,
     ProxitoProjectVersionHttp404,
@@ -62,12 +63,22 @@ def server_error_404(request, template_name="errors/404/base.html", exception=No
     this view is expected to serve an actual 404 message.
     """
 
+    context = {}
     # Properties are set by ProxitoHttp404. We could also have a look at the
     # subproject_slug
-    project_slug = getattr(exception, "project_slug", None)
-    version_slug = getattr(exception, "version_slug", None)
-    project = getattr(exception, "project", None)
-    subproject_slug = getattr(exception, "subproject_slug", None)
+    if isinstance(exception, ProxitoHttp404):
+        # These attributes are not guaranteed.
+        context.update(
+            {
+                "project": getattr(exception, "project", None),
+                "project_slug": getattr(exception, "project_slug", None),
+                "subproject_slug": getattr(exception, "subproject_slug", None),
+                "version_slug": getattr(exception, "version_slug", None),
+                "path_not_found": getattr(exception, "proxito_path", None),
+            }
+        )
+
+    context["path_not_found"] = context.get("path_not_found") or request.path
 
     if isinstance(exception, ProxitoProjectVersionHttp404):
         template_name = "errors/404/no_version.html"
@@ -81,12 +92,7 @@ def server_error_404(request, template_name="errors/404/base.html", exception=No
     r = render(
         request,
         template_name,
-        context={
-            "project": project,
-            "project_slug": project_slug,
-            "subproject_slug": subproject_slug,
-            "version_slug": version_slug,
-        },
+        context=context,
     )
     r.status_code = 404
     return r
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index fb8921faacf..9df8317e0d9 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -23,8 +23,9 @@ class ProxitoProjectHttp404(ProxitoHttp404):
 
     """Raised if a project was not found."""
 
-    def __init__(self, message, project_slug=None):
+    def __init__(self, message, project_slug=None, proxito_path=None):
         self.project_slug = project_slug
+        self.proxito_path = proxito_path
         super().__init__(message)
 
 
@@ -32,8 +33,15 @@ class ProxitoSubProjectHttp404(ProxitoProjectHttp404):
 
     """Raised if a subproject was not found."""
 
-    def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
-        super().__init__(message, project_slug=project_slug)
+    def __init__(
+        self,
+        message,
+        project_slug=None,
+        proxito_path=None,
+        project=None,
+        subproject_slug=None,
+    ):
+        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
         self.project = project
         self.subproject_slug = subproject_slug
 
@@ -42,8 +50,15 @@ class ProxitoProjectPageHttp404(ProxitoProjectHttp404):
 
     """Raised if a page inside an existing project was not found."""
 
-    def __init__(self, message, project_slug=None, project=None, subproject_slug=None):
-        super().__init__(message, project_slug=project_slug)
+    def __init__(
+        self,
+        message,
+        project_slug=None,
+        proxito_path=None,
+        project=None,
+        subproject_slug=None,
+    ):
+        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
         self.project = project
         self.subproject_slug = subproject_slug
 
@@ -60,11 +75,12 @@ def __init__(
         self,
         message,
         project_slug=None,
+        proxito_path=None,
         project=None,
         subproject_slug=None,
         version_slug=None,
     ):
-        super().__init__(message, project_slug=project_slug)
+        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
         self.project = project
         self.subproject_slug = subproject_slug
         self.version_slug = version_slug
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index 93bb55f161d..6762d651ff1 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -221,6 +221,7 @@ def process_request(self, request):  # noqa
             raise ProxitoProjectHttp404(
                 f"No project found for requested domain {exc.domain}.",
                 project_slug=exc.domain,
+                proxito_path=request.path,
             )
         except InvalidCustomDomainError as exc:
             # Some person is CNAMEing to us without configuring a domain - 404.
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index ea4b52a8d82..9bf59f086ad 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -479,6 +479,7 @@ def get(self, request, proxito_path, template_name='404.html'):
                 "Page not found and no custom 404",
                 project=final_project,
                 project_slug=final_project.slug,
+                proxito_path=proxito_path,
             )
         elif kwargs.get("subproject_slug"):
             raise ProxitoSubProjectHttp404(
@@ -486,12 +487,14 @@ def get(self, request, proxito_path, template_name='404.html'):
                 project=final_project,
                 project_slug=final_project.slug,
                 subproject_slug=kwargs.get("subproject_slug"),
+                proxito_path=proxito_path,
             )
         else:
             raise ProxitoProjectVersionHttp404(
                 "Version not found and no custom 404",
                 project=final_project,
                 project_slug=final_project.slug,
+                proxito_path=proxito_path,
             )
 
     def _register_broken_link(self, project, version, path, full_path):
diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
index 4b2e76fa7b7..99827f9f687 100644
--- a/readthedocs/settings/base.py
+++ b/readthedocs/settings/base.py
@@ -46,7 +46,7 @@ class CommunityBaseSettings(Settings):
     DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
     # Debug settings
-    DEBUG = True
+    DEBUG = False
     RTD_FORCE_SHOW_DEBUG_TOOLBAR = False
 
     @property
diff --git a/readthedocs/templates/errors/404/include_error_message.html b/readthedocs/templates/errors/404/include_error_message.html
index 3ec8de3482b..d3b3d20144f 100644
--- a/readthedocs/templates/errors/404/include_error_message.html
+++ b/readthedocs/templates/errors/404/include_error_message.html
@@ -1,9 +1,9 @@
 {% load i18n %}
 

   

-    {% blocktrans trimmed with request_path=request.path %}
+    {% blocktrans trimmed %}
       You are browsing the documentation of {{ project_slug }}.
-      The {{ not_found_subject }} you are looking for at {{ request_path }} was not found.
+      The {{ not_found_subject }} you are looking for at {{ path_not_found }} was not found.
     {% endblocktrans %}
   

 
diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
index c4ddc904217..5bccd7d4a17 100644
--- a/readthedocs/templates/errors/404/no_project.html
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -11,9 +11,9 @@
       {% endblocktrans %}
     

     

-      {% blocktrans trimmed with request_path=request.path %}
+      {% blocktrans trimmed %}
         You are browsing the documentation of {{ project_slug }}.
-        The page you are looking for at {{ request_path }} was not found.
+        The page you are looking for at {{ path_not_found }} was not found.
       {% endblocktrans %}
     

   


From 69f4886bda3b21d57948481996a7fc3dcaed6ec4 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Tue, 28 Feb 2023 17:16:32 -0500
Subject: [PATCH 30/70] Updates from review

---
 readthedocs/core/unresolver.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py
index 52bea1a902e..7bf6bbe3004 100644
--- a/readthedocs/core/unresolver.py
+++ b/readthedocs/core/unresolver.py
@@ -160,7 +160,7 @@ class Unresolver:
         re.VERBOSE,
     )
 
-    def unresolve(self, url, append_indexhtml=True):
+    def unresolve_url(self, url, append_indexhtml=True):
         """
         Turn a URL into the component parts that our views would use to process them.
 
@@ -211,7 +211,7 @@ def _unresolve(self, unresolved_domain, parsed_url, append_indexhtml):
         Extracted into a separate method so it can be re-used by
         the unresolve and unresolve_path methods.
         """
-        current_project, version, filename = self._unresolve_path(
+        current_project, version, filename = self._unresolve_path_with_parent_project(
             parent_project=unresolved_domain.project,
             path=parsed_url.path,
             external_version_slug=unresolved_domain.external_version_slug,
@@ -308,7 +308,7 @@ def _match_subproject(self, parent_project, path, external_version_slug=None):
             # We use the subproject as the new parent project
             # to resolve the rest of the path relative to it.
             subproject = project_relationship.child
-            response = self._unresolve_path(
+            response = self._unresolve_path_with_parent_project(
                 parent_project=subproject,
                 path=file,
                 check_subprojects=False,
@@ -350,7 +350,7 @@ def _match_single_version_project(
 
         return parent_project, version, file
 
-    def _unresolve_path(
+    def _unresolve_path_with_parent_project(
         self, parent_project, path, check_subprojects=True, external_version_slug=None
     ):
         """
@@ -540,4 +540,4 @@ def unresolve_domain_from_request(self, request):
 
 
 unresolver = Unresolver()
-unresolve = unresolver.unresolve
+unresolve = unresolver.unresolve_url

From 21841125aefe90b7e57e6890a5d78ad5550b86bd Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 1 Mar 2023 16:05:05 +0100
Subject: [PATCH 31/70] Missing imports

---
 readthedocs/proxito/views/serve.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 26eb4045ff3..887ec8202e6 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -25,6 +25,13 @@
 from readthedocs.redirects.exceptions import InfiniteRedirectException
 from readthedocs.storage import build_media_storage
 
+from ...core import unresolver
+from ...core.unresolver import (
+    InvalidExternalVersionError,
+    TranslationNotFoundError,
+    UnresolvedPathError,
+    VersionNotFoundError,
+)
 from .mixins import (
     InvalidPathError,
     ServeDocsMixin,

From ce28e12af643433eb78db4e688c07addd4615975 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 1 Mar 2023 16:06:02 +0100
Subject: [PATCH 32/70] Linting: Too many blank lines

---
 readthedocs/proxito/views/serve.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 887ec8202e6..fb2a5afbd84 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -314,7 +314,6 @@ def get(self, request, proxito_path, template_name='404.html'):
         if unresolved_domain.project.has_feature(Feature.USE_UNRESOLVER_WITH_PROXITO):
             return self.get_using_unresolver(request, proxito_path)
 
-
         # Parse the URL using the normal urlconf, so we get proper subdomain/translation data
         _, __, kwargs = url_resolve(
             proxito_path,

From 01f8cee492522eeedfe3ca5d328ad092b7b32ee8 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 1 Mar 2023 17:56:19 +0100
Subject: [PATCH 33/70] Add exception type for new 404 translation pages

---
 readthedocs/proxito/exceptions.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 9df8317e0d9..e28dcb1bb41 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -63,6 +63,25 @@ def __init__(
         self.subproject_slug = subproject_slug
 
 
+class ProxitoProjectTranslationHttp404(ProxitoProjectHttp404):
+
+    """Raised if a translation of some page was not found."""
+
+    def __init__(
+        self,
+        message,
+        project_slug=None,
+        proxito_path=None,
+        project=None,
+        subproject_slug=None,
+        translation_slug=None,
+    ):
+        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
+        self.project = project
+        self.subproject_slug = subproject_slug
+        self.translation_slug = translation_slug
+
+
 class ProxitoProjectVersionHttp404(ProxitoProjectHttp404):
 
     """

From 05384be067d7e1fb8e7b1264f428a9939201231d Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Wed, 1 Mar 2023 20:21:39 -0500
Subject: [PATCH 34/70] Remove duplicated imports

---
 readthedocs/proxito/views/serve.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 6f8b13f9a77..f0fbf10898b 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -30,13 +30,6 @@
 from readthedocs.redirects.exceptions import InfiniteRedirectException
 from readthedocs.storage import build_media_storage
 
-from ...core import unresolver
-from ...core.unresolver import (
-    InvalidExternalVersionError,
-    TranslationNotFoundError,
-    UnresolvedPathError,
-    VersionNotFoundError,
-)
 from .mixins import (
     InvalidPathError,
     ServeDocsMixin,

From 7af491af9a25ec26805d73a65743e25d40727fa5 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Wed, 1 Mar 2023 20:22:07 -0500
Subject: [PATCH 35/70] Black

---
 readthedocs/core/unresolver.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py
index 8446b58b5b4..ca3d057cf7b 100644
--- a/readthedocs/core/unresolver.py
+++ b/readthedocs/core/unresolver.py
@@ -52,7 +52,6 @@ def __init__(self, project, version_slug, filename):
 
 
 class TranslationNotFoundError(UnresolverError):
-
     def __init__(self, project, language, filename, version_slug):
         self.project = project
         self.language = language

From 86f67b2958d1b400613db2ad227104b1c4c0ae61 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Wed, 1 Mar 2023 20:24:24 -0500
Subject: [PATCH 36/70] Fix exc

---
 readthedocs/proxito/views/serve.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index f0fbf10898b..412ea7d805c 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -463,7 +463,6 @@ def get(self, request, proxito_path, template_name='404.html'):
         log.debug('Executing 404 handler.')
 
         unresolved_domain = request.unresolved_domain
-
         if unresolved_domain.is_from_external_domain:
             self.version_type = EXTERNAL
         if unresolved_domain.project.has_feature(Feature.USE_UNRESOLVER_WITH_PROXITO):
@@ -704,7 +703,7 @@ def get_using_unresolver(self, request, path):
             filename = exc.filename
         except InvalidExternalVersionError as exc:
             project = exc.project
-        except UnresolvedPathError as exc:
+        except InvalidPathForVersionedProjectError as exc:
             project = exc.project
             filename = exc.path
 

From 2d92061c2b11f55388a977a0653eed8a5bbd6d81 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Thu, 2 Mar 2023 18:11:07 -0500
Subject: [PATCH 37/70] Update tests

---
 .../proxito/tests/test_old_redirects.py       | 42 ++++++++++++++++++-
 readthedocs/proxito/tests/test_redirects.py   | 15 ++++++-
 readthedocs/proxito/tests/test_urls.py        | 17 +++++++-
 3 files changed, 71 insertions(+), 3 deletions(-)

diff --git a/readthedocs/proxito/tests/test_old_redirects.py b/readthedocs/proxito/tests/test_old_redirects.py
index aed84bbb6c8..cddd5ba4ae8 100644
--- a/readthedocs/proxito/tests/test_old_redirects.py
+++ b/readthedocs/proxito/tests/test_old_redirects.py
@@ -1039,7 +1039,16 @@ def test_redirect_prefix_crossdomain(self):
             self.assertEqual(r.status_code, 302, url)
             self.assertEqual(r["Location"], expected_location, url)
 
-        # These aren't even handled by Django.
+    def test_redirect_prefix_crossdomain_with_newline_chars(self):
+        fixture.get(
+            Redirect,
+            project=self.project,
+            redirect_type="prefix",
+            from_url="/",
+        )
+        # We make use of Django's URL resolve in the current implementation,
+        # which doesn't handle these chars and raises an exception
+        # instead of redirecting.
         urls = [
             # Trying to bypass the protocol check by including a `\n` char.
             "http://project.dev.readthedocs.io/http:/%0A/my.host/path.html",
@@ -1135,3 +1144,34 @@ def setUp(self):
             default_true=True,
             future_default_true=True,
         )
+
+    def test_redirect_prefix_crossdomain_with_newline_chars(self):
+        """
+        Overridden to make it compatible with the new implementation.
+
+        In the new implementation we will correctly redirect,
+        instead of raising an exception.
+        """
+        fixture.get(
+            Redirect,
+            project=self.project,
+            redirect_type="prefix",
+            from_url="/",
+        )
+        urls = [
+            (
+                "http://project.dev.readthedocs.io/http:/%0A/my.host/path.html",
+                "http://project.dev.readthedocs.io/en/latest/http://my.host/path.html",
+            ),
+            (
+                "http://project.dev.readthedocs.io/%0A/my.host/path.html",
+                "http://project.dev.readthedocs.io/en/latest/my.host/path.html",
+            ),
+        ]
+        for url, expected_location in urls:
+            r = self.client.get(
+                url,
+                HTTP_HOST="project.dev.readthedocs.io",
+            )
+            self.assertEqual(r.status_code, 302, url)
+            self.assertEqual(r["Location"], expected_location, url)
diff --git a/readthedocs/proxito/tests/test_redirects.py b/readthedocs/proxito/tests/test_redirects.py
index 1a1c5c5236d..a02f944f8c4 100644
--- a/readthedocs/proxito/tests/test_redirects.py
+++ b/readthedocs/proxito/tests/test_redirects.py
@@ -1,8 +1,9 @@
 # Copied from .org test_redirects
-
 import pytest
 from django.test import override_settings
+from django_dynamic_fixture import get
 
+from readthedocs.projects.models import Feature
 from readthedocs.proxito.constants import RedirectType
 
 from .base import BaseDocServing
@@ -280,3 +281,15 @@ def test_slash_redirect(self):
         # WARNING
         # The test client strips multiple slashes at the front of the URL
         # Additional tests for this are in ``test_middleware:test_front_slash``
+
+
+class ProxitoV2RedirectTests(RedirectTests):
+    # TODO: remove this class once the new implementation is the default.
+    def setUp(self):
+        super().setUp()
+        get(
+            Feature,
+            feature_id=Feature.USE_UNRESOLVER_WITH_PROXITO,
+            default_true=True,
+            future_default_true=True,
+        )
diff --git a/readthedocs/proxito/tests/test_urls.py b/readthedocs/proxito/tests/test_urls.py
index df958f9316b..6d8e8f1c6d1 100644
--- a/readthedocs/proxito/tests/test_urls.py
+++ b/readthedocs/proxito/tests/test_urls.py
@@ -3,8 +3,11 @@
 """Test URL config."""
 
 import pytest
-from django.test import TestCase, override_settings
+from django.test import TestCase
 from django.urls import resolve
+from django_dynamic_fixture import get
+
+from readthedocs.projects.models import Feature
 
 
 @pytest.mark.proxito
@@ -103,3 +106,15 @@ def test_single_version(self):
                 'filename': 'some/path/index.html',
             },
         )
+
+
+class ProxitoV2TestSingleVersionURLs(TestSingleVersionURLs):
+    # TODO: remove this class once the new implementation is the default.
+    def setUp(self):
+        super().setUp()
+        get(
+            Feature,
+            feature_id=Feature.USE_UNRESOLVER_WITH_PROXITO,
+            default_true=True,
+            future_default_true=True,
+        )

From c01750334b380a391075e544dd13950d8e3a7c19 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Thu, 2 Mar 2023 18:28:45 -0500
Subject: [PATCH 38/70] Move check to new implementation only

---
 readthedocs/proxito/views/serve.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 412ea7d805c..96871371c75 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -463,8 +463,6 @@ def get(self, request, proxito_path, template_name='404.html'):
         log.debug('Executing 404 handler.')
 
         unresolved_domain = request.unresolved_domain
-        if unresolved_domain.is_from_external_domain:
-            self.version_type = EXTERNAL
         if unresolved_domain.project.has_feature(Feature.USE_UNRESOLVER_WITH_PROXITO):
             return self.get_using_unresolver(request, proxito_path)
 
@@ -675,6 +673,11 @@ def get_using_unresolver(self, request, path):
         of the unresolver to extract the current project, version, and file.
         """
         unresolved_domain = request.unresolved_domain
+        # We force all storage calls to use the external versions storage,
+        # since we are serving an external version.
+        if unresolved_domain.is_from_external_domain:
+            self.version_type = EXTERNAL
+
         project = None
         version = None
         filename = None

From b5d96f41c78a88943fad0b1152f57f29f62f8b19 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Fri, 3 Mar 2023 16:12:26 +0100
Subject: [PATCH 39/70] Remove double error message for project 404s

---
 readthedocs/templates/errors/404/no_project.html | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
index 5bccd7d4a17..fc640c3b185 100644
--- a/readthedocs/templates/errors/404/no_project.html
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -10,12 +10,6 @@
         The project {{ project_slug }} that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
       {% endblocktrans %}
     

-    

-      {% blocktrans trimmed %}
-        You are browsing the documentation of {{ project_slug }}.
-        The page you are looking for at {{ path_not_found }} was not found.
-      {% endblocktrans %}
-    

   
 
   {% include "errors/404/include_search.html" %}

From a36eb90c0245564c7be22ba68d885ed7f8309598 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Fri, 3 Mar 2023 16:19:18 +0100
Subject: [PATCH 40/70] Add missing kwarg proxito_path for subproject 404
 exception

---
 readthedocs/proxito/views/serve.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 15d5b18f17a..86b5f192eb2 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -65,6 +65,7 @@ def get(self, request, subproject_slug=None, filename=""):
                     project=parent_project,
                     project_slug=parent_project.slug,
                     subproject_slug=subproject_slug,
+                    proxito_path=request.proxito_path,
                 )
 
         # Get the default version from the current project,
@@ -532,6 +533,7 @@ def get(self, request, proxito_path, template_name='404.html'):
                     f"Could not find subproject for {subproject_slug}",
                     project_slug=e.project_slug,
                     subproject_slug=subproject_slug,
+                    proxito_path=proxito_path,
                 )
             raise
 

From be2284880baeaa85e73c2c1743df457de47cb0f5 Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Mon, 6 Mar 2023 19:03:38 -0500
Subject: [PATCH 41/70] Updates from review

---
 readthedocs/proxito/views/serve.py | 25 +++++++++++++++++++------
 1 file changed, 19 insertions(+), 6 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 96871371c75..a6c880f6911 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -492,7 +492,9 @@ def get(self, request, proxito_path, template_name='404.html'):
             project=final_project, slug=version_slug
         ).first()
 
-        # Check if an index file exists, redirect to it if so.
+        # If we were able to resolve to a valid version, it means that the
+        # current file doesn't exist. So we check if we can redirect to its
+        # index file if it exists before doing anything else.
         if version:
             response = self._get_index_file_redirect(
                 request=request,
@@ -519,7 +521,9 @@ def get(self, request, proxito_path, template_name='404.html'):
             try:
                 return self.get_redirect_response(request, redirect_path, proxito_path, http_status)
             except InfiniteRedirectException:
-                # Continue with our normal 404 handling in this case
+                # ``get_redirect_response`` raises this when it's redirecting back to itself.
+                # We can safely ignore it here because it's logged in ``canonical_redirect``,
+                # and we don't want to issue infinite redirects.
                 pass
 
         # Register 404 pages into our database for user's analytics
@@ -530,7 +534,7 @@ def get(self, request, proxito_path, template_name='404.html'):
             full_path=proxito_path,
         )
 
-        response = self._serve_custom_404_page(
+        response = self._get_custom_404_page(
             request=request,
             project=final_project,
             version=version,
@@ -578,7 +582,7 @@ def _register_broken_link(self, project, version, path, full_path):
                 full_path=full_path,
             )
 
-    def _serve_custom_404_page(self, request, project, version=None):
+    def _get_custom_404_page(self, request, project, version=None):
         """
         Try to serve a custom 404 page from this project.
 
@@ -683,6 +687,9 @@ def get_using_unresolver(self, request, path):
         filename = None
         lang_slug = None
         version_slug = None
+        # Try to map the current path to a project/version/filename.
+        # If that fails, we fill the variables with the information we have
+        # available in the exceptions.
         try:
             unresolved = unresolver.unresolve_path(
                 unresolved_domain=unresolved_domain,
@@ -715,9 +722,13 @@ def get_using_unresolver(self, request, path):
             version_slug=version_slug,
         )
 
+        # TODO: find a better way to pass this to the middleware.
         request.path_project_slug = project.slug
         request.path_version_slug = version_slug
 
+        # If we were able to resolve to a valid version, it means that the
+        # current file doesn't exist. So we check if we can redirect to its
+        # index file if it exists before doing anything else.
         if version:
             response = self._get_index_file_redirect(
                 request=request,
@@ -746,7 +757,9 @@ def get_using_unresolver(self, request, path):
                     request, redirect_path, path, http_status
                 )
             except InfiniteRedirectException:
-                # Continue with our normal 404 handling in this case
+                # ``get_redirect_response`` raises this when it's redirecting back to itself.
+                # We can safely ignore it here because it's logged in ``canonical_redirect``,
+                # and we don't want to issue infinite redirects.
                 pass
 
         # Register 404 pages into our database for user's analytics
@@ -757,7 +770,7 @@ def get_using_unresolver(self, request, path):
             full_path=path,
         )
 
-        response = self._serve_custom_404_page(
+        response = self._get_custom_404_page(
             request=request,
             project=project,
             version=version,

From e3ff836b89d3c051c749e1f6b283ce796172feff Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Tue, 7 Mar 2023 15:22:30 +0100
Subject: [PATCH 42/70] Add a new 404 exception class and targeted template for
 missing project translations

---
 readthedocs/core/views/__init__.py            |  4 ++++
 readthedocs/proxito/exceptions.py             |  7 +++++-
 readthedocs/proxito/views/serve.py            |  1 +
 readthedocs/proxito/views/utils.py            | 23 +++++++++++--------
 .../templates/errors/404/no_language.html     | 10 ++++++++
 5 files changed, 35 insertions(+), 10 deletions(-)
 create mode 100644 readthedocs/templates/errors/404/no_language.html

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 325aec5885c..95cd875e0b1 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -17,6 +17,7 @@
     ProxitoHttp404,
     ProxitoProjectHttp404,
     ProxitoProjectPageHttp404,
+    ProxitoProjectTranslationHttp404,
     ProxitoProjectVersionHttp404,
     ProxitoSubProjectHttp404,
 )
@@ -79,6 +80,7 @@ def server_error_404(request, template_name="errors/404/base.html", exception=No
                 "project_slug": getattr(exception, "project_slug", None),
                 "subproject_slug": getattr(exception, "subproject_slug", None),
                 "version_slug": getattr(exception, "version_slug", None),
+                "language_slug": getattr(exception, "language_slug", None),
                 "path_not_found": getattr(exception, "proxito_path", None),
             }
         )
@@ -89,6 +91,8 @@ def server_error_404(request, template_name="errors/404/base.html", exception=No
         template_name = "errors/404/no_version.html"
     elif isinstance(exception, ProxitoSubProjectHttp404):
         template_name = "errors/404/no_subproject.html"
+    elif isinstance(exception, ProxitoProjectTranslationHttp404):
+        template_name = "errors/404/no_language.html"
     elif isinstance(exception, ProxitoProjectPageHttp404):
         template_name = "errors/404/no_project_page.html"
     elif isinstance(exception, ProxitoProjectHttp404):
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index e28dcb1bb41..0e8eb39a905 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -65,7 +65,12 @@ def __init__(
 
 class ProxitoProjectTranslationHttp404(ProxitoProjectHttp404):
 
-    """Raised if a translation of some page was not found."""
+    """
+    Raised if a translation of a project was not found.
+
+    This means that the project does not exist for requested language.
+    If a page isn't found, raise a ProxitoProjectPageHttp404.
+    """
 
     def __init__(
         self,
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 2ba2f0bcbcc..b06a09d8c78 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -525,6 +525,7 @@ def get(self, request, proxito_path, template_name='404.html'):
                 lang_slug=kwargs.get("lang_slug"),
                 version_slug=version_slug,
                 filename=kwargs.get("filename", ""),
+                explicit_proxito_path=proxito_path,
             )
         except ProxitoProjectHttp404 as e:
             if subproject_slug:
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index baa406d9ea9..4d60d1eeb30 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -4,7 +4,7 @@
 from django.http import HttpResponse
 
 from readthedocs.projects.models import Project
-from readthedocs.proxito.exceptions import ProxitoHttp404
+from readthedocs.proxito.exceptions import ProxitoProjectTranslationHttp404
 
 from ...core.views import server_error_404
 from .decorators import map_project_slug, map_subproject_slug
@@ -44,17 +44,21 @@ def proxito_404_page_handler(
 @map_project_slug
 @map_subproject_slug
 def _get_project_data_from_request(
-        request,
-        project,
-        subproject,
-        lang_slug=None,
-        version_slug=None,
-        filename='',
+    request,
+    project,
+    subproject,
+    lang_slug=None,
+    version_slug=None,
+    filename="",
+    explicit_proxito_path=None,
 ):
     """
     Get the proper project based on the request and URL.
 
     This is used in a few places and so we break out into a utility function.
+
+    :param str explicit_proxito_path: Set when called from the proxito 404 handler and
+      request.path contains additional prefix.
     """
     # Take the most relevant project so far
     current_project = subproject or project
@@ -77,11 +81,12 @@ def _get_project_data_from_request(
         try:
             final_project = current_project.translations.get(language=lang_slug)
         except Project.DoesNotExist:
-            raise ProxitoHttp404(
+            raise ProxitoProjectTranslationHttp404(
                 "Did not find translation",
                 project=current_project,
                 project_slug=current_project.slug,
-                subproject_slug=lang_slug,
+                translation_slug=lang_slug,
+                proxito_path=explicit_proxito_path or request.path,
             )
 
     # Handle single version by grabbing the default version
diff --git a/readthedocs/templates/errors/404/no_language.html b/readthedocs/templates/errors/404/no_language.html
new file mode 100644
index 00000000000..aba185f20d3
--- /dev/null
+++ b/readthedocs/templates/errors/404/no_language.html
@@ -0,0 +1,10 @@
+{% extends "errors/404/base.html" %}
+{% load i18n %}
+
+{# project exists but not in the requested language #}
+
+{% block 404_error_message %}
+  {% include "errors/404/include_error_message.html" with not_found_subject="language" %}
+  {% include "errors/404/include_search.html" %}
+  {% include "errors/404/include_tips.html" %}
+{% endblock %}

From 03bbc6d7aee143f01158333e47cbd30c8bd6254d Mon Sep 17 00:00:00 2001
From: Santos Gallegos 
Date: Tue, 7 Mar 2023 11:33:48 -0500
Subject: [PATCH 43/70] Updates from review

---
 readthedocs/core/unresolver.py     |  5 +++--
 readthedocs/proxito/views/serve.py | 15 ++++++++++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/readthedocs/core/unresolver.py b/readthedocs/core/unresolver.py
index ca3d057cf7b..1aa2986a34f 100644
--- a/readthedocs/core/unresolver.py
+++ b/readthedocs/core/unresolver.py
@@ -52,10 +52,11 @@ def __init__(self, project, version_slug, filename):
 
 
 class TranslationNotFoundError(UnresolverError):
-    def __init__(self, project, language, filename, version_slug):
+    def __init__(self, project, language, version_slug, filename):
         self.project = project
         self.language = language
         self.filename = filename
+        # The version doesn't exist, so we just have the slug.
         self.version_slug = version_slug
 
 
@@ -267,8 +268,8 @@ def _match_multiversion_project(
                 raise TranslationNotFoundError(
                     project=parent_project,
                     language=language,
-                    filename=file,
                     version_slug=version_slug,
+                    filename=file,
                 )
 
         if external_version_slug and external_version_slug != version_slug:
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index a6c880f6911..92399ab5475 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -495,6 +495,7 @@ def get(self, request, proxito_path, template_name='404.html'):
         # If we were able to resolve to a valid version, it means that the
         # current file doesn't exist. So we check if we can redirect to its
         # index file if it exists before doing anything else.
+        # This is /en/latest/foo -> /en/latest/foo/index.html.
         if version:
             response = self._get_index_file_redirect(
                 request=request,
@@ -632,7 +633,15 @@ def _get_custom_404_page(self, request, project, version=None):
         return None
 
     def _get_index_file_redirect(self, request, project, version, filename, full_path):
-        """Check if a file is a directory and redirect to its index/README file."""
+        """
+        Check if a file is a directory and redirect to its index/README file.
+
+        For example:
+
+        - /en/latest/foo -> /en/latest/foo/index.html
+        - /en/latest/foo -> /en/latest/foo/README.html
+        - /en/latest/foo/ -> /en/latest/foo/README.html
+        """
         storage_root_path = project.get_storage_path(
             type_="html",
             version_slug=version.slug,
@@ -679,6 +688,9 @@ def get_using_unresolver(self, request, path):
         unresolved_domain = request.unresolved_domain
         # We force all storage calls to use the external versions storage,
         # since we are serving an external version.
+        # The version that results from the unresolve_path() call already is
+        # validated to use the correct manager, this is here to add defense in
+        # depth against serving the wrong version.
         if unresolved_domain.is_from_external_domain:
             self.version_type = EXTERNAL
 
@@ -729,6 +741,7 @@ def get_using_unresolver(self, request, path):
         # If we were able to resolve to a valid version, it means that the
         # current file doesn't exist. So we check if we can redirect to its
         # index file if it exists before doing anything else.
+        # This is /en/latest/foo -> /en/latest/foo/index.html.
         if version:
             response = self._get_index_file_redirect(
                 request=request,

From b98dd0bc5f1e439daacd42c6c124d7440bb222c2 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 00:03:22 +0200
Subject: [PATCH 44/70] WIP: Simplify contextualized 404 messages into a set of
 exceptions. TODO: Remove from old Proxito implementation.

---
 readthedocs/core/exceptions.py     |  53 ++++++++++++++
 readthedocs/core/views/__init__.py |  40 +++--------
 readthedocs/proxito/exceptions.py  | 110 -----------------------------
 readthedocs/proxito/views/serve.py |  86 +++++++++++-----------
 4 files changed, 105 insertions(+), 184 deletions(-)
 create mode 100644 readthedocs/core/exceptions.py
 delete mode 100644 readthedocs/proxito/exceptions.py

diff --git a/readthedocs/core/exceptions.py b/readthedocs/core/exceptions.py
new file mode 100644
index 00000000000..36f3b887fcf
--- /dev/null
+++ b/readthedocs/core/exceptions.py
@@ -0,0 +1,53 @@
+from django.http import Http404
+
+
+class ContextualizedHttp404(Http404):
+
+    template_name = "errors/404/base.html"
+
+    def __init__(self, **kwargs):
+        self.context = kwargs
+
+
+class ProjectHttp404(ContextualizedHttp404):
+
+    """Raised if a project was not found."""
+
+    template_name = "errors/404/no_project.html"
+
+
+class SubProjectHttp404(ContextualizedHttp404):
+
+    """Raised if a subproject was not found."""
+
+    template_name = "errors/404/no_subproject.html"
+
+
+class ProjectPageHttp404(ContextualizedHttp404):
+
+    """Raised if a page inside an existing project was not found."""
+
+    template_name = "errors/404/no_project_page.html"
+
+
+class ProjectTranslationHttp404(ContextualizedHttp404):
+
+    """
+    Raised if a translation of a project was not found.
+
+    This means that the project does not exist for requested language.
+    If a page isn't found, raise a ProjectPageHttp404.
+    """
+
+    template_name = "errors/404/no_language.html"
+
+
+class ProjectVersionHttp404(ContextualizedHttp404):
+
+    """
+    Raised if a version was not found.
+
+    Note: The containing project can be both a project or a subproject inside another project.
+    """
+
+    template_name = "errors/404/no_version.html"
diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 571a2aba5aa..ceddccc21e0 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -12,16 +12,9 @@
 from django.urls import reverse
 from django.views.generic import TemplateView, View
 
+from readthedocs.core.exceptions import ContextualizedHttp404
 from readthedocs.core.mixins import CDNCacheControlMixin, PrivateViewMixin
 from readthedocs.projects.models import Project
-from readthedocs.proxito.exceptions import (
-    ProxitoHttp404,
-    ProxitoProjectHttp404,
-    ProxitoProjectPageHttp404,
-    ProxitoProjectTranslationHttp404,
-    ProxitoProjectVersionHttp404,
-    ProxitoSubProjectHttp404,
-)
 
 log = structlog.get_logger(__name__)
 
@@ -84,34 +77,17 @@ def server_error_404(request, template_name="errors/404/base.html", exception=No
     """
 
     context = {}
-    # Properties are set by ProxitoHttp404. We could also have a look at the
-    # subproject_slug
-    if isinstance(exception, ProxitoHttp404):
+
+    # Contextualized 404 exceptions:
+    # Context is defined by the views that raise these exceptions and handled
+    # in their templates.
+    if isinstance(exception, ContextualizedHttp404):
         # These attributes are not guaranteed.
-        context.update(
-            {
-                "project": getattr(exception, "project", None),
-                "project_slug": getattr(exception, "project_slug", None),
-                "subproject_slug": getattr(exception, "subproject_slug", None),
-                "version_slug": getattr(exception, "version_slug", None),
-                "language_slug": getattr(exception, "language_slug", None),
-                "path_not_found": getattr(exception, "proxito_path", None),
-            }
-        )
+        context.update(exception.context)
+        template_name = exception.template_name
 
     context["path_not_found"] = context.get("path_not_found") or request.path
 
-    if isinstance(exception, ProxitoProjectVersionHttp404):
-        template_name = "errors/404/no_version.html"
-    elif isinstance(exception, ProxitoSubProjectHttp404):
-        template_name = "errors/404/no_subproject.html"
-    elif isinstance(exception, ProxitoProjectTranslationHttp404):
-        template_name = "errors/404/no_language.html"
-    elif isinstance(exception, ProxitoProjectPageHttp404):
-        template_name = "errors/404/no_project_page.html"
-    elif isinstance(exception, ProxitoProjectHttp404):
-        template_name = "errors/404/no_project.html"
-
     r = render(
         request,
         template_name,
diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
deleted file mode 100644
index 0e8eb39a905..00000000000
--- a/readthedocs/proxito/exceptions.py
+++ /dev/null
@@ -1,110 +0,0 @@
-"""
-Exceptions for El Proxito.
-
-This module contains exceptions that may be raised in El Proxito application
-but are handled elsewhere in the Django project.
-
---------------
-404 exceptions
---------------
-
-Because an Http404 can be raised anywhere in the code,
-we want to add more context for targeted error handling and user-facing communication
-"""
-from django.http.response import Http404
-
-
-class ProxitoHttp404(Http404):
-
-    """General error class for proxity 404s."""
-
-
-class ProxitoProjectHttp404(ProxitoHttp404):
-
-    """Raised if a project was not found."""
-
-    def __init__(self, message, project_slug=None, proxito_path=None):
-        self.project_slug = project_slug
-        self.proxito_path = proxito_path
-        super().__init__(message)
-
-
-class ProxitoSubProjectHttp404(ProxitoProjectHttp404):
-
-    """Raised if a subproject was not found."""
-
-    def __init__(
-        self,
-        message,
-        project_slug=None,
-        proxito_path=None,
-        project=None,
-        subproject_slug=None,
-    ):
-        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
-        self.project = project
-        self.subproject_slug = subproject_slug
-
-
-class ProxitoProjectPageHttp404(ProxitoProjectHttp404):
-
-    """Raised if a page inside an existing project was not found."""
-
-    def __init__(
-        self,
-        message,
-        project_slug=None,
-        proxito_path=None,
-        project=None,
-        subproject_slug=None,
-    ):
-        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
-        self.project = project
-        self.subproject_slug = subproject_slug
-
-
-class ProxitoProjectTranslationHttp404(ProxitoProjectHttp404):
-
-    """
-    Raised if a translation of a project was not found.
-
-    This means that the project does not exist for requested language.
-    If a page isn't found, raise a ProxitoProjectPageHttp404.
-    """
-
-    def __init__(
-        self,
-        message,
-        project_slug=None,
-        proxito_path=None,
-        project=None,
-        subproject_slug=None,
-        translation_slug=None,
-    ):
-        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
-        self.project = project
-        self.subproject_slug = subproject_slug
-        self.translation_slug = translation_slug
-
-
-class ProxitoProjectVersionHttp404(ProxitoProjectHttp404):
-
-    """
-    Raised if a version was not found.
-
-    Note: The containing project can be both a project or a subproject inside another project.
-    """
-
-    def __init__(
-        self,
-        message,
-        project_slug=None,
-        proxito_path=None,
-        project=None,
-        subproject_slug=None,
-        version_slug=None,
-    ):
-        super().__init__(message, project_slug=project_slug, proxito_path=proxito_path)
-        self.project = project
-        self.subproject_slug = subproject_slug
-        self.version_slug = version_slug
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 6e833b88a5b..404e5599682 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -13,6 +13,14 @@
 from readthedocs.api.mixins import CDNCacheTagsMixin
 from readthedocs.builds.constants import EXTERNAL, INTERNAL, LATEST, STABLE
 from readthedocs.builds.models import Version
+from readthedocs.core.exceptions import (
+    ContextualizedHttp404,
+    ProjectHttp404,
+    ProjectPageHttp404,
+    ProjectTranslationHttp404,
+    ProjectVersionHttp404,
+    SubProjectHttp404,
+)
 from readthedocs.core.mixins import CDNCacheControlMixin
 from readthedocs.core.resolver import resolve_path, resolver
 from readthedocs.core.unresolver import (
@@ -27,13 +35,6 @@
 from readthedocs.projects.models import Domain, Feature, ProjectRelationship
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 from readthedocs.proxito.constants import RedirectType
-from readthedocs.proxito.exceptions import (
-    ProxitoHttp404,
-    ProxitoProjectHttp404,
-    ProxitoProjectPageHttp404,
-    ProxitoProjectVersionHttp404,
-    ProxitoSubProjectHttp404,
-)
 from readthedocs.redirects.exceptions import InfiniteRedirectException
 from readthedocs.storage import build_media_storage
 
@@ -72,8 +73,11 @@ def get(self, request, subproject_slug=None, filename=""):
             try:
                 project = project.subprojects.get(alias=subproject_slug).child
             except ProjectRelationship.DoesNotExist:
-                raise ProxitoSubProjectHttp404(
-                    f"Did not find subproject slug {subproject_slug} for project {project.slug}",
+                raise SubProjectHttp404(
+                    message=(
+                        f"Did not find subproject slug {subproject_slug} "
+                        f"for project {project.slug}"
+                    ),
                     project=parent_project,
                     project_slug=parent_project.slug,
                     subproject_slug=subproject_slug,
@@ -171,11 +175,11 @@ def get(
             )
         # This special treatment of ProxitoProjectHttp404 happens because the decorator that
         # resolves a project doesn't know if it's resolving a subproject or a normal project
-        except ProxitoProjectHttp404 as e:
+        except ProjectHttp404 as e:
             if subproject_slug:
                 log.debug("Project expected to be a subproject was not found")
-                raise ProxitoSubProjectHttp404(
-                    f"Could not find subproject for {subproject_slug}",
+                raise SubProjectHttp404(
+                    message=f"Could not find subproject for {subproject_slug}",
                     project_slug=e.project_slug,
                     subproject_slug=subproject_slug,
                 )
@@ -216,7 +220,9 @@ def get(
         # version on the database we want to return 404.
         if (version and not version.active) or (version_slug and not version):
             log.warning("Version does not exist or is not active.")
-            raise ProxitoHttp404("Version does not exist or is not active.")
+            raise ContextualizedHttp404(
+                message="Version does not exist or is not active."
+            )
 
         if version:
             # All public versions can be cached.
@@ -275,7 +281,7 @@ def get(
                 'Invalid URL for project with versions.',
                 filename=filename,
             )
-            raise ProxitoHttp404("Invalid URL for project with versions")
+            raise ContextualizedHttp404(message="Invalid URL for project with versions")
 
         redirect_path, http_status = self.get_redirect(
             project=final_project,
@@ -569,11 +575,11 @@ def get(self, request, proxito_path, template_name="404.html"):
                 filename=kwargs.get("filename", ""),
                 explicit_proxito_path=proxito_path,
             )
-        except ProxitoProjectHttp404 as e:
+        except ProjectHttp404 as e:
             if subproject_slug:
                 log.debug("Project expected to be a subproject was not found")
-                raise ProxitoSubProjectHttp404(
-                    f"Could not find subproject for {subproject_slug}",
+                raise SubProjectHttp404(
+                    message=f"Could not find subproject for {subproject_slug}",
                     project_slug=e.project_slug,
                     subproject_slug=subproject_slug,
                     proxito_path=proxito_path,
@@ -588,7 +594,7 @@ def get(self, request, proxito_path, template_name="404.html"):
         version = Version.objects.filter(
             project=final_project, slug=version_slug
         ).first()
-        version_found = bool(version)
+        bool(version)
 
         # If we were able to resolve to a valid version, it means that the
         # current file doesn't exist. So we check if we can redirect to its
@@ -641,27 +647,7 @@ def get(self, request, proxito_path, template_name="404.html"):
         if response:
             return response
 
-        if version_found:
-            raise ProxitoProjectPageHttp404(
-                "Page not found and no custom 404",
-                project=final_project,
-                project_slug=final_project.slug,
-                proxito_path=proxito_path,
-            )
-        if kwargs.get("subproject_slug"):
-            raise ProxitoSubProjectHttp404(
-                "Subproject not found and no custom 404",
-                project=final_project,
-                project_slug=final_project.slug,
-                subproject_slug=kwargs.get("subproject_slug"),
-                proxito_path=proxito_path,
-            )
-        raise ProxitoProjectVersionHttp404(
-            "Version not found and no custom 404",
-            project=final_project,
-            project_slug=final_project.slug,
-            proxito_path=proxito_path,
-        )
+        raise Http404("No custom 404 page found.")
 
     def _register_broken_link(self, project, version, path, full_path):
         try:
@@ -826,6 +812,9 @@ def get_using_unresolver(self, request, path):
         # Try to map the current path to a project/version/filename.
         # If that fails, we fill the variables with the information we have
         # available in the exceptions.
+
+        contextualized_404_class = ContextualizedHttp404
+
         try:
             unresolved = unresolver.unresolve_path(
                 unresolved_domain=unresolved_domain,
@@ -837,21 +826,26 @@ def get_using_unresolver(self, request, path):
             filename = unresolved.filename
             lang_slug = project.language
             version_slug = version.slug
+            contextualized_404_class = ProjectPageHttp404
         except VersionNotFoundError as exc:
             project = exc.project
             lang_slug = project.language
             version_slug = exc.version_slug
             filename = exc.filename
+            contextualized_404_class = ProjectVersionHttp404
         except TranslationNotFoundError as exc:
             project = exc.project
             lang_slug = exc.language
             version_slug = exc.version_slug
             filename = exc.filename
+            contextualized_404_class = ProjectTranslationHttp404
         except InvalidExternalVersionError as exc:
             project = exc.project
+            # TODO: Use a contextualized 404
         except InvalidPathForVersionedProjectError as exc:
             project = exc.project
             filename = exc.path
+            # TODO: Use a contextualized 404
 
         log.bind(
             project_slug=project.slug,
@@ -914,7 +908,15 @@ def get_using_unresolver(self, request, path):
         )
         if response:
             return response
-        raise Http404("No custom 404 page found.")
+
+        # No custom 404 page, use our contextualized 404 response
+        raise contextualized_404_class(
+            project=project,
+            version=version,
+            filename=filename,
+            lang_slug=project.language,
+            version_slug=version.slug,
+        )
 
 
 class ServeError404(SettingsOverrideObject):
@@ -973,7 +975,7 @@ def get(self, request):
 
         if no_serve_robots_txt:
             # ... we do return a 404
-            raise ProxitoHttp404()
+            raise Http404()
 
         log.bind(
             project_slug=project.slug,
@@ -1110,7 +1112,7 @@ def changefreqs_generator():
             only_active=True,
         )
         if not public_versions.exists():
-            raise ProxitoHttp404()
+            raise Http404()
 
         sorted_versions = sort_version_aware(public_versions)
 

From 9a366bba777f219316ca4c7fe959c508a907b439 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 00:16:58 +0200
Subject: [PATCH 45/70] Revert changes on old proxito decorators implementation

---
 readthedocs/proxito/views/decorators.py | 25 +++++--------------------
 1 file changed, 5 insertions(+), 20 deletions(-)

diff --git a/readthedocs/proxito/views/decorators.py b/readthedocs/proxito/views/decorators.py
index bffad18e684..c44051cd0e9 100644
--- a/readthedocs/proxito/views/decorators.py
+++ b/readthedocs/proxito/views/decorators.py
@@ -2,12 +2,9 @@
 
 import structlog
 from django.db.models import Q
+from django.http import Http404
 
 from readthedocs.projects.models import Project, ProjectRelationship
-from readthedocs.proxito.exceptions import (
-    ProxitoProjectHttp404,
-    ProxitoSubProjectHttp404,
-)
 
 log = structlog.get_logger(__name__)  # noqa
 
@@ -25,15 +22,12 @@ def map_subproject_slug(view_func):
     def inner_view(  # noqa
         request, subproject=None, subproject_slug=None, *args, **kwargs
     ):
-        # Something not entirely clear is happening when unpacking args and kwargs
-        # so the project is fetched from kwarg dictionary.
-        project = kwargs["project"]
         if subproject is None and subproject_slug:
             # Try to fetch by subproject alias first, otherwise we might end up
             # redirected to an unrelated project.
             # Depends on a project passed into kwargs
             rel = (
-                ProjectRelationship.objects.filter(parent=project)
+                ProjectRelationship.objects.filter(parent=kwargs["project"])
                 .filter(Q(alias=subproject_slug) | Q(child__slug=subproject_slug))
                 .first()
             )
@@ -43,15 +37,9 @@ def inner_view(  # noqa
                 log.warning(
                     "The slug is not subproject of project.",
                     subproject_slug=subproject_slug,
-                    project_slug=project.slug,
-                    project=project,
-                )
-                raise ProxitoSubProjectHttp404(
-                    "Invalid subproject slug",
-                    project_slug=project.slug,
-                    project=project,
-                    subproject_slug=subproject_slug,
+                    project_slug=kwargs["project"].slug,
                 )
+                raise Http404("Invalid subproject slug")
         return view_func(request, subproject=subproject, *args, **kwargs)
 
     return inner_view
@@ -83,10 +71,7 @@ def inner_view(  # noqa
                 try:
                     project = Project.objects.get(slug=project_slug)
                 except Project.DoesNotExist:
-                    log.debug(f"No project found with slug {project_slug}")
-                    raise ProxitoProjectHttp404(
-                        "Project does not exist.", project_slug=project_slug
-                    )
+                    raise Http404("Project does not exist.")
         return view_func(request, project=project, *args, **kwargs)
 
     return inner_view

From 651161e173efc3e5e6a265cc574da2efdc5d778b Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 11:25:34 +0200
Subject: [PATCH 46/70] WIP: refactor

---
 readthedocs/proxito/middleware.py  | 6 +++---
 readthedocs/proxito/views/utils.py | 6 +++---
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index ac5a4aa7b8b..247e52ddd86 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -16,6 +16,7 @@
 from django.urls import reverse
 from django.utils.deprecation import MiddlewareMixin
 
+from readthedocs.core.exceptions import ProjectHttp404
 from readthedocs.core.unresolver import (
     InvalidCustomDomainError,
     InvalidExternalDomainError,
@@ -26,7 +27,6 @@
 )
 from readthedocs.core.utils import get_cache_tag
 from readthedocs.projects.models import Feature, Project
-from readthedocs.proxito.exceptions import ProxitoProjectHttp404
 
 log = structlog.get_logger(__name__)
 
@@ -224,8 +224,8 @@ def process_request(self, request):  # noqa
             # We pass e.domain as 'project_slug' because we cannot unpack a slug from the
             # domain at this stage. The domain can be a custom domain docs.foobar.tld,
             # in which case 'docs' would be an incorrect slug.
-            raise ProxitoProjectHttp404(
-                f"No project found for requested domain {exc.domain}.",
+            raise ProjectHttp404(
+                message=f"No project found for requested domain {exc.domain}.",
                 project_slug=exc.domain,
                 proxito_path=request.path,
             )
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 4d60d1eeb30..65b642cf59c 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -3,8 +3,8 @@
 import structlog
 from django.http import HttpResponse
 
+from readthedocs.core.exceptions import ProjectTranslationHttp404
 from readthedocs.projects.models import Project
-from readthedocs.proxito.exceptions import ProxitoProjectTranslationHttp404
 
 from ...core.views import server_error_404
 from .decorators import map_project_slug, map_subproject_slug
@@ -81,8 +81,8 @@ def _get_project_data_from_request(
         try:
             final_project = current_project.translations.get(language=lang_slug)
         except Project.DoesNotExist:
-            raise ProxitoProjectTranslationHttp404(
-                "Did not find translation",
+            raise ProjectTranslationHttp404(
+                message="Did not find translation",
                 project=current_project,
                 project_slug=current_project.slug,
                 translation_slug=lang_slug,

From 3b7f4c06b2c9640455d3230bc7c0413aebefad61 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 14:21:15 +0200
Subject: [PATCH 47/70] Remove custom HTTP responses from most old proxito
 code, include CNAME DNS in contextualized 404 pattern, clean up context
 pattern a bit

---
 readthedocs/core/exceptions.py                | 41 ++++++++++++++++++-
 readthedocs/core/views/__init__.py            |  6 ++-
 readthedocs/proxito/middleware.py             | 20 ++++-----
 readthedocs/proxito/views/serve.py            | 41 +++++++------------
 .../errors/404/dns.html}                      | 26 ++++--------
 .../errors/404/include_error_message.html     |  2 +-
 .../templates/errors/404/include_tips.html    |  2 +-
 .../templates/errors/404/no_language.html     |  2 +-
 .../templates/errors/404/no_project.html      |  4 +-
 .../templates/errors/404/no_project_page.html |  4 +-
 .../templates/errors/404/no_subproject.html   |  2 +-
 .../templates/errors/404/no_version.html      |  2 +-
 12 files changed, 82 insertions(+), 70 deletions(-)
 rename readthedocs/{core/templates/core/dns-404.html => templates/errors/404/dns.html} (70%)

diff --git a/readthedocs/core/exceptions.py b/readthedocs/core/exceptions.py
index 36f3b887fcf..935c2b99a2f 100644
--- a/readthedocs/core/exceptions.py
+++ b/readthedocs/core/exceptions.py
@@ -1,19 +1,52 @@
 from django.http import Http404
+from django.utils.translation import gettext_lazy as _
 
 
 class ContextualizedHttp404(Http404):
 
+    """
+    Base class for contextualized HTTP 404 handling.
+
+    Subclasses may define their own template name,
+    HTTP status and object that was not found.
+
+    The contextualized exception is handled by the project's 404 view.
+    """
+
     template_name = "errors/404/base.html"
+    not_found_subject = _("page")
 
     def __init__(self, **kwargs):
-        self.context = kwargs
+        self.http_status = kwargs.pop("http_status", 404)
+        self.kwargs = kwargs
+
+    def get_context(self):
+        c = {
+            "not_found_subject": self.not_found_subject,
+        }
+        c.update(self.kwargs)
+        return c
+
+
+class DomainDNSHttp404(ContextualizedHttp404):
+
+    """Raised if a DNS record points to us but we do not know the domain."""
+
+    template_name = "errors/404/dns.html"
+    not_found_subject = _("domain matching DNS record")
 
 
 class ProjectHttp404(ContextualizedHttp404):
 
-    """Raised if a project was not found."""
+    """
+    Raised if a domain did not resolve to a project.
+
+    This is currently used very broadly.
+    It indicates a number of reasons for the user.
+    """
 
     template_name = "errors/404/no_project.html"
+    not_found_subject = _("project")
 
 
 class SubProjectHttp404(ContextualizedHttp404):
@@ -21,6 +54,7 @@ class SubProjectHttp404(ContextualizedHttp404):
     """Raised if a subproject was not found."""
 
     template_name = "errors/404/no_subproject.html"
+    not_found_subject = _("sub project")
 
 
 class ProjectPageHttp404(ContextualizedHttp404):
@@ -28,6 +62,7 @@ class ProjectPageHttp404(ContextualizedHttp404):
     """Raised if a page inside an existing project was not found."""
 
     template_name = "errors/404/no_project_page.html"
+    not_found_subject = _("documentation page")
 
 
 class ProjectTranslationHttp404(ContextualizedHttp404):
@@ -40,6 +75,7 @@ class ProjectTranslationHttp404(ContextualizedHttp404):
     """
 
     template_name = "errors/404/no_language.html"
+    not_found_subject = _("translation")
 
 
 class ProjectVersionHttp404(ContextualizedHttp404):
@@ -51,3 +87,4 @@ class ProjectVersionHttp404(ContextualizedHttp404):
     """
 
     template_name = "errors/404/no_version.html"
+    not_found_subject = _("documentation version")
diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index ceddccc21e0..4d57506ed0b 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -77,14 +77,16 @@ def server_error_404(request, template_name="errors/404/base.html", exception=No
     """
 
     context = {}
+    http_status = 404
 
     # Contextualized 404 exceptions:
     # Context is defined by the views that raise these exceptions and handled
     # in their templates.
     if isinstance(exception, ContextualizedHttp404):
         # These attributes are not guaranteed.
-        context.update(exception.context)
+        context.update(exception.get_context())
         template_name = exception.template_name
+        http_status = exception.http_status
 
     context["path_not_found"] = context.get("path_not_found") or request.path
 
@@ -93,7 +95,7 @@ def server_error_404(request, template_name="errors/404/base.html", exception=No
         template_name,
         context=context,
     )
-    r.status_code = 404
+    r.status_code = http_status
     return r
 
 
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index 247e52ddd86..41549c07c27 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -12,11 +12,11 @@
 import structlog
 from django.conf import settings
 from django.core.exceptions import SuspiciousOperation
-from django.shortcuts import redirect, render
+from django.shortcuts import redirect
 from django.urls import reverse
 from django.utils.deprecation import MiddlewareMixin
 
-from readthedocs.core.exceptions import ProjectHttp404
+from readthedocs.core.exceptions import DomainDNSHttp404, ProjectHttp404
 from readthedocs.core.unresolver import (
     InvalidCustomDomainError,
     InvalidExternalDomainError,
@@ -213,11 +213,9 @@ def process_request(self, request):  # noqa
             unresolved_domain = unresolver.unresolve_domain_from_request(request)
         except SuspiciousHostnameError as exc:
             log.warning("Weird variation on our hostname.", domain=exc.domain)
-            return render(
-                request,
-                "core/dns-404.html",
-                context={"host": exc.domain},
-                status=400,
+            raise DomainDNSHttp404(
+                http_status=400,
+                host=exc.domain,
             )
         except (InvalidSubdomainError, InvalidExternalDomainError) as exc:
             log.debug("Invalid project set on the subdomain.")
@@ -225,15 +223,13 @@ def process_request(self, request):  # noqa
             # domain at this stage. The domain can be a custom domain docs.foobar.tld,
             # in which case 'docs' would be an incorrect slug.
             raise ProjectHttp404(
-                message=f"No project found for requested domain {exc.domain}.",
-                project_slug=exc.domain,
-                proxito_path=request.path,
+                domain=exc.domain,
             )
         except InvalidCustomDomainError as exc:
             # Some person is CNAMEing to us without configuring a domain - 404.
             log.debug("CNAME 404.", domain=exc.domain)
-            return render(
-                request, "core/dns-404.html", context={"host": exc.domain}, status=404
+            raise DomainDNSHttp404(
+                host=exc.domain,
             )
         except InvalidXRTDSlugHeaderError:
             raise SuspiciousOperation("Invalid X-RTD-Slug header.")
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 404e5599682..bbd360a0af1 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -15,11 +15,9 @@
 from readthedocs.builds.models import Version
 from readthedocs.core.exceptions import (
     ContextualizedHttp404,
-    ProjectHttp404,
     ProjectPageHttp404,
     ProjectTranslationHttp404,
     ProjectVersionHttp404,
-    SubProjectHttp404,
 )
 from readthedocs.core.mixins import CDNCacheControlMixin
 from readthedocs.core.resolver import resolve_path, resolver
@@ -66,22 +64,17 @@ def get(self, request, subproject_slug=None, filename=""):
 
         unresolved_domain = request.unresolved_domain
         project = unresolved_domain.project
-        parent_project = project
 
         # Use the project from the domain, or use the subproject slug.
         if subproject_slug:
             try:
                 project = project.subprojects.get(alias=subproject_slug).child
             except ProjectRelationship.DoesNotExist:
-                raise SubProjectHttp404(
-                    message=(
+                raise Http404(
+                    (
                         f"Did not find subproject slug {subproject_slug} "
                         f"for project {project.slug}"
                     ),
-                    project=parent_project,
-                    project_slug=parent_project.slug,
-                    subproject_slug=subproject_slug,
-                    proxito_path=request.proxito_path,
                 )
 
         # Get the default version from the current project,
@@ -173,15 +166,11 @@ def get(
                 version_slug=version_slug,
                 filename=filename,
             )
-        # This special treatment of ProxitoProjectHttp404 happens because the decorator that
-        # resolves a project doesn't know if it's resolving a subproject or a normal project
-        except ProjectHttp404 as e:
+        except Http404:
             if subproject_slug:
                 log.debug("Project expected to be a subproject was not found")
-                raise SubProjectHttp404(
-                    message=f"Could not find subproject for {subproject_slug}",
-                    project_slug=e.project_slug,
-                    subproject_slug=subproject_slug,
+                raise Http404(
+                    f"Could not find subproject '{subproject_slug}' in project {project_slug}",
                 )
             raise
         version = final_project.versions.filter(slug=version_slug).first()
@@ -220,9 +209,7 @@ def get(
         # version on the database we want to return 404.
         if (version and not version.active) or (version_slug and not version):
             log.warning("Version does not exist or is not active.")
-            raise ContextualizedHttp404(
-                message="Version does not exist or is not active."
-            )
+            raise Http404("Version does not exist or is not active.")
 
         if version:
             # All public versions can be cached.
@@ -281,7 +268,7 @@ def get(
                 'Invalid URL for project with versions.',
                 filename=filename,
             )
-            raise ContextualizedHttp404(message="Invalid URL for project with versions")
+            raise Http404("Invalid URL for project with versions")
 
         redirect_path, http_status = self.get_redirect(
             project=final_project,
@@ -381,6 +368,9 @@ def get_using_unresolver(self, request):
         if unresolved_domain.is_from_external_domain:
             self.version_type = EXTERNAL
 
+        # 404 errors aren't contextualized because they are sent to the HTTP proxy
+        # The path will be 'unresolved' again when HTTP server handles the 404 error
+        # See: ServeError404Base
         try:
             unresolved = unresolver.unresolve_path(
                 unresolved_domain=unresolved_domain,
@@ -556,7 +546,7 @@ def get(self, request, proxito_path, template_name="404.html"):
 
         version_slug = kwargs.get('version_slug')
         version_slug = self.get_version_from_host(request, version_slug)
-        # This special treatment of ProxitoProjectHttp404 happens because the decorator that
+        # This special treatment of Http404 happens because the decorator that
         # resolves a project doesn't know if it's resolving a subproject or a normal project
         subproject_slug = kwargs.get("subproject_slug")
         project_slug = kwargs.get("project_slug")
@@ -575,14 +565,11 @@ def get(self, request, proxito_path, template_name="404.html"):
                 filename=kwargs.get("filename", ""),
                 explicit_proxito_path=proxito_path,
             )
-        except ProjectHttp404 as e:
+        except Http404 as e:
             if subproject_slug:
                 log.debug("Project expected to be a subproject was not found")
-                raise SubProjectHttp404(
-                    message=f"Could not find subproject for {subproject_slug}",
-                    project_slug=e.project_slug,
-                    subproject_slug=subproject_slug,
-                    proxito_path=proxito_path,
+                raise Http404(
+                    f"Could not find subproject '{subproject_slug}' for project {e.project_slug}",
                 )
             raise
 
diff --git a/readthedocs/core/templates/core/dns-404.html b/readthedocs/templates/errors/404/dns.html
similarity index 70%
rename from readthedocs/core/templates/core/dns-404.html
rename to readthedocs/templates/errors/404/dns.html
index d611955b1b0..22917c74820 100644
--- a/readthedocs/core/templates/core/dns-404.html
+++ b/readthedocs/templates/errors/404/dns.html
@@ -1,23 +1,9 @@
-{% extends "base.html" %}
+{% extends "errors/404/base.html" %}
 {% load core_tags %}
 {% load i18n %}
 
-{% block title %}
-  {% trans "Maze Found - Invalid Host" %}
-{% endblock %}
-
-{% block header-wrapper %}
-  {% include "error_header.html" %}
-{% endblock %}
-
-{% block notify %}{% endblock %}
+{% block 404_error_message %}
 
-{# Hide the language select form so we don't set a CSRF cookie #}
-{% block language-select-form %}{% endblock %}
-
-
-{% block content %}
-
{% trans '404 - Invalid Host' %}

 
{% blocktrans %}The host "{{ host }}" is unknown to Read the Docs{% endblocktrans %}

 
 

@@ -33,11 +19,15 @@ 
{% blocktrans %}The host "{{ host }}" is unknown to Read the Docs{% endblock
   {% endwith %}
 

 
+

+
{% trans "Are you the project owner?" %}

 
-
    
+
    {% trans "Here are some tips to address 404 errors:" %}
    
+
+
      
   
    • {% trans 'Ensure you have a CNAME record pointing to readthedocs.io' %}
      • 
   
    • {% trans 'Add your desired domain in the Read the Docs dashboard for your project (under Your Project >> Admin >> Domains)' %}
      • 
 
    
-
+

 
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/include_error_message.html b/readthedocs/templates/errors/404/include_error_message.html
index d3b3d20144f..8f9517fb5ec 100644
--- a/readthedocs/templates/errors/404/include_error_message.html
+++ b/readthedocs/templates/errors/404/include_error_message.html
@@ -1,7 +1,7 @@
 {% load i18n %}
 

   

-    {% blocktrans trimmed %}
+    {% blocktrans trimmed with project_slug=project.slug|default:_("unknown slug") %}
       You are browsing the documentation of {{ project_slug }}.
       The {{ not_found_subject }} you are looking for at {{ path_not_found }} was not found.
     {% endblocktrans %}
diff --git a/readthedocs/templates/errors/404/include_tips.html b/readthedocs/templates/errors/404/include_tips.html
index d507a918d33..ba8acad77c2 100644
--- a/readthedocs/templates/errors/404/include_tips.html
+++ b/readthedocs/templates/errors/404/include_tips.html
@@ -5,7 +5,7 @@ 
{% trans "Are you the project owner?" %}

 
{% trans "Here are some tips to address 404 errors:" %}

 
 
 

diff --git a/readthedocs/templates/errors/404/no_language.html b/readthedocs/templates/errors/404/no_language.html
index aba185f20d3..03d8c79670c 100644
--- a/readthedocs/templates/errors/404/no_language.html
+++ b/readthedocs/templates/errors/404/no_language.html
@@ -4,7 +4,7 @@
 {# project exists but not in the requested language #}
 
 {% block 404_error_message %}
-  {% include "errors/404/include_error_message.html" with not_found_subject="language" %}
+  {% include "errors/404/include_error_message.html" %}
   {% include "errors/404/include_search.html" %}
   {% include "errors/404/include_tips.html" %}
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
index fc640c3b185..8642ec0562f 100644
--- a/readthedocs/templates/errors/404/no_project.html
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -6,8 +6,8 @@
 {% block 404_error_message %}
   

     

-      {% blocktrans trimmed %}
-        The project {{ project_slug }} that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
+      {% blocktrans trimmed with domain=domain|default:_("no domain") %}
+        The project for {{ domain }} that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
       {% endblocktrans %}
     

   

diff --git a/readthedocs/templates/errors/404/no_project_page.html b/readthedocs/templates/errors/404/no_project_page.html
index 7cf480eaca0..4b10444fd8d 100644
--- a/readthedocs/templates/errors/404/no_project_page.html
+++ b/readthedocs/templates/errors/404/no_project_page.html
@@ -4,7 +4,7 @@
 {# project exists but page doesn't #}
 
 {% block 404_error_message %}
-  {% include "errors/404/include_error_message.html" with not_found_subject="page" %}
-  {% include "errors/404/include_search.html" with search_project=project_slug %}
+  {% include "errors/404/include_error_message.html" %}
+  {% include "errors/404/include_search.html" with search_project=project.slug %}
   {% include "errors/404/include_tips.html" %}
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/no_subproject.html b/readthedocs/templates/errors/404/no_subproject.html
index e85ce7d448f..c6635f691a7 100644
--- a/readthedocs/templates/errors/404/no_subproject.html
+++ b/readthedocs/templates/errors/404/no_subproject.html
@@ -4,7 +4,7 @@
 {# project exists but subproject doesn't #}
 
 {% block 404_error_message %}
-  {% include "errors/404/include_error_message.html" with not_found_subject="subproject" %}
+  {% include "errors/404/include_error_message.html" %}
   {% include "errors/404/include_search.html" %}
   {% include "errors/404/include_tips.html" %}
 {% endblock %}
diff --git a/readthedocs/templates/errors/404/no_version.html b/readthedocs/templates/errors/404/no_version.html
index 6405d3e80fa..4870c6daa09 100644
--- a/readthedocs/templates/errors/404/no_version.html
+++ b/readthedocs/templates/errors/404/no_version.html
@@ -4,7 +4,7 @@
 {# project exists but version doesn't #}
 
 {% block 404_error_message %}
-  {% include "errors/404/include_error_message.html" with not_found_subject="project version" %}
+  {% include "errors/404/include_error_message.html" %}
   {% include "errors/404/include_search.html" %}
   {% include "errors/404/include_tips.html" %}
 {% endblock %}

From 7c0c00cb878e2e11f29cd7c41e8e5bd27b8d9658 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 14:26:46 +0200
Subject: [PATCH 48/70] Remove more changes to old proxito code

---
 readthedocs/proxito/views/serve.py | 72 +++++++++++-------------------
 1 file changed, 27 insertions(+), 45 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index bbd360a0af1..bb0b728900d 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -152,28 +152,19 @@ def get(
         original_version_slug = version_slug
         version_slug = self.get_version_from_host(request, version_slug)
 
-        try:
-            (
-                final_project,
-                lang_slug,
-                version_slug,
-                filename,
-            ) = _get_project_data_from_request(  # noqa
-                request,
-                project_slug=project_slug,
-                subproject_slug=subproject_slug,
-                lang_slug=lang_slug,
-                version_slug=version_slug,
-                filename=filename,
-            )
-        except Http404:
-            if subproject_slug:
-                log.debug("Project expected to be a subproject was not found")
-                raise Http404(
-                    f"Could not find subproject '{subproject_slug}' in project {project_slug}",
-                )
-            raise
-        version = final_project.versions.filter(slug=version_slug).first()
+        (
+            final_project,
+            lang_slug,
+            version_slug,
+            filename,
+        ) = _get_project_data_from_request(  # noqa
+            request,
+            project_slug=project_slug,
+            subproject_slug=subproject_slug,
+            lang_slug=lang_slug,
+            version_slug=version_slug,
+            filename=filename,
+        )
 
         is_external = unresolved_domain.is_from_external_domain
         if (
@@ -550,28 +541,20 @@ def get(self, request, proxito_path, template_name="404.html"):
         # resolves a project doesn't know if it's resolving a subproject or a normal project
         subproject_slug = kwargs.get("subproject_slug")
         project_slug = kwargs.get("project_slug")
-        try:
-            (
-                final_project,
-                lang_slug,
-                version_slug,
-                filename,
-            ) = _get_project_data_from_request(  # noqa
-                request,
-                project_slug,
-                subproject_slug,
-                lang_slug=kwargs.get("lang_slug"),
-                version_slug=version_slug,
-                filename=kwargs.get("filename", ""),
-                explicit_proxito_path=proxito_path,
-            )
-        except Http404 as e:
-            if subproject_slug:
-                log.debug("Project expected to be a subproject was not found")
-                raise Http404(
-                    f"Could not find subproject '{subproject_slug}' for project {e.project_slug}",
-                )
-            raise
+        (
+            final_project,
+            lang_slug,
+            version_slug,
+            filename,
+        ) = _get_project_data_from_request(  # noqa
+            request,
+            project_slug,
+            subproject_slug,
+            lang_slug=kwargs.get("lang_slug"),
+            version_slug=version_slug,
+            filename=kwargs.get("filename", ""),
+            explicit_proxito_path=proxito_path,
+        )
 
         log.bind(
             project_slug=final_project.slug,
@@ -581,7 +564,6 @@ def get(self, request, proxito_path, template_name="404.html"):
         version = Version.objects.filter(
             project=final_project, slug=version_slug
         ).first()
-        bool(version)
 
         # If we were able to resolve to a valid version, it means that the
         # current file doesn't exist. So we check if we can redirect to its

From 93d7c48d25b91847c436aea6ce8c51b3120a508f Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 14:35:34 +0200
Subject: [PATCH 49/70] Remove more old proxito implementation changes

---
 readthedocs/proxito/views/serve.py |  7 ++-----
 readthedocs/proxito/views/utils.py | 15 ++-------------
 2 files changed, 4 insertions(+), 18 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index bb0b728900d..b7e6fedb093 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -539,8 +539,6 @@ def get(self, request, proxito_path, template_name="404.html"):
         version_slug = self.get_version_from_host(request, version_slug)
         # This special treatment of Http404 happens because the decorator that
         # resolves a project doesn't know if it's resolving a subproject or a normal project
-        subproject_slug = kwargs.get("subproject_slug")
-        project_slug = kwargs.get("project_slug")
         (
             final_project,
             lang_slug,
@@ -548,12 +546,11 @@ def get(self, request, proxito_path, template_name="404.html"):
             filename,
         ) = _get_project_data_from_request(  # noqa
             request,
-            project_slug,
-            subproject_slug,
+            project_slug=kwargs.get("project_slug"),
+            subproject_slug=kwargs.get("subproject_slug"),
             lang_slug=kwargs.get("lang_slug"),
             version_slug=version_slug,
             filename=kwargs.get("filename", ""),
-            explicit_proxito_path=proxito_path,
         )
 
         log.bind(
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 65b642cf59c..2350091afd1 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -1,9 +1,8 @@
 import os
 
 import structlog
-from django.http import HttpResponse
+from django.http import Http404, HttpResponse
 
-from readthedocs.core.exceptions import ProjectTranslationHttp404
 from readthedocs.projects.models import Project
 
 from ...core.views import server_error_404
@@ -50,15 +49,11 @@ def _get_project_data_from_request(
     lang_slug=None,
     version_slug=None,
     filename="",
-    explicit_proxito_path=None,
 ):
     """
     Get the proper project based on the request and URL.
 
     This is used in a few places and so we break out into a utility function.
-
-    :param str explicit_proxito_path: Set when called from the proxito 404 handler and
-      request.path contains additional prefix.
     """
     # Take the most relevant project so far
     current_project = subproject or project
@@ -81,13 +76,7 @@ def _get_project_data_from_request(
         try:
             final_project = current_project.translations.get(language=lang_slug)
         except Project.DoesNotExist:
-            raise ProjectTranslationHttp404(
-                message="Did not find translation",
-                project=current_project,
-                project_slug=current_project.slug,
-                translation_slug=lang_slug,
-                proxito_path=explicit_proxito_path or request.path,
-            )
+            raise Http404("Did not find translation")
 
     # Handle single version by grabbing the default version
     # We might have version_slug when we're serving a PR

From a1361d47029f3a2c2f9c4cb0e1c612548d33c27d Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 14:43:13 +0200
Subject: [PATCH 50/70] Use PRODUCTION_DOMAIN for search form action

---
 readthedocs/templates/errors/404/include_search.html | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/readthedocs/templates/errors/404/include_search.html b/readthedocs/templates/errors/404/include_search.html
index e95956d7042..db65c0c421e 100644
--- a/readthedocs/templates/errors/404/include_search.html
+++ b/readthedocs/templates/errors/404/include_search.html
@@ -5,9 +5,9 @@ 
{% trans "Try searching?" %}

 

   {% if search_project %}
     {% with "project:"|add:search_project as search_term %}
-      {% include "core/widesearchbar.html" with form_action="https://readthedocs.org/search/" term=search_term %}
+      {% include "core/widesearchbar.html" with form_action="//"|add:PRODUCTION_DOMAIN|add:"/search/" term=search_term %}
     {% endwith %}
   {% else %}
-    {% include "core/widesearchbar.html" with form_action="https://readthedocs.org/search/" %}
+    {% include "core/widesearchbar.html" with form_action="//"|add:PRODUCTION_DOMAIN|add:"/search/" %}
   {% endif %}
 


From d20187cb2e19711eb5d065aa5f6b7a841ac71e2d Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 16:30:28 +0200
Subject: [PATCH 51/70] Remove custom 404 handler from general app, have only
 that logic in proxito (for now?)

---
 readthedocs/core/views/__init__.py | 32 --------------------------
 readthedocs/proxito/views/utils.py | 36 ++++++++++++++++++++++--------
 readthedocs/urls.py                |  3 ---
 3 files changed, 27 insertions(+), 44 deletions(-)

diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py
index 4d57506ed0b..b53533c83e1 100644
--- a/readthedocs/core/views/__init__.py
+++ b/readthedocs/core/views/__init__.py
@@ -12,7 +12,6 @@
 from django.urls import reverse
 from django.views.generic import TemplateView, View
 
-from readthedocs.core.exceptions import ContextualizedHttp404
 from readthedocs.core.mixins import CDNCacheControlMixin, PrivateViewMixin
 from readthedocs.projects.models import Project
 
@@ -68,37 +67,6 @@ def get_context_data(self, **kwargs):
         return context
 
 
-def server_error_404(request, template_name="errors/404/base.html", exception=None):
-    """
-    Serves a 404 error message, handling 404 exception types raised throughout the app.
-
-    Notice that handling of 404 errors happens elsewhere in views and middleware,
-    this view is expected to serve an actual 404 message.
-    """
-
-    context = {}
-    http_status = 404
-
-    # Contextualized 404 exceptions:
-    # Context is defined by the views that raise these exceptions and handled
-    # in their templates.
-    if isinstance(exception, ContextualizedHttp404):
-        # These attributes are not guaranteed.
-        context.update(exception.get_context())
-        template_name = exception.template_name
-        http_status = exception.http_status
-
-    context["path_not_found"] = context.get("path_not_found") or request.path
-
-    r = render(
-        request,
-        template_name,
-        context=context,
-    )
-    r.status_code = http_status
-    return r
-
-
 def server_error_500(request, template_name='500.html'):
     """A simple 500 handler so we get media."""
     r = render(request, template_name)
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 2350091afd1..dc604a74879 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -2,10 +2,11 @@
 
 import structlog
 from django.http import Http404, HttpResponse
+from django.shortcuts import render
 
 from readthedocs.projects.models import Project
 
-from ...core.views import server_error_404
+from ...core.exceptions import ContextualizedHttp404
 from .decorators import map_project_slug, map_subproject_slug
 
 log = structlog.get_logger(__name__)  # noqa
@@ -22,22 +23,39 @@ def fast_404(request, *args, **kwargs):
 
 
 def proxito_404_page_handler(
-    request, exception=None, template_name="errors/404/base.html"
+    request, template_name="errors/404/base.html", exception=None
 ):
     """
-    Decide what 404 page return depending if it's an internal NGINX redirect.
+    Serves a 404 error message, handling 404 exception types raised throughout the app.
 
-    We want to return fast when the 404 is used as an internal NGINX redirect to
-    reach our ``ServeError404`` view. However, if the 404 exception was risen
-    inside ``ServeError404`` view, we want to render the default Read the Docs
-    Maze page.
+    Notice that handling of 404 errors happens elsewhere in views and middleware,
+    this view is expected to serve an actual 404 message.
     """
 
     if request.resolver_match and request.resolver_match.url_name != 'proxito_404_handler':
         return fast_404(request, exception, template_name)
 
-    # Serve the general 404 error message
-    return server_error_404(request, exception=exception, template_name=template_name)
+    context = {}
+    http_status = 404
+
+    # Contextualized 404 exceptions:
+    # Context is defined by the views that raise these exceptions and handled
+    # in their templates.
+    if isinstance(exception, ContextualizedHttp404):
+        # These attributes are not guaranteed.
+        context.update(exception.get_context())
+        template_name = exception.template_name
+        http_status = exception.http_status
+
+    context["path_not_found"] = context.get("path_not_found") or request.path
+
+    r = render(
+        request,
+        template_name,
+        context=context,
+    )
+    r.status_code = http_status
+    return r
 
 
 @map_project_slug
diff --git a/readthedocs/urls.py b/readthedocs/urls.py
index fef8fd0c4fe..f87a0b8b834 100644
--- a/readthedocs/urls.py
+++ b/readthedocs/urls.py
@@ -13,14 +13,12 @@
     HomepageView,
     SupportView,
     do_not_track,
-    server_error_404,
     server_error_500,
 )
 from readthedocs.search.views import GlobalSearchView
 
 admin.autodiscover()
 
-handler404 = server_error_404
 handler500 = server_error_500
 
 basic_urls = [
@@ -131,7 +129,6 @@
         document_root=os.path.join(settings.MEDIA_ROOT, build_format),
     )
 debug_urls += [
-    path("404/", server_error_404),
     path(
         "style-catalog/",
         TemplateView.as_view(template_name="style_catalog.html"),

From b0e8db45b251eb444f7aca53d93a4f4f6478e9ae Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 16:45:49 +0200
Subject: [PATCH 52/70] Remove a couple of additional unused patterns

---
 readthedocs/proxito/views/serve.py | 16 +++++-----------
 readthedocs/proxito/views/utils.py | 13 +++++--------
 2 files changed, 10 insertions(+), 19 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index b7e6fedb093..79b132e1474 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -5,7 +5,7 @@
 import structlog
 from django.conf import settings
 from django.http import Http404, HttpResponse, HttpResponseRedirect
-from django.shortcuts import render
+from django.shortcuts import get_object_or_404, render
 from django.urls import resolve as url_resolve
 from django.views import View
 
@@ -30,7 +30,7 @@
 )
 from readthedocs.core.utils.extend import SettingsOverrideObject
 from readthedocs.projects import constants
-from readthedocs.projects.models import Domain, Feature, ProjectRelationship
+from readthedocs.projects.models import Domain, Feature
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 from readthedocs.proxito.constants import RedirectType
 from readthedocs.redirects.exceptions import InfiniteRedirectException
@@ -67,15 +67,9 @@ def get(self, request, subproject_slug=None, filename=""):
 
         # Use the project from the domain, or use the subproject slug.
         if subproject_slug:
-            try:
-                project = project.subprojects.get(alias=subproject_slug).child
-            except ProjectRelationship.DoesNotExist:
-                raise Http404(
-                    (
-                        f"Did not find subproject slug {subproject_slug} "
-                        f"for project {project.slug}"
-                    ),
-                )
+            project = get_object_or_404(
+                project.subprojects, alias=subproject_slug
+            ).child
 
         # Get the default version from the current project,
         # or the version from the external domain.
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index dc604a74879..59837296ac7 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -1,10 +1,8 @@
 import os
 
 import structlog
-from django.http import Http404, HttpResponse
-from django.shortcuts import render
-
-from readthedocs.projects.models import Project
+from django.http import HttpResponse
+from django.shortcuts import get_object_or_404, render
 
 from ...core.exceptions import ContextualizedHttp404
 from .decorators import map_project_slug, map_subproject_slug
@@ -91,10 +89,9 @@ def _get_project_data_from_request(
     if not lang_slug or lang_slug == current_project.language:
         final_project = current_project
     else:
-        try:
-            final_project = current_project.translations.get(language=lang_slug)
-        except Project.DoesNotExist:
-            raise Http404("Did not find translation")
+        final_project = get_object_or_404(
+            current_project.translations.all(), language=lang_slug
+        )
 
     # Handle single version by grabbing the default version
     # We might have version_slug when we're serving a PR

From f0747c5b0078114aa9edf44acebb6d232e8d6105 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 17:03:23 +0200
Subject: [PATCH 53/70] Comment accuracy update

---
 readthedocs/proxito/middleware.py  | 6 +++---
 readthedocs/proxito/views/utils.py | 6 ++++--
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index 41549c07c27..02339803d77 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -213,21 +213,21 @@ def process_request(self, request):  # noqa
             unresolved_domain = unresolver.unresolve_domain_from_request(request)
         except SuspiciousHostnameError as exc:
             log.warning("Weird variation on our hostname.", domain=exc.domain)
+            # Raise a contextualized 404 that will be handled by proxito's 404 handler
             raise DomainDNSHttp404(
                 http_status=400,
                 host=exc.domain,
             )
         except (InvalidSubdomainError, InvalidExternalDomainError) as exc:
             log.debug("Invalid project set on the subdomain.")
-            # We pass e.domain as 'project_slug' because we cannot unpack a slug from the
-            # domain at this stage. The domain can be a custom domain docs.foobar.tld,
-            # in which case 'docs' would be an incorrect slug.
+            # Raise a contextualized 404 that will be handled by proxito's 404 handler
             raise ProjectHttp404(
                 domain=exc.domain,
             )
         except InvalidCustomDomainError as exc:
             # Some person is CNAMEing to us without configuring a domain - 404.
             log.debug("CNAME 404.", domain=exc.domain)
+            # Raise a contextualized 404 that will be handled by proxito's 404 handler
             raise DomainDNSHttp404(
                 host=exc.domain,
             )
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 59837296ac7..67c839096bd 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -26,10 +26,12 @@ def proxito_404_page_handler(
     """
     Serves a 404 error message, handling 404 exception types raised throughout the app.
 
-    Notice that handling of 404 errors happens elsewhere in views and middleware,
-    this view is expected to serve an actual 404 message.
+    We want to return fast when the 404 is used as an internal NGINX redirect to
+    reach our ``ServeError404`` view. However, if the 404 exception was risen
+    inside ``ServeError404`` view, we want to render a useful HTML response.
     """
 
+    # This type of request should have a fast response with no HTML rendered
     if request.resolver_match and request.resolver_match.url_name != 'proxito_404_handler':
         return fast_404(request, exception, template_name)
 

From 883aa503075192632a91a3dfe17def8752d3982f Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 17:26:02 +0200
Subject: [PATCH 54/70] Update another comment

---
 readthedocs/templates/404.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/templates/404.html b/readthedocs/templates/404.html
index 790202fa6a0..81bb1261bc4 100644
--- a/readthedocs/templates/404.html
+++ b/readthedocs/templates/404.html
@@ -1,2 +1,2 @@
 {% extends "errors/404/base.html" %}
-{# this is a deprecated template, will be removed. Please use one of the targeted 404s in templates/404/ #}
+{# this is the default template, used by Django's default 404 handler. Proxito has its own 404 view, but the main application doesn't (yet) #}

From e70cec460833252fabae3166bd21a769960cf167 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 17:30:21 +0200
Subject: [PATCH 55/70] Revert DEBUG=False

---
 readthedocs/settings/base.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py
index 0e761d27ca4..415e261cb5a 100644
--- a/readthedocs/settings/base.py
+++ b/readthedocs/settings/base.py
@@ -46,7 +46,7 @@ class CommunityBaseSettings(Settings):
     DEFAULT_AUTO_FIELD = 'django.db.models.AutoField'
 
     # Debug settings
-    DEBUG = False
+    DEBUG = True
     RTD_FORCE_SHOW_DEBUG_TOOLBAR = False
 
     @property

From f19541956e417d34855e0c13ceffb86598d1e87d Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 17:45:12 +0200
Subject: [PATCH 56/70] Move exceptions from core to proxito + use "path" from
 404 serve view in ContextualizedHttp404

---
 readthedocs/{core => proxito}/exceptions.py | 10 ++++++++++
 readthedocs/proxito/middleware.py           |  3 ++-
 readthedocs/proxito/views/serve.py          | 13 +++++++------
 readthedocs/proxito/views/utils.py          |  2 +-
 4 files changed, 20 insertions(+), 8 deletions(-)
 rename readthedocs/{core => proxito}/exceptions.py (82%)

diff --git a/readthedocs/core/exceptions.py b/readthedocs/proxito/exceptions.py
similarity index 82%
rename from readthedocs/core/exceptions.py
rename to readthedocs/proxito/exceptions.py
index 935c2b99a2f..5ab8e26f882 100644
--- a/readthedocs/core/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -17,12 +17,22 @@ class ContextualizedHttp404(Http404):
     not_found_subject = _("page")
 
     def __init__(self, **kwargs):
+        """
+        Constructor that all subclasses should call.
+
+        :param kwargs: all kwargs are added as page context for rendering the 404 template
+        :param http_status: 404 view should respect this and set the HTTP status.
+        :param path_not_found: Inform the template and 404 view about a different path from
+                               request.path
+        """
         self.http_status = kwargs.pop("http_status", 404)
+        self.path_not_found = kwargs.pop("path_not_found", None)
         self.kwargs = kwargs
 
     def get_context(self):
         c = {
             "not_found_subject": self.not_found_subject,
+            "path_not_found": self.path_not_found,
         }
         c.update(self.kwargs)
         return c
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index 02339803d77..3987c22d75a 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -16,7 +16,6 @@
 from django.urls import reverse
 from django.utils.deprecation import MiddlewareMixin
 
-from readthedocs.core.exceptions import DomainDNSHttp404, ProjectHttp404
 from readthedocs.core.unresolver import (
     InvalidCustomDomainError,
     InvalidExternalDomainError,
@@ -28,6 +27,8 @@
 from readthedocs.core.utils import get_cache_tag
 from readthedocs.projects.models import Feature, Project
 
+from .exceptions import DomainDNSHttp404, ProjectHttp404
+
 log = structlog.get_logger(__name__)
 
 
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 79b132e1474..b8d4be8ccd6 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -13,12 +13,6 @@
 from readthedocs.api.mixins import CDNCacheTagsMixin
 from readthedocs.builds.constants import EXTERNAL, INTERNAL, LATEST, STABLE
 from readthedocs.builds.models import Version
-from readthedocs.core.exceptions import (
-    ContextualizedHttp404,
-    ProjectPageHttp404,
-    ProjectTranslationHttp404,
-    ProjectVersionHttp404,
-)
 from readthedocs.core.mixins import CDNCacheControlMixin
 from readthedocs.core.resolver import resolve_path, resolver
 from readthedocs.core.unresolver import (
@@ -36,6 +30,12 @@
 from readthedocs.redirects.exceptions import InfiniteRedirectException
 from readthedocs.storage import build_media_storage
 
+from ..exceptions import (
+    ContextualizedHttp404,
+    ProjectPageHttp404,
+    ProjectTranslationHttp404,
+    ProjectVersionHttp404,
+)
 from .mixins import (
     InvalidPathError,
     ServeDocsMixin,
@@ -876,6 +876,7 @@ def get_using_unresolver(self, request, path):
             filename=filename,
             lang_slug=project.language,
             version_slug=version.slug,
+            path_not_found=path,
         )
 
 
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 67c839096bd..5d4935a400b 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -4,7 +4,7 @@
 from django.http import HttpResponse
 from django.shortcuts import get_object_or_404, render
 
-from ...core.exceptions import ContextualizedHttp404
+from ..exceptions import ContextualizedHttp404
 from .decorators import map_project_slug, map_subproject_slug
 
 log = structlog.get_logger(__name__)  # noqa

From 62df7958e2b2cfe289c3c642945578ee4f85ceae Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 17:53:00 +0200
Subject: [PATCH 57/70] Rename a few things

---
 readthedocs/proxito/exceptions.py  | 4 ++--
 readthedocs/proxito/views/serve.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 5ab8e26f882..cb7fec28c53 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -59,7 +59,7 @@ class ProjectHttp404(ContextualizedHttp404):
     not_found_subject = _("project")
 
 
-class SubProjectHttp404(ContextualizedHttp404):
+class SubprojectHttp404(ContextualizedHttp404):
 
     """Raised if a subproject was not found."""
 
@@ -67,7 +67,7 @@ class SubProjectHttp404(ContextualizedHttp404):
     not_found_subject = _("sub project")
 
 
-class ProjectPageHttp404(ContextualizedHttp404):
+class ProxitoProjectFilenameHttp404(ContextualizedHttp404):
 
     """Raised if a page inside an existing project was not found."""
 
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index b8d4be8ccd6..cab13c59de6 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -32,9 +32,9 @@
 
 from ..exceptions import (
     ContextualizedHttp404,
-    ProjectPageHttp404,
     ProjectTranslationHttp404,
     ProjectVersionHttp404,
+    ProxitoProjectFilenameHttp404,
 )
 from .mixins import (
     InvalidPathError,
@@ -786,7 +786,7 @@ def get_using_unresolver(self, request, path):
             filename = unresolved.filename
             lang_slug = project.language
             version_slug = version.slug
-            contextualized_404_class = ProjectPageHttp404
+            contextualized_404_class = ProxitoProjectFilenameHttp404
         except VersionNotFoundError as exc:
             project = exc.project
             lang_slug = project.language

From 11d099a76e466fcac3d18680fad97538f3532525 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 17:59:28 +0200
Subject: [PATCH 58/70] Removed unused kwarg

---
 readthedocs/proxito/views/serve.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index cab13c59de6..a7287b58d2c 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -875,7 +875,6 @@ def get_using_unresolver(self, request, path):
             version=version,
             filename=filename,
             lang_slug=project.language,
-            version_slug=version.slug,
             path_not_found=path,
         )
 

From 2fa8a5264498a93df5273485f3cfad9effdd8392 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 19:01:45 +0200
Subject: [PATCH 59/70] Update test cases to check for expected 404 exception
 and HTTP status

---
 readthedocs/proxito/tests/test_middleware.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/readthedocs/proxito/tests/test_middleware.py b/readthedocs/proxito/tests/test_middleware.py
index 858344e7d34..fedd53ad161 100644
--- a/readthedocs/proxito/tests/test_middleware.py
+++ b/readthedocs/proxito/tests/test_middleware.py
@@ -12,6 +12,7 @@
 from readthedocs.projects.constants import PUBLIC
 from readthedocs.projects.models import Domain, Feature, Project, ProjectRelationship
 from readthedocs.proxito.constants import RedirectType
+from readthedocs.proxito.exceptions import DomainDNSHttp404
 from readthedocs.proxito.middleware import ProxitoMiddleware
 from readthedocs.rtd_tests.base import RequestFactoryTestMixin
 from readthedocs.rtd_tests.storage import BuildMediaFileSystemStorageTest
@@ -139,9 +140,11 @@ def test_proper_cname_uppercase(self):
     def test_invalid_cname(self):
         self.assertFalse(Domain.objects.filter(domain='my.host.com').exists())
         request = self.request(method='get', path=self.url, HTTP_HOST='my.host.com')
-        r = self.run_middleware(request)
-        # We show the 404 error page
-        self.assertContains(r, 'my.host.com', status_code=404)
+
+        with self.assertRaises(DomainDNSHttp404) as cm:
+            self.run_middleware(request)
+
+        assert cm.exception.http_status == 404
 
     def test_proper_subdomain(self):
         request = self.request(method='get', path=self.url, HTTP_HOST='pip.dev.readthedocs.io')
@@ -194,8 +197,10 @@ def test_request_header_not_allowed(self):
     def test_long_bad_subdomain(self):
         domain = 'www.pip.dev.readthedocs.io'
         request = self.request(method='get', path=self.url, HTTP_HOST=domain)
-        res = self.run_middleware(request)
-        self.assertEqual(res.status_code, 400)
+        with self.assertRaises(DomainDNSHttp404) as cm:
+            self.run_middleware(request)
+
+        assert cm.exception.http_status == 400
 
     def test_front_slash(self):
         domain = 'pip.dev.readthedocs.io'

From 92abfb9a94df36cf1e19d3021afe00a68370479e Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 12 Apr 2023 23:25:04 +0200
Subject: [PATCH 60/70] Adds a test case variant to check the exception
 handling stack

---
 readthedocs/proxito/tests/test_middleware.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/readthedocs/proxito/tests/test_middleware.py b/readthedocs/proxito/tests/test_middleware.py
index fedd53ad161..d1d0d9229ea 100644
--- a/readthedocs/proxito/tests/test_middleware.py
+++ b/readthedocs/proxito/tests/test_middleware.py
@@ -4,7 +4,7 @@
 import pytest
 from django.core.exceptions import SuspiciousOperation
 from django.http import HttpRequest
-from django.test import TestCase
+from django.test import Client, TestCase
 from django.test.utils import override_settings
 from django_dynamic_fixture import get
 
@@ -146,6 +146,13 @@ def test_invalid_cname(self):
 
         assert cm.exception.http_status == 404
 
+        # test all the exception handling combined
+        client_without_exception_handling = Client(rause_request_exception=False)
+        r = client_without_exception_handling.request(
+            method="get", path=self.url, HTTP_HOST="my.host.com"
+        )
+        assert r.status_code == 404
+
     def test_proper_subdomain(self):
         request = self.request(method='get', path=self.url, HTTP_HOST='pip.dev.readthedocs.io')
         self.run_middleware(request)
@@ -202,6 +209,13 @@ def test_long_bad_subdomain(self):
 
         assert cm.exception.http_status == 400
 
+        # test all the exception handling combined
+        client_without_exception_handling = Client(rause_request_exception=False)
+        r = client_without_exception_handling.request(
+            method="get", path=self.url, HTTP_HOST=domain
+        )
+        assert r.status_code == 400
+
     def test_front_slash(self):
         domain = 'pip.dev.readthedocs.io'
 

From 084ccc716ad50b305ee817d545ddc83f094a9e8b Mon Sep 17 00:00:00 2001
From: Benjamin Balder Bach 
Date: Wed, 19 Apr 2023 22:52:52 +0200
Subject: [PATCH 61/70] Apply suggestions from @stsewd code review

Co-authored-by: Santos Gallegos 
---
 readthedocs/proxito/exceptions.py                      | 10 +++++-----
 readthedocs/proxito/tests/test_middleware.py           |  2 +-
 readthedocs/proxito/views/serve.py                     |  6 ++----
 readthedocs/proxito/views/utils.py                     |  3 +--
 .../templates/errors/404/include_error_message.html    |  2 +-
 readthedocs/templates/errors/404/no_project.html       |  2 +-
 6 files changed, 11 insertions(+), 14 deletions(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index cb7fec28c53..14f697e4abc 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -10,13 +10,13 @@ class ContextualizedHttp404(Http404):
     Subclasses may define their own template name,
     HTTP status and object that was not found.
 
-    The contextualized exception is handled by the project's 404 view.
+    The contextualized exception is handled by proxito's 404 handler
     """
 
     template_name = "errors/404/base.html"
     not_found_subject = _("page")
 
-    def __init__(self, **kwargs):
+    def __init__(self, http_status=404, path_not_found=None, **kwargs):
         """
         Constructor that all subclasses should call.
 
@@ -64,10 +64,10 @@ class SubprojectHttp404(ContextualizedHttp404):
     """Raised if a subproject was not found."""
 
     template_name = "errors/404/no_subproject.html"
-    not_found_subject = _("sub project")
+    not_found_subject = _("subproject")
 
 
-class ProxitoProjectFilenameHttp404(ContextualizedHttp404):
+class ProjectFilenameHttp404(ContextualizedHttp404):
 
     """Raised if a page inside an existing project was not found."""
 
@@ -93,7 +93,7 @@ class ProjectVersionHttp404(ContextualizedHttp404):
     """
     Raised if a version was not found.
 
-    Note: The containing project can be both a project or a subproject inside another project.
+    Note: The containing project can be a subproject.
     """
 
     template_name = "errors/404/no_version.html"
diff --git a/readthedocs/proxito/tests/test_middleware.py b/readthedocs/proxito/tests/test_middleware.py
index d1d0d9229ea..885f2279653 100644
--- a/readthedocs/proxito/tests/test_middleware.py
+++ b/readthedocs/proxito/tests/test_middleware.py
@@ -147,7 +147,7 @@ def test_invalid_cname(self):
         assert cm.exception.http_status == 404
 
         # test all the exception handling combined
-        client_without_exception_handling = Client(rause_request_exception=False)
+        client_without_exception_handling = Client(raise_request_exception=False)
         r = client_without_exception_handling.request(
             method="get", path=self.url, HTTP_HOST="my.host.com"
         )
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index a7287b58d2c..013a692c940 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -353,8 +353,8 @@ def get_using_unresolver(self, request):
         if unresolved_domain.is_from_external_domain:
             self.version_type = EXTERNAL
 
-        # 404 errors aren't contextualized because they are sent to the HTTP proxy
-        # The path will be 'unresolved' again when HTTP server handles the 404 error
+        # 404 errors aren't contextualized here because all 404s use the internal nginx redirect,
+        # where the path will be 'unresolved' again when handling the 404 error
         # See: ServeError404Base
         try:
             unresolved = unresolver.unresolve_path(
@@ -531,8 +531,6 @@ def get(self, request, proxito_path, template_name="404.html"):
 
         version_slug = kwargs.get('version_slug')
         version_slug = self.get_version_from_host(request, version_slug)
-        # This special treatment of Http404 happens because the decorator that
-        # resolves a project doesn't know if it's resolving a subproject or a normal project
         (
             final_project,
             lang_slug,
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index 5d4935a400b..cc2301f190e 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -31,7 +31,7 @@ def proxito_404_page_handler(
     inside ``ServeError404`` view, we want to render a useful HTML response.
     """
 
-    # This type of request should have a fast response with no HTML rendered
+    # 404 exceptions that don't originate from our proxito 404 handler should have a fast response with no HTML rendered, since they will be forwarded to our 404 handler again.
     if request.resolver_match and request.resolver_match.url_name != 'proxito_404_handler':
         return fast_404(request, exception, template_name)
 
@@ -42,7 +42,6 @@ def proxito_404_page_handler(
     # Context is defined by the views that raise these exceptions and handled
     # in their templates.
     if isinstance(exception, ContextualizedHttp404):
-        # These attributes are not guaranteed.
         context.update(exception.get_context())
         template_name = exception.template_name
         http_status = exception.http_status
diff --git a/readthedocs/templates/errors/404/include_error_message.html b/readthedocs/templates/errors/404/include_error_message.html
index 8f9517fb5ec..041a16fa08e 100644
--- a/readthedocs/templates/errors/404/include_error_message.html
+++ b/readthedocs/templates/errors/404/include_error_message.html
@@ -10,7 +10,7 @@
   

     {% blocktrans trimmed %}
       Documentation changes over time and pages are moved around.
-      You can try to navigate to the index page of the project and use its navigation or search function to find a similar page.
+      You can try to navigate to the index page of the project and use its navigation, or search for a similar page.
     {% endblocktrans %}
   

 
diff --git a/readthedocs/templates/errors/404/no_project.html b/readthedocs/templates/errors/404/no_project.html
index 8642ec0562f..376c49d0606 100644
--- a/readthedocs/templates/errors/404/no_project.html
+++ b/readthedocs/templates/errors/404/no_project.html
@@ -7,7 +7,7 @@
   

     

       {% blocktrans trimmed with domain=domain|default:_("no domain") %}
-        The project for {{ domain }} that you are looking for either does not exist, has been deleted or does not yet have any valid builds.
+        The project for {{ domain }} that you are looking for either does not exist, has been deleted or doesn't have any successful builds yet.
       {% endblocktrans %}
     

   


From 607c735587b272b7fea732f7de537c839eb43f7f Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 19 Apr 2023 23:01:19 +0200
Subject: [PATCH 62/70] use pgettext with context for 404 subjects

---
 readthedocs/proxito/exceptions.py | 32 ++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 14f697e4abc..87a220f8960 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -1,5 +1,5 @@
 from django.http import Http404
-from django.utils.translation import gettext_lazy as _
+from django.utils.translation import pgettext_lazy
 
 
 class ContextualizedHttp404(Http404):
@@ -14,7 +14,9 @@ class ContextualizedHttp404(Http404):
     """
 
     template_name = "errors/404/base.html"
-    not_found_subject = _("page")
+    not_found_subject = pgettext_lazy(
+        "Names an object not found in a 404 error", "page"
+    )
 
     def __init__(self, http_status=404, path_not_found=None, **kwargs):
         """
@@ -40,10 +42,12 @@ def get_context(self):
 
 class DomainDNSHttp404(ContextualizedHttp404):
 
-    """Raised if a DNS record points to us but we do not know the domain."""
+    """Raised if a DNS record points to us and we don't know the domain."""
 
     template_name = "errors/404/dns.html"
-    not_found_subject = _("domain matching DNS record")
+    not_found_subject = pgettext_lazy(
+        "Names an object not found in a 404 error", "matching DNS record"
+    )
 
 
 class ProjectHttp404(ContextualizedHttp404):
@@ -56,7 +60,9 @@ class ProjectHttp404(ContextualizedHttp404):
     """
 
     template_name = "errors/404/no_project.html"
-    not_found_subject = _("project")
+    not_found_subject = pgettext_lazy(
+        "Names an object not found in a 404 error", "project"
+    )
 
 
 class SubprojectHttp404(ContextualizedHttp404):
@@ -64,7 +70,9 @@ class SubprojectHttp404(ContextualizedHttp404):
     """Raised if a subproject was not found."""
 
     template_name = "errors/404/no_subproject.html"
-    not_found_subject = _("subproject")
+    not_found_subject = pgettext_lazy(
+        "Names an object not found in a 404 error", "subproject"
+    )
 
 
 class ProjectFilenameHttp404(ContextualizedHttp404):
@@ -72,7 +80,9 @@ class ProjectFilenameHttp404(ContextualizedHttp404):
     """Raised if a page inside an existing project was not found."""
 
     template_name = "errors/404/no_project_page.html"
-    not_found_subject = _("documentation page")
+    not_found_subject = pgettext_lazy(
+        "Names an object not found in a 404 error", "documentation page"
+    )
 
 
 class ProjectTranslationHttp404(ContextualizedHttp404):
@@ -85,7 +95,9 @@ class ProjectTranslationHttp404(ContextualizedHttp404):
     """
 
     template_name = "errors/404/no_language.html"
-    not_found_subject = _("translation")
+    not_found_subject = pgettext_lazy(
+        "Names an object not found in a 404 error", "translation"
+    )
 
 
 class ProjectVersionHttp404(ContextualizedHttp404):
@@ -97,4 +109,6 @@ class ProjectVersionHttp404(ContextualizedHttp404):
     """
 
     template_name = "errors/404/no_version.html"
-    not_found_subject = _("documentation version")
+    not_found_subject = pgettext_lazy(
+        "Names an object not found in a 404 error", "documentation version"
+    )

From 344a845ecca4daa6a57f65ab68bb1dd6d93dbd97 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 19 Apr 2023 23:21:43 +0200
Subject: [PATCH 63/70] change kwarg name, use some explicit kwargs

---
 readthedocs/proxito/exceptions.py         | 4 ++--
 readthedocs/proxito/middleware.py         | 4 ++--
 readthedocs/proxito/views/utils.py        | 3 ++-
 readthedocs/templates/errors/404/dns.html | 2 +-
 4 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 87a220f8960..e72bf5ab268 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -27,8 +27,8 @@ def __init__(self, http_status=404, path_not_found=None, **kwargs):
         :param path_not_found: Inform the template and 404 view about a different path from
                                request.path
         """
-        self.http_status = kwargs.pop("http_status", 404)
-        self.path_not_found = kwargs.pop("path_not_found", None)
+        self.http_status = http_status
+        self.path_not_found = path_not_found
         self.kwargs = kwargs
 
     def get_context(self):
diff --git a/readthedocs/proxito/middleware.py b/readthedocs/proxito/middleware.py
index 3987c22d75a..72220b92e2e 100644
--- a/readthedocs/proxito/middleware.py
+++ b/readthedocs/proxito/middleware.py
@@ -217,7 +217,7 @@ def process_request(self, request):  # noqa
             # Raise a contextualized 404 that will be handled by proxito's 404 handler
             raise DomainDNSHttp404(
                 http_status=400,
-                host=exc.domain,
+                domain=exc.domain,
             )
         except (InvalidSubdomainError, InvalidExternalDomainError) as exc:
             log.debug("Invalid project set on the subdomain.")
@@ -230,7 +230,7 @@ def process_request(self, request):  # noqa
             log.debug("CNAME 404.", domain=exc.domain)
             # Raise a contextualized 404 that will be handled by proxito's 404 handler
             raise DomainDNSHttp404(
-                host=exc.domain,
+                domain=exc.domain,
             )
         except InvalidXRTDSlugHeaderError:
             raise SuspiciousOperation("Invalid X-RTD-Slug header.")
diff --git a/readthedocs/proxito/views/utils.py b/readthedocs/proxito/views/utils.py
index cc2301f190e..2185719e6e0 100644
--- a/readthedocs/proxito/views/utils.py
+++ b/readthedocs/proxito/views/utils.py
@@ -31,7 +31,8 @@ def proxito_404_page_handler(
     inside ``ServeError404`` view, we want to render a useful HTML response.
     """
 
-    # 404 exceptions that don't originate from our proxito 404 handler should have a fast response with no HTML rendered, since they will be forwarded to our 404 handler again.
+    # 404 exceptions that don't originate from our proxito 404 handler should have a fast response
+    # with no HTML rendered, since they will be forwarded to our 404 handler again.
     if request.resolver_match and request.resolver_match.url_name != 'proxito_404_handler':
         return fast_404(request, exception, template_name)
 
diff --git a/readthedocs/templates/errors/404/dns.html b/readthedocs/templates/errors/404/dns.html
index 22917c74820..30ce574516a 100644
--- a/readthedocs/templates/errors/404/dns.html
+++ b/readthedocs/templates/errors/404/dns.html
@@ -4,7 +4,7 @@
 
 {% block 404_error_message %}
 
-
{% blocktrans %}The host "{{ host }}" is unknown to Read the Docs{% endblocktrans %}

+
{% blocktrans %}The host "{{ domain }}" is unknown to Read the Docs{% endblocktrans %}

 
 

   {% with docsurl='https://docs.readthedocs.io/en/stable/custom_domains.html' %}

From 6156040fdef95977235a13ded8073e456f80aeee Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 19 Apr 2023 23:24:28 +0200
Subject: [PATCH 64/70] Fix imports to be absolute

---
 readthedocs/proxito/views/serve.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 013a692c940..6b47e3a6d06 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -27,21 +27,21 @@
 from readthedocs.projects.models import Domain, Feature
 from readthedocs.projects.templatetags.projects_tags import sort_version_aware
 from readthedocs.proxito.constants import RedirectType
-from readthedocs.redirects.exceptions import InfiniteRedirectException
-from readthedocs.storage import build_media_storage
-
-from ..exceptions import (
+from readthedocs.proxito.exceptions import (
     ContextualizedHttp404,
+    ProjectFilenameHttp404,
     ProjectTranslationHttp404,
     ProjectVersionHttp404,
-    ProxitoProjectFilenameHttp404,
 )
-from .mixins import (
+from readthedocs.proxito.views.mixins import (
     InvalidPathError,
     ServeDocsMixin,
     ServeRedirectMixin,
     StorageFileNotFound,
 )
+from readthedocs.redirects.exceptions import InfiniteRedirectException
+from readthedocs.storage import build_media_storage
+
 from .utils import _get_project_data_from_request
 
 log = structlog.get_logger(__name__)  # noqa
@@ -784,7 +784,7 @@ def get_using_unresolver(self, request, path):
             filename = unresolved.filename
             lang_slug = project.language
             version_slug = version.slug
-            contextualized_404_class = ProxitoProjectFilenameHttp404
+            contextualized_404_class = ProjectFilenameHttp404
         except VersionNotFoundError as exc:
             project = exc.project
             lang_slug = project.language

From 9799f7e6b68e26fddca2f757f3541533fbe10370 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 19 Apr 2023 23:29:16 +0200
Subject: [PATCH 65/70] Remove h3 from search bar when included with a search
 term

---
 readthedocs/templates/core/widesearchbar.html | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/readthedocs/templates/core/widesearchbar.html b/readthedocs/templates/core/widesearchbar.html
index bcb91283106..7950c8096cd 100644
--- a/readthedocs/templates/core/widesearchbar.html
+++ b/readthedocs/templates/core/widesearchbar.html
@@ -1,9 +1,7 @@
 {% load i18n %}
 
         
-        {% if term|default_if_none:False %}
-        
{% trans "Search the docs" %}

-        {% else %}
+        {% if not term|default_if_none:False %}
         
{% trans "Search all the docs" %}

         {% endif %}
         


From 81e46a9eec1d3582e92eee7d07658afe758c5b18 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Thu, 20 Apr 2023 13:26:48 +0200
Subject: [PATCH 66/70] explicitly require certain exception arguments, trim
 the ones we're not (yet) using

---
 readthedocs/proxito/exceptions.py  | 60 ++++++++++++++++++++++++++++++
 readthedocs/proxito/views/serve.py |  5 +--
 2 files changed, 62 insertions(+), 3 deletions(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index e72bf5ab268..0439def6f0b 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -49,6 +49,16 @@ class DomainDNSHttp404(ContextualizedHttp404):
         "Names an object not found in a 404 error", "matching DNS record"
     )
 
+    def __init__(self, domain, **kwargs):
+        """
+        Raised when DNS for a custom domain is bad.
+
+        :param domain: The domain for which DNS is misconfigured.
+        :param kwargs:
+        """
+        kwargs["domain"] = domain
+        super().__init__(**kwargs)
+
 
 class ProjectHttp404(ContextualizedHttp404):
 
@@ -64,6 +74,16 @@ class ProjectHttp404(ContextualizedHttp404):
         "Names an object not found in a 404 error", "project"
     )
 
+    def __init__(self, domain=None, **kwargs):
+        """
+        Raised when a project wasn't found for a given domain.
+
+        :param domain: The domain (custom and hosted) that was not found.
+        :param kwargs:
+        """
+        kwargs["domain"] = domain
+        super().__init__(**kwargs)
+
 
 class SubprojectHttp404(ContextualizedHttp404):
 
@@ -74,6 +94,16 @@ class SubprojectHttp404(ContextualizedHttp404):
         "Names an object not found in a 404 error", "subproject"
     )
 
+    def __init__(self, project, **kwargs):
+        """
+        Raised if a subproject was not found.
+
+        :param project: The project in which the subproject could not be found
+        :param kwargs: Context dictionary of the rendered template
+        """
+        kwargs["project"] = project
+        super().__init__(**kwargs)
+
 
 class ProjectFilenameHttp404(ContextualizedHttp404):
 
@@ -84,6 +114,16 @@ class ProjectFilenameHttp404(ContextualizedHttp404):
         "Names an object not found in a 404 error", "documentation page"
     )
 
+    def __init__(self, project, **kwargs):
+        """
+        Raised if a page inside an existing project was not found.
+
+        :param project: The project in which the file could not be found
+        :param kwargs: Context dictionary of the rendered template
+        """
+        kwargs["project"] = project
+        super().__init__(**kwargs)
+
 
 class ProjectTranslationHttp404(ContextualizedHttp404):
 
@@ -99,6 +139,16 @@ class ProjectTranslationHttp404(ContextualizedHttp404):
         "Names an object not found in a 404 error", "translation"
     )
 
+    def __init__(self, project, **kwargs):
+        """
+        Raised if a translation of a project was not found.
+
+        :param project: The project in which the translation could not be found
+        :param kwargs: Context dictionary of the rendered template
+        """
+        kwargs["project"] = project
+        super().__init__(**kwargs)
+
 
 class ProjectVersionHttp404(ContextualizedHttp404):
 
@@ -112,3 +162,13 @@ class ProjectVersionHttp404(ContextualizedHttp404):
     not_found_subject = pgettext_lazy(
         "Names an object not found in a 404 error", "documentation version"
     )
+
+    def __init__(self, project, **kwargs):
+        """
+        Raised if a version was not found.
+
+        :param project: The project in which the version could not be found
+        :param kwargs: Context dictionary of the rendered template
+        """
+        kwargs["project"] = project
+        super().__init__(**kwargs)
diff --git a/readthedocs/proxito/views/serve.py b/readthedocs/proxito/views/serve.py
index 3d3602cf8e8..c14a5b0b400 100644
--- a/readthedocs/proxito/views/serve.py
+++ b/readthedocs/proxito/views/serve.py
@@ -849,11 +849,10 @@ def get_using_unresolver(self, request, path):
             return response
 
         # No custom 404 page, use our contextualized 404 response
+        # Several additional context variables can be added if the templates
+        # or other error handling is developed (version, language, filename).
         raise contextualized_404_class(
             project=project,
-            version=version,
-            filename=filename,
-            lang_slug=project.language,
             path_not_found=path,
         )
 

From d73d397400f1286548c79807ebb7688a4ebb6352 Mon Sep 17 00:00:00 2001
From: Benjamin Balder Bach 
Date: Wed, 26 Apr 2023 11:51:57 +0200
Subject: [PATCH 67/70] Apply suggestions from @stsewd code review

Co-authored-by: Santos Gallegos 
---
 readthedocs/proxito/exceptions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index 0439def6f0b..bdb52f9909e 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -74,7 +74,7 @@ class ProjectHttp404(ContextualizedHttp404):
         "Names an object not found in a 404 error", "project"
     )
 
-    def __init__(self, domain=None, **kwargs):
+    def __init__(self, domain, **kwargs):
         """
         Raised when a project wasn't found for a given domain.
 

From 18b5788c71c857327ef8fddf4054ef6514e2fc0f Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 26 Apr 2023 11:57:16 +0200
Subject: [PATCH 68/70] Remove |default filter and assume a value exists

---
 readthedocs/templates/errors/404/include_error_message.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/templates/errors/404/include_error_message.html b/readthedocs/templates/errors/404/include_error_message.html
index 041a16fa08e..2a0d0d45648 100644
--- a/readthedocs/templates/errors/404/include_error_message.html
+++ b/readthedocs/templates/errors/404/include_error_message.html
@@ -1,7 +1,7 @@
 {% load i18n %}
 

   

-    {% blocktrans trimmed with project_slug=project.slug|default:_("unknown slug") %}
+    {% blocktrans trimmed with project_slug=project.slug %}
       You are browsing the documentation of {{ project_slug }}.
       The {{ not_found_subject }} you are looking for at {{ path_not_found }} was not found.
     {% endblocktrans %}

From 8945bff14fa03da2a293e6f32aa9a95ce1aa1eda Mon Sep 17 00:00:00 2001
From: Benjamin Balder Bach 
Date: Wed, 26 Apr 2023 11:57:38 +0200
Subject: [PATCH 69/70] Update
 readthedocs/templates/errors/404/include_tips.html

Co-authored-by: Santos Gallegos 
---
 readthedocs/templates/errors/404/include_tips.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/readthedocs/templates/errors/404/include_tips.html b/readthedocs/templates/errors/404/include_tips.html
index ba8acad77c2..f6a507be273 100644
--- a/readthedocs/templates/errors/404/include_tips.html
+++ b/readthedocs/templates/errors/404/include_tips.html
@@ -5,7 +5,7 @@ 
{% trans "Are you the project owner?" %}

 
{% trans "Here are some tips to address 404 errors:" %}

 
 
 


From 5e9017e69781214296ad807e2f6aae4433e2a693 Mon Sep 17 00:00:00 2001
From: Benjamin Bach 
Date: Wed, 26 Apr 2023 12:03:28 +0200
Subject: [PATCH 70/70] Add more context for translators of not_found_subject

---
 readthedocs/proxito/exceptions.py | 20 +++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/readthedocs/proxito/exceptions.py b/readthedocs/proxito/exceptions.py
index bdb52f9909e..813037be633 100644
--- a/readthedocs/proxito/exceptions.py
+++ b/readthedocs/proxito/exceptions.py
@@ -1,6 +1,12 @@
 from django.http import Http404
 from django.utils.translation import pgettext_lazy
 
+_not_found_subject_translation_context = (
+    "Names a subject that was not found in a 404 error message. Used like "
+    "'The {{ not_found_subject }} you are looking for at {{ path_not_found }} "
+    "was not found.'"
+)
+
 
 class ContextualizedHttp404(Http404):
 
@@ -14,9 +20,7 @@ class ContextualizedHttp404(Http404):
     """
 
     template_name = "errors/404/base.html"
-    not_found_subject = pgettext_lazy(
-        "Names an object not found in a 404 error", "page"
-    )
+    not_found_subject = pgettext_lazy(_not_found_subject_translation_context, "page")
 
     def __init__(self, http_status=404, path_not_found=None, **kwargs):
         """
@@ -46,7 +50,7 @@ class DomainDNSHttp404(ContextualizedHttp404):
 
     template_name = "errors/404/dns.html"
     not_found_subject = pgettext_lazy(
-        "Names an object not found in a 404 error", "matching DNS record"
+        _not_found_subject_translation_context, "matching DNS record"
     )
 
     def __init__(self, domain, **kwargs):
@@ -70,9 +74,7 @@ class ProjectHttp404(ContextualizedHttp404):
     """
 
     template_name = "errors/404/no_project.html"
-    not_found_subject = pgettext_lazy(
-        "Names an object not found in a 404 error", "project"
-    )
+    not_found_subject = pgettext_lazy(_not_found_subject_translation_context, "project")
 
     def __init__(self, domain, **kwargs):
         """
@@ -111,7 +113,7 @@ class ProjectFilenameHttp404(ContextualizedHttp404):
 
     template_name = "errors/404/no_project_page.html"
     not_found_subject = pgettext_lazy(
-        "Names an object not found in a 404 error", "documentation page"
+        _not_found_subject_translation_context, "documentation page"
     )
 
     def __init__(self, project, **kwargs):
@@ -160,7 +162,7 @@ class ProjectVersionHttp404(ContextualizedHttp404):
 
     template_name = "errors/404/no_version.html"
     not_found_subject = pgettext_lazy(
-        "Names an object not found in a 404 error", "documentation version"
+        _not_found_subject_translation_context, "documentation version"
     )
 
     def __init__(self, project, **kwargs):