Skip to content

Search: use alias to link to search results of subprojects #7757

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Dec 16, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/server-side-search.rst
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,7 @@ This is ``https://docs.readthedocs.io/_/api/v2/search`` for the ``docs`` project

:>json string type: The type of the result, currently page is the only type.
:>json string project: The project slug
:>json string project_alias: Alias of the project if it's a subproject.
:>json string version: The version slug
:>json string title: The title of the page
:>json string domain: Canonical domain of the resulting page
Expand Down Expand Up @@ -143,6 +144,7 @@ This is ``https://docs.readthedocs.io/_/api/v2/search`` for the ``docs`` project
{
"type": "page",
"project": "docs",
"project_alias": null,
"version": "latest",
"title": "Server Side Search",
"domain": "https://docs.readthedocs.io",
Expand Down
4 changes: 2 additions & 2 deletions readthedocs/core/static-src/core/js/doc-embed/search.js
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ function attach_elastic_search_query_sphinx(data) {

// If the document is from a subproject, add extra information
if (result.project !== project) {
var text = " (from project " + result.project + ")";
var text = " (from project " + result.project_alias + ")";
var extra = $('<span>', {'text': text});
list_item.append(extra);
}
Expand Down Expand Up @@ -295,7 +295,7 @@ function attach_elastic_search_query_mkdocs(data) {
);

if (result.project !== project) {
var text = '(from project ' + result.project + ')';
var text = '(from project ' + result.project_alias + ')';
item.append($('<span>', {'text': text}));
}

Expand Down
2 changes: 1 addition & 1 deletion readthedocs/core/static/core/js/readthedocs-doc-embed.js

Large diffs are not rendered by default.

46 changes: 30 additions & 16 deletions readthedocs/search/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from readthedocs.search import tasks
from readthedocs.search.faceted_search import PageSearch

from .serializers import PageSearchSerializer, VersionData
from .serializers import PageSearchSerializer, ProjectData, VersionData

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -183,15 +183,21 @@ def _get_all_projects_data(self):
.. code::

{
"requests": VersionData(
"latest",
"sphinx",
"https://requests.readthedocs.io/en/latest/",
"requests": ProjectData(
alias='alias',
version=VersionData(
"latest",
"sphinx",
"https://requests.readthedocs.io/en/latest/",
),
),
"requests-oauth": VersionData(
"latest",
"sphinx_htmldir",
"https://requests-oauth.readthedocs.io/en/latest/",
"requests-oauth": ProjectData(
alias=None,
version=VersionData(
"latest",
"sphinx_htmldir",
"https://requests-oauth.readthedocs.io/en/latest/",
),
),
}

Expand All @@ -203,10 +209,13 @@ def _get_all_projects_data(self):
main_project = self._get_project()

projects_data = {
main_project.slug: VersionData(
slug=main_version.slug,
doctype=main_version.documentation_type,
docs_url=main_project.get_docs_url(version_slug=main_version.slug),
main_project.slug: ProjectData(
alias=None,
version=VersionData(
slug=main_version.slug,
doctype=main_version.documentation_type,
docs_url=main_project.get_docs_url(version_slug=main_version.slug),
),
)
}

Expand All @@ -232,11 +241,16 @@ def _get_all_projects_data(self):

if version and self._has_permission(self.request.user, version):
url = subproject.get_docs_url(version_slug=version.slug)
projects_data[subproject.slug] = VersionData(
project_alias = subproject.superprojects.values_list('alias', flat=True).first()
version_data = VersionData(
slug=version.slug,
doctype=version.documentation_type,
docs_url=url,
)
projects_data[subproject.slug] = ProjectData(
alias=project_alias,
version=version_data,
)

return projects_data

Expand Down Expand Up @@ -299,8 +313,8 @@ def get_queryset(self):

if main_project.has_feature(Feature.SEARCH_SUBPROJECTS_ON_DEFAULT_VERSION):
projects = {
project: version.slug
for project, version in self._get_all_projects_data().items()
project: project_data.version.slug
for project, project_data in self._get_all_projects_data().items()
}
# Check to avoid searching all projects in case it's empty.
if not projects:
Expand Down
1 change: 1 addition & 0 deletions readthedocs/search/documents.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ class PageDocument(RTDDocTypeMixin, Document):
# Metadata
project = fields.KeywordField(attr='project.slug')
version = fields.KeywordField(attr='version.slug')
doctype = fields.KeywordField(attr='version.documentation_type')
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Does this require reindexing?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yeah, but we can do it the next year, we aren't using it yet (still querying from the db or falling back to None). Tracking it here #7762.

path = fields.KeywordField(attr='processed_json.path')
full_path = fields.KeywordField(attr='path')
rank = fields.IntegerField()
Expand Down
70 changes: 43 additions & 27 deletions readthedocs/search/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@


# Structure used for storing cached data of a version mostly.
ProjectData = namedtuple('ProjectData', ['version', 'alias'])
VersionData = namedtuple('VersionData', ['slug', 'docs_url', 'doctype'])


Expand Down Expand Up @@ -58,13 +59,51 @@ class PageSearchSerializer(serializers.Serializer):

type = serializers.CharField(default='page', source=None, read_only=True)
project = serializers.CharField()
project_alias = serializers.SerializerMethodField()
version = serializers.CharField()
title = serializers.CharField()
path = serializers.SerializerMethodField()
domain = serializers.SerializerMethodField()
highlights = PageHighlightSerializer(source='meta.highlight', default=dict)
blocks = serializers.SerializerMethodField()

def _get_project_data(self, obj):
"""
Get and cache the project data.

Try to get the data from the ``projects_data`` context,
and fallback to get it from the database.
If the result is fetched from the database,
it's cached into ``projects_data``.
"""
project_data = self.context.get('projects_data', {}).get(obj.project)
if project_data:
return project_data

project = Project.objects.filter(slug=obj.project).first()
if project:
docs_url = project.get_docs_url(version_slug=obj.version)
project_alias = project.superprojects.values_list('alias', flat=True).first()

projects_data = self.context.setdefault('projects_data', {})
version_data = VersionData(
slug=obj.version,
docs_url=docs_url,
doctype=None,
)
projects_data[obj.project] = ProjectData(
alias=project_alias,
version=version_data,
)
return projects_data[obj.project]
return None

def get_project_alias(self, obj):
project_data = self._get_project_data(obj)
if project_data:
return project_data.alias
return None

def get_domain(self, obj):
full_path = self._get_full_path(obj)
if full_path:
Expand All @@ -80,40 +119,17 @@ def get_path(self, obj):
return None

def _get_full_path(self, obj):
"""
Get the page link.

Try to get the link from the ``project_data`` context,
and fallback to get it from the database.
If the result is fetched from the database,
it's cached into ``project_data``.
"""
# First try to build the URL from the context.
version_data = self.context.get('projects_data', {}).get(obj.project)
if version_data:
docs_url = version_data.docs_url
project_data = self._get_project_data(obj)
if project_data:
docs_url = project_data.version.docs_url
path = obj.full_path

# Generate an appropriate link for the doctypes that use htmldir,
# and always end it with / so it goes directly to proxito.
if version_data.doctype in {SPHINX_HTMLDIR, MKDOCS}:
if project_data.version.doctype in {SPHINX_HTMLDIR, MKDOCS}:
path = re.sub('(^|/)index.html$', '/', path)

return docs_url.rstrip('/') + '/' + path.lstrip('/')

# Fallback to build the URL querying the db.
project = Project.objects.filter(slug=obj.project).first()
if project:
docs_url = project.get_docs_url(version_slug=obj.version)
# cache the project URL
projects_data = self.context.setdefault('projects_data', {})
projects_data[obj.project] = VersionData(
slug=obj.version,
docs_url=docs_url,
doctype=None,
)
return docs_url + obj.full_path

return None

def get_blocks(self, obj):
Expand Down
2 changes: 2 additions & 0 deletions readthedocs/search/tests/test_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,7 @@ def test_doc_search_filter_by_version(self, api_client, project):
data = resp.data['results']
assert len(data) == 1
assert data[0]['project'] == project.slug
assert data[0]['project_alias'] is None

def test_doc_search_pagination(self, api_client, project):
"""Test Doc search result can be paginated"""
Expand Down Expand Up @@ -264,6 +265,7 @@ def test_doc_search_subprojects(self, api_client, all_projects):
# First result should be the subproject
first_result = data[0]
assert first_result['project'] == subproject.slug
assert first_result['project_alias'] == subproject.slug
# The result is from the same version as the main project.
assert first_result['version'] == version.slug
# Check the link is the subproject document link
Expand Down
13 changes: 9 additions & 4 deletions readthedocs/search/views.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@

from .serializers import (
PageSearchSerializer,
ProjectData,
ProjectSearchSerializer,
VersionData,
)
Expand Down Expand Up @@ -62,11 +63,15 @@ def _get_project_data(self, project, version_slug):
.get(slug=version_slug)
)
docs_url = project.get_docs_url(version_slug=version_slug)
version_data = VersionData(
slug=version_slug,
docs_url=docs_url,
doctype=version_doctype,
)
project_data = {
project.slug: VersionData(
slug=version_slug,
docs_url=docs_url,
doctype=version_doctype,
project.slug: ProjectData(
alias=None,
version=version_data,
)
}
return project_data
Expand Down