From 83c6f400dcdf9c0591c35e85f5f1a39b02f06352 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 23 Oct 2018 15:23:08 +0200 Subject: [PATCH 01/22] Remove old test references to intersphinx --- readthedocs/rtd_tests/tests/test_celery.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index 4e5ed8856e6..5760a672524 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -79,8 +79,7 @@ def test_update_docs(self): result = tasks.update_docs_task.delay( self.project.pk, build_pk=build.pk, - record=False, - intersphinx=False) + record=False) self.assertTrue(result.successful()) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_python_environment', new=MagicMock) @@ -96,8 +95,7 @@ def test_update_docs_unexpected_setup_exception(self, mock_setup_vcs): result = tasks.update_docs_task.delay( self.project.pk, build_pk=build.pk, - record=False, - intersphinx=False) + record=False) self.assertTrue(result.successful()) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_python_environment', new=MagicMock) @@ -113,8 +111,7 @@ def test_update_docs_unexpected_build_exception(self, mock_build_docs): result = tasks.update_docs_task.delay( self.project.pk, build_pk=build.pk, - record=False, - intersphinx=False) + record=False) self.assertTrue(result.successful()) def test_sync_repository(self): From db5fbd8d8efd3718db3f6b167a32f0f9367b398b Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 23 Oct 2018 15:24:47 +0200 Subject: [PATCH 02/22] Initial commit of domains app --- readthedocs/domains/__init__.py | 0 .../migrations/0001_create-domain-data.py | 32 ++++++++++++ readthedocs/domains/migrations/__init__.py | 0 readthedocs/domains/models.py | 50 +++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 readthedocs/domains/__init__.py create mode 100644 readthedocs/domains/migrations/0001_create-domain-data.py create mode 100644 readthedocs/domains/migrations/__init__.py create mode 100644 readthedocs/domains/models.py diff --git a/readthedocs/domains/__init__.py b/readthedocs/domains/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/domains/migrations/0001_create-domain-data.py b/readthedocs/domains/migrations/0001_create-domain-data.py new file mode 100644 index 00000000000..6e02dadf977 --- /dev/null +++ b/readthedocs/domains/migrations/0001_create-domain-data.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.16 on 2018-10-23 08:24 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0028_create-domain-data'), + ] + + operations = [ + migrations.CreateModel( + name='DomainData', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('domain', models.CharField(max_length=255, verbose_name='Domain')), + ('name', models.CharField(max_length=255, verbose_name='Name')), + ('display_name', models.CharField(max_length=255, verbose_name='Display Name')), + ('type', models.CharField(max_length=255, verbose_name='Type')), + ('doc_name', models.CharField(max_length=255, verbose_name='Doc Name')), + ('anchor', models.CharField(max_length=255, verbose_name='Anchor')), + ('priority', models.IntegerField(verbose_name='Priority')), + ('project', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='domain_data', to='projects.Project')), + ], + ), + ] diff --git a/readthedocs/domains/migrations/__init__.py b/readthedocs/domains/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/domains/models.py b/readthedocs/domains/models.py new file mode 100644 index 00000000000..5627d3c4eca --- /dev/null +++ b/readthedocs/domains/models.py @@ -0,0 +1,50 @@ +from django.db import models +from django.utils.encoding import python_2_unicode_compatible +from django.utils.translation import ugettext_lazy as _ + +from readthedocs.projects.models import Project + + +@python_2_unicode_compatible +class DomainData(models.Model): + + """ + Information from a project about it's Sphinx domains. + + This captures data about API objects that exist in that codebase. + """ + + project = models.OneToOneField( + Project, + on_delete=models.SET_NULL, + related_name='domain_data', + null=True, + blank=True, + ) + domain = models.CharField( + _('Domain'), + max_length=255, + ) + name = models.CharField( + _('Name'), + max_length=255, + ) + display_name = models.CharField( + _('Display Name'), + max_length=255, + ) + type = models.CharField( + _('Type'), + max_length=255, + ) + doc_name = models.CharField( + _('Doc Name'), + max_length=255, + ) + anchor = models.CharField( + _('Anchor'), + max_length=255, + ) + priority = models.IntegerField( + _('Priority'), + ) From 41b2857a29696c91b30d8b8e520f92370241d5f8 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 30 Oct 2018 15:04:57 -0500 Subject: [PATCH 03/22] Add initial domaindata modeling and API integration --- readthedocs/domaindata/__init__.py | 0 readthedocs/domaindata/admin.py | 9 +++++ readthedocs/domaindata/api.py | 29 ++++++++++++++++ readthedocs/domaindata/models.py | 55 ++++++++++++++++++++++++++++++ readthedocs/restapi/urls.py | 2 ++ readthedocs/settings/base.py | 3 ++ 6 files changed, 98 insertions(+) create mode 100644 readthedocs/domaindata/__init__.py create mode 100644 readthedocs/domaindata/admin.py create mode 100644 readthedocs/domaindata/api.py create mode 100644 readthedocs/domaindata/models.py diff --git a/readthedocs/domaindata/__init__.py b/readthedocs/domaindata/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/domaindata/admin.py b/readthedocs/domaindata/admin.py new file mode 100644 index 00000000000..3485ee1c572 --- /dev/null +++ b/readthedocs/domaindata/admin.py @@ -0,0 +1,9 @@ +from django.contrib import admin +from .models import DomainData + + +class DomainDataAdmin(admin.ModelAdmin): + list_filter = ('type', 'project') + + +admin.site.register(DomainData, DomainDataAdmin) diff --git a/readthedocs/domaindata/api.py b/readthedocs/domaindata/api.py new file mode 100644 index 00000000000..327b7510e40 --- /dev/null +++ b/readthedocs/domaindata/api.py @@ -0,0 +1,29 @@ +from rest_framework import serializers, viewsets + +from readthedocs.core.resolver import resolve +from .models import DomainData + + +class DomainDataSerializer(serializers.ModelSerializer): + project = serializers.SlugRelatedField(slug_field='slug', read_only=True) + version = serializers.SlugRelatedField(slug_field='slug', read_only=True) + doc_type = serializers.SerializerMethodField() + doc_url = serializers.SerializerMethodField() + + class Meta: + model = DomainData + fields = ('project', 'version', 'name', 'display_name', 'doc_type', 'doc_url') + + def get_doc_type(self, obj): + return f'{obj.domain}:{obj.type}' + + def get_doc_url(self, obj): + path = f'{obj.doc_name}#{obj.anchor}' + full_url = resolve(project=obj.project, version_slug=obj.version.slug, filename=path) + return full_url + + +class DomainDataAPIView(viewsets.ModelViewSet): + queryset = DomainData.objects.public() + serializer_class = DomainDataSerializer + filter_fields = ('project__slug', 'version__slug', 'domain', 'type', 'doc_name', 'name') diff --git a/readthedocs/domaindata/models.py b/readthedocs/domaindata/models.py new file mode 100644 index 00000000000..9271708821f --- /dev/null +++ b/readthedocs/domaindata/models.py @@ -0,0 +1,55 @@ +from django.db import models +from django.utils.encoding import python_2_unicode_compatible +from django.utils.translation import ugettext_lazy as _ + +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project +from readthedocs.projects.querysets import RelatedProjectQuerySet + + +@python_2_unicode_compatible +class DomainData(models.Model): + + """ + Information from a project about it's Sphinx domains. + + This captures data about API objects that exist in that codebase. + """ + + project = models.ForeignKey( + Project, + related_name='domain_data', + ) + version = models.ForeignKey(Version, verbose_name=_('Version'), + related_name='domain_data') + modified_date = models.DateTimeField(_('Publication date'), auto_now=True) + commit = models.CharField(_('Commit'), max_length=255) + + domain = models.CharField( + _('Domain'), + max_length=255, + ) + name = models.CharField( + _('Name'), + max_length=255, + ) + display_name = models.CharField( + _('Display Name'), + max_length=255, + ) + type = models.CharField( + _('Type'), + max_length=255, + ) + doc_name = models.CharField( + _('Doc Name'), + max_length=255, + ) + anchor = models.CharField( + _('Anchor'), + max_length=255, + ) + objects = RelatedProjectQuerySet.as_manager() + + def __str__(self): + return f'DomainData [{self.project.slug}:{self.version.slug}] [{self.domain}:{self.type}] {self.name} -> {self.doc_name}#{self.anchor}' diff --git a/readthedocs/restapi/urls.py b/readthedocs/restapi/urls.py index ffaad2405b6..bb8740881dd 100644 --- a/readthedocs/restapi/urls.py +++ b/readthedocs/restapi/urls.py @@ -34,6 +34,7 @@ SocialAccountViewSet, VersionViewSet, ) +from readthedocs.domaindata.api import DomainDataAPIView router = routers.DefaultRouter() router.register(r'build', BuildViewSet, base_name='build') @@ -42,6 +43,7 @@ router.register(r'project', ProjectViewSet, base_name='project') router.register(r'notification', NotificationViewSet, base_name='emailhook') router.register(r'domain', DomainViewSet, base_name='domain') +router.register(r'domaindata', DomainDataAPIView, base_name='domaindata') router.register( r'remote/org', RemoteOrganizationViewSet, diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 4770c009337..1d6ade0b897 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -84,6 +84,7 @@ def INSTALLED_APPS(self): # noqa 'django_extensions', 'messages_extends', 'tastypie', + 'django_filters', # our apps 'readthedocs.projects', @@ -99,6 +100,7 @@ def INSTALLED_APPS(self): # noqa 'readthedocs.notifications', 'readthedocs.integrations', 'readthedocs.analytics', + 'readthedocs.domaindata', # allauth @@ -295,6 +297,7 @@ def USE_PROMOS(self): # noqa # CORS CORS_ORIGIN_REGEX_WHITELIST = ( + '(.*)localhost(.*)', '^http://(.+)\.readthedocs\.io$', '^https://(.+)\.readthedocs\.io$' ) From 0f82c26accfb100f0fa3ab79e70bbd1032887d87 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 30 Oct 2018 18:37:12 -0500 Subject: [PATCH 04/22] Add indexing of DomainData to the builds. --- readthedocs/domaindata/api.py | 18 +++++++-- readthedocs/projects/tasks.py | 76 ++++++++++++++++++++++++++++++----- 2 files changed, 81 insertions(+), 13 deletions(-) diff --git a/readthedocs/domaindata/api.py b/readthedocs/domaindata/api.py index 327b7510e40..d3176d1a396 100644 --- a/readthedocs/domaindata/api.py +++ b/readthedocs/domaindata/api.py @@ -1,5 +1,6 @@ -from rest_framework import serializers, viewsets +from rest_framework import serializers +from readthedocs.restapi.views.model_views import UserSelectViewSet from readthedocs.core.resolver import resolve from .models import DomainData @@ -18,12 +19,21 @@ def get_doc_type(self, obj): return f'{obj.domain}:{obj.type}' def get_doc_url(self, obj): - path = f'{obj.doc_name}#{obj.anchor}' + path = obj.doc_name + if obj.anchor: + path += f'#{obj.anchor}' full_url = resolve(project=obj.project, version_slug=obj.version.slug, filename=path) return full_url -class DomainDataAPIView(viewsets.ModelViewSet): - queryset = DomainData.objects.public() +class DomainDataAdminSerializer(DomainDataSerializer): + + class Meta(DomainDataSerializer.Meta): + fields = '__all__' + + +class DomainDataAPIView(UserSelectViewSet): + model = DomainData serializer_class = DomainDataSerializer + admin_serializer_class = DomainDataAdminSerializer filter_fields = ('project__slug', 'version__slug', 'domain', 'type', 'doc_name', 'name') diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 73acebf46dd..453a0e5d6f9 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -14,6 +14,7 @@ import json import logging import os +import sys import shutil import socket from collections import Counter, defaultdict @@ -26,6 +27,8 @@ from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from slumber.exceptions import HttpClientError +from sphinx.ext import intersphinx + from readthedocs.builds.constants import ( BUILD_STATE_BUILDING, BUILD_STATE_CLONING, BUILD_STATE_FINISHED, @@ -46,6 +49,7 @@ VersionLockedError, YAMLParseError) from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv +from readthedocs.domaindata.models import DomainData from readthedocs.projects.models import APIProject from readthedocs.restapi.client import api as api_v2 from readthedocs.restapi.utils import index_search_request @@ -55,7 +59,7 @@ from .constants import LOG_TEMPLATE from .exceptions import RepositoryError -from .models import Domain, Feature, ImportedFile, Project +from .models import Domain, ImportedFile, Project from .signals import ( after_build, after_vcs, before_build, before_vcs, files_changed) @@ -323,16 +327,19 @@ def run(self, pk, version_pk=None, build_pk=None, record=True, # Catch unhandled errors in the setup step except Exception as e: # noqa + extra = { + 'stack': True, + 'tags': { + 'build': build_pk, + }, + } + if self.project: + extra['tags']['project'] = self.project.slug + if self.version: + extra['tags']['version'] = self.version.slug log.exception( 'An unhandled exception was raised during build setup', - extra={ - 'stack': True, - 'tags': { - 'build': build_pk, - 'project': self.project.slug, - 'version': self.version.slug, - }, - }, + extra=extra ) if self.setup_env is not None: self.setup_env.failure = BuildEnvironmentError( @@ -1027,6 +1034,7 @@ def fileify(version_pk, commit): ) ) _manage_imported_files(version, path, commit) + _update_intersphinx_data(version, path, commit) else: log.info( LOG_TEMPLATE.format( @@ -1037,6 +1045,56 @@ def fileify(version_pk, commit): ) +def _update_intersphinx_data(version, path, commit): + """ + Update intersphinx data for this version + + :param version: Version instance + :param path: Path to search + :param commit: Commit that updated path + """ + object_file = os.path.join(path, 'objects.inv') + + class MockConfig: + intersphinx_timeout = None # type: int + tls_verify = False + + class MockApp: + srcdir = '' + config = MockConfig() + + def warn(self, msg): + # type: (unicode) -> None + print(msg, file=sys.stderr) + + invdata = intersphinx.fetch_inventory(MockApp(), '', object_file) + for key in sorted(invdata or {}): + domain, _type = key.split(':') + for name, einfo in sorted(invdata[key].items()): + url = einfo[2] + if '#' in url: + doc_name, anchor = url.split('#') + else: + doc_name, anchor = url, '' + display_name = einfo[3] + obj, _ = DomainData.objects.get_or_create( + project=version.project, + version=version, + domain=domain, + name=name, + display_name=display_name, + type=_type, + doc_name=doc_name, + anchor=anchor, + ) + if obj.commit != commit: + obj.commit = commit + obj.save() + DomainData.objects.filter(project=version.project, + version=version + ).exclude(commit=commit).delete() + + def _manage_imported_files(version, path, commit): """ Update imported files for version. From 537c66adefb98c45b3ca3b8c202dcb781848e7b2 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 30 Jan 2019 20:05:46 -0500 Subject: [PATCH 05/22] Cleanup of some domain stuff --- readthedocs/domaindata/admin.py | 1 + readthedocs/domaindata/api.py | 13 ------------- readthedocs/domaindata/models.py | 19 ++++++++++++++++++- 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/readthedocs/domaindata/admin.py b/readthedocs/domaindata/admin.py index 3485ee1c572..9fe25b7012a 100644 --- a/readthedocs/domaindata/admin.py +++ b/readthedocs/domaindata/admin.py @@ -4,6 +4,7 @@ class DomainDataAdmin(admin.ModelAdmin): list_filter = ('type', 'project') + raw_id_fields = ('project', 'version') admin.site.register(DomainData, DomainDataAdmin) diff --git a/readthedocs/domaindata/api.py b/readthedocs/domaindata/api.py index d3176d1a396..5499e931260 100644 --- a/readthedocs/domaindata/api.py +++ b/readthedocs/domaindata/api.py @@ -1,30 +1,17 @@ from rest_framework import serializers from readthedocs.restapi.views.model_views import UserSelectViewSet -from readthedocs.core.resolver import resolve from .models import DomainData class DomainDataSerializer(serializers.ModelSerializer): project = serializers.SlugRelatedField(slug_field='slug', read_only=True) version = serializers.SlugRelatedField(slug_field='slug', read_only=True) - doc_type = serializers.SerializerMethodField() - doc_url = serializers.SerializerMethodField() class Meta: model = DomainData fields = ('project', 'version', 'name', 'display_name', 'doc_type', 'doc_url') - def get_doc_type(self, obj): - return f'{obj.domain}:{obj.type}' - - def get_doc_url(self, obj): - path = obj.doc_name - if obj.anchor: - path += f'#{obj.anchor}' - full_url = resolve(project=obj.project, version_slug=obj.version.slug, filename=path) - return full_url - class DomainDataAdminSerializer(DomainDataSerializer): diff --git a/readthedocs/domaindata/models.py b/readthedocs/domaindata/models.py index 9271708821f..9f3002bb247 100644 --- a/readthedocs/domaindata/models.py +++ b/readthedocs/domaindata/models.py @@ -3,6 +3,7 @@ from django.utils.translation import ugettext_lazy as _ from readthedocs.builds.models import Version +from readthedocs.core.resolver import resolve from readthedocs.projects.models import Project from readthedocs.projects.querysets import RelatedProjectQuerySet @@ -52,4 +53,20 @@ class DomainData(models.Model): objects = RelatedProjectQuerySet.as_manager() def __str__(self): - return f'DomainData [{self.project.slug}:{self.version.slug}] [{self.domain}:{self.type}] {self.name} -> {self.doc_name}#{self.anchor}' + return f''' + DomainData [{self.project.slug}:{self.version.slug}] + [{self.domain}:{self.type}] {self.name} -> {self.doc_name}#{self.anchor} + ''' + + @property + def doc_type(self): + return f'{self.domain}:{self.type}' + + @property + def doc_url(self): + path = self.doc_name + if self.anchor: + path += f'#{self.anchor}' + full_url = resolve( + project=self.project, version_slug=self.version.slug, filename=path) + return full_url From 647ae3f44a91ded8eed70210303a90837f3b0957 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 30 Jan 2019 20:05:54 -0500 Subject: [PATCH 06/22] Add search to Domains :tada: --- readthedocs/search/documents.py | 29 +++++++++++++++ readthedocs/search/faceted_search.py | 16 +++++++-- readthedocs/search/signals.py | 3 +- readthedocs/search/views.py | 10 +++++- readthedocs/settings/base.py | 6 ++++ .../templates/search/elastic_search.html | 35 ++++++++++++++++++- 6 files changed, 94 insertions(+), 5 deletions(-) diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index 779d0a52340..c68f1e4c0b4 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -4,6 +4,7 @@ from django_elasticsearch_dsl import DocType, Index, fields from readthedocs.projects.models import Project, HTMLFile +from readthedocs.domaindata.models import DomainData project_conf = settings.ES_INDEXES['project'] project_index = Index(project_conf['name']) @@ -13,10 +14,38 @@ page_index = Index(page_conf['name']) page_index.settings(**page_conf['settings']) +domain_conf = settings.ES_INDEXES['domain'] +domain_index = Index(domain_conf['name']) +domain_index.settings(**domain_conf['settings']) log = logging.getLogger(__name__) +@domain_index.doc_type +class DomainDocument(DocType): + project = fields.KeywordField(attr='project.slug') + version = fields.KeywordField(attr='version.slug') + doc_type = fields.KeywordField(attr='doc_type') + + class Meta(object): + model = DomainData + fields = ('name', 'display_name', 'doc_name', 'anchor') + ignore_signals = True + + @classmethod + def faceted_search(cls, query, user, doc_type=None): + from readthedocs.search.faceted_search import DomainSearch + kwargs = { + 'user': user, + 'query': query, + } + + if doc_type: + kwargs['filters'] = {'doc_type': doc_type} + + return DomainSearch(**kwargs) + + @project_index.doc_type class ProjectDocument(DocType): diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 31c4a34aa25..47f083a291b 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -3,8 +3,8 @@ from elasticsearch_dsl import FacetedSearch, TermsFacet from elasticsearch_dsl.query import SimpleQueryString, Bool -from readthedocs.search.documents import PageDocument, ProjectDocument -from readthedocs.search.signals import before_file_search, before_project_search +from readthedocs.search.documents import PageDocument, ProjectDocument, DomainDocument +from readthedocs.search.signals import before_file_search, before_project_search, before_domain_search log = logging.getLogger(__name__) @@ -50,6 +50,18 @@ def query(self, search, query): return search +class DomainSearch(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version'), + 'doc_type': TermsFacet(field='doc_type'), + } + signal = before_domain_search + doc_types = [DomainDocument] + index = DomainDocument._doc_type.index + fields = ('display_name^5', 'name') + + class ProjectSearch(RTDFacetedSearch): facets = { 'language': TermsFacet(field='language') diff --git a/readthedocs/search/signals.py b/readthedocs/search/signals.py index 93a102ca526..cafce4b9ea0 100644 --- a/readthedocs/search/signals.py +++ b/readthedocs/search/signals.py @@ -11,8 +11,9 @@ from readthedocs.search.tasks import index_objects_to_es, delete_objects_in_es -before_project_search = django.dispatch.Signal(providing_args=['user', 'search']) +before_domain_search = django.dispatch.Signal(providing_args=['user', 'search']) before_file_search = django.dispatch.Signal(providing_args=['user', 'search']) +before_project_search = django.dispatch.Signal(providing_args=['user', 'search']) @receiver(bulk_post_create, sender=HTMLFile) diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 2f9f7bf7cf5..04bd23fff68 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -8,7 +8,7 @@ from django.shortcuts import get_object_or_404, render from readthedocs.builds.constants import LATEST -from readthedocs.search.documents import PageDocument, ProjectDocument +from readthedocs.search.documents import PageDocument, ProjectDocument, DomainDocument from readthedocs.search.utils import get_project_list_or_404 from readthedocs.projects.models import Project @@ -25,6 +25,7 @@ 'version', 'taxonomy', 'language', + 'doc_type', ), ) @@ -38,6 +39,7 @@ def elastic_search(request): version=request.GET.get('version', LATEST), taxonomy=request.GET.get('taxonomy'), language=request.GET.get('language'), + doc_type=request.GET.get('doc_type'), ) results = '' user = '' @@ -53,6 +55,12 @@ def elastic_search(request): ) results = project_search.execute() facets = results.facets + elif user_input.type == 'domain': + project_search = DomainDocument.faceted_search( + query=user_input.query, user=user, doc_type=user_input.doc_type + ) + results = project_search.execute() + facets = results.facets elif user_input.type == 'file': kwargs = {} if user_input.project: diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index a4622e6a16c..c009cbb62bd 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -358,6 +358,12 @@ def USE_PROMOS(self): # noqa ES_TASK_CHUNK_SIZE = 100 ES_INDEXES = { + 'domain': { + 'name': 'domain_index', + 'settings': {'number_of_shards': 2, + 'number_of_replicas': 0 + } + }, 'project': { 'name': 'project_index', 'settings': {'number_of_shards': 2, diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index f46ef4f3a17..1c00311d4a4 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -19,6 +19,7 @@
  • {% trans 'Projects' %}
  • {% trans 'Files' %}
    • +
  • {% trans 'Domain objects' %}
    • @@ -41,6 +42,27 @@
    {% trans 'Language' %}
    {% endif %} + + {% if type == 'domain' %} + + {% if facets.doc_type %} +
    {% trans 'Type' %}
    + {% for name, count, selected in facets.doc_type %} +
  • + {% if doc_type == name %} + {{ name }} + {% else %} + {{ name }} + {% endif %} + ({{ count }}) + +
    • + {% endfor %} + {% endif %} + + {% endif %} + + {% if type == 'file' %} {% if facets.project %} @@ -124,7 +146,7 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo {% for result in results %}

  • - {% if result.name %} + {% if result.language %} {# Project #} {{ result.name }} ({{ result.slug }}) @@ -140,6 +162,17 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo {% endfor %} {# End Project #} + {% elif result.doc_type %} + + {# Domain Data #} + {{ result.project }} - {{ result.doc_type }} - {{ result.name|safe }} + {% for fragment in result.meta.highlight.display_name|slice:":3" %} +

    + ...{{ fragment|safe }}... +

    + {% endfor %} + {# End File #} + {% elif result.path %} {# File #} From 1dfabe84660b129379b8a0b8e4cb21ce862d0e3d Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Thu, 31 Jan 2019 14:22:13 -0500 Subject: [PATCH 07/22] Add initial search implementation to DomainData --- readthedocs/domaindata/admin.py | 1 + readthedocs/search/faceted_search.py | 45 +- .../static/search/readthedocs-client.js | 804 ++++++++++++++++++ readthedocs/search/views.py | 15 +- .../templates/search/elastic_search.html | 34 +- 5 files changed, 892 insertions(+), 7 deletions(-) create mode 100644 readthedocs/search/static/search/readthedocs-client.js diff --git a/readthedocs/domaindata/admin.py b/readthedocs/domaindata/admin.py index 9fe25b7012a..d24542669f9 100644 --- a/readthedocs/domaindata/admin.py +++ b/readthedocs/domaindata/admin.py @@ -5,6 +5,7 @@ class DomainDataAdmin(admin.ModelAdmin): list_filter = ('type', 'project') raw_id_fields = ('project', 'version') + search_fields = ('doc_name', 'name') admin.site.register(DomainData, DomainDataAdmin) diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index e88db0f6cb4..067f87d0fe9 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -1,10 +1,20 @@ +# -*- coding: utf-8 -*- import logging from elasticsearch_dsl import FacetedSearch, TermsFacet from elasticsearch_dsl.query import Bool, SimpleQueryString -from readthedocs.search.documents import PageDocument, ProjectDocument, DomainDocument -from readthedocs.search.signals import before_file_search, before_project_search, before_domain_search +from readthedocs.search.documents import ( + DomainDocument, + PageDocument, + ProjectDocument, +) +from readthedocs.search.signals import ( + before_domain_search, + before_file_search, + before_project_search, +) + log = logging.getLogger(__name__) @@ -62,6 +72,27 @@ class DomainSearch(RTDFacetedSearch): index = DomainDocument._doc_type.index fields = ('display_name^5', 'name') + def query(self, search, query): + """Use a custom SimpleQueryString instead of default query.""" + + search = super().query(search, query) + + all_queries = [] + + # need to search for both 'and' and 'or' operations + # the score of and should be higher as it satisfies both or and and + for operator in ['and', 'or']: + query_string = SimpleQueryString( + query=query, fields=self.fields, default_operator=operator + ) + all_queries.append(query_string) + + # run bool query with should, so it returns result where either of the query matches + bool_query = Bool(should=all_queries) + + search = search.query(bool_query) + return search + class ProjectSearch(RTDFacetedSearch): facets = {'language': TermsFacet(field='language')} @@ -101,3 +132,13 @@ def query(self, search, query): search = search.query(bool_query) return search + + +class AllSearch(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version') + } + signal = before_project_search + doc_types = [DomainDocument, PageDocument, ProjectDocument] + index = ['page_index', 'domain_index', 'project_index'] diff --git a/readthedocs/search/static/search/readthedocs-client.js b/readthedocs/search/static/search/readthedocs-client.js new file mode 100644 index 00000000000..8d5620d2f26 --- /dev/null +++ b/readthedocs/search/static/search/readthedocs-client.js @@ -0,0 +1,804 @@ +(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o 0) { + self._completeHandlers.shift()(resp) + } + } + + function success (resp) { + var type = o['type'] || resp && setType(resp.getResponseHeader('Content-Type')) // resp can be undefined in IE + resp = (type !== 'jsonp') ? self.request : resp + // use global data filter on response text + var filteredResponse = globalSetupOptions.dataFilter(resp.responseText, type) + , r = filteredResponse + try { + resp.responseText = r + } catch (e) { + // can't assign this in IE<=8, just ignore + } + if (r) { + switch (type) { + case 'json': + try { + resp = win.JSON ? win.JSON.parse(r) : eval('(' + r + ')') + } catch (err) { + return error(resp, 'Could not parse JSON in response', err) + } + break + case 'js': + resp = eval(r) + break + case 'html': + resp = r + break + case 'xml': + resp = resp.responseXML + && resp.responseXML.parseError // IE trololo + && resp.responseXML.parseError.errorCode + && resp.responseXML.parseError.reason + ? null + : resp.responseXML + break + } + } + + self._responseArgs.resp = resp + self._fulfilled = true + fn(resp) + self._successHandler(resp) + while (self._fulfillmentHandlers.length > 0) { + resp = self._fulfillmentHandlers.shift()(resp) + } + + complete(resp) + } + + function timedOut() { + self._timedOut = true + self.request.abort() + } + + function error(resp, msg, t) { + resp = self.request + self._responseArgs.resp = resp + self._responseArgs.msg = msg + self._responseArgs.t = t + self._erred = true + while (self._errorHandlers.length > 0) { + self._errorHandlers.shift()(resp, msg, t) + } + complete(resp) + } + + this.request = getRequest.call(this, success, error) + } + + Reqwest.prototype = { + abort: function () { + this._aborted = true + this.request.abort() + } + + , retry: function () { + init.call(this, this.o, this.fn) + } + + /** + * Small deviation from the Promises A CommonJs specification + * http://wiki.commonjs.org/wiki/Promises/A + */ + + /** + * `then` will execute upon successful requests + */ + , then: function (success, fail) { + success = success || function () {} + fail = fail || function () {} + if (this._fulfilled) { + this._responseArgs.resp = success(this._responseArgs.resp) + } else if (this._erred) { + fail(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) + } else { + this._fulfillmentHandlers.push(success) + this._errorHandlers.push(fail) + } + return this + } + + /** + * `always` will execute whether the request succeeds or fails + */ + , always: function (fn) { + if (this._fulfilled || this._erred) { + fn(this._responseArgs.resp) + } else { + this._completeHandlers.push(fn) + } + return this + } + + /** + * `fail` will execute when the request fails + */ + , fail: function (fn) { + if (this._erred) { + fn(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) + } else { + this._errorHandlers.push(fn) + } + return this + } + , 'catch': function (fn) { + return this.fail(fn) + } + } + + function reqwest(o, fn) { + return new Reqwest(o, fn) + } + + // normalize newline variants according to spec -> CRLF + function normalize(s) { + return s ? s.replace(/\r?\n/g, '\r\n') : '' + } + + function serial(el, cb) { + var n = el.name + , t = el.tagName.toLowerCase() + , optCb = function (o) { + // IE gives value="" even where there is no value attribute + // 'specified' ref: http://www.w3.org/TR/DOM-Level-3-Core/core.html#ID-862529273 + if (o && !o['disabled']) + cb(n, normalize(o['attributes']['value'] && o['attributes']['value']['specified'] ? o['value'] : o['text'])) + } + , ch, ra, val, i + + // don't serialize elements that are disabled or without a name + if (el.disabled || !n) return + + switch (t) { + case 'input': + if (!/reset|button|image|file/i.test(el.type)) { + ch = /checkbox/i.test(el.type) + ra = /radio/i.test(el.type) + val = el.value + // WebKit gives us "" instead of "on" if a checkbox has no value, so correct it here + ;(!(ch || ra) || el.checked) && cb(n, normalize(ch && val === '' ? 'on' : val)) + } + break + case 'textarea': + cb(n, normalize(el.value)) + break + case 'select': + if (el.type.toLowerCase() === 'select-one') { + optCb(el.selectedIndex >= 0 ? el.options[el.selectedIndex] : null) + } else { + for (i = 0; el.length && i < el.length; i++) { + el.options[i].selected && optCb(el.options[i]) + } + } + break + } + } + + // collect up all form elements found from the passed argument elements all + // the way down to child elements; pass a '
    ' or form fields. + // called with 'this'=callback to use for serial() on each element + function eachFormElement() { + var cb = this + , e, i + , serializeSubtags = function (e, tags) { + var i, j, fa + for (i = 0; i < tags.length; i++) { + fa = e[byTag](tags[i]) + for (j = 0; j < fa.length; j++) serial(fa[j], cb) + } + } + + for (i = 0; i < arguments.length; i++) { + e = arguments[i] + if (/input|select|textarea/i.test(e.tagName)) serial(e, cb) + serializeSubtags(e, [ 'input', 'select', 'textarea' ]) + } + } + + // standard query string style serialization + function serializeQueryString() { + return reqwest.toQueryString(reqwest.serializeArray.apply(null, arguments)) + } + + // { 'name': 'value', ... } style serialization + function serializeHash() { + var hash = {} + eachFormElement.apply(function (name, value) { + if (name in hash) { + hash[name] && !isArray(hash[name]) && (hash[name] = [hash[name]]) + hash[name].push(value) + } else hash[name] = value + }, arguments) + return hash + } + + // [ { name: 'name', value: 'value' }, ... ] style serialization + reqwest.serializeArray = function () { + var arr = [] + eachFormElement.apply(function (name, value) { + arr.push({name: name, value: value}) + }, arguments) + return arr + } + + reqwest.serialize = function () { + if (arguments.length === 0) return '' + var opt, fn + , args = Array.prototype.slice.call(arguments, 0) + + opt = args.pop() + opt && opt.nodeType && args.push(opt) && (opt = null) + opt && (opt = opt.type) + + if (opt == 'map') fn = serializeHash + else if (opt == 'array') fn = reqwest.serializeArray + else fn = serializeQueryString + + return fn.apply(null, args) + } + + reqwest.toQueryString = function (o, trad) { + var prefix, i + , traditional = trad || false + , s = [] + , enc = encodeURIComponent + , add = function (key, value) { + // If value is a function, invoke it and return its value + value = ('function' === typeof value) ? value() : (value == null ? '' : value) + s[s.length] = enc(key) + '=' + enc(value) + } + // If an array was passed in, assume that it is an array of form elements. + if (isArray(o)) { + for (i = 0; o && i < o.length; i++) add(o[i]['name'], o[i]['value']) + } else { + // If traditional, encode the "old" way (the way 1.3.2 or older + // did it), otherwise encode params recursively. + for (prefix in o) { + if (o.hasOwnProperty(prefix)) buildParams(prefix, o[prefix], traditional, add) + } + } + + // spaces should be + according to spec + return s.join('&').replace(/%20/g, '+') + } + + function buildParams(prefix, obj, traditional, add) { + var name, i, v + , rbracket = /\[\]$/ + + if (isArray(obj)) { + // Serialize array item. + for (i = 0; obj && i < obj.length; i++) { + v = obj[i] + if (traditional || rbracket.test(prefix)) { + // Treat each array item as a scalar. + add(prefix, v) + } else { + buildParams(prefix + '[' + (typeof v === 'object' ? i : '') + ']', v, traditional, add) + } + } + } else if (obj && obj.toString() === '[object Object]') { + // Serialize object item. + for (name in obj) { + buildParams(prefix + '[' + name + ']', obj[name], traditional, add) + } + + } else { + // Serialize scalar item. + add(prefix, obj) + } + } + + reqwest.getcallbackPrefix = function () { + return callbackPrefix + } + + // jQuery and Zepto compatibility, differences can be remapped here so you can call + // .ajax.compat(options, callback) + reqwest.compat = function (o, fn) { + if (o) { + o['type'] && (o['method'] = o['type']) && delete o['type'] + o['dataType'] && (o['type'] = o['dataType']) + o['jsonpCallback'] && (o['jsonpCallbackName'] = o['jsonpCallback']) && delete o['jsonpCallback'] + o['jsonp'] && (o['jsonpCallback'] = o['jsonp']) + } + return new Reqwest(o, fn) + } + + reqwest.ajaxSetup = function (options) { + options = options || {} + for (var k in options) { + globalSetupOptions[k] = options[k] + } + } + + return reqwest +}); + +},{}],2:[function(require,module,exports){ +// Document response + +// Page +var Page = function (project, version, doc) { + this.project = project; + this.version = version; + this.doc = doc; + + this.url = null; + this.sections = []; +}; + +Page.prototype.section = function (section) { + return new Section(this.project, this.version, this.doc, section); +}; + +// Section +var Section = function (project, version, doc, section) { + this.project = project; + this.version = version; + this.doc = doc; + this.section = section; + + this.url = null; + this.content = null; + this.wrapped = null; +} + +// Add iframe with returned content to page +Section.prototype.insertContent = function (elem) { + var iframe = document.createElement('iframe'), + self = this; + + iframe.style.display = 'none'; + + if (window.jQuery && elem instanceof window.jQuery) { + elem = elem.get(0); + } + + if (typeof(elem) != 'undefined') { + while (elem.children.length > 0) { + elem.firstChild.remove(); + } + elem.appendChild(iframe); + } + + var win = iframe.contentWindow; + + win.document.open(); + win.document.write(this.content); + win.document.close(); + + var head = win.document.head, + body = win.document.body, + base = null; + + if (head) { + base = win.document.createElement('base'); + base.target = '_parent'; + base.href = this.url; + head.appendChild(base); + + // Copy linked stylesheets from parent + var link_elems = document.head.getElementsByTagName('link'); + for (var n = 0; n < link_elems.length; n++) { + var link = link_elems[n]; + if (link.rel == 'stylesheet') { + head.appendChild(link.cloneNode()); + } + } + } + + win.onload = function () { + iframe.style.display = 'inline-block'; + }; + + return iframe; +}; + + +exports.Section = Section; +exports.Page = Page; + +},{}],3:[function(require,module,exports){ +/* Read the Docs Embed functions */ + +var doc = require('./doc'), + Section = doc.Section, + Page = doc.Page; + + +var Embed = function (config) { + this._api_host = 'https://api.grokthedocs.com'; + if (typeof config == 'object') { + if ('api_host' in config) { + this._api_host = config['api_host']; + } + } +}; + +Embed.prototype.section = function (project, version, doc, section, + callback, error_callback) { + callback = callback || function () {}; + error_callback = error_callback || function () {}; + + var self = this, + data = { + 'project': project, + 'version': version, + 'doc': doc, + 'section': section + }; + + this._getObject( + data, + function (resp) { + var section_ret = new Section(project, version, doc, section); + section_ret.url = resp.url; + section_ret.content = resp.content; + section_ret.wrapped = resp.wrapped; + callback(section_ret); + }, + function (error, msg) { + error_callback(error); + } + ); +}; + +Embed.prototype.page = function (project, version, doc, callback, + error_callback) { + + var self = this, + data = { + 'project': project, + 'version': version, + 'doc': doc, + }; + + this._getObject( + data, + function (resp) { + var page = new Page(project, version, doc); + page.url = resp.url; + // TODO headers is misleading here, rename it on the API + page.sections = resp.headers; + callback(page); + }, + function (error, msg) { + error_callback(error); + } + ) +}; + +Embed.prototype._getObject = function (data, callback, error_callback) { + var self = this, + reqwest = require("./../bower_components/reqwest/reqwest.js"); + callback = callback || function () {}; + error_callback = error_callback || function () {}; + + return reqwest({ + url: this._api_host + '/api/v1/embed/', + method: 'get', + contentType: 'application/json', + crossDomain: true, + headers: {'Accept': 'application/json'}, + data: data, + success: callback, + error: error_callback + }); +}; + +exports.Embed = Embed; + +},{"./../bower_components/reqwest/reqwest.js":1,"./doc":2}],4:[function(require,module,exports){ +/* Read the Docs Client */ + +var embed = require('./embed'); + + +exports.Embed = embed.Embed; + +if (typeof window != 'undefined') { + window.Embed = embed.Embed; +} + +},{"./embed":3}]},{},[4]) \ No newline at end of file diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 04bd23fff68..5ed00fc38c0 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -8,9 +8,14 @@ from django.shortcuts import get_object_or_404, render from readthedocs.builds.constants import LATEST -from readthedocs.search.documents import PageDocument, ProjectDocument, DomainDocument -from readthedocs.search.utils import get_project_list_or_404 from readthedocs.projects.models import Project +from readthedocs.search.documents import ( + DomainDocument, + PageDocument, + ProjectDocument, +) +from readthedocs.search.faceted_search import AllSearch +from readthedocs.search.utils import get_project_list_or_404 log = logging.getLogger(__name__) @@ -77,6 +82,12 @@ def elastic_search(request): ) results = page_search.execute() facets = results.facets + elif user_input.type == 'all': + project_search = AllSearch( + query=user_input.query, user=user + ) + results = project_search.execute() + facets = results.facets log.info( LOG_TEMPLATE.format( diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 1c00311d4a4..2b5a21d9265 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -1,6 +1,6 @@ {% extends "projects/base_project.html" %} -{% load core_tags i18n %} +{% load core_tags i18n static %} {% block title %}{% blocktrans with query=query|default:"" %}Search: {{ query }} {% endblocktrans %}{% endblock %} @@ -13,6 +13,31 @@ {% endblock %} +{% block extra_scripts %} + + + +{% endblock %} + {% block content %}
    @@ -20,6 +45,7 @@
  • {% trans 'Projects' %}
  • {% trans 'Files' %}
  • {% trans 'Domain objects' %}
    • +
  • {% trans 'All' %}
    • @@ -166,11 +192,13 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo {# Domain Data #} {{ result.project }} - {{ result.doc_type }} - {{ result.name|safe }} - {% for fragment in result.meta.highlight.display_name|slice:":3" %} -

    + {% for fragment in result.meta.highlight.name|slice:":3" %} +

    ...{{ fragment|safe }}...

    {% endfor %} +
    +
    {# End File #} {% elif result.path %} From 08296ae55c8470c847f85739e3b876445e3257f3 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Thu, 31 Jan 2019 20:25:58 -0500 Subject: [PATCH 08/22] Move project search to a subset of site search This removes the second entry point to site search, and unifies all our searching to use fancy facets and be nice. --- readthedocs/projects/urls/public.py | 2 +- readthedocs/search/faceted_search.py | 85 +++--- .../search/static/search/search-embed.js | 18 ++ readthedocs/search/views.py | 125 ++++---- .../search/elastic_project_search.html | 94 ------ .../templates/search/elastic_search.html | 273 ++++++++---------- 6 files changed, 238 insertions(+), 359 deletions(-) create mode 100644 readthedocs/search/static/search/search-embed.js delete mode 100644 readthedocs/templates/search/elastic_project_search.html diff --git a/readthedocs/projects/urls/public.py b/readthedocs/projects/urls/public.py index 7f3868287d6..101ca96fe4b 100644 --- a/readthedocs/projects/urls/public.py +++ b/readthedocs/projects/urls/public.py @@ -51,7 +51,7 @@ ), url( r'^(?P{project_slug})/search/$'.format(**pattern_opts), - search_views.elastic_project_search, + search_views.elastic_search, name='elastic_project_search', ), url( diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 067f87d0fe9..71a0efb1974 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -19,6 +19,9 @@ log = logging.getLogger(__name__) +ALL_FACETS = ['project', 'version', 'doc_type', 'language'] + + class RTDFacetedSearch(FacetedSearch): """Overwrite the initialization in order too meet our needs.""" @@ -28,6 +31,14 @@ class RTDFacetedSearch(FacetedSearch): def __init__(self, user, **kwargs): self.user = user + for facet in self.facets: + if facet in kwargs: + kwargs.setdefault('filters', {})[facet] = kwargs.pop(facet) + + # Don't pass along unnecessary filters + for f in ALL_FACETS: + if f in kwargs: + del kwargs[f] super(RTDFacetedSearch, self).__init__(**kwargs) def search(self): @@ -54,28 +65,12 @@ def query(self, search, query): """ Add query part to ``search`` when needed. - Also does HTML encoding of results to avoid XSS issues. + Also: + + * Adds SimpleQueryString instead of default query. + * Adds HTML encoding of results to avoid XSS issues. """ - search = super().query(search, query) search = search.highlight_options(encoder='html', number_of_fragments=3) - return search - - -class DomainSearch(RTDFacetedSearch): - facets = { - 'project': TermsFacet(field='project'), - 'version': TermsFacet(field='version'), - 'doc_type': TermsFacet(field='doc_type'), - } - signal = before_domain_search - doc_types = [DomainDocument] - index = DomainDocument._doc_type.index - fields = ('display_name^5', 'name') - - def query(self, search, query): - """Use a custom SimpleQueryString instead of default query.""" - - search = super().query(search, query) all_queries = [] @@ -94,8 +89,22 @@ def query(self, search, query): return search +class DomainSearch(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version'), + 'doc_type': TermsFacet(field='doc_type'), + } + signal = before_domain_search + doc_types = [DomainDocument] + index = DomainDocument._doc_type.index + fields = ('display_name^5', 'name') + + class ProjectSearch(RTDFacetedSearch): - facets = {'language': TermsFacet(field='language')} + facets = { + 'language': TermsFacet(field='language') + } signal = before_project_search doc_types = [ProjectDocument] index = ProjectDocument._doc_type.index @@ -112,33 +121,19 @@ class PageSearch(RTDFacetedSearch): index = PageDocument._doc_type.index fields = ['title^10', 'headers^5', 'content'] - def query(self, search, query): - """Use a custom SimpleQueryString instead of default query.""" - - search = super().query(search, query) - - all_queries = [] - - # need to search for both 'and' and 'or' operations - # the score of and should be higher as it satisfies both or and and - for operator in ['and', 'or']: - query_string = SimpleQueryString( - query=query, fields=self.fields, default_operator=operator - ) - all_queries.append(query_string) - - # run bool query with should, so it returns result where either of the query matches - bool_query = Bool(should=all_queries) - - search = search.query(bool_query) - return search - class AllSearch(RTDFacetedSearch): facets = { 'project': TermsFacet(field='project'), - 'version': TermsFacet(field='version') + 'version': TermsFacet(field='version'), + 'language': TermsFacet(field='language'), + 'doc_type': TermsFacet(field='doc_type'), + 'index': TermsFacet(field='_index'), } - signal = before_project_search + signal = before_file_search doc_types = [DomainDocument, PageDocument, ProjectDocument] - index = ['page_index', 'domain_index', 'project_index'] + index = [DomainDocument._doc_type.index, + PageDocument._doc_type.index, + ProjectDocument._doc_type.index] + fields = ('title^10', 'headers^5', 'content', 'name^10', + 'slug^5', 'description', 'display_name^5') diff --git a/readthedocs/search/static/search/search-embed.js b/readthedocs/search/static/search/search-embed.js new file mode 100644 index 00000000000..88440ad8254 --- /dev/null +++ b/readthedocs/search/static/search/search-embed.js @@ -0,0 +1,18 @@ + + $(document).ready(function () { + var config = {api_host: 'http://localhost:8000'} + var embed = new Embed(config); + $('.fragment').one("click mouseover", + function(elem) { + embed.section( + elem.currentTarget.getAttribute('project'), + elem.currentTarget.getAttribute('version'), + elem.currentTarget.getAttribute('doc_name').replace('.html', ''), + elem.currentTarget.getAttribute('anchor'), + function (section) { + section.insertContent(elem.currentTarget); + } + ) + } + ) + }) diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 5ed00fc38c0..208100c2fda 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -9,12 +9,7 @@ from readthedocs.builds.constants import LATEST from readthedocs.projects.models import Project -from readthedocs.search.documents import ( - DomainDocument, - PageDocument, - ProjectDocument, -) -from readthedocs.search.faceted_search import AllSearch +from readthedocs.search.faceted_search import AllSearch, ProjectSearch, PageSearch, DomainSearch from readthedocs.search.utils import get_project_list_or_404 @@ -31,20 +26,22 @@ 'taxonomy', 'language', 'doc_type', + 'index', ), ) -def elastic_search(request): +def elastic_search(request, project_slug=None): """Use Elasticsearch for global search.""" user_input = UserInput( query=request.GET.get('q'), - type=request.GET.get('type', 'project'), - project=request.GET.get('project'), + type=request.GET.get('type', 'all'), + project=project_slug or request.GET.get('project'), version=request.GET.get('version', LATEST), taxonomy=request.GET.get('taxonomy'), language=request.GET.get('language'), doc_type=request.GET.get('doc_type'), + index=request.GET.get('index'), ) results = '' user = '' @@ -54,37 +51,50 @@ def elastic_search(request): user = request.user if user_input.query: + kwargs = {} + + if user_input.project: + projects_list = get_project_list_or_404( + project_slug=user_input.project, user=user + ) + project_slug_list = [project.slug for project in projects_list] + kwargs['project'] = project_slug_list + log.info('Filtering to project list: %s' % project_slug_list) + + if user_input.version: + kwargs['version'] = [user_input.version] + + if user_input.doc_type: + kwargs['doc_type'] = user_input.doc_type + + if user_input.language: + kwargs['language'] = user_input.language + + if user_input.index: + kwargs['index'] = user_input.index + if user_input.type == 'project': - project_search = ProjectDocument.faceted_search( - query=user_input.query, user=user, language=user_input.language + project_search = ProjectSearch( + query=user_input.query, user=user, **kwargs ) results = project_search.execute() facets = results.facets elif user_input.type == 'domain': - project_search = DomainDocument.faceted_search( - query=user_input.query, user=user, doc_type=user_input.doc_type + project_search = DomainSearch( + query=user_input.query, user=user, **kwargs ) results = project_search.execute() facets = results.facets elif user_input.type == 'file': - kwargs = {} - if user_input.project: - projects_list = get_project_list_or_404( - project_slug=user_input.project, user=user - ) - project_slug_list = [project.slug for project in projects_list] - kwargs['projects_list'] = project_slug_list - if user_input.version: - kwargs['versions_list'] = user_input.version - - page_search = PageDocument.faceted_search( + + page_search = PageSearch( query=user_input.query, user=user, **kwargs ) results = page_search.execute() facets = results.facets elif user_input.type == 'all': project_search = AllSearch( - query=user_input.query, user=user + query=user_input.query, user=user, **kwargs ) results = project_search.execute() facets = results.facets @@ -100,6 +110,14 @@ def elastic_search(request): ), ) + # Make sure our selected facets are displayed even when they return 0 results + for avail_facet in ['project', 'version', 'doc_type', 'language']: + value = getattr(user_input, avail_facet) + if not value or avail_facet not in facets: + continue + if value not in [val[0] for val in facets[avail_facet]]: + facets[avail_facet].insert(0, (value, 0, True)) + if results: if user_input.type == 'file': # Change results to turn newlines in highlight into periods @@ -114,55 +132,18 @@ def elastic_search(request): log.debug('Search facets: %s', pformat(results.facets.to_dict())) template_vars = user_input._asdict() - template_vars.update({'results': results, 'facets': facets}) - return render( - request, - 'search/elastic_search.html', - template_vars, - ) - + template_vars.update({ + 'results': results, + 'facets': facets, + }) -def elastic_project_search(request, project_slug): - """Use elastic search to search in a project.""" - queryset = Project.objects.protected(request.user) - project = get_object_or_404(queryset, slug=project_slug) - version_slug = request.GET.get('version', LATEST) - query = request.GET.get('q', None) - results = None - - if query: - kwargs = {} - kwargs['projects_list'] = [project.slug] - kwargs['versions_list'] = version_slug - user = '' - if request.user.is_authenticated: - user = request.user - - page_search = PageDocument.faceted_search( - query=query, user=user, **kwargs - ) - results = page_search.execute() - - log.debug('Search results: %s', pformat(results.to_dict())) - log.debug('Search facets: %s', pformat(results.facets.to_dict())) - - log.info( - LOG_TEMPLATE.format( - user=user, - project=project or '', - type='inproject', - version=version_slug or '', - language='', - msg=query or '', - ), - ) + if project_slug: + queryset = Project.objects.protected(request.user) + project = get_object_or_404(queryset, slug=project_slug) + template_vars.update({'project_obj': project}) return render( request, - 'search/elastic_project_search.html', - { - 'project': project, - 'query': query, - 'results': results, - }, + 'search/elastic_search.html', + template_vars, ) diff --git a/readthedocs/templates/search/elastic_project_search.html b/readthedocs/templates/search/elastic_project_search.html deleted file mode 100644 index 1ce48e6e2b1..00000000000 --- a/readthedocs/templates/search/elastic_project_search.html +++ /dev/null @@ -1,94 +0,0 @@ -{% extends "projects/base_project.html" %} - -{% load core_tags i18n %} - -{% block title %}{% blocktrans with query=query|default:"" %}Search: {{ query }} {% endblocktrans %}{% endblock %} - -{% block project_editing %} - {% with search_active="active" %} - {% include "core/project_bar.html" %} - {% endwith %} -{% endblock %} - - -{% block content %} -
    -

    Search in this project:

    - -
    -
    - -
    - -
    -
    - {% comment %}Translators: This is about starting a search{% endcomment %} - -
    - -
    -
    - - - - {% if query %} - -
    -
    - -
    -

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblocktrans %}

    -
    - -
    -
    - -
      - {% for result in results %} -
    • -

      - {{ result.project }} - {{ result.title|safe }} -

      - {% for fragment in result.meta.highlight.content|slice:":3" %} -

      - {{ fragment|safe }} -

      - {% endfor %} -
      • - {% empty %} -
    • {% trans "No results found. Bummer." %}
      • - {% endfor %} - -
    - -
    -
    - -
    -
    - - - {% if page.has_previous or page.has_next %} - - - - {% endif %} - - {% else %} - {# Show some example queries to run, maybe query syntax, something else? #} - {% endif %} - -
    -{% endblock %} diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 2b5a21d9265..6ebe2dc31cd 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -15,41 +15,83 @@ {% block extra_scripts %} - + +{% endblock %} +{% block project_editing %} + {% if project_obj %} + {% with search_active="active" project=project_obj %} + {% include "core/project_bar.html" %} + {% endwith %} + {% endif %} {% endblock %} {% block content %}
    -

    {% trans 'Search' %}

    @@ -139,7 +143,7 @@

    {% trans 'Search' %}

    - + {% if type %} {% endif %} {% if project %} {% endif %} {% if taxonomy %} {% endif %} @@ -160,7 +164,6 @@

    {% trans 'Search' %}

    -

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblocktrans %}

    @@ -169,54 +172,53 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo
      + {% for result in results %} -
    • -

      - {% if result.language %} +

    • +

      + + {% if 'project' in result.meta.index %} + + {{ result.name }} ({{ result.slug }}) + + {% for fragment in result.meta.highlight.description %} +

      + ...{{ fragment|safe }}... +

      + {% empty %} +

      No description for project.

      + {% endfor %} + + {% elif 'domain' in result.meta.index %} + + + {{ result.project }} - {{ result.doc_type }} - {{ result.name }} + + {% for fragment in result.meta.highlight.name %} +

      + ...{{ fragment|safe }}... +

      + {% endfor %} + + {% elif 'page' in result.meta.index %} - {# Project #} - {{ result.name }} ({{ result.slug }}) + + {{ result.project }} - {{ result.title }} + + {% for fragment in result.meta.highlight.content %} +

      + ...{{ fragment|safe }}...

      - {{ result.meta.description }} - {% for fragment in result.meta.highlight.description|slice:":3" %} -

      - ...{{ fragment|safe }}... -

      -
      - {% empty %} -

      No description for project.

      - {% endfor %} - {# End Project #} - - {% elif result.doc_type %} - - {# Domain Data #} - {{ result.project }} - {{ result.doc_type }} - {{ result.name|safe }} - {% for fragment in result.meta.highlight.name|slice:":3" %} -

      - ...{{ fragment|safe }}... -

      - {% endfor %} -
      -
      - {# End File #} - - {% elif result.path %} - - {# File #} - {{ result.project }} - {{ result.title|safe }} - {% for fragment in result.meta.highlight.content|slice:":3" %} -

      - ...{{ fragment|safe }}... -

      - {% endfor %} - {# End File #} - - {% endif %} -
      • - {% empty %} -
    • {% trans "No results found. Bummer." %}
      • - {% endfor %} + {% endfor %} + + {% endif %} + +

      + + + {% empty %} +
    • {% trans "No results found." %}
      • + {% endfor %}
    @@ -225,31 +227,8 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo

    - - - {% if page.has_previous or page.has_next %} - - - - {% endif %} - - {% else %} - {# Show some example queries to run, maybe query syntax, something else? #} {% endif %} - - +
    {% endblock %} From 1e087fb7f3965124b36454ae2cf69c5fbe5c8c3e Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Fri, 1 Feb 2019 09:34:33 -0300 Subject: [PATCH 09/22] Small cleanup around faceting --- readthedocs/search/documents.py | 3 ++- readthedocs/search/faceted_search.py | 2 +- readthedocs/search/views.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index c68f1e4c0b4..70e70f06331 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -26,10 +26,11 @@ class DomainDocument(DocType): project = fields.KeywordField(attr='project.slug') version = fields.KeywordField(attr='version.slug') doc_type = fields.KeywordField(attr='doc_type') + anchor = fields.KeywordField(attr='anchor') class Meta(object): model = DomainData - fields = ('name', 'display_name', 'doc_name', 'anchor') + fields = ('name', 'display_name', 'doc_name') ignore_signals = True @classmethod diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 71a0efb1974..5a682f48ace 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -19,7 +19,7 @@ log = logging.getLogger(__name__) -ALL_FACETS = ['project', 'version', 'doc_type', 'language'] +ALL_FACETS = ['project', 'version', 'doc_type', 'language', 'index'] class RTDFacetedSearch(FacetedSearch): diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 208100c2fda..6078ad3a18a 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -9,7 +9,7 @@ from readthedocs.builds.constants import LATEST from readthedocs.projects.models import Project -from readthedocs.search.faceted_search import AllSearch, ProjectSearch, PageSearch, DomainSearch +from readthedocs.search.faceted_search import AllSearch, ProjectSearch, PageSearch, DomainSearch, ALL_FACETS from readthedocs.search.utils import get_project_list_or_404 @@ -111,7 +111,7 @@ def elastic_search(request, project_slug=None): ) # Make sure our selected facets are displayed even when they return 0 results - for avail_facet in ['project', 'version', 'doc_type', 'language']: + for avail_facet in ALL_FACETS: value = getattr(user_input, avail_facet) if not value or avail_facet not in facets: continue From 12bdf04cb2ada295e4167c4ea5061adc1c1ca1ea Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 4 Feb 2019 11:26:19 -0300 Subject: [PATCH 10/22] Limit resuts by DomainData type --- readthedocs/search/documents.py | 13 +++++++++++++ readthedocs/search/faceted_search.py | 5 +++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index 70e70f06331..89ad40f2fd6 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -46,6 +46,19 @@ def faceted_search(cls, query, user, doc_type=None): return DomainSearch(**kwargs) + def get_queryset(self): + """Overwrite default queryset to filter certain files to index""" + queryset = super().get_queryset() + + # Exclude some types to not index + excluded_types = ['std:doc', 'std:label'] + + # Do not index files that belong to non sphinx project + # Also do not index certain files + for exclude in excluded_types: + queryset = queryset.exclude(type=exclude) + return queryset + @project_index.doc_type class ProjectDocument(DocType): diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 5a682f48ace..aa5c35ae04f 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -59,7 +59,8 @@ def search(self): log.exception( 'Failed to return a search object from search signals' ) - return s + # Return 25 results + return s[:25] def query(self, search, query): """ @@ -135,5 +136,5 @@ class AllSearch(RTDFacetedSearch): index = [DomainDocument._doc_type.index, PageDocument._doc_type.index, ProjectDocument._doc_type.index] - fields = ('title^10', 'headers^5', 'content', 'name^10', + fields = ('title^10', 'headers^5', 'content', 'name^20', 'slug^5', 'description', 'display_name^5') From 444e739b98e9b2632af351f2953df31ffcf894c3 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 15:55:33 -0300 Subject: [PATCH 11/22] Remove old domains app --- readthedocs/domains/__init__.py | 0 .../migrations/0001_create-domain-data.py | 32 ------------ readthedocs/domains/migrations/__init__.py | 0 readthedocs/domains/models.py | 50 ------------------- 4 files changed, 82 deletions(-) delete mode 100644 readthedocs/domains/__init__.py delete mode 100644 readthedocs/domains/migrations/0001_create-domain-data.py delete mode 100644 readthedocs/domains/migrations/__init__.py delete mode 100644 readthedocs/domains/models.py diff --git a/readthedocs/domains/__init__.py b/readthedocs/domains/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/readthedocs/domains/migrations/0001_create-domain-data.py b/readthedocs/domains/migrations/0001_create-domain-data.py deleted file mode 100644 index 6e02dadf977..00000000000 --- a/readthedocs/domains/migrations/0001_create-domain-data.py +++ /dev/null @@ -1,32 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.16 on 2018-10-23 08:24 -from __future__ import unicode_literals - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ('projects', '0028_create-domain-data'), - ] - - operations = [ - migrations.CreateModel( - name='DomainData', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('domain', models.CharField(max_length=255, verbose_name='Domain')), - ('name', models.CharField(max_length=255, verbose_name='Name')), - ('display_name', models.CharField(max_length=255, verbose_name='Display Name')), - ('type', models.CharField(max_length=255, verbose_name='Type')), - ('doc_name', models.CharField(max_length=255, verbose_name='Doc Name')), - ('anchor', models.CharField(max_length=255, verbose_name='Anchor')), - ('priority', models.IntegerField(verbose_name='Priority')), - ('project', models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='domain_data', to='projects.Project')), - ], - ), - ] diff --git a/readthedocs/domains/migrations/__init__.py b/readthedocs/domains/migrations/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/readthedocs/domains/models.py b/readthedocs/domains/models.py deleted file mode 100644 index 5627d3c4eca..00000000000 --- a/readthedocs/domains/models.py +++ /dev/null @@ -1,50 +0,0 @@ -from django.db import models -from django.utils.encoding import python_2_unicode_compatible -from django.utils.translation import ugettext_lazy as _ - -from readthedocs.projects.models import Project - - -@python_2_unicode_compatible -class DomainData(models.Model): - - """ - Information from a project about it's Sphinx domains. - - This captures data about API objects that exist in that codebase. - """ - - project = models.OneToOneField( - Project, - on_delete=models.SET_NULL, - related_name='domain_data', - null=True, - blank=True, - ) - domain = models.CharField( - _('Domain'), - max_length=255, - ) - name = models.CharField( - _('Name'), - max_length=255, - ) - display_name = models.CharField( - _('Display Name'), - max_length=255, - ) - type = models.CharField( - _('Type'), - max_length=255, - ) - doc_name = models.CharField( - _('Doc Name'), - max_length=255, - ) - anchor = models.CharField( - _('Anchor'), - max_length=255, - ) - priority = models.IntegerField( - _('Priority'), - ) From 1672421815db5e9c46ef19222b8ba0fb1f125e70 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 16:00:32 -0300 Subject: [PATCH 12/22] Remove search changes --- readthedocs/search/documents.py | 47 +--- readthedocs/search/faceted_search.py | 132 ++++----- readthedocs/search/views.py | 123 ++++----- .../search/elastic_project_search.html | 94 +++++++ .../templates/search/elastic_search.html | 252 ++++++++---------- 5 files changed, 304 insertions(+), 344 deletions(-) create mode 100644 readthedocs/templates/search/elastic_project_search.html diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index 6b54d8c3f5a..0fcf0d69a3f 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -4,8 +4,8 @@ from django.conf import settings from django_elasticsearch_dsl import DocType, Index, fields -from readthedocs.projects.models import Project, HTMLFile -from readthedocs.domaindata.models import DomainData +from readthedocs.projects.models import HTMLFile, Project + project_conf = settings.ES_INDEXES['project'] project_index = Index(project_conf['name']) @@ -15,52 +15,9 @@ page_index = Index(page_conf['name']) page_index.settings(**page_conf['settings']) -domain_conf = settings.ES_INDEXES['domain'] -domain_index = Index(domain_conf['name']) -domain_index.settings(**domain_conf['settings']) - log = logging.getLogger(__name__) -@domain_index.doc_type -class DomainDocument(DocType): - project = fields.KeywordField(attr='project.slug') - version = fields.KeywordField(attr='version.slug') - doc_type = fields.KeywordField(attr='doc_type') - anchor = fields.KeywordField(attr='anchor') - - class Meta(object): - model = DomainData - fields = ('name', 'display_name', 'doc_name') - ignore_signals = True - - @classmethod - def faceted_search(cls, query, user, doc_type=None): - from readthedocs.search.faceted_search import DomainSearch - kwargs = { - 'user': user, - 'query': query, - } - - if doc_type: - kwargs['filters'] = {'doc_type': doc_type} - - return DomainSearch(**kwargs) - - def get_queryset(self): - """Overwrite default queryset to filter certain files to index""" - queryset = super().get_queryset() - - # Exclude some types to not index - excluded_types = ['std:doc', 'std:label'] - - # Do not index files that belong to non sphinx project - # Also do not index certain files - for exclude in excluded_types: - queryset = queryset.exclude(type=exclude) - return queryset - - @project_index.doc_type class ProjectDocument(DocType): diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 3a0c9600d37..627d78ffbe7 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -1,44 +1,17 @@ -# -*- coding: utf-8 -*- import logging from elasticsearch_dsl import FacetedSearch, TermsFacet from elasticsearch_dsl.query import Bool, SimpleQueryString -from readthedocs.search.documents import ( - DomainDocument, - PageDocument, - ProjectDocument, -) -from readthedocs.search.signals import ( - before_domain_search, - before_file_search, - before_project_search, -) - from readthedocs.core.utils.extend import SettingsOverrideObject +from readthedocs.search.documents import PageDocument, ProjectDocument log = logging.getLogger(__name__) -ALL_FACETS = ['project', 'version', 'doc_type', 'language', 'index'] - - class RTDFacetedSearch(FacetedSearch): def __init__(self, user, **kwargs): - self.user = user - self.filter_by_user = kwargs.pop('filter_by_user', None) - for facet in self.facets: - if facet in kwargs: - kwargs.setdefault('filters', {})[facet] = kwargs.pop(facet) - - # Don't pass along unnecessary filters - for f in ALL_FACETS: - if f in kwargs: - del kwargs[f] - super(RTDFacetedSearch, self).__init__(**kwargs) - - def search(self): """ Pass in a user in order to filter search results by privacy. @@ -47,36 +20,48 @@ def search(self): The `self.user` attribute isn't currently used on the .org, but is used on the .com """ - s = super().search() - s = s.source(exclude=['content', 'headers']) - resp = self.signal.send(sender=self, user=self.user, search=s) - if resp: - # Signal return a search object - try: - s = resp[0][1] - except AttributeError: - log.exception( - 'Failed to return a search object from search signals' - ) - # Return 25 results - return s[:25] + self.user = user + self.filter_by_user = kwargs.pop('filter_by_user', None) + super().__init__(**kwargs) def query(self, search, query): """ Add query part to ``search`` when needed. - Also: - - * Adds SimpleQueryString instead of default query. - * Adds HTML encoding of results to avoid XSS issues. + Also does HTML encoding of results to avoid XSS issues. """ + search = super().query(search, query) search = search.highlight_options(encoder='html', number_of_fragments=3) + search = search.source(exclude=['content', 'headers']) + return search + + +class ProjectSearchBase(RTDFacetedSearch): + facets = {'language': TermsFacet(field='language')} + doc_types = [ProjectDocument] + index = ProjectDocument._doc_type.index + fields = ('name^10', 'slug^5', 'description') + + +class PageSearchBase(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version') + } + doc_types = [PageDocument] + index = PageDocument._doc_type.index + fields = ['title^10', 'headers^5', 'content'] + + def query(self, search, query): + """Use a custom SimpleQueryString instead of default query.""" + + search = super().query(search, query) all_queries = [] # need to search for both 'and' and 'or' operations # the score of and should be higher as it satisfies both or and and - for operator in ['and', 'or']: + for operator in ['AND', 'OR']: query_string = SimpleQueryString( query=query, fields=self.fields, default_operator=operator ) @@ -89,50 +74,23 @@ def query(self, search, query): return search -class DomainSearch(RTDFacetedSearch): - facets = { - 'project': TermsFacet(field='project'), - 'version': TermsFacet(field='version'), - 'doc_type': TermsFacet(field='doc_type'), - } - signal = before_domain_search - doc_types = [DomainDocument] - index = DomainDocument._doc_type.index - fields = ('display_name^5', 'name') +class PageSearch(SettingsOverrideObject): + """ + Allow this class to be overridden based on CLASS_OVERRIDES setting. -class ProjectSearch(RTDFacetedSearch): - facets = { - 'language': TermsFacet(field='language') - } - signal = before_project_search - doc_types = [ProjectDocument] - index = ProjectDocument._doc_type.index - fields = ('name^10', 'slug^5', 'description') + This is primary used on the .com to adjust how we filter our search queries + """ + _default_class = PageSearchBase -class PageSearchBase(RTDFacetedSearch): - facets = { - 'project': TermsFacet(field='project'), - 'version': TermsFacet(field='version') - } - doc_types = [PageDocument] - index = PageDocument._doc_type.index - fields = ['title^10', 'headers^5', 'content'] +class ProjectSearch(SettingsOverrideObject): -class AllSearch(RTDFacetedSearch): - facets = { - 'project': TermsFacet(field='project'), - 'version': TermsFacet(field='version'), - 'language': TermsFacet(field='language'), - 'doc_type': TermsFacet(field='doc_type'), - 'index': TermsFacet(field='_index'), - } - signal = before_file_search - doc_types = [DomainDocument, PageDocument, ProjectDocument] - index = [DomainDocument._doc_type.index, - PageDocument._doc_type.index, - ProjectDocument._doc_type.index] - fields = ('title^10', 'headers^5', 'content', 'name^20', - 'slug^5', 'description', 'display_name^5') + """ + Allow this class to be overridden based on CLASS_OVERRIDES setting. + + This is primary used on the .com to adjust how we filter our search queries + """ + + _default_class = ProjectSearchBase diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 3a1bef1fd01..34366d1d042 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -8,11 +8,9 @@ from django.shortcuts import get_object_or_404, render from readthedocs.builds.constants import LATEST -from readthedocs.projects.models import Project -from readthedocs.search.faceted_search import ( - AllSearch, ProjectSearch, PageSearch, DomainSearch, ALL_FACETS -) +from readthedocs.search.documents import PageDocument, ProjectDocument from readthedocs.search.utils import get_project_list_or_404 +from readthedocs.projects.models import Project log = logging.getLogger(__name__) @@ -27,75 +25,42 @@ 'version', 'taxonomy', 'language', - 'doc_type', - 'index', ), ) -def elastic_search(request, project_slug=None): +def elastic_search(request): """Use Elasticsearch for global search.""" user_input = UserInput( query=request.GET.get('q'), - type=request.GET.get('type', 'all'), - project=project_slug or request.GET.get('project'), + type=request.GET.get('type', 'project'), + project=request.GET.get('project'), version=request.GET.get('version', LATEST), taxonomy=request.GET.get('taxonomy'), language=request.GET.get('language'), - doc_type=request.GET.get('doc_type'), - index=request.GET.get('index'), ) results = '' facets = {} if user_input.query: - kwargs = {} - - if user_input.project: - projects_list = get_project_list_or_404( - project_slug=user_input.project, user=request.user - ) - project_slug_list = [project.slug for project in projects_list] - kwargs['project'] = project_slug_list - log.info('Filtering to project list: %s' % project_slug_list) - - if user_input.version: - kwargs['version'] = [user_input.version] - - if user_input.doc_type: - kwargs['doc_type'] = user_input.doc_type - - if user_input.language: - kwargs['language'] = user_input.language - - if user_input.index: - kwargs['index'] = user_input.index - if user_input.type == 'project': - project_search = ProjectSearch( - query=user_input.query, user=request.user, **kwargs - ) - results = project_search.execute() - facets = results.facets - elif user_input.type == 'domain': - project_search = DomainSearch( - query=user_input.query, user=request.user, **kwargs + project_search = ProjectDocument.faceted_search( + query=user_input.query, user=request.user, language=user_input.language ) results = project_search.execute() facets = results.facets elif user_input.type == 'file': + kwargs = {} + if user_input.project: + kwargs['projects_list'] = [user_input.project] + if user_input.version: + kwargs['versions_list'] = [user_input.version] - page_search = PageSearch( + page_search = PageDocument.faceted_search( query=user_input.query, user=request.user, **kwargs ) results = page_search.execute() facets = results.facets - elif user_input.type == 'all': - project_search = AllSearch( - query=user_input.query, user=request.user, **kwargs - ) - results = project_search.execute() - facets = results.facets log.info( LOG_TEMPLATE.format( @@ -108,14 +73,6 @@ def elastic_search(request, project_slug=None): ), ) - # Make sure our selected facets are displayed even when they return 0 results - for avail_facet in ALL_FACETS: - value = getattr(user_input, avail_facet) - if not value or avail_facet not in facets: - continue - if value not in [val[0] for val in facets[avail_facet]]: - facets[avail_facet].insert(0, (value, 0, True)) - if results: if user_input.type == 'file': # Change results to turn newlines in highlight into periods @@ -129,18 +86,52 @@ def elastic_search(request, project_slug=None): log.debug('Search facets: %s', pformat(results.facets.to_dict())) template_vars = user_input._asdict() - template_vars.update({ - 'results': results, - 'facets': facets, - }) - - if project_slug: - queryset = Project.objects.protected(request.user) - project = get_object_or_404(queryset, slug=project_slug) - template_vars.update({'project_obj': project}) - + template_vars.update({'results': results, 'facets': facets}) return render( request, 'search/elastic_search.html', template_vars, ) + + +def elastic_project_search(request, project_slug): + """Use elastic search to search in a project.""" + queryset = Project.objects.protected(request.user) + project = get_object_or_404(queryset, slug=project_slug) + version_slug = request.GET.get('version', LATEST) + query = request.GET.get('q', None) + results = None + + if query: + kwargs = {} + kwargs['projects_list'] = [project.slug] + kwargs['versions_list'] = version_slug + + page_search = PageDocument.faceted_search( + query=query, user=request.user, **kwargs + ) + results = page_search.execute() + + log.debug('Search results: %s', pformat(results.to_dict())) + log.debug('Search facets: %s', pformat(results.facets.to_dict())) + + log.info( + LOG_TEMPLATE.format( + user=request.user, + project=project or '', + type='inproject', + version=version_slug or '', + language='', + msg=query or '', + ), + ) + + return render( + request, + 'search/elastic_project_search.html', + { + 'project': project, + 'query': query, + 'results': results, + }, + ) diff --git a/readthedocs/templates/search/elastic_project_search.html b/readthedocs/templates/search/elastic_project_search.html new file mode 100644 index 00000000000..1ce48e6e2b1 --- /dev/null +++ b/readthedocs/templates/search/elastic_project_search.html @@ -0,0 +1,94 @@ +{% extends "projects/base_project.html" %} + +{% load core_tags i18n %} + +{% block title %}{% blocktrans with query=query|default:"" %}Search: {{ query }} {% endblocktrans %}{% endblock %} + +{% block project_editing %} + {% with search_active="active" %} + {% include "core/project_bar.html" %} + {% endwith %} +{% endblock %} + + +{% block content %} +
    +

    Search in this project:

    + +
    +
    + +
    + +
    +
    + {% comment %}Translators: This is about starting a search{% endcomment %} + +
    + +
    +
    + + + + {% if query %} + +
    +
    + +
    +

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblocktrans %}

    +
    + +
    +
    + +
      + {% for result in results %} +
    • +

      + {{ result.project }} - {{ result.title|safe }} +

      + {% for fragment in result.meta.highlight.content|slice:":3" %} +

      + {{ fragment|safe }} +

      + {% endfor %} +
      • + {% empty %} +
    • {% trans "No results found. Bummer." %}
      • + {% endfor %} + +
    + +
    +
    + +
    +
    + + + {% if page.has_previous or page.has_next %} + + + + {% endif %} + + {% else %} + {# Show some example queries to run, maybe query syntax, something else? #} + {% endif %} + +
    +{% endblock %} diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 6ebe2dc31cd..f46ef4f3a17 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -1,6 +1,6 @@ {% extends "projects/base_project.html" %} -{% load core_tags i18n static %} +{% load core_tags i18n %} {% block title %}{% blocktrans with query=query|default:"" %}Search: {{ query }} {% endblocktrans %}{% endblock %} @@ -13,85 +13,16 @@ {% endblock %} -{% block extra_scripts %} - - -{% endblock %} - -{% block project_editing %} - {% if project_obj %} - {% with search_active="active" project=project_obj %} - {% include "core/project_bar.html" %} - {% endwith %} - {% endif %} -{% endblock %} - {% block content %}
    -

    {% trans 'Search' %}

    @@ -143,7 +91,7 @@

    {% trans 'Search' %}

    - + {% if type %} {% endif %} {% if project %} {% endif %} {% if taxonomy %} {% endif %} @@ -164,6 +112,7 @@

    {% trans 'Search' %}

    +

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblocktrans %}

    @@ -172,53 +121,41 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo
      - {% for result in results %} -
    • -

      - - {% if 'project' in result.meta.index %} - - {{ result.name }} ({{ result.slug }}) - - {% for fragment in result.meta.highlight.description %} -

      - ...{{ fragment|safe }}... -

      - {% empty %} -

      No description for project.

      - {% endfor %} - - {% elif 'domain' in result.meta.index %} - - - {{ result.project }} - {{ result.doc_type }} - {{ result.name }} - - {% for fragment in result.meta.highlight.name %} -

      - ...{{ fragment|safe }}... -

      - {% endfor %} - - {% elif 'page' in result.meta.index %} +
    • +

      + {% if result.name %} - - {{ result.project }} - {{ result.title }} - - {% for fragment in result.meta.highlight.content %} -

      - ...{{ fragment|safe }}... + {# Project #} + {{ result.name }} ({{ result.slug }})

      - {% endfor %} - - {% endif %} - -

      -
      • - - {% empty %} -
    • {% trans "No results found." %}
      • - {% endfor %} + {{ result.meta.description }} + {% for fragment in result.meta.highlight.description|slice:":3" %} +

      + ...{{ fragment|safe }}... +

      +
      + {% empty %} +

      No description for project.

      + {% endfor %} + {# End Project #} + + {% elif result.path %} + + {# File #} + {{ result.project }} - {{ result.title|safe }} + {% for fragment in result.meta.highlight.content|slice:":3" %} +

      + ...{{ fragment|safe }}... +

      + {% endfor %} + {# End File #} + + {% endif %} + + {% empty %} +
    • {% trans "No results found. Bummer." %}
      • + {% endfor %}
    @@ -227,8 +164,31 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo

    - {% endif %} + + {% if page.has_previous or page.has_next %} + + + + {% endif %} + + {% else %} + {# Show some example queries to run, maybe query syntax, something else? #} + {% endif %} + +
    {% endblock %} From 462545ead33043e6ac6099af909b14ffc5778e2b Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 16:01:10 -0300 Subject: [PATCH 13/22] Missed a few --- .../static/search/readthedocs-client.js | 804 ------------------ .../search/static/search/search-embed.js | 18 - 2 files changed, 822 deletions(-) delete mode 100644 readthedocs/search/static/search/readthedocs-client.js delete mode 100644 readthedocs/search/static/search/search-embed.js diff --git a/readthedocs/search/static/search/readthedocs-client.js b/readthedocs/search/static/search/readthedocs-client.js deleted file mode 100644 index 8d5620d2f26..00000000000 --- a/readthedocs/search/static/search/readthedocs-client.js +++ /dev/null @@ -1,804 +0,0 @@ -(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o 0) { - self._completeHandlers.shift()(resp) - } - } - - function success (resp) { - var type = o['type'] || resp && setType(resp.getResponseHeader('Content-Type')) // resp can be undefined in IE - resp = (type !== 'jsonp') ? self.request : resp - // use global data filter on response text - var filteredResponse = globalSetupOptions.dataFilter(resp.responseText, type) - , r = filteredResponse - try { - resp.responseText = r - } catch (e) { - // can't assign this in IE<=8, just ignore - } - if (r) { - switch (type) { - case 'json': - try { - resp = win.JSON ? win.JSON.parse(r) : eval('(' + r + ')') - } catch (err) { - return error(resp, 'Could not parse JSON in response', err) - } - break - case 'js': - resp = eval(r) - break - case 'html': - resp = r - break - case 'xml': - resp = resp.responseXML - && resp.responseXML.parseError // IE trololo - && resp.responseXML.parseError.errorCode - && resp.responseXML.parseError.reason - ? null - : resp.responseXML - break - } - } - - self._responseArgs.resp = resp - self._fulfilled = true - fn(resp) - self._successHandler(resp) - while (self._fulfillmentHandlers.length > 0) { - resp = self._fulfillmentHandlers.shift()(resp) - } - - complete(resp) - } - - function timedOut() { - self._timedOut = true - self.request.abort() - } - - function error(resp, msg, t) { - resp = self.request - self._responseArgs.resp = resp - self._responseArgs.msg = msg - self._responseArgs.t = t - self._erred = true - while (self._errorHandlers.length > 0) { - self._errorHandlers.shift()(resp, msg, t) - } - complete(resp) - } - - this.request = getRequest.call(this, success, error) - } - - Reqwest.prototype = { - abort: function () { - this._aborted = true - this.request.abort() - } - - , retry: function () { - init.call(this, this.o, this.fn) - } - - /** - * Small deviation from the Promises A CommonJs specification - * http://wiki.commonjs.org/wiki/Promises/A - */ - - /** - * `then` will execute upon successful requests - */ - , then: function (success, fail) { - success = success || function () {} - fail = fail || function () {} - if (this._fulfilled) { - this._responseArgs.resp = success(this._responseArgs.resp) - } else if (this._erred) { - fail(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) - } else { - this._fulfillmentHandlers.push(success) - this._errorHandlers.push(fail) - } - return this - } - - /** - * `always` will execute whether the request succeeds or fails - */ - , always: function (fn) { - if (this._fulfilled || this._erred) { - fn(this._responseArgs.resp) - } else { - this._completeHandlers.push(fn) - } - return this - } - - /** - * `fail` will execute when the request fails - */ - , fail: function (fn) { - if (this._erred) { - fn(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) - } else { - this._errorHandlers.push(fn) - } - return this - } - , 'catch': function (fn) { - return this.fail(fn) - } - } - - function reqwest(o, fn) { - return new Reqwest(o, fn) - } - - // normalize newline variants according to spec -> CRLF - function normalize(s) { - return s ? s.replace(/\r?\n/g, '\r\n') : '' - } - - function serial(el, cb) { - var n = el.name - , t = el.tagName.toLowerCase() - , optCb = function (o) { - // IE gives value="" even where there is no value attribute - // 'specified' ref: http://www.w3.org/TR/DOM-Level-3-Core/core.html#ID-862529273 - if (o && !o['disabled']) - cb(n, normalize(o['attributes']['value'] && o['attributes']['value']['specified'] ? o['value'] : o['text'])) - } - , ch, ra, val, i - - // don't serialize elements that are disabled or without a name - if (el.disabled || !n) return - - switch (t) { - case 'input': - if (!/reset|button|image|file/i.test(el.type)) { - ch = /checkbox/i.test(el.type) - ra = /radio/i.test(el.type) - val = el.value - // WebKit gives us "" instead of "on" if a checkbox has no value, so correct it here - ;(!(ch || ra) || el.checked) && cb(n, normalize(ch && val === '' ? 'on' : val)) - } - break - case 'textarea': - cb(n, normalize(el.value)) - break - case 'select': - if (el.type.toLowerCase() === 'select-one') { - optCb(el.selectedIndex >= 0 ? el.options[el.selectedIndex] : null) - } else { - for (i = 0; el.length && i < el.length; i++) { - el.options[i].selected && optCb(el.options[i]) - } - } - break - } - } - - // collect up all form elements found from the passed argument elements all - // the way down to child elements; pass a '' or form fields. - // called with 'this'=callback to use for serial() on each element - function eachFormElement() { - var cb = this - , e, i - , serializeSubtags = function (e, tags) { - var i, j, fa - for (i = 0; i < tags.length; i++) { - fa = e[byTag](tags[i]) - for (j = 0; j < fa.length; j++) serial(fa[j], cb) - } - } - - for (i = 0; i < arguments.length; i++) { - e = arguments[i] - if (/input|select|textarea/i.test(e.tagName)) serial(e, cb) - serializeSubtags(e, [ 'input', 'select', 'textarea' ]) - } - } - - // standard query string style serialization - function serializeQueryString() { - return reqwest.toQueryString(reqwest.serializeArray.apply(null, arguments)) - } - - // { 'name': 'value', ... } style serialization - function serializeHash() { - var hash = {} - eachFormElement.apply(function (name, value) { - if (name in hash) { - hash[name] && !isArray(hash[name]) && (hash[name] = [hash[name]]) - hash[name].push(value) - } else hash[name] = value - }, arguments) - return hash - } - - // [ { name: 'name', value: 'value' }, ... ] style serialization - reqwest.serializeArray = function () { - var arr = [] - eachFormElement.apply(function (name, value) { - arr.push({name: name, value: value}) - }, arguments) - return arr - } - - reqwest.serialize = function () { - if (arguments.length === 0) return '' - var opt, fn - , args = Array.prototype.slice.call(arguments, 0) - - opt = args.pop() - opt && opt.nodeType && args.push(opt) && (opt = null) - opt && (opt = opt.type) - - if (opt == 'map') fn = serializeHash - else if (opt == 'array') fn = reqwest.serializeArray - else fn = serializeQueryString - - return fn.apply(null, args) - } - - reqwest.toQueryString = function (o, trad) { - var prefix, i - , traditional = trad || false - , s = [] - , enc = encodeURIComponent - , add = function (key, value) { - // If value is a function, invoke it and return its value - value = ('function' === typeof value) ? value() : (value == null ? '' : value) - s[s.length] = enc(key) + '=' + enc(value) - } - // If an array was passed in, assume that it is an array of form elements. - if (isArray(o)) { - for (i = 0; o && i < o.length; i++) add(o[i]['name'], o[i]['value']) - } else { - // If traditional, encode the "old" way (the way 1.3.2 or older - // did it), otherwise encode params recursively. - for (prefix in o) { - if (o.hasOwnProperty(prefix)) buildParams(prefix, o[prefix], traditional, add) - } - } - - // spaces should be + according to spec - return s.join('&').replace(/%20/g, '+') - } - - function buildParams(prefix, obj, traditional, add) { - var name, i, v - , rbracket = /\[\]$/ - - if (isArray(obj)) { - // Serialize array item. - for (i = 0; obj && i < obj.length; i++) { - v = obj[i] - if (traditional || rbracket.test(prefix)) { - // Treat each array item as a scalar. - add(prefix, v) - } else { - buildParams(prefix + '[' + (typeof v === 'object' ? i : '') + ']', v, traditional, add) - } - } - } else if (obj && obj.toString() === '[object Object]') { - // Serialize object item. - for (name in obj) { - buildParams(prefix + '[' + name + ']', obj[name], traditional, add) - } - - } else { - // Serialize scalar item. - add(prefix, obj) - } - } - - reqwest.getcallbackPrefix = function () { - return callbackPrefix - } - - // jQuery and Zepto compatibility, differences can be remapped here so you can call - // .ajax.compat(options, callback) - reqwest.compat = function (o, fn) { - if (o) { - o['type'] && (o['method'] = o['type']) && delete o['type'] - o['dataType'] && (o['type'] = o['dataType']) - o['jsonpCallback'] && (o['jsonpCallbackName'] = o['jsonpCallback']) && delete o['jsonpCallback'] - o['jsonp'] && (o['jsonpCallback'] = o['jsonp']) - } - return new Reqwest(o, fn) - } - - reqwest.ajaxSetup = function (options) { - options = options || {} - for (var k in options) { - globalSetupOptions[k] = options[k] - } - } - - return reqwest -}); - -},{}],2:[function(require,module,exports){ -// Document response - -// Page -var Page = function (project, version, doc) { - this.project = project; - this.version = version; - this.doc = doc; - - this.url = null; - this.sections = []; -}; - -Page.prototype.section = function (section) { - return new Section(this.project, this.version, this.doc, section); -}; - -// Section -var Section = function (project, version, doc, section) { - this.project = project; - this.version = version; - this.doc = doc; - this.section = section; - - this.url = null; - this.content = null; - this.wrapped = null; -} - -// Add iframe with returned content to page -Section.prototype.insertContent = function (elem) { - var iframe = document.createElement('iframe'), - self = this; - - iframe.style.display = 'none'; - - if (window.jQuery && elem instanceof window.jQuery) { - elem = elem.get(0); - } - - if (typeof(elem) != 'undefined') { - while (elem.children.length > 0) { - elem.firstChild.remove(); - } - elem.appendChild(iframe); - } - - var win = iframe.contentWindow; - - win.document.open(); - win.document.write(this.content); - win.document.close(); - - var head = win.document.head, - body = win.document.body, - base = null; - - if (head) { - base = win.document.createElement('base'); - base.target = '_parent'; - base.href = this.url; - head.appendChild(base); - - // Copy linked stylesheets from parent - var link_elems = document.head.getElementsByTagName('link'); - for (var n = 0; n < link_elems.length; n++) { - var link = link_elems[n]; - if (link.rel == 'stylesheet') { - head.appendChild(link.cloneNode()); - } - } - } - - win.onload = function () { - iframe.style.display = 'inline-block'; - }; - - return iframe; -}; - - -exports.Section = Section; -exports.Page = Page; - -},{}],3:[function(require,module,exports){ -/* Read the Docs Embed functions */ - -var doc = require('./doc'), - Section = doc.Section, - Page = doc.Page; - - -var Embed = function (config) { - this._api_host = 'https://api.grokthedocs.com'; - if (typeof config == 'object') { - if ('api_host' in config) { - this._api_host = config['api_host']; - } - } -}; - -Embed.prototype.section = function (project, version, doc, section, - callback, error_callback) { - callback = callback || function () {}; - error_callback = error_callback || function () {}; - - var self = this, - data = { - 'project': project, - 'version': version, - 'doc': doc, - 'section': section - }; - - this._getObject( - data, - function (resp) { - var section_ret = new Section(project, version, doc, section); - section_ret.url = resp.url; - section_ret.content = resp.content; - section_ret.wrapped = resp.wrapped; - callback(section_ret); - }, - function (error, msg) { - error_callback(error); - } - ); -}; - -Embed.prototype.page = function (project, version, doc, callback, - error_callback) { - - var self = this, - data = { - 'project': project, - 'version': version, - 'doc': doc, - }; - - this._getObject( - data, - function (resp) { - var page = new Page(project, version, doc); - page.url = resp.url; - // TODO headers is misleading here, rename it on the API - page.sections = resp.headers; - callback(page); - }, - function (error, msg) { - error_callback(error); - } - ) -}; - -Embed.prototype._getObject = function (data, callback, error_callback) { - var self = this, - reqwest = require("./../bower_components/reqwest/reqwest.js"); - callback = callback || function () {}; - error_callback = error_callback || function () {}; - - return reqwest({ - url: this._api_host + '/api/v1/embed/', - method: 'get', - contentType: 'application/json', - crossDomain: true, - headers: {'Accept': 'application/json'}, - data: data, - success: callback, - error: error_callback - }); -}; - -exports.Embed = Embed; - -},{"./../bower_components/reqwest/reqwest.js":1,"./doc":2}],4:[function(require,module,exports){ -/* Read the Docs Client */ - -var embed = require('./embed'); - - -exports.Embed = embed.Embed; - -if (typeof window != 'undefined') { - window.Embed = embed.Embed; -} - -},{"./embed":3}]},{},[4]) \ No newline at end of file diff --git a/readthedocs/search/static/search/search-embed.js b/readthedocs/search/static/search/search-embed.js deleted file mode 100644 index 88440ad8254..00000000000 --- a/readthedocs/search/static/search/search-embed.js +++ /dev/null @@ -1,18 +0,0 @@ - - $(document).ready(function () { - var config = {api_host: 'http://localhost:8000'} - var embed = new Embed(config); - $('.fragment').one("click mouseover", - function(elem) { - embed.section( - elem.currentTarget.getAttribute('project'), - elem.currentTarget.getAttribute('version'), - elem.currentTarget.getAttribute('doc_name').replace('.html', ''), - elem.currentTarget.getAttribute('anchor'), - function (section) { - section.insertContent(elem.currentTarget); - } - ) - } - ) - }) From f53e5897f6a77d9f0c62ed5677f414cde41e79c5 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 16:03:01 -0300 Subject: [PATCH 14/22] Missed a bit more --- readthedocs/settings/base.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index c009cbb62bd..75fcc3c3ed1 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -84,7 +84,6 @@ def INSTALLED_APPS(self): # noqa 'django_extensions', 'messages_extends', 'tastypie', - 'django_filters', 'django_elasticsearch_dsl', # our apps @@ -325,9 +324,8 @@ def USE_PROMOS(self): # noqa # CORS CORS_ORIGIN_REGEX_WHITELIST = ( - '(.*)localhost(.*)', - '^http://(.+)\.readthedocs\.io$', - '^https://(.+)\.readthedocs\.io$' + r'^http://(.+)\.readthedocs\.io$', + r'^https://(.+)\.readthedocs\.io$', ) # So people can post to their accounts CORS_ALLOW_CREDENTIALS = True @@ -358,12 +356,6 @@ def USE_PROMOS(self): # noqa ES_TASK_CHUNK_SIZE = 100 ES_INDEXES = { - 'domain': { - 'name': 'domain_index', - 'settings': {'number_of_shards': 2, - 'number_of_replicas': 0 - } - }, 'project': { 'name': 'project_index', 'settings': {'number_of_shards': 2, From 98aae4645d9ba6c5b66b5db8960146c8d6198659 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 16:04:22 -0300 Subject: [PATCH 15/22] remove random bits that got merged in --- readthedocs/projects/tasks.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index d3a9bd0755c..bd409396bcd 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -379,19 +379,16 @@ def run( # Catch unhandled errors in the setup step except Exception as e: # noqa - extra = { - 'stack': True, - 'tags': { - 'build': build_pk, - }, - } - if self.project: - extra['tags']['project'] = self.project.slug - if self.version: - extra['tags']['version'] = self.version.slug log.exception( 'An unhandled exception was raised during build setup', - extra=extra + extra={ + 'stack': True, + 'tags': { + 'build': build_pk, + 'project': self.project.slug, + 'version': self.version.slug, + }, + }, ) if self.setup_env is not None: self.setup_env.failure = BuildEnvironmentError( From f2e355faccc2a96a8951d34292b2881a224b7cbc Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 16:13:03 -0300 Subject: [PATCH 16/22] Re-add Domain search --- readthedocs/search/documents.py | 47 +- readthedocs/search/faceted_search.py | 132 ++- .../static/search/readthedocs-client.js | 804 ++++++++++++++++++ .../search/static/search/search-embed.js | 18 + readthedocs/search/views.py | 123 +-- readthedocs/settings/base.py | 12 +- .../search/elastic_project_search.html | 94 -- .../templates/search/elastic_search.html | 252 +++--- 8 files changed, 1176 insertions(+), 306 deletions(-) create mode 100644 readthedocs/search/static/search/readthedocs-client.js create mode 100644 readthedocs/search/static/search/search-embed.js delete mode 100644 readthedocs/templates/search/elastic_project_search.html diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index 0fcf0d69a3f..6b54d8c3f5a 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -4,8 +4,8 @@ from django.conf import settings from django_elasticsearch_dsl import DocType, Index, fields -from readthedocs.projects.models import HTMLFile, Project - +from readthedocs.projects.models import Project, HTMLFile +from readthedocs.domaindata.models import DomainData project_conf = settings.ES_INDEXES['project'] project_index = Index(project_conf['name']) @@ -15,9 +15,52 @@ page_index = Index(page_conf['name']) page_index.settings(**page_conf['settings']) +domain_conf = settings.ES_INDEXES['domain'] +domain_index = Index(domain_conf['name']) +domain_index.settings(**domain_conf['settings']) + log = logging.getLogger(__name__) +@domain_index.doc_type +class DomainDocument(DocType): + project = fields.KeywordField(attr='project.slug') + version = fields.KeywordField(attr='version.slug') + doc_type = fields.KeywordField(attr='doc_type') + anchor = fields.KeywordField(attr='anchor') + + class Meta(object): + model = DomainData + fields = ('name', 'display_name', 'doc_name') + ignore_signals = True + + @classmethod + def faceted_search(cls, query, user, doc_type=None): + from readthedocs.search.faceted_search import DomainSearch + kwargs = { + 'user': user, + 'query': query, + } + + if doc_type: + kwargs['filters'] = {'doc_type': doc_type} + + return DomainSearch(**kwargs) + + def get_queryset(self): + """Overwrite default queryset to filter certain files to index""" + queryset = super().get_queryset() + + # Exclude some types to not index + excluded_types = ['std:doc', 'std:label'] + + # Do not index files that belong to non sphinx project + # Also do not index certain files + for exclude in excluded_types: + queryset = queryset.exclude(type=exclude) + return queryset + + @project_index.doc_type class ProjectDocument(DocType): diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 627d78ffbe7..3a0c9600d37 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -1,17 +1,44 @@ +# -*- coding: utf-8 -*- import logging from elasticsearch_dsl import FacetedSearch, TermsFacet from elasticsearch_dsl.query import Bool, SimpleQueryString +from readthedocs.search.documents import ( + DomainDocument, + PageDocument, + ProjectDocument, +) +from readthedocs.search.signals import ( + before_domain_search, + before_file_search, + before_project_search, +) + from readthedocs.core.utils.extend import SettingsOverrideObject -from readthedocs.search.documents import PageDocument, ProjectDocument log = logging.getLogger(__name__) +ALL_FACETS = ['project', 'version', 'doc_type', 'language', 'index'] + + class RTDFacetedSearch(FacetedSearch): def __init__(self, user, **kwargs): + self.user = user + self.filter_by_user = kwargs.pop('filter_by_user', None) + for facet in self.facets: + if facet in kwargs: + kwargs.setdefault('filters', {})[facet] = kwargs.pop(facet) + + # Don't pass along unnecessary filters + for f in ALL_FACETS: + if f in kwargs: + del kwargs[f] + super(RTDFacetedSearch, self).__init__(**kwargs) + + def search(self): """ Pass in a user in order to filter search results by privacy. @@ -20,48 +47,36 @@ def __init__(self, user, **kwargs): The `self.user` attribute isn't currently used on the .org, but is used on the .com """ - self.user = user - self.filter_by_user = kwargs.pop('filter_by_user', None) - super().__init__(**kwargs) + s = super().search() + s = s.source(exclude=['content', 'headers']) + resp = self.signal.send(sender=self, user=self.user, search=s) + if resp: + # Signal return a search object + try: + s = resp[0][1] + except AttributeError: + log.exception( + 'Failed to return a search object from search signals' + ) + # Return 25 results + return s[:25] def query(self, search, query): """ Add query part to ``search`` when needed. - Also does HTML encoding of results to avoid XSS issues. + Also: + + * Adds SimpleQueryString instead of default query. + * Adds HTML encoding of results to avoid XSS issues. """ - search = super().query(search, query) search = search.highlight_options(encoder='html', number_of_fragments=3) - search = search.source(exclude=['content', 'headers']) - return search - - -class ProjectSearchBase(RTDFacetedSearch): - facets = {'language': TermsFacet(field='language')} - doc_types = [ProjectDocument] - index = ProjectDocument._doc_type.index - fields = ('name^10', 'slug^5', 'description') - - -class PageSearchBase(RTDFacetedSearch): - facets = { - 'project': TermsFacet(field='project'), - 'version': TermsFacet(field='version') - } - doc_types = [PageDocument] - index = PageDocument._doc_type.index - fields = ['title^10', 'headers^5', 'content'] - - def query(self, search, query): - """Use a custom SimpleQueryString instead of default query.""" - - search = super().query(search, query) all_queries = [] # need to search for both 'and' and 'or' operations # the score of and should be higher as it satisfies both or and and - for operator in ['AND', 'OR']: + for operator in ['and', 'or']: query_string = SimpleQueryString( query=query, fields=self.fields, default_operator=operator ) @@ -74,23 +89,50 @@ def query(self, search, query): return search -class PageSearch(SettingsOverrideObject): - - """ - Allow this class to be overridden based on CLASS_OVERRIDES setting. - - This is primary used on the .com to adjust how we filter our search queries - """ +class DomainSearch(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version'), + 'doc_type': TermsFacet(field='doc_type'), + } + signal = before_domain_search + doc_types = [DomainDocument] + index = DomainDocument._doc_type.index + fields = ('display_name^5', 'name') - _default_class = PageSearchBase +class ProjectSearch(RTDFacetedSearch): + facets = { + 'language': TermsFacet(field='language') + } + signal = before_project_search + doc_types = [ProjectDocument] + index = ProjectDocument._doc_type.index + fields = ('name^10', 'slug^5', 'description') -class ProjectSearch(SettingsOverrideObject): - """ - Allow this class to be overridden based on CLASS_OVERRIDES setting. +class PageSearchBase(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version') + } + doc_types = [PageDocument] + index = PageDocument._doc_type.index + fields = ['title^10', 'headers^5', 'content'] - This is primary used on the .com to adjust how we filter our search queries - """ - _default_class = ProjectSearchBase +class AllSearch(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version'), + 'language': TermsFacet(field='language'), + 'doc_type': TermsFacet(field='doc_type'), + 'index': TermsFacet(field='_index'), + } + signal = before_file_search + doc_types = [DomainDocument, PageDocument, ProjectDocument] + index = [DomainDocument._doc_type.index, + PageDocument._doc_type.index, + ProjectDocument._doc_type.index] + fields = ('title^10', 'headers^5', 'content', 'name^20', + 'slug^5', 'description', 'display_name^5') diff --git a/readthedocs/search/static/search/readthedocs-client.js b/readthedocs/search/static/search/readthedocs-client.js new file mode 100644 index 00000000000..8d5620d2f26 --- /dev/null +++ b/readthedocs/search/static/search/readthedocs-client.js @@ -0,0 +1,804 @@ +(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o 0) { + self._completeHandlers.shift()(resp) + } + } + + function success (resp) { + var type = o['type'] || resp && setType(resp.getResponseHeader('Content-Type')) // resp can be undefined in IE + resp = (type !== 'jsonp') ? self.request : resp + // use global data filter on response text + var filteredResponse = globalSetupOptions.dataFilter(resp.responseText, type) + , r = filteredResponse + try { + resp.responseText = r + } catch (e) { + // can't assign this in IE<=8, just ignore + } + if (r) { + switch (type) { + case 'json': + try { + resp = win.JSON ? win.JSON.parse(r) : eval('(' + r + ')') + } catch (err) { + return error(resp, 'Could not parse JSON in response', err) + } + break + case 'js': + resp = eval(r) + break + case 'html': + resp = r + break + case 'xml': + resp = resp.responseXML + && resp.responseXML.parseError // IE trololo + && resp.responseXML.parseError.errorCode + && resp.responseXML.parseError.reason + ? null + : resp.responseXML + break + } + } + + self._responseArgs.resp = resp + self._fulfilled = true + fn(resp) + self._successHandler(resp) + while (self._fulfillmentHandlers.length > 0) { + resp = self._fulfillmentHandlers.shift()(resp) + } + + complete(resp) + } + + function timedOut() { + self._timedOut = true + self.request.abort() + } + + function error(resp, msg, t) { + resp = self.request + self._responseArgs.resp = resp + self._responseArgs.msg = msg + self._responseArgs.t = t + self._erred = true + while (self._errorHandlers.length > 0) { + self._errorHandlers.shift()(resp, msg, t) + } + complete(resp) + } + + this.request = getRequest.call(this, success, error) + } + + Reqwest.prototype = { + abort: function () { + this._aborted = true + this.request.abort() + } + + , retry: function () { + init.call(this, this.o, this.fn) + } + + /** + * Small deviation from the Promises A CommonJs specification + * http://wiki.commonjs.org/wiki/Promises/A + */ + + /** + * `then` will execute upon successful requests + */ + , then: function (success, fail) { + success = success || function () {} + fail = fail || function () {} + if (this._fulfilled) { + this._responseArgs.resp = success(this._responseArgs.resp) + } else if (this._erred) { + fail(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) + } else { + this._fulfillmentHandlers.push(success) + this._errorHandlers.push(fail) + } + return this + } + + /** + * `always` will execute whether the request succeeds or fails + */ + , always: function (fn) { + if (this._fulfilled || this._erred) { + fn(this._responseArgs.resp) + } else { + this._completeHandlers.push(fn) + } + return this + } + + /** + * `fail` will execute when the request fails + */ + , fail: function (fn) { + if (this._erred) { + fn(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) + } else { + this._errorHandlers.push(fn) + } + return this + } + , 'catch': function (fn) { + return this.fail(fn) + } + } + + function reqwest(o, fn) { + return new Reqwest(o, fn) + } + + // normalize newline variants according to spec -> CRLF + function normalize(s) { + return s ? s.replace(/\r?\n/g, '\r\n') : '' + } + + function serial(el, cb) { + var n = el.name + , t = el.tagName.toLowerCase() + , optCb = function (o) { + // IE gives value="" even where there is no value attribute + // 'specified' ref: http://www.w3.org/TR/DOM-Level-3-Core/core.html#ID-862529273 + if (o && !o['disabled']) + cb(n, normalize(o['attributes']['value'] && o['attributes']['value']['specified'] ? o['value'] : o['text'])) + } + , ch, ra, val, i + + // don't serialize elements that are disabled or without a name + if (el.disabled || !n) return + + switch (t) { + case 'input': + if (!/reset|button|image|file/i.test(el.type)) { + ch = /checkbox/i.test(el.type) + ra = /radio/i.test(el.type) + val = el.value + // WebKit gives us "" instead of "on" if a checkbox has no value, so correct it here + ;(!(ch || ra) || el.checked) && cb(n, normalize(ch && val === '' ? 'on' : val)) + } + break + case 'textarea': + cb(n, normalize(el.value)) + break + case 'select': + if (el.type.toLowerCase() === 'select-one') { + optCb(el.selectedIndex >= 0 ? el.options[el.selectedIndex] : null) + } else { + for (i = 0; el.length && i < el.length; i++) { + el.options[i].selected && optCb(el.options[i]) + } + } + break + } + } + + // collect up all form elements found from the passed argument elements all + // the way down to child elements; pass a '' or form fields. + // called with 'this'=callback to use for serial() on each element + function eachFormElement() { + var cb = this + , e, i + , serializeSubtags = function (e, tags) { + var i, j, fa + for (i = 0; i < tags.length; i++) { + fa = e[byTag](tags[i]) + for (j = 0; j < fa.length; j++) serial(fa[j], cb) + } + } + + for (i = 0; i < arguments.length; i++) { + e = arguments[i] + if (/input|select|textarea/i.test(e.tagName)) serial(e, cb) + serializeSubtags(e, [ 'input', 'select', 'textarea' ]) + } + } + + // standard query string style serialization + function serializeQueryString() { + return reqwest.toQueryString(reqwest.serializeArray.apply(null, arguments)) + } + + // { 'name': 'value', ... } style serialization + function serializeHash() { + var hash = {} + eachFormElement.apply(function (name, value) { + if (name in hash) { + hash[name] && !isArray(hash[name]) && (hash[name] = [hash[name]]) + hash[name].push(value) + } else hash[name] = value + }, arguments) + return hash + } + + // [ { name: 'name', value: 'value' }, ... ] style serialization + reqwest.serializeArray = function () { + var arr = [] + eachFormElement.apply(function (name, value) { + arr.push({name: name, value: value}) + }, arguments) + return arr + } + + reqwest.serialize = function () { + if (arguments.length === 0) return '' + var opt, fn + , args = Array.prototype.slice.call(arguments, 0) + + opt = args.pop() + opt && opt.nodeType && args.push(opt) && (opt = null) + opt && (opt = opt.type) + + if (opt == 'map') fn = serializeHash + else if (opt == 'array') fn = reqwest.serializeArray + else fn = serializeQueryString + + return fn.apply(null, args) + } + + reqwest.toQueryString = function (o, trad) { + var prefix, i + , traditional = trad || false + , s = [] + , enc = encodeURIComponent + , add = function (key, value) { + // If value is a function, invoke it and return its value + value = ('function' === typeof value) ? value() : (value == null ? '' : value) + s[s.length] = enc(key) + '=' + enc(value) + } + // If an array was passed in, assume that it is an array of form elements. + if (isArray(o)) { + for (i = 0; o && i < o.length; i++) add(o[i]['name'], o[i]['value']) + } else { + // If traditional, encode the "old" way (the way 1.3.2 or older + // did it), otherwise encode params recursively. + for (prefix in o) { + if (o.hasOwnProperty(prefix)) buildParams(prefix, o[prefix], traditional, add) + } + } + + // spaces should be + according to spec + return s.join('&').replace(/%20/g, '+') + } + + function buildParams(prefix, obj, traditional, add) { + var name, i, v + , rbracket = /\[\]$/ + + if (isArray(obj)) { + // Serialize array item. + for (i = 0; obj && i < obj.length; i++) { + v = obj[i] + if (traditional || rbracket.test(prefix)) { + // Treat each array item as a scalar. + add(prefix, v) + } else { + buildParams(prefix + '[' + (typeof v === 'object' ? i : '') + ']', v, traditional, add) + } + } + } else if (obj && obj.toString() === '[object Object]') { + // Serialize object item. + for (name in obj) { + buildParams(prefix + '[' + name + ']', obj[name], traditional, add) + } + + } else { + // Serialize scalar item. + add(prefix, obj) + } + } + + reqwest.getcallbackPrefix = function () { + return callbackPrefix + } + + // jQuery and Zepto compatibility, differences can be remapped here so you can call + // .ajax.compat(options, callback) + reqwest.compat = function (o, fn) { + if (o) { + o['type'] && (o['method'] = o['type']) && delete o['type'] + o['dataType'] && (o['type'] = o['dataType']) + o['jsonpCallback'] && (o['jsonpCallbackName'] = o['jsonpCallback']) && delete o['jsonpCallback'] + o['jsonp'] && (o['jsonpCallback'] = o['jsonp']) + } + return new Reqwest(o, fn) + } + + reqwest.ajaxSetup = function (options) { + options = options || {} + for (var k in options) { + globalSetupOptions[k] = options[k] + } + } + + return reqwest +}); + +},{}],2:[function(require,module,exports){ +// Document response + +// Page +var Page = function (project, version, doc) { + this.project = project; + this.version = version; + this.doc = doc; + + this.url = null; + this.sections = []; +}; + +Page.prototype.section = function (section) { + return new Section(this.project, this.version, this.doc, section); +}; + +// Section +var Section = function (project, version, doc, section) { + this.project = project; + this.version = version; + this.doc = doc; + this.section = section; + + this.url = null; + this.content = null; + this.wrapped = null; +} + +// Add iframe with returned content to page +Section.prototype.insertContent = function (elem) { + var iframe = document.createElement('iframe'), + self = this; + + iframe.style.display = 'none'; + + if (window.jQuery && elem instanceof window.jQuery) { + elem = elem.get(0); + } + + if (typeof(elem) != 'undefined') { + while (elem.children.length > 0) { + elem.firstChild.remove(); + } + elem.appendChild(iframe); + } + + var win = iframe.contentWindow; + + win.document.open(); + win.document.write(this.content); + win.document.close(); + + var head = win.document.head, + body = win.document.body, + base = null; + + if (head) { + base = win.document.createElement('base'); + base.target = '_parent'; + base.href = this.url; + head.appendChild(base); + + // Copy linked stylesheets from parent + var link_elems = document.head.getElementsByTagName('link'); + for (var n = 0; n < link_elems.length; n++) { + var link = link_elems[n]; + if (link.rel == 'stylesheet') { + head.appendChild(link.cloneNode()); + } + } + } + + win.onload = function () { + iframe.style.display = 'inline-block'; + }; + + return iframe; +}; + + +exports.Section = Section; +exports.Page = Page; + +},{}],3:[function(require,module,exports){ +/* Read the Docs Embed functions */ + +var doc = require('./doc'), + Section = doc.Section, + Page = doc.Page; + + +var Embed = function (config) { + this._api_host = 'https://api.grokthedocs.com'; + if (typeof config == 'object') { + if ('api_host' in config) { + this._api_host = config['api_host']; + } + } +}; + +Embed.prototype.section = function (project, version, doc, section, + callback, error_callback) { + callback = callback || function () {}; + error_callback = error_callback || function () {}; + + var self = this, + data = { + 'project': project, + 'version': version, + 'doc': doc, + 'section': section + }; + + this._getObject( + data, + function (resp) { + var section_ret = new Section(project, version, doc, section); + section_ret.url = resp.url; + section_ret.content = resp.content; + section_ret.wrapped = resp.wrapped; + callback(section_ret); + }, + function (error, msg) { + error_callback(error); + } + ); +}; + +Embed.prototype.page = function (project, version, doc, callback, + error_callback) { + + var self = this, + data = { + 'project': project, + 'version': version, + 'doc': doc, + }; + + this._getObject( + data, + function (resp) { + var page = new Page(project, version, doc); + page.url = resp.url; + // TODO headers is misleading here, rename it on the API + page.sections = resp.headers; + callback(page); + }, + function (error, msg) { + error_callback(error); + } + ) +}; + +Embed.prototype._getObject = function (data, callback, error_callback) { + var self = this, + reqwest = require("./../bower_components/reqwest/reqwest.js"); + callback = callback || function () {}; + error_callback = error_callback || function () {}; + + return reqwest({ + url: this._api_host + '/api/v1/embed/', + method: 'get', + contentType: 'application/json', + crossDomain: true, + headers: {'Accept': 'application/json'}, + data: data, + success: callback, + error: error_callback + }); +}; + +exports.Embed = Embed; + +},{"./../bower_components/reqwest/reqwest.js":1,"./doc":2}],4:[function(require,module,exports){ +/* Read the Docs Client */ + +var embed = require('./embed'); + + +exports.Embed = embed.Embed; + +if (typeof window != 'undefined') { + window.Embed = embed.Embed; +} + +},{"./embed":3}]},{},[4]) \ No newline at end of file diff --git a/readthedocs/search/static/search/search-embed.js b/readthedocs/search/static/search/search-embed.js new file mode 100644 index 00000000000..88440ad8254 --- /dev/null +++ b/readthedocs/search/static/search/search-embed.js @@ -0,0 +1,18 @@ + + $(document).ready(function () { + var config = {api_host: 'http://localhost:8000'} + var embed = new Embed(config); + $('.fragment').one("click mouseover", + function(elem) { + embed.section( + elem.currentTarget.getAttribute('project'), + elem.currentTarget.getAttribute('version'), + elem.currentTarget.getAttribute('doc_name').replace('.html', ''), + elem.currentTarget.getAttribute('anchor'), + function (section) { + section.insertContent(elem.currentTarget); + } + ) + } + ) + }) diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 34366d1d042..3a1bef1fd01 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -8,9 +8,11 @@ from django.shortcuts import get_object_or_404, render from readthedocs.builds.constants import LATEST -from readthedocs.search.documents import PageDocument, ProjectDocument -from readthedocs.search.utils import get_project_list_or_404 from readthedocs.projects.models import Project +from readthedocs.search.faceted_search import ( + AllSearch, ProjectSearch, PageSearch, DomainSearch, ALL_FACETS +) +from readthedocs.search.utils import get_project_list_or_404 log = logging.getLogger(__name__) @@ -25,42 +27,75 @@ 'version', 'taxonomy', 'language', + 'doc_type', + 'index', ), ) -def elastic_search(request): +def elastic_search(request, project_slug=None): """Use Elasticsearch for global search.""" user_input = UserInput( query=request.GET.get('q'), - type=request.GET.get('type', 'project'), - project=request.GET.get('project'), + type=request.GET.get('type', 'all'), + project=project_slug or request.GET.get('project'), version=request.GET.get('version', LATEST), taxonomy=request.GET.get('taxonomy'), language=request.GET.get('language'), + doc_type=request.GET.get('doc_type'), + index=request.GET.get('index'), ) results = '' facets = {} if user_input.query: + kwargs = {} + + if user_input.project: + projects_list = get_project_list_or_404( + project_slug=user_input.project, user=request.user + ) + project_slug_list = [project.slug for project in projects_list] + kwargs['project'] = project_slug_list + log.info('Filtering to project list: %s' % project_slug_list) + + if user_input.version: + kwargs['version'] = [user_input.version] + + if user_input.doc_type: + kwargs['doc_type'] = user_input.doc_type + + if user_input.language: + kwargs['language'] = user_input.language + + if user_input.index: + kwargs['index'] = user_input.index + if user_input.type == 'project': - project_search = ProjectDocument.faceted_search( - query=user_input.query, user=request.user, language=user_input.language + project_search = ProjectSearch( + query=user_input.query, user=request.user, **kwargs + ) + results = project_search.execute() + facets = results.facets + elif user_input.type == 'domain': + project_search = DomainSearch( + query=user_input.query, user=request.user, **kwargs ) results = project_search.execute() facets = results.facets elif user_input.type == 'file': - kwargs = {} - if user_input.project: - kwargs['projects_list'] = [user_input.project] - if user_input.version: - kwargs['versions_list'] = [user_input.version] - page_search = PageDocument.faceted_search( + page_search = PageSearch( query=user_input.query, user=request.user, **kwargs ) results = page_search.execute() facets = results.facets + elif user_input.type == 'all': + project_search = AllSearch( + query=user_input.query, user=request.user, **kwargs + ) + results = project_search.execute() + facets = results.facets log.info( LOG_TEMPLATE.format( @@ -73,6 +108,14 @@ def elastic_search(request): ), ) + # Make sure our selected facets are displayed even when they return 0 results + for avail_facet in ALL_FACETS: + value = getattr(user_input, avail_facet) + if not value or avail_facet not in facets: + continue + if value not in [val[0] for val in facets[avail_facet]]: + facets[avail_facet].insert(0, (value, 0, True)) + if results: if user_input.type == 'file': # Change results to turn newlines in highlight into periods @@ -86,52 +129,18 @@ def elastic_search(request): log.debug('Search facets: %s', pformat(results.facets.to_dict())) template_vars = user_input._asdict() - template_vars.update({'results': results, 'facets': facets}) - return render( - request, - 'search/elastic_search.html', - template_vars, - ) - + template_vars.update({ + 'results': results, + 'facets': facets, + }) -def elastic_project_search(request, project_slug): - """Use elastic search to search in a project.""" - queryset = Project.objects.protected(request.user) - project = get_object_or_404(queryset, slug=project_slug) - version_slug = request.GET.get('version', LATEST) - query = request.GET.get('q', None) - results = None - - if query: - kwargs = {} - kwargs['projects_list'] = [project.slug] - kwargs['versions_list'] = version_slug - - page_search = PageDocument.faceted_search( - query=query, user=request.user, **kwargs - ) - results = page_search.execute() - - log.debug('Search results: %s', pformat(results.to_dict())) - log.debug('Search facets: %s', pformat(results.facets.to_dict())) - - log.info( - LOG_TEMPLATE.format( - user=request.user, - project=project or '', - type='inproject', - version=version_slug or '', - language='', - msg=query or '', - ), - ) + if project_slug: + queryset = Project.objects.protected(request.user) + project = get_object_or_404(queryset, slug=project_slug) + template_vars.update({'project_obj': project}) return render( request, - 'search/elastic_project_search.html', - { - 'project': project, - 'query': query, - 'results': results, - }, + 'search/elastic_search.html', + template_vars, ) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 75fcc3c3ed1..c009cbb62bd 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -84,6 +84,7 @@ def INSTALLED_APPS(self): # noqa 'django_extensions', 'messages_extends', 'tastypie', + 'django_filters', 'django_elasticsearch_dsl', # our apps @@ -324,8 +325,9 @@ def USE_PROMOS(self): # noqa # CORS CORS_ORIGIN_REGEX_WHITELIST = ( - r'^http://(.+)\.readthedocs\.io$', - r'^https://(.+)\.readthedocs\.io$', + '(.*)localhost(.*)', + '^http://(.+)\.readthedocs\.io$', + '^https://(.+)\.readthedocs\.io$' ) # So people can post to their accounts CORS_ALLOW_CREDENTIALS = True @@ -356,6 +358,12 @@ def USE_PROMOS(self): # noqa ES_TASK_CHUNK_SIZE = 100 ES_INDEXES = { + 'domain': { + 'name': 'domain_index', + 'settings': {'number_of_shards': 2, + 'number_of_replicas': 0 + } + }, 'project': { 'name': 'project_index', 'settings': {'number_of_shards': 2, diff --git a/readthedocs/templates/search/elastic_project_search.html b/readthedocs/templates/search/elastic_project_search.html deleted file mode 100644 index 1ce48e6e2b1..00000000000 --- a/readthedocs/templates/search/elastic_project_search.html +++ /dev/null @@ -1,94 +0,0 @@ -{% extends "projects/base_project.html" %} - -{% load core_tags i18n %} - -{% block title %}{% blocktrans with query=query|default:"" %}Search: {{ query }} {% endblocktrans %}{% endblock %} - -{% block project_editing %} - {% with search_active="active" %} - {% include "core/project_bar.html" %} - {% endwith %} -{% endblock %} - - -{% block content %} -
    -

    Search in this project:

    - -
    -
    - -
    - -
    -
    - {% comment %}Translators: This is about starting a search{% endcomment %} - -
    - -
    -
    - - - - {% if query %} - -
    -
    - -
    -

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblocktrans %}

    -
    - -
    -
    - -
      - {% for result in results %} -
    • -

      - {{ result.project }} - {{ result.title|safe }} -

      - {% for fragment in result.meta.highlight.content|slice:":3" %} -

      - {{ fragment|safe }} -

      - {% endfor %} -
      • - {% empty %} -
    • {% trans "No results found. Bummer." %}
      • - {% endfor %} - -
    - -
    -
    - -
    -
    - - - {% if page.has_previous or page.has_next %} - - - - {% endif %} - - {% else %} - {# Show some example queries to run, maybe query syntax, something else? #} - {% endif %} - -
    -{% endblock %} diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index f46ef4f3a17..6ebe2dc31cd 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -1,6 +1,6 @@ {% extends "projects/base_project.html" %} -{% load core_tags i18n %} +{% load core_tags i18n static %} {% block title %}{% blocktrans with query=query|default:"" %}Search: {{ query }} {% endblocktrans %}{% endblock %} @@ -13,16 +13,85 @@ {% endblock %} +{% block extra_scripts %} + + +{% endblock %} + +{% block project_editing %} + {% if project_obj %} + {% with search_active="active" project=project_obj %} + {% include "core/project_bar.html" %} + {% endwith %} + {% endif %} +{% endblock %} + {% block content %}
    -

    {% trans 'Search' %}

    @@ -91,7 +143,7 @@

    {% trans 'Search' %}

    - + {% if type %} {% endif %} {% if project %} {% endif %} {% if taxonomy %} {% endif %} @@ -112,7 +164,6 @@

    {% trans 'Search' %}

    -

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblocktrans %}

    @@ -121,41 +172,53 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo
      + {% for result in results %} -
    • -

      - {% if result.name %} +

    • +

      + + {% if 'project' in result.meta.index %} + + {{ result.name }} ({{ result.slug }}) + + {% for fragment in result.meta.highlight.description %} +

      + ...{{ fragment|safe }}... +

      + {% empty %} +

      No description for project.

      + {% endfor %} + + {% elif 'domain' in result.meta.index %} + + + {{ result.project }} - {{ result.doc_type }} - {{ result.name }} + + {% for fragment in result.meta.highlight.name %} +

      + ...{{ fragment|safe }}... +

      + {% endfor %} + + {% elif 'page' in result.meta.index %} - {# Project #} - {{ result.name }} ({{ result.slug }}) + + {{ result.project }} - {{ result.title }} + + {% for fragment in result.meta.highlight.content %} +

      + ...{{ fragment|safe }}...

      - {{ result.meta.description }} - {% for fragment in result.meta.highlight.description|slice:":3" %} -

      - ...{{ fragment|safe }}... -

      -
      - {% empty %} -

      No description for project.

      - {% endfor %} - {# End Project #} - - {% elif result.path %} - - {# File #} - {{ result.project }} - {{ result.title|safe }} - {% for fragment in result.meta.highlight.content|slice:":3" %} -

      - ...{{ fragment|safe }}... -

      - {% endfor %} - {# End File #} - - {% endif %} -
      • - {% empty %} -
    • {% trans "No results found. Bummer." %}
      • - {% endfor %} + {% endfor %} + + {% endif %} + +

      + + + {% empty %} +
    • {% trans "No results found." %}
      • + {% endfor %}
    @@ -164,31 +227,8 @@

    {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo

    - - - {% if page.has_previous or page.has_next %} - - - - {% endif %} - - {% else %} - {# Show some example queries to run, maybe query syntax, something else? #} {% endif %} - - +
    {% endblock %} From 3b393b31b344d36c8897fa70d1aceaebefdd332d Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 16:21:28 -0300 Subject: [PATCH 17/22] Updates to faceted search --- readthedocs/search/faceted_search.py | 86 ++++++++++++++-------------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 3a0c9600d37..03e5b9fdbd1 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -9,20 +9,11 @@ PageDocument, ProjectDocument, ) -from readthedocs.search.signals import ( - before_domain_search, - before_file_search, - before_project_search, -) - from readthedocs.core.utils.extend import SettingsOverrideObject log = logging.getLogger(__name__) -ALL_FACETS = ['project', 'version', 'doc_type', 'language', 'index'] - - class RTDFacetedSearch(FacetedSearch): def __init__(self, user, **kwargs): @@ -31,12 +22,7 @@ def __init__(self, user, **kwargs): for facet in self.facets: if facet in kwargs: kwargs.setdefault('filters', {})[facet] = kwargs.pop(facet) - - # Don't pass along unnecessary filters - for f in ALL_FACETS: - if f in kwargs: - del kwargs[f] - super(RTDFacetedSearch, self).__init__(**kwargs) + super().__init__(**kwargs) def search(self): """ @@ -49,15 +35,6 @@ def search(self): """ s = super().search() s = s.source(exclude=['content', 'headers']) - resp = self.signal.send(sender=self, user=self.user, search=s) - if resp: - # Signal return a search object - try: - s = resp[0][1] - except AttributeError: - log.exception( - 'Failed to return a search object from search signals' - ) # Return 25 results return s[:25] @@ -89,23 +66,8 @@ def query(self, search, query): return search -class DomainSearch(RTDFacetedSearch): - facets = { - 'project': TermsFacet(field='project'), - 'version': TermsFacet(field='version'), - 'doc_type': TermsFacet(field='doc_type'), - } - signal = before_domain_search - doc_types = [DomainDocument] - index = DomainDocument._doc_type.index - fields = ('display_name^5', 'name') - - -class ProjectSearch(RTDFacetedSearch): - facets = { - 'language': TermsFacet(field='language') - } - signal = before_project_search +class ProjectSearchBase(RTDFacetedSearch): + facets = {'language': TermsFacet(field='language')} doc_types = [ProjectDocument] index = ProjectDocument._doc_type.index fields = ('name^10', 'slug^5', 'description') @@ -121,6 +83,47 @@ class PageSearchBase(RTDFacetedSearch): fields = ['title^10', 'headers^5', 'content'] +class DomainSearchBase(RTDFacetedSearch): + facets = { + 'project': TermsFacet(field='project'), + 'version': TermsFacet(field='version'), + 'doc_type': TermsFacet(field='doc_type'), + } + doc_types = [DomainDocument] + index = DomainDocument._doc_type.index + fields = ('display_name^5', 'name') + + +class PageSearch(SettingsOverrideObject): + + """ + Allow this class to be overridden based on CLASS_OVERRIDES setting. + This is primary used on the .com to adjust how we filter our search queries + """ + + _default_class = PageSearchBase + + +class ProjectSearch(SettingsOverrideObject): + + """ + Allow this class to be overridden based on CLASS_OVERRIDES setting. + This is primary used on the .com to adjust how we filter our search queries + """ + + _default_class = ProjectSearchBase + + +class DomainSearch(SettingsOverrideObject): + + """ + Allow this class to be overridden based on CLASS_OVERRIDES setting. + This is primary used on the .com to adjust how we filter our search queries + """ + + _default_class = DomainSearchBase + + class AllSearch(RTDFacetedSearch): facets = { 'project': TermsFacet(field='project'), @@ -129,7 +132,6 @@ class AllSearch(RTDFacetedSearch): 'doc_type': TermsFacet(field='doc_type'), 'index': TermsFacet(field='_index'), } - signal = before_file_search doc_types = [DomainDocument, PageDocument, ProjectDocument] index = [DomainDocument._doc_type.index, PageDocument._doc_type.index, From 6680006466dc8a88e94a23f47ad4f2f9c85ca654 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 16:56:37 -0300 Subject: [PATCH 18/22] Cleanup search display and logic. --- readthedocs/search/faceted_search.py | 13 ++++ readthedocs/search/views.py | 68 ++++++++----------- .../templates/search/elastic_search.html | 4 +- 3 files changed, 43 insertions(+), 42 deletions(-) diff --git a/readthedocs/search/faceted_search.py b/readthedocs/search/faceted_search.py index 03e5b9fdbd1..c9d29bff6d9 100644 --- a/readthedocs/search/faceted_search.py +++ b/readthedocs/search/faceted_search.py @@ -13,15 +13,25 @@ log = logging.getLogger(__name__) +ALL_FACETS = ['project', 'version', 'doc_type', 'language', 'index'] + class RTDFacetedSearch(FacetedSearch): def __init__(self, user, **kwargs): self.user = user self.filter_by_user = kwargs.pop('filter_by_user', None) + + # Set filters properly for facet in self.facets: if facet in kwargs: kwargs.setdefault('filters', {})[facet] = kwargs.pop(facet) + + # Don't pass along unnecessary filters + for f in ALL_FACETS: + if f in kwargs: + del kwargs[f] + super().__init__(**kwargs) def search(self): @@ -98,6 +108,7 @@ class PageSearch(SettingsOverrideObject): """ Allow this class to be overridden based on CLASS_OVERRIDES setting. + This is primary used on the .com to adjust how we filter our search queries """ @@ -108,6 +119,7 @@ class ProjectSearch(SettingsOverrideObject): """ Allow this class to be overridden based on CLASS_OVERRIDES setting. + This is primary used on the .com to adjust how we filter our search queries """ @@ -118,6 +130,7 @@ class DomainSearch(SettingsOverrideObject): """ Allow this class to be overridden based on CLASS_OVERRIDES setting. + This is primary used on the .com to adjust how we filter our search queries """ diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 3a1bef1fd01..83700663bb4 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -12,8 +12,6 @@ from readthedocs.search.faceted_search import ( AllSearch, ProjectSearch, PageSearch, DomainSearch, ALL_FACETS ) -from readthedocs.search.utils import get_project_list_or_404 - log = logging.getLogger(__name__) LOG_TEMPLATE = '(Elastic Search) [{user}:{type}] [{project}:{version}:{language}] {msg}' @@ -34,10 +32,21 @@ def elastic_search(request, project_slug=None): - """Use Elasticsearch for global search.""" + """ + Global user search on the dashboard + + This is for both the main search and project search. + + :param project_slug: Sent when the view is a project search + """ + + if project_slug: + queryset = Project.objects.protected(request.user) + project_obj = get_object_or_404(queryset, slug=project_slug) + user_input = UserInput( query=request.GET.get('q'), - type=request.GET.get('type', 'all'), + type=request.GET.get('type', 'project'), project=project_slug or request.GET.get('project'), version=request.GET.get('version', LATEST), taxonomy=request.GET.get('taxonomy'), @@ -45,57 +54,40 @@ def elastic_search(request, project_slug=None): doc_type=request.GET.get('doc_type'), index=request.GET.get('index'), ) + results = '' facets = {} if user_input.query: kwargs = {} - if user_input.project: - projects_list = get_project_list_or_404( - project_slug=user_input.project, user=request.user - ) - project_slug_list = [project.slug for project in projects_list] - kwargs['project'] = project_slug_list - log.info('Filtering to project list: %s' % project_slug_list) - - if user_input.version: - kwargs['version'] = [user_input.version] - - if user_input.doc_type: - kwargs['doc_type'] = user_input.doc_type - - if user_input.language: - kwargs['language'] = user_input.language - - if user_input.index: - kwargs['index'] = user_input.index + for avail_facet in ALL_FACETS: + value = getattr(user_input, avail_facet, None) + if value: + kwargs[avail_facet] = value if user_input.type == 'project': - project_search = ProjectSearch( + search = ProjectSearch( query=user_input.query, user=request.user, **kwargs ) - results = project_search.execute() - facets = results.facets + elif user_input.type == 'domain': - project_search = DomainSearch( + search = DomainSearch( query=user_input.query, user=request.user, **kwargs ) - results = project_search.execute() - facets = results.facets - elif user_input.type == 'file': - page_search = PageSearch( + elif user_input.type == 'file': + search = PageSearch( query=user_input.query, user=request.user, **kwargs ) - results = page_search.execute() - facets = results.facets + elif user_input.type == 'all': - project_search = AllSearch( + search = AllSearch( query=user_input.query, user=request.user, **kwargs ) - results = project_search.execute() - facets = results.facets + + results = search.execute() + facets = results.facets log.info( LOG_TEMPLATE.format( @@ -135,9 +127,7 @@ def elastic_search(request, project_slug=None): }) if project_slug: - queryset = Project.objects.protected(request.user) - project = get_object_or_404(queryset, slug=project_slug) - template_vars.update({'project_obj': project}) + template_vars.update({'project_obj': project_obj}) return render( request, diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 6ebe2dc31cd..88293379cef 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -30,15 +30,13 @@
      - {% if facets.index %} From a3018672e3d428299f489318a710d308905c774c Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 17:04:52 -0300 Subject: [PATCH 19/22] Remove search embed tests --- .../static/search/readthedocs-client.js | 804 ------------------ .../search/static/search/search-embed.js | 18 - .../templates/search/elastic_search.html | 4 - 3 files changed, 826 deletions(-) delete mode 100644 readthedocs/search/static/search/readthedocs-client.js delete mode 100644 readthedocs/search/static/search/search-embed.js diff --git a/readthedocs/search/static/search/readthedocs-client.js b/readthedocs/search/static/search/readthedocs-client.js deleted file mode 100644 index 8d5620d2f26..00000000000 --- a/readthedocs/search/static/search/readthedocs-client.js +++ /dev/null @@ -1,804 +0,0 @@ -(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o 0) { - self._completeHandlers.shift()(resp) - } - } - - function success (resp) { - var type = o['type'] || resp && setType(resp.getResponseHeader('Content-Type')) // resp can be undefined in IE - resp = (type !== 'jsonp') ? self.request : resp - // use global data filter on response text - var filteredResponse = globalSetupOptions.dataFilter(resp.responseText, type) - , r = filteredResponse - try { - resp.responseText = r - } catch (e) { - // can't assign this in IE<=8, just ignore - } - if (r) { - switch (type) { - case 'json': - try { - resp = win.JSON ? win.JSON.parse(r) : eval('(' + r + ')') - } catch (err) { - return error(resp, 'Could not parse JSON in response', err) - } - break - case 'js': - resp = eval(r) - break - case 'html': - resp = r - break - case 'xml': - resp = resp.responseXML - && resp.responseXML.parseError // IE trololo - && resp.responseXML.parseError.errorCode - && resp.responseXML.parseError.reason - ? null - : resp.responseXML - break - } - } - - self._responseArgs.resp = resp - self._fulfilled = true - fn(resp) - self._successHandler(resp) - while (self._fulfillmentHandlers.length > 0) { - resp = self._fulfillmentHandlers.shift()(resp) - } - - complete(resp) - } - - function timedOut() { - self._timedOut = true - self.request.abort() - } - - function error(resp, msg, t) { - resp = self.request - self._responseArgs.resp = resp - self._responseArgs.msg = msg - self._responseArgs.t = t - self._erred = true - while (self._errorHandlers.length > 0) { - self._errorHandlers.shift()(resp, msg, t) - } - complete(resp) - } - - this.request = getRequest.call(this, success, error) - } - - Reqwest.prototype = { - abort: function () { - this._aborted = true - this.request.abort() - } - - , retry: function () { - init.call(this, this.o, this.fn) - } - - /** - * Small deviation from the Promises A CommonJs specification - * http://wiki.commonjs.org/wiki/Promises/A - */ - - /** - * `then` will execute upon successful requests - */ - , then: function (success, fail) { - success = success || function () {} - fail = fail || function () {} - if (this._fulfilled) { - this._responseArgs.resp = success(this._responseArgs.resp) - } else if (this._erred) { - fail(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) - } else { - this._fulfillmentHandlers.push(success) - this._errorHandlers.push(fail) - } - return this - } - - /** - * `always` will execute whether the request succeeds or fails - */ - , always: function (fn) { - if (this._fulfilled || this._erred) { - fn(this._responseArgs.resp) - } else { - this._completeHandlers.push(fn) - } - return this - } - - /** - * `fail` will execute when the request fails - */ - , fail: function (fn) { - if (this._erred) { - fn(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) - } else { - this._errorHandlers.push(fn) - } - return this - } - , 'catch': function (fn) { - return this.fail(fn) - } - } - - function reqwest(o, fn) { - return new Reqwest(o, fn) - } - - // normalize newline variants according to spec -> CRLF - function normalize(s) { - return s ? s.replace(/\r?\n/g, '\r\n') : '' - } - - function serial(el, cb) { - var n = el.name - , t = el.tagName.toLowerCase() - , optCb = function (o) { - // IE gives value="" even where there is no value attribute - // 'specified' ref: http://www.w3.org/TR/DOM-Level-3-Core/core.html#ID-862529273 - if (o && !o['disabled']) - cb(n, normalize(o['attributes']['value'] && o['attributes']['value']['specified'] ? o['value'] : o['text'])) - } - , ch, ra, val, i - - // don't serialize elements that are disabled or without a name - if (el.disabled || !n) return - - switch (t) { - case 'input': - if (!/reset|button|image|file/i.test(el.type)) { - ch = /checkbox/i.test(el.type) - ra = /radio/i.test(el.type) - val = el.value - // WebKit gives us "" instead of "on" if a checkbox has no value, so correct it here - ;(!(ch || ra) || el.checked) && cb(n, normalize(ch && val === '' ? 'on' : val)) - } - break - case 'textarea': - cb(n, normalize(el.value)) - break - case 'select': - if (el.type.toLowerCase() === 'select-one') { - optCb(el.selectedIndex >= 0 ? el.options[el.selectedIndex] : null) - } else { - for (i = 0; el.length && i < el.length; i++) { - el.options[i].selected && optCb(el.options[i]) - } - } - break - } - } - - // collect up all form elements found from the passed argument elements all - // the way down to child elements; pass a '' or form fields. - // called with 'this'=callback to use for serial() on each element - function eachFormElement() { - var cb = this - , e, i - , serializeSubtags = function (e, tags) { - var i, j, fa - for (i = 0; i < tags.length; i++) { - fa = e[byTag](tags[i]) - for (j = 0; j < fa.length; j++) serial(fa[j], cb) - } - } - - for (i = 0; i < arguments.length; i++) { - e = arguments[i] - if (/input|select|textarea/i.test(e.tagName)) serial(e, cb) - serializeSubtags(e, [ 'input', 'select', 'textarea' ]) - } - } - - // standard query string style serialization - function serializeQueryString() { - return reqwest.toQueryString(reqwest.serializeArray.apply(null, arguments)) - } - - // { 'name': 'value', ... } style serialization - function serializeHash() { - var hash = {} - eachFormElement.apply(function (name, value) { - if (name in hash) { - hash[name] && !isArray(hash[name]) && (hash[name] = [hash[name]]) - hash[name].push(value) - } else hash[name] = value - }, arguments) - return hash - } - - // [ { name: 'name', value: 'value' }, ... ] style serialization - reqwest.serializeArray = function () { - var arr = [] - eachFormElement.apply(function (name, value) { - arr.push({name: name, value: value}) - }, arguments) - return arr - } - - reqwest.serialize = function () { - if (arguments.length === 0) return '' - var opt, fn - , args = Array.prototype.slice.call(arguments, 0) - - opt = args.pop() - opt && opt.nodeType && args.push(opt) && (opt = null) - opt && (opt = opt.type) - - if (opt == 'map') fn = serializeHash - else if (opt == 'array') fn = reqwest.serializeArray - else fn = serializeQueryString - - return fn.apply(null, args) - } - - reqwest.toQueryString = function (o, trad) { - var prefix, i - , traditional = trad || false - , s = [] - , enc = encodeURIComponent - , add = function (key, value) { - // If value is a function, invoke it and return its value - value = ('function' === typeof value) ? value() : (value == null ? '' : value) - s[s.length] = enc(key) + '=' + enc(value) - } - // If an array was passed in, assume that it is an array of form elements. - if (isArray(o)) { - for (i = 0; o && i < o.length; i++) add(o[i]['name'], o[i]['value']) - } else { - // If traditional, encode the "old" way (the way 1.3.2 or older - // did it), otherwise encode params recursively. - for (prefix in o) { - if (o.hasOwnProperty(prefix)) buildParams(prefix, o[prefix], traditional, add) - } - } - - // spaces should be + according to spec - return s.join('&').replace(/%20/g, '+') - } - - function buildParams(prefix, obj, traditional, add) { - var name, i, v - , rbracket = /\[\]$/ - - if (isArray(obj)) { - // Serialize array item. - for (i = 0; obj && i < obj.length; i++) { - v = obj[i] - if (traditional || rbracket.test(prefix)) { - // Treat each array item as a scalar. - add(prefix, v) - } else { - buildParams(prefix + '[' + (typeof v === 'object' ? i : '') + ']', v, traditional, add) - } - } - } else if (obj && obj.toString() === '[object Object]') { - // Serialize object item. - for (name in obj) { - buildParams(prefix + '[' + name + ']', obj[name], traditional, add) - } - - } else { - // Serialize scalar item. - add(prefix, obj) - } - } - - reqwest.getcallbackPrefix = function () { - return callbackPrefix - } - - // jQuery and Zepto compatibility, differences can be remapped here so you can call - // .ajax.compat(options, callback) - reqwest.compat = function (o, fn) { - if (o) { - o['type'] && (o['method'] = o['type']) && delete o['type'] - o['dataType'] && (o['type'] = o['dataType']) - o['jsonpCallback'] && (o['jsonpCallbackName'] = o['jsonpCallback']) && delete o['jsonpCallback'] - o['jsonp'] && (o['jsonpCallback'] = o['jsonp']) - } - return new Reqwest(o, fn) - } - - reqwest.ajaxSetup = function (options) { - options = options || {} - for (var k in options) { - globalSetupOptions[k] = options[k] - } - } - - return reqwest -}); - -},{}],2:[function(require,module,exports){ -// Document response - -// Page -var Page = function (project, version, doc) { - this.project = project; - this.version = version; - this.doc = doc; - - this.url = null; - this.sections = []; -}; - -Page.prototype.section = function (section) { - return new Section(this.project, this.version, this.doc, section); -}; - -// Section -var Section = function (project, version, doc, section) { - this.project = project; - this.version = version; - this.doc = doc; - this.section = section; - - this.url = null; - this.content = null; - this.wrapped = null; -} - -// Add iframe with returned content to page -Section.prototype.insertContent = function (elem) { - var iframe = document.createElement('iframe'), - self = this; - - iframe.style.display = 'none'; - - if (window.jQuery && elem instanceof window.jQuery) { - elem = elem.get(0); - } - - if (typeof(elem) != 'undefined') { - while (elem.children.length > 0) { - elem.firstChild.remove(); - } - elem.appendChild(iframe); - } - - var win = iframe.contentWindow; - - win.document.open(); - win.document.write(this.content); - win.document.close(); - - var head = win.document.head, - body = win.document.body, - base = null; - - if (head) { - base = win.document.createElement('base'); - base.target = '_parent'; - base.href = this.url; - head.appendChild(base); - - // Copy linked stylesheets from parent - var link_elems = document.head.getElementsByTagName('link'); - for (var n = 0; n < link_elems.length; n++) { - var link = link_elems[n]; - if (link.rel == 'stylesheet') { - head.appendChild(link.cloneNode()); - } - } - } - - win.onload = function () { - iframe.style.display = 'inline-block'; - }; - - return iframe; -}; - - -exports.Section = Section; -exports.Page = Page; - -},{}],3:[function(require,module,exports){ -/* Read the Docs Embed functions */ - -var doc = require('./doc'), - Section = doc.Section, - Page = doc.Page; - - -var Embed = function (config) { - this._api_host = 'https://api.grokthedocs.com'; - if (typeof config == 'object') { - if ('api_host' in config) { - this._api_host = config['api_host']; - } - } -}; - -Embed.prototype.section = function (project, version, doc, section, - callback, error_callback) { - callback = callback || function () {}; - error_callback = error_callback || function () {}; - - var self = this, - data = { - 'project': project, - 'version': version, - 'doc': doc, - 'section': section - }; - - this._getObject( - data, - function (resp) { - var section_ret = new Section(project, version, doc, section); - section_ret.url = resp.url; - section_ret.content = resp.content; - section_ret.wrapped = resp.wrapped; - callback(section_ret); - }, - function (error, msg) { - error_callback(error); - } - ); -}; - -Embed.prototype.page = function (project, version, doc, callback, - error_callback) { - - var self = this, - data = { - 'project': project, - 'version': version, - 'doc': doc, - }; - - this._getObject( - data, - function (resp) { - var page = new Page(project, version, doc); - page.url = resp.url; - // TODO headers is misleading here, rename it on the API - page.sections = resp.headers; - callback(page); - }, - function (error, msg) { - error_callback(error); - } - ) -}; - -Embed.prototype._getObject = function (data, callback, error_callback) { - var self = this, - reqwest = require("./../bower_components/reqwest/reqwest.js"); - callback = callback || function () {}; - error_callback = error_callback || function () {}; - - return reqwest({ - url: this._api_host + '/api/v1/embed/', - method: 'get', - contentType: 'application/json', - crossDomain: true, - headers: {'Accept': 'application/json'}, - data: data, - success: callback, - error: error_callback - }); -}; - -exports.Embed = Embed; - -},{"./../bower_components/reqwest/reqwest.js":1,"./doc":2}],4:[function(require,module,exports){ -/* Read the Docs Client */ - -var embed = require('./embed'); - - -exports.Embed = embed.Embed; - -if (typeof window != 'undefined') { - window.Embed = embed.Embed; -} - -},{"./embed":3}]},{},[4]) \ No newline at end of file diff --git a/readthedocs/search/static/search/search-embed.js b/readthedocs/search/static/search/search-embed.js deleted file mode 100644 index 88440ad8254..00000000000 --- a/readthedocs/search/static/search/search-embed.js +++ /dev/null @@ -1,18 +0,0 @@ - - $(document).ready(function () { - var config = {api_host: 'http://localhost:8000'} - var embed = new Embed(config); - $('.fragment').one("click mouseover", - function(elem) { - embed.section( - elem.currentTarget.getAttribute('project'), - elem.currentTarget.getAttribute('version'), - elem.currentTarget.getAttribute('doc_name').replace('.html', ''), - elem.currentTarget.getAttribute('anchor'), - function (section) { - section.insertContent(elem.currentTarget); - } - ) - } - ) - }) diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 88293379cef..1d9346c0898 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -13,10 +13,6 @@ {% endblock %} -{% block extra_scripts %} - - -{% endblock %} {% block project_editing %} {% if project_obj %} From 9dd55afb3b9d5a15352c97797b4c68b7c4057db1 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 13 Feb 2019 17:04:58 -0300 Subject: [PATCH 20/22] Cleanup file search --- readthedocs/search/views.py | 4 +++- readthedocs/templates/search/elastic_search.html | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 83700663bb4..4ae9e5a4154 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -39,14 +39,16 @@ def elastic_search(request, project_slug=None): :param project_slug: Sent when the view is a project search """ + _type = None if project_slug: queryset = Project.objects.protected(request.user) project_obj = get_object_or_404(queryset, slug=project_slug) + _type = request.GET.get('type', 'file') user_input = UserInput( query=request.GET.get('q'), - type=request.GET.get('type', 'project'), + type=_type or request.GET.get('type', 'project'), project=project_slug or request.GET.get('project'), version=request.GET.get('version', LATEST), taxonomy=request.GET.get('taxonomy'), diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 1d9346c0898..7a1b89a17a8 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -28,7 +28,9 @@
        {% trans 'Object Type' %}
        {#
      • {% trans 'All' %}
        • #} + {% if not project %}
      • {% trans 'Projects' %}
        • + {% endif %}
      • {% trans 'Files' %}
      • {% trans 'Code API' %}
        • From 4af11e8bb4463887c892f30980a7d51691dd194b Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Thu, 14 Feb 2019 10:53:33 -0300 Subject: [PATCH 21/22] Pass along the variables in the search form if they are set --- readthedocs/templates/search/elastic_search.html | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 7a1b89a17a8..06cf12a87d6 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -27,6 +27,7 @@
          {% trans 'Object Type' %}
          + {# Hidden until we make the UX nicer here. #} {#
        • {% trans 'All' %}
          • #} {% if not project %}
        • {% trans 'Projects' %}
          • @@ -142,9 +143,10 @@

          {% trans 'Search' %}

          {% if type %} {% endif %} {% if project %} {% endif %} - {% if taxonomy %} {% endif %} {% if version %} {% endif %} {% if language %} {% endif %} + {% if doc_type %} {% endif %} + {% if index %} {% endif %}
        {% comment %}Translators: This is about starting a search{% endcomment %} From 3addaacd68d4765bd26e97e1f3b915f41193775d Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Thu, 14 Feb 2019 12:27:05 -0300 Subject: [PATCH 22/22] Use the embed API to show search results inline This is a work in progress, just showing what can be done. --- .../static/search/readthedocs-client.js | 804 ++++++++++++++++++ .../search/static/search/search-embed.js | 18 + .../templates/search/elastic_search.html | 4 + 3 files changed, 826 insertions(+) create mode 100644 readthedocs/search/static/search/readthedocs-client.js create mode 100644 readthedocs/search/static/search/search-embed.js diff --git a/readthedocs/search/static/search/readthedocs-client.js b/readthedocs/search/static/search/readthedocs-client.js new file mode 100644 index 00000000000..8d5620d2f26 --- /dev/null +++ b/readthedocs/search/static/search/readthedocs-client.js @@ -0,0 +1,804 @@ +(function e(t,n,r){function s(o,u){if(!n[o]){if(!t[o]){var a=typeof require=="function"&&require;if(!u&&a)return a(o,!0);if(i)return i(o,!0);throw new Error("Cannot find module '"+o+"'")}var f=n[o]={exports:{}};t[o][0].call(f.exports,function(e){var n=t[o][1][e];return s(n?n:e)},f,f.exports,e,t,n,r)}return n[o].exports}var i=typeof require=="function"&&require;for(var o=0;o 0) { + self._completeHandlers.shift()(resp) + } + } + + function success (resp) { + var type = o['type'] || resp && setType(resp.getResponseHeader('Content-Type')) // resp can be undefined in IE + resp = (type !== 'jsonp') ? self.request : resp + // use global data filter on response text + var filteredResponse = globalSetupOptions.dataFilter(resp.responseText, type) + , r = filteredResponse + try { + resp.responseText = r + } catch (e) { + // can't assign this in IE<=8, just ignore + } + if (r) { + switch (type) { + case 'json': + try { + resp = win.JSON ? win.JSON.parse(r) : eval('(' + r + ')') + } catch (err) { + return error(resp, 'Could not parse JSON in response', err) + } + break + case 'js': + resp = eval(r) + break + case 'html': + resp = r + break + case 'xml': + resp = resp.responseXML + && resp.responseXML.parseError // IE trololo + && resp.responseXML.parseError.errorCode + && resp.responseXML.parseError.reason + ? null + : resp.responseXML + break + } + } + + self._responseArgs.resp = resp + self._fulfilled = true + fn(resp) + self._successHandler(resp) + while (self._fulfillmentHandlers.length > 0) { + resp = self._fulfillmentHandlers.shift()(resp) + } + + complete(resp) + } + + function timedOut() { + self._timedOut = true + self.request.abort() + } + + function error(resp, msg, t) { + resp = self.request + self._responseArgs.resp = resp + self._responseArgs.msg = msg + self._responseArgs.t = t + self._erred = true + while (self._errorHandlers.length > 0) { + self._errorHandlers.shift()(resp, msg, t) + } + complete(resp) + } + + this.request = getRequest.call(this, success, error) + } + + Reqwest.prototype = { + abort: function () { + this._aborted = true + this.request.abort() + } + + , retry: function () { + init.call(this, this.o, this.fn) + } + + /** + * Small deviation from the Promises A CommonJs specification + * http://wiki.commonjs.org/wiki/Promises/A + */ + + /** + * `then` will execute upon successful requests + */ + , then: function (success, fail) { + success = success || function () {} + fail = fail || function () {} + if (this._fulfilled) { + this._responseArgs.resp = success(this._responseArgs.resp) + } else if (this._erred) { + fail(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) + } else { + this._fulfillmentHandlers.push(success) + this._errorHandlers.push(fail) + } + return this + } + + /** + * `always` will execute whether the request succeeds or fails + */ + , always: function (fn) { + if (this._fulfilled || this._erred) { + fn(this._responseArgs.resp) + } else { + this._completeHandlers.push(fn) + } + return this + } + + /** + * `fail` will execute when the request fails + */ + , fail: function (fn) { + if (this._erred) { + fn(this._responseArgs.resp, this._responseArgs.msg, this._responseArgs.t) + } else { + this._errorHandlers.push(fn) + } + return this + } + , 'catch': function (fn) { + return this.fail(fn) + } + } + + function reqwest(o, fn) { + return new Reqwest(o, fn) + } + + // normalize newline variants according to spec -> CRLF + function normalize(s) { + return s ? s.replace(/\r?\n/g, '\r\n') : '' + } + + function serial(el, cb) { + var n = el.name + , t = el.tagName.toLowerCase() + , optCb = function (o) { + // IE gives value="" even where there is no value attribute + // 'specified' ref: http://www.w3.org/TR/DOM-Level-3-Core/core.html#ID-862529273 + if (o && !o['disabled']) + cb(n, normalize(o['attributes']['value'] && o['attributes']['value']['specified'] ? o['value'] : o['text'])) + } + , ch, ra, val, i + + // don't serialize elements that are disabled or without a name + if (el.disabled || !n) return + + switch (t) { + case 'input': + if (!/reset|button|image|file/i.test(el.type)) { + ch = /checkbox/i.test(el.type) + ra = /radio/i.test(el.type) + val = el.value + // WebKit gives us "" instead of "on" if a checkbox has no value, so correct it here + ;(!(ch || ra) || el.checked) && cb(n, normalize(ch && val === '' ? 'on' : val)) + } + break + case 'textarea': + cb(n, normalize(el.value)) + break + case 'select': + if (el.type.toLowerCase() === 'select-one') { + optCb(el.selectedIndex >= 0 ? el.options[el.selectedIndex] : null) + } else { + for (i = 0; el.length && i < el.length; i++) { + el.options[i].selected && optCb(el.options[i]) + } + } + break + } + } + + // collect up all form elements found from the passed argument elements all + // the way down to child elements; pass a '' or form fields. + // called with 'this'=callback to use for serial() on each element + function eachFormElement() { + var cb = this + , e, i + , serializeSubtags = function (e, tags) { + var i, j, fa + for (i = 0; i < tags.length; i++) { + fa = e[byTag](tags[i]) + for (j = 0; j < fa.length; j++) serial(fa[j], cb) + } + } + + for (i = 0; i < arguments.length; i++) { + e = arguments[i] + if (/input|select|textarea/i.test(e.tagName)) serial(e, cb) + serializeSubtags(e, [ 'input', 'select', 'textarea' ]) + } + } + + // standard query string style serialization + function serializeQueryString() { + return reqwest.toQueryString(reqwest.serializeArray.apply(null, arguments)) + } + + // { 'name': 'value', ... } style serialization + function serializeHash() { + var hash = {} + eachFormElement.apply(function (name, value) { + if (name in hash) { + hash[name] && !isArray(hash[name]) && (hash[name] = [hash[name]]) + hash[name].push(value) + } else hash[name] = value + }, arguments) + return hash + } + + // [ { name: 'name', value: 'value' }, ... ] style serialization + reqwest.serializeArray = function () { + var arr = [] + eachFormElement.apply(function (name, value) { + arr.push({name: name, value: value}) + }, arguments) + return arr + } + + reqwest.serialize = function () { + if (arguments.length === 0) return '' + var opt, fn + , args = Array.prototype.slice.call(arguments, 0) + + opt = args.pop() + opt && opt.nodeType && args.push(opt) && (opt = null) + opt && (opt = opt.type) + + if (opt == 'map') fn = serializeHash + else if (opt == 'array') fn = reqwest.serializeArray + else fn = serializeQueryString + + return fn.apply(null, args) + } + + reqwest.toQueryString = function (o, trad) { + var prefix, i + , traditional = trad || false + , s = [] + , enc = encodeURIComponent + , add = function (key, value) { + // If value is a function, invoke it and return its value + value = ('function' === typeof value) ? value() : (value == null ? '' : value) + s[s.length] = enc(key) + '=' + enc(value) + } + // If an array was passed in, assume that it is an array of form elements. + if (isArray(o)) { + for (i = 0; o && i < o.length; i++) add(o[i]['name'], o[i]['value']) + } else { + // If traditional, encode the "old" way (the way 1.3.2 or older + // did it), otherwise encode params recursively. + for (prefix in o) { + if (o.hasOwnProperty(prefix)) buildParams(prefix, o[prefix], traditional, add) + } + } + + // spaces should be + according to spec + return s.join('&').replace(/%20/g, '+') + } + + function buildParams(prefix, obj, traditional, add) { + var name, i, v + , rbracket = /\[\]$/ + + if (isArray(obj)) { + // Serialize array item. + for (i = 0; obj && i < obj.length; i++) { + v = obj[i] + if (traditional || rbracket.test(prefix)) { + // Treat each array item as a scalar. + add(prefix, v) + } else { + buildParams(prefix + '[' + (typeof v === 'object' ? i : '') + ']', v, traditional, add) + } + } + } else if (obj && obj.toString() === '[object Object]') { + // Serialize object item. + for (name in obj) { + buildParams(prefix + '[' + name + ']', obj[name], traditional, add) + } + + } else { + // Serialize scalar item. + add(prefix, obj) + } + } + + reqwest.getcallbackPrefix = function () { + return callbackPrefix + } + + // jQuery and Zepto compatibility, differences can be remapped here so you can call + // .ajax.compat(options, callback) + reqwest.compat = function (o, fn) { + if (o) { + o['type'] && (o['method'] = o['type']) && delete o['type'] + o['dataType'] && (o['type'] = o['dataType']) + o['jsonpCallback'] && (o['jsonpCallbackName'] = o['jsonpCallback']) && delete o['jsonpCallback'] + o['jsonp'] && (o['jsonpCallback'] = o['jsonp']) + } + return new Reqwest(o, fn) + } + + reqwest.ajaxSetup = function (options) { + options = options || {} + for (var k in options) { + globalSetupOptions[k] = options[k] + } + } + + return reqwest +}); + +},{}],2:[function(require,module,exports){ +// Document response + +// Page +var Page = function (project, version, doc) { + this.project = project; + this.version = version; + this.doc = doc; + + this.url = null; + this.sections = []; +}; + +Page.prototype.section = function (section) { + return new Section(this.project, this.version, this.doc, section); +}; + +// Section +var Section = function (project, version, doc, section) { + this.project = project; + this.version = version; + this.doc = doc; + this.section = section; + + this.url = null; + this.content = null; + this.wrapped = null; +} + +// Add iframe with returned content to page +Section.prototype.insertContent = function (elem) { + var iframe = document.createElement('iframe'), + self = this; + + iframe.style.display = 'none'; + + if (window.jQuery && elem instanceof window.jQuery) { + elem = elem.get(0); + } + + if (typeof(elem) != 'undefined') { + while (elem.children.length > 0) { + elem.firstChild.remove(); + } + elem.appendChild(iframe); + } + + var win = iframe.contentWindow; + + win.document.open(); + win.document.write(this.content); + win.document.close(); + + var head = win.document.head, + body = win.document.body, + base = null; + + if (head) { + base = win.document.createElement('base'); + base.target = '_parent'; + base.href = this.url; + head.appendChild(base); + + // Copy linked stylesheets from parent + var link_elems = document.head.getElementsByTagName('link'); + for (var n = 0; n < link_elems.length; n++) { + var link = link_elems[n]; + if (link.rel == 'stylesheet') { + head.appendChild(link.cloneNode()); + } + } + } + + win.onload = function () { + iframe.style.display = 'inline-block'; + }; + + return iframe; +}; + + +exports.Section = Section; +exports.Page = Page; + +},{}],3:[function(require,module,exports){ +/* Read the Docs Embed functions */ + +var doc = require('./doc'), + Section = doc.Section, + Page = doc.Page; + + +var Embed = function (config) { + this._api_host = 'https://api.grokthedocs.com'; + if (typeof config == 'object') { + if ('api_host' in config) { + this._api_host = config['api_host']; + } + } +}; + +Embed.prototype.section = function (project, version, doc, section, + callback, error_callback) { + callback = callback || function () {}; + error_callback = error_callback || function () {}; + + var self = this, + data = { + 'project': project, + 'version': version, + 'doc': doc, + 'section': section + }; + + this._getObject( + data, + function (resp) { + var section_ret = new Section(project, version, doc, section); + section_ret.url = resp.url; + section_ret.content = resp.content; + section_ret.wrapped = resp.wrapped; + callback(section_ret); + }, + function (error, msg) { + error_callback(error); + } + ); +}; + +Embed.prototype.page = function (project, version, doc, callback, + error_callback) { + + var self = this, + data = { + 'project': project, + 'version': version, + 'doc': doc, + }; + + this._getObject( + data, + function (resp) { + var page = new Page(project, version, doc); + page.url = resp.url; + // TODO headers is misleading here, rename it on the API + page.sections = resp.headers; + callback(page); + }, + function (error, msg) { + error_callback(error); + } + ) +}; + +Embed.prototype._getObject = function (data, callback, error_callback) { + var self = this, + reqwest = require("./../bower_components/reqwest/reqwest.js"); + callback = callback || function () {}; + error_callback = error_callback || function () {}; + + return reqwest({ + url: this._api_host + '/api/v1/embed/', + method: 'get', + contentType: 'application/json', + crossDomain: true, + headers: {'Accept': 'application/json'}, + data: data, + success: callback, + error: error_callback + }); +}; + +exports.Embed = Embed; + +},{"./../bower_components/reqwest/reqwest.js":1,"./doc":2}],4:[function(require,module,exports){ +/* Read the Docs Client */ + +var embed = require('./embed'); + + +exports.Embed = embed.Embed; + +if (typeof window != 'undefined') { + window.Embed = embed.Embed; +} + +},{"./embed":3}]},{},[4]) \ No newline at end of file diff --git a/readthedocs/search/static/search/search-embed.js b/readthedocs/search/static/search/search-embed.js new file mode 100644 index 00000000000..e1c970aed8a --- /dev/null +++ b/readthedocs/search/static/search/search-embed.js @@ -0,0 +1,18 @@ +$(document).ready(function () { + var config = {api_host: 'http://localhost:8000'} + var embed = new Embed(config); + $('.fragment').one("click mouseover", + function(elem) { + embed.section( + elem.currentTarget.getAttribute('project'), + elem.currentTarget.getAttribute('version'), + elem.currentTarget.getAttribute('doc_name').replace('.html', ''), + elem.currentTarget.getAttribute('anchor'), + function (section) { + section.insertContent(elem.currentTarget); + } + ) + } + ) +}) + diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 06cf12a87d6..dd27642cee5 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -13,6 +13,10 @@ {% endblock %} +{% block extra_scripts %} + + +{% endblock %} {% block project_editing %} {% if project_obj %}