Index more domain data into elasticsearch

readthedocs/projects/models.py

@@ -1271,6 +1271,7 @@ def get_processed_json(self):
             'path': file_path,
             'title': '',
             'sections': [],
+            'domain_data': {},
         }
 
     @cached_property

readthedocs/search/documents.py

@@ -94,6 +94,8 @@ class PageDocument(RTDDocTypeMixin, DocType):
             # For showing in the search result
             'type_display': fields.TextField(),
             'doc_display': fields.TextField(),
+            'docstrings': fields.TextField(),
+            'signature': fields.TextField(),
 
             # Simple analyzer breaks on `.`,
             # otherwise search results are too strict for this use case
@@ -123,6 +125,10 @@ def prepare_domains(self, html_file):
                 'anchor': domain.anchor,
                 'type_display': domain.type_display,
                 'doc_display': domain.doc_display,
+                'docstrings': self._get_domain_data(html_file.processed_json['domain_data'],
+                                                    'docstrings', domain.anchor),
+                'signature': self._get_domain_data(html_file.processed_json['domain_data'],
+                                                   'signature', domain.anchor),
                 'name': domain.name,
                 'display_name': domain.display_name if domain.display_name != '-' else '',
             }
@@ -131,6 +137,18 @@ def prepare_domains(self, html_file):
 
         return all_domains
 
+    def _get_domain_data(self, data, data_type, anchor):
+        """Returns sphinx domains data from the ``data`` dict."""
+        domain_data = data.get(anchor, None)
+
+        if not domain_data:
+            return ''
+        text = domain_data.get(data_type, '')
+        if data_type == 'signature':
+            # '[source]' is present in every domain signature at the end.
+            text = text.replace('[source]', '')
+        return text
+
     @classmethod
     def faceted_search(
             cls, query, user, projects_list=None, versions_list=None,

readthedocs/search/parse_json.py

@@ -25,7 +25,7 @@ def generate_sections_from_pyquery(body):
                 if 'section' in next_p[0].attrib['class']:
                     break
 
-            h1_content += parse_content(next_p.text())
+            h1_content += parse_content(next_p.text(), remove_first_line=True)
             next_p = next_p.next()
         if h1_content:
             yield {
@@ -43,7 +43,7 @@ def generate_sections_from_pyquery(body):
         section_id = div.attr('id')
 
         content = div.text()
-        content = parse_content(content)
+        content = parse_content(content, remove_first_line=True)
 
         yield {
             'id': section_id,
@@ -64,6 +64,7 @@ def process_file(fjson_filename):
     sections = []
     path = ''
     title = ''
+    domain_data = {}
 
     if 'current_page_name' in data:
         path = data['current_page_name']
@@ -73,6 +74,7 @@ def process_file(fjson_filename):
     if data.get('body'):
         body = PyQuery(data['body'])
         sections.extend(generate_sections_from_pyquery(body))
+        domain_data = generate_domains_data_from_pyquery(body, fjson_filename)
     else:
         log.info('Unable to index content for: %s', fjson_filename)
 
@@ -86,10 +88,11 @@ def process_file(fjson_filename):
         'path': path,
         'title': title,
         'sections': sections,
+        'domain_data': domain_data,
     }
 
 
-def parse_content(content):
+def parse_content(content, remove_first_line=False):
     """
     Removes the starting text and ¶.
 
@@ -101,9 +104,71 @@ def parse_content(content):
 
     # removing the starting text of each
     content = content.split('\n')
-    if len(content) > 1:  # there were \n
+    if remove_first_line and len(content) > 1:
         content = content[1:]
 
     # converting newlines to ". "
     content = '. '.join([text.strip().rstrip('.') for text in content])
     return content
+
+
+def _get_text_for_domain_data(desc_contents):
+    """Returns the text from the PyQuery object ``desc_contents``."""
+    # remove the 'dl', 'dt' and 'dd' tags from it
+    # because all the 'dd' and 'dt' tags are inside 'dl'
+    # and all 'dl' tags are already captured.
+    desc_contents.remove('dl')
+    desc_contents.remove('dt')
+    desc_contents.remove('dd')
+
+    # remove multiple spaces, new line characters and '¶' symbol.
+    docstrings = parse_content(desc_contents.text())
+    return docstrings
+
+
+def generate_domains_data_from_pyquery(body, fjson_filename):
+    """
+    Given a pyquery object, generate sphinx domain objects' docstrings.
+
+    Returns a dict with the generated data.
+    The returned dict is in the following form::
+
+        {
+            "domain-id-1": {
+                "signature": "domain_signature_1(*args, **kwargs),
+                "docstrings": "docstrings for the domain-id-1",
+            },
+            "domain-id-2": {
+                "signature": "domain_signature_2(*args, **kwargs),
+                "docstrings": "docstrings for the domain-id-2",
+            }
+        }
+    """
+
+    domain_data = {}
+    try:
+        dl_tags = body('dl')
+        for dl_tag in dl_tags:
+
+            dt = dl_tag.findall('dt')
+            dd = dl_tag.findall('dd')
+
+            # len(dt) should be equal to len(dd)
+            # because these tags go together.
+            for title, desc in zip(dt, dd):
+                id_ = title.attrib.get('id')
+                if id_:
+                    # clone the PyQuery objects so that
+                    # the original one remains undisturbed
+                    signature = _get_text_for_domain_data(PyQuery(title).clone())
+                    docstrings = _get_text_for_domain_data(PyQuery(desc).clone())
+
+                    domain_data[id_] = {
+                        'signature': signature,
+                        'docstrings': docstrings
+                    }
+
+    except Exception:
+        log.exception('Error parsing docstrings for domains in file %s', fjson_filename)
+
+    return domain_data