Merge pull request #4909 from safwanrahman/search_fix

Tuning Elasticsearch for search improvements

Author: ericholscher

readthedocs/search/documents.py

@@ -59,7 +59,7 @@ class Meta(object):
     title = fields.TextField(attr='processed_json.title')
     headers = fields.TextField(attr='processed_json.headers')
     content = fields.TextField(attr='processed_json.content')
-    path = fields.TextField(attr='processed_json.path')
+    path = fields.KeywordField(attr='processed_json.path')
 
     # Fields to perform search with weight
     search_fields = ['title^10', 'headers^5', 'content']

readthedocs/settings/base.py

@@ -343,18 +343,29 @@ def USE_PROMOS(self):  # noqa
     ES_INDEXES = {
         'project': {
             'name': 'project_index',
-            'settings': {'number_of_shards': 5,
+            # We do not have much data in the project index, therefore only 1 shard with
+            # 1 replica is appropriate project index
+            'settings': {'number_of_shards': 1,
                          'number_of_replicas': 1
                          }
         },
         'page': {
             'name': 'page_index',
             'settings': {
-                'number_of_shards': 5,
-                'number_of_replicas': 1,
+                # We have 3 nodes, therefore having 3 shards and each one having 3 replica
+                # will be good fit for our infrastructure. So all the 9(3*3) shards will be
+                # allocated to 3 nodes. Therefore, if one nodes get failed, the data will be
+                # inside other nodes and Elasticsearch can serve properly.
+                'number_of_shards': 3,
+                'number_of_replicas': 3,
+                "index": {
+                    "sort.field": ["project", "version"]
+                }
             }
         },
     }
+    # Disable auto refresh for increasing index performance
+    ELASTICSEARCH_DSL_AUTO_REFRESH = False
 
     ALLOWED_HOSTS = ['*']
 

readthedocs/settings/test.py

@@ -18,6 +18,7 @@ class CommunityTestSettings(CommunityDevSettings):
     TEMPLATE_DEBUG = False
     ES_PAGE_IGNORE_SIGNALS = False
     ELASTICSEARCH_DSL_AUTOSYNC = False
+    ELASTICSEARCH_DSL_AUTO_REFRESH = True
 
     @property
     def ES_INDEXES(self):  # noqa - avoid pep8 N802