From 536874eadd75a56fd077c2820edb519f3a85e920 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 17 Nov 2018 00:29:34 +0600 Subject: [PATCH 1/4] optimize for elasticsearch --- readthedocs/settings/base.py | 11 ++++++++--- readthedocs/settings/test.py | 1 + 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 3e06c78347f..44f1bfd96d5 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -343,18 +343,23 @@ def USE_PROMOS(self): # noqa ES_INDEXES = { 'project': { 'name': 'project_index', - 'settings': {'number_of_shards': 5, + 'settings': {'number_of_shards': 1, 'number_of_replicas': 1 } }, 'page': { 'name': 'page_index', 'settings': { - 'number_of_shards': 5, - 'number_of_replicas': 1, + 'number_of_shards': 3, + 'number_of_replicas': 3, + "index": { + "sort.field": ["project", "version"] + } } }, } + # Disable auto refresh for increasing index performance + ELASTICSEARCH_DSL_AUTO_REFRESH = False ALLOWED_HOSTS = ['*'] diff --git a/readthedocs/settings/test.py b/readthedocs/settings/test.py index 6bd1e3f6752..ca4adca929d 100644 --- a/readthedocs/settings/test.py +++ b/readthedocs/settings/test.py @@ -18,6 +18,7 @@ class CommunityTestSettings(CommunityDevSettings): TEMPLATE_DEBUG = False ES_PAGE_IGNORE_SIGNALS = False ELASTICSEARCH_DSL_AUTOSYNC = False + ELASTICSEARCH_DSL_AUTO_REFRESH = False @property def ES_INDEXES(self): # noqa - avoid pep8 N802 From 40865ae59a0f157275c9aae2537805a9a0901295 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 17 Nov 2018 00:54:28 +0600 Subject: [PATCH 2/4] fixing test --- readthedocs/settings/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/settings/test.py b/readthedocs/settings/test.py index ca4adca929d..ee8132c053f 100644 --- a/readthedocs/settings/test.py +++ b/readthedocs/settings/test.py @@ -18,7 +18,7 @@ class CommunityTestSettings(CommunityDevSettings): TEMPLATE_DEBUG = False ES_PAGE_IGNORE_SIGNALS = False ELASTICSEARCH_DSL_AUTOSYNC = False - ELASTICSEARCH_DSL_AUTO_REFRESH = False + ELASTICSEARCH_DSL_AUTO_REFRESH = True @property def ES_INDEXES(self): # noqa - avoid pep8 N802 From 10a6590f13dc99fb88ea02c2d9febba919312044 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 17 Nov 2018 01:50:48 +0600 Subject: [PATCH 3/4] adding comments --- readthedocs/search/documents.py | 2 +- readthedocs/settings/base.py | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/readthedocs/search/documents.py b/readthedocs/search/documents.py index 8520f90e71e..770c9cb9384 100644 --- a/readthedocs/search/documents.py +++ b/readthedocs/search/documents.py @@ -59,7 +59,7 @@ class Meta(object): title = fields.TextField(attr='processed_json.title') headers = fields.TextField(attr='processed_json.headers') content = fields.TextField(attr='processed_json.content') - path = fields.TextField(attr='processed_json.path') + path = fields.KeywordField(attr='processed_json.path') # Fields to perform search with weight search_fields = ['title^10', 'headers^5', 'content'] diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 44f1bfd96d5..532ab6e13e0 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -343,6 +343,8 @@ def USE_PROMOS(self): # noqa ES_INDEXES = { 'project': { 'name': 'project_index', + # We do not have much data in the project node, therefore only 1 shard with + # 1 replica is appropriate project index 'settings': {'number_of_shards': 1, 'number_of_replicas': 1 } @@ -350,6 +352,10 @@ def USE_PROMOS(self): # noqa 'page': { 'name': 'page_index', 'settings': { + # We have 3 nodes, therefore having 3 shards and each one having 3 replica + # will be good fit for our infrastructure. So all the 9(3*3) shards will be + # allocated to 3 nodes. Therefore, if one nodes get failed, the data will be + # inside other nodes and Elasticsearch can serve properly. 'number_of_shards': 3, 'number_of_replicas': 3, "index": { From 7bea6d8a87490c65b1310e793a98ba836eff04f4 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 17 Nov 2018 02:17:22 +0600 Subject: [PATCH 4/4] more fix --- readthedocs/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 532ab6e13e0..4eaae861235 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -343,7 +343,7 @@ def USE_PROMOS(self): # noqa ES_INDEXES = { 'project': { 'name': 'project_index', - # We do not have much data in the project node, therefore only 1 shard with + # We do not have much data in the project index, therefore only 1 shard with # 1 replica is appropriate project index 'settings': {'number_of_shards': 1, 'number_of_replicas': 1