From 8d3ae90bb1c490116c77510f32fcfd2d448413ad Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 25 Sep 2023 19:16:53 -0500 Subject: [PATCH 1/4] Upgrade ES to 8.x We need to wait for https://github.com/django-es/django-elasticsearch-dsl/pull/458 or https://github.com/readthedocs/readthedocs.org/issues/10730. There are no breaking changes in the API, so this can be deployed without any downtime. The only change is that it now has authentication enabled by default, so our docker image now uses a password. How to deploy: - Deploy web extra - The new packages will continue working with the old ES, but we can make sure by searching from web-extra. - Finish the deploy - Upgrade the ES cluster from cloud.elastic.co --- .circleci/config.yml | 1 + readthedocs/settings/base.py | 3 +-- requirements/deploy.txt | 19 +++++++++++-------- requirements/docker.txt | 19 +++++++++++-------- requirements/pip.in | 7 ++++--- requirements/pip.txt | 17 +++++++++-------- requirements/testing.txt | 19 +++++++++++-------- 7 files changed, 48 insertions(+), 37 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9a66ccaad3e..17b805950e6 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -16,6 +16,7 @@ jobs: environment: discovery.type: single-node ES_JAVA_OPTS: -Xms750m -Xmx750m + ELASTIC_PASSWORD: password steps: - checkout - run: git submodule sync diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index abb54cad4cc..485ee29467e 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -790,10 +790,9 @@ def DOCKER_LIMITS(self): RTD_ORG_TRIAL_PERIOD_DAYS = 30 # Elasticsearch settings. - ES_HOSTS = ['search:9200'] ELASTICSEARCH_DSL = { 'default': { - 'hosts': 'search:9200' + 'hosts': 'http://elastic:password@search:9200', }, } # Chunk size for elasticsearch reindex celery tasks diff --git a/requirements/deploy.txt b/requirements/deploy.txt index a394868fcb1..85fa90a9eea 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -24,11 +24,11 @@ billiard==3.6.4.0 # via # -r requirements/pip.txt # celery -boto3==1.28.53 +boto3==1.28.54 # via # -r requirements/pip.txt # django-storages -botocore==1.31.53 +botocore==1.31.54 # via # -r requirements/pip.txt # boto3 @@ -40,7 +40,7 @@ celery==5.2.7 certifi==2023.7.22 # via # -r requirements/pip.txt - # elasticsearch + # elastic-transport # requests # sentry-sdk cffi==1.15.1 @@ -135,7 +135,7 @@ django-csp==3.7 # via -r requirements/pip.txt django-debug-toolbar==4.2.0 # via -r requirements/pip.txt -django-elasticsearch-dsl==7.3 +django-elasticsearch-dsl @ git+https://github.com/oehrlein/django-elasticsearch-dsl.git@elasticsearch-8 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -185,11 +185,15 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via + # -r requirements/pip.txt + # elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.txt # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.txt # django-elasticsearch-dsl @@ -341,7 +345,6 @@ six==1.16.0 # asttokens # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # unicode-slugify slumber==0.7.1 @@ -395,7 +398,7 @@ urllib3==1.26.16 # -r requirements/pip.txt # botocore # docker - # elasticsearch + # elastic-transport # requests # sentry-sdk user-agents==2.2.0 diff --git a/requirements/docker.txt b/requirements/docker.txt index b1a0afd5eeb..37039a5a923 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -26,11 +26,11 @@ billiard==3.6.4.0 # via # -r requirements/pip.txt # celery -boto3==1.28.53 +boto3==1.28.54 # via # -r requirements/pip.txt # django-storages -botocore==1.31.53 +botocore==1.31.54 # via # -r requirements/pip.txt # boto3 @@ -44,7 +44,7 @@ celery==5.2.7 certifi==2023.7.22 # via # -r requirements/pip.txt - # elasticsearch + # elastic-transport # requests cffi==1.15.1 # via @@ -146,7 +146,7 @@ django-csp==3.7 # via -r requirements/pip.txt django-debug-toolbar==4.2.0 # via -r requirements/pip.txt -django-elasticsearch-dsl==7.3 +django-elasticsearch-dsl @ git+https://github.com/oehrlein/django-elasticsearch-dsl.git@elasticsearch-8 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -196,11 +196,15 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via + # -r requirements/pip.txt + # elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.txt # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.txt # django-elasticsearch-dsl @@ -374,7 +378,6 @@ six==1.16.0 # asttokens # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # unicode-slugify slumber==0.7.1 @@ -430,7 +433,7 @@ urllib3==1.26.16 # -r requirements/pip.txt # botocore # docker - # elasticsearch + # elastic-transport # requests user-agents==2.2.0 # via -r requirements/pip.txt diff --git a/requirements/pip.in b/requirements/pip.in index 6c59f225e64..7d1b1249a37 100644 --- a/requirements/pip.in +++ b/requirements/pip.in @@ -87,9 +87,10 @@ django-allauth==0.51.0 requests-oauthlib # Search -elasticsearch<8.0 -elasticsearch-dsl<8.0 -django-elasticsearch-dsl<8.0 +elasticsearch~=8.0 +elasticsearch-dsl~=8.0 +django-elasticsearch-dsl@git+https://github.com/oehrlein/django-elasticsearch-dsl.git@elasticsearch-8 + selectolax # embed diff --git a/requirements/pip.txt b/requirements/pip.txt index ba3f96a608a..38208a62f2d 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -12,9 +12,9 @@ async-timeout==4.0.3 # via redis billiard==3.6.4.0 # via celery -boto3==1.28.53 +boto3==1.28.54 # via django-storages -botocore==1.31.53 +botocore==1.31.54 # via # boto3 # s3transfer @@ -24,7 +24,7 @@ celery==5.2.7 # django-celery-beat certifi==2023.7.22 # via - # elasticsearch + # elastic-transport # requests cffi==1.15.1 # via cryptography @@ -95,7 +95,7 @@ django-csp==3.7 # via -r requirements/pip.in django-debug-toolbar==4.2.0 # via -r requirements/pip.in -django-elasticsearch-dsl==7.3 +django-elasticsearch-dsl @ git+https://github.com/oehrlein/django-elasticsearch-dsl.git@elasticsearch-8 # via -r requirements/pip.in django-extensions==3.2.3 # via -r requirements/pip.in @@ -141,11 +141,13 @@ drf-extensions==0.7.1 # via -r requirements/pip.in drf-flex-fields==1.0.2 # via -r requirements/pip.in -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.in # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.in # django-elasticsearch-dsl @@ -241,7 +243,6 @@ six==1.16.0 # via # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # unicode-slugify slumber==0.7.1 @@ -276,7 +277,7 @@ urllib3==1.26.16 # via # botocore # docker - # elasticsearch + # elastic-transport # requests user-agents==2.2.0 # via -r requirements/pip.in diff --git a/requirements/testing.txt b/requirements/testing.txt index 2145850307d..9d3f50a579e 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -24,11 +24,11 @@ billiard==3.6.4.0 # via # -r requirements/pip.txt # celery -boto3==1.28.53 +boto3==1.28.54 # via # -r requirements/pip.txt # django-storages -botocore==1.31.53 +botocore==1.31.54 # via # -r requirements/pip.txt # boto3 @@ -40,7 +40,7 @@ celery==5.2.7 certifi==2023.7.22 # via # -r requirements/pip.txt - # elasticsearch + # elastic-transport # requests cffi==1.15.1 # via @@ -136,7 +136,7 @@ django-debug-toolbar==4.2.0 # via -r requirements/pip.txt django-dynamic-fixture==4.0.1 # via -r requirements/testing.in -django-elasticsearch-dsl==7.3 +django-elasticsearch-dsl @ git+https://github.com/oehrlein/django-elasticsearch-dsl.git@elasticsearch-8 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -188,11 +188,15 @@ drf-extensions==0.7.1 # via -r requirements/pip.txt drf-flex-fields==1.0.2 # via -r requirements/pip.txt -elasticsearch==7.17.9 +elastic-transport==8.4.1 + # via + # -r requirements/pip.txt + # elasticsearch +elasticsearch==8.10.0 # via # -r requirements/pip.txt # elasticsearch-dsl -elasticsearch-dsl==7.4.1 +elasticsearch-dsl==8.9.0 # via # -r requirements/pip.txt # django-elasticsearch-dsl @@ -355,7 +359,6 @@ six==1.16.0 # -r requirements/pip.txt # django-annoying # django-elasticsearch-dsl - # elasticsearch-dsl # python-dateutil # requests-mock # unicode-slugify @@ -425,7 +428,7 @@ urllib3==1.26.16 # -r requirements/pip.txt # botocore # docker - # elasticsearch + # elastic-transport # requests user-agents==2.2.0 # via -r requirements/pip.txt From c200b0a88cbab3b4d1ced624d5240fdc1127c7d8 Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 25 Sep 2023 19:25:10 -0500 Subject: [PATCH 2/4] Forgot about this --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 17b805950e6..2d21a822d55 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,7 +11,7 @@ jobs: environment: TOX_POSARGS: '' PYTEST_COVERAGE: --cov-report=xml --cov-config .coveragerc --cov=. --cov-append - - image: 'docker.elastic.co/elasticsearch/elasticsearch:7.17.13' + - image: 'docker.elastic.co/elasticsearch/elasticsearch:8.10.2' name: search environment: discovery.type: single-node From 20bf3259676eef7b298f17399e3cbc616bf449ca Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Tue, 26 Sep 2023 09:18:45 -0500 Subject: [PATCH 3/4] Fix tests --- .circleci/config.yml | 2 ++ readthedocs/search/tests/test_faceted_search.py | 4 ++-- readthedocs/search/tests/test_views.py | 4 ++-- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 2d21a822d55..a6387736db0 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -17,6 +17,8 @@ jobs: discovery.type: single-node ES_JAVA_OPTS: -Xms750m -Xmx750m ELASTIC_PASSWORD: password + # Disabled SSL for testing. + xpack.security.transport.ssl.enabled: 'false' steps: - checkout - run: git submodule sync diff --git a/readthedocs/search/tests/test_faceted_search.py b/readthedocs/search/tests/test_faceted_search.py index b4b4d9c59e8..54406d44f9c 100644 --- a/readthedocs/search/tests/test_faceted_search.py +++ b/readthedocs/search/tests/test_faceted_search.py @@ -26,11 +26,11 @@ def test_search_exact_match(self, client, project, case): assert len(results) == 2 assert results[0]["project"] == "kuma" assert results[0]["path"] == "testdocumentation" - assert results[0]["version"] == "stable" + assert results[0]["version"] == "latest" assert results[1]["project"] == "kuma" assert results[1]["path"] == "testdocumentation" - assert results[1]["version"] == "latest" + assert results[1]["version"] == "stable" def test_search_combined_result(self, client, project): """Check search result are combined of both `AND` and `OR` operator diff --git a/readthedocs/search/tests/test_views.py b/readthedocs/search/tests/test_views.py index 213467aaa19..3ad1c708c9b 100644 --- a/readthedocs/search/tests/test_views.py +++ b/readthedocs/search/tests/test_views.py @@ -260,8 +260,8 @@ def test_file_search_exact_match(self, client, project): # There are two results, # one from each version of the "kuma" project. assert len(results) == 2 - assert results[0]["version"] == {"slug": "stable"} - assert results[1]["version"] == {"slug": "latest"} + assert results[0]["version"] == {"slug": "latest"} + assert results[1]["version"] == {"slug": "stable"} for result in results: assert result["project"] == {"alias": None, "slug": "kuma"} assert result["domain"] == "http://readthedocs.org" From 5c57999945f9c3cf595d4c9788b11628283f7a1b Mon Sep 17 00:00:00 2001 From: Santos Gallegos Date: Mon, 9 Oct 2023 12:48:23 -0500 Subject: [PATCH 4/4] Upgrade dependencies --- requirements/deploy.txt | 8 ++++---- requirements/docker.txt | 8 ++++---- requirements/pip.txt | 8 ++++---- requirements/testing.txt | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/requirements/deploy.txt b/requirements/deploy.txt index 491a55078f1..9c0ed9c59b6 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -135,7 +135,7 @@ django-csp==3.7 # via -r requirements/pip.txt django-debug-toolbar==4.2.0 # via -r requirements/pip.txt -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -155,7 +155,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.txt django-simple-history==3.0.0 # via -r requirements/pip.txt -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.txt django-structlog==2.2.0 # via -r requirements/pip.txt @@ -360,7 +360,7 @@ stripe==4.2.0 # via # -r requirements/pip.txt # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.txt # django-structlog @@ -415,7 +415,7 @@ wcwidth==0.2.8 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements/pip.txt # docker diff --git a/requirements/docker.txt b/requirements/docker.txt index 16eb991d8b8..b62d4a4383f 100644 --- a/requirements/docker.txt +++ b/requirements/docker.txt @@ -146,7 +146,7 @@ django-csp==3.7 # via -r requirements/pip.txt django-debug-toolbar==4.2.0 # via -r requirements/pip.txt -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -166,7 +166,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.txt django-simple-history==3.0.0 # via -r requirements/pip.txt -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.txt django-structlog==2.2.0 # via -r requirements/pip.txt @@ -393,7 +393,7 @@ stripe==4.2.0 # via # -r requirements/pip.txt # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.txt # django-structlog @@ -451,7 +451,7 @@ wcwidth==0.2.8 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements/pip.txt # docker diff --git a/requirements/pip.txt b/requirements/pip.txt index b64e67c1a10..332bc225a92 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -95,7 +95,7 @@ django-csp==3.7 # via -r requirements/pip.in django-debug-toolbar==4.2.0 # via -r requirements/pip.in -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.in django-extensions==3.2.3 # via -r requirements/pip.in @@ -113,7 +113,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.in django-simple-history==3.0.0 # via -r requirements/pip.in -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.in django-structlog==2.2.0 # via -r requirements/pip.in @@ -255,7 +255,7 @@ stripe==4.2.0 # via # -r requirements/pip.in # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.in # django-structlog @@ -290,7 +290,7 @@ virtualenv==20.24.5 # via -r requirements/pip.in wcwidth==0.2.8 # via prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via docker # The following packages are considered to be unsafe in a requirements file: diff --git a/requirements/testing.txt b/requirements/testing.txt index 8e4d002f16d..84196625fa5 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -136,7 +136,7 @@ django-debug-toolbar==4.2.0 # via -r requirements/pip.txt django-dynamic-fixture==4.0.1 # via -r requirements/testing.in -django-elasticsearch-dsl==7.4 +django-elasticsearch-dsl==8.0 # via -r requirements/pip.txt django-extensions==3.2.3 # via -r requirements/pip.txt @@ -156,7 +156,7 @@ django-polymorphic==3.1.0 # via -r requirements/pip.txt django-simple-history==3.0.0 # via -r requirements/pip.txt -django-storages[boto3]==1.14.1 +django-storages[boto3]==1.14.2 # via -r requirements/pip.txt django-structlog==2.2.0 # via -r requirements/pip.txt @@ -395,7 +395,7 @@ stripe==4.2.0 # via # -r requirements/pip.txt # dj-stripe -structlog==23.1.0 +structlog==23.2.0 # via # -r requirements/pip.txt # django-structlog @@ -444,7 +444,7 @@ wcwidth==0.2.8 # via # -r requirements/pip.txt # prompt-toolkit -websocket-client==1.6.3 +websocket-client==1.6.4 # via # -r requirements/pip.txt # docker