From c83cca662ae611cbdf8ec02e68c4f18d717854f3 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Wed, 7 Nov 2018 02:27:44 +0600 Subject: [PATCH 01/16] WIP: compatable with latest elasticsearch-dsl --- django_elasticsearch_dsl/documents.py | 86 +++---------------- django_elasticsearch_dsl/indices.py | 4 +- .../management/commands/search_index.py | 2 +- django_elasticsearch_dsl/registries.py | 62 +++++++++++-- requirements_dev.txt | 2 + requirements_test.txt | 1 + runtests.py | 2 +- tests/documents.py | 57 +++++++----- tests/fixtures.py | 14 +-- tests/test_documents.py | 35 +++++--- tests/test_indices.py | 1 + tox.ini | 2 +- 12 files changed, 141 insertions(+), 127 deletions(-) diff --git a/django_elasticsearch_dsl/documents.py b/django_elasticsearch_dsl/documents.py index e0df4155..2b33c7c3 100644 --- a/django_elasticsearch_dsl/documents.py +++ b/django_elasticsearch_dsl/documents.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals +from copy import deepcopy + from django.db import models from django.core.paginator import Paginator from django.utils.six import add_metaclass, iteritems from elasticsearch.helpers import bulk -from elasticsearch_dsl import DocType as DSLDocType -from elasticsearch_dsl.document import DocTypeMeta as DSLDocTypeMeta +from elasticsearch_dsl import Document as DSLDocument from elasticsearch_dsl.field import Field from .apps import DEDConfig @@ -50,70 +51,7 @@ } -class DocTypeMeta(DSLDocTypeMeta): - def __new__(cls, name, bases, attrs): - """ - Subclass default DocTypeMeta to generate ES fields from django - models fields - """ - super_new = super(DocTypeMeta, cls).__new__ - - parents = [b for b in bases if isinstance(b, DocTypeMeta)] - if not parents: - return super_new(cls, name, bases, attrs) - - model = attrs['Meta'].model - - ignore_signals = getattr(attrs['Meta'], "ignore_signals", False) - auto_refresh = getattr( - attrs['Meta'], 'auto_refresh', DEDConfig.auto_refresh_enabled() - ) - model_field_names = getattr(attrs['Meta'], "fields", []) - related_models = getattr(attrs['Meta'], "related_models", []) - queryset_pagination = getattr( - attrs['Meta'], "queryset_pagination", None - ) - - class_fields = set( - name for name, field in iteritems(attrs) - if isinstance(field, Field) - ) - - cls = super_new(cls, name, bases, attrs) - - cls._doc_type.model = model - cls._doc_type.ignore_signals = ignore_signals - cls._doc_type.auto_refresh = auto_refresh - cls._doc_type.related_models = related_models - cls._doc_type.queryset_pagination = queryset_pagination - - fields = model._meta.get_fields() - fields_lookup = dict((field.name, field) for field in fields) - - for field_name in model_field_names: - if field_name in class_fields: - raise RedeclaredFieldError( - "You cannot redeclare the field named '{}' on {}" - .format(field_name, cls.__name__) - ) - - field_instance = cls.to_field(field_name, - fields_lookup[field_name]) - cls._doc_type.mapping.field(field_name, field_instance) - - cls._doc_type._fields = ( - lambda: cls._doc_type.mapping.properties.properties.to_dict()) - - if getattr(cls._doc_type, 'index'): - index = Index(cls._doc_type.index) - index.doc_type(cls) - registry.register(index, cls) - - return cls - - -@add_metaclass(DocTypeMeta) -class DocType(DSLDocType): +class DocType(DSLDocument): def __init__(self, related_instance_to_ignore=None, **kwargs): super(DocType, self).__init__(**kwargs) self._related_instance_to_ignore = related_instance_to_ignore @@ -130,14 +68,14 @@ def search(cls, using=None, index=None): using=using or cls._doc_type.using, index=index or cls._doc_type.index, doc_type=[cls], - model=cls._doc_type.model + model=cls.django.model ) def get_queryset(self): """ Return the queryset that should be indexed by this doc type. """ - return self._doc_type.model._default_manager.all() + return self.django.model._default_manager.all() def prepare(self, instance): """ @@ -145,7 +83,7 @@ def prepare(self, instance): based on the fields defined on this DocType subclass """ data = {} - for name, field in iteritems(self._doc_type._fields()): + for name, field in iteritems(instance._doc_type.mapping): if not isinstance(field, DEDField): continue @@ -188,12 +126,12 @@ def to_field(cls, field_name, model_field): ) def bulk(self, actions, **kwargs): - return bulk(client=self.connection, actions=actions, **kwargs) + return bulk(client=self._get_connection, actions=actions, **kwargs) def _prepare_action(self, object_instance, action): return { '_op_type': action, - '_index': str(self._doc_type.index), + '_index': str(self._index), '_type': self._doc_type.mapping.doc_type, '_id': object_instance.pk, '_source': ( @@ -202,9 +140,9 @@ def _prepare_action(self, object_instance, action): } def _get_actions(self, object_list, action): - if self._doc_type.queryset_pagination is not None: + if self.django.queryset_pagination is not None: paginator = Paginator( - object_list, self._doc_type.queryset_pagination + object_list, self.django.queryset_pagination ) for page in paginator.page_range: for object_instance in paginator.page(page).object_list: @@ -218,7 +156,7 @@ def update(self, thing, refresh=None, action='index', **kwargs): Update each document in ES for a model, iterable of models or queryset """ if refresh is True or ( - refresh is None and self._doc_type.auto_refresh + refresh is None and self.django.auto_refresh ): kwargs['refresh'] = True diff --git a/django_elasticsearch_dsl/indices.py b/django_elasticsearch_dsl/indices.py index 1d208e0d..4dba3286 100644 --- a/django_elasticsearch_dsl/indices.py +++ b/django_elasticsearch_dsl/indices.py @@ -9,8 +9,8 @@ @python_2_unicode_compatible class Index(DSLIndex): - def __init__(self, name, using='default'): - super(Index, self).__init__(name, using) + def __init__(self, *args, **kwargs): + super(Index, self).__init__(*args, **kwargs) self._settings = deepcopy(DEDConfig.default_index_settings()) def doc_type(self, doc_type, *args, **kwargs): diff --git a/django_elasticsearch_dsl/management/commands/search_index.py b/django_elasticsearch_dsl/management/commands/search_index.py index f6f505f4..20179bcc 100644 --- a/django_elasticsearch_dsl/management/commands/search_index.py +++ b/django_elasticsearch_dsl/management/commands/search_index.py @@ -87,7 +87,7 @@ def _populate(self, models, options): for doc in registry.get_documents(models): qs = doc().get_queryset() self.stdout.write("Indexing {} '{}' objects".format( - qs.count(), doc._doc_type.model.__name__) + qs.count(), doc.django.model.__name__) ) doc().update(qs) diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index 89701019..260ef0aa 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -2,8 +2,11 @@ from itertools import chain from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ImproperlyConfigured from django.utils.six import itervalues, iterkeys, iteritems +from elasticsearch_dsl import Field, AttrDict +from django_elasticsearch_dsl.exceptions import RedeclaredFieldError from .apps import DEDConfig @@ -18,10 +21,10 @@ def __init__(self): def register(self, index, doc_class): """Register the model with the registry""" - self._models[doc_class._doc_type.model].add(doc_class) + self._models[doc_class.django.model].add(doc_class) - for related in doc_class._doc_type.related_models: - self._related_models[related].add(doc_class._doc_type.model) + for related in doc_class.django.related_models: + self._related_models[related].add(doc_class.django.model) for idx, docs in iteritems(self._indices): if index._name == idx._name: @@ -30,10 +33,57 @@ def register(self, index, doc_class): self._indices[index].add(doc_class) + def register_document(self, document): + django_meta = getattr(document, 'Django') + # Raise error if Django class can not be found + if not django_meta: + message = "You must declare the Django class inside {}".format(document.__name__) + raise ImproperlyConfigured(message) + + # Keep all django related attribute in a django_attr AttrDict + data = {'model': getattr(document.Django, 'model')} + django_attr = AttrDict(data) + + if not django_attr.model: + raise ImproperlyConfigured("You must specify the django model") + + # Add The model fields into elasticsearch mapping field + model_field_names = getattr(document.Django, "fields", []) + class_fields = set( + name for name, field in document.__dict__.items() + if isinstance(field, Field) + ) + + for field_name in model_field_names: + if field_name in class_fields: + raise RedeclaredFieldError( + "You cannot redeclare the field named '{}' on {}" + .format(field_name, document.__name__) + ) + + django_field = django_attr.model._meta.get_field(field_name) + + field_instance = document.to_field(field_name, django_field) + document._doc_type.mapping.field(field_name, field_instance) + + django_attr.ignore_signals = getattr(django_meta, "ignore_signals", False) + django_attr.auto_refresh = getattr(django_meta, + "auto_refresh", DEDConfig.auto_refresh_enabled()) + django_attr.related_models = getattr(django_meta, "related_models", []) + django_attr.queryset_pagination = getattr(django_meta, "queryset_pagination", None) + + # Add django attribute in the document class with all the django attribute + setattr(document, 'django', django_attr) + + # Register the document and index class to our registry + self.register(index=document._index, doc_class=document) + + return document + def _get_related_doc(self, instance): for model in self._related_models.get(instance.__class__, []): for doc in self._models[model]: - if instance.__class__ in doc._doc_type.related_models: + if instance.__class__ in doc.django.related_models: yield doc def update_related(self, instance, **kwargs): @@ -80,7 +130,7 @@ def update(self, instance, **kwargs): if instance.__class__ in self._models: for doc in self._models[instance.__class__]: - if not doc._doc_type.ignore_signals: + if not doc.django.ignore_signals: doc().update(instance, **kwargs) def delete(self, instance, **kwargs): @@ -112,7 +162,7 @@ def get_indices(self, models=None): if models is not None: return set( indice for indice, docs in iteritems(self._indices) - for doc in docs if doc._doc_type.model in models + for doc in docs if doc.django.model in models ) return set(iterkeys(self._indices)) diff --git a/requirements_dev.txt b/requirements_dev.txt index 05be29ad..41164ffe 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,4 +3,6 @@ wheel==0.32.2 django>=2.0,<2.2 elasticsearch-dsl>=2.1.0,<6.2.0 twine +wheel==0.29.0 +django>=2.0,<2.1 -e . diff --git a/requirements_test.txt b/requirements_test.txt index 1afa9d74..01b39764 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -6,3 +6,4 @@ Pillow==4.2.1 # Additional test requirements go here +https://github.com/elastic/elasticsearch-dsl-py/archive/master.tar.gz diff --git a/runtests.py b/runtests.py index 157374a0..6a6679f6 100644 --- a/runtests.py +++ b/runtests.py @@ -27,7 +27,7 @@ def get_settings(): ELASTICSEARCH_DSL={ 'default': { 'hosts': os.environ.get('ELASTICSEARCH_URL', - 'localhost:9200') + '127.0.0.1:9200') }, }, ) diff --git a/tests/documents.py b/tests/documents.py index cd9d1213..e1a093ec 100644 --- a/tests/documents.py +++ b/tests/documents.py @@ -1,5 +1,6 @@ from elasticsearch_dsl import analyzer from django_elasticsearch_dsl import DocType, Index, fields +from django_elasticsearch_dsl.registries import registry from .models import Ad, Category, Car, Manufacturer @@ -9,9 +10,6 @@ } -car_index = Index('test_cars').settings(**index_settings) - - html_strip = analyzer( 'html_strip', tokenizer="standard", @@ -20,7 +18,7 @@ ) -@car_index.doc_type +@registry.register_document class CarDocument(DocType): # test can override __init__ def __init__(self, *args, **kwargs): @@ -43,7 +41,7 @@ def __init__(self, *args, **kwargs): 'icon': fields.FileField(), }) - class Meta: + class Django: model = Car related_models = [Ad, Manufacturer, Category] fields = [ @@ -51,7 +49,10 @@ class Meta: 'launched', 'type', ] - doc_type = 'car_document' + + class Index: + name = 'test_cars' + settings = index_settings def get_queryset(self): return super(CarDocument, self).get_queryset().select_related( @@ -65,14 +66,11 @@ def get_instances_from_related(self, related_instance): return related_instance.car_set.all() -manufacturer_index = Index('test_manufacturers').settings(**index_settings) - - -@manufacturer_index.doc_type +@registry.register_document class ManufacturerDocument(DocType): country = fields.StringField() - class Meta: + class Django: model = Manufacturer fields = [ 'name', @@ -80,9 +78,13 @@ class Meta: 'country_code', 'logo', ] - doc_type = 'manufacturer_document' + + class Index: + name = 'index_settings' + settings = index_settings +@registry.register_document class CarWithPrepareDocument(DocType): manufacturer = fields.ObjectField(properties={ 'name': fields.StringField(), @@ -93,16 +95,18 @@ class CarWithPrepareDocument(DocType): 'name': fields.StringField(), }) - class Meta: + class Django: model = Car related_models = [Manufacturer] - index = 'car_with_prepare_index' fields = [ 'name', 'launched', 'type', ] + class Index: + name = 'car_with_prepare_index' + def prepare_manufacturer_with_related(self, car, related_to_ignore): if (car.manufacturer is not None and car.manufacturer != related_to_ignore): @@ -123,17 +127,14 @@ def get_instances_from_related(self, related_instance): return related_instance.car_set.all() -ad_index = Index('test_ads').settings(**index_settings) - - -@ad_index.doc_type +@registry.register_document class AdDocument(DocType): description = fields.TextField( analyzer=html_strip, fields={'raw': fields.KeywordField()} ) - class Meta: + class Django: model = Ad fields = [ 'title', @@ -141,13 +142,17 @@ class Meta: 'modified', 'url', ] - doc_type = 'ad_document' + class Index: + name = 'test_ads' + settings = index_settings + +@registry.register_document class PaginatedAdDocument(DocType): - class Meta: + + class Django: model = Ad - index = 'ad_index' queryset_pagination = 2 fields = [ 'title', @@ -155,7 +160,13 @@ class Meta: 'modified', 'url', ] - doc_type = 'paginated_ad_document' + + class Index: + name = 'ad_index' def get_queryset(self): return Ad.objects.all().order_by('-id') + + +ad_index = AdDocument._index +car_index = CarDocument._index diff --git a/tests/fixtures.py b/tests/fixtures.py index 521eb4a0..674a5f25 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -29,12 +29,17 @@ def _generate_doc_mock( self, _model, index=None, mock_qs=None, _ignore_signals=False, _related_models=None ): + _index = index + class Doc(DocType): class Meta: - model = _model + index = _index ignore_signals = _ignore_signals - related_models = _related_models if ( - _related_models) is not None else [] + + class Django: + model = _model + related_models = _related_models if _related_models is not None else [] + Doc.update = Mock() if mock_qs: @@ -42,7 +47,6 @@ class Meta: if _related_models: Doc.get_instances_from_related = Mock() - if index: - self.registry.register(index, Doc) + self.registry.register_document(document=Doc) return Doc diff --git a/tests/test_documents.py b/tests/test_documents.py index 637dedf0..2c7f3f49 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -2,13 +2,14 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ -from elasticsearch_dsl import GeoPoint +from elasticsearch_dsl import GeoPoint, MetaField from mock import patch from django_elasticsearch_dsl import fields from django_elasticsearch_dsl.documents import DocType from django_elasticsearch_dsl.exceptions import (ModelFieldNotMappedError, RedeclaredFieldError) +from django_elasticsearch_dsl.registries import registry from tests import ES_MAJOR_VERSION @@ -34,6 +35,7 @@ class Meta: app_label = 'car' +@registry.register_document class CarDocument(DocType): color = fields.TextField() type = fields.StringField() @@ -42,35 +44,40 @@ def prepare_color(self, instance): return "blue" class Meta: + doc_type = MetaField('car_document') + + class Django: fields = ['name', 'price'] model = Car - index = 'car_index' related_models = [Manufacturer] + + class Index: + name = 'car_index' doc_type = 'car_document' class DocTypeTestCase(TestCase): def test_model_class_added(self): - self.assertEqual(CarDocument._doc_type.model, Car) + self.assertEqual(CarDocument.django.model, Car) def test_ignore_signal_default(self): - self.assertFalse(CarDocument._doc_type.ignore_signals) + self.assertFalse(CarDocument.django.ignore_signals) def test_auto_refresh_default(self): - self.assertTrue(CarDocument._doc_type.auto_refresh) + self.assertTrue(CarDocument.django.auto_refresh) def test_ignore_signal_added(self): class CarDocument2(DocType): - class Meta: + class Django: model = Car ignore_signals = True - self.assertTrue(CarDocument2._doc_type.ignore_signals) + self.assertTrue(CarDocument2.django.ignore_signals) def test_auto_refresh_added(self): class CarDocument2(DocType): - class Meta: + class Django: model = Car auto_refresh = False @@ -78,12 +85,12 @@ class Meta: def test_queryset_pagination_added(self): class CarDocument2(DocType): - class Meta: + class Django: model = Car queryset_pagination = 120 - self.assertIsNone(CarDocument._doc_type.queryset_pagination) - self.assertEqual(CarDocument2._doc_type.queryset_pagination, 120) + self.assertIsNone(CarDocument.django.queryset_pagination) + self.assertEqual(CarDocument2.django.queryset_pagination, 120) def test_fields_populated(self): mapping = CarDocument._doc_type.mapping @@ -93,7 +100,7 @@ def test_fields_populated(self): ) def test_related_models_added(self): - related_models = CarDocument._doc_type.related_models + related_models = CarDocument.django.related_models self.assertEqual([Manufacturer], related_models) def test_duplicate_field_names_not_allowed(self): @@ -121,7 +128,7 @@ def test_mapping(self): text_type = 'string' if ES_MAJOR_VERSION == 2 else 'text' self.assertEqual( - CarDocument._doc_type.mapping.to_dict(), { + CarDocument.meta.mapping.to_dict(), { 'car_document': { 'properties': { 'name': { @@ -256,7 +263,7 @@ def test_model_instance_update_no_refresh(self): def test_model_instance_iterable_update_with_pagination(self): class CarDocument2(DocType): - class Meta: + class Django: model = Car queryset_pagination = 2 diff --git a/tests/test_indices.py b/tests/test_indices.py index 2035db2b..b9ff69eb 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -16,6 +16,7 @@ def test_documents_add_to_register(self): index = Index('test') doc_a1 = self._generate_doc_mock(self.ModelA) doc_a2 = self._generate_doc_mock(self.ModelA) + print(index._mapping) index.doc_type(doc_a1) docs = list(registry.get_documents()) self.assertEqual(len(docs), 1) diff --git a/tox.ini b/tox.ini index 0259587e..5bec62a2 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = django-21: Django>=2.1,<2.2 es2: elasticsearch-dsl>=2.1.0,<3.0.0 es5: elasticsearch-dsl>=5.0.0,<6.0.0 - es6: elasticsearch-dsl>=6.0.0,<7.0.0 + es6: https://github.com/elastic/elasticsearch-dsl-py/archive/master.tar.gz -r{toxinidir}/requirements_test.txt basepython = From 03290025827edc66ed49fe0a6357fdfcdc9fd4b8 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Wed, 7 Nov 2018 03:18:30 +0600 Subject: [PATCH 02/16] more fixup --- django_elasticsearch_dsl/documents.py | 2 +- django_elasticsearch_dsl/registries.py | 4 ++++ tests/fixtures.py | 10 +++++----- tests/test_documents.py | 18 +++++++++++++----- 4 files changed, 23 insertions(+), 11 deletions(-) diff --git a/django_elasticsearch_dsl/documents.py b/django_elasticsearch_dsl/documents.py index 2b33c7c3..52752424 100644 --- a/django_elasticsearch_dsl/documents.py +++ b/django_elasticsearch_dsl/documents.py @@ -83,7 +83,7 @@ def prepare(self, instance): based on the fields defined on this DocType subclass """ data = {} - for name, field in iteritems(instance._doc_type.mapping): + for name, field in iteritems(self._fields): if not isinstance(field, DEDField): continue diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index 260ef0aa..f52cb727 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -75,6 +75,10 @@ def register_document(self, document): # Add django attribute in the document class with all the django attribute setattr(document, 'django', django_attr) + # Set the fields of the mappings + fields = document._doc_type.mapping.properties.properties.to_dict() + setattr(document, '_fields', fields) + # Register the document and index class to our registry self.register(index=document._index, doc_class=document) diff --git a/tests/fixtures.py b/tests/fixtures.py index 674a5f25..5b7ae69b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -3,6 +3,7 @@ from django.db import models from django_elasticsearch_dsl.documents import DocType +from django_elasticsearch_dsl.registries import registry class WithFixturesMixin(object): @@ -31,14 +32,15 @@ def _generate_doc_mock( ): _index = index + @registry.register_document class Doc(DocType): - class Meta: - index = _index - ignore_signals = _ignore_signals + class Index: + name = _index class Django: model = _model related_models = _related_models if _related_models is not None else [] + ignore_signals = _ignore_signals Doc.update = Mock() @@ -47,6 +49,4 @@ class Django: if _related_models: Doc.get_instances_from_related = Mock() - self.registry.register_document(document=Doc) - return Doc diff --git a/tests/test_documents.py b/tests/test_documents.py index 2c7f3f49..214162bf 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -68,6 +68,8 @@ def test_auto_refresh_default(self): self.assertTrue(CarDocument.django.auto_refresh) def test_ignore_signal_added(self): + + @registry.register_document class CarDocument2(DocType): class Django: model = Car @@ -76,14 +78,16 @@ class Django: self.assertTrue(CarDocument2.django.ignore_signals) def test_auto_refresh_added(self): + @registry.register_document class CarDocument2(DocType): class Django: model = Car auto_refresh = False - self.assertFalse(CarDocument2._doc_type.auto_refresh) + self.assertFalse(CarDocument2.django.auto_refresh) def test_queryset_pagination_added(self): + @registry.register_document class CarDocument2(DocType): class Django: model = Car @@ -105,11 +109,12 @@ def test_related_models_added(self): def test_duplicate_field_names_not_allowed(self): with self.assertRaises(RedeclaredFieldError): + @registry.register_document class CarDocument(DocType): color = fields.StringField() name = fields.StringField() - class Meta: + class Django: fields = ['name'] model = Car @@ -128,7 +133,7 @@ def test_mapping(self): text_type = 'string' if ES_MAJOR_VERSION == 2 else 'text' self.assertEqual( - CarDocument.meta.mapping.to_dict(), { + CarDocument._doc_type.mapping.to_dict(), { 'car_document': { 'properties': { 'name': { @@ -167,14 +172,17 @@ def test_prepare(self): ) def test_prepare_ignore_dsl_base_field(self): + @registry.register_document class CarDocumentDSlBaseField(DocType): position = GeoPoint() - class Meta: + class Django: model = Car - index = 'car_index' fields = ['name', 'price'] + class Index: + name = 'car_index' + car = Car(name="Type 57", price=5400000.0, not_indexed="not_indexex") doc = CarDocumentDSlBaseField() prepared_data = doc.prepare(car) From 2965be37bfcfe986e618de0453a64553e3cc39fb Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 29 Dec 2018 03:29:36 +0600 Subject: [PATCH 03/16] fixing some tests --- django_elasticsearch_dsl/registries.py | 9 +++++++++ tests/fixtures.py | 9 +++------ tests/test_registries.py | 10 +++++----- 3 files changed, 17 insertions(+), 11 deletions(-) diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index f52cb727..0b08ebcc 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -27,10 +27,14 @@ def register(self, index, doc_class): self._related_models[related].add(doc_class.django.model) for idx, docs in iteritems(self._indices): + # import pdb + # pdb.set_trace() if index._name == idx._name: docs.add(doc_class) return + print(self._indices, index) + self._indices[index].add(doc_class) def register_document(self, document): @@ -97,6 +101,11 @@ def update_related(self, instance, **kwargs): if not DEDConfig.autosync_enabled(): return + # print(list(self._get_related_doc(instance))) + # import pdb + # pdb.set_trace() + print(instance.__class__, "ffff") + for doc in self._get_related_doc(instance): doc_instance = doc() try: diff --git a/tests/fixtures.py b/tests/fixtures.py index 5b7ae69b..3b7a87bd 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -3,7 +3,6 @@ from django.db import models from django_elasticsearch_dsl.documents import DocType -from django_elasticsearch_dsl.registries import registry class WithFixturesMixin(object): @@ -26,13 +25,11 @@ class ModelD(models.Model): class ModelE(models.Model): pass - def _generate_doc_mock( - self, _model, index=None, mock_qs=None, - _ignore_signals=False, _related_models=None - ): + def _generate_doc_mock(self, _model, index=None, mock_qs=None, + _ignore_signals=False, _related_models=None): _index = index - @registry.register_document + @self.registry.register_document class Doc(DocType): class Index: name = _index diff --git a/tests/test_registries.py b/tests/test_registries.py index c7282963..c72894cb 100644 --- a/tests/test_registries.py +++ b/tests/test_registries.py @@ -3,6 +3,7 @@ from django.conf import settings +from django_elasticsearch_dsl import Index from django_elasticsearch_dsl.registries import DocumentRegistry from .fixtures import WithFixturesMixin @@ -11,8 +12,8 @@ class DocumentRegistryTestCase(WithFixturesMixin, TestCase): def setUp(self): self.registry = DocumentRegistry() - self.index_1 = Mock() - self.index_2 = Mock() + self.index_1 = Index(name='index_1') + self.index_2 = Index(name='index_2') self.doc_a1 = self._generate_doc_mock(self.ModelA, self.index_1) self.doc_a2 = self._generate_doc_mock(self.ModelA, self.index_1) @@ -111,9 +112,8 @@ def test_update_related_instances(self): doc_d2.update.assert_not_called() def test_update_related_instances_not_defined(self): - doc_d1 = self._generate_doc_mock( - self.ModelD, self.index_1, _related_models=[self.ModelE] - ) + doc_d1 = self._generate_doc_mock(_model=self.ModelD, index=self.index_1, + _related_models=[self.ModelE]) instance = self.ModelE() From 31b6ecbc63bc49da172e3085bb9285cf860918ca Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Mon, 31 Dec 2018 06:56:54 +0600 Subject: [PATCH 04/16] fixing more tests --- django_elasticsearch_dsl/registries.py | 4 ---- requirements_test.txt | 2 +- tests/fixtures.py | 3 +-- tox.ini | 2 +- 4 files changed, 3 insertions(+), 8 deletions(-) diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index 0b08ebcc..41e0c1a0 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -27,14 +27,10 @@ def register(self, index, doc_class): self._related_models[related].add(doc_class.django.model) for idx, docs in iteritems(self._indices): - # import pdb - # pdb.set_trace() if index._name == idx._name: docs.add(doc_class) return - print(self._indices, index) - self._indices[index].add(doc_class) def register_document(self, document): diff --git a/requirements_test.txt b/requirements_test.txt index 01b39764..06a3e6d7 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -6,4 +6,4 @@ Pillow==4.2.1 # Additional test requirements go here -https://github.com/elastic/elasticsearch-dsl-py/archive/master.tar.gz +https://github.com/safwanrahman/elasticsearch-dsl-py/archive/document_method.tar.gz#egg=elasticsearch-dsl-py==6.3.3 diff --git a/tests/fixtures.py b/tests/fixtures.py index 3b7a87bd..da0b2b3f 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -30,9 +30,8 @@ def _generate_doc_mock(self, _model, index=None, mock_qs=None, _index = index @self.registry.register_document + @_index.document class Doc(DocType): - class Index: - name = _index class Django: model = _model diff --git a/tox.ini b/tox.ini index 5bec62a2..7caaf0cd 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ deps = django-21: Django>=2.1,<2.2 es2: elasticsearch-dsl>=2.1.0,<3.0.0 es5: elasticsearch-dsl>=5.0.0,<6.0.0 - es6: https://github.com/elastic/elasticsearch-dsl-py/archive/master.tar.gz + es6: https://github.com/safwanrahman/elasticsearch-dsl-py/archive/document_method.tar.gz#egg=elasticsearch-dsl-py==6.3.3 -r{toxinidir}/requirements_test.txt basepython = From c0eda4c2409aaa200a29ab622b91310be6894463 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Mon, 31 Dec 2018 07:49:52 +0600 Subject: [PATCH 05/16] more test fix --- django_elasticsearch_dsl/documents.py | 4 ++-- tests/test_documents.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/django_elasticsearch_dsl/documents.py b/django_elasticsearch_dsl/documents.py index 52752424..e4a2da7a 100644 --- a/django_elasticsearch_dsl/documents.py +++ b/django_elasticsearch_dsl/documents.py @@ -131,8 +131,8 @@ def bulk(self, actions, **kwargs): def _prepare_action(self, object_instance, action): return { '_op_type': action, - '_index': str(self._index), - '_type': self._doc_type.mapping.doc_type, + '_index': self._index._name, + '_type': self._doc_type.name, '_id': object_instance.pk, '_source': ( self.prepare(object_instance) if action != 'delete' else None diff --git a/tests/test_documents.py b/tests/test_documents.py index 214162bf..2cddb45f 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -44,7 +44,7 @@ def prepare_color(self, instance): return "blue" class Meta: - doc_type = MetaField('car_document') + doc_type = 'car_document' class Django: fields = ['name', 'price'] @@ -217,7 +217,7 @@ def test_model_instance_update(self): ) self.assertTrue(mock.call_args_list[0][1]['refresh']) self.assertEqual( - doc.connection, mock.call_args_list[0][1]['client'] + doc._index.connection, mock.call_args_list[0][1]['client']() ) def test_model_instance_iterable_update(self): @@ -258,7 +258,7 @@ def test_model_instance_iterable_update(self): ) self.assertTrue(mock.call_args_list[0][1]['refresh']) self.assertEqual( - doc.connection, mock.call_args_list[0][1]['client'] + doc._index.connection, mock.call_args_list[0][1]['client']() ) def test_model_instance_update_no_refresh(self): From 1bfb9e2854ec30cf691c2ffd772ac51339d8e98f Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 26 Jan 2019 03:00:51 +0600 Subject: [PATCH 06/16] More fixing --- django_elasticsearch_dsl/registries.py | 7 ++----- tests/test_documents.py | 2 +- tests/test_indices.py | 10 ++++++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index 41e0c1a0..4dac024a 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -49,13 +49,10 @@ def register_document(self, document): # Add The model fields into elasticsearch mapping field model_field_names = getattr(document.Django, "fields", []) - class_fields = set( - name for name, field in document.__dict__.items() - if isinstance(field, Field) - ) + mapping_fields = document._doc_type.mapping.properties.properties.to_dict().keys() for field_name in model_field_names: - if field_name in class_fields: + if field_name in mapping_fields: raise RedeclaredFieldError( "You cannot redeclare the field named '{}' on {}" .format(field_name, document.__name__) diff --git a/tests/test_documents.py b/tests/test_documents.py index 2cddb45f..844655b6 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -263,7 +263,7 @@ def test_model_instance_iterable_update(self): def test_model_instance_update_no_refresh(self): doc = CarDocument() - doc._doc_type.auto_refresh = False + doc.django.auto_refresh = False car = Car() with patch('django_elasticsearch_dsl.documents.bulk') as mock: doc.update(car) diff --git a/tests/test_indices.py b/tests/test_indices.py index b9ff69eb..6fbff354 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -10,13 +10,15 @@ class IndexTestCase(WithFixturesMixin, TestCase): + def setUp(self): + self.registry = DocumentRegistry() + def test_documents_add_to_register(self): - registry = DocumentRegistry() + registry = self.registry with patch('django_elasticsearch_dsl.indices.registry', new=registry): index = Index('test') - doc_a1 = self._generate_doc_mock(self.ModelA) - doc_a2 = self._generate_doc_mock(self.ModelA) - print(index._mapping) + doc_a1 = self._generate_doc_mock(self.ModelA, index) + doc_a2 = self._generate_doc_mock(self.ModelA, index) index.doc_type(doc_a1) docs = list(registry.get_documents()) self.assertEqual(len(docs), 1) From bbeeb2e8811ec311b5a2928bfbd424cc7581568a Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 26 Jan 2019 22:44:26 +0600 Subject: [PATCH 07/16] Fixing more index and tests --- django_elasticsearch_dsl/indices.py | 15 +++++---- .../management/commands/search_index.py | 1 + django_elasticsearch_dsl/registries.py | 7 ++++ tests/fixtures.py | 5 +-- tests/test_commands.py | 32 +++++++++++++------ tests/test_indices.py | 8 ++--- 6 files changed, 47 insertions(+), 21 deletions(-) diff --git a/django_elasticsearch_dsl/indices.py b/django_elasticsearch_dsl/indices.py index 4dba3286..23147a25 100644 --- a/django_elasticsearch_dsl/indices.py +++ b/django_elasticsearch_dsl/indices.py @@ -11,15 +11,18 @@ class Index(DSLIndex): def __init__(self, *args, **kwargs): super(Index, self).__init__(*args, **kwargs) - self._settings = deepcopy(DEDConfig.default_index_settings()) + default_index_settings = deepcopy(DEDConfig.default_index_settings()) + self.settings(**default_index_settings) - def doc_type(self, doc_type, *args, **kwargs): + def document(self, document): """ - Extend to register the doc_type in the global document registry + Extend to register the document in the global document registry """ - doc_type = super(Index, self).doc_type(doc_type, *args, **kwargs) - registry.register(self, doc_type) - return doc_type + document = super(Index, self).document(document) + registry.register_document(document) + return document + + doc_type = document def __str__(self): return self._name diff --git a/django_elasticsearch_dsl/management/commands/search_index.py b/django_elasticsearch_dsl/management/commands/search_index.py index 20179bcc..d598404a 100644 --- a/django_elasticsearch_dsl/management/commands/search_index.py +++ b/django_elasticsearch_dsl/management/commands/search_index.py @@ -59,6 +59,7 @@ def _get_models(self, args): for arg in args: arg = arg.lower() match_found = False + print(registry.get_models()) for model in registry.get_models(): if model._meta.app_label == arg: diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index 4dac024a..fea6c821 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -1,4 +1,6 @@ from collections import defaultdict +from copy import deepcopy + from itertools import chain from django.core.exceptions import ObjectDoesNotExist @@ -76,6 +78,11 @@ def register_document(self, document): fields = document._doc_type.mapping.properties.properties.to_dict() setattr(document, '_fields', fields) + # Update settings of the document index + default_index_settings = deepcopy(DEDConfig.default_index_settings()) + document._index.settings(**default_index_settings) + print(type(document._index)) + # Register the document and index class to our registry self.register(index=document._index, doc_class=document) diff --git a/tests/fixtures.py b/tests/fixtures.py index da0b2b3f..4d3bad9e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -29,8 +29,6 @@ def _generate_doc_mock(self, _model, index=None, mock_qs=None, _ignore_signals=False, _related_models=None): _index = index - @self.registry.register_document - @_index.document class Doc(DocType): class Django: @@ -38,6 +36,9 @@ class Django: related_models = _related_models if _related_models is not None else [] ignore_signals = _ignore_signals + if _index: + _index.document(Doc) + self.registry.register_document(Doc) Doc.update = Mock() if mock_qs: diff --git a/tests/test_commands.py b/tests/test_commands.py index e32f61d3..764665d4 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -5,6 +5,7 @@ from django.core.management import call_command from django.utils.six import StringIO +from django_elasticsearch_dsl import Index from django_elasticsearch_dsl.management.commands.search_index import Command from django_elasticsearch_dsl.registries import DocumentRegistry @@ -12,11 +13,26 @@ class SearchIndexTestCase(WithFixturesMixin, TestCase): + def _mock_setup(self): + # Mock Patch object + patch_registry = patch( + 'django_elasticsearch_dsl.management.commands.search_index.registry', self.registry) + + patch_registry.start() + + methods = ['delete', 'create'] + for index in [self.index_a, self.index_b]: + for method in methods: + obj_patch = patch.object(index, method) + obj_patch.start() + + self.addCleanup(patch.stopall) + def setUp(self): self.out = StringIO() self.registry = DocumentRegistry() - self.index_a = Mock() - self.index_b = Mock() + self.index_a = Index('foo') + self.index_b = Index('bar') self.doc_a1_qs = Mock() self.doc_a1 = self._generate_doc_mock( @@ -38,6 +54,8 @@ def setUp(self): self.ModelC, self.index_b, self.doc_c1_qs ) + self._mock_setup() + def test_get_models(self): cmd = Command() with patch( @@ -97,15 +115,11 @@ def test_force_delete_all_indices(self): self.index_b.delete.assert_called_once() def test_force_delete_bar_model_c_index(self): - - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): + with patch.object(self.index_b, 'delete'): call_command('search_index', stdout=self.out, - models=['bar.ModelC'], + models=[self.ModelC._meta.label], action='delete', force=True) + print(dir(self.index_b.delete)) self.index_b.delete.assert_called_once() self.assertFalse(self.index_a.delete.called) diff --git a/tests/test_indices.py b/tests/test_indices.py index 6fbff354..11f4f5bb 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -17,14 +17,14 @@ def test_documents_add_to_register(self): registry = self.registry with patch('django_elasticsearch_dsl.indices.registry', new=registry): index = Index('test') - doc_a1 = self._generate_doc_mock(self.ModelA, index) - doc_a2 = self._generate_doc_mock(self.ModelA, index) - index.doc_type(doc_a1) + doc_a1 = self._generate_doc_mock(self.ModelA) + doc_a2 = self._generate_doc_mock(self.ModelA) + index.document(doc_a1) docs = list(registry.get_documents()) self.assertEqual(len(docs), 1) self.assertIs(docs[0], doc_a1) - index.doc_type(doc_a2) + index.document(doc_a2) docs = registry.get_documents() self.assertEqual(docs, set([doc_a1, doc_a2])) From 7a851877418546512d4b1a6e6c69707e08e8b41d Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sun, 27 Jan 2019 00:04:09 +0600 Subject: [PATCH 08/16] Fix debug --- .../management/commands/search_index.py | 1 - django_elasticsearch_dsl/registries.py | 6 - tests/test_commands.py | 116 +++++++----------- 3 files changed, 43 insertions(+), 80 deletions(-) diff --git a/django_elasticsearch_dsl/management/commands/search_index.py b/django_elasticsearch_dsl/management/commands/search_index.py index d598404a..20179bcc 100644 --- a/django_elasticsearch_dsl/management/commands/search_index.py +++ b/django_elasticsearch_dsl/management/commands/search_index.py @@ -59,7 +59,6 @@ def _get_models(self, args): for arg in args: arg = arg.lower() match_found = False - print(registry.get_models()) for model in registry.get_models(): if model._meta.app_label == arg: diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index fea6c821..1084955c 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -81,7 +81,6 @@ def register_document(self, document): # Update settings of the document index default_index_settings = deepcopy(DEDConfig.default_index_settings()) document._index.settings(**default_index_settings) - print(type(document._index)) # Register the document and index class to our registry self.register(index=document._index, doc_class=document) @@ -101,11 +100,6 @@ def update_related(self, instance, **kwargs): if not DEDConfig.autosync_enabled(): return - # print(list(self._get_related_doc(instance))) - # import pdb - # pdb.set_trace() - print(instance.__class__, "ffff") - for doc in self._get_related_doc(instance): doc_instance = doc() try: diff --git a/tests/test_commands.py b/tests/test_commands.py index 764665d4..2267d75c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -58,27 +58,22 @@ def setUp(self): def test_get_models(self): cmd = Command() - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - self.assertEqual( - cmd._get_models(['foo']), - set([self.ModelA, self.ModelB]) - ) - - self.assertEqual( - cmd._get_models(['foo', 'bar.ModelC']), - set([self.ModelA, self.ModelB, self.ModelC]) - ) - - self.assertEqual( - cmd._get_models([]), - set([self.ModelA, self.ModelB, self.ModelC]) - ) - with self.assertRaises(CommandError): - cmd._get_models(['unknown']) + self.assertEqual( + cmd._get_models(['foo']), + set([self.ModelA, self.ModelB]) + ) + + self.assertEqual( + cmd._get_models(['foo', 'bar.ModelC']), + set([self.ModelA, self.ModelB, self.ModelC]) + ) + + self.assertEqual( + cmd._get_models([]), + set([self.ModelA, self.ModelB, self.ModelC]) + ) + with self.assertRaises(CommandError): + cmd._get_models(['unknown']) def test_no_action_error(self): cmd = Command() @@ -88,68 +83,43 @@ def test_no_action_error(self): def test_delete_foo_index(self): with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry + 'django_elasticsearch_dsl.management.commands.search_index.input', + Mock(return_value="y") ): - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.input', - Mock(return_value="y") - ): - call_command('search_index', stdout=self.out, - action='delete', models=['foo']) - self.index_a.delete.assert_called_once() - self.assertFalse(self.index_b.delete.called) + call_command('search_index', stdout=self.out, + action='delete', models=['foo']) + self.index_a.delete.assert_called_once() + self.assertFalse(self.index_b.delete.called) def test_force_delete_all_indices(self): - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - call_command('search_index', stdout=self.out, - action='delete', force=True) - self.index_a.delete.assert_called_once() - self.index_b.delete.assert_called_once() + call_command('search_index', stdout=self.out, + action='delete', force=True) + self.index_a.delete.assert_called_once() + self.index_b.delete.assert_called_once() def test_force_delete_bar_model_c_index(self): - with patch.object(self.index_b, 'delete'): - call_command('search_index', stdout=self.out, - models=[self.ModelC._meta.label], - action='delete', force=True) - print(dir(self.index_b.delete)) - self.index_b.delete.assert_called_once() - self.assertFalse(self.index_a.delete.called) + call_command('search_index', stdout=self.out, + models=[self.ModelC._meta.label], + action='delete', force=True) + self.index_b.delete.assert_called_once() + self.assertFalse(self.index_a.delete.called) def test_create_all_indices(self): - - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - call_command('search_index', stdout=self.out, action='create') - self.index_a.create.assert_called_once() - self.index_b.create.assert_called_once() + call_command('search_index', stdout=self.out, action='create') + self.index_a.create.assert_called_once() + self.index_b.create.assert_called_once() def test_populate_all_doc_type(self): - - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - call_command('search_index', stdout=self.out, action='populate') - self.doc_a1.get_queryset.assert_called_once() - self.doc_a1.update.assert_called_once_with(self.doc_a1_qs) - self.doc_a2.get_queryset.assert_called_once() - self.doc_a2.update.assert_called_once_with(self.doc_a2_qs) - self.doc_b1.get_queryset.assert_called_once() - self.doc_b1.update.assert_called_once_with(self.doc_b1_qs) - self.doc_c1.get_queryset.assert_called_once() - self.doc_c1.update.assert_called_once_with(self.doc_c1_qs) + call_command('search_index', stdout=self.out, action='populate') + self.doc_a1.get_queryset.assert_called_once() + self.doc_a1.update.assert_called_once_with(self.doc_a1_qs) + self.doc_a2.get_queryset.assert_called_once() + self.doc_a2.update.assert_called_once_with(self.doc_a2_qs) + self.doc_b1.get_queryset.assert_called_once() + self.doc_b1.update.assert_called_once_with(self.doc_b1_qs) + self.doc_c1.get_queryset.assert_called_once() + self.doc_c1.update.assert_called_once_with(self.doc_c1_qs) def test_rebuild_indices(self): From 01e7eb2591b94286c74675831af6040788921e8e Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sun, 27 Jan 2019 00:23:53 +0600 Subject: [PATCH 09/16] More fixup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 40c231e2..b699e4a4 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ ], include_package_data=True, install_requires=[ - 'elasticsearch-dsl>=2.1.0,<6.2.0', + 'elasticsearch-dsl>=2.1.0', ], license="Apache Software License 2.0", zip_safe=False, From 8bccc83d2fb59f63d85ce600b87f76663fff10f5 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 9 Feb 2019 04:09:26 +0600 Subject: [PATCH 10/16] Fixup travis --- .travis.yml | 52 ---------------------------------------------------- README.rst | 17 ++++++----------- setup.py | 2 +- tox.ini | 10 ++++------ 4 files changed, 11 insertions(+), 70 deletions(-) diff --git a/.travis.yml b/.travis.yml index 68e959d9..9a70246e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,68 +7,24 @@ dist: trusty # default "precise" distro doesn't include Java 8 for Elasticsearch matrix: include: - - env: TOX_ENV=py36-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.6 - - env: TOX_ENV=py36-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.6 - env: TOX_ENV=py36-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.6 - - env: TOX_ENV=py37-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - sudo: true - dist: xenial - - env: TOX_ENV=py37-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - sudo: true - dist: xenial - env: TOX_ENV=py37-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.7 sudo: true dist: xenial - - env: TOX_ENV=py27-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 2.7 - - env: TOX_ENV=py27-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 2.7 - env: TOX_ENV=py27-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 - - env: TOX_ENV=py36-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.6 - - env: TOX_ENV=py36-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.6 - env: TOX_ENV=py36-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.6 - - env: TOX_ENV=py37-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - sudo: true - dist: xenial - - env: TOX_ENV=py37-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - sudo: true - dist: xenial - env: TOX_ENV=py37-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.7 sudo: true dist: xenial - - env: TOX_ENV=py27-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 2.7 - - env: TOX_ENV=py27-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 2.7 - env: TOX_ENV=py27-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 - - env: TOX_ENV=py36-django-2-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.6 - - env: TOX_ENV=py36-django-2-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.6 - env: TOX_ENV=py36-django-2-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.6 - - env: TOX_ENV=py37-django-2-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - sudo: true - dist: xenial - - env: TOX_ENV=py37-django-2-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - sudo: true - dist: xenial - env: TOX_ENV=py37-django-2-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.7 sudo: true @@ -77,14 +33,6 @@ matrix: python: 3.7 dist: xenial sudo: true - - env: TOX_ENV=py37-django-21-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - dist: xenial - sudo: true - - env: TOX_ENV=py37-django-21-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - dist: xenial - sudo: true cache: pip diff --git a/README.rst b/README.rst index a93a6480..f00ad5e0 100644 --- a/README.rst +++ b/README.rst @@ -27,7 +27,7 @@ Features - Django >= 1.10 - Python 2.7, 3.5, 3.6, 3.7 - - Elasticsearch >= 2.0 < 7.0 + - Elasticsearch >= 6.0 < 7.0 .. _Search: http://elasticsearch-dsl.readthedocs.io/en/stable/search_dsl.html @@ -39,13 +39,8 @@ Install Django Elasticsearch DSL:: pip install django-elasticsearch-dsl # Elasticsearch 6.x - pip install 'elasticsearch-dsl>=6.0,<6.2' + pip install 'elasticsearch-dsl>=6.3.0,<7.0' - # Elasticsearch 5.x - pip install 'elasticsearch-dsl>=5.0,<6.0' - - # Elasticsearch 2.x - pip install 'elasticsearch-dsl>=2.1,<3.0' Then add ``django_elasticsearch_dsl`` to the INSTALLED_APPS @@ -89,7 +84,7 @@ defined in a ``documents.py`` file. # documents.py - from django_elasticsearch_dsl import DocType, Index + from django_elasticsearch_dsl import Document, Index from .models import Car # Name of the Elasticsearch index @@ -101,9 +96,9 @@ defined in a ``documents.py`` file. ) - @car.doc_type - class CarDocument(DocType): - class Meta: + @car.document + class CarDocument(Document): + class Django: model = Car # The model associated with this DocType # The fields of the model you want to be indexed in Elasticsearch diff --git a/setup.py b/setup.py index b699e4a4..feaf41ff 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ ], include_package_data=True, install_requires=[ - 'elasticsearch-dsl>=2.1.0', + 'elasticsearch-dsl>=6.3.0', ], license="Apache Software License 2.0", zip_safe=False, diff --git a/tox.ini b/tox.ini index 7caaf0cd..6a2903d6 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - {py27,py36,py37}-django-110-{es2,es5,es6} - {py27,py36,py37}-django-111-{es2,es5,es6} - {py36,py37}-django-2-{es2,es5,es6} - {py36,py37}-django-21-{es2,es5,es6} + {py27,py36,py37}-django-110-{es6} + {py27,py36,py37}-django-111-{es6} + {py36,py37}-django-2-{es6} + {py36,py37}-django-21-{es6} [testenv] setenv = @@ -16,8 +16,6 @@ deps = django-111: Django>=1.11,<2.0 django-2: Django>=2.0,<2.1 django-21: Django>=2.1,<2.2 - es2: elasticsearch-dsl>=2.1.0,<3.0.0 - es5: elasticsearch-dsl>=5.0.0,<6.0.0 es6: https://github.com/safwanrahman/elasticsearch-dsl-py/archive/document_method.tar.gz#egg=elasticsearch-dsl-py==6.3.3 -r{toxinidir}/requirements_test.txt From 36b6892754d41ffdb8f698ff67666fdcd1cb00d2 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 9 Feb 2019 04:17:29 +0600 Subject: [PATCH 11/16] fixup test --- django_elasticsearch_dsl/test/testcases.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/django_elasticsearch_dsl/test/testcases.py b/django_elasticsearch_dsl/test/testcases.py index ae1d9b0a..83ea3401 100644 --- a/django_elasticsearch_dsl/test/testcases.py +++ b/django_elasticsearch_dsl/test/testcases.py @@ -7,7 +7,7 @@ class ESTestCase(object): def setUp(self): for doc in registry.get_documents(): - doc._doc_type.index += self._index_suffixe + doc._index._name += self._index_suffixe for index in registry.get_indices(): index._name += self._index_suffixe @@ -20,7 +20,7 @@ def tearDown(self): pattern = re.compile(self._index_suffixe + '$') for doc in registry.get_documents(): - doc._doc_type.index = pattern.sub('', doc._doc_type.index) + doc._index._name = pattern.sub('', doc._index._name) for index in registry.get_indices(): index.delete(ignore=[404, 400]) From a14562b0e7aeca8032c416c41dd822ad718718d3 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 9 Feb 2019 04:25:51 +0600 Subject: [PATCH 12/16] fixup test --- django_elasticsearch_dsl/documents.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_elasticsearch_dsl/documents.py b/django_elasticsearch_dsl/documents.py index e4a2da7a..b2817c5e 100644 --- a/django_elasticsearch_dsl/documents.py +++ b/django_elasticsearch_dsl/documents.py @@ -126,7 +126,7 @@ def to_field(cls, field_name, model_field): ) def bulk(self, actions, **kwargs): - return bulk(client=self._get_connection, actions=actions, **kwargs) + return bulk(client=self._get_connection(), actions=actions, **kwargs) def _prepare_action(self, object_instance, action): return { From 57b37350c8a303244ca575121fe9d1508bd61d0c Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 9 Feb 2019 04:39:15 +0600 Subject: [PATCH 13/16] more fixup --- django_elasticsearch_dsl/documents.py | 4 ++-- tests/test_documents.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/django_elasticsearch_dsl/documents.py b/django_elasticsearch_dsl/documents.py index b2817c5e..a1560a28 100644 --- a/django_elasticsearch_dsl/documents.py +++ b/django_elasticsearch_dsl/documents.py @@ -65,8 +65,8 @@ def __hash__(self): @classmethod def search(cls, using=None, index=None): return Search( - using=using or cls._doc_type.using, - index=index or cls._doc_type.index, + using=cls._get_using(using), + index=cls._default_index(index), doc_type=[cls], model=cls.django.model ) diff --git a/tests/test_documents.py b/tests/test_documents.py index 844655b6..d302d7b9 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -217,7 +217,7 @@ def test_model_instance_update(self): ) self.assertTrue(mock.call_args_list[0][1]['refresh']) self.assertEqual( - doc._index.connection, mock.call_args_list[0][1]['client']() + doc._index.connection, mock.call_args_list[0][1]['client'] ) def test_model_instance_iterable_update(self): @@ -258,7 +258,7 @@ def test_model_instance_iterable_update(self): ) self.assertTrue(mock.call_args_list[0][1]['refresh']) self.assertEqual( - doc._index.connection, mock.call_args_list[0][1]['client']() + doc._index.connection, mock.call_args_list[0][1]['client'] ) def test_model_instance_update_no_refresh(self): From f08a36665e74f313eb274e0cd4903b0874d83ecf Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 9 Feb 2019 06:27:03 +0600 Subject: [PATCH 14/16] fixup --- django_elasticsearch_dsl/search.py | 3 ++- tests/test_integration.py | 22 ++++++---------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/django_elasticsearch_dsl/search.py b/django_elasticsearch_dsl/search.py index 9a3d1e71..0b69d0eb 100644 --- a/django_elasticsearch_dsl/search.py +++ b/django_elasticsearch_dsl/search.py @@ -24,8 +24,9 @@ def to_queryset(self, keep_order=True): if not hasattr(self, '_response'): # We only need the meta fields with the models ids s = self.source(excludes=['*']) + s = s.execute() - pks = [result._id for result in s] + pks = [result.meta.id for result in s] qs = self._model.objects.filter(pk__in=pks) diff --git a/tests/test_integration.py b/tests/test_integration.py index 98511e23..26960b49 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -183,7 +183,6 @@ def test_index_to_dict(self): test_index = DSLIndex('test_index').settings(**index_settings) test_index.doc_type(CarDocument) - test_index.doc_type(ManufacturerDocument) index_dict = test_index.to_dict() @@ -203,16 +202,7 @@ def test_index_to_dict(self): } }) self.assertEqual(index_dict['mappings'], { - 'manufacturer_document': { - 'properties': { - 'created': {'type': 'date'}, - 'name': {'type': text_type}, - 'country': {'type': text_type}, - 'country_code': {'type': text_type}, - 'logo': {'type': text_type}, - } - }, - 'car_document': { + 'doc': { 'properties': { 'ads': { 'type': 'nested', @@ -222,7 +212,7 @@ def test_index_to_dict(self): 'html_strip' }, 'pk': {'type': 'integer'}, - 'title': {'type': text_type}, + 'title': {'type': text_type} }, }, 'categories': { @@ -230,21 +220,21 @@ def test_index_to_dict(self): 'properties': { 'title': {'type': text_type}, 'slug': {'type': text_type}, - 'icon': {'type': text_type}, + 'icon': {'type': text_type} }, }, 'manufacturer': { 'type': 'object', 'properties': { 'country': {'type': text_type}, - 'name': {'type': text_type}, + 'name': {'type': text_type} }, }, 'name': {'type': text_type}, 'launched': {'type': 'date'}, - 'type': {'type': text_type}, + 'type': {'type': text_type} } - }, + } }) def test_related_docs_are_updated(self): From 42378325958cebf6e9101736bde82ac9dc6425ef Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 9 Feb 2019 07:45:15 +0600 Subject: [PATCH 15/16] updating documentation --- README.rst | 112 ++++++++++++++++++++++++++--------------------------- 1 file changed, 55 insertions(+), 57 deletions(-) diff --git a/README.rst b/README.rst index f00ad5e0..7e241ce5 100644 --- a/README.rst +++ b/README.rst @@ -76,30 +76,31 @@ Then for a model: (4, "SUV"), ]) -To make this model work with Elasticsearch, create a subclass of ``django_elasticsearch_dsl.DocType`` -and create a ``django_elasticsearch_dsl.Index`` to define your Elasticsearch indices, names, and settings. This classes must be -defined in a ``documents.py`` file. +To make this model work with Elasticsearch, create a subclass of ``django_elasticsearch_dsl.Document``, +create a ``class Index`` inside the ``Document`` class +to define your Elasticsearch indices, names, settings etc and at last register the class using +``registry.register_document`` decorator. .. code-block:: python # documents.py - from django_elasticsearch_dsl import Document, Index + from django_elasticsearch_dsl import Document + from django_elasticsearch_dsl.registries import registry from .models import Car - # Name of the Elasticsearch index - car = Index('cars') - # See Elasticsearch Indices API reference for available settings - car.settings( - number_of_shards=1, - number_of_replicas=0 - ) - - @car.document + @registry.register_document class CarDocument(Document): + class Index: + # Name of the Elasticsearch index + name = 'cars' + # See Elasticsearch Indices API reference for available settings + settings = {'number_of_shards': 1, + 'number_of_replicas': 0} + class Django: - model = Car # The model associated with this DocType + model = Car # The model associated with this Document # The fields of the model you want to be indexed in Elasticsearch fields = [ @@ -200,7 +201,7 @@ the model to a string, so we'll just add a method for it: else: return "SUV" -Now we need to tell our ``DocType`` subclass to use that method instead of just +Now we need to tell our ``Document`` subclass to use that method instead of just accessing the ``type`` field on the model directly. Change the CarDocument to look like this: @@ -208,17 +209,17 @@ like this: # documents.py - from django_elasticsearch_dsl import DocType, fields + from django_elasticsearch_dsl import Document, fields # ... # - @car.doc_type - class CarDocument(DocType): + @registry.register_document + class CarDocument(Document): # add a string field to the Elasticsearch mapping called type, the # value of which is derived from the model's type_to_string attribute type = fields.TextField(attr="type_to_string") - class Meta: + class Django: model = Car # we removed the type field from here fields = [ @@ -235,7 +236,7 @@ Using prepare_field ~~~~~~~~~~~~~~~~~~~ Sometimes, you need to do some extra prepping before a field should be saved to -Elasticsearch. You can add a ``prepare_foo(self, instance)`` method to a DocType +Elasticsearch. You can add a ``prepare_foo(self, instance)`` method to a Document (where foo is the name of the field), and that will be called when the field needs to be saved. @@ -245,7 +246,7 @@ needs to be saved. # ... # - class CarDocument(DocType): + class CarDocument(Document): # ... # foo = TextField() @@ -287,18 +288,11 @@ You can use an ObjectField or a NestedField. # documents.py - from django_elasticsearch_dsl import DocType, Index, fields + from django_elasticsearch_dsl import Document, fields from .models import Car, Manufacturer, Ad - car = Index('cars') - car.settings( - number_of_shards=1, - number_of_replicas=0 - ) - - - @car.doc_type - class CarDocument(DocType): + @registry.register_document + class CarDocument(Document): manufacturer = fields.ObjectField(properties={ 'name': fields.TextField(), 'country_code': fields.TextField(), @@ -309,7 +303,10 @@ You can use an ObjectField or a NestedField. 'pk': fields.IntegerField(), }) - class Meta: + class Index: + name = 'cars' + + class Django: model = Car fields = [ 'name', @@ -362,14 +359,14 @@ So for example you can use a custom analyzer_: char_filter=["html_strip"] ) - @car.doc_type - class CarDocument(DocType): + @registry.register_document + class CarDocument(Document): description = fields.TextField( analyzer=html_strip, fields={'raw': fields.KeywordField()} ) - class Meta: + class Django: model = Car fields = [ 'name', @@ -412,20 +409,19 @@ instance. Index ----- +In typical scenario using `class Index` on a `Document` class is sufficient to perform any action. +In a few cases though it can be useful to manipulate an Index object directly. +To define an Elasticsearch index you must instantiate a ``elasticsearch_dsl.Index`` class and set the name +and settings of the index. +After you instantiate your class, you need to associate it with the Document you +want to put in this Elasticsearch index and also add the `registry.register_document` decorator. -To define an Elasticsearch index you must instantiate a ``django_elasticsearch_dsl.Index`` class and set the name -and settings of the index. This class inherits from elasticsearch-dsl-py Index_. -After you instantiate your class, you need to associate it with the DocType you -want to put in this Elasticsearch index. - - -.. _Index: http://elasticsearch-dsl.readthedocs.io/en/stable/persistence.html#index .. code-block:: python # documents.py - - from django_elasticsearch_dsl import DocType, Index + from elasticsearch_dsl import Index + from django_elasticsearch_dsl import Document from .models import Car, Manufacturer # The name of your index @@ -436,24 +432,27 @@ want to put in this Elasticsearch index. number_of_replicas=0 ) - - @car.doc_type - class CarDocument(DocType): - class Meta: + @registry.register_document + @car.document + class CarDocument(Document): + class Django: model = Car fields = [ 'name', 'color', ] - @car.doc_type - class ManufacturerDocument(DocType): - class Meta: + @registry.register_document + class ManufacturerDocument(Document): + class Index: + name = 'manufacture' + settings = {'number_of_shards': 1, + 'number_of_replicas': 0} + + class Django: model = Car fields = [ - 'name', # If a field as the same name in multiple DocType of - # the same Index, the field type must be identical - # (here fields.TextField) + 'name', 'country_code', ] @@ -461,8 +460,7 @@ When you execute the command:: $ ./manage.py search_index --rebuild -This will create an index named ``cars`` in Elasticsearch with two mappings: -``manufacturer_document`` and ``car_document``. +This will create two index named ``cars`` and ``manufacture`` in Elasticsearch with appropriate mapping. Management Commands @@ -560,6 +558,6 @@ TODO - Add support for --using (use another Elasticsearch cluster) in management commands. - Add management commands for mapping level operations (like update_mapping....). - Dedicated documentation. -- Generate ObjectField/NestField properties from a DocType class. +- Generate ObjectField/NestField properties from a Document class. - More examples. - Better ``ESTestCase`` and documentation for testing From 489200d99dcff60e81b0b7a23f3af68b20b5fbf1 Mon Sep 17 00:00:00 2001 From: Safwan Rahman Date: Sat, 1 Jun 2019 07:35:21 +0600 Subject: [PATCH 16/16] use released version --- requirements.txt | 2 +- requirements_test.txt | 1 - tox.ini | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index 21d1190d..dbaef8d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ django>=1.9.6 -elasticsearch-dsl>=2.1.0,<6.2.0 +elasticsearch-dsl>=6.4.0,<7.0.0 diff --git a/requirements_test.txt b/requirements_test.txt index 06a3e6d7..1afa9d74 100644 --- a/requirements_test.txt +++ b/requirements_test.txt @@ -6,4 +6,3 @@ Pillow==4.2.1 # Additional test requirements go here -https://github.com/safwanrahman/elasticsearch-dsl-py/archive/document_method.tar.gz#egg=elasticsearch-dsl-py==6.3.3 diff --git a/tox.ini b/tox.ini index 6a2903d6..d20f3375 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,7 @@ deps = django-111: Django>=1.11,<2.0 django-2: Django>=2.0,<2.1 django-21: Django>=2.1,<2.2 - es6: https://github.com/safwanrahman/elasticsearch-dsl-py/archive/document_method.tar.gz#egg=elasticsearch-dsl-py==6.3.3 + es6: elasticsearch-dsl>=6.4.0,<7.0.0 -r{toxinidir}/requirements_test.txt basepython =