diff --git a/.travis.yml b/.travis.yml index 68e959d9..9a70246e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,68 +7,24 @@ dist: trusty # default "precise" distro doesn't include Java 8 for Elasticsearch matrix: include: - - env: TOX_ENV=py36-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.6 - - env: TOX_ENV=py36-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.6 - env: TOX_ENV=py36-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.6 - - env: TOX_ENV=py37-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - sudo: true - dist: xenial - - env: TOX_ENV=py37-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - sudo: true - dist: xenial - env: TOX_ENV=py37-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.7 sudo: true dist: xenial - - env: TOX_ENV=py27-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 2.7 - - env: TOX_ENV=py27-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 2.7 - env: TOX_ENV=py27-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 - - env: TOX_ENV=py36-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.6 - - env: TOX_ENV=py36-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.6 - env: TOX_ENV=py36-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.6 - - env: TOX_ENV=py37-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - sudo: true - dist: xenial - - env: TOX_ENV=py37-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - sudo: true - dist: xenial - env: TOX_ENV=py37-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.7 sudo: true dist: xenial - - env: TOX_ENV=py27-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 2.7 - - env: TOX_ENV=py27-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 2.7 - env: TOX_ENV=py27-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 - - env: TOX_ENV=py36-django-2-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.6 - - env: TOX_ENV=py36-django-2-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.6 - env: TOX_ENV=py36-django-2-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.6 - - env: TOX_ENV=py37-django-2-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - sudo: true - dist: xenial - - env: TOX_ENV=py37-django-2-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - sudo: true - dist: xenial - env: TOX_ENV=py37-django-2-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 3.7 sudo: true @@ -77,14 +33,6 @@ matrix: python: 3.7 dist: xenial sudo: true - - env: TOX_ENV=py37-django-21-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt - python: 3.7 - dist: xenial - sudo: true - - env: TOX_ENV=py37-django-21-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian - python: 3.7 - dist: xenial - sudo: true cache: pip diff --git a/README.rst b/README.rst index a93a6480..7e241ce5 100644 --- a/README.rst +++ b/README.rst @@ -27,7 +27,7 @@ Features - Django >= 1.10 - Python 2.7, 3.5, 3.6, 3.7 - - Elasticsearch >= 2.0 < 7.0 + - Elasticsearch >= 6.0 < 7.0 .. _Search: http://elasticsearch-dsl.readthedocs.io/en/stable/search_dsl.html @@ -39,13 +39,8 @@ Install Django Elasticsearch DSL:: pip install django-elasticsearch-dsl # Elasticsearch 6.x - pip install 'elasticsearch-dsl>=6.0,<6.2' + pip install 'elasticsearch-dsl>=6.3.0,<7.0' - # Elasticsearch 5.x - pip install 'elasticsearch-dsl>=5.0,<6.0' - - # Elasticsearch 2.x - pip install 'elasticsearch-dsl>=2.1,<3.0' Then add ``django_elasticsearch_dsl`` to the INSTALLED_APPS @@ -81,30 +76,31 @@ Then for a model: (4, "SUV"), ]) -To make this model work with Elasticsearch, create a subclass of ``django_elasticsearch_dsl.DocType`` -and create a ``django_elasticsearch_dsl.Index`` to define your Elasticsearch indices, names, and settings. This classes must be -defined in a ``documents.py`` file. +To make this model work with Elasticsearch, create a subclass of ``django_elasticsearch_dsl.Document``, +create a ``class Index`` inside the ``Document`` class +to define your Elasticsearch indices, names, settings etc and at last register the class using +``registry.register_document`` decorator. .. code-block:: python # documents.py - from django_elasticsearch_dsl import DocType, Index + from django_elasticsearch_dsl import Document + from django_elasticsearch_dsl.registries import registry from .models import Car - # Name of the Elasticsearch index - car = Index('cars') - # See Elasticsearch Indices API reference for available settings - car.settings( - number_of_shards=1, - number_of_replicas=0 - ) + @registry.register_document + class CarDocument(Document): + class Index: + # Name of the Elasticsearch index + name = 'cars' + # See Elasticsearch Indices API reference for available settings + settings = {'number_of_shards': 1, + 'number_of_replicas': 0} - @car.doc_type - class CarDocument(DocType): - class Meta: - model = Car # The model associated with this DocType + class Django: + model = Car # The model associated with this Document # The fields of the model you want to be indexed in Elasticsearch fields = [ @@ -205,7 +201,7 @@ the model to a string, so we'll just add a method for it: else: return "SUV" -Now we need to tell our ``DocType`` subclass to use that method instead of just +Now we need to tell our ``Document`` subclass to use that method instead of just accessing the ``type`` field on the model directly. Change the CarDocument to look like this: @@ -213,17 +209,17 @@ like this: # documents.py - from django_elasticsearch_dsl import DocType, fields + from django_elasticsearch_dsl import Document, fields # ... # - @car.doc_type - class CarDocument(DocType): + @registry.register_document + class CarDocument(Document): # add a string field to the Elasticsearch mapping called type, the # value of which is derived from the model's type_to_string attribute type = fields.TextField(attr="type_to_string") - class Meta: + class Django: model = Car # we removed the type field from here fields = [ @@ -240,7 +236,7 @@ Using prepare_field ~~~~~~~~~~~~~~~~~~~ Sometimes, you need to do some extra prepping before a field should be saved to -Elasticsearch. You can add a ``prepare_foo(self, instance)`` method to a DocType +Elasticsearch. You can add a ``prepare_foo(self, instance)`` method to a Document (where foo is the name of the field), and that will be called when the field needs to be saved. @@ -250,7 +246,7 @@ needs to be saved. # ... # - class CarDocument(DocType): + class CarDocument(Document): # ... # foo = TextField() @@ -292,18 +288,11 @@ You can use an ObjectField or a NestedField. # documents.py - from django_elasticsearch_dsl import DocType, Index, fields + from django_elasticsearch_dsl import Document, fields from .models import Car, Manufacturer, Ad - car = Index('cars') - car.settings( - number_of_shards=1, - number_of_replicas=0 - ) - - - @car.doc_type - class CarDocument(DocType): + @registry.register_document + class CarDocument(Document): manufacturer = fields.ObjectField(properties={ 'name': fields.TextField(), 'country_code': fields.TextField(), @@ -314,7 +303,10 @@ You can use an ObjectField or a NestedField. 'pk': fields.IntegerField(), }) - class Meta: + class Index: + name = 'cars' + + class Django: model = Car fields = [ 'name', @@ -367,14 +359,14 @@ So for example you can use a custom analyzer_: char_filter=["html_strip"] ) - @car.doc_type - class CarDocument(DocType): + @registry.register_document + class CarDocument(Document): description = fields.TextField( analyzer=html_strip, fields={'raw': fields.KeywordField()} ) - class Meta: + class Django: model = Car fields = [ 'name', @@ -417,20 +409,19 @@ instance. Index ----- - -To define an Elasticsearch index you must instantiate a ``django_elasticsearch_dsl.Index`` class and set the name -and settings of the index. This class inherits from elasticsearch-dsl-py Index_. -After you instantiate your class, you need to associate it with the DocType you -want to put in this Elasticsearch index. +In typical scenario using `class Index` on a `Document` class is sufficient to perform any action. +In a few cases though it can be useful to manipulate an Index object directly. +To define an Elasticsearch index you must instantiate a ``elasticsearch_dsl.Index`` class and set the name +and settings of the index. +After you instantiate your class, you need to associate it with the Document you +want to put in this Elasticsearch index and also add the `registry.register_document` decorator. -.. _Index: http://elasticsearch-dsl.readthedocs.io/en/stable/persistence.html#index - .. code-block:: python # documents.py - - from django_elasticsearch_dsl import DocType, Index + from elasticsearch_dsl import Index + from django_elasticsearch_dsl import Document from .models import Car, Manufacturer # The name of your index @@ -441,24 +432,27 @@ want to put in this Elasticsearch index. number_of_replicas=0 ) - - @car.doc_type - class CarDocument(DocType): - class Meta: + @registry.register_document + @car.document + class CarDocument(Document): + class Django: model = Car fields = [ 'name', 'color', ] - @car.doc_type - class ManufacturerDocument(DocType): - class Meta: + @registry.register_document + class ManufacturerDocument(Document): + class Index: + name = 'manufacture' + settings = {'number_of_shards': 1, + 'number_of_replicas': 0} + + class Django: model = Car fields = [ - 'name', # If a field as the same name in multiple DocType of - # the same Index, the field type must be identical - # (here fields.TextField) + 'name', 'country_code', ] @@ -466,8 +460,7 @@ When you execute the command:: $ ./manage.py search_index --rebuild -This will create an index named ``cars`` in Elasticsearch with two mappings: -``manufacturer_document`` and ``car_document``. +This will create two index named ``cars`` and ``manufacture`` in Elasticsearch with appropriate mapping. Management Commands @@ -565,6 +558,6 @@ TODO - Add support for --using (use another Elasticsearch cluster) in management commands. - Add management commands for mapping level operations (like update_mapping....). - Dedicated documentation. -- Generate ObjectField/NestField properties from a DocType class. +- Generate ObjectField/NestField properties from a Document class. - More examples. - Better ``ESTestCase`` and documentation for testing diff --git a/django_elasticsearch_dsl/documents.py b/django_elasticsearch_dsl/documents.py index e0df4155..a1560a28 100644 --- a/django_elasticsearch_dsl/documents.py +++ b/django_elasticsearch_dsl/documents.py @@ -1,11 +1,12 @@ from __future__ import unicode_literals +from copy import deepcopy + from django.db import models from django.core.paginator import Paginator from django.utils.six import add_metaclass, iteritems from elasticsearch.helpers import bulk -from elasticsearch_dsl import DocType as DSLDocType -from elasticsearch_dsl.document import DocTypeMeta as DSLDocTypeMeta +from elasticsearch_dsl import Document as DSLDocument from elasticsearch_dsl.field import Field from .apps import DEDConfig @@ -50,70 +51,7 @@ } -class DocTypeMeta(DSLDocTypeMeta): - def __new__(cls, name, bases, attrs): - """ - Subclass default DocTypeMeta to generate ES fields from django - models fields - """ - super_new = super(DocTypeMeta, cls).__new__ - - parents = [b for b in bases if isinstance(b, DocTypeMeta)] - if not parents: - return super_new(cls, name, bases, attrs) - - model = attrs['Meta'].model - - ignore_signals = getattr(attrs['Meta'], "ignore_signals", False) - auto_refresh = getattr( - attrs['Meta'], 'auto_refresh', DEDConfig.auto_refresh_enabled() - ) - model_field_names = getattr(attrs['Meta'], "fields", []) - related_models = getattr(attrs['Meta'], "related_models", []) - queryset_pagination = getattr( - attrs['Meta'], "queryset_pagination", None - ) - - class_fields = set( - name for name, field in iteritems(attrs) - if isinstance(field, Field) - ) - - cls = super_new(cls, name, bases, attrs) - - cls._doc_type.model = model - cls._doc_type.ignore_signals = ignore_signals - cls._doc_type.auto_refresh = auto_refresh - cls._doc_type.related_models = related_models - cls._doc_type.queryset_pagination = queryset_pagination - - fields = model._meta.get_fields() - fields_lookup = dict((field.name, field) for field in fields) - - for field_name in model_field_names: - if field_name in class_fields: - raise RedeclaredFieldError( - "You cannot redeclare the field named '{}' on {}" - .format(field_name, cls.__name__) - ) - - field_instance = cls.to_field(field_name, - fields_lookup[field_name]) - cls._doc_type.mapping.field(field_name, field_instance) - - cls._doc_type._fields = ( - lambda: cls._doc_type.mapping.properties.properties.to_dict()) - - if getattr(cls._doc_type, 'index'): - index = Index(cls._doc_type.index) - index.doc_type(cls) - registry.register(index, cls) - - return cls - - -@add_metaclass(DocTypeMeta) -class DocType(DSLDocType): +class DocType(DSLDocument): def __init__(self, related_instance_to_ignore=None, **kwargs): super(DocType, self).__init__(**kwargs) self._related_instance_to_ignore = related_instance_to_ignore @@ -127,17 +65,17 @@ def __hash__(self): @classmethod def search(cls, using=None, index=None): return Search( - using=using or cls._doc_type.using, - index=index or cls._doc_type.index, + using=cls._get_using(using), + index=cls._default_index(index), doc_type=[cls], - model=cls._doc_type.model + model=cls.django.model ) def get_queryset(self): """ Return the queryset that should be indexed by this doc type. """ - return self._doc_type.model._default_manager.all() + return self.django.model._default_manager.all() def prepare(self, instance): """ @@ -145,7 +83,7 @@ def prepare(self, instance): based on the fields defined on this DocType subclass """ data = {} - for name, field in iteritems(self._doc_type._fields()): + for name, field in iteritems(self._fields): if not isinstance(field, DEDField): continue @@ -188,13 +126,13 @@ def to_field(cls, field_name, model_field): ) def bulk(self, actions, **kwargs): - return bulk(client=self.connection, actions=actions, **kwargs) + return bulk(client=self._get_connection(), actions=actions, **kwargs) def _prepare_action(self, object_instance, action): return { '_op_type': action, - '_index': str(self._doc_type.index), - '_type': self._doc_type.mapping.doc_type, + '_index': self._index._name, + '_type': self._doc_type.name, '_id': object_instance.pk, '_source': ( self.prepare(object_instance) if action != 'delete' else None @@ -202,9 +140,9 @@ def _prepare_action(self, object_instance, action): } def _get_actions(self, object_list, action): - if self._doc_type.queryset_pagination is not None: + if self.django.queryset_pagination is not None: paginator = Paginator( - object_list, self._doc_type.queryset_pagination + object_list, self.django.queryset_pagination ) for page in paginator.page_range: for object_instance in paginator.page(page).object_list: @@ -218,7 +156,7 @@ def update(self, thing, refresh=None, action='index', **kwargs): Update each document in ES for a model, iterable of models or queryset """ if refresh is True or ( - refresh is None and self._doc_type.auto_refresh + refresh is None and self.django.auto_refresh ): kwargs['refresh'] = True diff --git a/django_elasticsearch_dsl/indices.py b/django_elasticsearch_dsl/indices.py index 1d208e0d..23147a25 100644 --- a/django_elasticsearch_dsl/indices.py +++ b/django_elasticsearch_dsl/indices.py @@ -9,17 +9,20 @@ @python_2_unicode_compatible class Index(DSLIndex): - def __init__(self, name, using='default'): - super(Index, self).__init__(name, using) - self._settings = deepcopy(DEDConfig.default_index_settings()) + def __init__(self, *args, **kwargs): + super(Index, self).__init__(*args, **kwargs) + default_index_settings = deepcopy(DEDConfig.default_index_settings()) + self.settings(**default_index_settings) - def doc_type(self, doc_type, *args, **kwargs): + def document(self, document): """ - Extend to register the doc_type in the global document registry + Extend to register the document in the global document registry """ - doc_type = super(Index, self).doc_type(doc_type, *args, **kwargs) - registry.register(self, doc_type) - return doc_type + document = super(Index, self).document(document) + registry.register_document(document) + return document + + doc_type = document def __str__(self): return self._name diff --git a/django_elasticsearch_dsl/management/commands/search_index.py b/django_elasticsearch_dsl/management/commands/search_index.py index f6f505f4..20179bcc 100644 --- a/django_elasticsearch_dsl/management/commands/search_index.py +++ b/django_elasticsearch_dsl/management/commands/search_index.py @@ -87,7 +87,7 @@ def _populate(self, models, options): for doc in registry.get_documents(models): qs = doc().get_queryset() self.stdout.write("Indexing {} '{}' objects".format( - qs.count(), doc._doc_type.model.__name__) + qs.count(), doc.django.model.__name__) ) doc().update(qs) diff --git a/django_elasticsearch_dsl/registries.py b/django_elasticsearch_dsl/registries.py index 89701019..1084955c 100644 --- a/django_elasticsearch_dsl/registries.py +++ b/django_elasticsearch_dsl/registries.py @@ -1,9 +1,14 @@ from collections import defaultdict +from copy import deepcopy + from itertools import chain from django.core.exceptions import ObjectDoesNotExist +from django.core.exceptions import ImproperlyConfigured from django.utils.six import itervalues, iterkeys, iteritems +from elasticsearch_dsl import Field, AttrDict +from django_elasticsearch_dsl.exceptions import RedeclaredFieldError from .apps import DEDConfig @@ -18,10 +23,10 @@ def __init__(self): def register(self, index, doc_class): """Register the model with the registry""" - self._models[doc_class._doc_type.model].add(doc_class) + self._models[doc_class.django.model].add(doc_class) - for related in doc_class._doc_type.related_models: - self._related_models[related].add(doc_class._doc_type.model) + for related in doc_class.django.related_models: + self._related_models[related].add(doc_class.django.model) for idx, docs in iteritems(self._indices): if index._name == idx._name: @@ -30,10 +35,62 @@ def register(self, index, doc_class): self._indices[index].add(doc_class) + def register_document(self, document): + django_meta = getattr(document, 'Django') + # Raise error if Django class can not be found + if not django_meta: + message = "You must declare the Django class inside {}".format(document.__name__) + raise ImproperlyConfigured(message) + + # Keep all django related attribute in a django_attr AttrDict + data = {'model': getattr(document.Django, 'model')} + django_attr = AttrDict(data) + + if not django_attr.model: + raise ImproperlyConfigured("You must specify the django model") + + # Add The model fields into elasticsearch mapping field + model_field_names = getattr(document.Django, "fields", []) + mapping_fields = document._doc_type.mapping.properties.properties.to_dict().keys() + + for field_name in model_field_names: + if field_name in mapping_fields: + raise RedeclaredFieldError( + "You cannot redeclare the field named '{}' on {}" + .format(field_name, document.__name__) + ) + + django_field = django_attr.model._meta.get_field(field_name) + + field_instance = document.to_field(field_name, django_field) + document._doc_type.mapping.field(field_name, field_instance) + + django_attr.ignore_signals = getattr(django_meta, "ignore_signals", False) + django_attr.auto_refresh = getattr(django_meta, + "auto_refresh", DEDConfig.auto_refresh_enabled()) + django_attr.related_models = getattr(django_meta, "related_models", []) + django_attr.queryset_pagination = getattr(django_meta, "queryset_pagination", None) + + # Add django attribute in the document class with all the django attribute + setattr(document, 'django', django_attr) + + # Set the fields of the mappings + fields = document._doc_type.mapping.properties.properties.to_dict() + setattr(document, '_fields', fields) + + # Update settings of the document index + default_index_settings = deepcopy(DEDConfig.default_index_settings()) + document._index.settings(**default_index_settings) + + # Register the document and index class to our registry + self.register(index=document._index, doc_class=document) + + return document + def _get_related_doc(self, instance): for model in self._related_models.get(instance.__class__, []): for doc in self._models[model]: - if instance.__class__ in doc._doc_type.related_models: + if instance.__class__ in doc.django.related_models: yield doc def update_related(self, instance, **kwargs): @@ -80,7 +137,7 @@ def update(self, instance, **kwargs): if instance.__class__ in self._models: for doc in self._models[instance.__class__]: - if not doc._doc_type.ignore_signals: + if not doc.django.ignore_signals: doc().update(instance, **kwargs) def delete(self, instance, **kwargs): @@ -112,7 +169,7 @@ def get_indices(self, models=None): if models is not None: return set( indice for indice, docs in iteritems(self._indices) - for doc in docs if doc._doc_type.model in models + for doc in docs if doc.django.model in models ) return set(iterkeys(self._indices)) diff --git a/django_elasticsearch_dsl/search.py b/django_elasticsearch_dsl/search.py index 9a3d1e71..0b69d0eb 100644 --- a/django_elasticsearch_dsl/search.py +++ b/django_elasticsearch_dsl/search.py @@ -24,8 +24,9 @@ def to_queryset(self, keep_order=True): if not hasattr(self, '_response'): # We only need the meta fields with the models ids s = self.source(excludes=['*']) + s = s.execute() - pks = [result._id for result in s] + pks = [result.meta.id for result in s] qs = self._model.objects.filter(pk__in=pks) diff --git a/django_elasticsearch_dsl/test/testcases.py b/django_elasticsearch_dsl/test/testcases.py index ae1d9b0a..83ea3401 100644 --- a/django_elasticsearch_dsl/test/testcases.py +++ b/django_elasticsearch_dsl/test/testcases.py @@ -7,7 +7,7 @@ class ESTestCase(object): def setUp(self): for doc in registry.get_documents(): - doc._doc_type.index += self._index_suffixe + doc._index._name += self._index_suffixe for index in registry.get_indices(): index._name += self._index_suffixe @@ -20,7 +20,7 @@ def tearDown(self): pattern = re.compile(self._index_suffixe + '$') for doc in registry.get_documents(): - doc._doc_type.index = pattern.sub('', doc._doc_type.index) + doc._index._name = pattern.sub('', doc._index._name) for index in registry.get_indices(): index.delete(ignore=[404, 400]) diff --git a/requirements.txt b/requirements.txt index 21d1190d..dbaef8d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ django>=1.9.6 -elasticsearch-dsl>=2.1.0,<6.2.0 +elasticsearch-dsl>=6.4.0,<7.0.0 diff --git a/requirements_dev.txt b/requirements_dev.txt index 05be29ad..41164ffe 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -3,4 +3,6 @@ wheel==0.32.2 django>=2.0,<2.2 elasticsearch-dsl>=2.1.0,<6.2.0 twine +wheel==0.29.0 +django>=2.0,<2.1 -e . diff --git a/runtests.py b/runtests.py index 157374a0..6a6679f6 100644 --- a/runtests.py +++ b/runtests.py @@ -27,7 +27,7 @@ def get_settings(): ELASTICSEARCH_DSL={ 'default': { 'hosts': os.environ.get('ELASTICSEARCH_URL', - 'localhost:9200') + '127.0.0.1:9200') }, }, ) diff --git a/setup.py b/setup.py index 40c231e2..feaf41ff 100755 --- a/setup.py +++ b/setup.py @@ -42,7 +42,7 @@ ], include_package_data=True, install_requires=[ - 'elasticsearch-dsl>=2.1.0,<6.2.0', + 'elasticsearch-dsl>=6.3.0', ], license="Apache Software License 2.0", zip_safe=False, diff --git a/tests/documents.py b/tests/documents.py index cd9d1213..e1a093ec 100644 --- a/tests/documents.py +++ b/tests/documents.py @@ -1,5 +1,6 @@ from elasticsearch_dsl import analyzer from django_elasticsearch_dsl import DocType, Index, fields +from django_elasticsearch_dsl.registries import registry from .models import Ad, Category, Car, Manufacturer @@ -9,9 +10,6 @@ } -car_index = Index('test_cars').settings(**index_settings) - - html_strip = analyzer( 'html_strip', tokenizer="standard", @@ -20,7 +18,7 @@ ) -@car_index.doc_type +@registry.register_document class CarDocument(DocType): # test can override __init__ def __init__(self, *args, **kwargs): @@ -43,7 +41,7 @@ def __init__(self, *args, **kwargs): 'icon': fields.FileField(), }) - class Meta: + class Django: model = Car related_models = [Ad, Manufacturer, Category] fields = [ @@ -51,7 +49,10 @@ class Meta: 'launched', 'type', ] - doc_type = 'car_document' + + class Index: + name = 'test_cars' + settings = index_settings def get_queryset(self): return super(CarDocument, self).get_queryset().select_related( @@ -65,14 +66,11 @@ def get_instances_from_related(self, related_instance): return related_instance.car_set.all() -manufacturer_index = Index('test_manufacturers').settings(**index_settings) - - -@manufacturer_index.doc_type +@registry.register_document class ManufacturerDocument(DocType): country = fields.StringField() - class Meta: + class Django: model = Manufacturer fields = [ 'name', @@ -80,9 +78,13 @@ class Meta: 'country_code', 'logo', ] - doc_type = 'manufacturer_document' + + class Index: + name = 'index_settings' + settings = index_settings +@registry.register_document class CarWithPrepareDocument(DocType): manufacturer = fields.ObjectField(properties={ 'name': fields.StringField(), @@ -93,16 +95,18 @@ class CarWithPrepareDocument(DocType): 'name': fields.StringField(), }) - class Meta: + class Django: model = Car related_models = [Manufacturer] - index = 'car_with_prepare_index' fields = [ 'name', 'launched', 'type', ] + class Index: + name = 'car_with_prepare_index' + def prepare_manufacturer_with_related(self, car, related_to_ignore): if (car.manufacturer is not None and car.manufacturer != related_to_ignore): @@ -123,17 +127,14 @@ def get_instances_from_related(self, related_instance): return related_instance.car_set.all() -ad_index = Index('test_ads').settings(**index_settings) - - -@ad_index.doc_type +@registry.register_document class AdDocument(DocType): description = fields.TextField( analyzer=html_strip, fields={'raw': fields.KeywordField()} ) - class Meta: + class Django: model = Ad fields = [ 'title', @@ -141,13 +142,17 @@ class Meta: 'modified', 'url', ] - doc_type = 'ad_document' + class Index: + name = 'test_ads' + settings = index_settings + +@registry.register_document class PaginatedAdDocument(DocType): - class Meta: + + class Django: model = Ad - index = 'ad_index' queryset_pagination = 2 fields = [ 'title', @@ -155,7 +160,13 @@ class Meta: 'modified', 'url', ] - doc_type = 'paginated_ad_document' + + class Index: + name = 'ad_index' def get_queryset(self): return Ad.objects.all().order_by('-id') + + +ad_index = AdDocument._index +car_index = CarDocument._index diff --git a/tests/fixtures.py b/tests/fixtures.py index 521eb4a0..4d3bad9e 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -25,16 +25,20 @@ class ModelD(models.Model): class ModelE(models.Model): pass - def _generate_doc_mock( - self, _model, index=None, mock_qs=None, - _ignore_signals=False, _related_models=None - ): + def _generate_doc_mock(self, _model, index=None, mock_qs=None, + _ignore_signals=False, _related_models=None): + _index = index + class Doc(DocType): - class Meta: + + class Django: model = _model + related_models = _related_models if _related_models is not None else [] ignore_signals = _ignore_signals - related_models = _related_models if ( - _related_models) is not None else [] + + if _index: + _index.document(Doc) + self.registry.register_document(Doc) Doc.update = Mock() if mock_qs: @@ -42,7 +46,4 @@ class Meta: if _related_models: Doc.get_instances_from_related = Mock() - if index: - self.registry.register(index, Doc) - return Doc diff --git a/tests/test_commands.py b/tests/test_commands.py index e32f61d3..2267d75c 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -5,6 +5,7 @@ from django.core.management import call_command from django.utils.six import StringIO +from django_elasticsearch_dsl import Index from django_elasticsearch_dsl.management.commands.search_index import Command from django_elasticsearch_dsl.registries import DocumentRegistry @@ -12,11 +13,26 @@ class SearchIndexTestCase(WithFixturesMixin, TestCase): + def _mock_setup(self): + # Mock Patch object + patch_registry = patch( + 'django_elasticsearch_dsl.management.commands.search_index.registry', self.registry) + + patch_registry.start() + + methods = ['delete', 'create'] + for index in [self.index_a, self.index_b]: + for method in methods: + obj_patch = patch.object(index, method) + obj_patch.start() + + self.addCleanup(patch.stopall) + def setUp(self): self.out = StringIO() self.registry = DocumentRegistry() - self.index_a = Mock() - self.index_b = Mock() + self.index_a = Index('foo') + self.index_b = Index('bar') self.doc_a1_qs = Mock() self.doc_a1 = self._generate_doc_mock( @@ -38,29 +54,26 @@ def setUp(self): self.ModelC, self.index_b, self.doc_c1_qs ) + self._mock_setup() + def test_get_models(self): cmd = Command() - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - self.assertEqual( - cmd._get_models(['foo']), - set([self.ModelA, self.ModelB]) - ) - - self.assertEqual( - cmd._get_models(['foo', 'bar.ModelC']), - set([self.ModelA, self.ModelB, self.ModelC]) - ) - - self.assertEqual( - cmd._get_models([]), - set([self.ModelA, self.ModelB, self.ModelC]) - ) - with self.assertRaises(CommandError): - cmd._get_models(['unknown']) + self.assertEqual( + cmd._get_models(['foo']), + set([self.ModelA, self.ModelB]) + ) + + self.assertEqual( + cmd._get_models(['foo', 'bar.ModelC']), + set([self.ModelA, self.ModelB, self.ModelC]) + ) + + self.assertEqual( + cmd._get_models([]), + set([self.ModelA, self.ModelB, self.ModelC]) + ) + with self.assertRaises(CommandError): + cmd._get_models(['unknown']) def test_no_action_error(self): cmd = Command() @@ -70,72 +83,43 @@ def test_no_action_error(self): def test_delete_foo_index(self): with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry + 'django_elasticsearch_dsl.management.commands.search_index.input', + Mock(return_value="y") ): - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.input', - Mock(return_value="y") - ): - call_command('search_index', stdout=self.out, - action='delete', models=['foo']) - self.index_a.delete.assert_called_once() - self.assertFalse(self.index_b.delete.called) + call_command('search_index', stdout=self.out, + action='delete', models=['foo']) + self.index_a.delete.assert_called_once() + self.assertFalse(self.index_b.delete.called) def test_force_delete_all_indices(self): - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - call_command('search_index', stdout=self.out, - action='delete', force=True) - self.index_a.delete.assert_called_once() - self.index_b.delete.assert_called_once() + call_command('search_index', stdout=self.out, + action='delete', force=True) + self.index_a.delete.assert_called_once() + self.index_b.delete.assert_called_once() def test_force_delete_bar_model_c_index(self): - - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - call_command('search_index', stdout=self.out, - models=['bar.ModelC'], - action='delete', force=True) - self.index_b.delete.assert_called_once() - self.assertFalse(self.index_a.delete.called) + call_command('search_index', stdout=self.out, + models=[self.ModelC._meta.label], + action='delete', force=True) + self.index_b.delete.assert_called_once() + self.assertFalse(self.index_a.delete.called) def test_create_all_indices(self): - - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - call_command('search_index', stdout=self.out, action='create') - self.index_a.create.assert_called_once() - self.index_b.create.assert_called_once() + call_command('search_index', stdout=self.out, action='create') + self.index_a.create.assert_called_once() + self.index_b.create.assert_called_once() def test_populate_all_doc_type(self): - - with patch( - 'django_elasticsearch_dsl.management.commands.' - 'search_index.registry', - self.registry - ): - call_command('search_index', stdout=self.out, action='populate') - self.doc_a1.get_queryset.assert_called_once() - self.doc_a1.update.assert_called_once_with(self.doc_a1_qs) - self.doc_a2.get_queryset.assert_called_once() - self.doc_a2.update.assert_called_once_with(self.doc_a2_qs) - self.doc_b1.get_queryset.assert_called_once() - self.doc_b1.update.assert_called_once_with(self.doc_b1_qs) - self.doc_c1.get_queryset.assert_called_once() - self.doc_c1.update.assert_called_once_with(self.doc_c1_qs) + call_command('search_index', stdout=self.out, action='populate') + self.doc_a1.get_queryset.assert_called_once() + self.doc_a1.update.assert_called_once_with(self.doc_a1_qs) + self.doc_a2.get_queryset.assert_called_once() + self.doc_a2.update.assert_called_once_with(self.doc_a2_qs) + self.doc_b1.get_queryset.assert_called_once() + self.doc_b1.update.assert_called_once_with(self.doc_b1_qs) + self.doc_c1.get_queryset.assert_called_once() + self.doc_c1.update.assert_called_once_with(self.doc_c1_qs) def test_rebuild_indices(self): diff --git a/tests/test_documents.py b/tests/test_documents.py index 637dedf0..d302d7b9 100644 --- a/tests/test_documents.py +++ b/tests/test_documents.py @@ -2,13 +2,14 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ -from elasticsearch_dsl import GeoPoint +from elasticsearch_dsl import GeoPoint, MetaField from mock import patch from django_elasticsearch_dsl import fields from django_elasticsearch_dsl.documents import DocType from django_elasticsearch_dsl.exceptions import (ModelFieldNotMappedError, RedeclaredFieldError) +from django_elasticsearch_dsl.registries import registry from tests import ES_MAJOR_VERSION @@ -34,6 +35,7 @@ class Meta: app_label = 'car' +@registry.register_document class CarDocument(DocType): color = fields.TextField() type = fields.StringField() @@ -42,48 +44,57 @@ def prepare_color(self, instance): return "blue" class Meta: + doc_type = 'car_document' + + class Django: fields = ['name', 'price'] model = Car - index = 'car_index' related_models = [Manufacturer] + + class Index: + name = 'car_index' doc_type = 'car_document' class DocTypeTestCase(TestCase): def test_model_class_added(self): - self.assertEqual(CarDocument._doc_type.model, Car) + self.assertEqual(CarDocument.django.model, Car) def test_ignore_signal_default(self): - self.assertFalse(CarDocument._doc_type.ignore_signals) + self.assertFalse(CarDocument.django.ignore_signals) def test_auto_refresh_default(self): - self.assertTrue(CarDocument._doc_type.auto_refresh) + self.assertTrue(CarDocument.django.auto_refresh) def test_ignore_signal_added(self): + + @registry.register_document class CarDocument2(DocType): - class Meta: + class Django: model = Car ignore_signals = True - self.assertTrue(CarDocument2._doc_type.ignore_signals) + self.assertTrue(CarDocument2.django.ignore_signals) def test_auto_refresh_added(self): + @registry.register_document class CarDocument2(DocType): - class Meta: + class Django: model = Car auto_refresh = False - self.assertFalse(CarDocument2._doc_type.auto_refresh) + self.assertFalse(CarDocument2.django.auto_refresh) def test_queryset_pagination_added(self): + @registry.register_document class CarDocument2(DocType): - class Meta: + class Django: model = Car queryset_pagination = 120 - self.assertIsNone(CarDocument._doc_type.queryset_pagination) - self.assertEqual(CarDocument2._doc_type.queryset_pagination, 120) + self.assertIsNone(CarDocument.django.queryset_pagination) + self.assertEqual(CarDocument2.django.queryset_pagination, 120) def test_fields_populated(self): mapping = CarDocument._doc_type.mapping @@ -93,16 +104,17 @@ def test_fields_populated(self): ) def test_related_models_added(self): - related_models = CarDocument._doc_type.related_models + related_models = CarDocument.django.related_models self.assertEqual([Manufacturer], related_models) def test_duplicate_field_names_not_allowed(self): with self.assertRaises(RedeclaredFieldError): + @registry.register_document class CarDocument(DocType): color = fields.StringField() name = fields.StringField() - class Meta: + class Django: fields = ['name'] model = Car @@ -160,14 +172,17 @@ def test_prepare(self): ) def test_prepare_ignore_dsl_base_field(self): + @registry.register_document class CarDocumentDSlBaseField(DocType): position = GeoPoint() - class Meta: + class Django: model = Car - index = 'car_index' fields = ['name', 'price'] + class Index: + name = 'car_index' + car = Car(name="Type 57", price=5400000.0, not_indexed="not_indexex") doc = CarDocumentDSlBaseField() prepared_data = doc.prepare(car) @@ -202,7 +217,7 @@ def test_model_instance_update(self): ) self.assertTrue(mock.call_args_list[0][1]['refresh']) self.assertEqual( - doc.connection, mock.call_args_list[0][1]['client'] + doc._index.connection, mock.call_args_list[0][1]['client'] ) def test_model_instance_iterable_update(self): @@ -243,12 +258,12 @@ def test_model_instance_iterable_update(self): ) self.assertTrue(mock.call_args_list[0][1]['refresh']) self.assertEqual( - doc.connection, mock.call_args_list[0][1]['client'] + doc._index.connection, mock.call_args_list[0][1]['client'] ) def test_model_instance_update_no_refresh(self): doc = CarDocument() - doc._doc_type.auto_refresh = False + doc.django.auto_refresh = False car = Car() with patch('django_elasticsearch_dsl.documents.bulk') as mock: doc.update(car) @@ -256,7 +271,7 @@ def test_model_instance_update_no_refresh(self): def test_model_instance_iterable_update_with_pagination(self): class CarDocument2(DocType): - class Meta: + class Django: model = Car queryset_pagination = 2 diff --git a/tests/test_indices.py b/tests/test_indices.py index 2035db2b..11f4f5bb 100644 --- a/tests/test_indices.py +++ b/tests/test_indices.py @@ -10,18 +10,21 @@ class IndexTestCase(WithFixturesMixin, TestCase): + def setUp(self): + self.registry = DocumentRegistry() + def test_documents_add_to_register(self): - registry = DocumentRegistry() + registry = self.registry with patch('django_elasticsearch_dsl.indices.registry', new=registry): index = Index('test') doc_a1 = self._generate_doc_mock(self.ModelA) doc_a2 = self._generate_doc_mock(self.ModelA) - index.doc_type(doc_a1) + index.document(doc_a1) docs = list(registry.get_documents()) self.assertEqual(len(docs), 1) self.assertIs(docs[0], doc_a1) - index.doc_type(doc_a2) + index.document(doc_a2) docs = registry.get_documents() self.assertEqual(docs, set([doc_a1, doc_a2])) diff --git a/tests/test_integration.py b/tests/test_integration.py index 98511e23..26960b49 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -183,7 +183,6 @@ def test_index_to_dict(self): test_index = DSLIndex('test_index').settings(**index_settings) test_index.doc_type(CarDocument) - test_index.doc_type(ManufacturerDocument) index_dict = test_index.to_dict() @@ -203,16 +202,7 @@ def test_index_to_dict(self): } }) self.assertEqual(index_dict['mappings'], { - 'manufacturer_document': { - 'properties': { - 'created': {'type': 'date'}, - 'name': {'type': text_type}, - 'country': {'type': text_type}, - 'country_code': {'type': text_type}, - 'logo': {'type': text_type}, - } - }, - 'car_document': { + 'doc': { 'properties': { 'ads': { 'type': 'nested', @@ -222,7 +212,7 @@ def test_index_to_dict(self): 'html_strip' }, 'pk': {'type': 'integer'}, - 'title': {'type': text_type}, + 'title': {'type': text_type} }, }, 'categories': { @@ -230,21 +220,21 @@ def test_index_to_dict(self): 'properties': { 'title': {'type': text_type}, 'slug': {'type': text_type}, - 'icon': {'type': text_type}, + 'icon': {'type': text_type} }, }, 'manufacturer': { 'type': 'object', 'properties': { 'country': {'type': text_type}, - 'name': {'type': text_type}, + 'name': {'type': text_type} }, }, 'name': {'type': text_type}, 'launched': {'type': 'date'}, - 'type': {'type': text_type}, + 'type': {'type': text_type} } - }, + } }) def test_related_docs_are_updated(self): diff --git a/tests/test_registries.py b/tests/test_registries.py index c7282963..c72894cb 100644 --- a/tests/test_registries.py +++ b/tests/test_registries.py @@ -3,6 +3,7 @@ from django.conf import settings +from django_elasticsearch_dsl import Index from django_elasticsearch_dsl.registries import DocumentRegistry from .fixtures import WithFixturesMixin @@ -11,8 +12,8 @@ class DocumentRegistryTestCase(WithFixturesMixin, TestCase): def setUp(self): self.registry = DocumentRegistry() - self.index_1 = Mock() - self.index_2 = Mock() + self.index_1 = Index(name='index_1') + self.index_2 = Index(name='index_2') self.doc_a1 = self._generate_doc_mock(self.ModelA, self.index_1) self.doc_a2 = self._generate_doc_mock(self.ModelA, self.index_1) @@ -111,9 +112,8 @@ def test_update_related_instances(self): doc_d2.update.assert_not_called() def test_update_related_instances_not_defined(self): - doc_d1 = self._generate_doc_mock( - self.ModelD, self.index_1, _related_models=[self.ModelE] - ) + doc_d1 = self._generate_doc_mock(_model=self.ModelD, index=self.index_1, + _related_models=[self.ModelE]) instance = self.ModelE() diff --git a/tox.ini b/tox.ini index 0259587e..d20f3375 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - {py27,py36,py37}-django-110-{es2,es5,es6} - {py27,py36,py37}-django-111-{es2,es5,es6} - {py36,py37}-django-2-{es2,es5,es6} - {py36,py37}-django-21-{es2,es5,es6} + {py27,py36,py37}-django-110-{es6} + {py27,py36,py37}-django-111-{es6} + {py36,py37}-django-2-{es6} + {py36,py37}-django-21-{es6} [testenv] setenv = @@ -16,9 +16,7 @@ deps = django-111: Django>=1.11,<2.0 django-2: Django>=2.0,<2.1 django-21: Django>=2.1,<2.2 - es2: elasticsearch-dsl>=2.1.0,<3.0.0 - es5: elasticsearch-dsl>=5.0.0,<6.0.0 - es6: elasticsearch-dsl>=6.0.0,<7.0.0 + es6: elasticsearch-dsl>=6.4.0,<7.0.0 -r{toxinidir}/requirements_test.txt basepython =