Skip to content

Commit d86b43f

Browse files
Grendel7sabricot
Grendel7
authored and
sabricot
committed
Update DED for Elasticsearch 6
1 parent 69ccb5a commit d86b43f

File tree

14 files changed

+225
-129
lines changed

14 files changed

+225
-129
lines changed

‎.travis.yml

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,50 +11,74 @@ matrix:
1111
python: 3.5
1212
- env: TOX_ENV=py35-django-18-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
1313
python: 3.5
14+
- env: TOX_ENV=py35-django-18-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
15+
python: 3.5
1416
- env: TOX_ENV=py27-django-18-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
1517
python: 2.7
1618
- env: TOX_ENV=py27-django-18-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
1719
python: 2.7
20+
- env: TOX_ENV=py27-django-18-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
21+
python: 2.7
1822
- env: TOX_ENV=py35-django-19-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
1923
python: 3.5
2024
- env: TOX_ENV=py35-django-19-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
2125
python: 3.5
26+
- env: TOX_ENV=py35-django-19-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
27+
python: 3.5
2228
- env: TOX_ENV=py27-django-19-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
2329
python: 2.7
2430
- env: TOX_ENV=py27-django-19-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
2531
python: 2.7
32+
- env: TOX_ENV=py27-django-19-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
33+
python: 2.7
2634
- env: TOX_ENV=py36-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
2735
python: 3.6
2836
- env: TOX_ENV=py36-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
2937
python: 3.6
38+
- env: TOX_ENV=py36-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
39+
python: 3.6
3040
- env: TOX_ENV=py35-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
3141
python: 3.5
3242
- env: TOX_ENV=py35-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
3343
python: 3.5
44+
- env: TOX_ENV=py35-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
45+
python: 3.5
3446
- env: TOX_ENV=py27-django-110-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
3547
python: 2.7
3648
- env: TOX_ENV=py27-django-110-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
3749
python: 2.7
50+
- env: TOX_ENV=py27-django-110-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
51+
python: 2.7
3852
- env: TOX_ENV=py36-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
3953
python: 3.6
4054
- env: TOX_ENV=py36-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
4155
python: 3.6
56+
- env: TOX_ENV=py36-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
57+
python: 3.6
4258
- env: TOX_ENV=py35-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
4359
python: 3.5
4460
- env: TOX_ENV=py35-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
4561
python: 3.5
62+
- env: TOX_ENV=py35-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
63+
python: 3.5
4664
- env: TOX_ENV=py27-django-111-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
4765
python: 2.7
4866
- env: TOX_ENV=py27-django-111-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
4967
python: 2.7
68+
- env: TOX_ENV=py27-django-111-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
69+
python: 2.7
5070
- env: TOX_ENV=py36-django-2-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
5171
python: 3.6
5272
- env: TOX_ENV=py36-django-2-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
5373
python: 3.6
74+
- env: TOX_ENV=py36-django-2-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
75+
python: 3.6
5476
- env: TOX_ENV=py35-django-2-es2 ES_APT_URL=https://packages.elastic.co/elasticsearch/2.x/debian
5577
python: 3.5
5678
- env: TOX_ENV=py35-django-2-es5 ES_APT_URL=https://artifacts.elastic.co/packages/5.x/apt
5779
python: 3.5
80+
- env: TOX_ENV=py35-django-2-es6 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
81+
python: 3.5
5882

5983
before_install:
6084
# work around https://github.com/travis-ci/travis-ci/issues/8363

‎README.rst

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ Features
2626
- Requirements
2727
- Django >= 1.8
2828
- Python 2.7, 3.4, 3.5, 3.6
29-
- Elasticsearch >= 2.0 < 6.0
29+
- Elasticsearch >= 2.0 < 7.0
3030

3131
.. _Search: http://elasticsearch-dsl.readthedocs.io/en/stable/search_dsl.html
3232

@@ -37,6 +37,9 @@ Install Django Elasticsearch DSL::
3737

3838
pip install django-elasticsearch-dsl
3939

40+
# Elasticsearch 6.x
41+
pip install 'elasticsearch-dsl>=6.0,<7.0'
42+
4043
# Elasticsearch 5.x
4144
pip install 'elasticsearch-dsl>=5.0,<6.0'
4245

‎django_elasticsearch_dsl/documents.py

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,10 @@
1717
DoubleField,
1818
FileField,
1919
IntegerField,
20+
KeywordField,
2021
LongField,
2122
ShortField,
22-
StringField,
23+
TextField,
2324
)
2425
from .indices import Index
2526
from .registries import registry
@@ -29,23 +30,23 @@
2930
models.AutoField: IntegerField,
3031
models.BigIntegerField: LongField,
3132
models.BooleanField: BooleanField,
32-
models.CharField: StringField,
33+
models.CharField: TextField,
3334
models.DateField: DateField,
3435
models.DateTimeField: DateField,
35-
models.EmailField: StringField,
36+
models.EmailField: TextField,
3637
models.FileField: FileField,
37-
models.FilePathField: StringField,
38+
models.FilePathField: KeywordField,
3839
models.FloatField: DoubleField,
3940
models.ImageField: FileField,
4041
models.IntegerField: IntegerField,
4142
models.NullBooleanField: BooleanField,
4243
models.PositiveIntegerField: IntegerField,
4344
models.PositiveSmallIntegerField: ShortField,
44-
models.SlugField: StringField,
45+
models.SlugField: KeywordField,
4546
models.SmallIntegerField: ShortField,
46-
models.TextField: StringField,
47+
models.TextField: TextField,
4748
models.TimeField: LongField,
48-
models.URLField: StringField,
49+
models.URLField: TextField,
4950
}
5051

5152

‎django_elasticsearch_dsl/fields.py

Lines changed: 74 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import collections
22
from types import MethodType
3+
import warnings
34

45
from django.core.exceptions import ObjectDoesNotExist
56
from django.db import models
@@ -8,7 +9,6 @@
89
from django.utils.functional import Promise
910

1011
from elasticsearch_dsl.field import (
11-
Attachment,
1212
Boolean,
1313
Byte,
1414
Completion,
@@ -24,7 +24,6 @@
2424
Nested,
2525
Object,
2626
Short,
27-
String,
2827
)
2928
from .exceptions import VariableLookupError
3029

@@ -91,16 +90,29 @@ def get_value_from_instance(self, instance, field_value_to_ignore=None):
9190
class ObjectField(DEDField, Object):
9291
def _get_inner_field_data(self, obj, field_value_to_ignore=None):
9392
data = {}
94-
for name, field in self.properties.to_dict().items():
95-
if not isinstance(field, DEDField):
96-
continue
9793

98-
if field._path == []:
99-
field._path = [name]
94+
if hasattr(self, 'properties'):
95+
for name, field in self.properties.to_dict().items():
96+
if not isinstance(field, DEDField):
97+
continue
10098

101-
data[name] = field.get_value_from_instance(
102-
obj, field_value_to_ignore
103-
)
99+
if field._path == []:
100+
field._path = [name]
101+
102+
data[name] = field.get_value_from_instance(
103+
obj, field_value_to_ignore
104+
)
105+
else:
106+
for name, field in self._doc_class._doc_type.mapping.properties._params.get('properties', {}).items():
107+
if not isinstance(field, DEDField):
108+
continue
109+
110+
if field._path == []:
111+
field._path = [name]
112+
113+
data[name] = field.get_value_from_instance(
114+
obj, field_value_to_ignore
115+
)
104116

105117
return data
106118

@@ -136,10 +148,6 @@ def get_value_from_instance(self, instance, field_value_to_ignore=None):
136148
return field
137149

138150

139-
class AttachmentField(DEDField, Attachment):
140-
pass
141-
142-
143151
class BooleanField(DEDField, Boolean):
144152
pass
145153

@@ -192,31 +200,71 @@ class ShortField(DEDField, Short):
192200
pass
193201

194202

195-
class StringField(DEDField, String):
196-
pass
197-
198-
199-
class FileField(DEDField, String):
203+
class FileFieldMixin(object):
200204
def get_value_from_instance(self, instance, field_value_to_ignore=None):
201-
_file = super(FileField, self).get_value_from_instance(
205+
_file = super(FileFieldMixin, self).get_value_from_instance(
202206
instance, field_value_to_ignore)
203207

204208
if isinstance(_file, FieldFile):
205209
return _file.url if _file else ''
206210
return _file
207211

208212

209-
# ES5 specific fields
213+
# ES5+ has text types Keyword and Text, ES2 has type String.
210214
try:
211-
from elasticsearch_dsl.field import (
212-
Keyword,
213-
Text,
214-
)
215+
from elasticsearch_dsl.field import Keyword, Text
215216

216217
class KeywordField(DEDField, Keyword):
217218
pass
218219

219220
class TextField(DEDField, Text):
220221
pass
222+
223+
class StringField(DEDField, Text):
224+
warnings.warn(
225+
'StringField is deprecated in Elasticsearch 5 and removed in '
226+
'Elasticsearch 6. Please use TextField and KeywordField instead.',
227+
DeprecationWarning
228+
)
229+
230+
class FileField(FileFieldMixin, DEDField, Text):
231+
pass
232+
233+
221234
except ImportError:
222-
pass
235+
from elasticsearch_dsl.field import String
236+
237+
class KeywordField(DEDField, String):
238+
def __init__(self, **kwargs):
239+
if 'index' not in kwargs or kwargs['index'] is True:
240+
# Unless a custom 'index' setting was provided, create the
241+
# Keyword field as a "not_analyzed" field.
242+
kwargs['index'] = 'not_analyzed'
243+
super(KeywordField, self).__init__(**kwargs)
244+
245+
class TextField(DEDField, String):
246+
pass
247+
248+
class StringField(DEDField, String):
249+
pass
250+
251+
class FileField(FileFieldMixin, DEDField, String):
252+
pass
253+
254+
255+
# Elasticsearch 2 and 5 have an Attachment field, ES 6 doesn't.
256+
try:
257+
from elasticsearch_dsl.field import Attachment
258+
259+
class AttachmentField(DEDField, Attachment):
260+
pass
261+
except ImportError:
262+
from elasticsearch_dsl.field import Text
263+
264+
class AttachmentField(DEDField, Text):
265+
name = 'attachment'
266+
warnings.warn(
267+
'AttachmentField is deprecated in Elasticsearch 5 and removed in '
268+
'Elasticsearch 6.',
269+
DeprecationWarning
270+
)

‎example/test_app/documents.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ def get_instances_from_related(self, related_instance):
6161

6262
@car.doc_type
6363
class ManufacturerDocument(DocType):
64-
country = fields.StringField()
64+
country = fields.TextField()
6565

6666
class Meta:
6767
model = Manufacturer
@@ -74,9 +74,9 @@ class Meta:
7474

7575

7676
class AdDocument(DocType):
77-
description = fields.StringField(
77+
description = fields.TextField(
7878
analyzer=html_strip,
79-
fields={'raw': fields.StringField(index='not_analyzed')}
79+
fields={'raw': fields.KeywordField()}
8080
)
8181

8282
class Meta:

‎requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
django>=1.9.6
2-
elasticsearch-dsl>=2.1.0,<6.0.0
2+
elasticsearch-dsl>=2.1.0,<7.0.0

‎requirements_dev.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
bumpversion==0.5.3
22
wheel==0.29.0
33
django>=2.0,<2.1
4-
elasticsearch-dsl>=2.1.0,<6.0.0
4+
elasticsearch-dsl>=2.1.0,<7.0.0
55

‎setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242
],
4343
include_package_data=True,
4444
install_requires=[
45-
'elasticsearch-dsl>=2.1.0,<6.0.0',
45+
'elasticsearch-dsl>=2.1.0,<7.0.0',
4646
],
4747
license="Apache Software License 2.0",
4848
zip_safe=False,

‎tests/__init__.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
from elasticsearch_dsl import VERSION
2+
3+
ES_MAJOR_VERSION = VERSION[0]

‎tests/documents.py

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ class Meta:
5151
'launched',
5252
'type',
5353
]
54+
doc_type = 'car_document'
5455

5556
def get_queryset(self):
5657
return super(CarDocument, self).get_queryset().select_related(
@@ -64,7 +65,10 @@ def get_instances_from_related(self, related_instance):
6465
return related_instance.car_set.all()
6566

6667

67-
@car_index.doc_type
68+
manufacturer_index = Index('test_manufacturers').settings(**index_settings)
69+
70+
71+
@manufacturer_index.doc_type
6872
class ManufacturerDocument(DocType):
6973
country = fields.StringField()
7074

@@ -76,6 +80,7 @@ class Meta:
7680
'country_code',
7781
'logo',
7882
]
83+
doc_type = 'manufacturer_document'
7984

8085

8186
class CarWithPrepareDocument(DocType):
@@ -124,9 +129,9 @@ def get_instances_from_related(self, related_instance):
124129

125130
@ad_index.doc_type
126131
class AdDocument(DocType):
127-
description = fields.StringField(
132+
description = fields.TextField(
128133
analyzer=html_strip,
129-
fields={'raw': fields.StringField(index='not_analyzed')}
134+
fields={'raw': fields.KeywordField()}
130135
)
131136

132137
class Meta:
@@ -137,6 +142,7 @@ class Meta:
137142
'modified',
138143
'url',
139144
]
145+
doc_type = 'ad_document'
140146

141147

142148
class PaginatedAdDocument(DocType):
@@ -150,6 +156,7 @@ class Meta:
150156
'modified',
151157
'url',
152158
]
159+
doc_type = 'paginated_ad_document'
153160

154161
def get_queryset(self):
155162
return Ad.objects.all().order_by('-id')

0 commit comments

Comments
 (0)