Skip to content

Commit 17908d3

Browse files
xrmxxrmx
committed
search: fixup elasticsearch query syntax for 5.5
fields is now _source and filter should be inside the query. aggs is the new facets and we can drop filtering on aggs as they should be calculated on the query results so after filtering.
1 parent b9461f5 commit 17908d3

File tree

4 files changed

+52
-59
lines changed

4 files changed

+52
-59
lines changed

‎readthedocs/projects/views/public.py

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -305,6 +305,10 @@ def elastic_project_search(request, project_slug):
305305
{'match': {'title': {'query': query, 'boost': 10}}},
306306
{'match': {'headers': {'query': query, 'boost': 5}}},
307307
{'match': {'content': {'query': query}}},
308+
],
309+
'filter': [
310+
{'term': {'project': project_slug}},
311+
{'term': {'version': version_slug}},
308312
]
309313
}
310314
},
@@ -315,13 +319,7 @@ def elastic_project_search(request, project_slug):
315319
'content': {},
316320
}
317321
},
318-
'fields': ['title', 'project', 'version', 'path'],
319-
'filter': {
320-
'and': [
321-
{'term': {'project': project_slug}},
322-
{'term': {'version': version_slug}},
323-
]
324-
},
322+
'_source': ['title', 'project', 'version', 'path'],
325323
'size': 50, # TODO: Support pagination.
326324
}
327325

@@ -335,9 +333,12 @@ def elastic_project_search(request, project_slug):
335333
if results:
336334
# pre and post 1.0 compat
337335
for num, hit in enumerate(results['hits']['hits']):
338-
for key, val in list(hit['fields'].items()):
336+
for key, val in list(hit['_source'].items()):
339337
if isinstance(val, list):
340-
results['hits']['hits'][num]['fields'][key] = val[0]
338+
results['hits']['hits'][num]['_source'][key] = val[0]
339+
# we cannot render attributes starting with an underscore
340+
hit['fields'] = hit['_source']
341+
del hit['_source']
341342

342343
return render(
343344
request,

‎readthedocs/restapi/views/search_views.py

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@ def search(request):
6464
# Supplement result paths with domain information on project
6565
hits = results.get('hits', {}).get('hits', [])
6666
for (n, hit) in enumerate(hits):
67-
fields = hit.get('fields', {})
67+
fields = hit.get('_source', {})
6868
search_project = fields.get('project')[0]
6969
search_version = fields.get('version')[0]
7070
path = fields.get('path')[0]
@@ -77,9 +77,12 @@ def search(request):
7777
)
7878
except ProjectRelationship.DoesNotExist:
7979
pass
80-
results['hits']['hits'][n]['fields']['link'] = (
80+
results['hits']['hits'][n]['_source']['link'] = (
8181
canonical_url + path
8282
)
83+
# we cannot render attributes starting with an underscore
84+
results['hits']['hits'][n]['fields'] = results['hits']['hits'][n]['_source']
85+
del results['hits']['hits'][n]['_source']
8386

8487
return Response({'results': results})
8588

‎readthedocs/search/lib.py

Lines changed: 28 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -25,9 +25,9 @@ def search_project(request, query, language=None):
2525
]
2626
},
2727
},
28-
"facets": {
28+
"aggs": {
2929
"language": {
30-
"terms": {"field": "lang"},
30+
"terms": {"field": "lang.keyword"},
3131
},
3232
},
3333
"highlight": {
@@ -36,13 +36,12 @@ def search_project(request, query, language=None):
3636
"description": {},
3737
}
3838
},
39-
"fields": ["name", "slug", "description", "lang", "url"],
39+
"_source": ["name", "slug", "description", "lang", "url"],
4040
"size": 50 # TODO: Support pagination.
4141
}
4242

4343
if language:
44-
body['facets']['language']['facet_filter'] = {"term": {"lang": language}}
45-
body['filter'] = {"term": {"lang": language}}
44+
body['query']['bool']['filter'] = {"term": {"lang": language}}
4645

4746
before_project_search.send(request=request, sender=ProjectIndex, body=body)
4847

@@ -89,15 +88,15 @@ def search_file(request, query, project_slug=None, version_slug=LATEST, taxonomy
8988
]
9089
}
9190
},
92-
"facets": {
91+
"aggs": {
9392
"taxonomy": {
94-
"terms": {"field": "taxonomy"},
93+
"terms": {"field": "taxonomy.keyword"},
9594
},
9695
"project": {
97-
"terms": {"field": "project"},
96+
"terms": {"field": "project.keyword"},
9897
},
9998
"version": {
100-
"terms": {"field": "version"},
99+
"terms": {"field": "version.keyword"},
101100
},
102101
},
103102
"highlight": {
@@ -107,12 +106,12 @@ def search_file(request, query, project_slug=None, version_slug=LATEST, taxonomy
107106
"content": {},
108107
}
109108
},
110-
"fields": ["title", "project", "version", "path"],
109+
"_source": ["title", "project", "version", "path"],
111110
"size": 50 # TODO: Support pagination.
112111
}
113112

114113
if project_slug or version_slug or taxonomy:
115-
final_filter = {"and": []}
114+
final_filter = []
116115

117116
if project_slug:
118117
try:
@@ -126,7 +125,7 @@ def search_file(request, query, project_slug=None, version_slug=LATEST, taxonomy
126125
in Project.objects.public(
127126
request.user).filter(
128127
superprojects__parent__slug=project.slug))
129-
final_filter['and'].append({"terms": {"project": project_slugs}})
128+
final_filter.append({"terms": {"project": project_slugs}})
130129

131130
# Add routing to optimize search by hitting the right shard.
132131
# This purposely doesn't apply routing if the project has more
@@ -141,15 +140,12 @@ def search_file(request, query, project_slug=None, version_slug=LATEST, taxonomy
141140
return None
142141

143142
if version_slug:
144-
final_filter['and'].append({'term': {'version': version_slug}})
143+
final_filter.append({'term': {'version': version_slug}})
145144

146145
if taxonomy:
147-
final_filter['and'].append({'term': {'taxonomy': taxonomy}})
146+
final_filter.append({'term': {'taxonomy': taxonomy}})
148147

149-
body['filter'] = final_filter
150-
body['facets']['project']['facet_filter'] = final_filter
151-
body['facets']['version']['facet_filter'] = final_filter
152-
body['facets']['taxonomy']['facet_filter'] = final_filter
148+
body['query']['bool']['filter'] = final_filter
153149

154150
if settings.DEBUG:
155151
print("Before Signal")
@@ -167,9 +163,9 @@ def search_section(request, query, project_slug=None, version_slug=LATEST,
167163
"""
168164
Search for a section of content.
169165
170-
When you search, you will have a ``project`` facet, which includes the
166+
When you search, you will have a ``project`` facet (aggs), which includes the
171167
number of matching sections per project. When you search inside a project,
172-
the ``path`` facet will show the number of matching sections per page.
168+
the ``path`` aggs will show the number of matching sections per page.
173169
174170
:param request: Request instance
175171
:param query: string to use in query
@@ -198,12 +194,9 @@ def search_section(request, query, project_slug=None, version_slug=LATEST,
198194
]
199195
}
200196
},
201-
"facets": {
197+
"aggs": {
202198
"project": {
203-
"terms": {"field": "project"},
204-
"facet_filter": {
205-
"term": {"version": version_slug},
206-
}
199+
"terms": {"field": "project.keyword"},
207200
},
208201
},
209202
"highlight": {
@@ -212,36 +205,29 @@ def search_section(request, query, project_slug=None, version_slug=LATEST,
212205
"content": {},
213206
}
214207
},
215-
"fields": ["title", "project", "version", "path", "page_id", "content"],
208+
"_source": ["title", "project", "version", "path", "page_id", "content"],
216209
"size": 10 # TODO: Support pagination.
217210
}
218211

219212
if project_slug:
220-
body['filter'] = {
221-
"and": [
222-
{"term": {"project": project_slug}},
223-
{"term": {"version": version_slug}},
224-
]
225-
}
226-
body['facets']['path'] = {
213+
body['query']['bool']['filter'] = [
214+
{"term": {"project": project_slug}},
215+
{"term": {"version": version_slug}},
216+
]
217+
body['aggs']['path'] = {
227218
"terms": {"field": "path"},
228-
"facet_filter": {
229-
"term": {"project": project_slug},
230-
}
231219
},
232220
# Add routing to optimize search by hitting the right shard.
233221
kwargs['routing'] = project_slug
234222

235223
if path:
236-
body['filter'] = {
237-
"and": [
238-
{"term": {"path": path}},
239-
]
240-
}
224+
body['query']['bool']['filter'] = [
225+
{"term": {"path": path}},
226+
]
241227

242228
if path and not project_slug:
243229
# Show facets when we only have a path
244-
body['facets']['path'] = {
230+
body['aggs']['path'] = {
245231
"terms": {"field": "path"}
246232
}
247233

‎readthedocs/search/views.py

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -55,16 +55,19 @@ def elastic_search(request):
5555
if results:
5656
# pre and post 1.0 compat
5757
for num, hit in enumerate(results['hits']['hits']):
58-
for key, val in list(hit['fields'].items()):
58+
for key, val in list(hit['_source'].items()):
5959
if isinstance(val, list):
60-
results['hits']['hits'][num]['fields'][key] = val[0]
60+
results['hits']['hits'][num]['_source'][key] = val[0]
61+
# we cannot render attributes starting with an underscore
62+
hit['fields'] = hit['_source']
63+
del hit['_source']
6164

62-
if 'facets' in results:
65+
if 'aggregations' in results:
6366
for facet_type in ['project', 'version', 'taxonomy', 'language']:
64-
if facet_type in results['facets']:
67+
if facet_type in results['aggregations']:
6568
facets[facet_type] = collections.OrderedDict()
66-
for term in results['facets'][facet_type]['terms']:
67-
facets[facet_type][term['term']] = term['count']
69+
for term in results['aggregations'][facet_type]['buckets']:
70+
facets[facet_type][term['key']] = term['doc_count']
6871

6972
if settings.DEBUG:
7073
print(pprint(results))

0 commit comments

Comments
 (0)