Merge pull request #7080 from readthedocs/track-total-results

Search: track total results

Author: stsewd

readthedocs/projects/views/private.py

@@ -1013,7 +1013,7 @@ def get_context_data(self, **kwargs):
                 qs.values('query')
                 .annotate(count=Count('id'))
                 .order_by('-count', 'query')
-                .values_list('query', 'count')
+                .values_list('query', 'count', 'total_results')
             )
 
             # only show top 100 queries
@@ -1040,7 +1040,7 @@ def _search_analytics_csv_data(self):
                 created__date__lte=now,
             )
             .order_by('-created')
-            .values_list('created', 'query')
+            .values_list('created', 'query', 'total_results')
         )
 
         file_name = '{project_slug}_from_{start}_to_{end}.csv'.format(
@@ -1052,8 +1052,8 @@ def _search_analytics_csv_data(self):
         file_name = '-'.join([text for text in file_name.split() if text])
 
         csv_data = (
-            [timezone.datetime.strftime(time, '%Y-%m-%d %H:%M:%S'), query]
-            for time, query in data
+            [timezone.datetime.strftime(time, '%Y-%m-%d %H:%M:%S'), query, total_results]
+            for time, query, total_results in data
         )
         pseudo_buffer = Echo()
         writer = csv.writer(pseudo_buffer)

readthedocs/rtd_tests/tests/test_views.py

@@ -286,15 +286,15 @@ def test_top_queries(self):
             test_time.return_value = self.test_time
 
             expected_result = [
-                ('hello world', 5),
-                ('documentation', 4),
-                ('read the docs', 4),
-                ('advertising', 3),
-                ('elasticsearch', 2),
-                ('sphinx', 2),
-                ('github', 1),
-                ('hello', 1),
-                ('search', 1),
+                ('hello world', 5, 0),
+                ('documentation', 4, 0),
+                ('read the docs', 4, 0),
+                ('advertising', 3, 0),
+                ('elasticsearch', 2, 0),
+                ('sphinx', 2, 0),
+                ('github', 1, 0),
+                ('hello', 1, 0),
+                ('search', 1, 0),
             ]
 
             resp = self.client.get(self.analyics_page)

readthedocs/search/migrations/0002_add_total_results_field.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.12 on 2020-05-14 17:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('search', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='searchquery',
+            name='total_results',
+            field=models.IntegerField(default=0, null=True, verbose_name='Total results'),
+        ),
+    ]

readthedocs/search/models.py

@@ -32,6 +32,12 @@ class SearchQuery(TimeStampedModel):
         _('Query'),
         max_length=4092,
     )
+    total_results = models.IntegerField(
+        _('Total results'),
+        default=0,
+        # TODO: to avoid downtime, remove later.
+        null=True,
+    )
     objects = RelatedProjectQuerySet.as_manager()
 
     class Meta:

readthedocs/search/tasks.py

@@ -144,7 +144,7 @@ def delete_old_search_queries_from_db():
 
 @app.task(queue='web')
 def record_search_query(project_slug, version_slug, query, total_results, time_string):
-    """Record/update search query in database."""
+    """Record/update a search query for analytics."""
     if not project_slug or not version_slug or not query:
         log.debug(
             'Not recording the search query. Passed arguments: '
@@ -162,11 +162,12 @@ def record_search_query(project_slug, version_slug, query, total_results, time_s
         modified__gte=before_10_sec,
     ).order_by('-modified')
 
-    # check if partial query exists,
-    # if yes, then just update the object.
+    # If a partial query exists,
+    # then just update that object.
     for partial_query in partial_query_qs.iterator():
         if query.startswith(partial_query.query):
             partial_query.query = query
+            partial_query.total_results = total_results
             partial_query.save()
             return
 
@@ -190,9 +191,9 @@ def record_search_query(project_slug, version_slug, query, total_results, time_s
         )
         return
 
-    version_qs = Version.objects.filter(project=project, slug=version_slug)
+    version = Version.objects.filter(project=project, slug=version_slug).first()
 
-    if not version_qs.exists():
+    if not version:
         log.debug(
             'Not recording the search query because version does not exist. '
             'project_slug: %s, version_slug: %s' % (
@@ -201,11 +202,10 @@ def record_search_query(project_slug, version_slug, query, total_results, time_s
         )
         return
 
-    version = version_qs.first()
-
-    # make a new SearchQuery object.
+    # Create a new SearchQuery object.
     SearchQuery.objects.create(
         project=project,
         version=version,
         query=query,
+        total_results=total_results,
     )

readthedocs/templates/projects/projects_search_analytics.html

@@ -16,9 +16,10 @@ <h3>{% trans "Top queries" %}</h3>
   <div class="module-list">
     <div class="module-list-wrapper">
       <ul class="long-list-overflow">
-        {% for query, count in queries %}
+        {% for query, count, total_results in queries %}
           <li class="module-item">
             {{ query }}
+            ({{ total_results }} result{{ total_results|pluralize:"s" }})
             <span class="right quiet">
               {{ count }} search{{ count|pluralize:"es" }}
             </span>