Store Pageviews in DB

readthedocs/analytics/admin.py

@@ -0,0 +1,16 @@
+"""Analytics Admin classes."""
+
+from django.contrib import admin
+
+from .models import PageView
+
+
+class PageViewAdmin(admin.ModelAdmin):
+    raw_id_fields = ('project', 'version')
+    list_display = ('project', 'version', 'path', 'view_count', 'date')
+    search_fields = ('project__slug', 'version__slug', 'path')
+    readonly_fields = ('date',)
+    list_select_related = ('project', 'version', 'version__project')
+
+
+admin.site.register(PageView, PageViewAdmin)

readthedocs/analytics/migrations/0001_initial.py

@@ -0,0 +1,32 @@
+# Generated by Django 2.2.12 on 2020-05-19 00:45
+
+import datetime
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+        ('builds', '0022_migrate_protected_versions'),
+        ('projects', '0048_remove_version_privacy_field'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='PageView',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('path', models.CharField(max_length=4096)),
+                ('view_count', models.PositiveIntegerField(default=0)),
+                ('date', models.DateField(db_index=True, default=datetime.date.today)),
+                ('project', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='page_views', to='projects.Project')),
+                ('version', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='page_views', to='builds.Version', verbose_name='Version')),
+            ],
+            options={
+                'unique_together': {('project', 'version', 'path', 'date')},
+            },
+        ),
+    ]

readthedocs/analytics/models.py

@@ -0,0 +1,130 @@
+"""Analytics modeling to help understand the projects on Read the Docs."""
+
+import datetime
+
+from django.db import models
+from django.db.models import Sum
+from django.utils import timezone
+from django.utils.translation import ugettext_lazy as _
+
+from readthedocs.builds.models import Version
+from readthedocs.projects.models import Project
+
+
+def _last_30_days_iter():
+    """Returns iterator for previous 30 days (including today)."""
+    thirty_days_ago = timezone.now().date() - timezone.timedelta(days=30)
+
+    # this includes the current day, len() = 31
+    return (thirty_days_ago + timezone.timedelta(days=n) for n in range(31))
+
+
+class PageView(models.Model):
+
+    """PageView counts per day for a project, version, and path."""
+
+    project = models.ForeignKey(
+        Project,
+        related_name='page_views',
+        on_delete=models.CASCADE,
+    )
+    version = models.ForeignKey(
+        Version,
+        verbose_name=_('Version'),
+        related_name='page_views',
+        on_delete=models.CASCADE,
+    )
+    path = models.CharField(max_length=4096)
+    view_count = models.PositiveIntegerField(default=0)
+    date = models.DateField(default=datetime.date.today, db_index=True)
+
+    class Meta:
+        unique_together = ("project", "version", "path", "date")
+
+    def __str__(self):
+        return f'PageView: [{self.project.slug}:{self.version.slug}] - {self.path} for {self.date}'
+
+    @classmethod
+    def top_viewed_pages(cls, project, since=None):
+        """
+        Returns top 10 pages according to view counts.
+
+        Structure of returned data is compatible to make graphs.
+        Sample returned data::
+        {
+            'pages': ['index', 'config-file/v1', 'intro/import-guide'],
+            'view_counts': [150, 120, 100]
+        }
+        This data shows that `index` is the most viewed page having 150 total views,
+        followed by `config-file/v1` and `intro/import-guide` having 120 and
+        100 total page views respectively.
+        """
+        if since is None:
+            since = timezone.now().date() - timezone.timedelta(days=30)
+
+        qs = (
+            cls.objects
+            .filter(project=project, date__gte=since)
+            .values_list('path')
+            .annotate(total_views=Sum('view_count'))
+            .values_list('path', 'total_views')
+            .order_by('-total_views')[:10]
+        )
+
+        pages = []
+        view_counts = []
+
+        for data in qs.iterator():
+            pages.append(data[0])
+            view_counts.append(data[1])
+
+        final_data = {
+            'pages': pages,
+            'view_counts': view_counts,
+        }
+
+        return final_data
+
+    @classmethod
+    def page_views_by_date(cls, project_slug, since=None):
+        """
+        Returns the total page views count for last 30 days for a particular project.
+
+        Structure of returned data is compatible to make graphs.
+        Sample returned data::
+            {
+                'labels': ['01 Jul', '02 Jul', '03 Jul'],
+                'int_data': [150, 200, 143]
+            }
+        This data shows that there were 150 page views on 01 July,
+        200 page views on 02 July and 143 page views on 03 July.
+        """
+        if since is None:
+            since = timezone.now().date() - timezone.timedelta(days=30)
+
+        qs = cls.objects.filter(
+            project__slug=project_slug,
+            date__gt=since,
+        ).values('date').annotate(total_views=Sum('view_count')).order_by('date')
+
+        count_dict = dict(
+            qs.order_by('date').values_list('date', 'total_views')
+        )
+
+        # This fills in any dates where there is no data
+        # to make sure we have a full 30 days of dates
+        count_data = [count_dict.get(date) or 0 for date in _last_30_days_iter()]
+
+        # format the date value to a more readable form
+        # Eg. `16 Jul`
+        last_30_days_str = [
+            timezone.datetime.strftime(date, '%d %b')
+            for date in _last_30_days_iter()
+        ]
+
+        final_data = {
+            'labels': last_30_days_str,
+            'int_data': count_data,
+        }
+
+        return final_data

readthedocs/analytics/tasks.py

@@ -3,10 +3,15 @@
 """Tasks for Read the Docs' analytics."""
 
 from django.conf import settings
+from django.db.models import F
+from django.utils import timezone
 
 import readthedocs
 from readthedocs.worker import app
+from readthedocs.builds.models import Version
+from readthedocs.projects.models import Project
 
+from .models import PageView
 from .utils import send_to_analytics
 
 
@@ -70,3 +75,30 @@ def analytics_event(
     data.update(DEFAULT_PARAMETERS)
     data.update(kwargs)
     send_to_analytics(data)
+
+
+@app.task(queue='web')
+def increase_page_view_count(project_slug, version_slug, path):
+    """Increase the page view count for the given project."""
+    project = Project.objects.get(slug=project_slug)
+
+    page_view, _ = PageView.objects.get_or_create(
+        project=project,
+        version=Version.objects.get(project=project, slug=version_slug),
+        path=path,
+        date=timezone.now().date(),
+    )
+    PageView.objects.filter(pk=page_view.pk).update(
+        view_count=F('view_count') + 1
+    )
+
+
+@app.task(queue='web')
+def delete_old_page_counts():
+    """
+    Delete page counts older than 30 days.
+
+    This is intended to run from a periodic task daily.
+    """
+    thirty_days_ago = timezone.now().date() - timezone.timedelta(days=30)
+    return PageView.objects.filter(date__lt=thirty_days_ago).delete()

readthedocs/analytics/tests.py

@@ -1,6 +1,14 @@
-# -*- coding: utf-8 -*-
+from unittest import mock
+
+from django_dynamic_fixture import get
 from django.test import TestCase, RequestFactory
+from django.utils import timezone
+
+from readthedocs.builds.models import Version
+from readthedocs.projects.models import Project
 
+from .models import PageView
+from .tasks import increase_page_view_count
 from .utils import (
     anonymize_ip_address,
     anonymize_user_agent,
@@ -87,3 +95,82 @@ def test_get_client_ip_with_remote_addr(self):
         request.META['REMOTE_ADDR'] = '203.0.113.195'
         client_ip = get_client_ip(request)
         self.assertEqual(client_ip, '203.0.113.195')
+
+
+class AnalyticsTasksTests(TestCase):
+    def test_increase_page_view_count(self):
+        project = get(
+            Project,
+            slug='project-1',
+        )
+        version = get(Version, slug='1.8', project=project)
+
+        today = timezone.now()
+        tomorrow = timezone.now() + timezone.timedelta(days=1)
+        yesterday = timezone.now() - timezone.timedelta(days=1)
+
+        assert (
+            PageView.objects.all().count() == 0
+        ), 'There\'s no PageView object created yet.'
+
+        # testing for yesterday
+        with mock.patch('readthedocs.analytics.tasks.timezone.now') as mocked_timezone:
+            mocked_timezone.return_value = yesterday
+
+            increase_page_view_count(
+                project_slug=project.slug,
+                version_slug=version.slug,
+                path='index',
+            )
+
+            assert (
+                PageView.objects.all().count() == 1
+            ), 'PageView object for path \'index\' is created'
+            assert (
+                PageView.objects.all().first().view_count == 1
+            ), '\'index\' has 1 view'
+
+            increase_page_view_count(
+                project_slug=project.slug,
+                version_slug=version.slug,
+                path='index',
+            )
+
+            assert (
+                PageView.objects.all().count() == 1
+            ), 'PageView object for path \'index\' is already created'
+            assert (
+                PageView.objects.all().first().view_count == 2
+            ), '\'index\' has 2 views now'
+
+        # testing for today
+        with mock.patch('readthedocs.analytics.tasks.timezone.now') as mocked_timezone:
+            mocked_timezone.return_value = today
+            increase_page_view_count(
+                project_slug=project.slug,
+                version_slug=version.slug,
+                path='index',
+            )
+
+            assert (
+                PageView.objects.all().count() == 2
+            ), 'PageView object for path \'index\' is created for two days (yesterday and today)'
+            assert (
+                PageView.objects.all().order_by('-date').first().view_count == 1
+            ), '\'index\' has 1 view today'
+
+        # testing for tomorrow
+        with mock.patch('readthedocs.analytics.tasks.timezone.now') as mocked_timezone:
+            mocked_timezone.return_value = tomorrow
+            increase_page_view_count(
+                project_slug=project.slug,
+                version_slug=version.slug,
+                path='index',
+            )
+
+            assert (
+                PageView.objects.all().count() == 3
+            ), 'PageView object for path \'index\' is created for three days (yesterday, today & tomorrow)'
+            assert (
+                PageView.objects.all().order_by('-date').first().view_count == 1
+            ), '\'index\' has 1 view tomorrow'

readthedocs/api/v2/views/footer_views.py

@@ -20,6 +20,7 @@
     highest_version,
     parse_version_failsafe,
 )
+from readthedocs.analytics.tasks import increase_page_view_count
 
 
 def get_version_compare_data(project, base_version=None):
@@ -221,6 +222,15 @@ def get(self, request, format=None):
             'version_supported': version.supported,
         }
 
+        # increase the page view count for the given page
+        page_slug = request.GET.get('page', '')
+        if page_slug:
+            increase_page_view_count.delay(
+                project_slug=context['project'].slug,
+                version_slug=context['version'].slug,
+                path=page_slug
+            )
+
         # Allow folks to hook onto the footer response for various information
         # collection, or to modify the resp_data.
         footer_response.send(

readthedocs/projects/urls/private.py

@@ -26,6 +26,7 @@
     IntegrationExchangeDetail,
     IntegrationList,
     IntegrationWebhookSync,
+    TrafficAnalyticsView,
     ProjectAdvancedUpdate,
     ProjectAdvertisingUpdate,
     ProjectDashboard,
@@ -139,6 +140,10 @@
         SearchAnalytics.as_view(),
         name='projects_search_analytics',
     ),
+    url(
+        r'^(?P<project_slug>[-\w]+)/traffic-analytics/$',
+        TrafficAnalyticsView.as_view(), name='projects_traffic_analytics',
+    ),
 ]
 
 domain_urls = [