Merge branch 'master' into allow-query-params-in-redirects

Author: stsewd

.github/mergeable.yml

@@ -0,0 +1,28 @@
+# ProBot Mergeable Bot
+# https://github.com/jusx/mergeable
+
+mergeable:
+  pull_requests:
+    approvals:
+      # Minimum of approvals needed.
+      min: 1
+        message: 'The PR must have a minimum of 1 approvals.'
+
+    description:
+      no_empty:
+        # Do not allow empty descriptions on PR.
+        enabled: false
+        message: 'Description can not be empty.'
+
+      must_exclude:
+        # Do not allow 'DO NOT MERGE' phrase on PR's description.
+        regex: 'DO NOT MERGE'
+        message: 'Description says that the PR should not be merged yet.'
+
+    # Do not allow 'WIP' on PR's title.
+    title: 'WIP'
+
+    label:
+      # Do not allow PR with label 'PR: work in progress'
+      must_exclude: 'PR: work in progress'
+      message: 'This PR is work in progress.'

.travis.yml

@@ -1,11 +1,10 @@
 language: python
 python:
-  - 2.7
   - 3.6
-env:
-  - ES_VERSION=1.3.9 ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
 matrix:
   include:
+    - python: 3.6
+      env: TOXENV=py36 ES_VERSION=1.3.9 ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
     - python: 3.6
       env: TOXENV=docs
     - python: 3.6
@@ -45,6 +44,6 @@ notifications:
 
 branches:
   only:
-  - master 
+  - master
   - rel # Community release branch
   - relcorp # Corporate release branch

LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2010-2017 Read the Docs, Inc & contributors
+Copyright (c) 2010-2019 Read the Docs, Inc & contributors
 
 Permission is hereby granted, free of charge, to any person
 obtaining a copy of this software and associated documentation

README.rst

@@ -78,6 +78,6 @@ when you push to GitHub.
 License
 -------
 
-`MIT`_ © 2010-2017 Read the Docs, Inc & contributors
+`MIT`_ © 2010-2019 Read the Docs, Inc & contributors
 
 .. _MIT: LICENSE

common

@@ -230,3 +230,49 @@ What commit of Read the Docs is in production?
 ----------------------------------------------
 
 We deploy readthedocs.org from the `rel` branch in our GitHub repository. You can see the latest commits that have been deployed by looking on GitHub: https://github.com/rtfd/readthedocs.org/commits/rel
+
+
+How can I avoid search results having a deprecated version of my docs?
+---------------------------------------------------------------------
+
+If readers search something related to your docs in Google, it will probably return the most relevant version of your documentation.
+It may happen that this version is already deprecated and you want to stop Google indexing it as a result,
+and start suggesting the latest (or newer) one.
+
+To accomplish this, you can add a ``robots.txt`` file to your documentation's root so it ends up served at the root URL of your project
+(for example, https://yourproject.readthedocs.io/robots.txt).
+
+
+Minimal example of ``robots.txt``
++++++++++++++++++++++++++++++++++
+
+::
+
+   User-agent: *
+   Disallow: /en/deprecated-version/
+   Disallow: /en/2.0/
+
+.. note::
+
+   See `Google's docs`_ for its full syntax.
+
+This file has to be served as is under ``/robots.txt``.
+Depending if you are using Sphinx or MkDocs, you will need a different configuration for this.
+
+
+Sphinx
+~~~~~~
+
+Sphinx uses `html_extra`_ option to add static files to the output.
+You need to create a ``robots.txt`` file and put it under the path defined in ``html_extra``.
+
+
+MkDocs
+~~~~~~
+
+MkDocs needs the ``robots.txt`` to be at the directory defined at `docs_dir`_ config.
+
+
+.. _Google's docs: https://support.google.com/webmasters/answer/6062608
+.. _html_extra: https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_extra_path
+.. _docs_dir: https://www.mkdocs.org/user-guide/configuration/#docs_dir

docs/faq.rst

@@ -161,9 +161,9 @@ that your VCS provider is not configured correctly. If there is payload
 information on your Read the Docs project, you might need to verify that your
 versions are configured to build correctly.
 
-Either way, it may help to either resync your webhook intergration (see
+Either way, it may help to either resync your webhook integration (see
 `Resyncing webhooks`_ for information on this process), or set up an entirely
-new webhook intergration.
+new webhook integration.
 
 .. _webhook-github-services:
 
@@ -180,7 +180,7 @@ In order for your project to continue automatically building, you will need to
 configure your GitHub repository with a new webhook. You can use either a
 connected GitHub account and a :ref:`GitHub webhook integration <webhook-integration-github>`
 on your Read the Docs project, or you can use a
-:ref:`generic webhook integraiton <webhook-integration-generic>` without a connected
+:ref:`generic webhook integration <webhook-integration-generic>` without a connected
 account.
 
 .. [1] https://developer.github.com/changes/2018-04-25-github-services-deprecation/

docs/webhooks.rst

@@ -1,9 +1,10 @@
 # -*- coding: utf-8 -*-
+
 """Read the Docs."""
 
 import os.path
 
-from future.moves.configparser import RawConfigParser
+from configparser import RawConfigParser
 
 
 def get_version(setupcfg_path):

readthedocs/__init__.py

@@ -1,3 +1,5 @@
-"""App init"""
+# -*- coding: utf-8 -*-
 
-default_app_config = 'readthedocs.analytics.apps.AnalyticsAppConfig'    # noqa
+"""App init."""
+
+default_app_config = 'readthedocs.analytics.apps.AnalyticsAppConfig'  # noqa

readthedocs/analytics/__init__.py

@@ -1,12 +1,13 @@
+# -*- coding: utf-8 -*-
+
 """Django app config for the analytics app."""
 
-from __future__ import absolute_import
 from django.apps import AppConfig
 
 
 class AnalyticsAppConfig(AppConfig):
 
-    """Analytics app init code"""
+    """Analytics app init code."""
 
     name = 'readthedocs.analytics'
     verbose_name = 'Analytics'

readthedocs/analytics/apps.py

@@ -1,6 +1,6 @@
-"""Tasks for Read the Docs' analytics"""
+# -*- coding: utf-8 -*-
 
-from __future__ import absolute_import
+"""Tasks for Read the Docs' analytics."""
 
 from django.conf import settings
 
@@ -11,24 +11,24 @@
 
 
 DEFAULT_PARAMETERS = {
-    'v': '1',               # analytics version (always 1)
-    'aip': '1',             # anonymize IP
+    'v': '1',  # analytics version (always 1)
+    'aip': '1',  # anonymize IP
     'tid': settings.GLOBAL_ANALYTICS_CODE,
 
     # User data
-    'uip': None,            # User IP address
-    'ua': None,             # User agent
+    'uip': None,  # User IP address
+    'ua': None,  # User agent
 
     # Application info
     'an': 'Read the Docs',
-    'av': readthedocs.__version__,    # App version
+    'av': readthedocs.__version__,  # App version
 }
 
 
 @app.task(queue='web')
 def analytics_pageview(url, title=None, **kwargs):
     """
-    Send a pageview to Google Analytics
+    Send a pageview to Google Analytics.
 
     :see: https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters
     :param url: the URL of the pageview
@@ -37,18 +37,21 @@ def analytics_pageview(url, title=None, **kwargs):
     """
     data = {
         't': 'pageview',
-        'dl': url,          # URL of the pageview (required)
-        'dt': title,        # Title of the page
+        'dl': url,  # URL of the pageview (required)
+        'dt': title,  # Title of the page
     }
     data.update(DEFAULT_PARAMETERS)
     data.update(kwargs)
     send_to_analytics(data)
 
 
 @app.task(queue='web')
-def analytics_event(event_category, event_action, event_label=None, event_value=None, **kwargs):
+def analytics_event(
+        event_category, event_action, event_label=None, event_value=None,
+        **kwargs
+):
     """
-    Send an analytics event to Google Analytics
+    Send an analytics event to Google Analytics.
 
     :see: https://developers.google.com/analytics/devguides/collection/protocol/v1/devguide#event
     :param event_category: the category of the event
@@ -58,11 +61,11 @@ def analytics_event(event_category, event_action, event_label=None, event_value=
     :param kwargs: extra event parameters to send to GA
     """
     data = {
-        't': 'event',           # GA event - don't change
-        'ec': event_category,   # Event category (required)
-        'ea': event_action,     # Event action (required)
-        'el': event_label,      # Event label
-        'ev': event_value,      # Event value (numeric)
+        't': 'event',  # GA event - don't change
+        'ec': event_category,  # Event category (required)
+        'ea': event_action,  # Event action (required)
+        'el': event_label,  # Event label
+        'ev': event_value,  # Event value (numeric)
     }
     data.update(DEFAULT_PARAMETERS)
     data.update(kwargs)

readthedocs/analytics/tasks.py

@@ -1,5 +1,4 @@
-from __future__ import absolute_import, unicode_literals
-
+# -*- coding: utf-8 -*-
 from django.test import TestCase
 
 from .utils import anonymize_ip_address, anonymize_user_agent
@@ -29,4 +28,3 @@ def test_anonymize_ua(self):
             anonymize_user_agent('Some rare user agent'),
             'Rare user agent',
         )
-

readthedocs/analytics/tests.py

@@ -1,26 +1,23 @@
-"""Utilities related to analytics"""
+# -*- coding: utf-8 -*-
+
+"""Utilities related to analytics."""
 
-from __future__ import absolute_import, unicode_literals
 import hashlib
+import ipaddress
 import logging
 
+import requests
 from django.conf import settings
-from django.utils.encoding import force_text, force_bytes
 from django.utils.crypto import get_random_string
-import requests
+from django.utils.encoding import force_bytes, force_text
 from user_agents import parse
 
-try:
-    # Python 3.3+ only
-    import ipaddress
-except ImportError:
-    from .vendor import ipaddress
 
-log = logging.getLogger(__name__)   # noqa
+log = logging.getLogger(__name__)  # noqa
 
 
 def get_client_ip(request):
-    """Gets the real IP based on a request object"""
+    """Gets the real IP based on a request object."""
     ip_address = request.META.get('REMOTE_ADDR')
 
     # Get the original IP address (eg. "X-Forwarded-For: client, proxy1, proxy2")
@@ -32,7 +29,7 @@ def get_client_ip(request):
 
 
 def anonymize_ip_address(ip_address):
-    """Anonymizes an IP address by zeroing the last 2 bytes"""
+    """Anonymizes an IP address by zeroing the last 2 bytes."""
     # Used to anonymize an IP by zero-ing out the last 2 bytes
     ip_mask = int('0xFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000', 16)
 
@@ -46,7 +43,7 @@ def anonymize_ip_address(ip_address):
 
 
 def anonymize_user_agent(user_agent):
-    """Anonymizes rare user agents"""
+    """Anonymizes rare user agents."""
     # If the browser family is not recognized, this is a rare user agent
     parsed_ua = parse(user_agent)
     if parsed_ua.browser.family == 'Other' or parsed_ua.os.family == 'Other':
@@ -56,7 +53,7 @@ def anonymize_user_agent(user_agent):
 
 
 def send_to_analytics(data):
-    """Sends data to Google Analytics"""
+    """Sends data to Google Analytics."""
     if data.get('uip') and data.get('ua'):
         data['cid'] = generate_client_id(data['uip'], data['ua'])
 
@@ -74,7 +71,7 @@ def send_to_analytics(data):
         resp = requests.post(
             'https://www.google-analytics.com/collect',
             data=data,
-            timeout=3,      # seconds
+            timeout=3,  # seconds
         )
     except requests.Timeout:
         log.warning('Timeout sending to Google Analytics')
@@ -85,10 +82,10 @@ def send_to_analytics(data):
 
 def generate_client_id(ip_address, user_agent):
     """
-    Create an advertising ID
+    Create an advertising ID.
 
-    This simplifies things but essentially if a user has the same IP and same UA,
-    this will treat them as the same user for analytics purposes
+    This simplifies things but essentially if a user has the same IP and same
+    UA, this will treat them as the same user for analytics purposes
     """
     salt = b'advertising-client-id'