Merge pull request #5186 from rtfd/humitos/use-unicode-slugify

Use unicode-slugify to generate Version.slug

Author: humitos

readthedocs/builds/version_slug.py

@@ -26,6 +26,7 @@
 
 from django.db import models
 from django.utils.encoding import force_text
+from slugify import slugify as unicode_slugify
 
 
 def get_fields_with_model(cls):
@@ -53,13 +54,15 @@ def get_fields_with_model(cls):
 
 class VersionSlugField(models.CharField):
 
-    """Inspired by ``django_extensions.db.fields.AutoSlugField``."""
+    """
+    Inspired by ``django_extensions.db.fields.AutoSlugField``.
 
-    invalid_chars_re = re.compile('[^-._a-z0-9]')
-    leading_punctuation_re = re.compile('^[-._]+')
-    placeholder = '-'
-    fallback_slug = 'unknown'
+    Uses ``unicode-slugify`` to generate the slug.
+    """
+
+    ok_chars = '-._'  # dash, dot, underscore
     test_pattern = re.compile('^{pattern}$'.format(pattern=VERSION_SLUG_REGEX))
+    fallback_slug = 'unknown'
 
     def __init__(self, *args, **kwargs):
         kwargs.setdefault('db_index', True)
@@ -78,13 +81,42 @@ def get_queryset(self, model_cls, slug_field):
                 return model._default_manager.all()
         return model_cls._default_manager.all()
 
+    def _normalize(self, content):
+        """
+        Normalize some invalid characters (/, %, !, ?) to become a dash (``-``).
+
+        .. note::
+
+            We replace these characters to a dash to keep compatibility with the
+            old behavior and also because it makes this more readable.
+
+        For example, ``release/1.0`` will become ``release-1.0``.
+        """
+        return re.sub('[/%!?]', '-', content)
+
     def slugify(self, content):
+        """
+        Make ``content`` a valid slug.
+
+        It uses ``unicode-slugify`` behind the scenes which works properly with
+        Unicode characters.
+        """
         if not content:
             return ''
 
-        slugified = content.lower()
-        slugified = self.invalid_chars_re.sub(self.placeholder, slugified)
-        slugified = self.leading_punctuation_re.sub('', slugified)
+        normalized = self._normalize(content)
+        slugified = unicode_slugify(
+            normalized,
+            only_ascii=True,
+            spaces=False,
+            lower=True,
+            ok=self.ok_chars,
+            space_replacement='-',
+        )
+
+        # Remove first character wile it's an invalid character for the
+        # beginning of the slug
+        slugified = slugified.lstrip(self.ok_chars)
 
         if not slugified:
             return self.fallback_slug

readthedocs/rtd_tests/tests/test_version_slug.py

@@ -105,3 +105,15 @@ def test_uniquifying_suffix(self):
         self.assertEqual(field.uniquifying_suffix(25), '_z')
         self.assertEqual(field.uniquifying_suffix(26), '_ba')
         self.assertEqual(field.uniquifying_suffix(52), '_ca')
+
+    def test_unicode(self):
+        version = Version.objects.create(
+            verbose_name='camión',
+            project=self.pip,
+        )
+        self.assertEqual(version.slug, 'camion')
+        version = Version.objects.create(
+            verbose_name='ŭñíč°də-branch',
+            project=self.pip,
+        )
+        self.assertEqual(version.slug, 'unicd-branch')

requirements/pip.txt

@@ -68,6 +68,9 @@ django-kombu==0.9.4
 mock==2.0.0
 stripe==2.21.0
 
+# unicode-slugify==0.1.5 is not released on PyPI yet
+git+https://github.com/mozilla/unicode-slugify@b696c37#egg=unicode-slugify
+
 django-formtools==2.1
 
 # docker is pinned to 3.1.3 because we found some strange behavior