Django3: add new django.db.models.JSONField

readthedocs/builds/migrations/0038_add_new_jsonfields.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.11 on 2022-01-31 11:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0037_alter_build_cold_storage'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='build',
+            name='_config_json',
+            field=models.JSONField(default=dict, verbose_name='Configuration used in the build'),
+        ),
+    ]

readthedocs/builds/migrations/0039_migrate_config_data.py

@@ -0,0 +1,24 @@
+from django.db import migrations, models
+from django.db.models import F
+from django.db.models.functions import Cast
+
+
+def forwards_func(apps, schema_editor):
+    Build = apps.get_model('builds', 'Build')
+    (
+        Build.objects
+        .annotate(_config_in_json=Cast('_config', output_field=models.JSONField()))
+        # Copy `_config` JSONField (text) into `_config_json` (jsonb)
+        .update(_config_json=F('_config_in_json'))
+    )
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('builds', '0038_add_new_jsonfields'),
+    ]
+
+    operations = [
+        migrations.RunPython(forwards_func)
+    ]

readthedocs/builds/models.py

@@ -651,6 +651,7 @@ class Build(models.Model):
         blank=True,
     )
     _config = JSONField(_('Configuration used in the build'), default=dict)
+    _config_json = models.JSONField(_('Configuration used in the build'), default=dict)
 
     length = models.IntegerField(_('Build Length'), null=True, blank=True)
 
@@ -719,6 +720,10 @@ def config(self):
         Build object (it could be stored in this object or one of the previous
         ones).
         """
+        # TODO: now that we are using a proper JSONField here, we could
+        # probably change this field to be a ForeignKey to avoid repeating the
+        # config file over and over again and re-use them to save db data as
+        # well
         if self.CONFIG_KEY in self._config:
             return (
                 Build.objects
@@ -762,6 +767,11 @@ def save(self, *args, **kwargs):  # noqa
             self.version_name = self.version.verbose_name
             self.version_slug = self.version.slug
             self.version_type = self.version.type
+
+        # TODO: delete copying config after deploy
+        # Copy `_config` into the new `_config_json` JSONField
+        self._config_json = self._config
+
         super().save(*args, **kwargs)
         self._config_changed = False
 

readthedocs/integrations/admin.py

@@ -93,6 +93,7 @@ class IntegrationAdmin(admin.ModelAdmin):
     search_fields = ('project__slug', 'project__name')
     readonly_fields = ['exchanges']
 
+    # TODO: review this now that we are using official Django's JSONField
     def exchanges(self, obj):
         """
         Manually make an inline-ish block.

readthedocs/integrations/migrations/0008_add_new_jsonfields.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.11 on 2022-01-31 11:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('integrations', '0007_update-provider-data'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='httpexchange',
+            name='request_headers_json',
+            field=models.JSONField(default=None, null=True, verbose_name='Request headers'),
+        ),
+        migrations.AddField(
+            model_name='httpexchange',
+            name='response_headers_json',
+            field=models.JSONField(default=None, null=True, verbose_name='Request headers'),
+        ),
+        migrations.AddField(
+            model_name='integration',
+            name='provider_data_json',
+            field=models.JSONField(default=dict, verbose_name='Provider data'),
+        ),
+    ]

readthedocs/integrations/migrations/0009_migrate_headers_data.py

@@ -0,0 +1,42 @@
+from django.db import migrations, models
+from django.db.models import F
+from django.db.models.functions import Cast
+
+
+def forwards_func(apps, schema_editor):
+    HttpExchange = apps.get_model('integrations', 'HttpExchange')
+    (
+        HttpExchange.objects
+        .annotate(
+            request_headers_in_json=Cast('request_headers', output_field=models.JSONField()),
+            response_headers_in_json=Cast('response_headers', output_field=models.JSONField()),
+        )
+        .update(
+            request_headers_json=F('request_headers_in_json'),
+            response_headers_json=F('response_headers_in_json'),
+        )
+    )
+
+
+    Integration = apps.get_model('integrations', 'Integration')
+    (
+        Integration.objects
+        .annotate(
+            provider_data_in_json=Cast('provider_data', output_field=models.JSONField()),
+        )
+        .update(
+            provider_data_json=F('provider_data_in_json'),
+        )
+    )
+
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('integrations', '0008_add_new_jsonfields'),
+    ]
+
+    operations = [
+        migrations.RunPython(forwards_func)
+    ]

readthedocs/integrations/models.py

@@ -145,9 +145,21 @@ class HttpExchange(models.Model):
     date = models.DateTimeField(_('Date'), auto_now_add=True)
 
     request_headers = JSONField(_('Request headers'))
+    request_headers_json = models.JSONField(
+        _('Request headers'),
+        # Delete after deploy
+        null=True,
+        default=None,
+    )
     request_body = models.TextField(_('Request body'))
 
     response_headers = JSONField(_('Request headers'))
+    response_headers_json = models.JSONField(
+        _('Request headers'),
+        # Delete after deploy
+        null=True,
+        default=None,
+    )
     response_body = models.TextField(_('Response body'))
 
     status_code = models.IntegerField(
@@ -163,6 +175,15 @@ class Meta:
     def __str__(self):
         return _('Exchange {0}').format(self.pk)
 
+    # TODO: delete .save method after deploy
+    # Copy headers into new JSONField
+    def save(self, *args, **kwargs):
+        # NOTE: cast headers into a regular dict because Django does not know
+        # how to serialize ``requests.structures.CaseInsensitiveDict``
+        self.request_headers_json = dict(self.request_headers)
+        self.response_headers_json = dict(self.response_headers)
+        super().save(*args, **kwargs)
+
     @property
     def failed(self):
         # Assume anything that isn't 2xx level status code is an error
@@ -278,6 +299,7 @@ class Integration(models.Model):
         choices=INTEGRATIONS,
     )
     provider_data = JSONField(_('Provider data'), default=dict)
+    provider_data_json = models.JSONField(_('Provider data'), default=dict)
     exchanges = GenericRelation(
         'HttpExchange',
         related_query_name='integrations',
@@ -303,6 +325,12 @@ def remove_secret(self):
         self.secret = None
         self.save(update_fields=['secret'])
 
+    # TODO: delete .save method after deploy
+    # Copy `provider_data` into new JSONField
+    def save(self, *args, **kwargs):
+        self.provider_data_json = self.provider_data
+        super().save(*args, **kwargs)
+
     def __str__(self):
         return (
             _('{0} for {1}')

readthedocs/projects/forms.py

@@ -682,7 +682,10 @@ class IntegrationForm(forms.ModelForm):
 
     class Meta:
         model = Integration
-        exclude = ['provider_data', 'exchanges', 'secret']  # pylint: disable=modelform-uses-exclude
+        fields = [
+            'project',
+            'integration_type',
+        ]
 
     def __init__(self, *args, **kwargs):
         self.project = kwargs.pop('project', None)

readthedocs/rtd_tests/tests/test_oauth.py

@@ -346,7 +346,7 @@ def test_update_webhook_404_error(self, setup_webhook, session):
     @mock.patch('readthedocs.oauth.services.github.GitHubService.get_session')
     @mock.patch('readthedocs.oauth.services.github.GitHubService.setup_webhook')
     def test_update_webhook_no_provider_data(self, setup_webhook, session):
-        self.integration.provider_data = None
+        self.integration.provider_data = {}
         self.integration.save()
 
         session().patch.side_effect = AttributeError
@@ -794,7 +794,7 @@ def test_update_webhook_404_error(self, setup_webhook, session):
     @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.get_session')
     @mock.patch('readthedocs.oauth.services.bitbucket.BitbucketService.setup_webhook')
     def test_update_webhook_no_provider_data(self, setup_webhook, session):
-        self.integration.provider_data = None
+        self.integration.provider_data = {}
         self.integration.save()
 
         session().put.side_effect = AttributeError
@@ -1279,7 +1279,7 @@ def test_update_webhook_404_error(self, repo_id, setup_webhook, session):
     @mock.patch('readthedocs.oauth.services.gitlab.GitLabService.setup_webhook')
     @mock.patch('readthedocs.oauth.services.gitlab.GitLabService._get_repo_id')
     def test_update_webhook_no_provider_data(self, repo_id, setup_webhook, session):
-        self.integration.provider_data = None
+        self.integration.provider_data = {}
         self.integration.save()
 
         repo_id.return_value = '9999'

readthedocs/rtd_tests/tests/test_privacy_urls.py

@@ -167,8 +167,8 @@ def setUp(self):
         # For whatever reason, fixtures hates JSONField
         self.integration_exchange = HttpExchange.objects.create(
             related_object=self.integration,
-            request_headers='{"foo": "bar"}',
-            response_headers='{"foo": "bar"}',
+            request_headers={'foo': 'bar'},
+            response_headers={'foo': 'bar'},
             status_code=200,
         )
         self.domain = get(Domain, domain='docs.foobar.com', project=self.pip)
@@ -183,8 +183,8 @@ def setUp(self):
         self.webhook = get(WebHook, project=self.pip)
         self.webhook_exchange = HttpExchange.objects.create(
             related_object=self.webhook,
-            request_headers='{"foo": "bar"}',
-            response_headers='{"foo": "bar"}',
+            request_headers={'foo': 'bar'},
+            response_headers={'foo': 'bar'},
             status_code=200,
         )
         self.default_kwargs = {