diff --git a/.github/config.yml b/.github/config.yml deleted file mode 100644 index 3575f85609e..00000000000 --- a/.github/config.yml +++ /dev/null @@ -1,9 +0,0 @@ -# ProBot TODO bot -# https://probot.github.io/apps/todo/ - -todo: - autoAssign: false - blobLines: 7 - caseSensitive: true - keyword: "TODO" - diff --git a/.github/mergeable.yml b/.github/mergeable.yml deleted file mode 100644 index 6027bbb2da1..00000000000 --- a/.github/mergeable.yml +++ /dev/null @@ -1,28 +0,0 @@ -# ProBot Mergeable Bot -# https://github.com/jusx/mergeable - -mergeable: - pull_requests: - approvals: - # Minimum of approvals needed. - min: 1 - message: 'The PR must have a minimum of 1 approvals.' - - description: - no_empty: - # Do not allow empty descriptions on PR. - enabled: false - message: 'Description can not be empty.' - - must_exclude: - # Do not allow 'DO NOT MERGE' phrase on PR's description. - regex: 'DO NOT MERGE' - message: 'Description says that the PR should not be merged yet.' - - # Do not allow 'WIP' on PR's title. - title: 'WIP' - - label: - # Do not allow PR with label 'PR: work in progress' - must_exclude: 'PR: work in progress' - message: 'This PR is work in progress.' diff --git a/.github/no-response.yml b/.github/no-response.yml deleted file mode 100644 index 338215ccdcf..00000000000 --- a/.github/no-response.yml +++ /dev/null @@ -1,17 +0,0 @@ -# ProBot No Response Bot -# https://probot.github.io/apps/no-response/ - -# Number of days of inactivity before an Issue is closed for lack of response -daysUntilClose: 14 - -# Label requiring a response -responseRequiredLabel: 'Needed: more information' - -# Comment to post when closing an Issue for lack of response. Set to `false` to disable -closeComment: > - This issue has been automatically closed because - [there has been no response to our request for more information](https://docs.readthedocs.io/en/latest/contribute.html#initial-triage) - from the original author. With only the information that is currently in the issue, - we don't have enough information to take action. - Please reach out if you have or find the answers we need so that we can investigate further. - Thanks! diff --git a/.github/stale.yml b/.github/stale.yml deleted file mode 100644 index 704b8d634f1..00000000000 --- a/.github/stale.yml +++ /dev/null @@ -1,26 +0,0 @@ -# ProBot Stale Bot -# https://probot.github.io/apps/stale/ - -# Number of days of inactivity before an issue becomes stale -daysUntilStale: 45 - -# Number of days of inactivity before a stale issue is closed -daysUntilClose: 7 - -# Issues with these labels will never be considered stale -exemptLabels: - - 'Accepted' - - 'Needed: design decision' - - 'Status: blocked' - -# Label to use when marking an issue as stale -staleLabel: 'Status: stale' - -# Comment to post when marking an issue as stale. Set to `false` to disable -markComment: > - This issue has been automatically marked as stale because it has not had - recent activity. It will be closed if no further activity occurs. Thank you - for your contributions. - -# Comment to post when closing a stale issue. Set to `false` to disable -closeComment: false diff --git a/.gitignore b/.gitignore index 720c05adeb0..d4939081a95 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,6 @@ .DS_Store .cache .coverage -.coverage.* .idea .vagrant .vscode @@ -23,7 +22,7 @@ celerybeat-schedule.* deploy/.vagrant dist/* local_settings.py -locks/** +locks/* logs/* media/dash media/epub diff --git a/.readthedocs.yml b/.readthedocs.yml index 3be62e08d07..0e55d253be9 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,4 +3,4 @@ formats: all sphinx: configuration: docs/conf.py python: - requirements: requirements/local-docs-build.txt + requirements: requirements.txt diff --git a/.travis.yml b/.travis.yml index 802c8986cc0..6743bf289bb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,11 @@ language: python python: + - 2.7 - 3.6 +env: + - ES_VERSION=1.3.9 ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz matrix: include: - - python: 3.6 - env: TOXENV=py36 ES_VERSION=1.3.9 ES_DOWNLOAD_URL=https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz - python: 3.6 env: TOXENV=docs - python: 3.6 @@ -44,6 +45,6 @@ notifications: branches: only: - - master + - master - rel # Community release branch - relcorp # Corporate release branch diff --git a/CHANGELOG.rst b/CHANGELOG.rst index f51469f7fa1..5a5d68a46db 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,56 +1,3 @@ -Version 2.8.5 -------------- - -:Date: January 15, 2019 - -* `@stsewd `__: Use the python path from virtualenv in Conda (`#5110 `__) -* `@humitos `__: Feature flag to use `readthedocs/build:testing` image (`#5109 `__) -* `@stsewd `__: Use python from virtualenv's bin directory when executing commands (`#5107 `__) -* `@humitos `__: Do not build projects from banned users (`#5096 `__) -* `@agjohnson `__: Fix common pieces (`#5095 `__) -* `@rainwoodman `__: Suppress progress bar of the conda command. (`#5094 `__) -* `@humitos `__: Remove unused suggestion block from 404 pages (`#5087 `__) -* `@humitos `__: Remove header nav (Login/Logout button) on 404 pages (`#5085 `__) -* `@stsewd `__: Fix little typo (`#5084 `__) -* `@agjohnson `__: Split up deprecated view notification to GitHub and other webhook endpoints (`#5083 `__) -* `@humitos `__: Install ProBot (`#5082 `__) -* `@stsewd `__: Update docs about contributing to docs (`#5077 `__) -* `@humitos `__: Declare and improve invoke tasks (`#5075 `__) -* `@davidfischer `__: Fire a signal for domain verification (eg. for SSL) (`#5071 `__) -* `@agjohnson `__: Update copy on notifications for github services deprecation (`#5067 `__) -* `@humitos `__: Upgrade all packages with pur (`#5059 `__) -* `@dojutsu-user `__: Reduce logging to sentry (`#5054 `__) -* `@discdiver `__: fixed missing apostrophe for possessive "project's" (`#5052 `__) -* `@dojutsu-user `__: Template improvements in "gold/subscription_form.html" (`#5049 `__) -* `@merwok `__: Fix link in features page (`#5048 `__) -* `@stsewd `__: Update webhook docs (`#5040 `__) -* `@stsewd `__: Remove sphinx static and template dir (`#5039 `__) -* `@stephenfin `__: Add temporary method for disabling shallow cloning (#5031) (`#5036 `__) -* `@stsewd `__: Raise exception in failed checkout (`#5035 `__) -* `@dojutsu-user `__: Change default_branch value from Version.slug to Version.identifier (`#5034 `__) -* `@humitos `__: Make wipe view not CSRF exempt (`#5025 `__) -* `@humitos `__: Convert an IRI path to URI before setting as NGINX header (`#5024 `__) -* `@safwanrahman `__: index project asynchronously (`#5023 `__) -* `@stsewd `__: Keep command output when it's killed (`#5015 `__) -* `@stsewd `__: More hints for invalid submodules (`#5012 `__) -* `@ericholscher `__: Release 2.8.4 (`#5011 `__) -* `@stsewd `__: Remove `auto` doctype (`#5010 `__) -* `@davidfischer `__: Tweak sidebar ad priority (`#5005 `__) -* `@stsewd `__: Replace git status and git submodules status for gitpython (`#5002 `__) -* `@davidfischer `__: Backport jquery 2432 to Read the Docs (`#5001 `__) -* `@stsewd `__: Refactor remove_dir (`#4994 `__) -* `@humitos `__: Skip builds when project is not active (`#4991 `__) -* `@dojutsu-user `__: Make $ unselectable in docs (`#4990 `__) -* `@dojutsu-user `__: Remove deprecated "models.permalink" (`#4975 `__) -* `@dojutsu-user `__: Add validation for tags of length greater than 100 characters (`#4967 `__) -* `@dojutsu-user `__: Add test case for send_notifications on VersionLockedError (`#4958 `__) -* `@dojutsu-user `__: Remove trailing slashes on svn checkout (`#4951 `__) -* `@stsewd `__: Safe symlink on version deletion (`#4937 `__) -* `@humitos `__: CRUD for EnvironmentVariables from Project's admin (`#4899 `__) -* `@humitos `__: Notify users about the usage of deprecated webhooks (`#4898 `__) -* `@dojutsu-user `__: Disable django guardian warning (`#4892 `__) -* `@humitos `__: Handle 401, 403 and 404 status codes when hitting GitHub for webhook (`#4805 `__) - Version 2.8.4 ------------- diff --git a/LICENSE b/LICENSE index 37484f246ea..2447f29c36a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2010-2019 Read the Docs, Inc & contributors +Copyright (c) 2010-2017 Read the Docs, Inc & contributors Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation diff --git a/README.rst b/README.rst index e578b996181..5095d4a5481 100644 --- a/README.rst +++ b/README.rst @@ -78,6 +78,6 @@ when you push to GitHub. License ------- -`MIT`_ © 2010-2019 Read the Docs, Inc & contributors +`MIT`_ © 2010-2017 Read the Docs, Inc & contributors .. _MIT: LICENSE diff --git a/common b/common index 2c428603279..a6cb6bbafb3 160000 --- a/common +++ b/common @@ -1 +1 @@ -Subproject commit 2c42860327916ec66f3aed7cf3d7bab809438ab4 +Subproject commit a6cb6bbafb3cf93bfad7bab98c6636021719cc48 diff --git a/docs/.rstcheck.cfg b/docs/.rstcheck.cfg index 0ebdd74410e..1b093d3f696 100644 --- a/docs/.rstcheck.cfg +++ b/docs/.rstcheck.cfg @@ -1,4 +1,4 @@ [rstcheck] -ignore_directives=automodule,http:get,tabs,tab,prompt +ignore_directives=automodule,http:get,tabs,tab ignore_roles=djangosetting,setting ignore_messages=(Duplicate implicit target name: ".*")|(Hyperlink target ".*" is not referenced) diff --git a/docs/_static/css/sphinx_prompt_css.css b/docs/_static/css/sphinx_prompt_css.css deleted file mode 100644 index d1d06f8e67c..00000000000 --- a/docs/_static/css/sphinx_prompt_css.css +++ /dev/null @@ -1,13 +0,0 @@ -/* CSS for sphinx-prompt */ - -pre.highlight { - border: 1px solid #e1e4e5; - overflow-x: auto; - margin: 1px 0 24px 0; - padding: 12px 12px; -} - -pre.highlight span.prompt1 { - font-size: 12px; - line-height: 1.4; -} diff --git a/docs/api/v2.rst b/docs/api/v2.rst index d11e2c28472..eed418ff6fa 100644 --- a/docs/api/v2.rst +++ b/docs/api/v2.rst @@ -53,9 +53,9 @@ Project list **Example request**: - .. prompt:: bash $ + .. sourcecode:: bash - curl https://readthedocs.org/api/v2/project/?slug=pip + $ curl https://readthedocs.org/api/v2/project/?slug=pip **Example response**: @@ -232,9 +232,9 @@ Build list **Example request**: - .. prompt:: bash $ + .. sourcecode:: bash - curl https://readthedocs.org/api/v2/build/?project__slug=pip + $ curl https://readthedocs.org/api/v2/build/?project__slug=pip **Example response**: diff --git a/docs/builds.rst b/docs/builds.rst index 807a4a0f6fd..75fcdce3a2c 100644 --- a/docs/builds.rst +++ b/docs/builds.rst @@ -225,8 +225,3 @@ The *Sphinx* and *Mkdocs* builders set the following RTD-specific environment va +-------------------------+--------------------------------------------------+----------------------+ | ``READTHEDOCS_PROJECT`` | The RTD name of the project which is being built | ``myexampleproject`` | +-------------------------+--------------------------------------------------+----------------------+ - -.. tip:: - - In case extra environment variables are needed to the build process (like secrets, tokens, etc), - you can add them going to **Admin > Environment Variables** in your project. See :doc:`guides/environment-variables`. diff --git a/docs/commercial/sharing.rst b/docs/commercial/sharing.rst index 1893d498bc4..a35f21aab5d 100644 --- a/docs/commercial/sharing.rst +++ b/docs/commercial/sharing.rst @@ -4,7 +4,7 @@ Sharing .. note:: This feature only exists on our Business offering at `readthedocs.com `_. You can share your project with users outside of your company. -There are two ways to do this: +There are two way to do this: * by sending them a *secret link*, * by giving them a *password*. diff --git a/docs/conf.py b/docs/conf.py index 958c0c77aa5..d5ddc85ad50 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -28,7 +28,6 @@ 'djangodocs', 'doc_extensions', 'sphinx_tabs.tabs', - 'sphinx-prompt', ] templates_path = ['_templates'] @@ -83,7 +82,3 @@ # Activate autosectionlabel plugin autosectionlabel_prefix_document = True - - -def setup(app): - app.add_stylesheet('css/sphinx_prompt_css.css') diff --git a/docs/contribute.rst b/docs/contribute.rst index 61f0a5cab1f..65d9d50424a 100644 --- a/docs/contribute.rst +++ b/docs/contribute.rst @@ -49,26 +49,20 @@ install `pre-commit`_ and it will automatically run different linting tools and `yapf`_) to check your changes before you commit them. `pre-commit` will let you know if there were any problems that is wasn't able to fix automatically. -To run the `pre-commit` command and check your changes: +To run the `pre-commit` command and check your changes:: -.. prompt:: bash $ + $ pip install -U pre-commit + $ git add + $ pre-commit run - pip install -U pre-commit - git add - pre-commit run +or to run against a specific file:: -or to run against a specific file: - -.. prompt:: bash $ - - pre-commit run --files + $ pre-commit run --files `pre-commit` can also be run as a git pre-commit hook. You can set this up -with: - -.. prompt:: bash $ +with:: - pre-commit install + $ pre-commit install After this installation, the next time you run `git commit` the `pre-commit run` command will be run immediately and will inform you of the changes and errors. diff --git a/docs/custom_installs/elasticsearch.rst b/docs/custom_installs/elasticsearch.rst index 0abe74ee41e..3612374594a 100644 --- a/docs/custom_installs/elasticsearch.rst +++ b/docs/custom_installs/elasticsearch.rst @@ -12,11 +12,9 @@ Installing Java Elasticsearch requires Java 8 or later. Use `Oracle official documentation `_. or opensource distribution like `OpenJDK `_. -After installing java, verify the installation by,: +After installing java, verify the installation by,:: -.. prompt:: bash $ - - java -version + $ java -version The result should be something like this:: @@ -33,68 +31,52 @@ Elasticsearch can be downloaded directly from elastic.co. For Ubuntu, it's best RTD currently uses elasticsearch 1.x which can be easily downloaded and installed from `elastic.co `_. -Install the downloaded package by following command: - -.. prompt:: bash $ +Install the downloaded package by following command:: - sudo apt install .{path-to-downloaded-file}/elasticsearch-1.3.8.deb + $ sudo apt install .{path-to-downloaded-file}/elasticsearch-1.3.8.deb Custom setup ------------ -You need the icu plugin: - -.. prompt:: bash $ +You need the icu plugin:: - elasticsearch/bin/plugin -install elasticsearch/elasticsearch-analysis-icu/2.3.0 + $ elasticsearch/bin/plugin -install elasticsearch/elasticsearch-analysis-icu/2.3.0 Running Elasticsearch from command line --------------------------------------- -Elasticsearch is not started automatically after installation. How to start and stop Elasticsearch depends on whether your system uses SysV init or systemd (used by newer distributions). You can tell which is being used by running this command: +Elasticsearch is not started automatically after installation. How to start and stop Elasticsearch depends on whether your system uses SysV init or systemd (used by newer distributions). You can tell which is being used by running this command:: -.. prompt:: bash $ - - ps -p 1 + $ ps -p 1 **Running Elasticsearch with SysV init** -Use the ``update-rc.d command`` to configure Elasticsearch to start automatically when the system boots up: - -.. prompt:: bash $ - - sudo update-rc.d elasticsearch defaults 95 10 +Use the ``update-rc.d command`` to configure Elasticsearch to start automatically when the system boots up:: -Elasticsearch can be started and stopped using the service command: + $ sudo update-rc.d elasticsearch defaults 95 10 -.. prompt:: bash $ +Elasticsearch can be started and stopped using the service command:: - sudo -i service elasticsearch start - sudo -i service elasticsearch stop + $ sudo -i service elasticsearch start + $ sudo -i service elasticsearch stop If Elasticsearch fails to start for any reason, it will print the reason for failure to STDOUT. Log files can be found in /var/log/elasticsearch/. **Running Elasticsearch with systemd** -To configure Elasticsearch to start automatically when the system boots up, run the following commands: +To configure Elasticsearch to start automatically when the system boots up, run the following commands:: -.. prompt:: bash $ + $ sudo /bin/systemctl daemon-reload + $ sudo /bin/systemctl enable elasticsearch.service - sudo /bin/systemctl daemon-reload - sudo /bin/systemctl enable elasticsearch.service +Elasticsearch can be started and stopped as follows:: -Elasticsearch can be started and stopped as follows: + $ sudo systemctl start elasticsearch.service + $ sudo systemctl stop elasticsearch.service -.. prompt:: bash $ +To verify run:: - sudo systemctl start elasticsearch.service - sudo systemctl stop elasticsearch.service - -To verify run: - -.. prompt:: bash $ - - curl http://localhost:9200 + $ curl http://localhost:9200 You should get something like:: @@ -115,16 +97,12 @@ You should get something like:: Index the data available at RTD database ---------------------------------------- -You need to create the indexes: - -.. prompt:: bash $ - - python manage.py provision_elasticsearch +You need to create the indexes:: -In order to search through the RTD database, you need to index it into the elasticsearch index: + $ python manage.py provision_elasticsearch -.. prompt:: bash $ +In order to search through the RTD database, you need to index it into the elasticsearch index:: - python manage.py reindex_elasticsearch + $ python manage.py reindex_elasticsearch You are ready to go! diff --git a/docs/custom_installs/local_rtd_vm.rst b/docs/custom_installs/local_rtd_vm.rst index 101f5bb974b..42f14a2d220 100644 --- a/docs/custom_installs/local_rtd_vm.rst +++ b/docs/custom_installs/local_rtd_vm.rst @@ -5,29 +5,23 @@ Assumptions and Prerequisites ----------------------------- * Debian VM provisioned with python 2.7.x -* All python dependencies and setup tools are installed: +* All python dependencies and setup tools are installed :: -.. prompt:: bash $ + $ sudo apt-get install python-setuptools + $ sudo apt-get install build-essential + $ sudo apt-get install python-dev + $ sudo apt-get install libevent-dev + $ sudo easy_install pip - sudo apt-get install python-setuptools - sudo apt-get install build-essential - sudo apt-get install python-dev - sudo apt-get install libevent-dev - sudo easy_install pip +* Git :: -* Git: - -.. prompt:: bash $ - - sudo apt-get install git + $ sudo apt-get install git * Git repo is ``git.corp.company.com:git/docs/documentation.git`` * Source documents are in ``../docs/source`` -* Sphinx: +* Sphinx :: -.. prompt:: bash $ - - sudo pip install sphinx + $ sudo pip install sphinx .. note:: Not using sudo may prevent access. “error: could not create '/usr/local/lib/python2.7/dist-packages/markupsafe': Permission denied” @@ -37,52 +31,42 @@ Local RTD Setup Install RTD ~~~~~~~~~~~ -To host your documentation on a local RTD installation, set it up in your VM: - -.. prompt:: bash $ +To host your documentation on a local RTD installation, set it up in your VM. :: - mkdir checkouts - cd checkouts - git clone https://github.com/rtfd/readthedocs.org.git - cd readthedocs.org - sudo pip install -r requirements.txt + $ mkdir checkouts + $ cd checkouts + $ git clone https://github.com/rtfd/readthedocs.org.git + $ cd readthedocs.org + $ sudo pip install -r requirements.txt Possible Error and Resolution ````````````````````````````` **Error**: ``error: command 'gcc' failed with exit status 1`` -**Resolution**: Run the following commands: - -.. prompt:: bash $ - - sudo apt-get update - sudo apt-get install python2.7-dev tk8.5 tcl8.5 tk8.5-dev tcl8.5-dev libxml2-devel libxslt-devel - sudo apt-get build-dep python-imaging --fix-missing - -On Debian 8 (jessie) the command is slightly different: +**Resolution**: Run the following commands. :: -.. prompt:: bash $ + $ sudo apt-get update + $ sudo apt-get install python2.7-dev tk8.5 tcl8.5 tk8.5-dev tcl8.5-dev libxml2-devel libxslt-devel + $ sudo apt-get build-dep python-imaging --fix-missing - sudo apt-get update - sudo apt-get install python2.7-dev tk8.5 tcl8.5 tk8.5-dev tcl8.5-dev libxml2-dev libxslt-dev - sudo apt-get build-dep python-imaging --fix-missing +On Debian 8 (jessie) the command is slightly different :: -Also don't forget to re-run the dependency installation + $ sudo apt-get update + $ sudo apt-get install python2.7-dev tk8.5 tcl8.5 tk8.5-dev tcl8.5-dev libxml2-dev libxslt-dev + $ sudo apt-get build-dep python-imaging --fix-missing -.. prompt:: bash $ +Also don't forget to re-run the dependency installation :: - sudo pip install -r requirements.txt + $ sudo pip install -r requirements.txt Configure the RTD Server and Superuser ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -1. Run the following commands: +1. Run the following commands. :: - .. prompt:: bash $ - - ./manage.py migrate - ./manage.py createsuperuser + $ ./manage.py migrate + $ ./manage.py createsuperuser 2. This will prompt you to create a superuser account for Django. Enter appropriate details. For example: :: @@ -93,12 +77,10 @@ Configure the RTD Server and Superuser RTD Server Administration ~~~~~~~~~~~~~~~~~~~~~~~~~ -Navigate to the ``../checkouts/readthedocs.org`` folder in your VM and run the following command: - -.. prompt:: bash $ +Navigate to the ``../checkouts/readthedocs.org`` folder in your VM and run the following command. :: - ./manage.py runserver [VM IP ADDRESS]:8000 - curl -i http://[VM IP ADDRESS]:8000 + $ ./manage.py runserver [VM IP ADDRESS]:8000 + $ curl -i http://[VM IP ADDRESS]:8000 You should now be able to log into the admin interface from any PC in your LAN at ``http://[VM IP ADDRESS]:8000/admin`` using the superuser account created in django. @@ -108,11 +90,9 @@ Go to the dashboard at ``http://[VM IP ADDRESS]:8000/dashboard`` and follow the Example: ``git.corp.company.com:/git/docs/documentation.git``. 2. Clone the documentation sources from Git in the VM. 3. Navigate to the root path for documentation. -4. Run the following Sphinx commands: - -.. prompt:: bash $ +4. Run the following Sphinx commands. :: - make html + $ make html This generates the HTML documentation site using the default Sphinx theme. Verify the output in your local documentation folder under ``../build/html`` @@ -125,30 +105,24 @@ Possible Error and Resolution **Workaround-1** -1. In your machine, navigate to the ``.ssh`` folder: +1. In your machine, navigate to the ``.ssh`` folder. :: - .. prompt:: bash $ - - cd .ssh/ - cat id_rsa + $ cd .ssh/ + $ cat id_rsa 2. Copy the entire Private Key. 3. Now, SSH to the VM. -4. Open the ``id_rsa`` file in the VM: - -.. prompt:: bash $ +4. Open the ``id_rsa`` file in the VM. :: - vim /home//.ssh/id_rsa + $ vim /home//.ssh/id_rsa 5. Paste the RSA key copied from your machine and save file (``Esc``. ``:wq!``). **Workaround 2** -SSH to the VM using the ``-A`` directive: - -.. prompt:: bash $ +SSH to the VM using the ``-A`` directive. :: - ssh document-vm -A + $ ssh document-vm -A This provides all permissions for that particular remote session, which are revoked when you logout. diff --git a/docs/docs.rst b/docs/docs.rst index 5a1ec9c6413..81ea77a227c 100644 --- a/docs/docs.rst +++ b/docs/docs.rst @@ -36,6 +36,4 @@ Content but rather break them on semantic meaning (e.g. periods or commas). Read more about this `here `_. * If you are cross-referencing to a different page within our website, - use the ``doc`` role and not a hyperlink. -* If you are cross-referencing to a section within our website, - use the ``ref`` role with the label from the `autosectionlabel extension `__. + use the ``doc`` directive and not a hyperlink. diff --git a/docs/faq.rst b/docs/faq.rst index c782632012f..30025963c18 100644 --- a/docs/faq.rst +++ b/docs/faq.rst @@ -230,49 +230,3 @@ What commit of Read the Docs is in production? ---------------------------------------------- We deploy readthedocs.org from the `rel` branch in our GitHub repository. You can see the latest commits that have been deployed by looking on GitHub: https://github.com/rtfd/readthedocs.org/commits/rel - - -How can I avoid search results having a deprecated version of my docs? ---------------------------------------------------------------------- - -If readers search something related to your docs in Google, it will probably return the most relevant version of your documentation. -It may happen that this version is already deprecated and you want to stop Google indexing it as a result, -and start suggesting the latest (or newer) one. - -To accomplish this, you can add a ``robots.txt`` file to your documentation's root so it ends up served at the root URL of your project -(for example, https://yourproject.readthedocs.io/robots.txt). - - -Minimal example of ``robots.txt`` -+++++++++++++++++++++++++++++++++ - -:: - - User-agent: * - Disallow: /en/deprecated-version/ - Disallow: /en/2.0/ - -.. note:: - - See `Google's docs`_ for its full syntax. - -This file has to be served as is under ``/robots.txt``. -Depending if you are using Sphinx or MkDocs, you will need a different configuration for this. - - -Sphinx -~~~~~~ - -Sphinx uses `html_extra`_ option to add static files to the output. -You need to create a ``robots.txt`` file and put it under the path defined in ``html_extra``. - - -MkDocs -~~~~~~ - -MkDocs needs the ``robots.txt`` to be at the directory defined at `docs_dir`_ config. - - -.. _Google's docs: https://support.google.com/webmasters/answer/6062608 -.. _html_extra: https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_extra_path -.. _docs_dir: https://www.mkdocs.org/user-guide/configuration/#docs_dir diff --git a/docs/guides/environment-variables.rst b/docs/guides/environment-variables.rst deleted file mode 100644 index 91ca94cab01..00000000000 --- a/docs/guides/environment-variables.rst +++ /dev/null @@ -1,37 +0,0 @@ -I Need Secrets (or Environment Variables) in my Build -===================================================== - -It may happen that your documentation depends on an authenticated service to be built properly. -In this case, you will require some secrets to access these services. - -Read the Docs provides a way to define environment variables for your project to be used in the build process. -All these variables will be exposed to all the commands executed when building your documentation. - -To define an environment variable, you need to - -#. Go to your project **Admin > Environment Variables** -#. Click on "Add Environment Variable" button -#. Input a ``Name`` and ``Value`` (your secret needed here) -#. Click "Save" button - -.. note:: - - Values will never be exposed to users, even to owners of the project. Once you create an environment variable you won't be able to see its value anymore because of security purposes. - -After adding an environment variable from your project's admin, you can access it from your build process using Python, for example: - -.. code-block:: python - - # conf.py - import os - import requests - - # Access to our custom environment variables - username = os.environ.get('USERNAME') - password = os.environ.get('PASSWORD') - - # Request a username/password protected URL - response = requests.get( - 'https://httpbin.org/basic-auth/username/password', - auth=(username, password), - ) diff --git a/docs/guides/manage-translations.rst b/docs/guides/manage-translations.rst index d4d2db8767f..b651ae0415f 100644 --- a/docs/guides/manage-translations.rst +++ b/docs/guides/manage-translations.rst @@ -31,9 +31,9 @@ Create translatable files To generate these ``.pot`` files it's needed to run this command from your ``docs/`` directory: -.. prompt:: bash $ +.. code-block:: console - sphinx-build -b gettext . _build/gettext + $ sphinx-build -b gettext . _build/gettext .. tip:: @@ -57,18 +57,18 @@ We recommend using `sphinx-intl`_ tool for this workflow. First, you need to install it: -.. prompt:: bash $ +.. code-block:: console - pip install sphinx-intl + $ pip install sphinx-intl As a second step, we want to create a directory with each translated file per target language (in this example we are using Spanish/Argentina and Portuguese/Brazil). This can be achieved with the following command: -.. prompt:: bash $ +.. code-block:: console - sphinx-intl update -p _build/gettext -l es_AR -l pt_BR + $ sphinx-intl update -p _build/gettext -l es_AR -l pt_BR This command will create a directory structure similar to the following (with one ``.po`` file per ``.rst`` file in your documentation):: @@ -113,9 +113,9 @@ To do this, run this command: .. _transifex-client: https://docs.transifex.com/client/introduction -.. prompt:: bash $ +.. code-block:: console - pip install transifex-client + $ pip install transifex-client After installing it, you need to configure your account. For this, you need to create an API Token for your user to access this service through the command line. @@ -126,17 +126,17 @@ This can be done under your `User's Settings`_. Now, you need to setup it to use this token: -.. prompt:: bash $ +.. code-block:: console - tx init --token $TOKEN --no-interactive + $ tx init --token $TOKEN --no-interactive The next step is to map every ``.pot`` file you have created in the previous step to a resource under Transifex. To achieve this, you need to run this command: -.. prompt:: bash $ +.. code-block:: console - tx config mapping-bulk \ + $ tx config mapping-bulk \ --project $TRANSIFEX_PROJECT \ --file-extension '.pot' \ --source-file-dir docs/_build/gettext \ @@ -150,17 +150,17 @@ This command will generate a file at ``.tx/config`` with all the information nee Finally, you need to upload these files to Transifex platform so translators can start their work. To do this, you can run this command: -.. prompt:: bash $ +.. code-block:: console - tx push --source + $ tx push --source Now, you can go to your Transifex's project and check that there is one resource per ``.rst`` file of your documentation. After the source files are translated using Transifex, you can download all the translations for all the languages by running: -.. prompt:: bash $ +.. code-block:: console - tx pull --all + $ tx pull --all This command will leave the ``.po`` files needed for building the documentation in the target language under ``locale//LC_MESSAGES``. @@ -176,9 +176,9 @@ Build the documentation in target language Finally, to build our documentation in Spanish(Argentina) we need to tell Sphinx builder the target language with the following command: -.. prompt:: bash $ +.. code-block:: console - sphinx-build -b html -D language=es_AR . _build/html/es_AR + $ sphinx-build -b html -D language=es_AR . _build/html/es_AR .. note:: @@ -197,21 +197,21 @@ Once you have done changes in your documentation, you may want to make these add #. Create the ``.pot`` files: - .. prompt:: bash $ + .. code-block:: console - sphinx-build -b gettext . _build/gettext + $ sphinx-build -b gettext . _build/gettext - .. For the manual workflow, we need to run this command +.. For the manual workflow, we need to run this command - $ sphinx-intl update -p _build/gettext -l es_AR -l pt_BR + $ sphinx-intl update -p _build/gettext -l es_AR -l pt_BR #. Push new files to Transifex - .. prompt:: bash $ + .. code-block:: console - tx push --sources + $ tx push --sources Build documentation from up to date translation @@ -221,16 +221,16 @@ When translators have finished their job, you may want to update the documentati #. Pull up to date translations from Transifex: - .. prompt:: bash $ + .. code-block:: console - tx pull --all + $ tx pull --all #. Commit and push these changes to our repo - .. prompt:: bash $ + .. code-block:: console - git add locale/ - git commit -m "Update translations" - git push + $ git add locale/ + $ git commit -m "Update translations" + $ git push The last ``git push`` will trigger a build per translation defined as part of your project under Read the Docs and make it immediately available. diff --git a/docs/guides/specifying-dependencies.rst b/docs/guides/specifying-dependencies.rst index bac7c5fefba..0019598a625 100644 --- a/docs/guides/specifying-dependencies.rst +++ b/docs/guides/specifying-dependencies.rst @@ -35,7 +35,7 @@ Using the project admin dashboard Once the requirements file has been created; - Login to Read the Docs and go to the project admin dashboard. -- Go to **Admin > Advanced Settings > Requirements file**. +- Go to ``Admin > Advanced Settings > Requirements file``. - Specify the path of the requirements file you just created. The path should be relative to the root directory of the documentation project. Using a conda environment file diff --git a/docs/i18n.rst b/docs/i18n.rst index c8bd827af6d..19ec3896ddc 100644 --- a/docs/i18n.rst +++ b/docs/i18n.rst @@ -269,11 +269,9 @@ Compiling to MO Gettext doesn't parse any text files, it reads a binary format for faster performance. To compile the latest PO files in the repository, Django provides the ``compilemessages`` management command. For example, to compile all the -available localizations, just run: +available localizations, just run:: -.. prompt:: bash $ - - python manage.py compilemessages -a + $ python manage.py compilemessages -a You will need to do this every time you want to push updated translations to the live site. @@ -306,12 +304,12 @@ help pages `_. #. Update files and push sources (English) to Transifex: - .. prompt:: bash $ + .. code-block:: console - fab i18n_push_source + $ fab i18n_push_source #. Pull changes (new translations) from Transifex: - .. prompt:: bash $ + .. code-block:: console - fab i18n_pull + $ fab i18n_pull diff --git a/docs/intro/getting-started-with-mkdocs.rst b/docs/intro/getting-started-with-mkdocs.rst index 5f6139db7a8..04b09f945f7 100644 --- a/docs/intro/getting-started-with-mkdocs.rst +++ b/docs/intro/getting-started-with-mkdocs.rst @@ -20,15 +20,15 @@ Quick start Assuming you have Python already, `install MkDocs`_: -.. prompt:: bash $ +.. sourcecode:: bash - pip install mkdocs + $ pip install mkdocs Setup your MkDocs project: -.. prompt:: bash $ +.. sourcecode:: bash - mkdocs new . + $ mkdocs new . This command creates ``mkdocs.yml`` which holds your MkDocs configuration, and ``docs/index.md`` which is the Markdown file @@ -37,9 +37,9 @@ that is the entry point for your documentation. You can edit this ``index.md`` file to add more details about your project and then you can build your documentation: -.. prompt:: bash $ +.. sourcecode:: bash - mkdocs serve + $ mkdocs serve This command builds your Markdown files into HTML and starts a development server to browse your documentation. diff --git a/docs/intro/getting-started-with-sphinx.rst b/docs/intro/getting-started-with-sphinx.rst index f2351752d5d..4f967c70263 100644 --- a/docs/intro/getting-started-with-sphinx.rst +++ b/docs/intro/getting-started-with-sphinx.rst @@ -33,23 +33,23 @@ Quick start Assuming you have Python already, `install Sphinx`_: -.. prompt:: bash $ +.. sourcecode:: bash - pip install sphinx + $ pip install sphinx Create a directory inside your project to hold your docs: -.. prompt:: bash $ +.. sourcecode:: bash - cd /path/to/project - mkdir docs + $ cd /path/to/project + $ mkdir docs Run ``sphinx-quickstart`` in there: -.. prompt:: bash $ +.. sourcecode:: bash - cd docs - sphinx-quickstart + $ cd docs + $ sphinx-quickstart This quick start will walk you through creating the basic configuration; in most cases, you can just accept the defaults. When it's done, you'll have an ``index.rst``, a @@ -59,9 +59,9 @@ Now, edit your ``index.rst`` and add some information about your project. Include as much detail as you like (refer to the reStructuredText_ syntax or `this template`_ if you need help). Build them to see how they look: -.. prompt:: bash $ +.. sourcecode:: bash - make html + $ make html Your ``index.rst`` has been built into ``index.html`` in your documentation output directory (typically ``_build/html/index.html``). @@ -88,9 +88,9 @@ Using Markdown with Sphinx You can use Markdown and reStructuredText in the same Sphinx project. We support this natively on Read the Docs, and you can do it locally: -.. prompt:: bash $ +.. sourcecode:: bash - pip install recommonmark + $ pip install recommonmark Then in your ``conf.py``: diff --git a/docs/webhooks.rst b/docs/webhooks.rst index 7d3702f12d3..128efcaebf2 100644 --- a/docs/webhooks.rst +++ b/docs/webhooks.rst @@ -20,8 +20,6 @@ details and a list of HTTP exchanges that have taken place for the integration. You need this information for the URL, webhook, or Payload URL needed by the repository provider such as GitHub, GitLab, or Bitbucket. -.. _webhook-creation: - Webhook Creation ---------------- @@ -38,8 +36,6 @@ As an example, the URL pattern looks like this: *readthedocs.org/api/v2/webhook/ Use this URL when setting up a new webhook with your provider -- these steps vary depending on the provider: -.. _webhook-integration-github: - GitHub ~~~~~~ @@ -58,8 +54,6 @@ For a 403 error, it's likely that the Payload URL is incorrect. .. note:: The webhook token, intended for the GitHub **Secret** field, is not yet implemented. -.. _webhook-integration-bitbucket: - Bitbucket ~~~~~~~~~ @@ -70,8 +64,6 @@ Bitbucket * Under **Triggers**, **Repository push** should be selected * Finish by clicking **Save** -.. _webhook-integration-gitlab: - GitLab ~~~~~~ @@ -82,8 +74,6 @@ GitLab * Leave the default **Push events** selected and mark **Tag push events** also * Finish by clicking **Add Webhook** -.. _webhook-integration-generic: - Using the generic API integration --------------------------------- @@ -147,69 +137,3 @@ Resyncing webhooks It might be necessary to re-establish a webhook if you are noticing problems. To resync a webhook from Read the Docs, visit the integration detail page and follow the directions for re-syncing your repository webhook. - -Troubleshooting ---------------- - -My project isn't automatically building -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If your project isn't automatically building, you can check your integration on -Read the Docs to see the payload sent to our servers. If there is no recent -activity on your Read the Docs project webhook integration, then it's likely -that your VCS provider is not configured correctly. If there is payload -information on your Read the Docs project, you might need to verify that your -versions are configured to build correctly. - -Either way, it may help to either resync your webhook integration (see -`Resyncing webhooks`_ for information on this process), or set up an entirely -new webhook integration. - -.. _webhook-github-services: - -I was warned I shouldn't use GitHub Services -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Last year, GitHub announced that effective Jan 31st, 2019, GitHub Services will stop -working [1]_. This means GitHub will stop sending notifications to Read the Docs -for projects configured with the ``ReadTheDocs`` GitHub Service. If your project -has been configured on Read the Docs for a long time, you are most likely still -using this service to automatically build your project on Read the Docs. - -In order for your project to continue automatically building, you will need to -configure your GitHub repository with a new webhook. You can use either a -connected GitHub account and a :ref:`GitHub webhook integration ` -on your Read the Docs project, or you can use a -:ref:`generic webhook integration ` without a connected -account. - -.. [1] https://developer.github.com/changes/2018-04-25-github-services-deprecation/ - -.. _webhook-deprecated-endpoints: - -I was warned that my project won't automatically build after April 1st -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In addition to :ref:`no longer supporting GitHub Services `, -we have decided to no longer support several other legacy incoming webhook -endpoints that were used before we introduced project webhook integrations. When -we introduced our webhook integrations, we added several features and improved -security for incoming webhooks and these features were not added to our leagcy -incoming webhooks. New projects have not been able to use our legacy incoming -webhooks since, however if you have a project that has been established for a -while, you may still be using these endpoints. - -After March 1st, 2019, we will stop accepting incoming webhook notifications for -these legacy incoming webhooks. Your project will need to be reconfigured and -have a webhook integration configured, pointing to a new webhook with your VCS -provider. - -In particular, the incoming webhook URLs that will be removed are: - -* ``https://readthedocs.org/build`` -* ``https://readthedocs.org/bitbucket`` -* ``https://readthedocs.org/github`` (as noted :ref:`above `) -* ``https://readthedocs.org/gitlab`` - -In order to establish a new project webhook integration, :ref:`follow -the directions for your VCS provider ` diff --git a/docs/yaml-config.rst b/docs/yaml-config.rst index 9aac9521baf..616ace4cc7d 100644 --- a/docs/yaml-config.rst +++ b/docs/yaml-config.rst @@ -278,9 +278,9 @@ installed in addition to the default ``requests`` and ``simplejson``, use the Behind the scene the following Pip command will be run: -.. prompt:: bash $ +.. code-block:: shell - pip install .[tests,docs] + $ pip install .[tests,docs] .. _issue: https://github.com/rtfd/readthedocs.org/issues diff --git a/readthedocs/__init__.py b/readthedocs/__init__.py index 8f8c9ee7c80..bf6f944d7e6 100644 --- a/readthedocs/__init__.py +++ b/readthedocs/__init__.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- - """Read the Docs.""" import os.path -from configparser import RawConfigParser +from future.moves.configparser import RawConfigParser def get_version(setupcfg_path): diff --git a/readthedocs/analytics/__init__.py b/readthedocs/analytics/__init__.py index b25cef94087..f2531d76edb 100644 --- a/readthedocs/analytics/__init__.py +++ b/readthedocs/analytics/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- +"""App init""" -"""App init.""" - -default_app_config = 'readthedocs.analytics.apps.AnalyticsAppConfig' # noqa +default_app_config = 'readthedocs.analytics.apps.AnalyticsAppConfig' # noqa diff --git a/readthedocs/analytics/apps.py b/readthedocs/analytics/apps.py index c2abb221a18..afdea7f5dec 100644 --- a/readthedocs/analytics/apps.py +++ b/readthedocs/analytics/apps.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - """Django app config for the analytics app.""" +from __future__ import absolute_import from django.apps import AppConfig class AnalyticsAppConfig(AppConfig): - """Analytics app init code.""" + """Analytics app init code""" name = 'readthedocs.analytics' verbose_name = 'Analytics' diff --git a/readthedocs/analytics/tasks.py b/readthedocs/analytics/tasks.py index 4e2cc957ee8..6c1ec2cfce1 100644 --- a/readthedocs/analytics/tasks.py +++ b/readthedocs/analytics/tasks.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- +"""Tasks for Read the Docs' analytics""" -"""Tasks for Read the Docs' analytics.""" +from __future__ import absolute_import from django.conf import settings @@ -11,24 +11,24 @@ DEFAULT_PARAMETERS = { - 'v': '1', # analytics version (always 1) - 'aip': '1', # anonymize IP + 'v': '1', # analytics version (always 1) + 'aip': '1', # anonymize IP 'tid': settings.GLOBAL_ANALYTICS_CODE, # User data - 'uip': None, # User IP address - 'ua': None, # User agent + 'uip': None, # User IP address + 'ua': None, # User agent # Application info 'an': 'Read the Docs', - 'av': readthedocs.__version__, # App version + 'av': readthedocs.__version__, # App version } @app.task(queue='web') def analytics_pageview(url, title=None, **kwargs): """ - Send a pageview to Google Analytics. + Send a pageview to Google Analytics :see: https://developers.google.com/analytics/devguides/collection/protocol/v1/parameters :param url: the URL of the pageview @@ -37,8 +37,8 @@ def analytics_pageview(url, title=None, **kwargs): """ data = { 't': 'pageview', - 'dl': url, # URL of the pageview (required) - 'dt': title, # Title of the page + 'dl': url, # URL of the pageview (required) + 'dt': title, # Title of the page } data.update(DEFAULT_PARAMETERS) data.update(kwargs) @@ -46,12 +46,9 @@ def analytics_pageview(url, title=None, **kwargs): @app.task(queue='web') -def analytics_event( - event_category, event_action, event_label=None, event_value=None, - **kwargs -): +def analytics_event(event_category, event_action, event_label=None, event_value=None, **kwargs): """ - Send an analytics event to Google Analytics. + Send an analytics event to Google Analytics :see: https://developers.google.com/analytics/devguides/collection/protocol/v1/devguide#event :param event_category: the category of the event @@ -61,11 +58,11 @@ def analytics_event( :param kwargs: extra event parameters to send to GA """ data = { - 't': 'event', # GA event - don't change - 'ec': event_category, # Event category (required) - 'ea': event_action, # Event action (required) - 'el': event_label, # Event label - 'ev': event_value, # Event value (numeric) + 't': 'event', # GA event - don't change + 'ec': event_category, # Event category (required) + 'ea': event_action, # Event action (required) + 'el': event_label, # Event label + 'ev': event_value, # Event value (numeric) } data.update(DEFAULT_PARAMETERS) data.update(kwargs) diff --git a/readthedocs/analytics/tests.py b/readthedocs/analytics/tests.py index d3507d8642e..37b26957033 100644 --- a/readthedocs/analytics/tests.py +++ b/readthedocs/analytics/tests.py @@ -1,4 +1,5 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import, unicode_literals + from django.test import TestCase from .utils import anonymize_ip_address, anonymize_user_agent @@ -28,3 +29,4 @@ def test_anonymize_ua(self): anonymize_user_agent('Some rare user agent'), 'Rare user agent', ) + diff --git a/readthedocs/analytics/utils.py b/readthedocs/analytics/utils.py index c358423e499..44eef551125 100644 --- a/readthedocs/analytics/utils.py +++ b/readthedocs/analytics/utils.py @@ -1,23 +1,26 @@ -# -*- coding: utf-8 -*- - -"""Utilities related to analytics.""" +"""Utilities related to analytics""" +from __future__ import absolute_import, unicode_literals import hashlib -import ipaddress import logging -import requests from django.conf import settings +from django.utils.encoding import force_text, force_bytes from django.utils.crypto import get_random_string -from django.utils.encoding import force_bytes, force_text +import requests from user_agents import parse +try: + # Python 3.3+ only + import ipaddress +except ImportError: + from .vendor import ipaddress -log = logging.getLogger(__name__) # noqa +log = logging.getLogger(__name__) # noqa def get_client_ip(request): - """Gets the real IP based on a request object.""" + """Gets the real IP based on a request object""" ip_address = request.META.get('REMOTE_ADDR') # Get the original IP address (eg. "X-Forwarded-For: client, proxy1, proxy2") @@ -29,7 +32,7 @@ def get_client_ip(request): def anonymize_ip_address(ip_address): - """Anonymizes an IP address by zeroing the last 2 bytes.""" + """Anonymizes an IP address by zeroing the last 2 bytes""" # Used to anonymize an IP by zero-ing out the last 2 bytes ip_mask = int('0xFFFFFFFFFFFFFFFFFFFFFFFFFFFF0000', 16) @@ -43,7 +46,7 @@ def anonymize_ip_address(ip_address): def anonymize_user_agent(user_agent): - """Anonymizes rare user agents.""" + """Anonymizes rare user agents""" # If the browser family is not recognized, this is a rare user agent parsed_ua = parse(user_agent) if parsed_ua.browser.family == 'Other' or parsed_ua.os.family == 'Other': @@ -53,7 +56,7 @@ def anonymize_user_agent(user_agent): def send_to_analytics(data): - """Sends data to Google Analytics.""" + """Sends data to Google Analytics""" if data.get('uip') and data.get('ua'): data['cid'] = generate_client_id(data['uip'], data['ua']) @@ -71,7 +74,7 @@ def send_to_analytics(data): resp = requests.post( 'https://www.google-analytics.com/collect', data=data, - timeout=3, # seconds + timeout=3, # seconds ) except requests.Timeout: log.warning('Timeout sending to Google Analytics') @@ -82,10 +85,10 @@ def send_to_analytics(data): def generate_client_id(ip_address, user_agent): """ - Create an advertising ID. + Create an advertising ID - This simplifies things but essentially if a user has the same IP and same - UA, this will treat them as the same user for analytics purposes + This simplifies things but essentially if a user has the same IP and same UA, + this will treat them as the same user for analytics purposes """ salt = b'advertising-client-id' diff --git a/readthedocs/analytics/vendor/__init__.py b/readthedocs/analytics/vendor/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/analytics/vendor/ipaddress.py b/readthedocs/analytics/vendor/ipaddress.py new file mode 100644 index 00000000000..b81d477bf96 --- /dev/null +++ b/readthedocs/analytics/vendor/ipaddress.py @@ -0,0 +1,2420 @@ +# flake8: noqa +# Copyright 2007 Google Inc. +# Licensed to PSF under a Contributor Agreement. + +"""A fast, lightweight IPv4/IPv6 manipulation library in Python. + +This library is used to create/poke/manipulate IPv4 and IPv6 addresses +and networks. + +""" + +from __future__ import unicode_literals + + +import itertools +import struct + +__version__ = '1.0.22' + +# Compatibility functions +_compat_int_types = (int,) +try: + _compat_int_types = (int, long) +except NameError: + pass +try: + _compat_str = unicode +except NameError: + _compat_str = str + assert bytes != str +if b'\0'[0] == 0: # Python 3 semantics + def _compat_bytes_to_byte_vals(byt): + return byt +else: + def _compat_bytes_to_byte_vals(byt): + return [struct.unpack(b'!B', b)[0] for b in byt] +try: + _compat_int_from_byte_vals = int.from_bytes +except AttributeError: + def _compat_int_from_byte_vals(bytvals, endianess): + assert endianess == 'big' + res = 0 + for bv in bytvals: + assert isinstance(bv, _compat_int_types) + res = (res << 8) + bv + return res + + +def _compat_to_bytes(intval, length, endianess): + assert isinstance(intval, _compat_int_types) + assert endianess == 'big' + if length == 4: + if intval < 0 or intval >= 2 ** 32: + raise struct.error("integer out of range for 'I' format code") + return struct.pack(b'!I', intval) + elif length == 16: + if intval < 0 or intval >= 2 ** 128: + raise struct.error("integer out of range for 'QQ' format code") + return struct.pack(b'!QQ', intval >> 64, intval & 0xffffffffffffffff) + else: + raise NotImplementedError() + + +if hasattr(int, 'bit_length'): + # Not int.bit_length , since that won't work in 2.7 where long exists + def _compat_bit_length(i): + return i.bit_length() +else: + def _compat_bit_length(i): + for res in itertools.count(): + if i >> res == 0: + return res + + +def _compat_range(start, end, step=1): + assert step > 0 + i = start + while i < end: + yield i + i += step + + +class _TotalOrderingMixin(object): + __slots__ = () + + # Helper that derives the other comparison operations from + # __lt__ and __eq__ + # We avoid functools.total_ordering because it doesn't handle + # NotImplemented correctly yet (http://bugs.python.org/issue10042) + def __eq__(self, other): + raise NotImplementedError + + def __ne__(self, other): + equal = self.__eq__(other) + if equal is NotImplemented: + return NotImplemented + return not equal + + def __lt__(self, other): + raise NotImplementedError + + def __le__(self, other): + less = self.__lt__(other) + if less is NotImplemented or not less: + return self.__eq__(other) + return less + + def __gt__(self, other): + less = self.__lt__(other) + if less is NotImplemented: + return NotImplemented + equal = self.__eq__(other) + if equal is NotImplemented: + return NotImplemented + return not (less or equal) + + def __ge__(self, other): + less = self.__lt__(other) + if less is NotImplemented: + return NotImplemented + return not less + + +IPV4LENGTH = 32 +IPV6LENGTH = 128 + + +class AddressValueError(ValueError): + """A Value Error related to the address.""" + + +class NetmaskValueError(ValueError): + """A Value Error related to the netmask.""" + + +def ip_address(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Address or IPv6Address object. + + Raises: + ValueError: if the *address* passed isn't either a v4 or a v6 + address + + """ + try: + return IPv4Address(address) + except (AddressValueError, NetmaskValueError): + pass + + try: + return IPv6Address(address) + except (AddressValueError, NetmaskValueError): + pass + + if isinstance(address, bytes): + raise AddressValueError( + '%r does not appear to be an IPv4 or IPv6 address. ' + 'Did you pass in a bytes (str in Python 2) instead of' + ' a unicode object?' % address) + + raise ValueError('%r does not appear to be an IPv4 or IPv6 address' % + address) + + +def ip_network(address, strict=True): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP network. Either IPv4 or + IPv6 networks may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Network or IPv6Network object. + + Raises: + ValueError: if the string passed isn't either a v4 or a v6 + address. Or if the network has host bits set. + + """ + try: + return IPv4Network(address, strict) + except (AddressValueError, NetmaskValueError): + pass + + try: + return IPv6Network(address, strict) + except (AddressValueError, NetmaskValueError): + pass + + if isinstance(address, bytes): + raise AddressValueError( + '%r does not appear to be an IPv4 or IPv6 network. ' + 'Did you pass in a bytes (str in Python 2) instead of' + ' a unicode object?' % address) + + raise ValueError('%r does not appear to be an IPv4 or IPv6 network' % + address) + + +def ip_interface(address): + """Take an IP string/int and return an object of the correct type. + + Args: + address: A string or integer, the IP address. Either IPv4 or + IPv6 addresses may be supplied; integers less than 2**32 will + be considered to be IPv4 by default. + + Returns: + An IPv4Interface or IPv6Interface object. + + Raises: + ValueError: if the string passed isn't either a v4 or a v6 + address. + + Notes: + The IPv?Interface classes describe an Address on a particular + Network, so they're basically a combination of both the Address + and Network classes. + + """ + try: + return IPv4Interface(address) + except (AddressValueError, NetmaskValueError): + pass + + try: + return IPv6Interface(address) + except (AddressValueError, NetmaskValueError): + pass + + raise ValueError('%r does not appear to be an IPv4 or IPv6 interface' % + address) + + +def v4_int_to_packed(address): + """Represent an address as 4 packed bytes in network (big-endian) order. + + Args: + address: An integer representation of an IPv4 IP address. + + Returns: + The integer address packed as 4 bytes in network (big-endian) order. + + Raises: + ValueError: If the integer is negative or too large to be an + IPv4 IP address. + + """ + try: + return _compat_to_bytes(address, 4, 'big') + except (struct.error, OverflowError): + raise ValueError("Address negative or too large for IPv4") + + +def v6_int_to_packed(address): + """Represent an address as 16 packed bytes in network (big-endian) order. + + Args: + address: An integer representation of an IPv6 IP address. + + Returns: + The integer address packed as 16 bytes in network (big-endian) order. + + """ + try: + return _compat_to_bytes(address, 16, 'big') + except (struct.error, OverflowError): + raise ValueError("Address negative or too large for IPv6") + + +def _split_optional_netmask(address): + """Helper to split the netmask and raise AddressValueError if needed""" + addr = _compat_str(address).split('/') + if len(addr) > 2: + raise AddressValueError("Only one '/' permitted in %r" % address) + return addr + + +def _find_address_range(addresses): + """Find a sequence of sorted deduplicated IPv#Address. + + Args: + addresses: a list of IPv#Address objects. + + Yields: + A tuple containing the first and last IP addresses in the sequence. + + """ + it = iter(addresses) + first = last = next(it) + for ip in it: + if ip._ip != last._ip + 1: + yield first, last + first = ip + last = ip + yield first, last + + +def _count_righthand_zero_bits(number, bits): + """Count the number of zero bits on the right hand side. + + Args: + number: an integer. + bits: maximum number of bits to count. + + Returns: + The number of zero bits on the right hand side of the number. + + """ + if number == 0: + return bits + return min(bits, _compat_bit_length(~number & (number - 1))) + + +def summarize_address_range(first, last): + """Summarize a network range given the first and last IP addresses. + + Example: + >>> list(summarize_address_range(IPv4Address('192.0.2.0'), + ... IPv4Address('192.0.2.130'))) + ... #doctest: +NORMALIZE_WHITESPACE + [IPv4Network('192.0.2.0/25'), IPv4Network('192.0.2.128/31'), + IPv4Network('192.0.2.130/32')] + + Args: + first: the first IPv4Address or IPv6Address in the range. + last: the last IPv4Address or IPv6Address in the range. + + Returns: + An iterator of the summarized IPv(4|6) network objects. + + Raise: + TypeError: + If the first and last objects are not IP addresses. + If the first and last objects are not the same version. + ValueError: + If the last object is not greater than the first. + If the version of the first address is not 4 or 6. + + """ + if (not (isinstance(first, _BaseAddress) and + isinstance(last, _BaseAddress))): + raise TypeError('first and last must be IP addresses, not networks') + if first.version != last.version: + raise TypeError("%s and %s are not of the same version" % ( + first, last)) + if first > last: + raise ValueError('last IP address must be greater than first') + + if first.version == 4: + ip = IPv4Network + elif first.version == 6: + ip = IPv6Network + else: + raise ValueError('unknown IP version') + + ip_bits = first._max_prefixlen + first_int = first._ip + last_int = last._ip + while first_int <= last_int: + nbits = min(_count_righthand_zero_bits(first_int, ip_bits), + _compat_bit_length(last_int - first_int + 1) - 1) + net = ip((first_int, ip_bits - nbits)) + yield net + first_int += 1 << nbits + if first_int - 1 == ip._ALL_ONES: + break + + +def _collapse_addresses_internal(addresses): + """Loops through the addresses, collapsing concurrent netblocks. + + Example: + + ip1 = IPv4Network('192.0.2.0/26') + ip2 = IPv4Network('192.0.2.64/26') + ip3 = IPv4Network('192.0.2.128/26') + ip4 = IPv4Network('192.0.2.192/26') + + _collapse_addresses_internal([ip1, ip2, ip3, ip4]) -> + [IPv4Network('192.0.2.0/24')] + + This shouldn't be called directly; it is called via + collapse_addresses([]). + + Args: + addresses: A list of IPv4Network's or IPv6Network's + + Returns: + A list of IPv4Network's or IPv6Network's depending on what we were + passed. + + """ + # First merge + to_merge = list(addresses) + subnets = {} + while to_merge: + net = to_merge.pop() + supernet = net.supernet() + existing = subnets.get(supernet) + if existing is None: + subnets[supernet] = net + elif existing != net: + # Merge consecutive subnets + del subnets[supernet] + to_merge.append(supernet) + # Then iterate over resulting networks, skipping subsumed subnets + last = None + for net in sorted(subnets.values()): + if last is not None: + # Since they are sorted, + # last.network_address <= net.network_address is a given. + if last.broadcast_address >= net.broadcast_address: + continue + yield net + last = net + + +def collapse_addresses(addresses): + """Collapse a list of IP objects. + + Example: + collapse_addresses([IPv4Network('192.0.2.0/25'), + IPv4Network('192.0.2.128/25')]) -> + [IPv4Network('192.0.2.0/24')] + + Args: + addresses: An iterator of IPv4Network or IPv6Network objects. + + Returns: + An iterator of the collapsed IPv(4|6)Network objects. + + Raises: + TypeError: If passed a list of mixed version objects. + + """ + addrs = [] + ips = [] + nets = [] + + # split IP addresses and networks + for ip in addresses: + if isinstance(ip, _BaseAddress): + if ips and ips[-1]._version != ip._version: + raise TypeError("%s and %s are not of the same version" % ( + ip, ips[-1])) + ips.append(ip) + elif ip._prefixlen == ip._max_prefixlen: + if ips and ips[-1]._version != ip._version: + raise TypeError("%s and %s are not of the same version" % ( + ip, ips[-1])) + try: + ips.append(ip.ip) + except AttributeError: + ips.append(ip.network_address) + else: + if nets and nets[-1]._version != ip._version: + raise TypeError("%s and %s are not of the same version" % ( + ip, nets[-1])) + nets.append(ip) + + # sort and dedup + ips = sorted(set(ips)) + + # find consecutive address ranges in the sorted sequence and summarize them + if ips: + for first, last in _find_address_range(ips): + addrs.extend(summarize_address_range(first, last)) + + return _collapse_addresses_internal(addrs + nets) + + +def get_mixed_type_key(obj): + """Return a key suitable for sorting between networks and addresses. + + Address and Network objects are not sortable by default; they're + fundamentally different so the expression + + IPv4Address('192.0.2.0') <= IPv4Network('192.0.2.0/24') + + doesn't make any sense. There are some times however, where you may wish + to have ipaddress sort these for you anyway. If you need to do this, you + can use this function as the key= argument to sorted(). + + Args: + obj: either a Network or Address object. + Returns: + appropriate key. + + """ + if isinstance(obj, _BaseNetwork): + return obj._get_networks_key() + elif isinstance(obj, _BaseAddress): + return obj._get_address_key() + return NotImplemented + + +class _IPAddressBase(_TotalOrderingMixin): + + """The mother class.""" + + __slots__ = () + + @property + def exploded(self): + """Return the longhand version of the IP address as a string.""" + return self._explode_shorthand_ip_string() + + @property + def compressed(self): + """Return the shorthand version of the IP address as a string.""" + return _compat_str(self) + + @property + def reverse_pointer(self): + """The name of the reverse DNS pointer for the IP address, e.g.: + >>> ipaddress.ip_address("127.0.0.1").reverse_pointer + '1.0.0.127.in-addr.arpa' + >>> ipaddress.ip_address("2001:db8::1").reverse_pointer + '1.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.0.8.b.d.0.1.0.0.2.ip6.arpa' + + """ + return self._reverse_pointer() + + @property + def version(self): + msg = '%200s has no version specified' % (type(self),) + raise NotImplementedError(msg) + + def _check_int_address(self, address): + if address < 0: + msg = "%d (< 0) is not permitted as an IPv%d address" + raise AddressValueError(msg % (address, self._version)) + if address > self._ALL_ONES: + msg = "%d (>= 2**%d) is not permitted as an IPv%d address" + raise AddressValueError(msg % (address, self._max_prefixlen, + self._version)) + + def _check_packed_address(self, address, expected_len): + address_len = len(address) + if address_len != expected_len: + msg = ( + '%r (len %d != %d) is not permitted as an IPv%d address. ' + 'Did you pass in a bytes (str in Python 2) instead of' + ' a unicode object?') + raise AddressValueError(msg % (address, address_len, + expected_len, self._version)) + + @classmethod + def _ip_int_from_prefix(cls, prefixlen): + """Turn the prefix length into a bitwise netmask + + Args: + prefixlen: An integer, the prefix length. + + Returns: + An integer. + + """ + return cls._ALL_ONES ^ (cls._ALL_ONES >> prefixlen) + + @classmethod + def _prefix_from_ip_int(cls, ip_int): + """Return prefix length from the bitwise netmask. + + Args: + ip_int: An integer, the netmask in expanded bitwise format + + Returns: + An integer, the prefix length. + + Raises: + ValueError: If the input intermingles zeroes & ones + """ + trailing_zeroes = _count_righthand_zero_bits(ip_int, + cls._max_prefixlen) + prefixlen = cls._max_prefixlen - trailing_zeroes + leading_ones = ip_int >> trailing_zeroes + all_ones = (1 << prefixlen) - 1 + if leading_ones != all_ones: + byteslen = cls._max_prefixlen // 8 + details = _compat_to_bytes(ip_int, byteslen, 'big') + msg = 'Netmask pattern %r mixes zeroes & ones' + raise ValueError(msg % details) + return prefixlen + + @classmethod + def _report_invalid_netmask(cls, netmask_str): + msg = '%r is not a valid netmask' % netmask_str + raise NetmaskValueError(msg) + + @classmethod + def _prefix_from_prefix_string(cls, prefixlen_str): + """Return prefix length from a numeric string + + Args: + prefixlen_str: The string to be converted + + Returns: + An integer, the prefix length. + + Raises: + NetmaskValueError: If the input is not a valid netmask + """ + # int allows a leading +/- as well as surrounding whitespace, + # so we ensure that isn't the case + if not _BaseV4._DECIMAL_DIGITS.issuperset(prefixlen_str): + cls._report_invalid_netmask(prefixlen_str) + try: + prefixlen = int(prefixlen_str) + except ValueError: + cls._report_invalid_netmask(prefixlen_str) + if not (0 <= prefixlen <= cls._max_prefixlen): + cls._report_invalid_netmask(prefixlen_str) + return prefixlen + + @classmethod + def _prefix_from_ip_string(cls, ip_str): + """Turn a netmask/hostmask string into a prefix length + + Args: + ip_str: The netmask/hostmask to be converted + + Returns: + An integer, the prefix length. + + Raises: + NetmaskValueError: If the input is not a valid netmask/hostmask + """ + # Parse the netmask/hostmask like an IP address. + try: + ip_int = cls._ip_int_from_string(ip_str) + except AddressValueError: + cls._report_invalid_netmask(ip_str) + + # Try matching a netmask (this would be /1*0*/ as a bitwise regexp). + # Note that the two ambiguous cases (all-ones and all-zeroes) are + # treated as netmasks. + try: + return cls._prefix_from_ip_int(ip_int) + except ValueError: + pass + + # Invert the bits, and try matching a /0+1+/ hostmask instead. + ip_int ^= cls._ALL_ONES + try: + return cls._prefix_from_ip_int(ip_int) + except ValueError: + cls._report_invalid_netmask(ip_str) + + def __reduce__(self): + return self.__class__, (_compat_str(self),) + + +class _BaseAddress(_IPAddressBase): + + """A generic IP object. + + This IP class contains the version independent methods which are + used by single IP addresses. + """ + + __slots__ = () + + def __int__(self): + return self._ip + + def __eq__(self, other): + try: + return (self._ip == other._ip and + self._version == other._version) + except AttributeError: + return NotImplemented + + def __lt__(self, other): + if not isinstance(other, _IPAddressBase): + return NotImplemented + if not isinstance(other, _BaseAddress): + raise TypeError('%s and %s are not of the same type' % ( + self, other)) + if self._version != other._version: + raise TypeError('%s and %s are not of the same version' % ( + self, other)) + if self._ip != other._ip: + return self._ip < other._ip + return False + + # Shorthand for Integer addition and subtraction. This is not + # meant to ever support addition/subtraction of addresses. + def __add__(self, other): + if not isinstance(other, _compat_int_types): + return NotImplemented + return self.__class__(int(self) + other) + + def __sub__(self, other): + if not isinstance(other, _compat_int_types): + return NotImplemented + return self.__class__(int(self) - other) + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, _compat_str(self)) + + def __str__(self): + return _compat_str(self._string_from_ip_int(self._ip)) + + def __hash__(self): + return hash(hex(int(self._ip))) + + def _get_address_key(self): + return (self._version, self) + + def __reduce__(self): + return self.__class__, (self._ip,) + + +class _BaseNetwork(_IPAddressBase): + + """A generic IP network object. + + This IP class contains the version independent methods which are + used by networks. + + """ + def __init__(self, address): + self._cache = {} + + def __repr__(self): + return '%s(%r)' % (self.__class__.__name__, _compat_str(self)) + + def __str__(self): + return '%s/%d' % (self.network_address, self.prefixlen) + + def hosts(self): + """Generate Iterator over usable hosts in a network. + + This is like __iter__ except it doesn't return the network + or broadcast addresses. + + """ + network = int(self.network_address) + broadcast = int(self.broadcast_address) + for x in _compat_range(network + 1, broadcast): + yield self._address_class(x) + + def __iter__(self): + network = int(self.network_address) + broadcast = int(self.broadcast_address) + for x in _compat_range(network, broadcast + 1): + yield self._address_class(x) + + def __getitem__(self, n): + network = int(self.network_address) + broadcast = int(self.broadcast_address) + if n >= 0: + if network + n > broadcast: + raise IndexError('address out of range') + return self._address_class(network + n) + else: + n += 1 + if broadcast + n < network: + raise IndexError('address out of range') + return self._address_class(broadcast + n) + + def __lt__(self, other): + if not isinstance(other, _IPAddressBase): + return NotImplemented + if not isinstance(other, _BaseNetwork): + raise TypeError('%s and %s are not of the same type' % ( + self, other)) + if self._version != other._version: + raise TypeError('%s and %s are not of the same version' % ( + self, other)) + if self.network_address != other.network_address: + return self.network_address < other.network_address + if self.netmask != other.netmask: + return self.netmask < other.netmask + return False + + def __eq__(self, other): + try: + return (self._version == other._version and + self.network_address == other.network_address and + int(self.netmask) == int(other.netmask)) + except AttributeError: + return NotImplemented + + def __hash__(self): + return hash(int(self.network_address) ^ int(self.netmask)) + + def __contains__(self, other): + # always false if one is v4 and the other is v6. + if self._version != other._version: + return False + # dealing with another network. + if isinstance(other, _BaseNetwork): + return False + # dealing with another address + else: + # address + return (int(self.network_address) <= int(other._ip) <= + int(self.broadcast_address)) + + def overlaps(self, other): + """Tell if self is partly contained in other.""" + return self.network_address in other or ( + self.broadcast_address in other or ( + other.network_address in self or ( + other.broadcast_address in self))) + + @property + def broadcast_address(self): + x = self._cache.get('broadcast_address') + if x is None: + x = self._address_class(int(self.network_address) | + int(self.hostmask)) + self._cache['broadcast_address'] = x + return x + + @property + def hostmask(self): + x = self._cache.get('hostmask') + if x is None: + x = self._address_class(int(self.netmask) ^ self._ALL_ONES) + self._cache['hostmask'] = x + return x + + @property + def with_prefixlen(self): + return '%s/%d' % (self.network_address, self._prefixlen) + + @property + def with_netmask(self): + return '%s/%s' % (self.network_address, self.netmask) + + @property + def with_hostmask(self): + return '%s/%s' % (self.network_address, self.hostmask) + + @property + def num_addresses(self): + """Number of hosts in the current subnet.""" + return int(self.broadcast_address) - int(self.network_address) + 1 + + @property + def _address_class(self): + # Returning bare address objects (rather than interfaces) allows for + # more consistent behaviour across the network address, broadcast + # address and individual host addresses. + msg = '%200s has no associated address class' % (type(self),) + raise NotImplementedError(msg) + + @property + def prefixlen(self): + return self._prefixlen + + def address_exclude(self, other): + """Remove an address from a larger block. + + For example: + + addr1 = ip_network('192.0.2.0/28') + addr2 = ip_network('192.0.2.1/32') + list(addr1.address_exclude(addr2)) = + [IPv4Network('192.0.2.0/32'), IPv4Network('192.0.2.2/31'), + IPv4Network('192.0.2.4/30'), IPv4Network('192.0.2.8/29')] + + or IPv6: + + addr1 = ip_network('2001:db8::1/32') + addr2 = ip_network('2001:db8::1/128') + list(addr1.address_exclude(addr2)) = + [ip_network('2001:db8::1/128'), + ip_network('2001:db8::2/127'), + ip_network('2001:db8::4/126'), + ip_network('2001:db8::8/125'), + ... + ip_network('2001:db8:8000::/33')] + + Args: + other: An IPv4Network or IPv6Network object of the same type. + + Returns: + An iterator of the IPv(4|6)Network objects which is self + minus other. + + Raises: + TypeError: If self and other are of differing address + versions, or if other is not a network object. + ValueError: If other is not completely contained by self. + + """ + if not self._version == other._version: + raise TypeError("%s and %s are not of the same version" % ( + self, other)) + + if not isinstance(other, _BaseNetwork): + raise TypeError("%s is not a network object" % other) + + if not other.subnet_of(self): + raise ValueError('%s not contained in %s' % (other, self)) + if other == self: + return + + # Make sure we're comparing the network of other. + other = other.__class__('%s/%s' % (other.network_address, + other.prefixlen)) + + s1, s2 = self.subnets() + while s1 != other and s2 != other: + if other.subnet_of(s1): + yield s2 + s1, s2 = s1.subnets() + elif other.subnet_of(s2): + yield s1 + s1, s2 = s2.subnets() + else: + # If we got here, there's a bug somewhere. + raise AssertionError('Error performing exclusion: ' + 's1: %s s2: %s other: %s' % + (s1, s2, other)) + if s1 == other: + yield s2 + elif s2 == other: + yield s1 + else: + # If we got here, there's a bug somewhere. + raise AssertionError('Error performing exclusion: ' + 's1: %s s2: %s other: %s' % + (s1, s2, other)) + + def compare_networks(self, other): + """Compare two IP objects. + + This is only concerned about the comparison of the integer + representation of the network addresses. This means that the + host bits aren't considered at all in this method. If you want + to compare host bits, you can easily enough do a + 'HostA._ip < HostB._ip' + + Args: + other: An IP object. + + Returns: + If the IP versions of self and other are the same, returns: + + -1 if self < other: + eg: IPv4Network('192.0.2.0/25') < IPv4Network('192.0.2.128/25') + IPv6Network('2001:db8::1000/124') < + IPv6Network('2001:db8::2000/124') + 0 if self == other + eg: IPv4Network('192.0.2.0/24') == IPv4Network('192.0.2.0/24') + IPv6Network('2001:db8::1000/124') == + IPv6Network('2001:db8::1000/124') + 1 if self > other + eg: IPv4Network('192.0.2.128/25') > IPv4Network('192.0.2.0/25') + IPv6Network('2001:db8::2000/124') > + IPv6Network('2001:db8::1000/124') + + Raises: + TypeError if the IP versions are different. + + """ + # does this need to raise a ValueError? + if self._version != other._version: + raise TypeError('%s and %s are not of the same type' % ( + self, other)) + # self._version == other._version below here: + if self.network_address < other.network_address: + return -1 + if self.network_address > other.network_address: + return 1 + # self.network_address == other.network_address below here: + if self.netmask < other.netmask: + return -1 + if self.netmask > other.netmask: + return 1 + return 0 + + def _get_networks_key(self): + """Network-only key function. + + Returns an object that identifies this address' network and + netmask. This function is a suitable "key" argument for sorted() + and list.sort(). + + """ + return (self._version, self.network_address, self.netmask) + + def subnets(self, prefixlen_diff=1, new_prefix=None): + """The subnets which join to make the current subnet. + + In the case that self contains only one IP + (self._prefixlen == 32 for IPv4 or self._prefixlen == 128 + for IPv6), yield an iterator with just ourself. + + Args: + prefixlen_diff: An integer, the amount the prefix length + should be increased by. This should not be set if + new_prefix is also set. + new_prefix: The desired new prefix length. This must be a + larger number (smaller prefix) than the existing prefix. + This should not be set if prefixlen_diff is also set. + + Returns: + An iterator of IPv(4|6) objects. + + Raises: + ValueError: The prefixlen_diff is too small or too large. + OR + prefixlen_diff and new_prefix are both set or new_prefix + is a smaller number than the current prefix (smaller + number means a larger network) + + """ + if self._prefixlen == self._max_prefixlen: + yield self + return + + if new_prefix is not None: + if new_prefix < self._prefixlen: + raise ValueError('new prefix must be longer') + if prefixlen_diff != 1: + raise ValueError('cannot set prefixlen_diff and new_prefix') + prefixlen_diff = new_prefix - self._prefixlen + + if prefixlen_diff < 0: + raise ValueError('prefix length diff must be > 0') + new_prefixlen = self._prefixlen + prefixlen_diff + + if new_prefixlen > self._max_prefixlen: + raise ValueError( + 'prefix length diff %d is invalid for netblock %s' % ( + new_prefixlen, self)) + + start = int(self.network_address) + end = int(self.broadcast_address) + 1 + step = (int(self.hostmask) + 1) >> prefixlen_diff + for new_addr in _compat_range(start, end, step): + current = self.__class__((new_addr, new_prefixlen)) + yield current + + def supernet(self, prefixlen_diff=1, new_prefix=None): + """The supernet containing the current network. + + Args: + prefixlen_diff: An integer, the amount the prefix length of + the network should be decreased by. For example, given a + /24 network and a prefixlen_diff of 3, a supernet with a + /21 netmask is returned. + + Returns: + An IPv4 network object. + + Raises: + ValueError: If self.prefixlen - prefixlen_diff < 0. I.e., you have + a negative prefix length. + OR + If prefixlen_diff and new_prefix are both set or new_prefix is a + larger number than the current prefix (larger number means a + smaller network) + + """ + if self._prefixlen == 0: + return self + + if new_prefix is not None: + if new_prefix > self._prefixlen: + raise ValueError('new prefix must be shorter') + if prefixlen_diff != 1: + raise ValueError('cannot set prefixlen_diff and new_prefix') + prefixlen_diff = self._prefixlen - new_prefix + + new_prefixlen = self.prefixlen - prefixlen_diff + if new_prefixlen < 0: + raise ValueError( + 'current prefixlen is %d, cannot have a prefixlen_diff of %d' % + (self.prefixlen, prefixlen_diff)) + return self.__class__(( + int(self.network_address) & (int(self.netmask) << prefixlen_diff), + new_prefixlen)) + + @property + def is_multicast(self): + """Test if the address is reserved for multicast use. + + Returns: + A boolean, True if the address is a multicast address. + See RFC 2373 2.7 for details. + + """ + return (self.network_address.is_multicast and + self.broadcast_address.is_multicast) + + @staticmethod + def _is_subnet_of(a, b): + try: + # Always false if one is v4 and the other is v6. + if a._version != b._version: + raise TypeError("%s and %s are not of the same version" (a, b)) + return (b.network_address <= a.network_address and + b.broadcast_address >= a.broadcast_address) + except AttributeError: + raise TypeError("Unable to test subnet containment " + "between %s and %s" % (a, b)) + + def subnet_of(self, other): + """Return True if this network is a subnet of other.""" + return self._is_subnet_of(self, other) + + def supernet_of(self, other): + """Return True if this network is a supernet of other.""" + return self._is_subnet_of(other, self) + + @property + def is_reserved(self): + """Test if the address is otherwise IETF reserved. + + Returns: + A boolean, True if the address is within one of the + reserved IPv6 Network ranges. + + """ + return (self.network_address.is_reserved and + self.broadcast_address.is_reserved) + + @property + def is_link_local(self): + """Test if the address is reserved for link-local. + + Returns: + A boolean, True if the address is reserved per RFC 4291. + + """ + return (self.network_address.is_link_local and + self.broadcast_address.is_link_local) + + @property + def is_private(self): + """Test if this address is allocated for private networks. + + Returns: + A boolean, True if the address is reserved per + iana-ipv4-special-registry or iana-ipv6-special-registry. + + """ + return (self.network_address.is_private and + self.broadcast_address.is_private) + + @property + def is_global(self): + """Test if this address is allocated for public networks. + + Returns: + A boolean, True if the address is not reserved per + iana-ipv4-special-registry or iana-ipv6-special-registry. + + """ + return not self.is_private + + @property + def is_unspecified(self): + """Test if the address is unspecified. + + Returns: + A boolean, True if this is the unspecified address as defined in + RFC 2373 2.5.2. + + """ + return (self.network_address.is_unspecified and + self.broadcast_address.is_unspecified) + + @property + def is_loopback(self): + """Test if the address is a loopback address. + + Returns: + A boolean, True if the address is a loopback address as defined in + RFC 2373 2.5.3. + + """ + return (self.network_address.is_loopback and + self.broadcast_address.is_loopback) + + +class _BaseV4(object): + + """Base IPv4 object. + + The following methods are used by IPv4 objects in both single IP + addresses and networks. + + """ + + __slots__ = () + _version = 4 + # Equivalent to 255.255.255.255 or 32 bits of 1's. + _ALL_ONES = (2 ** IPV4LENGTH) - 1 + _DECIMAL_DIGITS = frozenset('0123456789') + + # the valid octets for host and netmasks. only useful for IPv4. + _valid_mask_octets = frozenset([255, 254, 252, 248, 240, 224, 192, 128, 0]) + + _max_prefixlen = IPV4LENGTH + # There are only a handful of valid v4 netmasks, so we cache them all + # when constructed (see _make_netmask()). + _netmask_cache = {} + + def _explode_shorthand_ip_string(self): + return _compat_str(self) + + @classmethod + def _make_netmask(cls, arg): + """Make a (netmask, prefix_len) tuple from the given argument. + + Argument can be: + - an integer (the prefix length) + - a string representing the prefix length (e.g. "24") + - a string representing the prefix netmask (e.g. "255.255.255.0") + """ + if arg not in cls._netmask_cache: + if isinstance(arg, _compat_int_types): + prefixlen = arg + else: + try: + # Check for a netmask in prefix length form + prefixlen = cls._prefix_from_prefix_string(arg) + except NetmaskValueError: + # Check for a netmask or hostmask in dotted-quad form. + # This may raise NetmaskValueError. + prefixlen = cls._prefix_from_ip_string(arg) + netmask = IPv4Address(cls._ip_int_from_prefix(prefixlen)) + cls._netmask_cache[arg] = netmask, prefixlen + return cls._netmask_cache[arg] + + @classmethod + def _ip_int_from_string(cls, ip_str): + """Turn the given IP string into an integer for comparison. + + Args: + ip_str: A string, the IP ip_str. + + Returns: + The IP ip_str as an integer. + + Raises: + AddressValueError: if ip_str isn't a valid IPv4 Address. + + """ + if not ip_str: + raise AddressValueError('Address cannot be empty') + + octets = ip_str.split('.') + if len(octets) != 4: + raise AddressValueError("Expected 4 octets in %r" % ip_str) + + try: + return _compat_int_from_byte_vals( + map(cls._parse_octet, octets), 'big') + except ValueError as exc: + raise AddressValueError("%s in %r" % (exc, ip_str)) + + @classmethod + def _parse_octet(cls, octet_str): + """Convert a decimal octet into an integer. + + Args: + octet_str: A string, the number to parse. + + Returns: + The octet as an integer. + + Raises: + ValueError: if the octet isn't strictly a decimal from [0..255]. + + """ + if not octet_str: + raise ValueError("Empty octet not permitted") + # Whitelist the characters, since int() allows a lot of bizarre stuff. + if not cls._DECIMAL_DIGITS.issuperset(octet_str): + msg = "Only decimal digits permitted in %r" + raise ValueError(msg % octet_str) + # We do the length check second, since the invalid character error + # is likely to be more informative for the user + if len(octet_str) > 3: + msg = "At most 3 characters permitted in %r" + raise ValueError(msg % octet_str) + # Convert to integer (we know digits are legal) + octet_int = int(octet_str, 10) + # Any octets that look like they *might* be written in octal, + # and which don't look exactly the same in both octal and + # decimal are rejected as ambiguous + if octet_int > 7 and octet_str[0] == '0': + msg = "Ambiguous (octal/decimal) value in %r not permitted" + raise ValueError(msg % octet_str) + if octet_int > 255: + raise ValueError("Octet %d (> 255) not permitted" % octet_int) + return octet_int + + @classmethod + def _string_from_ip_int(cls, ip_int): + """Turns a 32-bit integer into dotted decimal notation. + + Args: + ip_int: An integer, the IP address. + + Returns: + The IP address as a string in dotted decimal notation. + + """ + return '.'.join(_compat_str(struct.unpack(b'!B', b)[0] + if isinstance(b, bytes) + else b) + for b in _compat_to_bytes(ip_int, 4, 'big')) + + def _is_hostmask(self, ip_str): + """Test if the IP string is a hostmask (rather than a netmask). + + Args: + ip_str: A string, the potential hostmask. + + Returns: + A boolean, True if the IP string is a hostmask. + + """ + bits = ip_str.split('.') + try: + parts = [x for x in map(int, bits) if x in self._valid_mask_octets] + except ValueError: + return False + if len(parts) != len(bits): + return False + if parts[0] < parts[-1]: + return True + return False + + def _reverse_pointer(self): + """Return the reverse DNS pointer name for the IPv4 address. + + This implements the method described in RFC1035 3.5. + + """ + reverse_octets = _compat_str(self).split('.')[::-1] + return '.'.join(reverse_octets) + '.in-addr.arpa' + + @property + def max_prefixlen(self): + return self._max_prefixlen + + @property + def version(self): + return self._version + + +class IPv4Address(_BaseV4, _BaseAddress): + + """Represent and manipulate single IPv4 Addresses.""" + + __slots__ = ('_ip', '__weakref__') + + def __init__(self, address): + + """ + Args: + address: A string or integer representing the IP + + Additionally, an integer can be passed, so + IPv4Address('192.0.2.1') == IPv4Address(3221225985). + or, more generally + IPv4Address(int(IPv4Address('192.0.2.1'))) == + IPv4Address('192.0.2.1') + + Raises: + AddressValueError: If ipaddress isn't a valid IPv4 address. + + """ + # Efficient constructor from integer. + if isinstance(address, _compat_int_types): + self._check_int_address(address) + self._ip = address + return + + # Constructing from a packed address + if isinstance(address, bytes): + self._check_packed_address(address, 4) + bvs = _compat_bytes_to_byte_vals(address) + self._ip = _compat_int_from_byte_vals(bvs, 'big') + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP string. + addr_str = _compat_str(address) + if '/' in addr_str: + raise AddressValueError("Unexpected '/' in %r" % address) + self._ip = self._ip_int_from_string(addr_str) + + @property + def packed(self): + """The binary representation of this address.""" + return v4_int_to_packed(self._ip) + + @property + def is_reserved(self): + """Test if the address is otherwise IETF reserved. + + Returns: + A boolean, True if the address is within the + reserved IPv4 Network range. + + """ + return self in self._constants._reserved_network + + @property + def is_private(self): + """Test if this address is allocated for private networks. + + Returns: + A boolean, True if the address is reserved per + iana-ipv4-special-registry. + + """ + return any(self in net for net in self._constants._private_networks) + + @property + def is_global(self): + return ( + self not in self._constants._public_network and + not self.is_private) + + @property + def is_multicast(self): + """Test if the address is reserved for multicast use. + + Returns: + A boolean, True if the address is multicast. + See RFC 3171 for details. + + """ + return self in self._constants._multicast_network + + @property + def is_unspecified(self): + """Test if the address is unspecified. + + Returns: + A boolean, True if this is the unspecified address as defined in + RFC 5735 3. + + """ + return self == self._constants._unspecified_address + + @property + def is_loopback(self): + """Test if the address is a loopback address. + + Returns: + A boolean, True if the address is a loopback per RFC 3330. + + """ + return self in self._constants._loopback_network + + @property + def is_link_local(self): + """Test if the address is reserved for link-local. + + Returns: + A boolean, True if the address is link-local per RFC 3927. + + """ + return self in self._constants._linklocal_network + + +class IPv4Interface(IPv4Address): + + def __init__(self, address): + if isinstance(address, (bytes, _compat_int_types)): + IPv4Address.__init__(self, address) + self.network = IPv4Network(self._ip) + self._prefixlen = self._max_prefixlen + return + + if isinstance(address, tuple): + IPv4Address.__init__(self, address[0]) + if len(address) > 1: + self._prefixlen = int(address[1]) + else: + self._prefixlen = self._max_prefixlen + + self.network = IPv4Network(address, strict=False) + self.netmask = self.network.netmask + self.hostmask = self.network.hostmask + return + + addr = _split_optional_netmask(address) + IPv4Address.__init__(self, addr[0]) + + self.network = IPv4Network(address, strict=False) + self._prefixlen = self.network._prefixlen + + self.netmask = self.network.netmask + self.hostmask = self.network.hostmask + + def __str__(self): + return '%s/%d' % (self._string_from_ip_int(self._ip), + self.network.prefixlen) + + def __eq__(self, other): + address_equal = IPv4Address.__eq__(self, other) + if not address_equal or address_equal is NotImplemented: + return address_equal + try: + return self.network == other.network + except AttributeError: + # An interface with an associated network is NOT the + # same as an unassociated address. That's why the hash + # takes the extra info into account. + return False + + def __lt__(self, other): + address_less = IPv4Address.__lt__(self, other) + if address_less is NotImplemented: + return NotImplemented + try: + return (self.network < other.network or + self.network == other.network and address_less) + except AttributeError: + # We *do* allow addresses and interfaces to be sorted. The + # unassociated address is considered less than all interfaces. + return False + + def __hash__(self): + return self._ip ^ self._prefixlen ^ int(self.network.network_address) + + __reduce__ = _IPAddressBase.__reduce__ + + @property + def ip(self): + return IPv4Address(self._ip) + + @property + def with_prefixlen(self): + return '%s/%s' % (self._string_from_ip_int(self._ip), + self._prefixlen) + + @property + def with_netmask(self): + return '%s/%s' % (self._string_from_ip_int(self._ip), + self.netmask) + + @property + def with_hostmask(self): + return '%s/%s' % (self._string_from_ip_int(self._ip), + self.hostmask) + + +class IPv4Network(_BaseV4, _BaseNetwork): + + """This class represents and manipulates 32-bit IPv4 network + addresses.. + + Attributes: [examples for IPv4Network('192.0.2.0/27')] + .network_address: IPv4Address('192.0.2.0') + .hostmask: IPv4Address('0.0.0.31') + .broadcast_address: IPv4Address('192.0.2.32') + .netmask: IPv4Address('255.255.255.224') + .prefixlen: 27 + + """ + # Class to use when creating address objects + _address_class = IPv4Address + + def __init__(self, address, strict=True): + + """Instantiate a new IPv4 network object. + + Args: + address: A string or integer representing the IP [& network]. + '192.0.2.0/24' + '192.0.2.0/255.255.255.0' + '192.0.0.2/0.0.0.255' + are all functionally the same in IPv4. Similarly, + '192.0.2.1' + '192.0.2.1/255.255.255.255' + '192.0.2.1/32' + are also functionally equivalent. That is to say, failing to + provide a subnetmask will create an object with a mask of /32. + + If the mask (portion after the / in the argument) is given in + dotted quad form, it is treated as a netmask if it starts with a + non-zero field (e.g. /255.0.0.0 == /8) and as a hostmask if it + starts with a zero field (e.g. 0.255.255.255 == /8), with the + single exception of an all-zero mask which is treated as a + netmask == /0. If no mask is given, a default of /32 is used. + + Additionally, an integer can be passed, so + IPv4Network('192.0.2.1') == IPv4Network(3221225985) + or, more generally + IPv4Interface(int(IPv4Interface('192.0.2.1'))) == + IPv4Interface('192.0.2.1') + + Raises: + AddressValueError: If ipaddress isn't a valid IPv4 address. + NetmaskValueError: If the netmask isn't valid for + an IPv4 address. + ValueError: If strict is True and a network address is not + supplied. + + """ + _BaseNetwork.__init__(self, address) + + # Constructing from a packed address or integer + if isinstance(address, (_compat_int_types, bytes)): + self.network_address = IPv4Address(address) + self.netmask, self._prefixlen = self._make_netmask( + self._max_prefixlen) + # fixme: address/network test here. + return + + if isinstance(address, tuple): + if len(address) > 1: + arg = address[1] + else: + # We weren't given an address[1] + arg = self._max_prefixlen + self.network_address = IPv4Address(address[0]) + self.netmask, self._prefixlen = self._make_netmask(arg) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: + raise ValueError('%s has host bits set' % self) + else: + self.network_address = IPv4Address(packed & + int(self.netmask)) + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP prefix string. + addr = _split_optional_netmask(address) + self.network_address = IPv4Address(self._ip_int_from_string(addr[0])) + + if len(addr) == 2: + arg = addr[1] + else: + arg = self._max_prefixlen + self.netmask, self._prefixlen = self._make_netmask(arg) + + if strict: + if (IPv4Address(int(self.network_address) & int(self.netmask)) != + self.network_address): + raise ValueError('%s has host bits set' % self) + self.network_address = IPv4Address(int(self.network_address) & + int(self.netmask)) + + if self._prefixlen == (self._max_prefixlen - 1): + self.hosts = self.__iter__ + + @property + def is_global(self): + """Test if this address is allocated for public networks. + + Returns: + A boolean, True if the address is not reserved per + iana-ipv4-special-registry. + + """ + return (not (self.network_address in IPv4Network('100.64.0.0/10') and + self.broadcast_address in IPv4Network('100.64.0.0/10')) and + not self.is_private) + + +class _IPv4Constants(object): + + _linklocal_network = IPv4Network('169.254.0.0/16') + + _loopback_network = IPv4Network('127.0.0.0/8') + + _multicast_network = IPv4Network('224.0.0.0/4') + + _public_network = IPv4Network('100.64.0.0/10') + + _private_networks = [ + IPv4Network('0.0.0.0/8'), + IPv4Network('10.0.0.0/8'), + IPv4Network('127.0.0.0/8'), + IPv4Network('169.254.0.0/16'), + IPv4Network('172.16.0.0/12'), + IPv4Network('192.0.0.0/29'), + IPv4Network('192.0.0.170/31'), + IPv4Network('192.0.2.0/24'), + IPv4Network('192.168.0.0/16'), + IPv4Network('198.18.0.0/15'), + IPv4Network('198.51.100.0/24'), + IPv4Network('203.0.113.0/24'), + IPv4Network('240.0.0.0/4'), + IPv4Network('255.255.255.255/32'), + ] + + _reserved_network = IPv4Network('240.0.0.0/4') + + _unspecified_address = IPv4Address('0.0.0.0') + + +IPv4Address._constants = _IPv4Constants + + +class _BaseV6(object): + + """Base IPv6 object. + + The following methods are used by IPv6 objects in both single IP + addresses and networks. + + """ + + __slots__ = () + _version = 6 + _ALL_ONES = (2 ** IPV6LENGTH) - 1 + _HEXTET_COUNT = 8 + _HEX_DIGITS = frozenset('0123456789ABCDEFabcdef') + _max_prefixlen = IPV6LENGTH + + # There are only a bunch of valid v6 netmasks, so we cache them all + # when constructed (see _make_netmask()). + _netmask_cache = {} + + @classmethod + def _make_netmask(cls, arg): + """Make a (netmask, prefix_len) tuple from the given argument. + + Argument can be: + - an integer (the prefix length) + - a string representing the prefix length (e.g. "24") + - a string representing the prefix netmask (e.g. "255.255.255.0") + """ + if arg not in cls._netmask_cache: + if isinstance(arg, _compat_int_types): + prefixlen = arg + else: + prefixlen = cls._prefix_from_prefix_string(arg) + netmask = IPv6Address(cls._ip_int_from_prefix(prefixlen)) + cls._netmask_cache[arg] = netmask, prefixlen + return cls._netmask_cache[arg] + + @classmethod + def _ip_int_from_string(cls, ip_str): + """Turn an IPv6 ip_str into an integer. + + Args: + ip_str: A string, the IPv6 ip_str. + + Returns: + An int, the IPv6 address + + Raises: + AddressValueError: if ip_str isn't a valid IPv6 Address. + + """ + if not ip_str: + raise AddressValueError('Address cannot be empty') + + parts = ip_str.split(':') + + # An IPv6 address needs at least 2 colons (3 parts). + _min_parts = 3 + if len(parts) < _min_parts: + msg = "At least %d parts expected in %r" % (_min_parts, ip_str) + raise AddressValueError(msg) + + # If the address has an IPv4-style suffix, convert it to hexadecimal. + if '.' in parts[-1]: + try: + ipv4_int = IPv4Address(parts.pop())._ip + except AddressValueError as exc: + raise AddressValueError("%s in %r" % (exc, ip_str)) + parts.append('%x' % ((ipv4_int >> 16) & 0xFFFF)) + parts.append('%x' % (ipv4_int & 0xFFFF)) + + # An IPv6 address can't have more than 8 colons (9 parts). + # The extra colon comes from using the "::" notation for a single + # leading or trailing zero part. + _max_parts = cls._HEXTET_COUNT + 1 + if len(parts) > _max_parts: + msg = "At most %d colons permitted in %r" % ( + _max_parts - 1, ip_str) + raise AddressValueError(msg) + + # Disregarding the endpoints, find '::' with nothing in between. + # This indicates that a run of zeroes has been skipped. + skip_index = None + for i in _compat_range(1, len(parts) - 1): + if not parts[i]: + if skip_index is not None: + # Can't have more than one '::' + msg = "At most one '::' permitted in %r" % ip_str + raise AddressValueError(msg) + skip_index = i + + # parts_hi is the number of parts to copy from above/before the '::' + # parts_lo is the number of parts to copy from below/after the '::' + if skip_index is not None: + # If we found a '::', then check if it also covers the endpoints. + parts_hi = skip_index + parts_lo = len(parts) - skip_index - 1 + if not parts[0]: + parts_hi -= 1 + if parts_hi: + msg = "Leading ':' only permitted as part of '::' in %r" + raise AddressValueError(msg % ip_str) # ^: requires ^:: + if not parts[-1]: + parts_lo -= 1 + if parts_lo: + msg = "Trailing ':' only permitted as part of '::' in %r" + raise AddressValueError(msg % ip_str) # :$ requires ::$ + parts_skipped = cls._HEXTET_COUNT - (parts_hi + parts_lo) + if parts_skipped < 1: + msg = "Expected at most %d other parts with '::' in %r" + raise AddressValueError(msg % (cls._HEXTET_COUNT - 1, ip_str)) + else: + # Otherwise, allocate the entire address to parts_hi. The + # endpoints could still be empty, but _parse_hextet() will check + # for that. + if len(parts) != cls._HEXTET_COUNT: + msg = "Exactly %d parts expected without '::' in %r" + raise AddressValueError(msg % (cls._HEXTET_COUNT, ip_str)) + if not parts[0]: + msg = "Leading ':' only permitted as part of '::' in %r" + raise AddressValueError(msg % ip_str) # ^: requires ^:: + if not parts[-1]: + msg = "Trailing ':' only permitted as part of '::' in %r" + raise AddressValueError(msg % ip_str) # :$ requires ::$ + parts_hi = len(parts) + parts_lo = 0 + parts_skipped = 0 + + try: + # Now, parse the hextets into a 128-bit integer. + ip_int = 0 + for i in range(parts_hi): + ip_int <<= 16 + ip_int |= cls._parse_hextet(parts[i]) + ip_int <<= 16 * parts_skipped + for i in range(-parts_lo, 0): + ip_int <<= 16 + ip_int |= cls._parse_hextet(parts[i]) + return ip_int + except ValueError as exc: + raise AddressValueError("%s in %r" % (exc, ip_str)) + + @classmethod + def _parse_hextet(cls, hextet_str): + """Convert an IPv6 hextet string into an integer. + + Args: + hextet_str: A string, the number to parse. + + Returns: + The hextet as an integer. + + Raises: + ValueError: if the input isn't strictly a hex number from + [0..FFFF]. + + """ + # Whitelist the characters, since int() allows a lot of bizarre stuff. + if not cls._HEX_DIGITS.issuperset(hextet_str): + raise ValueError("Only hex digits permitted in %r" % hextet_str) + # We do the length check second, since the invalid character error + # is likely to be more informative for the user + if len(hextet_str) > 4: + msg = "At most 4 characters permitted in %r" + raise ValueError(msg % hextet_str) + # Length check means we can skip checking the integer value + return int(hextet_str, 16) + + @classmethod + def _compress_hextets(cls, hextets): + """Compresses a list of hextets. + + Compresses a list of strings, replacing the longest continuous + sequence of "0" in the list with "" and adding empty strings at + the beginning or at the end of the string such that subsequently + calling ":".join(hextets) will produce the compressed version of + the IPv6 address. + + Args: + hextets: A list of strings, the hextets to compress. + + Returns: + A list of strings. + + """ + best_doublecolon_start = -1 + best_doublecolon_len = 0 + doublecolon_start = -1 + doublecolon_len = 0 + for index, hextet in enumerate(hextets): + if hextet == '0': + doublecolon_len += 1 + if doublecolon_start == -1: + # Start of a sequence of zeros. + doublecolon_start = index + if doublecolon_len > best_doublecolon_len: + # This is the longest sequence of zeros so far. + best_doublecolon_len = doublecolon_len + best_doublecolon_start = doublecolon_start + else: + doublecolon_len = 0 + doublecolon_start = -1 + + if best_doublecolon_len > 1: + best_doublecolon_end = (best_doublecolon_start + + best_doublecolon_len) + # For zeros at the end of the address. + if best_doublecolon_end == len(hextets): + hextets += [''] + hextets[best_doublecolon_start:best_doublecolon_end] = [''] + # For zeros at the beginning of the address. + if best_doublecolon_start == 0: + hextets = [''] + hextets + + return hextets + + @classmethod + def _string_from_ip_int(cls, ip_int=None): + """Turns a 128-bit integer into hexadecimal notation. + + Args: + ip_int: An integer, the IP address. + + Returns: + A string, the hexadecimal representation of the address. + + Raises: + ValueError: The address is bigger than 128 bits of all ones. + + """ + if ip_int is None: + ip_int = int(cls._ip) + + if ip_int > cls._ALL_ONES: + raise ValueError('IPv6 address is too large') + + hex_str = '%032x' % ip_int + hextets = ['%x' % int(hex_str[x:x + 4], 16) for x in range(0, 32, 4)] + + hextets = cls._compress_hextets(hextets) + return ':'.join(hextets) + + def _explode_shorthand_ip_string(self): + """Expand a shortened IPv6 address. + + Args: + ip_str: A string, the IPv6 address. + + Returns: + A string, the expanded IPv6 address. + + """ + if isinstance(self, IPv6Network): + ip_str = _compat_str(self.network_address) + elif isinstance(self, IPv6Interface): + ip_str = _compat_str(self.ip) + else: + ip_str = _compat_str(self) + + ip_int = self._ip_int_from_string(ip_str) + hex_str = '%032x' % ip_int + parts = [hex_str[x:x + 4] for x in range(0, 32, 4)] + if isinstance(self, (_BaseNetwork, IPv6Interface)): + return '%s/%d' % (':'.join(parts), self._prefixlen) + return ':'.join(parts) + + def _reverse_pointer(self): + """Return the reverse DNS pointer name for the IPv6 address. + + This implements the method described in RFC3596 2.5. + + """ + reverse_chars = self.exploded[::-1].replace(':', '') + return '.'.join(reverse_chars) + '.ip6.arpa' + + @property + def max_prefixlen(self): + return self._max_prefixlen + + @property + def version(self): + return self._version + + +class IPv6Address(_BaseV6, _BaseAddress): + + """Represent and manipulate single IPv6 Addresses.""" + + __slots__ = ('_ip', '__weakref__') + + def __init__(self, address): + """Instantiate a new IPv6 address object. + + Args: + address: A string or integer representing the IP + + Additionally, an integer can be passed, so + IPv6Address('2001:db8::') == + IPv6Address(42540766411282592856903984951653826560) + or, more generally + IPv6Address(int(IPv6Address('2001:db8::'))) == + IPv6Address('2001:db8::') + + Raises: + AddressValueError: If address isn't a valid IPv6 address. + + """ + # Efficient constructor from integer. + if isinstance(address, _compat_int_types): + self._check_int_address(address) + self._ip = address + return + + # Constructing from a packed address + if isinstance(address, bytes): + self._check_packed_address(address, 16) + bvs = _compat_bytes_to_byte_vals(address) + self._ip = _compat_int_from_byte_vals(bvs, 'big') + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP string. + addr_str = _compat_str(address) + if '/' in addr_str: + raise AddressValueError("Unexpected '/' in %r" % address) + self._ip = self._ip_int_from_string(addr_str) + + @property + def packed(self): + """The binary representation of this address.""" + return v6_int_to_packed(self._ip) + + @property + def is_multicast(self): + """Test if the address is reserved for multicast use. + + Returns: + A boolean, True if the address is a multicast address. + See RFC 2373 2.7 for details. + + """ + return self in self._constants._multicast_network + + @property + def is_reserved(self): + """Test if the address is otherwise IETF reserved. + + Returns: + A boolean, True if the address is within one of the + reserved IPv6 Network ranges. + + """ + return any(self in x for x in self._constants._reserved_networks) + + @property + def is_link_local(self): + """Test if the address is reserved for link-local. + + Returns: + A boolean, True if the address is reserved per RFC 4291. + + """ + return self in self._constants._linklocal_network + + @property + def is_site_local(self): + """Test if the address is reserved for site-local. + + Note that the site-local address space has been deprecated by RFC 3879. + Use is_private to test if this address is in the space of unique local + addresses as defined by RFC 4193. + + Returns: + A boolean, True if the address is reserved per RFC 3513 2.5.6. + + """ + return self in self._constants._sitelocal_network + + @property + def is_private(self): + """Test if this address is allocated for private networks. + + Returns: + A boolean, True if the address is reserved per + iana-ipv6-special-registry. + + """ + return any(self in net for net in self._constants._private_networks) + + @property + def is_global(self): + """Test if this address is allocated for public networks. + + Returns: + A boolean, true if the address is not reserved per + iana-ipv6-special-registry. + + """ + return not self.is_private + + @property + def is_unspecified(self): + """Test if the address is unspecified. + + Returns: + A boolean, True if this is the unspecified address as defined in + RFC 2373 2.5.2. + + """ + return self._ip == 0 + + @property + def is_loopback(self): + """Test if the address is a loopback address. + + Returns: + A boolean, True if the address is a loopback address as defined in + RFC 2373 2.5.3. + + """ + return self._ip == 1 + + @property + def ipv4_mapped(self): + """Return the IPv4 mapped address. + + Returns: + If the IPv6 address is a v4 mapped address, return the + IPv4 mapped address. Return None otherwise. + + """ + if (self._ip >> 32) != 0xFFFF: + return None + return IPv4Address(self._ip & 0xFFFFFFFF) + + @property + def teredo(self): + """Tuple of embedded teredo IPs. + + Returns: + Tuple of the (server, client) IPs or None if the address + doesn't appear to be a teredo address (doesn't start with + 2001::/32) + + """ + if (self._ip >> 96) != 0x20010000: + return None + return (IPv4Address((self._ip >> 64) & 0xFFFFFFFF), + IPv4Address(~self._ip & 0xFFFFFFFF)) + + @property + def sixtofour(self): + """Return the IPv4 6to4 embedded address. + + Returns: + The IPv4 6to4-embedded address if present or None if the + address doesn't appear to contain a 6to4 embedded address. + + """ + if (self._ip >> 112) != 0x2002: + return None + return IPv4Address((self._ip >> 80) & 0xFFFFFFFF) + + +class IPv6Interface(IPv6Address): + + def __init__(self, address): + if isinstance(address, (bytes, _compat_int_types)): + IPv6Address.__init__(self, address) + self.network = IPv6Network(self._ip) + self._prefixlen = self._max_prefixlen + return + if isinstance(address, tuple): + IPv6Address.__init__(self, address[0]) + if len(address) > 1: + self._prefixlen = int(address[1]) + else: + self._prefixlen = self._max_prefixlen + self.network = IPv6Network(address, strict=False) + self.netmask = self.network.netmask + self.hostmask = self.network.hostmask + return + + addr = _split_optional_netmask(address) + IPv6Address.__init__(self, addr[0]) + self.network = IPv6Network(address, strict=False) + self.netmask = self.network.netmask + self._prefixlen = self.network._prefixlen + self.hostmask = self.network.hostmask + + def __str__(self): + return '%s/%d' % (self._string_from_ip_int(self._ip), + self.network.prefixlen) + + def __eq__(self, other): + address_equal = IPv6Address.__eq__(self, other) + if not address_equal or address_equal is NotImplemented: + return address_equal + try: + return self.network == other.network + except AttributeError: + # An interface with an associated network is NOT the + # same as an unassociated address. That's why the hash + # takes the extra info into account. + return False + + def __lt__(self, other): + address_less = IPv6Address.__lt__(self, other) + if address_less is NotImplemented: + return NotImplemented + try: + return (self.network < other.network or + self.network == other.network and address_less) + except AttributeError: + # We *do* allow addresses and interfaces to be sorted. The + # unassociated address is considered less than all interfaces. + return False + + def __hash__(self): + return self._ip ^ self._prefixlen ^ int(self.network.network_address) + + __reduce__ = _IPAddressBase.__reduce__ + + @property + def ip(self): + return IPv6Address(self._ip) + + @property + def with_prefixlen(self): + return '%s/%s' % (self._string_from_ip_int(self._ip), + self._prefixlen) + + @property + def with_netmask(self): + return '%s/%s' % (self._string_from_ip_int(self._ip), + self.netmask) + + @property + def with_hostmask(self): + return '%s/%s' % (self._string_from_ip_int(self._ip), + self.hostmask) + + @property + def is_unspecified(self): + return self._ip == 0 and self.network.is_unspecified + + @property + def is_loopback(self): + return self._ip == 1 and self.network.is_loopback + + +class IPv6Network(_BaseV6, _BaseNetwork): + + """This class represents and manipulates 128-bit IPv6 networks. + + Attributes: [examples for IPv6('2001:db8::1000/124')] + .network_address: IPv6Address('2001:db8::1000') + .hostmask: IPv6Address('::f') + .broadcast_address: IPv6Address('2001:db8::100f') + .netmask: IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff:ffff:fff0') + .prefixlen: 124 + + """ + + # Class to use when creating address objects + _address_class = IPv6Address + + def __init__(self, address, strict=True): + """Instantiate a new IPv6 Network object. + + Args: + address: A string or integer representing the IPv6 network or the + IP and prefix/netmask. + '2001:db8::/128' + '2001:db8:0000:0000:0000:0000:0000:0000/128' + '2001:db8::' + are all functionally the same in IPv6. That is to say, + failing to provide a subnetmask will create an object with + a mask of /128. + + Additionally, an integer can be passed, so + IPv6Network('2001:db8::') == + IPv6Network(42540766411282592856903984951653826560) + or, more generally + IPv6Network(int(IPv6Network('2001:db8::'))) == + IPv6Network('2001:db8::') + + strict: A boolean. If true, ensure that we have been passed + A true network address, eg, 2001:db8::1000/124 and not an + IP address on a network, eg, 2001:db8::1/124. + + Raises: + AddressValueError: If address isn't a valid IPv6 address. + NetmaskValueError: If the netmask isn't valid for + an IPv6 address. + ValueError: If strict was True and a network address was not + supplied. + + """ + _BaseNetwork.__init__(self, address) + + # Efficient constructor from integer or packed address + if isinstance(address, (bytes, _compat_int_types)): + self.network_address = IPv6Address(address) + self.netmask, self._prefixlen = self._make_netmask( + self._max_prefixlen) + return + + if isinstance(address, tuple): + if len(address) > 1: + arg = address[1] + else: + arg = self._max_prefixlen + self.netmask, self._prefixlen = self._make_netmask(arg) + self.network_address = IPv6Address(address[0]) + packed = int(self.network_address) + if packed & int(self.netmask) != packed: + if strict: + raise ValueError('%s has host bits set' % self) + else: + self.network_address = IPv6Address(packed & + int(self.netmask)) + return + + # Assume input argument to be string or any object representation + # which converts into a formatted IP prefix string. + addr = _split_optional_netmask(address) + + self.network_address = IPv6Address(self._ip_int_from_string(addr[0])) + + if len(addr) == 2: + arg = addr[1] + else: + arg = self._max_prefixlen + self.netmask, self._prefixlen = self._make_netmask(arg) + + if strict: + if (IPv6Address(int(self.network_address) & int(self.netmask)) != + self.network_address): + raise ValueError('%s has host bits set' % self) + self.network_address = IPv6Address(int(self.network_address) & + int(self.netmask)) + + if self._prefixlen == (self._max_prefixlen - 1): + self.hosts = self.__iter__ + + def hosts(self): + """Generate Iterator over usable hosts in a network. + + This is like __iter__ except it doesn't return the + Subnet-Router anycast address. + + """ + network = int(self.network_address) + broadcast = int(self.broadcast_address) + for x in _compat_range(network + 1, broadcast + 1): + yield self._address_class(x) + + @property + def is_site_local(self): + """Test if the address is reserved for site-local. + + Note that the site-local address space has been deprecated by RFC 3879. + Use is_private to test if this address is in the space of unique local + addresses as defined by RFC 4193. + + Returns: + A boolean, True if the address is reserved per RFC 3513 2.5.6. + + """ + return (self.network_address.is_site_local and + self.broadcast_address.is_site_local) + + +class _IPv6Constants(object): + + _linklocal_network = IPv6Network('fe80::/10') + + _multicast_network = IPv6Network('ff00::/8') + + _private_networks = [ + IPv6Network('::1/128'), + IPv6Network('::/128'), + IPv6Network('::ffff:0:0/96'), + IPv6Network('100::/64'), + IPv6Network('2001::/23'), + IPv6Network('2001:2::/48'), + IPv6Network('2001:db8::/32'), + IPv6Network('2001:10::/28'), + IPv6Network('fc00::/7'), + IPv6Network('fe80::/10'), + ] + + _reserved_networks = [ + IPv6Network('::/8'), IPv6Network('100::/8'), + IPv6Network('200::/7'), IPv6Network('400::/6'), + IPv6Network('800::/5'), IPv6Network('1000::/4'), + IPv6Network('4000::/3'), IPv6Network('6000::/3'), + IPv6Network('8000::/3'), IPv6Network('A000::/3'), + IPv6Network('C000::/3'), IPv6Network('E000::/4'), + IPv6Network('F000::/5'), IPv6Network('F800::/6'), + IPv6Network('FE00::/9'), + ] + + _sitelocal_network = IPv6Network('fec0::/10') + + +IPv6Address._constants = _IPv6Constants diff --git a/readthedocs/api/base.py b/readthedocs/api/base.py index a70fb024599..73b64e431cf 100644 --- a/readthedocs/api/base.py +++ b/readthedocs/api/base.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- - """API resources.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging +from builtins import object import redis from django.conf.urls import url @@ -22,7 +25,6 @@ from .utils import PostAuthentication - log = logging.getLogger(__name__) @@ -32,7 +34,7 @@ class ProjectResource(ModelResource): users = fields.ToManyField('readthedocs.api.base.UserResource', 'users') - class Meta: + class Meta(object): include_absolute_url = True allowed_methods = ['get', 'post', 'put'] queryset = Project.objects.api() @@ -46,7 +48,7 @@ class Meta: def get_object_list(self, request): self._meta.queryset = Project.objects.api(user=request.user) - return super().get_object_list(request) + return super(ProjectResource, self).get_object_list(request) def dehydrate(self, bundle): bundle.data['downloads'] = bundle.obj.get_downloads() @@ -70,9 +72,7 @@ def post_list(self, request, **kwargs): # Force this in an ugly way, at least should do "reverse" deserialized['users'] = ['/api/v1/user/%s/' % request.user.id] bundle = self.build_bundle( - data=dict_strip_unicode_keys(deserialized), - request=request, - ) + data=dict_strip_unicode_keys(deserialized), request=request) self.is_valid(bundle) updated_bundle = self.obj_create(bundle, request=request) return HttpCreated(location=self.get_resource_uri(updated_bundle)) @@ -81,20 +81,14 @@ def prepend_urls(self): return [ url( r'^(?P%s)/schema/$' % self._meta.resource_name, - self.wrap_view('get_schema'), - name='api_get_schema', - ), + self.wrap_view('get_schema'), name='api_get_schema'), url( r'^(?P%s)/search%s$' % (self._meta.resource_name, trailing_slash()), - self.wrap_view('get_search'), - name='api_get_search', - ), - url( - (r'^(?P%s)/(?P[a-z-_]+)/$') % self._meta.resource_name, - self.wrap_view('dispatch_detail'), - name='api_dispatch_detail', - ), + self.wrap_view('get_search'), name='api_get_search'), + url((r'^(?P%s)/(?P[a-z-_]+)/$') % + self._meta.resource_name, self.wrap_view('dispatch_detail'), + name='api_dispatch_detail'), ] @@ -104,7 +98,7 @@ class VersionResource(ModelResource): project = fields.ForeignKey(ProjectResource, 'project', full=True) - class Meta: + class Meta(object): allowed_methods = ['get', 'put', 'post'] always_return_data = True queryset = Version.objects.api() @@ -118,7 +112,7 @@ class Meta: def get_object_list(self, request): self._meta.queryset = Version.objects.api(user=request.user) - return super().get_object_list(request) + return super(VersionResource, self).get_object_list(request) def build_version(self, request, **kwargs): project = get_object_or_404(Project, slug=kwargs['project_slug']) @@ -131,23 +125,17 @@ def prepend_urls(self): return [ url( r'^(?P%s)/schema/$' % self._meta.resource_name, - self.wrap_view('get_schema'), - name='api_get_schema', - ), + self.wrap_view('get_schema'), name='api_get_schema'), url( r'^(?P%s)/(?P[a-z-_]+[a-z0-9-_]+)/$' # noqa % self._meta.resource_name, self.wrap_view('dispatch_list'), - name='api_version_list', - ), - url( - ( - r'^(?P%s)/(?P[a-z-_]+[a-z0-9-_]+)/(?P' - r'[a-z0-9-_.]+)/build/$' - ) % self._meta.resource_name, - self.wrap_view('build_version'), - name='api_version_build_slug', - ), + name='api_version_list'), + url(( + r'^(?P%s)/(?P[a-z-_]+[a-z0-9-_]+)/(?P' + r'[a-z0-9-_.]+)/build/$') % + self._meta.resource_name, self.wrap_view('build_version'), + name='api_version_build_slug'), ] @@ -157,7 +145,7 @@ class FileResource(ModelResource): project = fields.ForeignKey(ProjectResource, 'project', full=True) - class Meta: + class Meta(object): allowed_methods = ['get', 'post'] queryset = ImportedFile.objects.all() excludes = ['md5', 'slug'] @@ -169,15 +157,11 @@ def prepend_urls(self): return [ url( r'^(?P%s)/schema/$' % self._meta.resource_name, - self.wrap_view('get_schema'), - name='api_get_schema', - ), + self.wrap_view('get_schema'), name='api_get_schema'), url( r'^(?P%s)/anchor%s$' % (self._meta.resource_name, trailing_slash()), - self.wrap_view('get_anchor'), - name='api_get_anchor', - ), + self.wrap_view('get_anchor'), name='api_get_anchor'), ] def get_anchor(self, request, **__): @@ -206,7 +190,7 @@ class UserResource(ModelResource): """Read-only API resource for User model.""" - class Meta: + class Meta(object): allowed_methods = ['get'] queryset = User.objects.all() fields = ['username', 'id'] @@ -218,12 +202,9 @@ def prepend_urls(self): return [ url( r'^(?P%s)/schema/$' % self._meta.resource_name, - self.wrap_view('get_schema'), - name='api_get_schema', - ), + self.wrap_view('get_schema'), name='api_get_schema'), url( - r'^(?P%s)/(?P[a-z-_]+)/$' % self._meta.resource_name, - self.wrap_view('dispatch_detail'), - name='api_dispatch_detail', - ), + r'^(?P%s)/(?P[a-z-_]+)/$' % + self._meta.resource_name, self.wrap_view('dispatch_detail'), + name='api_dispatch_detail'), ] diff --git a/readthedocs/api/client.py b/readthedocs/api/client.py index a3020163449..dd742198b15 100644 --- a/readthedocs/api/client.py +++ b/readthedocs/api/client.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- """Slumber API client.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging -import requests from django.conf import settings +import requests from requests_toolbelt.adapters import host_header_ssl from slumber import API - log = logging.getLogger(__name__) PRODUCTION_DOMAIN = getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org') diff --git a/readthedocs/api/utils.py b/readthedocs/api/utils.py index c5ecd60cd1e..1daa2deb963 100644 --- a/readthedocs/api/utils.py +++ b/readthedocs/api/utils.py @@ -1,13 +1,13 @@ -# -*- coding: utf-8 -*- - -"""Utility classes for api module.""" +"""Utility classes for api module""" +from __future__ import absolute_import import logging from django.utils.translation import ugettext + from tastypie.authentication import BasicAuthentication from tastypie.authorization import Authorization -from tastypie.exceptions import NotFound from tastypie.resources import ModelResource +from tastypie.exceptions import NotFound log = logging.getLogger(__name__) @@ -18,14 +18,14 @@ class PostAuthentication(BasicAuthentication): """Require HTTP Basic authentication for any method other than GET.""" def is_authenticated(self, request, **kwargs): - val = super().is_authenticated(request, **kwargs) - if request.method == 'GET': + val = super(PostAuthentication, self).is_authenticated(request, + **kwargs) + if request.method == "GET": return True return val class EnhancedModelResource(ModelResource): - def obj_get_list(self, request=None, *_, **kwargs): # noqa """ A ORM-specific implementation of ``obj_get_list``. @@ -44,16 +44,12 @@ def obj_get_list(self, request=None, *_, **kwargs): # noqa try: return self.get_object_list(request).filter(**applicable_filters) except ValueError as e: - raise NotFound( - ugettext( - 'Invalid resource lookup data provided ' - '(mismatched type).: %(error)s', - ) % {'error': e}, - ) + raise NotFound(ugettext("Invalid resource lookup data provided " + "(mismatched type).: %(error)s") + % {'error': e}) class OwnerAuthorization(Authorization): - def apply_limits(self, request, object_list): if request and hasattr(request, 'user') and request.method != 'GET': if request.user.is_authenticated: diff --git a/readthedocs/builds/admin.py b/readthedocs/builds/admin.py index 571536f03a2..66c046f9c3e 100644 --- a/readthedocs/builds/admin.py +++ b/readthedocs/builds/admin.py @@ -1,12 +1,10 @@ -# -*- coding: utf-8 -*- - """Django admin interface for `~builds.models.Build` and related models.""" +from __future__ import absolute_import from django.contrib import admin +from readthedocs.builds.models import Build, Version, BuildCommandResult from guardian.admin import GuardedModelAdmin -from readthedocs.builds.models import Build, BuildCommandResult, Version - class BuildCommandResultInline(admin.TabularInline): model = BuildCommandResult @@ -14,25 +12,8 @@ class BuildCommandResultInline(admin.TabularInline): class BuildAdmin(admin.ModelAdmin): - fields = ( - 'project', - 'version', - 'type', - 'state', - 'error', - 'success', - 'length', - 'cold_storage', - ) - list_display = ( - 'id', - 'project', - 'version_name', - 'success', - 'type', - 'state', - 'date', - ) + fields = ('project', 'version', 'type', 'state', 'error', 'success', 'length', 'cold_storage') + list_display = ('id', 'project', 'version_name', 'success', 'type', 'state', 'date') list_filter = ('type', 'state', 'success') list_select_related = ('project', 'version') raw_id_fields = ('project', 'version') @@ -45,14 +26,7 @@ def version_name(self, obj): class VersionAdmin(GuardedModelAdmin): search_fields = ('slug', 'project__name') - list_display = ( - 'slug', - 'type', - 'project', - 'privacy_level', - 'active', - 'built', - ) + list_display = ('slug', 'type', 'project', 'privacy_level', 'active', 'built') list_filter = ('type', 'privacy_level', 'active', 'built') raw_id_fields = ('project',) diff --git a/readthedocs/builds/constants.py b/readthedocs/builds/constants.py index 94858300c78..99d816814ba 100644 --- a/readthedocs/builds/constants.py +++ b/readthedocs/builds/constants.py @@ -1,10 +1,8 @@ -# -*- coding: utf-8 -*- - """Constants for the builds app.""" -from django.conf import settings +from __future__ import absolute_import from django.utils.translation import ugettext_lazy as _ - +from django.conf import settings BUILD_STATE_TRIGGERED = 'triggered' BUILD_STATE_CLONING = 'cloning' diff --git a/readthedocs/builds/forms.py b/readthedocs/builds/forms.py index 406da7417ab..74fe0dba504 100644 --- a/readthedocs/builds/forms.py +++ b/readthedocs/builds/forms.py @@ -1,7 +1,13 @@ -# -*- coding: utf-8 -*- - """Django forms for the builds app.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from builtins import object from django import forms from django.utils.translation import ugettext_lazy as _ @@ -11,7 +17,7 @@ class VersionForm(forms.ModelForm): - class Meta: + class Meta(object): model = Version fields = ['active', 'privacy_level', 'tags'] @@ -20,10 +26,10 @@ def clean_active(self): if self._is_default_version() and not active: msg = _( '{version} is the default version of the project, ' - 'it should be active.', + 'it should be active.' ) raise forms.ValidationError( - msg.format(version=self.instance.verbose_name), + msg.format(version=self.instance.verbose_name) ) return active @@ -32,7 +38,7 @@ def _is_default_version(self): return project.default_version == self.instance.slug def save(self, commit=True): - obj = super().save(commit=commit) + obj = super(VersionForm, self).save(commit=commit) if obj.active and not obj.built and not obj.uploaded: trigger_build(project=obj.project, version=obj) return obj diff --git a/readthedocs/builds/managers.py b/readthedocs/builds/managers.py index be9c7050ea5..9ef1b2836e5 100644 --- a/readthedocs/builds/managers.py +++ b/readthedocs/builds/managers.py @@ -1,23 +1,14 @@ -# -*- coding: utf-8 -*- +"""Build and Version class model Managers""" -"""Build and Version class model Managers.""" +from __future__ import absolute_import from django.db import models -from readthedocs.core.utils.extend import ( - SettingsOverrideObject, - get_override_class, -) - -from .constants import ( - BRANCH, - LATEST, - LATEST_VERBOSE_NAME, - STABLE, - STABLE_VERBOSE_NAME, - TAG, -) +from .constants import (BRANCH, TAG, LATEST, LATEST_VERBOSE_NAME, STABLE, + STABLE_VERBOSE_NAME) from .querysets import VersionQuerySet +from readthedocs.core.utils.extend import (SettingsOverrideObject, + get_override_class) __all__ = ['VersionManager'] @@ -39,9 +30,9 @@ def from_queryset(cls, queryset_class, class_name=None): # no direct members. queryset_class = get_override_class( VersionQuerySet, - VersionQuerySet._default_class, # pylint: disable=protected-access + VersionQuerySet._default_class # pylint: disable=protected-access ) - return super().from_queryset(queryset_class, class_name) + return super(VersionManagerBase, cls).from_queryset(queryset_class, class_name) def create_stable(self, **kwargs): defaults = { diff --git a/readthedocs/builds/migrations/0001_initial.py b/readthedocs/builds/migrations/0001_initial.py index 27f61efb797..32e6e0eb1fd 100644 --- a/readthedocs/builds/migrations/0001_initial.py +++ b/readthedocs/builds/migrations/0001_initial.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- -import taggit.managers -from django.db import migrations, models +from __future__ import unicode_literals +from __future__ import absolute_import +from django.db import models, migrations import readthedocs.builds.version_slug +import taggit.managers class Migration(migrations.Migration): @@ -75,10 +77,10 @@ class Migration(migrations.Migration): ), migrations.AlterUniqueTogether( name='version', - unique_together={('project', 'slug')}, + unique_together=set([('project', 'slug')]), ), migrations.AlterIndexTogether( name='build', - index_together={('version', 'state', 'type')}, + index_together=set([('version', 'state', 'type')]), ), ] diff --git a/readthedocs/builds/migrations/0002_build_command_initial.py b/readthedocs/builds/migrations/0002_build_command_initial.py index d78b9d13d2f..7b45f946830 100644 --- a/readthedocs/builds/migrations/0002_build_command_initial.py +++ b/readthedocs/builds/migrations/0002_build_command_initial.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals +from __future__ import absolute_import +from django.db import models, migrations import readthedocs.builds.models diff --git a/readthedocs/builds/migrations/0003_add-cold-storage.py b/readthedocs/builds/migrations/0003_add-cold-storage.py index 2c53cc144dd..7c474a973c0 100644 --- a/readthedocs/builds/migrations/0003_add-cold-storage.py +++ b/readthedocs/builds/migrations/0003_add-cold-storage.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-10-09 20:14 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/builds/migrations/0004_add-apiversion-proxy-model.py b/readthedocs/builds/migrations/0004_add-apiversion-proxy-model.py index 8247d9a0249..b96db28e95a 100644 --- a/readthedocs/builds/migrations/0004_add-apiversion-proxy-model.py +++ b/readthedocs/builds/migrations/0004_add-apiversion-proxy-model.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-10-27 00:17 +from __future__ import unicode_literals + from django.db import migrations diff --git a/readthedocs/builds/migrations/0005_remove-version-alias.py b/readthedocs/builds/migrations/0005_remove-version-alias.py index 65f6aadd3a5..a41af51e2df 100644 --- a/readthedocs/builds/migrations/0005_remove-version-alias.py +++ b/readthedocs/builds/migrations/0005_remove-version-alias.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-10-17 04:20 -from django.db import migrations, models +from __future__ import unicode_literals +from django.db import migrations, models import readthedocs.builds.version_slug diff --git a/readthedocs/builds/migrations/0006_add_config_field.py b/readthedocs/builds/migrations/0006_add_config_field.py index deb3700278c..7af36e8ad79 100644 --- a/readthedocs/builds/migrations/0006_add_config_field.py +++ b/readthedocs/builds/migrations/0006_add_config_field.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-02 13:24 -import jsonfield.fields +from __future__ import unicode_literals + from django.db import migrations +import jsonfield.fields class Migration(migrations.Migration): diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index e44d426b381..93678975232 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -1,15 +1,21 @@ # -*- coding: utf-8 -*- - """Models for the builds app.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import os.path import re from shutil import rmtree +from builtins import object from django.conf import settings from django.db import models -from django.urls import reverse from django.utils import timezone from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext @@ -17,6 +23,7 @@ from guardian.shortcuts import assign from jsonfield import JSONField from taggit.managers import TaggableManager +from django.urls import reverse from readthedocs.core.utils import broadcast from readthedocs.projects.constants import ( @@ -48,12 +55,8 @@ ) from .version_slug import VersionSlugField - DEFAULT_VERSION_PRIVACY_LEVEL = getattr( - settings, - 'DEFAULT_VERSION_PRIVACY_LEVEL', - 'public', -) + settings, 'DEFAULT_VERSION_PRIVACY_LEVEL', 'public') log = logging.getLogger(__name__) @@ -93,10 +96,7 @@ class Version(models.Model): #: filesystem to determine how the paths for this version are called. It #: must not be used for any other identifying purposes. slug = VersionSlugField( - _('Slug'), - max_length=255, - populate_from='verbose_name', - ) + _('Slug'), max_length=255, populate_from='verbose_name') supported = models.BooleanField(_('Supported'), default=True) active = models.BooleanField(_('Active'), default=False) @@ -114,14 +114,13 @@ class Version(models.Model): objects = VersionManager.from_queryset(VersionQuerySet)() - class Meta: + class Meta(object): unique_together = [('project', 'slug')] ordering = ['-verbose_name'] permissions = ( # Translators: Permission around whether a user can view the # version - ('view_version', _('View Version')), - ) + ('view_version', _('View Version')),) def __str__(self): return ugettext( @@ -129,8 +128,7 @@ def __str__(self): version=self.verbose_name, project=self.project, pk=self.pk, - ), - ) + )) @property def config(self): @@ -141,10 +139,9 @@ def config(self): :rtype: dict """ last_build = ( - self.builds.filter( - state='finished', - success=True, - ).order_by('-date').first() + self.builds.filter(state='finished', success=True) + .order_by('-date') + .first() ) return last_build.config @@ -187,9 +184,7 @@ def commit_name(self): # If we came that far it's not a special version nor a branch or tag. # Therefore just return the identifier to make a safe guess. - log.debug( - 'TODO: Raise an exception here. Testing what cases it happens', - ) + log.debug('TODO: Raise an exception here. Testing what cases it happens') return self.identifier def get_absolute_url(self): @@ -203,38 +198,25 @@ def get_absolute_url(self): ) private = self.privacy_level == PRIVATE return self.project.get_docs_url( - version_slug=self.slug, - private=private, - ) + version_slug=self.slug, private=private) def save(self, *args, **kwargs): # pylint: disable=arguments-differ """Add permissions to the Version for all owners on save.""" from readthedocs.projects import tasks - obj = super().save(*args, **kwargs) + obj = super(Version, self).save(*args, **kwargs) for owner in self.project.users.all(): assign('view_version', owner, self) broadcast( - type='app', - task=tasks.symlink_project, - args=[self.project.pk], - ) + type='app', task=tasks.symlink_project, args=[self.project.pk]) return obj def delete(self, *args, **kwargs): # pylint: disable=arguments-differ from readthedocs.projects import tasks log.info('Removing files for version %s', self.slug) + broadcast(type='app', task=tasks.clear_artifacts, args=[self.get_artifact_paths()]) broadcast( - type='app', - task=tasks.remove_dirs, - args=[self.get_artifact_paths()], - ) - project_pk = self.project.pk - super().delete(*args, **kwargs) - broadcast( - type='app', - task=tasks.symlink_project, - args=[project_pk], - ) + type='app', task=tasks.symlink_project, args=[self.project.pk]) + super(Version, self).delete(*args, **kwargs) @property def identifier_friendly(self): @@ -263,27 +245,19 @@ def get_downloads(self, pretty=False): data['PDF'] = project.get_production_media_url('pdf', self.slug) if project.has_htmlzip(self.slug): data['HTML'] = project.get_production_media_url( - 'htmlzip', - self.slug, - ) + 'htmlzip', self.slug) if project.has_epub(self.slug): data['Epub'] = project.get_production_media_url( - 'epub', - self.slug, - ) + 'epub', self.slug) else: if project.has_pdf(self.slug): data['pdf'] = project.get_production_media_url('pdf', self.slug) if project.has_htmlzip(self.slug): data['htmlzip'] = project.get_production_media_url( - 'htmlzip', - self.slug, - ) + 'htmlzip', self.slug) if project.has_epub(self.slug): data['epub'] = project.get_production_media_url( - 'epub', - self.slug, - ) + 'epub', self.slug) return data def get_conf_py_path(self): @@ -309,8 +283,9 @@ def get_artifact_paths(self): for type_ in ('pdf', 'epub', 'htmlzip'): paths.append( - self.project - .get_production_media_path(type_=type_, version_slug=self.slug), + self.project.get_production_media_path( + type_=type_, + version_slug=self.slug), ) paths.append(self.project.rtd_build_path(version=self.slug)) @@ -332,12 +307,7 @@ def clean_build_path(self): log.exception('Build path cleanup failed') def get_github_url( - self, - docroot, - filename, - source_suffix='.rst', - action='view', - ): + self, docroot, filename, source_suffix='.rst', action='view'): """ Return a GitHub URL for a given filename. @@ -379,12 +349,7 @@ def get_github_url( ) def get_gitlab_url( - self, - docroot, - filename, - source_suffix='.rst', - action='view', - ): + self, docroot, filename, source_suffix='.rst', action='view'): repo_url = self.project.repo if 'gitlab' not in repo_url: return '' @@ -469,7 +434,7 @@ def __init__(self, *args, **kwargs): del kwargs[key] except KeyError: pass - super().__init__(*args, **kwargs) + super(APIVersion, self).__init__(*args, **kwargs) def save(self, *args, **kwargs): return 0 @@ -481,28 +446,13 @@ class Build(models.Model): """Build data.""" project = models.ForeignKey( - Project, - verbose_name=_('Project'), - related_name='builds', - ) + Project, verbose_name=_('Project'), related_name='builds') version = models.ForeignKey( - Version, - verbose_name=_('Version'), - null=True, - related_name='builds', - ) + Version, verbose_name=_('Version'), null=True, related_name='builds') type = models.CharField( - _('Type'), - max_length=55, - choices=BUILD_TYPES, - default='html', - ) + _('Type'), max_length=55, choices=BUILD_TYPES, default='html') state = models.CharField( - _('State'), - max_length=55, - choices=BUILD_STATE, - default='finished', - ) + _('State'), max_length=55, choices=BUILD_STATE, default='finished') date = models.DateTimeField(_('Date'), auto_now_add=True) success = models.BooleanField(_('Success'), default=True) @@ -512,26 +462,16 @@ class Build(models.Model): error = models.TextField(_('Error'), default='', blank=True) exit_code = models.IntegerField(_('Exit code'), null=True, blank=True) commit = models.CharField( - _('Commit'), - max_length=255, - null=True, - blank=True, - ) + _('Commit'), max_length=255, null=True, blank=True) _config = JSONField(_('Configuration used in the build'), default=dict) length = models.IntegerField(_('Build Length'), null=True, blank=True) builder = models.CharField( - _('Builder'), - max_length=255, - null=True, - blank=True, - ) + _('Builder'), max_length=255, null=True, blank=True) cold_storage = models.NullBooleanField( - _('Cold Storage'), - help_text='Build steps stored outside the database.', - ) + _('Cold Storage'), help_text='Build steps stored outside the database.') # Manager @@ -539,13 +479,13 @@ class Build(models.Model): CONFIG_KEY = '__config' - class Meta: + class Meta(object): ordering = ['-date'] get_latest_by = 'date' index_together = [['version', 'state', 'type']] def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(Build, self).__init__(*args, **kwargs) self._config_changed = False @property @@ -558,11 +498,14 @@ def previous(self): date = self.date or timezone.now() if self.project is not None and self.version is not None: return ( - Build.objects.filter( + Build.objects + .filter( project=self.project, version=self.version, date__lt=date, - ).order_by('-date').first() + ) + .order_by('-date') + .first() ) return None @@ -572,9 +515,9 @@ def config(self): Get the config used for this build. Since we are saving the config into the JSON field only when it differs - from the previous one, this helper returns the correct JSON used in this - Build object (it could be stored in this object or one of the previous - ones). + from the previous one, this helper returns the correct JSON used in + this Build object (it could be stored in this object or one of the + previous ones). """ if self.CONFIG_KEY in self._config: return Build.objects.get(pk=self._config[self.CONFIG_KEY])._config @@ -602,13 +545,11 @@ def save(self, *args, **kwargs): # noqa """ if self.pk is None or self._config_changed: previous = self.previous - if ( - previous is not None and self._config and - self._config == previous.config - ): + if (previous is not None and + self._config and self._config == previous.config): previous_pk = previous._config.get(self.CONFIG_KEY, previous.pk) self._config = {self.CONFIG_KEY: previous_pk} - super().save(*args, **kwargs) + super(Build, self).save(*args, **kwargs) self._config_changed = False def __str__(self): @@ -619,8 +560,7 @@ def __str__(self): self.project.users.all().values_list('username', flat=True), ), pk=self.pk, - ), - ) + )) def get_absolute_url(self): return reverse('builds_detail', args=[self.project.slug, self.pk]) @@ -631,7 +571,7 @@ def finished(self): return self.state == BUILD_STATE_FINISHED -class BuildCommandResultMixin: +class BuildCommandResultMixin(object): """ Mixin for common command result methods/properties. @@ -661,10 +601,7 @@ class BuildCommandResult(BuildCommandResultMixin, models.Model): """Build command for a ``Build``.""" build = models.ForeignKey( - Build, - verbose_name=_('Build'), - related_name='commands', - ) + Build, verbose_name=_('Build'), related_name='commands') command = models.TextField(_('Command')) description = models.TextField(_('Description'), blank=True) @@ -674,7 +611,7 @@ class BuildCommandResult(BuildCommandResultMixin, models.Model): start_time = models.DateTimeField(_('Start time')) end_time = models.DateTimeField(_('End time')) - class Meta: + class Meta(object): ordering = ['start_time'] get_latest_by = 'start_time' @@ -683,8 +620,7 @@ class Meta: def __str__(self): return ( ugettext('Build command {pk} for build {build}') - .format(pk=self.pk, build=self.build) - ) + .format(pk=self.pk, build=self.build)) @property def run_time(self): diff --git a/readthedocs/builds/querysets.py b/readthedocs/builds/querysets.py index ae7e5a8fb79..34408b9d982 100644 --- a/readthedocs/builds/querysets.py +++ b/readthedocs/builds/querysets.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- +"""Build and Version QuerySet classes""" -"""Build and Version QuerySet classes.""" +from __future__ import absolute_import from django.db import models from guardian.shortcuts import get_objects_for_user @@ -37,9 +37,7 @@ def public(self, user=None, project=None, only_active=True): return queryset def protected(self, user=None, project=None, only_active=True): - queryset = self.filter( - privacy_level__in=[constants.PUBLIC, constants.PROTECTED], - ) + queryset = self.filter(privacy_level__in=[constants.PUBLIC, constants.PROTECTED]) if user: queryset = self._add_user_repos(queryset, user) if project: @@ -62,10 +60,10 @@ def api(self, user=None): return self.public(user, only_active=False) def for_project(self, project): - """Return all versions for a project, including translations.""" + """Return all versions for a project, including translations""" return self.filter( models.Q(project=project) | - models.Q(project__main_language_project=project), + models.Q(project__main_language_project=project) ) @@ -121,7 +119,8 @@ def _add_user_repos(self, queryset, user=None): if user.is_authenticated: user_queryset = get_objects_for_user(user, 'builds.view_version') pks = user_queryset.values_list('pk', flat=True) - queryset = self.filter(build__version__pk__in=pks,) | queryset + queryset = self.filter( + build__version__pk__in=pks) | queryset return queryset.distinct() def public(self, user=None, project=None): diff --git a/readthedocs/builds/signals.py b/readthedocs/builds/signals.py index df87930f9b7..c80f7d3e42d 100644 --- a/readthedocs/builds/signals.py +++ b/readthedocs/builds/signals.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Build signals.""" +"""Build signals""" +from __future__ import absolute_import import django.dispatch diff --git a/readthedocs/builds/syncers.py b/readthedocs/builds/syncers.py index 6b7b1d337bf..0ac0c6dedf6 100644 --- a/readthedocs/builds/syncers.py +++ b/readthedocs/builds/syncers.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Classes to copy files between build and web servers. @@ -7,23 +5,26 @@ local machine. """ +from __future__ import absolute_import + import getpass import logging import os import shutil +from builtins import object from django.conf import settings -from readthedocs.core.utils import safe_makedirs from readthedocs.core.utils.extend import SettingsOverrideObject +from readthedocs.core.utils import safe_makedirs log = logging.getLogger(__name__) -class BaseSyncer: +class BaseSyncer(object): - """A base object for syncers and pullers.""" + """A base object for syncers and pullers""" @classmethod def copy(cls, path, target, is_file=False, **kwargs): @@ -35,7 +36,7 @@ class LocalSyncer(BaseSyncer): @classmethod def copy(cls, path, target, is_file=False, **kwargs): """A copy command that works with files or directories.""" - log.info('Local Copy %s to %s', path, target) + log.info("Local Copy %s to %s", path, target) if is_file: if path == target: # Don't copy the same file over itself @@ -61,31 +62,28 @@ def copy(cls, path, target, is_file=False, **kwargs): sync_user = getattr(settings, 'SYNC_USER', getpass.getuser()) app_servers = getattr(settings, 'MULTIPLE_APP_SERVERS', []) if app_servers: - log.info('Remote Copy %s to %s on %s', path, target, app_servers) + log.info("Remote Copy %s to %s on %s", path, target, app_servers) for server in app_servers: - mkdir_cmd = ( - 'ssh {}@{} mkdir -p {}'.format(sync_user, server, target) - ) + mkdir_cmd = ("ssh %s@%s mkdir -p %s" % (sync_user, server, target)) ret = os.system(mkdir_cmd) if ret != 0: - log.debug('Copy error to app servers: cmd=%s', mkdir_cmd) + log.debug("Copy error to app servers: cmd=%s", mkdir_cmd) if is_file: - slash = '' + slash = "" else: - slash = '/' + slash = "/" # Add a slash when copying directories sync_cmd = ( - "rsync -e 'ssh -T' -av --delete {path}{slash} {user}@{server}:{target}".format( + "rsync -e 'ssh -T' -av --delete {path}{slash} {user}@{server}:{target}" + .format( path=path, slash=slash, user=sync_user, server=server, - target=target, - ) - ) + target=target)) ret = os.system(sync_cmd) if ret != 0: - log.debug('Copy error to app servers: cmd=%s', sync_cmd) + log.debug("Copy error to app servers: cmd=%s", sync_cmd) class DoubleRemotePuller(BaseSyncer): @@ -100,32 +98,29 @@ def copy(cls, path, target, host, is_file=False, **kwargs): # pylint: disable=a sync_user = getattr(settings, 'SYNC_USER', getpass.getuser()) app_servers = getattr(settings, 'MULTIPLE_APP_SERVERS', []) if not is_file: - path += '/' - log.info('Remote Copy %s to %s', path, target) + path += "/" + log.info("Remote Copy %s to %s", path, target) for server in app_servers: if not is_file: - mkdir_cmd = 'ssh {user}@{server} mkdir -p {target}'.format( - user=sync_user, - server=server, - target=target, + mkdir_cmd = "ssh {user}@{server} mkdir -p {target}".format( + user=sync_user, server=server, target=target ) ret = os.system(mkdir_cmd) if ret != 0: - log.debug('MkDir error to app servers: cmd=%s', mkdir_cmd) + log.debug("MkDir error to app servers: cmd=%s", mkdir_cmd) # Add a slash when copying directories sync_cmd = ( "ssh {user}@{server} 'rsync -av " - "--delete --exclude projects {user}@{host}:{path} {target}'".format( + "--delete --exclude projects {user}@{host}:{path} {target}'" + .format( host=host, path=path, user=sync_user, server=server, - target=target, - ) - ) + target=target)) ret = os.system(sync_cmd) if ret != 0: - log.debug('Copy error to app servers: cmd=%s', sync_cmd) + log.debug("Copy error to app servers: cmd=%s", sync_cmd) class RemotePuller(BaseSyncer): @@ -139,8 +134,8 @@ def copy(cls, path, target, host, is_file=False, **kwargs): # pylint: disable=a """ sync_user = getattr(settings, 'SYNC_USER', getpass.getuser()) if not is_file: - path += '/' - log.info('Remote Pull %s to %s', path, target) + path += "/" + log.info("Remote Pull %s to %s", path, target) if not is_file and not os.path.exists(target): safe_makedirs(target) # Add a slash when copying directories @@ -153,7 +148,7 @@ def copy(cls, path, target, host, is_file=False, **kwargs): # pylint: disable=a ret = os.system(sync_cmd) if ret != 0: log.debug( - 'Copy error to app servers. Command: [%s] Return: [%s]', + "Copy error to app servers. Command: [%s] Return: [%s]", sync_cmd, ret, ) diff --git a/readthedocs/builds/urls.py b/readthedocs/builds/urls.py index 9c421662047..a148362a51b 100644 --- a/readthedocs/builds/urls.py +++ b/readthedocs/builds/urls.py @@ -1,11 +1,12 @@ # -*- coding: utf-8 -*- - """URL configuration for builds app.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + from django.conf.urls import url from .views import builds_redirect_detail, builds_redirect_list - urlpatterns = [ url( r'^(?P[-\w]+)/(?P\d+)/$', diff --git a/readthedocs/builds/utils.py b/readthedocs/builds/utils.py index e0025dc77ae..7fcc245dbb8 100644 --- a/readthedocs/builds/utils.py +++ b/readthedocs/builds/utils.py @@ -1,12 +1,11 @@ # -*- coding: utf-8 -*- - """Utilities for the builds app.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + from readthedocs.projects.constants import ( - BITBUCKET_REGEXS, - GITHUB_REGEXS, - GITLAB_REGEXS, -) + BITBUCKET_REGEXS, GITHUB_REGEXS, GITLAB_REGEXS) def get_github_username_repo(url): diff --git a/readthedocs/builds/version_slug.py b/readthedocs/builds/version_slug.py index b7f88cd585f..61622369da7 100644 --- a/readthedocs/builds/version_slug.py +++ b/readthedocs/builds/version_slug.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Contains logic for handling version slugs. @@ -19,6 +17,8 @@ another number would be confusing. """ +from __future__ import absolute_import + import math import re import string @@ -26,6 +26,7 @@ from django.db import models from django.utils.encoding import force_text +from builtins import range def get_fields_with_model(cls): @@ -36,10 +37,12 @@ def get_fields_with_model(cls): prescrived in the Django docs. https://docs.djangoproject.com/en/1.11/ref/models/meta/#migrating-from-the-old-api """ - return [(f, f.model if f.model != cls else None) - for f in cls._meta.get_fields() - if not f.is_relation or f.one_to_one or - (f.many_to_one and f.related_model)] + return [ + (f, f.model if f.model != cls else None) + for f in cls._meta.get_fields() + if not f.is_relation or f.one_to_one or + (f.many_to_one and f.related_model) + ] # Regex breakdown: @@ -69,7 +72,7 @@ def __init__(self, *args, **kwargs): raise ValueError("missing 'populate_from' argument") else: self._populate_from = populate_from - super().__init__(*args, **kwargs) + super(VersionSlugField, self).__init__(*args, **kwargs) def get_queryset(self, model_cls, slug_field): # pylint: disable=protected-access @@ -165,8 +168,7 @@ def create_slug(self, model_instance): count += 1 assert self.test_pattern.match(slug), ( - 'Invalid generated slug: {slug}'.format(slug=slug) - ) + 'Invalid generated slug: {slug}'.format(slug=slug)) return slug def pre_save(self, model_instance, add): @@ -178,6 +180,6 @@ def pre_save(self, model_instance, add): return value def deconstruct(self): - name, path, args, kwargs = super().deconstruct() + name, path, args, kwargs = super(VersionSlugField, self).deconstruct() kwargs['populate_from'] = self._populate_from return name, path, args, kwargs diff --git a/readthedocs/builds/views.py b/readthedocs/builds/views.py index e86a673dc27..7cbe310fbb5 100644 --- a/readthedocs/builds/views.py +++ b/readthedocs/builds/views.py @@ -2,8 +2,16 @@ """Views for builds app.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging +from builtins import object from django.contrib import messages from django.contrib.auth.decorators import login_required from django.http import ( @@ -25,7 +33,7 @@ log = logging.getLogger(__name__) -class BuildBase: +class BuildBase(object): model = Build def get_queryset(self): @@ -35,14 +43,13 @@ def get_queryset(self): slug=self.project_slug, ) queryset = Build.objects.public( - user=self.request.user, - project=self.project, + user=self.request.user, project=self.project ) return queryset -class BuildTriggerMixin: +class BuildTriggerMixin(object): @method_decorator(login_required) def post(self, request, project_slug): @@ -58,10 +65,7 @@ def post(self, request, project_slug): slug=version_slug, ) - update_docs_task, build = trigger_build( - project=project, - version=version, - ) + update_docs_task, build = trigger_build(project=project, version=version) if (update_docs_task, build) == (None, None): # Build was skipped messages.add_message( @@ -81,17 +85,15 @@ def post(self, request, project_slug): class BuildList(BuildBase, BuildTriggerMixin, ListView): def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(BuildList, self).get_context_data(**kwargs) - active_builds = self.get_queryset().exclude( - state='finished', - ).values('id') + active_builds = self.get_queryset().exclude(state='finished' + ).values('id') context['project'] = self.project context['active_builds'] = active_builds context['versions'] = Version.objects.public( - user=self.request.user, - project=self.project, + user=self.request.user, project=self.project ) context['build_qs'] = self.get_queryset() @@ -102,7 +104,7 @@ class BuildDetail(BuildBase, DetailView): pk_url_kwarg = 'build_pk' def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(BuildDetail, self).get_context_data(**kwargs) context['project'] = self.project return context @@ -112,11 +114,11 @@ def get_context_data(self, **kwargs): def builds_redirect_list(request, project_slug): # pylint: disable=unused-argument return HttpResponsePermanentRedirect( - reverse('builds_project_list', args=[project_slug]), + reverse('builds_project_list', args=[project_slug]) ) def builds_redirect_detail(request, project_slug, pk): # pylint: disable=unused-argument return HttpResponsePermanentRedirect( - reverse('builds_detail', args=[project_slug, pk]), + reverse('builds_detail', args=[project_slug, pk]) ) diff --git a/readthedocs/config/__init__.py b/readthedocs/config/__init__.py index 23006b4c46e..314f6c394cc 100644 --- a/readthedocs/config/__init__.py +++ b/readthedocs/config/__init__.py @@ -1,5 +1,2 @@ -# -*- coding: utf-8 -*- - -"""Logic to parse and validate ``readthedocs.yaml`` file.""" from .config import * # noqa from .parser import * # noqa diff --git a/readthedocs/config/config.py b/readthedocs/config/config.py index e493d09db22..4b7c48f5649 100644 --- a/readthedocs/config/config.py +++ b/readthedocs/config/config.py @@ -3,9 +3,14 @@ # pylint: disable=too-many-lines """Build configuration for rtd.""" +from __future__ import division, print_function, unicode_literals + import os +import re from contextlib import contextmanager +import six + from readthedocs.projects.constants import DOCUMENTATION_CHOICES from .find import find_one @@ -17,12 +22,12 @@ validate_bool, validate_choice, validate_dict, + validate_directory, validate_file, validate_list, validate_string, ) - __all__ = ( 'ALL', 'load', @@ -38,8 +43,12 @@ CONFIG_NOT_SUPPORTED = 'config-not-supported' VERSION_INVALID = 'version-invalid' +BASE_INVALID = 'base-invalid' +BASE_NOT_A_DIR = 'base-not-a-directory' CONFIG_SYNTAX_INVALID = 'config-syntax-invalid' CONFIG_REQUIRED = 'config-required' +NAME_REQUIRED = 'name-required' +NAME_INVALID = 'name-invalid' CONF_FILE_REQUIRED = 'conf-file-required' PYTHON_INVALID = 'python-invalid' SUBMODULES_INVALID = 'submodules-invalid' @@ -76,7 +85,7 @@ class ConfigError(Exception): def __init__(self, message, code): self.code = code - super().__init__(message) + super(ConfigError, self).__init__(message) class ConfigOptionNotSupportedError(ConfigError): @@ -88,9 +97,9 @@ def __init__(self, configuration): template = ( 'The "{}" configuration option is not supported in this version' ) - super().__init__( + super(ConfigOptionNotSupportedError, self).__init__( template.format(self.configuration), - CONFIG_NOT_SUPPORTED, + CONFIG_NOT_SUPPORTED ) @@ -109,10 +118,10 @@ def __init__(self, key, code, error_message, source_file=None): code=code, error=error_message, ) - super().__init__(message, code=code) + super(InvalidConfig, self).__init__(message, code=code) -class BuildConfigBase: +class BuildConfigBase(object): """ Config that handles the build of one particular documentation. @@ -131,15 +140,9 @@ class BuildConfigBase: """ PUBLIC_ATTRIBUTES = [ - 'version', - 'formats', - 'python', - 'conda', - 'build', - 'doctype', - 'sphinx', - 'mkdocs', - 'submodules', + 'version', 'formats', 'python', + 'conda', 'build', 'doctype', + 'sphinx', 'mkdocs', 'submodules', ] version = None @@ -226,7 +229,7 @@ def validate(self): @property def python_interpreter(self): ver = self.python_full_version - return 'python{}'.format(ver) + return 'python{0}'.format(ver) @property def python_full_version(self): @@ -260,6 +263,12 @@ class BuildConfigV1(BuildConfigBase): """Version 1 of the configuration file.""" + BASE_INVALID_MESSAGE = 'Invalid value for base: {base}' + BASE_NOT_A_DIR_MESSAGE = '"base" is not a directory: {base}' + NAME_REQUIRED_MESSAGE = 'Missing key "name"' + NAME_INVALID_MESSAGE = ( + 'Invalid name "{name}". Valid values must match {name_re}' + ) CONF_FILE_REQUIRED_MESSAGE = 'Missing key "conf_file"' PYTHON_INVALID_MESSAGE = '"python" section must be a mapping.' PYTHON_EXTRA_REQUIREMENTS_INVALID_MESSAGE = ( @@ -297,17 +306,63 @@ def validate(self): ``readthedocs.yml`` config file if not set """ # Validate env_config. + # TODO: this isn't used + self._config['output_base'] = self.validate_output_base() + # Validate the build environment first # Must happen before `validate_python`! self._config['build'] = self.validate_build() # Validate raw_config. Order matters. + # TODO: this isn't used + self._config['name'] = self.validate_name() + # TODO: this isn't used + self._config['base'] = self.validate_base() self._config['python'] = self.validate_python() self._config['formats'] = self.validate_formats() self._config['conda'] = self.validate_conda() self._config['requirements_file'] = self.validate_requirements_file() + def validate_output_base(self): + """Validates that ``output_base`` exists and set its absolute path.""" + assert 'output_base' in self.env_config, ( + '"output_base" required in "env_config"') + output_base = os.path.abspath( + os.path.join( + self.env_config.get('output_base', self.base_path), + ) + ) + return output_base + + def validate_name(self): + """Validates that name exists.""" + name = self.raw_config.get('name', None) + if not name: + name = self.env_config.get('name', None) + if not name: + self.error('name', self.NAME_REQUIRED_MESSAGE, code=NAME_REQUIRED) + name_re = r'^[-_.0-9a-zA-Z]+$' + if not re.match(name_re, name): + self.error( + 'name', + self.NAME_INVALID_MESSAGE.format( + name=name, + name_re=name_re), + code=NAME_INVALID) + + return name + + def validate_base(self): + """Validates that path is a valid directory.""" + if 'base' in self.raw_config: + base = self.raw_config['base'] + else: + base = self.base_path + with self.catch_validation_error('base'): + base = validate_directory(base, self.base_path) + return base + def validate_build(self): """ Validate the build config settings. @@ -343,11 +398,19 @@ def validate_build(self): # Prepend proper image name to user's image name build['image'] = '{}:{}'.format( DOCKER_DEFAULT_IMAGE, - build['image'], + build['image'] ) # Update docker default settings from image name if build['image'] in DOCKER_IMAGE_SETTINGS: - self.env_config.update(DOCKER_IMAGE_SETTINGS[build['image']]) + self.env_config.update( + DOCKER_IMAGE_SETTINGS[build['image']] + ) + # Update docker settings from user config + if 'DOCKER_IMAGE_SETTINGS' in self.env_config and \ + build['image'] in self.env_config['DOCKER_IMAGE_SETTINGS']: + self.env_config.update( + self.env_config['DOCKER_IMAGE_SETTINGS'][build['image']] + ) # Allow to override specific project config_image = self.defaults.get('build_image') @@ -374,22 +437,20 @@ def validate_python(self): self.error( 'python', self.PYTHON_INVALID_MESSAGE, - code=PYTHON_INVALID, - ) + code=PYTHON_INVALID) # Validate use_system_site_packages. if 'use_system_site_packages' in raw_python: - with self.catch_validation_error('python.use_system_site_packages'): + with self.catch_validation_error( + 'python.use_system_site_packages'): python['use_system_site_packages'] = validate_bool( - raw_python['use_system_site_packages'], - ) + raw_python['use_system_site_packages']) # Validate pip_install. if 'pip_install' in raw_python: with self.catch_validation_error('python.pip_install'): python['install_with_pip'] = validate_bool( - raw_python['pip_install'], - ) + raw_python['pip_install']) # Validate extra_requirements. if 'extra_requirements' in raw_python: @@ -398,30 +459,29 @@ def validate_python(self): self.error( 'python.extra_requirements', self.PYTHON_EXTRA_REQUIREMENTS_INVALID_MESSAGE, - code=PYTHON_INVALID, - ) + code=PYTHON_INVALID) if not python['install_with_pip']: python['extra_requirements'] = [] else: for extra_name in raw_extra_requirements: - with self.catch_validation_error('python.extra_requirements'): + with self.catch_validation_error( + 'python.extra_requirements'): python['extra_requirements'].append( - validate_string(extra_name), + validate_string(extra_name) ) # Validate setup_py_install. if 'setup_py_install' in raw_python: with self.catch_validation_error('python.setup_py_install'): python['install_with_setup'] = validate_bool( - raw_python['setup_py_install'], - ) + raw_python['setup_py_install']) if 'version' in raw_python: with self.catch_validation_error('python.version'): # Try to convert strings to an int first, to catch '2', then # a float, to catch '2.7' version = raw_python['version'] - if isinstance(version, str): + if isinstance(version, six.string_types): try: version = int(version) except ValueError: @@ -448,8 +508,7 @@ def validate_conda(self): if 'file' in raw_conda: with self.catch_validation_error('conda.file'): conda_environment = validate_file( - raw_conda['file'], - self.base_path, + raw_conda['file'], self.base_path ) conda['environment'] = conda_environment @@ -483,6 +542,21 @@ def validate_formats(self): return formats + @property + def name(self): + """The project name.""" + return self._config['name'] + + @property + def base(self): + """The base directory.""" + return self._config['base'] + + @property + def output_base(self): + """The output base.""" + return self._config['output_base'] + @property def formats(self): """The documentation formats to be built.""" @@ -655,7 +729,7 @@ def validate_python(self): python = {} with self.catch_validation_error('python.version'): version = self.pop_config('python.version', 3) - if isinstance(version, str): + if isinstance(version, six.string_types): try: version = int(version) except ValueError: @@ -687,8 +761,7 @@ def validate_python(self): with self.catch_validation_error('python.extra_requirements'): extra_requirements = self.pop_config( - 'python.extra_requirements', - [], + 'python.extra_requirements', [] ) extra_requirements = validate_list(extra_requirements) if extra_requirements and not python['install_with_pip']: @@ -806,8 +879,7 @@ def validate_sphinx(self): if not configuration: configuration = None configuration = self.pop_config( - 'sphinx.configuration', - configuration, + 'sphinx.configuration', configuration ) if configuration is not None: configuration = validate_file(configuration, self.base_path) @@ -823,8 +895,9 @@ def validate_final_doc_type(self): """ Validates that the doctype is the same as the admin panel. - This a temporal validation, as the configuration file should support per - version doctype, but we need to adapt the rtd code for that. + This a temporal validation, as the configuration file + should support per version doctype, but we need to + adapt the rtd code for that. """ dashboard_doctype = self.defaults.get('doctype', 'sphinx') if self.doctype != dashboard_doctype: @@ -834,7 +907,7 @@ def validate_final_doc_type(self): if dashboard_doctype == 'mkdocs' or not self.sphinx: error_msg += ' but there is no "{}" key specified.'.format( - 'mkdocs' if dashboard_doctype == 'mkdocs' else 'sphinx', + 'mkdocs' if dashboard_doctype == 'mkdocs' else 'sphinx' ) else: error_msg += ' but your "sphinx.builder" key does not match.' @@ -896,8 +969,8 @@ def validate_keys(self): """ Checks that we don't have extra keys (invalid ones). - This should be called after all the validations are done and all keys - are popped from `self.raw_config`. + This should be called after all the validations are done + and all keys are popped from `self.raw_config`. """ msg = ( 'Invalid configuration option: {}. ' @@ -987,7 +1060,10 @@ def load(path, env_config): filename = find_one(path, CONFIG_FILENAME_REGEX) if not filename: - raise ConfigError('No configuration file found', code=CONFIG_REQUIRED) + raise ConfigError( + 'No configuration file found', + code=CONFIG_REQUIRED + ) with open(filename, 'r') as configuration_file: try: config = parse(configuration_file.read()) diff --git a/readthedocs/config/find.py b/readthedocs/config/find.py index 2fe5292c45e..cb3e5e0c56d 100644 --- a/readthedocs/config/find.py +++ b/readthedocs/config/find.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- - """Helper functions to search files.""" +from __future__ import division, print_function, unicode_literals + import os import re diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index bc6f381e805..bf12ddfa6d4 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- - """Models for the response of the configuration object.""" +from __future__ import division, print_function, unicode_literals + from collections import namedtuple diff --git a/readthedocs/config/parser.py b/readthedocs/config/parser.py index 376774d6144..655b1601bf7 100644 --- a/readthedocs/config/parser.py +++ b/readthedocs/config/parser.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- - """YAML parser for the RTD configuration file.""" -import yaml +from __future__ import division, print_function, unicode_literals +import yaml __all__ = ('parse', 'ParseError') @@ -12,6 +12,8 @@ class ParseError(Exception): """Parser related errors.""" + pass + def parse(stream): """ diff --git a/readthedocs/config/tests/test_config.py b/readthedocs/config/tests/test_config.py index 98e579b1ceb..f623881a615 100644 --- a/readthedocs/config/tests/test_config.py +++ b/readthedocs/config/tests/test_config.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + import os import re import textwrap @@ -22,6 +24,8 @@ CONFIG_NOT_SUPPORTED, CONFIG_REQUIRED, INVALID_KEY, + NAME_INVALID, + NAME_REQUIRED, PYTHON_INVALID, VERSION_INVALID, ) @@ -30,20 +34,49 @@ INVALID_BOOL, INVALID_CHOICE, INVALID_LIST, + INVALID_PATH, + INVALID_STRING, VALUE_NOT_FOUND, ValidationError, ) from .utils import apply_fs +env_config = { + 'output_base': '/tmp', +} + +minimal_config = { + 'name': 'docs', +} -yaml_config_dir = { - 'readthedocs.yml': textwrap.dedent( - ''' - formats: - - pdf - ''' - ), +config_with_explicit_empty_list = { + 'readthedocs.yml': ''' +name: docs +formats: [] +''', +} + +minimal_config_dir = { + 'readthedocs.yml': '''\ +name: docs +''', +} + +multiple_config_dir = { + 'readthedocs.yml': ''' +name: first +--- +name: second + ''', + 'nested': minimal_config_dir, +} + +yaml_extension_config_dir = { + 'readthedocs.yaml': '''\ +name: docs +type: sphinx +''' } @@ -55,145 +88,156 @@ def get_build_config(config, env_config=None, source_file='readthedocs.yml'): ) -@pytest.mark.parametrize( - 'files', [ - {'readthedocs.ymlmore': ''}, {'first': {'readthedocs.yml': ''}}, - {'startreadthedocs.yml': ''}, {'second': {'confuser.txt': 'content'}}, - {'noroot': {'readthedocs.ymlmore': ''}}, {'third': {'readthedocs.yml': 'content', 'Makefile': ''}}, - {'noroot': {'startreadthedocs.yml': ''}}, {'fourth': {'samplefile.yaml': 'content'}}, - {'readthebots.yaml': ''}, {'fifth': {'confuser.txt': '', 'readthedocs.yml': 'content'}}, - ], -) +def get_env_config(extra=None): + """Get the minimal env_config for the configuration object.""" + defaults = { + 'output_base': '', + 'name': 'name', + } + if extra is None: + extra = {} + defaults.update(extra) + return defaults + + +@pytest.mark.parametrize('files', [ + {'readthedocs.ymlmore': ''}, {'first': {'readthedocs.yml': ''}}, + {'startreadthedocs.yml': ''}, {'second': {'confuser.txt': 'content'}}, + {'noroot': {'readthedocs.ymlmore': ''}}, {'third': {'readthedocs.yml': 'content', 'Makefile': ''}}, + {'noroot': {'startreadthedocs.yml': ''}}, {'fourth': {'samplefile.yaml': 'content'}}, + {'readthebots.yaml': ''}, {'fifth': {'confuser.txt': '', 'readthedocs.yml': 'content'}}, +]) def test_load_no_config_file(tmpdir, files): apply_fs(tmpdir, files) base = str(tmpdir) with raises(ConfigError) as e: - load(base, {}) + load(base, env_config) assert e.value.code == CONFIG_REQUIRED def test_load_empty_config_file(tmpdir): - apply_fs( - tmpdir, { - 'readthedocs.yml': '', - }, - ) + apply_fs(tmpdir, { + 'readthedocs.yml': '' + }) base = str(tmpdir) with raises(ConfigError): - load(base, {}) + load(base, env_config) def test_minimal_config(tmpdir): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config_dir) base = str(tmpdir) - build = load(base, {}) + build = load(base, env_config) assert isinstance(build, BuildConfigV1) def test_load_version1(tmpdir): - apply_fs( - tmpdir, { - 'readthedocs.yml': textwrap.dedent(''' + apply_fs(tmpdir, { + 'readthedocs.yml': textwrap.dedent(''' version: 1 - '''), - }, - ) + ''') + }) base = str(tmpdir) - build = load(base, {'allow_v2': True}) + build = load(base, get_env_config({'allow_v2': True})) assert isinstance(build, BuildConfigV1) def test_load_version2(tmpdir): - apply_fs( - tmpdir, { - 'readthedocs.yml': textwrap.dedent(''' + apply_fs(tmpdir, { + 'readthedocs.yml': textwrap.dedent(''' version: 2 - '''), - }, - ) + ''') + }) base = str(tmpdir) - build = load(base, {'allow_v2': True}) + build = load(base, get_env_config({'allow_v2': True})) assert isinstance(build, BuildConfigV2) def test_load_unknow_version(tmpdir): - apply_fs( - tmpdir, { - 'readthedocs.yml': textwrap.dedent(''' + apply_fs(tmpdir, { + 'readthedocs.yml': textwrap.dedent(''' version: 9 - '''), - }, - ) + ''') + }) base = str(tmpdir) with raises(ConfigError) as excinfo: - load(base, {'allow_v2': True}) + load(base, get_env_config({'allow_v2': True})) assert excinfo.value.code == VERSION_INVALID def test_yaml_extension(tmpdir): - """Make sure loading the 'readthedocs' file with a 'yaml' extension.""" - apply_fs( - tmpdir, { - 'readthedocs.yaml': textwrap.dedent( - ''' - python: - version: 3 - ''' - ), - }, - ) + """Make sure it's capable of loading the 'readthedocs' file with a 'yaml' extension.""" + apply_fs(tmpdir, yaml_extension_config_dir) base = str(tmpdir) - config = load(base, {}) + config = load(base, env_config) assert isinstance(config, BuildConfigV1) def test_build_config_has_source_file(tmpdir): - base = str(apply_fs(tmpdir, yaml_config_dir)) - build = load(base, {}) + base = str(apply_fs(tmpdir, minimal_config_dir)) + build = load(base, env_config) assert build.source_file == os.path.join(base, 'readthedocs.yml') def test_build_config_has_list_with_single_empty_value(tmpdir): - base = str(apply_fs( - tmpdir, { - 'readthedocs.yml': textwrap.dedent( - ''' - formats: [] - ''' - ), - }, - )) - build = load(base, {}) + base = str(apply_fs(tmpdir, config_with_explicit_empty_list)) + build = load(base, env_config) assert isinstance(build, BuildConfigV1) assert build.formats == [] +def test_config_requires_name(): + build = BuildConfigV1( + {'output_base': ''}, + {}, + source_file='readthedocs.yml', + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'name' + assert excinfo.value.code == NAME_REQUIRED + + +def test_build_requires_valid_name(): + build = BuildConfigV1( + {'output_base': ''}, + {'name': 'with/slashes'}, + source_file='readthedocs.yml', + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'name' + assert excinfo.value.code == NAME_INVALID + + def test_version(): - build = get_build_config({}) + build = get_build_config({}, get_env_config()) assert build.version == '1' def test_doc_type(): build = get_build_config( {}, - { - 'defaults': { - 'doctype': 'sphinx', - }, - }, + get_env_config( + { + 'defaults': { + 'doctype': 'sphinx', + }, + } + ) ) build.validate() assert build.doctype == 'sphinx' def test_empty_python_section_is_valid(): - build = get_build_config({'python': {}}) + build = get_build_config({'python': {}}, get_env_config()) build.validate() assert build.python def test_python_section_must_be_dict(): - build = get_build_config({'python': 123}) + build = get_build_config({'python': 123}, get_env_config()) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'python' @@ -201,7 +245,7 @@ def test_python_section_must_be_dict(): def test_use_system_site_packages_defaults_to_false(): - build = get_build_config({'python': {}}) + build = get_build_config({'python': {}}, get_env_config()) build.validate() # Default is False. assert not build.python.use_system_site_packages @@ -212,22 +256,22 @@ def test_use_system_site_packages_repects_default_value(value): defaults = { 'use_system_packages': value, } - build = get_build_config({}, {'defaults': defaults}) + build = get_build_config({}, get_env_config({'defaults': defaults})) build.validate() assert build.python.use_system_site_packages is value def test_python_pip_install_default(): - build = get_build_config({'python': {}}) + build = get_build_config({'python': {}}, get_env_config()) build.validate() # Default is False. assert build.python.install_with_pip is False -class TestValidatePythonExtraRequirements: +class TestValidatePythonExtraRequirements(object): def test_it_defaults_to_list(self): - build = get_build_config({'python': {}}) + build = get_build_config({'python': {}}, get_env_config()) build.validate() # Default is an empty list. assert build.python.extra_requirements == [] @@ -235,6 +279,7 @@ def test_it_defaults_to_list(self): def test_it_validates_is_a_list(self): build = get_build_config( {'python': {'extra_requirements': 'invalid'}}, + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -251,21 +296,23 @@ def test_it_uses_validate_string(self, validate_string): 'extra_requirements': ['tests'], }, }, + get_env_config(), ) build.validate() validate_string.assert_any_call('tests') -class TestValidateUseSystemSitePackages: +class TestValidateUseSystemSitePackages(object): def test_it_defaults_to_false(self): - build = get_build_config({'python': {}}) + build = get_build_config({'python': {}}, get_env_config()) build.validate() assert build.python.use_system_site_packages is False def test_it_validates_value(self): build = get_build_config( {'python': {'use_system_site_packages': 'invalid'}}, + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -277,21 +324,23 @@ def test_it_uses_validate_bool(self, validate_bool): validate_bool.return_value = True build = get_build_config( {'python': {'use_system_site_packages': 'to-validate'}}, + get_env_config(), ) build.validate() validate_bool.assert_any_call('to-validate') -class TestValidateSetupPyInstall: +class TestValidateSetupPyInstall(object): def test_it_defaults_to_false(self): - build = get_build_config({'python': {}}) + build = get_build_config({'python': {}}, get_env_config()) build.validate() assert build.python.install_with_setup is False def test_it_validates_value(self): build = get_build_config( {'python': {'setup_py_install': 'this-is-string'}}, + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -303,15 +352,16 @@ def test_it_uses_validate_bool(self, validate_bool): validate_bool.return_value = True build = get_build_config( {'python': {'setup_py_install': 'to-validate'}}, + get_env_config(), ) build.validate() validate_bool.assert_any_call('to-validate') -class TestValidatePythonVersion: +class TestValidatePythonVersion(object): def test_it_defaults_to_a_valid_version(self): - build = get_build_config({'python': {}}) + build = get_build_config({'python': {}}, get_env_config()) build.validate() assert build.python.version == 2 assert build.python_interpreter == 'python2.7' @@ -320,6 +370,7 @@ def test_it_defaults_to_a_valid_version(self): def test_it_supports_other_versions(self): build = get_build_config( {'python': {'version': 3.5}}, + get_env_config(), ) build.validate() assert build.python.version == 3.5 @@ -329,6 +380,7 @@ def test_it_supports_other_versions(self): def test_it_validates_versions_out_of_range(self): build = get_build_config( {'python': {'version': 1.0}}, + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -338,6 +390,7 @@ def test_it_validates_versions_out_of_range(self): def test_it_validates_wrong_type(self): build = get_build_config( {'python': {'version': 'this-is-string'}}, + get_env_config(), ) with raises(InvalidConfig) as excinfo: build.validate() @@ -347,6 +400,7 @@ def test_it_validates_wrong_type(self): def test_it_validates_wrong_type_right_value(self): build = get_build_config( {'python': {'version': '3.5'}}, + get_env_config(), ) build.validate() assert build.python.version == 3.5 @@ -355,6 +409,7 @@ def test_it_validates_wrong_type_right_value(self): build = get_build_config( {'python': {'version': '3'}}, + get_env_config(), ) build.validate() assert build.python.version == 3 @@ -364,10 +419,12 @@ def test_it_validates_wrong_type_right_value(self): def test_it_validates_env_supported_versions(self): build = get_build_config( {'python': {'version': 3.6}}, - env_config={ - 'python': {'supported_versions': [3.5]}, - 'build': {'image': 'custom'}, - }, + env_config=get_env_config( + { + 'python': {'supported_versions': [3.5]}, + 'build': {'image': 'custom'}, + } + ) ) with raises(InvalidConfig) as excinfo: build.validate() @@ -376,10 +433,12 @@ def test_it_validates_env_supported_versions(self): build = get_build_config( {'python': {'version': 3.6}}, - env_config={ - 'python': {'supported_versions': [3.5, 3.6]}, - 'build': {'image': 'custom'}, - }, + env_config=get_env_config( + { + 'python': {'supported_versions': [3.5, 3.6]}, + 'build': {'image': 'custom'}, + } + ) ) build.validate() assert build.python.version == 3.6 @@ -393,42 +452,43 @@ def test_it_respects_default_value(self, value): } build = get_build_config( {}, - {'defaults': defaults}, + get_env_config({'defaults': defaults}), ) build.validate() assert build.python.version == value -class TestValidateFormats: +class TestValidateFormats(object): def test_it_defaults_to_empty(self): - build = get_build_config({}) + build = get_build_config({}, get_env_config()) build.validate() assert build.formats == [] def test_it_gets_set_correctly(self): - build = get_build_config({'formats': ['pdf']}) + build = get_build_config({'formats': ['pdf']}, get_env_config()) build.validate() assert build.formats == ['pdf'] def test_formats_can_be_null(self): - build = get_build_config({'formats': None}) + build = get_build_config({'formats': None}, get_env_config()) build.validate() assert build.formats == [] def test_formats_with_previous_none(self): - build = get_build_config({'formats': ['none']}) + build = get_build_config({'formats': ['none']}, get_env_config()) build.validate() assert build.formats == [] def test_formats_can_be_empty(self): - build = get_build_config({'formats': []}) + build = get_build_config({'formats': []}, get_env_config()) build.validate() assert build.formats == [] def test_all_valid_formats(self): build = get_build_config( {'formats': ['pdf', 'htmlzip', 'epub']}, + get_env_config() ) build.validate() assert build.formats == ['pdf', 'htmlzip', 'epub'] @@ -436,6 +496,7 @@ def test_all_valid_formats(self): def test_cant_have_none_as_format(self): build = get_build_config( {'formats': ['htmlzip', None]}, + get_env_config() ) with raises(InvalidConfig) as excinfo: build.validate() @@ -445,6 +506,7 @@ def test_cant_have_none_as_format(self): def test_formats_have_only_allowed_values(self): build = get_build_config( {'formats': ['htmlzip', 'csv']}, + get_env_config() ) with raises(InvalidConfig) as excinfo: build.validate() @@ -452,7 +514,7 @@ def test_formats_have_only_allowed_values(self): assert excinfo.value.code == INVALID_CHOICE def test_only_list_type(self): - build = get_build_config({'formats': 'no-list'}) + build = get_build_config({'formats': 'no-list'}, get_env_config()) with raises(InvalidConfig) as excinfo: build.validate() assert excinfo.value.key == 'format' @@ -461,23 +523,75 @@ def test_only_list_type(self): def test_valid_build_config(): build = BuildConfigV1( - {}, - {}, + env_config, + minimal_config, source_file='readthedocs.yml', ) build.validate() + assert build.name == 'docs' + assert build.base assert build.python assert build.python.install_with_setup is False assert build.python.install_with_pip is False assert build.python.use_system_site_packages is False + assert build.output_base + +class TestValidateBase(object): -class TestValidateBuild: + def test_it_validates_to_abspath(self, tmpdir): + apply_fs(tmpdir, {'configs': minimal_config, 'docs': {}}) + with tmpdir.as_cwd(): + source_file = str(tmpdir.join('configs', 'readthedocs.yml')) + build = BuildConfigV1( + get_env_config(), + {'base': '../docs'}, + source_file=source_file, + ) + build.validate() + assert build.base == str(tmpdir.join('docs')) + + @patch('readthedocs.config.config.validate_directory') + def test_it_uses_validate_directory(self, validate_directory): + validate_directory.return_value = 'path' + build = get_build_config({'base': '../my-path'}, get_env_config()) + build.validate() + # Test for first argument to validate_directory + args, kwargs = validate_directory.call_args + assert args[0] == '../my-path' + + def test_it_fails_if_base_is_not_a_string(self, tmpdir): + apply_fs(tmpdir, minimal_config) + with tmpdir.as_cwd(): + build = BuildConfigV1( + get_env_config(), + {'base': 1}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'base' + assert excinfo.value.code == INVALID_STRING + + def test_it_fails_if_base_does_not_exist(self, tmpdir): + apply_fs(tmpdir, minimal_config) + build = BuildConfigV1( + get_env_config(), + {'base': 'docs'}, + source_file=str(tmpdir.join('readthedocs.yml')), + ) + with raises(InvalidConfig) as excinfo: + build.validate() + assert excinfo.value.key == 'base' + assert excinfo.value.code == INVALID_PATH + + +class TestValidateBuild(object): def test_it_fails_if_build_is_invalid_option(self, tmpdir): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config) build = BuildConfigV1( - {}, + get_env_config(), {'build': {'image': 3.0}}, source_file=str(tmpdir.join('readthedocs.yml')), ) @@ -487,7 +601,7 @@ def test_it_fails_if_build_is_invalid_option(self, tmpdir): assert excinfo.value.code == INVALID_CHOICE def test_it_fails_on_python_validation(self, tmpdir): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config) build = BuildConfigV1( {}, { @@ -503,7 +617,7 @@ def test_it_fails_on_python_validation(self, tmpdir): assert excinfo.value.code == INVALID_CHOICE def test_it_works_on_python_validation(self, tmpdir): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config) build = BuildConfigV1( {}, { @@ -516,9 +630,9 @@ def test_it_works_on_python_validation(self, tmpdir): build.validate_python() def test_it_works(self, tmpdir): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config) build = BuildConfigV1( - {}, + get_env_config(), {'build': {'image': 'latest'}}, source_file=str(tmpdir.join('readthedocs.yml')), ) @@ -526,9 +640,9 @@ def test_it_works(self, tmpdir): assert build.build.image == 'readthedocs/build:latest' def test_default(self, tmpdir): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config) build = BuildConfigV1( - {}, + get_env_config(), {}, source_file=str(tmpdir.join('readthedocs.yml')), ) @@ -536,15 +650,14 @@ def test_default(self, tmpdir): assert build.build.image == 'readthedocs/build:2.0' @pytest.mark.parametrize( - 'image', ['latest', 'readthedocs/build:3.0', 'rtd/build:latest'], - ) + 'image', ['latest', 'readthedocs/build:3.0', 'rtd/build:latest']) def test_it_priorities_image_from_env_config(self, tmpdir, image): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config) defaults = { 'build_image': image, } build = BuildConfigV1( - {'defaults': defaults}, + get_env_config({'defaults': defaults}), {'build': {'image': 'latest'}}, source_file=str(tmpdir.join('readthedocs.yml')), ) @@ -553,7 +666,7 @@ def test_it_priorities_image_from_env_config(self, tmpdir, image): def test_use_conda_default_false(): - build = get_build_config({}) + build = get_build_config({}, get_env_config()) build.validate() assert build.conda is None @@ -561,6 +674,7 @@ def test_use_conda_default_false(): def test_use_conda_respects_config(): build = get_build_config( {'conda': {}}, + get_env_config(), ) build.validate() assert isinstance(build.conda, Conda) @@ -570,6 +684,7 @@ def test_validates_conda_file(tmpdir): apply_fs(tmpdir, {'environment.yml': ''}) build = get_build_config( {'conda': {'file': 'environment.yml'}}, + get_env_config(), source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() @@ -578,7 +693,7 @@ def test_validates_conda_file(tmpdir): def test_requirements_file_empty(): - build = get_build_config({}) + build = get_build_config({}, get_env_config()) build.validate() assert build.python.requirements is None @@ -590,7 +705,7 @@ def test_requirements_file_repects_default_value(tmpdir): } build = get_build_config( {}, - {'defaults': defaults}, + get_env_config({'defaults': defaults}), source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() @@ -601,6 +716,7 @@ def test_requirements_file_respects_configuration(tmpdir): apply_fs(tmpdir, {'requirements.txt': ''}) build = get_build_config( {'requirements_file': 'requirements.txt'}, + get_env_config(), source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() @@ -610,6 +726,7 @@ def test_requirements_file_respects_configuration(tmpdir): def test_requirements_file_is_null(tmpdir): build = get_build_config( {'requirements_file': None}, + get_env_config(), source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() @@ -619,6 +736,7 @@ def test_requirements_file_is_null(tmpdir): def test_requirements_file_is_blank(tmpdir): build = get_build_config( {'requirements_file': ''}, + get_env_config(), source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() @@ -626,7 +744,7 @@ def test_requirements_file_is_blank(tmpdir): def test_build_validate_calls_all_subvalidators(tmpdir): - apply_fs(tmpdir, {}) + apply_fs(tmpdir, minimal_config) build = BuildConfigV1( {}, {}, @@ -634,22 +752,28 @@ def test_build_validate_calls_all_subvalidators(tmpdir): ) with patch.multiple( BuildConfigV1, + validate_base=DEFAULT, + validate_name=DEFAULT, validate_python=DEFAULT, + validate_output_base=DEFAULT, ): build.validate() + BuildConfigV1.validate_base.assert_called_with() + BuildConfigV1.validate_name.assert_called_with() BuildConfigV1.validate_python.assert_called_with() + BuildConfigV1.validate_output_base.assert_called_with() def test_load_calls_validate(tmpdir): - apply_fs(tmpdir, yaml_config_dir) + apply_fs(tmpdir, minimal_config_dir) base = str(tmpdir) with patch.object(BuildConfigV1, 'validate') as build_validate: - load(base, {}) + load(base, env_config) assert build_validate.call_count == 1 def test_raise_config_not_supported(): - build = get_build_config({}) + build = get_build_config({}, get_env_config()) build.validate() with raises(ConfigOptionNotSupportedError) as excinfo: build.redirects @@ -657,11 +781,9 @@ def test_raise_config_not_supported(): assert excinfo.value.code == CONFIG_NOT_SUPPORTED -@pytest.mark.parametrize( - 'correct_config_filename', - [prefix + 'readthedocs.' + extension for prefix in {'', '.'} - for extension in {'yml', 'yaml'}], -) +@pytest.mark.parametrize('correct_config_filename', + [prefix + 'readthedocs.' + extension for prefix in {"", "."} + for extension in {"yml", "yaml"}]) def test_config_filenames_regex(correct_config_filename): assert re.match(CONFIG_FILENAME_REGEX, correct_config_filename) @@ -677,12 +799,12 @@ def test_as_dict(tmpdir): }, 'requirements_file': 'requirements.txt', }, - { + get_env_config({ 'defaults': { 'doctype': 'sphinx', 'sphinx_configuration': None, }, - }, + }), source_file=str(tmpdir.join('readthedocs.yml')), ) build.validate() @@ -720,11 +842,10 @@ def test_as_dict(tmpdir): assert build.as_dict() == expected_dict -class TestBuildConfigV2: +class TestBuildConfigV2(object): def get_build_config( - self, config, env_config=None, source_file='readthedocs.yml', - ): + self, config, env_config=None, source_file='readthedocs.yml'): return BuildConfigV2( env_config or {}, config, @@ -757,7 +878,7 @@ def test_formats_check_invalid_value(self, value): def test_formats_check_invalid_type(self): build = self.get_build_config( - {'formats': ['htmlzip', 'invalid', 'epub']}, + {'formats': ['htmlzip', 'invalid', 'epub']} ) with raises(InvalidConfig) as excinfo: build.validate() @@ -854,8 +975,7 @@ def test_build_image_check_invalid(self, value): assert excinfo.value.key == 'build.image' @pytest.mark.parametrize( - 'image', ['latest', 'readthedocs/build:3.0', 'rtd/build:latest'], - ) + 'image', ['latest', 'readthedocs/build:3.0', 'rtd/build:latest']) def test_build_image_priorities_default(self, image): build = self.get_build_config( {'build': {'image': 'latest'}}, @@ -899,13 +1019,9 @@ def test_python_check_invalid_types(self, value): build.validate() assert excinfo.value.key == 'python' - @pytest.mark.parametrize( - 'image,versions', - [ - ('latest', [2, 2.7, 3, 3.5, 3.6]), - ('stable', [2, 2.7, 3, 3.5, 3.6]), - ], - ) + @pytest.mark.parametrize('image,versions', + [('latest', [2, 2.7, 3, 3.5, 3.6]), + ('stable', [2, 2.7, 3, 3.5, 3.6])]) def test_python_version(self, image, versions): for version in versions: build = self.get_build_config({ @@ -931,13 +1047,9 @@ def test_python_version_accepts_string(self): build.validate() assert build.python.version == 3.6 - @pytest.mark.parametrize( - 'image,versions', - [ - ('latest', [1, 2.8, 4, 3.8]), - ('stable', [1, 2.8, 4, 3.8]), - ], - ) + @pytest.mark.parametrize('image,versions', + [('latest', [1, 2.8, 4, 3.8]), + ('stable', [1, 2.8, 4, 3.8])]) def test_python_version_invalid(self, image, versions): for version in versions: build = self.get_build_config({ @@ -1113,7 +1225,7 @@ def test_python_extra_requirements_and_pip(self): 'python': { 'install': 'pip', 'extra_requirements': ['docs', 'tests'], - }, + } }) build.validate() assert build.python.extra_requirements == ['docs', 'tests'] @@ -1122,7 +1234,7 @@ def test_python_extra_requirements_not_install(self): build = self.get_build_config({ 'python': { 'extra_requirements': ['docs', 'tests'], - }, + } }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1133,7 +1245,7 @@ def test_python_extra_requirements_and_setup(self): 'python': { 'install': 'setup.py', 'extra_requirements': ['docs', 'tests'], - }, + } }) with raises(InvalidConfig) as excinfo: build.validate() @@ -1229,14 +1341,10 @@ def test_sphinx_is_default_doc_type(self): assert build.mkdocs is None assert build.doctype == 'sphinx' - @pytest.mark.parametrize( - 'value,expected', - [ - ('html', 'sphinx'), - ('htmldir', 'sphinx_htmldir'), - ('singlehtml', 'sphinx_singlehtml'), - ], - ) + @pytest.mark.parametrize('value,expected', + [('html', 'sphinx'), + ('htmldir', 'sphinx_htmldir'), + ('singlehtml', 'sphinx_singlehtml')]) def test_sphinx_builder_check_valid(self, value, expected): build = self.get_build_config( {'sphinx': {'builder': value}}, @@ -1259,8 +1367,7 @@ def test_sphinx_builder_default(self): build.sphinx.builder == 'sphinx' @pytest.mark.skip( - 'This test is not compatible with the new validation around doctype.', - ) + 'This test is not compatible with the new validation around doctype.') def test_sphinx_builder_ignores_default(self): build = self.get_build_config( {}, @@ -1492,7 +1599,7 @@ def test_submodules_check_invalid_type(self, value): def test_submodules_include_check_valid(self): build = self.get_build_config({ 'submodules': { - 'include': ['one', 'two'], + 'include': ['one', 'two'] }, }) build.validate() @@ -1525,8 +1632,8 @@ def test_submodules_include_allows_all_keyword(self): def test_submodules_exclude_check_valid(self): build = self.get_build_config({ 'submodules': { - 'exclude': ['one', 'two'], - }, + 'exclude': ['one', 'two'] + } }) build.validate() assert build.submodules.include == [] @@ -1626,28 +1733,26 @@ def test_submodules_recursive_explict_default(self): assert build.submodules.exclude == [] assert build.submodules.recursive is False - @pytest.mark.parametrize( - 'value,key', [ - ({'typo': 'something'}, 'typo'), - ( - { - 'pyton': { - 'version': 'another typo', - }, - }, - 'pyton.version', - ), - ( - { - 'build': { - 'image': 'latest', - 'extra': 'key', - }, - }, - 'build.extra', - ), - ], - ) + @pytest.mark.parametrize('value,key', [ + ({'typo': 'something'}, 'typo'), + ( + { + 'pyton': { + 'version': 'another typo', + } + }, + 'pyton.version' + ), + ( + { + 'build': { + 'image': 'latest', + 'extra': 'key', + } + }, + 'build.extra' + ) + ]) def test_strict_validation(self, value, key): build = self.get_build_config(value) with raises(InvalidConfig) as excinfo: @@ -1665,15 +1770,13 @@ def test_strict_validation_pops_all_keys(self): build.validate() assert build.raw_config == {} - @pytest.mark.parametrize( - 'value,expected', [ - ({}, []), - ({'one': 1}, ['one']), - ({'one': {'two': 3}}, ['one', 'two']), - (OrderedDict([('one', 1), ('two', 2)]), ['one']), - (OrderedDict([('one', {'two': 2}), ('three', 3)]), ['one', 'two']), - ], - ) + @pytest.mark.parametrize('value,expected', [ + ({}, []), + ({'one': 1}, ['one']), + ({'one': {'two': 3}}, ['one', 'two']), + (OrderedDict([('one', 1), ('two', 2)]), ['one']), + (OrderedDict([('one', {'two': 2}), ('three', 3)]), ['one', 'two']), + ]) def test_get_extra_key(self, value, expected): build = self.get_build_config({}) assert build._get_extra_key(value) == expected diff --git a/readthedocs/config/tests/test_find.py b/readthedocs/config/tests/test_find.py index ab813cdb6a5..7ba651059e3 100644 --- a/readthedocs/config/tests/test_find.py +++ b/readthedocs/config/tests/test_find.py @@ -1,6 +1,8 @@ -# -*- coding: utf-8 -*- -import os +from __future__ import division, print_function, unicode_literals +import os +import pytest +import six from readthedocs.config.find import find_one from .utils import apply_fs @@ -17,3 +19,16 @@ def test_find_at_root(tmpdir): base = str(tmpdir) path = find_one(base, r'readthedocs\.yml') assert path == os.path.abspath(os.path.join(base, 'readthedocs.yml')) + + +@pytest.mark.skipif(not six.PY2, reason='Only for python2') +def test_find_unicode_path(tmpdir): + base_path = os.path.abspath( + os.path.join(os.path.dirname(__file__), 'fixtures/bad_encode_project') + ) + path = find_one(base_path, r'readthedocs\.yml') + assert path == '' + unicode_base_path = base_path.decode('utf-8') + assert isinstance(unicode_base_path, unicode) + path = find_one(unicode_base_path, r'readthedocs\.yml') + assert path == '' diff --git a/readthedocs/config/tests/test_parser.py b/readthedocs/config/tests/test_parser.py index 64a7e1f7292..5c37c3c5cb0 100644 --- a/readthedocs/config/tests/test_parser.py +++ b/readthedocs/config/tests/test_parser.py @@ -1,4 +1,5 @@ -# -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from io import StringIO from pytest import raises @@ -7,64 +8,63 @@ def test_parse_empty_config_file(): - buf = StringIO('') + buf = StringIO(u'') with raises(ParseError): parse(buf) def test_parse_invalid_yaml(): - buf = StringIO('- - !asdf') + buf = StringIO(u'- - !asdf') with raises(ParseError): parse(buf) def test_parse_bad_type(): - buf = StringIO('Hello') + buf = StringIO(u'Hello') with raises(ParseError): parse(buf) def test_parse_single_config(): - buf = StringIO('base: path') + buf = StringIO(u'base: path') config = parse(buf) assert isinstance(config, dict) assert config['base'] == 'path' def test_parse_null_value(): - buf = StringIO('base: null') + buf = StringIO(u'base: null') config = parse(buf) assert config['base'] is None def test_parse_empty_value(): - buf = StringIO('base:') + buf = StringIO(u'base:') config = parse(buf) assert config['base'] is None def test_parse_empty_string_value(): - buf = StringIO('base: ""') + buf = StringIO(u'base: ""') config = parse(buf) assert config['base'] == '' def test_parse_empty_list(): - buf = StringIO('base: []') + buf = StringIO(u'base: []') config = parse(buf) assert config['base'] == [] def test_do_not_parse_multiple_configs_in_one_file(): buf = StringIO( - ''' + u''' base: path --- base: other_path name: second nested: works: true - ''' - ) + ''') with raises(ParseError): parse(buf) diff --git a/readthedocs/config/tests/test_utils.py b/readthedocs/config/tests/test_utils.py index 3d4b57254e3..3d89cb0d0c1 100644 --- a/readthedocs/config/tests/test_utils.py +++ b/readthedocs/config/tests/test_utils.py @@ -1,4 +1,5 @@ -# -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from .utils import apply_fs diff --git a/readthedocs/config/tests/test_validation.py b/readthedocs/config/tests/test_validation.py index 8a0e98b98c4..8c2519570d2 100644 --- a/readthedocs/config/tests/test_validation.py +++ b/readthedocs/config/tests/test_validation.py @@ -1,29 +1,20 @@ # -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + import os from mock import patch from pytest import raises +from six import text_type from readthedocs.config.validation import ( - INVALID_BOOL, - INVALID_CHOICE, - INVALID_DIRECTORY, - INVALID_FILE, - INVALID_LIST, - INVALID_PATH, - INVALID_STRING, - ValidationError, - validate_bool, - validate_choice, - validate_directory, - validate_file, - validate_list, - validate_path, - validate_string, -) - - -class TestValidateBool: + INVALID_BOOL, INVALID_CHOICE, INVALID_DIRECTORY, INVALID_FILE, INVALID_LIST, + INVALID_PATH, INVALID_STRING, ValidationError, validate_bool, + validate_choice, validate_directory, validate_file, validate_list, + validate_path, validate_string) + + +class TestValidateBool(object): def test_it_accepts_true(self): assert validate_bool(True) is True @@ -42,7 +33,7 @@ def test_it_fails_on_string(self): assert excinfo.value.code == INVALID_BOOL -class TestValidateChoice: +class TestValidateChoice(object): def test_it_accepts_valid_choice(self): result = validate_choice('choice', ('choice', 'another_choice')) @@ -58,7 +49,7 @@ def test_it_rejects_invalid_choice(self): assert excinfo.value.code == INVALID_CHOICE -class TestValidateList: +class TestValidateList(object): def test_it_accepts_list_types(self): result = validate_list(['choice', 'another_choice']) @@ -79,16 +70,16 @@ def iterator(): def test_it_rejects_string_types(self): with raises(ValidationError) as excinfo: - validate_list('choice') + result = validate_list('choice') assert excinfo.value.code == INVALID_LIST -class TestValidateDirectory: +class TestValidateDirectory(object): def test_it_uses_validate_path(self, tmpdir): patcher = patch('readthedocs.config.validation.validate_path') with patcher as validate_path: - path = str(tmpdir.mkdir('a directory')) + path = text_type(tmpdir.mkdir('a directory')) validate_path.return_value = path validate_directory(path, str(tmpdir)) validate_path.assert_called_with(path, str(tmpdir)) @@ -100,7 +91,7 @@ def test_it_rejects_files(self, tmpdir): assert excinfo.value.code == INVALID_DIRECTORY -class TestValidateFile: +class TestValidateFile(object): def test_it_uses_validate_path(self, tmpdir): patcher = patch('readthedocs.config.validation.validate_path') @@ -119,7 +110,7 @@ def test_it_rejects_directories(self, tmpdir): assert excinfo.value.code == INVALID_FILE -class TestValidatePath: +class TestValidatePath(object): def test_it_accepts_relative_path(self, tmpdir): tmpdir.mkdir('a directory') @@ -149,15 +140,15 @@ def test_it_rejects_non_existent_path(self, tmpdir): assert excinfo.value.code == INVALID_PATH -class TestValidateString: +class TestValidateString(object): def test_it_accepts_unicode(self): - result = validate_string('Unicöde') - assert isinstance(result, str) + result = validate_string(u'Unicöde') + assert isinstance(result, text_type) def test_it_accepts_nonunicode(self): result = validate_string('Unicode') - assert isinstance(result, str) + assert isinstance(result, text_type) def test_it_rejects_float(self): with raises(ValidationError) as excinfo: diff --git a/readthedocs/config/tests/utils.py b/readthedocs/config/tests/utils.py index 4dd6a53313c..b1b312420bb 100644 --- a/readthedocs/config/tests/utils.py +++ b/readthedocs/config/tests/utils.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + def apply_fs(tmpdir, contents): """ - Create the directory structure specified in ``contents``. - - It's a dict of filenames as keys and the file contents as values. If the - value is another dict, it's a subdirectory. + Create the directory structure specified in ``contents``. It's a dict of + filenames as keys and the file contents as values. If the value is another + dict, it's a subdirectory. """ for filename, content in contents.items(): if hasattr(content, 'items'): diff --git a/readthedocs/config/validation.py b/readthedocs/config/validation.py index 5d7651dffd8..ab9164f335e 100644 --- a/readthedocs/config/validation.py +++ b/readthedocs/config/validation.py @@ -1,8 +1,9 @@ -# -*- coding: utf-8 -*- - """Validations for the RTD configuration file.""" +from __future__ import division, print_function, unicode_literals + import os +from six import string_types, text_type INVALID_BOOL = 'invalid-bool' INVALID_CHOICE = 'invalid-choice' @@ -28,7 +29,7 @@ class ValidationError(Exception): INVALID_PATH: 'path {value} does not exist', INVALID_STRING: 'expected string', INVALID_LIST: 'expected list', - VALUE_NOT_FOUND: '{value} not found', + VALUE_NOT_FOUND: '{value} not found' } def __init__(self, value, code, format_kwargs=None): @@ -40,12 +41,12 @@ def __init__(self, value, code, format_kwargs=None): if format_kwargs is not None: defaults.update(format_kwargs) message = self.messages[code].format(**defaults) - super().__init__(message) + super(ValidationError, self).__init__(message) def validate_list(value): """Check if ``value`` is an iterable.""" - if isinstance(value, (dict, str)): + if isinstance(value, (dict, string_types)): raise ValidationError(value, INVALID_LIST) if not hasattr(value, '__iter__'): raise ValidationError(value, INVALID_LIST) @@ -62,13 +63,9 @@ def validate_choice(value, choices): """Check that ``value`` is in ``choices``.""" choices = validate_list(choices) if value not in choices: - raise ValidationError( - value, - INVALID_CHOICE, - { - 'choices': ', '.join(map(str, choices)), - }, - ) + raise ValidationError(value, INVALID_CHOICE, { + 'choices': ', '.join(map(str, choices)) + }) return value @@ -116,6 +113,6 @@ def validate_path(value, base_path): def validate_string(value): """Check that ``value`` is a string type.""" - if not isinstance(value, str): + if not isinstance(value, string_types): raise ValidationError(value, INVALID_STRING) - return str(value) + return text_type(value) diff --git a/readthedocs/constants.py b/readthedocs/constants.py index 579b937715a..b9796e8f998 100644 --- a/readthedocs/constants.py +++ b/readthedocs/constants.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Common constants.""" +"""Common constants""" +from __future__ import absolute_import from readthedocs.builds.version_slug import VERSION_SLUG_REGEX from readthedocs.projects.constants import LANGUAGES_REGEX, PROJECT_SLUG_REGEX diff --git a/readthedocs/core/__init__.py b/readthedocs/core/__init__.py index 21fae505765..ed1c53debf0 100644 --- a/readthedocs/core/__init__.py +++ b/readthedocs/core/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """App initialization.""" default_app_config = 'readthedocs.core.apps.CoreAppConfig' diff --git a/readthedocs/core/adapters.py b/readthedocs/core/adapters.py index c91ad9c3619..170f8954e26 100644 --- a/readthedocs/core/adapters.py +++ b/readthedocs/core/adapters.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Allauth overrides.""" +"""Allauth overrides""" +from __future__ import absolute_import import json import logging @@ -10,7 +9,6 @@ from readthedocs.core.utils import send_email - try: from django.utils.encoding import force_text except ImportError: @@ -21,17 +19,16 @@ class AccountAdapter(DefaultAccountAdapter): - """Customize Allauth emails to match our current patterns.""" + """Customize Allauth emails to match our current patterns""" def format_email_subject(self, subject): return force_text(subject) def send_mail(self, template_prefix, email, context): subject = render_to_string( - '{}_subject.txt'.format(template_prefix), - context, + '{0}_subject.txt'.format(template_prefix), context ) - subject = ' '.join(subject.splitlines()).strip() + subject = " ".join(subject.splitlines()).strip() subject = self.format_email_subject(subject) # Allauth sends some additional data in the context, remove it if the @@ -44,15 +41,13 @@ def send_mail(self, template_prefix, email, context): removed_keys.append(key) del context[key] if removed_keys: - log.debug( - 'Removed context we were unable to serialize: %s', - removed_keys, - ) + log.debug('Removed context we were unable to serialize: %s', + removed_keys) send_email( recipient=email, subject=subject, - template='{}_message.txt'.format(template_prefix), - template_html='{}_message.html'.format(template_prefix), - context=context, + template='{0}_message.txt'.format(template_prefix), + template_html='{0}_message.html'.format(template_prefix), + context=context ) diff --git a/readthedocs/core/admin.py b/readthedocs/core/admin.py index d0f2051d47c..b30f5460484 100644 --- a/readthedocs/core/admin.py +++ b/readthedocs/core/admin.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- - """Django admin interface for core models.""" +from __future__ import absolute_import from datetime import timedelta from django.contrib import admin -from django.contrib.auth.admin import UserAdmin from django.contrib.auth.models import User -from django.utils import timezone +from django.contrib.auth.admin import UserAdmin from django.utils.translation import ugettext_lazy as _ +from django.utils import timezone from readthedocs.core.models import UserProfile from readthedocs.projects.models import Project @@ -60,14 +59,8 @@ class UserAdminExtra(UserAdmin): """Admin configuration for User.""" - list_display = ( - 'username', - 'email', - 'first_name', - 'last_name', - 'is_staff', - 'is_banned', - ) + list_display = ('username', 'email', 'first_name', + 'last_name', 'is_staff', 'is_banned') list_filter = (UserProjectFilter,) + UserAdmin.list_filter actions = ['ban_user'] inlines = [UserProjectInline] diff --git a/readthedocs/core/apps.py b/readthedocs/core/apps.py index ac5a39d63fd..4a9875ffb13 100644 --- a/readthedocs/core/apps.py +++ b/readthedocs/core/apps.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - """App configurations for core app.""" +from __future__ import absolute_import from django.apps import AppConfig diff --git a/readthedocs/core/backends.py b/readthedocs/core/backends.py index b7f9bf788a7..6a8b8ec9007 100644 --- a/readthedocs/core/backends.py +++ b/readthedocs/core/backends.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- - """Email backends for core app.""" +from __future__ import absolute_import import smtplib -from django.core.mail.backends.smtp import EmailBackend from django.core.mail.utils import DNS_NAME +from django.core.mail.backends.smtp import EmailBackend class SSLEmailBackend(EmailBackend): @@ -14,11 +13,8 @@ def open(self): if self.connection: return False try: - self.connection = smtplib.SMTP_SSL( - self.host, - self.port, - local_hostname=DNS_NAME.get_fqdn(), - ) + self.connection = smtplib.SMTP_SSL(self.host, self.port, + local_hostname=DNS_NAME.get_fqdn()) if self.username and self.password: self.connection.login(self.username, self.password) return True diff --git a/readthedocs/core/context_processors.py b/readthedocs/core/context_processors.py index 8bddb9bd10f..6ac71df2ac7 100644 --- a/readthedocs/core/context_processors.py +++ b/readthedocs/core/context_processors.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - """Template context processors for core app.""" +from __future__ import absolute_import from django.conf import settings @@ -12,17 +11,10 @@ def readthedocs_processor(request): 'PRODUCTION_DOMAIN': getattr(settings, 'PRODUCTION_DOMAIN', None), 'USE_SUBDOMAINS': getattr(settings, 'USE_SUBDOMAINS', None), 'GLOBAL_ANALYTICS_CODE': getattr(settings, 'GLOBAL_ANALYTICS_CODE'), - 'DASHBOARD_ANALYTICS_CODE': getattr( - settings, - 'DASHBOARD_ANALYTICS_CODE', - ), + 'DASHBOARD_ANALYTICS_CODE': getattr(settings, 'DASHBOARD_ANALYTICS_CODE'), 'SITE_ROOT': getattr(settings, 'SITE_ROOT', '') + '/', 'TEMPLATE_ROOT': getattr(settings, 'TEMPLATE_ROOT', '') + '/', - 'DO_NOT_TRACK_ENABLED': getattr( - settings, - 'DO_NOT_TRACK_ENABLED', - False, - ), + 'DO_NOT_TRACK_ENABLED': getattr(settings, 'DO_NOT_TRACK_ENABLED', False), 'USE_PROMOS': getattr(settings, 'USE_PROMOS', False), } return exports diff --git a/readthedocs/core/fields.py b/readthedocs/core/fields.py index 5801d30146b..87f06090870 100644 --- a/readthedocs/core/fields.py +++ b/readthedocs/core/fields.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- - -"""Shared model fields and defaults.""" +"""Shared model fields and defaults""" +from __future__ import absolute_import import binascii import os def default_token(): - """Generate default value for token field.""" + """Generate default value for token field""" return binascii.hexlify(os.urandom(20)).decode() diff --git a/readthedocs/core/fixtures/flag_types.json b/readthedocs/core/fixtures/flag_types.json index 52f13740581..afe8ef1cd79 100644 --- a/readthedocs/core/fixtures/flag_types.json +++ b/readthedocs/core/fixtures/flag_types.json @@ -1,28 +1,28 @@ [ { - "pk": 1, - "model": "flagging.flagtype", + "pk": 1, + "model": "flagging.flagtype", "fields": { - "description": "This item is inappropriate to the purpose of the site", - "slug": "inappropriate", + "description": "This item is inappropriate to the purpose of the site", + "slug": "inappropriate", "title": "Inappropriate" } - }, + }, { - "pk": 2, - "model": "flagging.flagtype", + "pk": 2, + "model": "flagging.flagtype", "fields": { - "description": "This item is spam", - "slug": "spam", + "description": "This item is spam", + "slug": "spam", "title": "Spam" } - }, + }, { - "pk": 3, - "model": "flagging.flagtype", + "pk": 3, + "model": "flagging.flagtype", "fields": { - "description": "These docs are a duplicate of other, official docs, on the site", - "slug": "duplicate", + "description": "These docs are a duplicate of other, official docs, on the site", + "slug": "duplicate", "title": "Duplicate" } } diff --git a/readthedocs/core/forms.py b/readthedocs/core/forms.py index 34ebfbd0d2e..bc062286547 100644 --- a/readthedocs/core/forms.py +++ b/readthedocs/core/forms.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- - """Forms for core app.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging +from builtins import object from django import forms from django.contrib.auth.models import User @@ -11,7 +14,6 @@ from .models import UserProfile - log = logging.getLogger(__name__) @@ -19,13 +21,13 @@ class UserProfileForm(forms.ModelForm): first_name = CharField(label=_('First name'), required=False, max_length=30) last_name = CharField(label=_('Last name'), required=False, max_length=30) - class Meta: + class Meta(object): model = UserProfile # Don't allow users edit someone else's user page fields = ['first_name', 'last_name', 'homepage'] def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(UserProfileForm, self).__init__(*args, **kwargs) try: self.fields['first_name'].initial = self.instance.user.first_name self.fields['last_name'].initial = self.instance.user.last_name @@ -35,7 +37,7 @@ def __init__(self, *args, **kwargs): def save(self, commit=True): first_name = self.cleaned_data.pop('first_name', None) last_name = self.cleaned_data.pop('last_name', None) - profile = super().save(commit=commit) + profile = super(UserProfileForm, self).save(commit=commit) if commit: user = profile.user user.first_name = first_name @@ -50,7 +52,7 @@ class UserDeleteForm(forms.ModelForm): help_text=_('Please type your username to confirm.'), ) - class Meta: + class Meta(object): model = User fields = ['username'] @@ -64,8 +66,7 @@ def clean_username(self): class UserAdvertisingForm(forms.ModelForm): - - class Meta: + class Meta(object): model = UserProfile fields = ['allow_ads'] diff --git a/readthedocs/core/management/commands/archive.py b/readthedocs/core/management/commands/archive.py index 037b7319883..33a35bc56c3 100644 --- a/readthedocs/core/management/commands/archive.py +++ b/readthedocs/core/management/commands/archive.py @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- +"""Rebuild documentation for all projects""" -"""Rebuild documentation for all projects.""" - -import logging -import os +from __future__ import absolute_import +from __future__ import print_function from glob import glob +import os +import logging from django.conf import settings from django.core.management.base import BaseCommand from django.template import loader as template_loader - log = logging.getLogger(__name__) @@ -22,10 +21,10 @@ def handle(self, *args, **options): doc_index = {} os.chdir(settings.DOCROOT) - for directory in glob('*'): + for directory in glob("*"): doc_index[directory] = [] path = os.path.join(directory, 'rtd-builds') - for version in glob(os.path.join(path, '*')): + for version in glob(os.path.join(path, "*")): v = version.replace(path + '/', '') doc_index[directory].append(v) @@ -33,7 +32,5 @@ def handle(self, *args, **options): 'doc_index': doc_index, 'MEDIA_URL': settings.MEDIA_URL, } - html = template_loader.get_template( - 'archive/index.html', - ).render(context) + html = template_loader.get_template('archive/index.html').render(context) print(html) diff --git a/readthedocs/core/management/commands/clean_builds.py b/readthedocs/core/management/commands/clean_builds.py index d46b8cd1089..e47a651cf8a 100644 --- a/readthedocs/core/management/commands/clean_builds.py +++ b/readthedocs/core/management/commands/clean_builds.py @@ -1,9 +1,9 @@ -# -*- coding: utf-8 -*- +"""Clean up stable build paths per project version""" -"""Clean up stable build paths per project version.""" - -import logging +from __future__ import absolute_import from datetime import timedelta +import logging +from optparse import make_option from django.core.management.base import BaseCommand from django.db.models import Max @@ -11,7 +11,6 @@ from readthedocs.builds.models import Build, Version - log = logging.getLogger(__name__) @@ -25,24 +24,24 @@ def add_arguments(self, parser): dest='days', type='int', default=365, - help='Find builds older than DAYS days, default: 365', + help='Find builds older than DAYS days, default: 365' ) parser.add_argument( '--dryrun', action='store_true', dest='dryrun', - help='Perform dry run on build cleanup', + help='Perform dry run on build cleanup' ) def handle(self, *args, **options): - """Find stale builds and remove build paths.""" + """Find stale builds and remove build paths""" max_date = timezone.now() - timedelta(days=options['days']) - queryset = ( - Build.objects.values('project', 'version').annotate( - max_date=Max('date'), - ).filter(max_date__lt=max_date).order_by('-max_date') - ) + queryset = (Build.objects + .values('project', 'version') + .annotate(max_date=Max('date')) + .filter(max_date__lt=max_date) + .order_by('-max_date')) for build in queryset: try: # Get version from build version id, perform sanity check on diff --git a/readthedocs/core/management/commands/import_github.py b/readthedocs/core/management/commands/import_github.py index 90ce69e9e6a..3bb34b1b4ad 100644 --- a/readthedocs/core/management/commands/import_github.py +++ b/readthedocs/core/management/commands/import_github.py @@ -1,9 +1,8 @@ -# -*- coding: utf-8 -*- +"""Resync GitHub project for user""" -"""Resync GitHub project for user.""" - -from django.contrib.auth.models import User +from __future__ import absolute_import from django.core.management.base import BaseCommand +from django.contrib.auth.models import User from readthedocs.oauth.services import GitHubService @@ -16,8 +15,6 @@ def handle(self, *args, **options): if args: for slug in args: for service in GitHubService.for_user( - User.objects.get( - username=slug, - ), + User.objects.get(username=slug) ): service.sync() diff --git a/readthedocs/core/management/commands/import_github_language.py b/readthedocs/core/management/commands/import_github_language.py index bf99ac265c0..ef2945ea557 100644 --- a/readthedocs/core/management/commands/import_github_language.py +++ b/readthedocs/core/management/commands/import_github_language.py @@ -1,7 +1,5 @@ -# -*- coding: utf-8 -*- - """ -Import a project's programming language from GitHub. +Import a project's programming language from GitHub This builds a basic management command that will set a projects language to the most used one in GitHub. @@ -10,15 +8,16 @@ which should contain a proper GitHub Oauth Token for rate limiting. """ +from __future__ import absolute_import +from __future__ import print_function import os - import requests -from django.core.cache import cache + from django.core.management.base import BaseCommand +from django.core.cache import cache -from readthedocs.projects.constants import GITHUB_REGEXS, PROGRAMMING_LANGUAGES from readthedocs.projects.models import Project - +from readthedocs.projects.constants import GITHUB_REGEXS, PROGRAMMING_LANGUAGES PL_DICT = {} @@ -37,7 +36,11 @@ def handle(self, *args, **options): print('Invalid GitHub token, exiting') return - for project in Project.objects.filter(programming_language__in=['none', '', 'words']).filter(repo__contains='github'): # noqa + for project in Project.objects.filter( + programming_language__in=['none', '', 'words'] + ).filter( + repo__contains='github' + ): user = repo = '' repo_url = project.repo for regex in GITHUB_REGEXS: @@ -50,7 +53,7 @@ def handle(self, *args, **options): print('No GitHub repo for %s' % repo_url) continue - cache_key = '{}-{}'.format(user, repo) + cache_key = '%s-%s' % (user, repo) top_lang = cache.get(cache_key, None) if not top_lang: url = 'https://api.github.com/repos/{user}/{repo}/languages'.format( @@ -63,21 +66,15 @@ def handle(self, *args, **options): languages = resp.json() if not languages: continue - sorted_langs = sorted( - list(languages.items()), - key=lambda x: x[1], - reverse=True, - ) + sorted_langs = sorted(list(languages.items()), key=lambda x: x[1], reverse=True) print('Sorted langs: %s ' % sorted_langs) top_lang = sorted_langs[0][0] else: print('Cached top_lang: %s' % top_lang) if top_lang in PL_DICT: slug = PL_DICT[top_lang] - print('Setting {} to {}'.format(repo_url, slug)) - Project.objects.filter( - pk=project.pk, - ).update(programming_language=slug) + print('Setting %s to %s' % (repo_url, slug)) + Project.objects.filter(pk=project.pk).update(programming_language=slug) else: print('Language unknown: %s' % top_lang) cache.set(cache_key, top_lang, 60 * 600) diff --git a/readthedocs/core/management/commands/provision_elasticsearch.py b/readthedocs/core/management/commands/provision_elasticsearch.py index b1efe643c61..9f29fa37a9e 100644 --- a/readthedocs/core/management/commands/provision_elasticsearch.py +++ b/readthedocs/core/management/commands/provision_elasticsearch.py @@ -1,18 +1,11 @@ -# -*- coding: utf-8 -*- - -"""Provision Elastic Search.""" +"""Provision Elastic Search""" +from __future__ import absolute_import import logging from django.core.management.base import BaseCommand -from readthedocs.search.indexes import ( - Index, - PageIndex, - ProjectIndex, - SectionIndex, -) - +from readthedocs.search.indexes import Index, PageIndex, ProjectIndex, SectionIndex log = logging.getLogger(__name__) @@ -22,19 +15,19 @@ class Command(BaseCommand): help = __doc__ def handle(self, *args, **options): - """Provision new ES instance.""" + """Provision new ES instance""" index = Index() index_name = index.timestamped_index() - log.info('Creating indexes..') + log.info("Creating indexes..") index.create_index(index_name) index.update_aliases(index_name) - log.info('Updating mappings..') + log.info("Updating mappings..") proj = ProjectIndex() proj.put_mapping() page = PageIndex() page.put_mapping() sec = SectionIndex() sec.put_mapping() - log.info('Done!') + log.info("Done!") diff --git a/readthedocs/core/management/commands/pull.py b/readthedocs/core/management/commands/pull.py index ae62f9da10c..3540a35c077 100644 --- a/readthedocs/core/management/commands/pull.py +++ b/readthedocs/core/management/commands/pull.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Trigger build for project slug.""" +"""Trigger build for project slug""" +from __future__ import absolute_import import logging from django.core.management.base import BaseCommand diff --git a/readthedocs/core/management/commands/reindex_elasticsearch.py b/readthedocs/core/management/commands/reindex_elasticsearch.py index bcc199a26c1..7a5f25a065a 100644 --- a/readthedocs/core/management/commands/reindex_elasticsearch.py +++ b/readthedocs/core/management/commands/reindex_elasticsearch.py @@ -1,17 +1,17 @@ -# -*- coding: utf-8 -*- - -"""Reindex Elastic Search indexes.""" +"""Reindex Elastic Search indexes""" +from __future__ import absolute_import import logging +from optparse import make_option +from django.core.management.base import BaseCommand +from django.core.management.base import CommandError from django.conf import settings -from django.core.management.base import BaseCommand, CommandError from readthedocs.builds.constants import LATEST from readthedocs.builds.models import Version from readthedocs.projects.tasks import update_search - log = logging.getLogger(__name__) @@ -24,11 +24,11 @@ def add_arguments(self, parser): '-p', dest='project', default='', - help='Project to index', + help='Project to index' ) def handle(self, *args, **options): - """Build/index all versions or a single project's version.""" + """Build/index all versions or a single project's version""" project = options['project'] queryset = Version.objects.all() @@ -37,14 +37,13 @@ def handle(self, *args, **options): queryset = queryset.filter(project__slug=project) if not queryset.exists(): raise CommandError( - 'No project with slug: {slug}'.format(slug=project), - ) - log.info('Building all versions for %s', project) + 'No project with slug: {slug}'.format(slug=project)) + log.info("Building all versions for %s", project) elif getattr(settings, 'INDEX_ONLY_LATEST', True): queryset = queryset.filter(slug=LATEST) for version in queryset: - log.info('Reindexing %s', version) + log.info("Reindexing %s", version) try: commit = version.project.vcs_repo(version.slug).commit except: # noqa @@ -53,10 +52,7 @@ def handle(self, *args, **options): commit = None try: - update_search( - version.pk, - commit, - delete_non_commit_files=False, - ) + update_search(version.pk, commit, + delete_non_commit_files=False) except Exception as e: log.exception('Reindex failed for %s, %s', version, e) diff --git a/readthedocs/core/management/commands/set_metadata.py b/readthedocs/core/management/commands/set_metadata.py index 62384fe0e0e..dbefcddbdd0 100644 --- a/readthedocs/core/management/commands/set_metadata.py +++ b/readthedocs/core/management/commands/set_metadata.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- - -"""Generate metadata for all projects.""" +"""Generate metadata for all projects""" +from __future__ import absolute_import import logging from django.core.management.base import BaseCommand -from readthedocs.core.utils import broadcast from readthedocs.projects import tasks from readthedocs.projects.models import Project - +from readthedocs.core.utils import broadcast log = logging.getLogger(__name__) @@ -21,12 +19,8 @@ class Command(BaseCommand): def handle(self, *args, **options): queryset = Project.objects.all() for p in queryset: - log.info('Generating metadata for %s', p) + log.info("Generating metadata for %s", p) try: - broadcast( - type='app', - task=tasks.update_static_metadata, - args=[p.pk], - ) + broadcast(type='app', task=tasks.update_static_metadata, args=[p.pk]) except Exception: log.exception('Build failed for %s', p) diff --git a/readthedocs/core/management/commands/symlink.py b/readthedocs/core/management/commands/symlink.py index 2d843b9baab..e5c039c622b 100644 --- a/readthedocs/core/management/commands/symlink.py +++ b/readthedocs/core/management/commands/symlink.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- - -"""Update symlinks for projects.""" +"""Update symlinks for projects""" +from __future__ import absolute_import import logging from django.core.management.base import BaseCommand from readthedocs.projects import tasks -from readthedocs.projects.models import Project +from readthedocs.projects.models import Project log = logging.getLogger(__name__) @@ -25,9 +24,7 @@ def handle(self, *args, **options): if 'all' in projects: pks = Project.objects.values_list('pk', flat=True) else: - pks = Project.objects.filter( - slug__in=projects, - ).values_list('pk', flat=True) + pks = Project.objects.filter(slug__in=projects).values_list('pk', flat=True) for proj in pks: try: tasks.symlink_project(project_pk=proj) diff --git a/readthedocs/core/management/commands/update_api.py b/readthedocs/core/management/commands/update_api.py index 68b695c4d52..e95b23b1899 100644 --- a/readthedocs/core/management/commands/update_api.py +++ b/readthedocs/core/management/commands/update_api.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Build documentation using the API and not hitting a database. @@ -8,6 +6,7 @@ ./manage.py update_api """ +from __future__ import absolute_import import logging from django.core.management.base import BaseCommand @@ -33,6 +32,6 @@ def handle(self, *args, **options): for slug in options['projects']: project_data = api.project(slug).get() p = APIProject(**project_data) - log.info('Building %s', p) + log.info("Building %s", p) # pylint: disable=no-value-for-parameter tasks.update_docs_task(p.pk, docker=docker) diff --git a/readthedocs/core/management/commands/update_repos.py b/readthedocs/core/management/commands/update_repos.py index 17d0951909e..b5233d2d6df 100644 --- a/readthedocs/core/management/commands/update_repos.py +++ b/readthedocs/core/management/commands/update_repos.py @@ -6,6 +6,9 @@ Invoked via ``./manage.py update_repos``. """ +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging from django.core.management.base import BaseCommand @@ -15,7 +18,6 @@ from readthedocs.projects import tasks from readthedocs.projects.models import Project - log = logging.getLogger(__name__) diff --git a/readthedocs/core/management/commands/update_versions.py b/readthedocs/core/management/commands/update_versions.py index aacb47635a1..8961d6706cf 100644 --- a/readthedocs/core/management/commands/update_versions.py +++ b/readthedocs/core/management/commands/update_versions.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Rebuild documentation for all projects.""" +"""Rebuild documentation for all projects""" +from __future__ import absolute_import from django.core.management.base import BaseCommand from readthedocs.builds.models import Version @@ -18,5 +17,5 @@ def handle(self, *args, **options): update_docs_task( version.project_id, record=False, - version_pk=version.pk, + version_pk=version.pk ) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 6fc5bbe55c8..3ccd22d6864 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -1,34 +1,37 @@ -# -*- coding: utf-8 -*- - """Middleware for core app.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging from django.conf import settings from django.contrib.sessions.middleware import SessionMiddleware -from django.core.cache import cache from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.http import Http404, HttpResponseBadRequest from django.urls.base import set_urlconf from django.utils.deprecation import MiddlewareMixin from django.utils.translation import ugettext_lazy as _ -from readthedocs.core.utils import cname_to_slug from readthedocs.projects.models import Domain, Project log = logging.getLogger(__name__) -LOG_TEMPLATE = '(Middleware) {msg} [{host}{path}]' +LOG_TEMPLATE = u"(Middleware) {msg} [{host}{path}]" SUBDOMAIN_URLCONF = getattr( settings, 'SUBDOMAIN_URLCONF', - 'readthedocs.core.urls.subdomain', + 'readthedocs.core.urls.subdomain' ) SINGLE_VERSION_URLCONF = getattr( settings, 'SINGLE_VERSION_URLCONF', - 'readthedocs.core.urls.single_version', + 'readthedocs.core.urls.single_version' ) @@ -54,7 +57,7 @@ def process_request(self, request): production_domain = getattr( settings, 'PRODUCTION_DOMAIN', - 'readthedocs.org', + 'readthedocs.org' ) if public_domain is None: @@ -67,8 +70,9 @@ def process_request(self, request): if len(domain_parts) == len(public_domain.split('.')) + 1: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - if not is_www and ( # Support ports during local dev - public_domain in host or public_domain in full_host + if not is_www and ( + # Support ports during local dev + public_domain in host or public_domain in full_host ): if not Project.objects.filter(slug=subdomain).exists(): raise Http404(_('Project not found')) @@ -78,10 +82,10 @@ def process_request(self, request): return None # Serve CNAMEs - if ( - public_domain not in host and production_domain not in host and - 'localhost' not in host and 'testserver' not in host - ): + if (public_domain not in host and + production_domain not in host and + 'localhost' not in host and + 'testserver' not in host): request.cname = True domains = Domain.objects.filter(domain=host) if domains.count(): @@ -90,68 +94,35 @@ def process_request(self, request): request.slug = domain.project.slug request.urlconf = SUBDOMAIN_URLCONF request.domain_object = True - log.debug( - LOG_TEMPLATE.format( - msg='Domain Object Detected: %s' % domain.domain, - **log_kwargs - ), - ) + log.debug(LOG_TEMPLATE.format( + msg='Domain Object Detected: %s' % domain.domain, + **log_kwargs)) break - if ( - not hasattr(request, 'domain_object') and - 'HTTP_X_RTD_SLUG' in request.META - ): + if (not hasattr(request, 'domain_object') and + 'HTTP_X_RTD_SLUG' in request.META): request.slug = request.META['HTTP_X_RTD_SLUG'].lower() request.urlconf = SUBDOMAIN_URLCONF request.rtdheader = True - log.debug( - LOG_TEMPLATE.format( - msg='X-RTD-Slug header detected: %s' % request.slug, - **log_kwargs - ), - ) + log.debug(LOG_TEMPLATE.format( + msg='X-RTD-Slug header detected: %s' % request.slug, + **log_kwargs)) # Try header first, then DNS elif not hasattr(request, 'domain_object'): - try: - slug = cache.get(host) - if not slug: - slug = cname_to_slug(host) - cache.set(host, slug, 60 * 60) - # Cache the slug -> host mapping permanently. - log.info( - LOG_TEMPLATE.format( - msg='CNAME cached: {}->{}'.format(slug, host), - **log_kwargs - ), - ) - request.slug = slug - request.urlconf = SUBDOMAIN_URLCONF - log.warning( - LOG_TEMPLATE.format( - msg='CNAME detected: %s' % request.slug, - **log_kwargs - ), - ) - except: # noqa - # Some crazy person is CNAMEing to us. 404. - log.warning( - LOG_TEMPLATE.format(msg='CNAME 404', **log_kwargs), - ) - raise Http404(_('Invalid hostname')) + # Some person is CNAMEing to us. 404. + log.warning(LOG_TEMPLATE.format(msg='CNAME 404', **log_kwargs)) + raise Http404(_('Invalid hostname')) # Google was finding crazy www.blah.readthedocs.org domains. # Block these explicitly after trying CNAME logic. if len(domain_parts) > 3 and not settings.DEBUG: # Stop www.fooo.readthedocs.org if domain_parts[0] == 'www': - log.debug( - LOG_TEMPLATE.format(msg='404ing long domain', **log_kwargs), - ) + log.debug(LOG_TEMPLATE.format( + msg='404ing long domain', **log_kwargs + )) return HttpResponseBadRequest(_('Invalid hostname')) - log.debug( - LOG_TEMPLATE - .format(msg='Allowing long domain name', **log_kwargs), - ) - # raise Http404(_('Invalid hostname')) + log.debug(LOG_TEMPLATE.format( + msg='Allowing long domain name', **log_kwargs + )) # Normal request. return None @@ -208,9 +179,8 @@ def process_request(self, request): host = request.get_host() path = request.get_full_path() log_kwargs = dict(host=host, path=path) - log.debug( - LOG_TEMPLATE. - format(msg='Handling single_version request', **log_kwargs), + log.debug(LOG_TEMPLATE.format( + msg='Handling single_version request', **log_kwargs) ) return None @@ -240,7 +210,7 @@ def process_request(self, request): else: # HTTP_X_FORWARDED_FOR can be a comma-separated list of IPs. The # client's IP will be the first one. - real_ip = real_ip.split(',')[0].strip() + real_ip = real_ip.split(",")[0].strip() request.META['REMOTE_ADDR'] = real_ip @@ -252,26 +222,20 @@ class FooterNoSessionMiddleware(SessionMiddleware): This will reduce the size of our session table drastically. """ - IGNORE_URLS = [ - '/api/v2/footer_html', '/sustainability/view', '/sustainability/click', - ] + IGNORE_URLS = ['/api/v2/footer_html', '/sustainability/view', '/sustainability/click'] def process_request(self, request): for url in self.IGNORE_URLS: - if ( - request.path_info.startswith(url) and - settings.SESSION_COOKIE_NAME not in request.COOKIES - ): + if (request.path_info.startswith(url) and + settings.SESSION_COOKIE_NAME not in request.COOKIES): # Hack request.session otherwise the Authentication middleware complains. request.session = {} return - super().process_request(request) + super(FooterNoSessionMiddleware, self).process_request(request) def process_response(self, request, response): for url in self.IGNORE_URLS: - if ( - request.path_info.startswith(url) and - settings.SESSION_COOKIE_NAME not in request.COOKIES - ): + if (request.path_info.startswith(url) and + settings.SESSION_COOKIE_NAME not in request.COOKIES): return response - return super().process_response(request, response) + return super(FooterNoSessionMiddleware, self).process_response(request, response) diff --git a/readthedocs/core/migrations/0001_initial.py b/readthedocs/core/migrations/0001_initial.py index bb5bde04285..e3d5b948469 100644 --- a/readthedocs/core/migrations/0001_initial.py +++ b/readthedocs/core/migrations/0001_initial.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations from django.conf import settings -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/core/migrations/0002_make_userprofile_user_a_onetoonefield.py b/readthedocs/core/migrations/0002_make_userprofile_user_a_onetoonefield.py index 5f7d04ff910..f5e6255cc6b 100644 --- a/readthedocs/core/migrations/0002_make_userprofile_user_a_onetoonefield.py +++ b/readthedocs/core/migrations/0002_make_userprofile_user_a_onetoonefield.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations from django.conf import settings -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/core/migrations/0003_add_banned_status.py b/readthedocs/core/migrations/0003_add_banned_status.py index 95d26eefc42..f3dfbb8b777 100644 --- a/readthedocs/core/migrations/0003_add_banned_status.py +++ b/readthedocs/core/migrations/0003_add_banned_status.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/core/migrations/0004_ad-opt-out.py b/readthedocs/core/migrations/0004_ad-opt-out.py index f5d6ae3d029..9e8c5bf3209 100644 --- a/readthedocs/core/migrations/0004_ad-opt-out.py +++ b/readthedocs/core/migrations/0004_ad-opt-out.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-06-14 18:06 +from __future__ import unicode_literals + import annoying.fields -import django.db.models.deletion from django.conf import settings from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): diff --git a/readthedocs/core/migrations/0005_migrate-old-passwords.py b/readthedocs/core/migrations/0005_migrate-old-passwords.py index 8a44107c90b..2ef614d0db6 100644 --- a/readthedocs/core/migrations/0005_migrate-old-passwords.py +++ b/readthedocs/core/migrations/0005_migrate-old-passwords.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-11 17:28 -from django.contrib.auth.hashers import make_password +from __future__ import unicode_literals + from django.db import migrations +from django.contrib.auth.hashers import make_password def forwards_func(apps, schema_editor): diff --git a/readthedocs/core/mixins.py b/readthedocs/core/mixins.py index 4d6b1160c1c..7655db20bd4 100644 --- a/readthedocs/core/mixins.py +++ b/readthedocs/core/mixins.py @@ -1,24 +1,24 @@ -# -*- coding: utf-8 -*- - -"""Common mixin classes for views.""" +"""Common mixin classes for views""" +from __future__ import absolute_import +from builtins import object +from vanilla import ListView from django.contrib.auth.decorators import login_required from django.utils.decorators import method_decorator -from vanilla import ListView class ListViewWithForm(ListView): - """List view that also exposes a create form.""" + """List view that also exposes a create form""" def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(ListViewWithForm, self).get_context_data(**kwargs) context['form'] = self.get_form(data=None, files=None) return context -class LoginRequiredMixin: +class LoginRequiredMixin(object): @method_decorator(login_required) def dispatch(self, *args, **kwargs): - return super().dispatch(*args, **kwargs) + return super(LoginRequiredMixin, self).dispatch(*args, **kwargs) diff --git a/readthedocs/core/models.py b/readthedocs/core/models.py index bee057bdb6e..8b72a79199f 100644 --- a/readthedocs/core/models.py +++ b/readthedocs/core/models.py @@ -1,15 +1,16 @@ # -*- coding: utf-8 -*- - """Models for the core app.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging from annoying.fields import AutoOneToOneField from django.db import models -from django.urls import reverse from django.utils.encoding import python_2_unicode_compatible -from django.utils.translation import ugettext from django.utils.translation import ugettext_lazy as _ - +from django.utils.translation import ugettext +from django.urls import reverse STANDARD_EMAIL = 'anonymous@readthedocs.org' @@ -22,10 +23,7 @@ class UserProfile(models.Model): """Additional information about a User.""" user = AutoOneToOneField( - 'auth.User', - verbose_name=_('User'), - related_name='profile', - ) + 'auth.User', verbose_name=_('User'), related_name='profile') whitelisted = models.BooleanField(_('Whitelisted'), default=False) banned = models.BooleanField(_('Banned'), default=False) homepage = models.CharField(_('Homepage'), max_length=100, blank=True) @@ -43,14 +41,10 @@ class UserProfile(models.Model): def __str__(self): return ( ugettext("%(username)s's profile") % - {'username': self.user.username} - ) + {'username': self.user.username}) def get_absolute_url(self): - return reverse( - 'profiles_profile_detail', - kwargs={'username': self.user.username}, - ) + return reverse('profiles_profile_detail', kwargs={'username': self.user.username}) def get_contribution_details(self): """ diff --git a/readthedocs/core/permissions.py b/readthedocs/core/permissions.py index 1c397eae11c..c8a7fe6821b 100644 --- a/readthedocs/core/permissions.py +++ b/readthedocs/core/permissions.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- +"""Objects for User permission checks""" -"""Objects for User permission checks.""" +from __future__ import absolute_import from readthedocs.core.utils.extend import SettingsOverrideObject -class AdminPermissionBase: +class AdminPermissionBase(object): @classmethod def is_admin(cls, user, project): diff --git a/readthedocs/core/resolver.py b/readthedocs/core/resolver.py index 7758226cc2f..43fce68b3f6 100644 --- a/readthedocs/core/resolver.py +++ b/readthedocs/core/resolver.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- - """URL resolver for documentation.""" +from __future__ import absolute_import +from builtins import object import re from django.conf import settings -from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.projects.constants import PRIVATE, PUBLIC +from readthedocs.core.utils.extend import SettingsOverrideObject -class ResolverBase: +class ResolverBase(object): """ Read the Docs URL Resolver. @@ -51,55 +51,35 @@ class ResolverBase: /docs//projects// """ - def base_resolve_path( - self, - project_slug, - filename, - version_slug=None, - language=None, - private=False, - single_version=None, - subproject_slug=None, - subdomain=None, - cname=None, - ): + def base_resolve_path(self, project_slug, filename, version_slug=None, + language=None, private=False, single_version=None, + subproject_slug=None, subdomain=None, cname=None): """Resolve a with nothing smart, just filling in the blanks.""" # Only support `/docs/project' URLs outside our normal environment. Normally # the path should always have a subdomain or CNAME domain # pylint: disable=unused-argument if subdomain or cname or (self._use_subdomain()): - url = '/' + url = u'/' else: - url = '/docs/{project_slug}/' + url = u'/docs/{project_slug}/' if subproject_slug: - url += 'projects/{subproject_slug}/' + url += u'projects/{subproject_slug}/' if single_version: - url += '{filename}' + url += u'{filename}' else: - url += '{language}/{version_slug}/{filename}' + url += u'{language}/{version_slug}/{filename}' return url.format( - project_slug=project_slug, - filename=filename, - version_slug=version_slug, - language=language, - single_version=single_version, - subproject_slug=subproject_slug, + project_slug=project_slug, filename=filename, + version_slug=version_slug, language=language, + single_version=single_version, subproject_slug=subproject_slug, ) - def resolve_path( - self, - project, - filename='', - version_slug=None, - language=None, - single_version=None, - subdomain=None, - cname=None, - private=None, - ): + def resolve_path(self, project, filename='', version_slug=None, + language=None, single_version=None, subdomain=None, + cname=None, private=None): """Resolve a URL with a subset of fields defined.""" cname = cname or project.domains.filter(canonical=True).first() version_slug = version_slug or project.get_default_version() @@ -158,10 +138,8 @@ def resolve_domain(self, project, private=None): return getattr(settings, 'PRODUCTION_DOMAIN') - def resolve( - self, project, require_https=False, filename='', private=None, - **kwargs - ): + def resolve(self, project, require_https=False, filename='', private=None, + **kwargs): if private is None: version_slug = kwargs.get('version_slug') if version_slug is None: @@ -195,8 +173,8 @@ def resolve( return '{protocol}://{domain}{path}'.format( protocol=protocol, domain=domain, - path=self. - resolve_path(project, filename=filename, private=private, **kwargs), + path=self.resolve_path(project, filename=filename, private=private, + **kwargs), ) def _get_canonical_project(self, project, projects=None): @@ -234,7 +212,7 @@ def _get_project_subdomain(self, project): if self._use_subdomain(): project = self._get_canonical_project(project) subdomain_slug = project.slug.replace('_', '-') - return '{}.{}'.format(subdomain_slug, public_domain) + return "%s.%s" % (subdomain_slug, public_domain) def _get_project_custom_domain(self, project): return project.domains.filter(canonical=True).first() @@ -245,11 +223,7 @@ def _get_private(self, project, version_slug): version = project.versions.get(slug=version_slug) private = version.privacy_level == PRIVATE except Version.DoesNotExist: - private = getattr( - settings, - 'DEFAULT_PRIVACY_LEVEL', - PUBLIC, - ) == PRIVATE + private = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', PUBLIC) == PRIVATE return private def _fix_filename(self, project, filename): @@ -267,17 +241,17 @@ def _fix_filename(self, project, filename): if filename: if filename.endswith('/') or filename.endswith('.html'): path = filename - elif project.documentation_type == 'sphinx_singlehtml': - path = 'index.html#document-' + filename - elif project.documentation_type in ['sphinx_htmldir', 'mkdocs']: - path = filename + '/' + elif project.documentation_type == "sphinx_singlehtml": + path = "index.html#document-" + filename + elif project.documentation_type in ["sphinx_htmldir", "mkdocs"]: + path = filename + "/" elif '#' in filename: # do nothing if the filename contains URL fragments path = filename else: - path = filename + '.html' + path = filename + ".html" else: - path = '' + path = "" return path def _use_custom_domain(self, custom_domain): diff --git a/readthedocs/core/settings.py b/readthedocs/core/settings.py index 5967893cc0e..d66c6d02d28 100644 --- a/readthedocs/core/settings.py +++ b/readthedocs/core/settings.py @@ -1,10 +1,12 @@ """Class based settings for complex settings inheritance.""" +from __future__ import absolute_import +from builtins import object import inspect import sys -class Settings: +class Settings(object): """Class-based settings wrapper.""" diff --git a/readthedocs/core/signals.py b/readthedocs/core/signals.py index a4f029c6b3e..d8b90b57d82 100644 --- a/readthedocs/core/signals.py +++ b/readthedocs/core/signals.py @@ -2,19 +2,21 @@ """Signal handling for core app.""" +from __future__ import absolute_import + import logging -from urllib.parse import urlparse from corsheaders import signals from django.conf import settings -from django.db.models import Count, Q from django.db.models.signals import pre_delete -from django.dispatch import Signal, receiver +from django.dispatch import Signal +from django.db.models import Q, Count +from django.dispatch import receiver +from future.backports.urllib.parse import urlparse from rest_framework.permissions import SAFE_METHODS from readthedocs.oauth.models import RemoteOrganization -from readthedocs.projects.models import Domain, Project - +from readthedocs.projects.models import Project, Domain log = logging.getLogger(__name__) @@ -90,17 +92,15 @@ def delete_projects_and_organizations(sender, instance, *args, **kwargs): # https://github.com/rtfd/readthedocs.org/pull/4577 # https://docs.djangoproject.com/en/2.1/topics/db/aggregation/#order-of-annotate-and-filter-clauses # noqa projects = ( - Project.objects.annotate(num_users=Count('users') - ).filter(users=instance.id - ).exclude(num_users__gt=1) + Project.objects.annotate(num_users=Count('users')) + .filter(users=instance.id).exclude(num_users__gt=1) ) # Here we count the users list from the organization that the user belong # Then exclude the organizations where there are more than one user oauth_organizations = ( - RemoteOrganization.objects.annotate(num_users=Count('users') - ).filter(users=instance.id - ).exclude(num_users__gt=1) + RemoteOrganization.objects.annotate(num_users=Count('users')) + .filter(users=instance.id).exclude(num_users__gt=1) ) projects.delete() diff --git a/readthedocs/core/static.py b/readthedocs/core/static.py index 71d433b259c..89cd883877e 100644 --- a/readthedocs/core/static.py +++ b/readthedocs/core/static.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from django.contrib.staticfiles.finders import FileSystemFinder @@ -13,4 +15,4 @@ class SelectiveFileSystemFinder(FileSystemFinder): def list(self, ignore_patterns): ignore_patterns.extend(['epub', 'pdf', 'htmlzip', 'json', 'man']) - return super().list(ignore_patterns) + return super(SelectiveFileSystemFinder, self).list(ignore_patterns) diff --git a/readthedocs/core/static/core/font/fontawesome-webfont.svg b/readthedocs/core/static/core/font/fontawesome-webfont.svg index 52c0773359b..855c845e538 100644 --- a/readthedocs/core/static/core/font/fontawesome-webfont.svg +++ b/readthedocs/core/static/core/font/fontawesome-webfont.svg @@ -8,7 +8,7 @@ Copyright Dave Gandy 2016. All rights reserved. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/readthedocs/core/symlink.py b/readthedocs/core/symlink.py index 6bd19f5dd28..ec110cf6a15 100644 --- a/readthedocs/core/symlink.py +++ b/readthedocs/core/symlink.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ A class that manages the symlinks for nginx to serve public files. @@ -54,11 +52,19 @@ fabric -> rtd-builds/fabric/en/latest/ # single version """ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import os import shutil from collections import OrderedDict +from builtins import object from django.conf import settings from readthedocs.builds.models import Version @@ -68,23 +74,20 @@ from readthedocs.projects import constants from readthedocs.projects.models import Domain - log = logging.getLogger(__name__) -class Symlink: +class Symlink(object): """Base class for symlinking of projects.""" def __init__(self, project): self.project = project self.project_root = os.path.join( - self.WEB_ROOT, - project.slug, + self.WEB_ROOT, project.slug ) self.subproject_root = os.path.join( - self.project_root, - 'projects', + self.project_root, 'projects' ) self.environment = LocalEnvironment(project) self.sanity_check() @@ -96,13 +99,9 @@ def sanity_check(self): This will leave it in the proper state for the single_project setting. """ if os.path.islink(self.project_root) and not self.project.single_version: - log.info( - constants.LOG_TEMPLATE.format( - project=self.project.slug, - version='', - msg='Removing single version symlink', - ), - ) + log.info(constants.LOG_TEMPLATE.format( + project=self.project.slug, version='', + msg="Removing single version symlink")) safe_unlink(self.project_root) safe_makedirs(self.project_root) elif (self.project.single_version and @@ -155,15 +154,13 @@ def symlink_cnames(self, domain=None): domains = Domain.objects.filter(project=self.project) for dom in domains: log_msg = 'Symlinking CNAME: {} -> {}'.format( - dom.domain, - self.project.slug, + dom.domain, self.project.slug ) log.info( constants.LOG_TEMPLATE.format( project=self.project.slug, - version='', - msg=log_msg, - ), + version='', msg=log_msg + ) ) # CNAME to doc root @@ -172,26 +169,17 @@ def symlink_cnames(self, domain=None): # Project symlink project_cname_symlink = os.path.join( - self.PROJECT_CNAME_ROOT, - dom.domain, + self.PROJECT_CNAME_ROOT, dom.domain ) self.environment.run( - 'ln', - '-nsf', - self.project.doc_path, - project_cname_symlink, + 'ln', '-nsf', self.project.doc_path, project_cname_symlink ) def remove_symlink_cname(self, domain): """Remove CNAME symlink.""" - log_msg = 'Removing symlink for CNAME {}'.format(domain.domain) - log.info( - constants.LOG_TEMPLATE.format( - project=self.project.slug, - version='', - msg=log_msg, - ), - ) + log_msg = "Removing symlink for CNAME {0}".format(domain.domain) + log.info(constants.LOG_TEMPLATE.format(project=self.project.slug, + version='', msg=log_msg)) symlink = os.path.join(self.CNAME_ROOT, domain.domain) safe_unlink(symlink) @@ -199,7 +187,8 @@ def symlink_subprojects(self): """ Symlink project subprojects. - Link from $WEB_ROOT/projects/ -> $WEB_ROOT/ + Link from $WEB_ROOT/projects/ -> + $WEB_ROOT/ """ subprojects = set() rels = self.get_subprojects() @@ -216,21 +205,12 @@ def symlink_subprojects(self): from_to[rel.alias] = rel.child.slug subprojects.add(rel.alias) for from_slug, to_slug in list(from_to.items()): - log_msg = 'Symlinking subproject: {} -> {}'.format( - from_slug, - to_slug, - ) - log.info( - constants.LOG_TEMPLATE.format( - project=self.project.slug, - version='', - msg=log_msg, - ), - ) + log_msg = "Symlinking subproject: {0} -> {1}".format(from_slug, to_slug) + log.info(constants.LOG_TEMPLATE.format(project=self.project.slug, + version='', msg=log_msg)) symlink = os.path.join(self.subproject_root, from_slug) docs_dir = os.path.join( - self.WEB_ROOT, - to_slug, + self.WEB_ROOT, to_slug ) symlink_dir = os.sep.join(symlink.split(os.path.sep)[:-1]) if not os.path.lexists(symlink_dir): @@ -242,8 +222,7 @@ def symlink_subprojects(self): if result.exit_code > 0: log.error( 'Could not symlink path: status=%d error=%s', - result.exit_code, - result.error, + result.exit_code, result.error ) # Remove old symlinks @@ -257,7 +236,7 @@ def symlink_translations(self): Symlink project translations. Link from $WEB_ROOT/// -> - $WEB_ROOT/// + $WEB_ROOT/// """ translations = {} @@ -277,9 +256,8 @@ def symlink_translations(self): log.info( constants.LOG_TEMPLATE.format( project=self.project.slug, - version='', - msg=log_msg, - ), + version='', msg=log_msg + ) ) symlink = os.path.join(self.project_root, language) docs_dir = os.path.join(self.WEB_ROOT, slug, language) @@ -299,9 +277,8 @@ def symlink_single_version(self): """ Symlink project single version. - Link from: - - $WEB_ROOT/ -> HOME/user_builds//rtd-builds/latest/ + Link from $WEB_ROOT/ -> + HOME/user_builds//rtd-builds/latest/ """ version = self.get_default_version() @@ -318,7 +295,7 @@ def symlink_single_version(self): settings.DOCROOT, self.project.slug, 'rtd-builds', - version.slug, + version.slug ) self.environment.run('ln', '-nsf', docs_dir, symlink) @@ -327,13 +304,11 @@ def symlink_versions(self): Symlink project's versions. Link from $WEB_ROOT//// -> - HOME/user_builds//rtd-builds/ + HOME/user_builds//rtd-builds/ """ versions = set() version_dir = os.path.join( - self.WEB_ROOT, - self.project.slug, - self.project.language, + self.WEB_ROOT, self.project.slug, self.project.language ) # Include active public versions, # as well as public versions that are built but not active, for archived versions @@ -347,15 +322,15 @@ def symlink_versions(self): constants.LOG_TEMPLATE.format( project=self.project.slug, version='', - msg=log_msg, - ), + msg=log_msg + ) ) symlink = os.path.join(version_dir, version.slug) docs_dir = os.path.join( settings.DOCROOT, self.project.slug, 'rtd-builds', - version.slug, + version.slug ) self.environment.run('ln', '-nsf', docs_dir, symlink) versions.add(version.slug) @@ -378,18 +353,11 @@ def get_default_version(self): class PublicSymlinkBase(Symlink): CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_root') WEB_ROOT = os.path.join(settings.SITE_ROOT, 'public_web_root') - PROJECT_CNAME_ROOT = os.path.join( - settings.SITE_ROOT, - 'public_cname_project', - ) + PROJECT_CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'public_cname_project') def get_version_queryset(self): - return ( - self.project.versions.protected( - only_active=False, - ).filter(built=True) | - self.project.versions.protected(only_active=True) - ) + return (self.project.versions.protected(only_active=False).filter(built=True) | + self.project.versions.protected(only_active=True)) def get_subprojects(self): return self.project.subprojects.protected() @@ -401,16 +369,11 @@ def get_translations(self): class PrivateSymlinkBase(Symlink): CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'private_cname_root') WEB_ROOT = os.path.join(settings.SITE_ROOT, 'private_web_root') - PROJECT_CNAME_ROOT = os.path.join( - settings.SITE_ROOT, - 'private_cname_project', - ) + PROJECT_CNAME_ROOT = os.path.join(settings.SITE_ROOT, 'private_cname_project') def get_version_queryset(self): - return ( - self.project.versions.private(only_active=False).filter(built=True) | - self.project.versions.private(only_active=True) - ) + return (self.project.versions.private(only_active=False).filter(built=True) | + self.project.versions.private(only_active=True)) def get_subprojects(self): return self.project.subprojects.private() diff --git a/readthedocs/core/tasks.py b/readthedocs/core/tasks.py index 7c1b86f477f..8ed81f1bb1e 100644 --- a/readthedocs/core/tasks.py +++ b/readthedocs/core/tasks.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - """Basic tasks.""" +from __future__ import absolute_import import logging from django.conf import settings from django.core.mail import EmailMultiAlternatives -from django.template import TemplateDoesNotExist from django.template.loader import get_template +from django.template import TemplateDoesNotExist from django.utils import timezone from messages_extends.models import Message as PersistentMessage @@ -20,10 +19,8 @@ @app.task(queue='web', time_limit=EMAIL_TIME_LIMIT) -def send_email_task( - recipient, subject, template, template_html, context=None, - from_email=None, **kwargs -): +def send_email_task(recipient, subject, template, template_html, + context=None, from_email=None, **kwargs): """ Send multipart email. @@ -47,15 +44,14 @@ def send_email_task( """ msg = EmailMultiAlternatives( subject, - get_template(template).render(context), from_email or - settings.DEFAULT_FROM_EMAIL, - [recipient], **kwargs + get_template(template).render(context), + from_email or settings.DEFAULT_FROM_EMAIL, + [recipient], + **kwargs ) try: - msg.attach_alternative( - get_template(template_html).render(context), - 'text/html', - ) + msg.attach_alternative(get_template(template_html).render(context), + 'text/html') except TemplateDoesNotExist: pass msg.send() @@ -66,7 +62,5 @@ def send_email_task( def clear_persistent_messages(): # Delete all expired message_extend's messages log.info("Deleting all expired message_extend's messages") - expired_messages = PersistentMessage.objects.filter( - expires__lt=timezone.now(), - ) + expired_messages = PersistentMessage.objects.filter(expires__lt=timezone.now()) expired_messages.delete() diff --git a/readthedocs/core/templatetags/core_tags.py b/readthedocs/core/templatetags/core_tags.py index a94e511acf0..e91c1fb667e 100644 --- a/readthedocs/core/templatetags/core_tags.py +++ b/readthedocs/core/templatetags/core_tags.py @@ -1,14 +1,15 @@ -# -*- coding: utf-8 -*- - """Template tags for core app.""" +from __future__ import absolute_import + import hashlib -from urllib.parse import urlencode +from builtins import str # pylint: disable=redefined-builtin from django import template from django.conf import settings from django.utils.encoding import force_bytes, force_text from django.utils.safestring import mark_safe +from future.backports.urllib.parse import urlencode from readthedocs import __version__ from readthedocs.core.resolver import resolve @@ -21,25 +22,23 @@ @register.filter def gravatar(email, size=48): """ - Hacked from djangosnippets.org, but basically given an email address. + Hacked from djangosnippets.org, but basically given an email address render an img tag with the hashed up bits needed for leetness omgwtfstillreading """ - url = 'http://www.gravatar.com/avatar.php?%s' % urlencode({ + url = "http://www.gravatar.com/avatar.php?%s" % urlencode({ 'gravatar_id': hashlib.md5(email).hexdigest(), - 'size': str(size), + 'size': str(size) }) - return ( - 'gravatar' % (url, size, size) - ) + return ('gravatar' % (url, size, size)) -@register.simple_tag(name='doc_url') +@register.simple_tag(name="doc_url") def make_document_url(project, version=None, page=''): if not project: - return '' + return "" return resolve(project=project, version_slug=version, filename=page) @@ -52,7 +51,7 @@ def restructuredtext(value, short=False): if settings.DEBUG: raise template.TemplateSyntaxError( "Error in 'restructuredtext' filter: " - "The Python docutils library isn't installed.", + "The Python docutils library isn't installed." ) return force_text(value) else: @@ -60,22 +59,20 @@ def restructuredtext(value, short=False): 'raw_enabled': False, 'file_insertion_enabled': False, } - docutils_settings.update( - getattr(settings, 'RESTRUCTUREDTEXT_FILTER_SETTINGS', {}), - ) + docutils_settings.update(getattr(settings, 'RESTRUCTUREDTEXT_FILTER_SETTINGS', {})) try: parts = publish_parts( source=force_bytes(value), - writer_name='html4css1', + writer_name="html4css1", settings_overrides=docutils_settings, ) except ApplicationError: return force_text(value) - out = force_text(parts['fragment']) + out = force_text(parts["fragment"]) try: if short: - out = out.split('\n')[0] + out = out.split("\n")[0] except IndexError: pass return mark_safe(out) diff --git a/readthedocs/core/templatetags/privacy_tags.py b/readthedocs/core/templatetags/privacy_tags.py index 12d29f6624c..d18778778f6 100644 --- a/readthedocs/core/templatetags/privacy_tags.py +++ b/readthedocs/core/templatetags/privacy_tags.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- - """Template tags to query projects by privacy.""" +from __future__ import absolute_import + from django import template from readthedocs.core.permissions import AdminPermission @@ -18,9 +18,6 @@ def is_admin(user, project): @register.simple_tag(takes_context=True) def get_public_projects(context, user): - projects = Project.objects.for_user_and_viewer( - user=user, - viewer=context['request'].user, - ) + projects = Project.objects.for_user_and_viewer(user=user, viewer=context['request'].user) context['public_projects'] = projects return '' diff --git a/readthedocs/core/tests/test_signals.py b/readthedocs/core/tests/test_signals.py index 40dccdaa3a7..c38705a1f8c 100644 --- a/readthedocs/core/tests/test_signals.py +++ b/readthedocs/core/tests/test_signals.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- -import django_dynamic_fixture import pytest +import django_dynamic_fixture + from django.contrib.auth.models import User from readthedocs.oauth.models import RemoteOrganization @@ -8,13 +8,15 @@ @pytest.mark.django_db -class TestProjectOrganizationSignal: +class TestProjectOrganizationSignal(object): @pytest.mark.parametrize('model_class', [Project, RemoteOrganization]) def test_project_organization_get_deleted_upon_user_delete(self, model_class): - """If the user has Project or RemoteOrganization where he is the only - user, upon deleting his account, the Project or RemoteOrganization - should also get deleted.""" + """ + If the user has Project or RemoteOrganization where he is the only user, + upon deleting his account, the Project or RemoteOrganization should also get + deleted. + """ obj = django_dynamic_fixture.get(model_class) user1 = django_dynamic_fixture.get(User) @@ -31,8 +33,10 @@ def test_project_organization_get_deleted_upon_user_delete(self, model_class): @pytest.mark.parametrize('model_class', [Project, RemoteOrganization]) def test_multiple_users_project_organization_not_delete(self, model_class): - """Check Project or RemoteOrganization which have multiple users do not - get deleted when any of the user delete his account.""" + """ + Check Project or RemoteOrganization which have multiple users do not get deleted + when any of the user delete his account. + """ obj = django_dynamic_fixture.get(model_class) user1 = django_dynamic_fixture.get(User) diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py index 60e7fd32325..48b2e9a614a 100644 --- a/readthedocs/core/urls/__init__.py +++ b/readthedocs/core/urls/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """URL configuration for core app.""" from __future__ import absolute_import @@ -10,77 +8,53 @@ from readthedocs.core.views import hooks, serve from readthedocs.projects.feeds import LatestProjectsFeed, NewProjectsFeed + docs_urls = [ - url( - ( - r'^docs/(?P{project_slug})/page/' - r'(?P{filename_slug})$'.format(**pattern_opts) - ), + url((r'^docs/(?P{project_slug})/page/' + r'(?P{filename_slug})$'.format(**pattern_opts)), serve.redirect_page_with_filename, - name='docs_detail', - ), - url( - ( - r'^docs/(?P{project_slug})/' - r'(?:|projects/(?P{project_slug})/)$'.format( - **pattern_opts - ) - ), + name='docs_detail'), + + url((r'^docs/(?P{project_slug})/' + r'(?:|projects/(?P{project_slug})/)$'.format(**pattern_opts)), serve.redirect_project_slug, - name='docs_detail', - ), - url( - ( - r'^docs/(?P{project_slug})/' - r'(?:|projects/(?P{project_slug})/)' - r'(?P{lang_slug})/' - r'(?P{version_slug})/' - r'(?P{filename_slug})'.format(**pattern_opts) - ), + name='docs_detail'), + + url((r'^docs/(?P{project_slug})/' + r'(?:|projects/(?P{project_slug})/)' + r'(?P{lang_slug})/' + r'(?P{version_slug})/' + r'(?P{filename_slug})'.format(**pattern_opts)), serve.serve_docs, - name='docs_detail', - ), + name='docs_detail'), ] + core_urls = [ # Hooks url(r'^github', hooks.github_build, name='github_build'), url(r'^gitlab', hooks.gitlab_build, name='gitlab_build'), url(r'^bitbucket', hooks.bitbucket_build, name='bitbucket_build'), - url( - ( - r'^build/' - r'(?P{project_slug})'.format(**pattern_opts) - ), + url((r'^build/' + r'(?P{project_slug})'.format(**pattern_opts)), hooks.generic_build, - name='generic_build', - ), + name='generic_build'), # Random other stuff - url( - r'^random/(?P{project_slug})'.format(**pattern_opts), + url(r'^random/(?P{project_slug})'.format(**pattern_opts), views.random_page, - name='random_page', - ), + name='random_page'), url(r'^random/$', views.random_page, name='random_page'), - url( - ( - r'^wipe/(?P{project_slug})/' - r'(?P{version_slug})/$'.format(**pattern_opts) - ), + url((r'^wipe/(?P{project_slug})/' + r'(?P{version_slug})/$'.format(**pattern_opts)), views.wipe_version, - name='wipe_version', - ), + name='wipe_version'), ] deprecated_urls = [ - url( - r'^feeds/new/$', + url(r'^feeds/new/$', NewProjectsFeed(), - name='new_feed', - ), - url( - r'^feeds/latest/$', + name="new_feed"), + url(r'^feeds/latest/$', LatestProjectsFeed(), - name='latest_feed', - ), + name="latest_feed"), ] diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 253cae6a319..afc84bda83f 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -1,59 +1,47 @@ -# -*- coding: utf-8 -*- - """URL configuration for a single version.""" +from __future__ import absolute_import + from functools import reduce from operator import add -from django.conf import settings from django.conf.urls import url +from django.conf import settings from django.conf.urls.static import static from readthedocs.constants import pattern_opts from readthedocs.core.views import serve - handler500 = 'readthedocs.core.views.server_error_500' handler404 = 'readthedocs.core.views.server_error_404' single_version_urls = [ - url( - r'^(?:|projects/(?P{project_slug})/)' + url(r'^(?:|projects/(?P{project_slug})/)' r'page/(?P.*)$'.format(**pattern_opts), serve.redirect_page_with_filename, - name='docs_detail', - ), - url( - ( - r'^(?:|projects/(?P{project_slug})/)' - r'(?P{filename_slug})$'.format(**pattern_opts) - ), + name='docs_detail'), + + url((r'^(?:|projects/(?P{project_slug})/)' + r'(?P{filename_slug})$'.format(**pattern_opts)), serve.serve_docs, - name='docs_detail', - ), + name='docs_detail'), ] groups = [single_version_urls] # Needed to serve media locally if getattr(settings, 'DEBUG', False): - groups.insert( - 0, - static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT), - ) + groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) # Allow `/docs/` URL's when not using subdomains or during local dev if not getattr(settings, 'USE_SUBDOMAIN', False) or settings.DEBUG: docs_url = [ - url( - ( - r'^docs/(?P[-\w]+)/' - r'(?:|projects/(?P{project_slug})/)' - r'(?P{filename_slug})$'.format(**pattern_opts) - ), + url((r'^docs/(?P[-\w]+)/' + r'(?:|projects/(?P{project_slug})/)' + r'(?P{filename_slug})$'.format(**pattern_opts)), serve.serve_docs, - name='docs_detail', - ), + name='docs_detail') ] groups.insert(1, docs_url) + urlpatterns = reduce(add, groups) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index 4e4a6775e3a..826c6443660 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -1,63 +1,48 @@ -# -*- coding: utf-8 -*- - """URL configurations for subdomains.""" +from __future__ import absolute_import + from functools import reduce from operator import add -from django.conf import settings from django.conf.urls import url +from django.conf import settings from django.conf.urls.static import static -from readthedocs.constants import pattern_opts -from readthedocs.core.views import server_error_404, server_error_500 from readthedocs.core.views.serve import ( redirect_page_with_filename, - redirect_project_slug, - robots_txt, - serve_docs, + redirect_project_slug, serve_docs ) - +from readthedocs.core.views import ( + server_error_500, + server_error_404, +) +from readthedocs.constants import pattern_opts handler500 = server_error_500 handler404 = server_error_404 subdomain_urls = [ - url(r'robots.txt$', robots_txt, name='robots_txt'), - url( - r'^(?:|projects/(?P{project_slug})/)' + url(r'^(?:|projects/(?P{project_slug})/)' r'page/(?P.*)$'.format(**pattern_opts), redirect_page_with_filename, - name='docs_detail', - ), - url( - (r'^(?:|projects/(?P{project_slug})/)$').format( - **pattern_opts - ), + name='docs_detail'), + + url((r'^(?:|projects/(?P{project_slug})/)$').format(**pattern_opts), redirect_project_slug, - name='redirect_project_slug', - ), - url( - ( - r'^(?:|projects/(?P{project_slug})/)' - r'(?P{lang_slug})/' - r'(?P{version_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts) - ), + name='redirect_project_slug'), + + url((r'^(?:|projects/(?P{project_slug})/)' + r'(?P{lang_slug})/' + r'(?P{version_slug})/' + r'(?P{filename_slug})$'.format(**pattern_opts)), serve_docs, - name='docs_detail', - ), + name='docs_detail'), ] groups = [subdomain_urls] # Needed to serve media locally if getattr(settings, 'DEBUG', False): - groups.insert( - 0, - static( - settings.MEDIA_URL, - document_root=settings.MEDIA_ROOT, - ), - ) + groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) urlpatterns = reduce(add, groups) diff --git a/readthedocs/core/utils/__init__.py b/readthedocs/core/utils/__init__.py index 2123ce5a326..4f447486459 100644 --- a/readthedocs/core/utils/__init__.py +++ b/readthedocs/core/utils/__init__.py @@ -11,6 +11,7 @@ import re from django.conf import settings +from django.utils import six from django.utils.functional import allow_lazy from django.utils.safestring import SafeText, mark_safe from django.utils.text import slugify as slugify_base @@ -19,6 +20,7 @@ from readthedocs.builds.constants import LATEST, BUILD_STATE_TRIGGERED from readthedocs.doc_builder.constants import DOCKER_LIMITS + log = logging.getLogger(__name__) SYNC_USER = getattr(settings, 'SYNC_USER', getpass.getuser()) @@ -59,15 +61,6 @@ def broadcast(type, task, args, kwargs=None, callback=None): # pylint: disable= return task_promise -def cname_to_slug(host): - # TODO: remove - from dns import resolver - answer = [ans for ans in resolver.query(host, 'CNAME')][0] - domain = answer.target.to_unicode() - slug = domain.split('.')[0] - return slug - - def prepare_build( project, version=None, @@ -219,7 +212,7 @@ def slugify(value, *args, **kwargs): return value -slugify = allow_lazy(slugify, str, SafeText) +slugify = allow_lazy(slugify, six.text_type, SafeText) def safe_makedirs(directory_name): diff --git a/readthedocs/core/utils/extend.py b/readthedocs/core/utils/extend.py index a74b1175835..567c0c23a4d 100644 --- a/readthedocs/core/utils/extend.py +++ b/readthedocs/core/utils/extend.py @@ -1,11 +1,11 @@ -# -*- coding: utf-8 -*- - """Patterns for extending Read the Docs.""" +from __future__ import absolute_import import inspect from django.conf import settings from django.utils.module_loading import import_string +import six def get_override_class(proxy_class, default_class=None): @@ -21,7 +21,7 @@ def get_override_class(proxy_class, default_class=None): default_class = getattr(proxy_class, '_default_class') class_id = '.'.join([ inspect.getmodule(proxy_class).__name__, - proxy_class.__name__, + proxy_class.__name__ ]) class_path = getattr(settings, 'CLASS_OVERRIDES', {}).get(class_id) # pylint: disable=protected-access @@ -34,18 +34,14 @@ def get_override_class(proxy_class, default_class=None): class SettingsOverrideMeta(type): - """ - Meta class to manage our Setting configurations. - - Meta class for passing along classmethod class to the underlying class. - """ + """Meta class for passing along classmethod class to the underlying class.""" # noqa def __getattr__(cls, attr): # noqa: pep8 false positive proxy_class = get_override_class(cls, getattr(cls, '_default_class')) return getattr(proxy_class, attr) -class SettingsOverrideObject(metaclass=SettingsOverrideMeta): +class SettingsOverrideObject(six.with_metaclass(SettingsOverrideMeta, object)): """ Base class for creating class that can be overridden. diff --git a/readthedocs/core/utils/tasks/__init__.py b/readthedocs/core/utils/tasks/__init__.py index 7aede3ac1dc..344215036f9 100644 --- a/readthedocs/core/utils/tasks/__init__.py +++ b/readthedocs/core/utils/tasks/__init__.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- - -"""Common task exports.""" +"""Common task exports""" from .permission_checks import user_id_matches # noqa for unused import from .public import PublicTask # noqa diff --git a/readthedocs/core/utils/tasks/permission_checks.py b/readthedocs/core/utils/tasks/permission_checks.py index 84f545830e1..1643e866015 100644 --- a/readthedocs/core/utils/tasks/permission_checks.py +++ b/readthedocs/core/utils/tasks/permission_checks.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- - -"""Permission checks for tasks.""" +"""Permission checks for tasks""" __all__ = ('user_id_matches',) diff --git a/readthedocs/core/utils/tasks/public.py b/readthedocs/core/utils/tasks/public.py index 8b8f2421ac7..9fb2948ef71 100644 --- a/readthedocs/core/utils/tasks/public.py +++ b/readthedocs/core/utils/tasks/public.py @@ -1,19 +1,22 @@ -# -*- coding: utf-8 -*- +"""Celery tasks with publicly viewable status""" -"""Celery tasks with publicly viewable status.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) from celery import Task, states from django.conf import settings from .retrieve import TaskNotFound, get_task_data - __all__ = ( - 'PublicTask', - 'TaskNoPermission', - 'get_public_task_data', + 'PublicTask', 'TaskNoPermission', 'get_public_task_data' ) + STATUS_UPDATES_ENABLED = not getattr(settings, 'CELERY_ALWAYS_EAGER', False) @@ -48,7 +51,7 @@ def update_progress_data(self): def set_permission_context(self, context): """ - Set data that can be used by ``check_permission`` to authorize a. + Set data that can be used by ``check_permission`` to authorize a request for the this task. By default it will be the ``kwargs`` passed into the task. @@ -106,26 +109,22 @@ def permission_check(check): def my_public_task(user_id): pass """ - def decorator(func): func.check_permission = check return func - return decorator class TaskNoPermission(Exception): - def __init__(self, task_id, *args, **kwargs): message = 'No permission to access task with id {id}'.format( - id=task_id, - ) - super().__init__(message, *args, **kwargs) + id=task_id) + super(TaskNoPermission, self).__init__(message, *args, **kwargs) def get_public_task_data(request, task_id): """ - Return task details as tuple. + Return task details as tuple Will raise `TaskNoPermission` if `request` has no permission to access info of the task with id `task_id`. This is also the case of no task with the diff --git a/readthedocs/core/utils/tasks/retrieve.py b/readthedocs/core/utils/tasks/retrieve.py index 9281ad8a1af..c96b7823706 100644 --- a/readthedocs/core/utils/tasks/retrieve.py +++ b/readthedocs/core/utils/tasks/retrieve.py @@ -1,24 +1,27 @@ -# -*- coding: utf-8 -*- - """Utilities for retrieving task data.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from celery import states from celery.result import AsyncResult - __all__ = ('TaskNotFound', 'get_task_data') class TaskNotFound(Exception): - def __init__(self, task_id, *args, **kwargs): message = 'No public task found with id {id}'.format(id=task_id) - super().__init__(message, *args, **kwargs) + super(TaskNotFound, self).__init__(message, *args, **kwargs) def get_task_data(task_id): """ - Will raise `TaskNotFound` if the task is in state ``PENDING`` or the task. + Will raise `TaskNotFound` if the task is in state ``PENDING`` or the task meta data has no ``'task_name'`` key set. """ diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index fd33e5a9e37..6f25e9bd92b 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -19,7 +19,7 @@ from readthedocs.builds.models import Version from readthedocs.core.utils import broadcast from readthedocs.projects.models import Project, ImportedFile -from readthedocs.projects.tasks import remove_dirs +from readthedocs.projects.tasks import remove_dir from readthedocs.redirects.utils import get_redirect_response log = logging.getLogger(__name__) @@ -35,7 +35,7 @@ class HomepageView(TemplateView): def get_context_data(self, **kwargs): """Add latest builds and featured projects.""" - context = super().get_context_data(**kwargs) + context = super(HomepageView, self).get_context_data(**kwargs) context['featured_list'] = Project.objects.filter(featured=True) context['projects_count'] = Project.objects.count() return context @@ -45,7 +45,7 @@ class SupportView(TemplateView): template_name = 'support.html' def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(SupportView, self).get_context_data(**kwargs) support_email = getattr(settings, 'SUPPORT_EMAIL', None) if not support_email: support_email = 'support@{domain}'.format( @@ -89,7 +89,7 @@ def wipe_version(request, project_slug, version_slug): os.path.join(version.project.doc_path, 'conda', version.slug), ] for del_dir in del_dirs: - broadcast(type='build', task=remove_dirs, args=[(del_dir,)]) + broadcast(type='build', task=remove_dir, args=[del_dir]) return redirect('project_version_list', project_slug) return render( request, @@ -133,15 +133,13 @@ def do_not_track(request): dnt_header = request.META.get('HTTP_DNT') # https://w3c.github.io/dnt/drafts/tracking-dnt.html#status-representation - return JsonResponse( # pylint: disable=redundant-content-type-for-json-response - { - 'policy': 'https://docs.readthedocs.io/en/latest/privacy-policy.html', - 'same-party': [ - 'readthedocs.org', - 'readthedocs.com', - 'readthedocs.io', # .org Documentation Sites - 'readthedocs-hosted.com', # .com Documentation Sites - ], - 'tracking': 'N' if dnt_header == '1' else 'T', - }, content_type='application/tracking-status+json', - ) + return JsonResponse({ # pylint: disable=redundant-content-type-for-json-response + 'policy': 'https://docs.readthedocs.io/en/latest/privacy-policy.html', + 'same-party': [ + 'readthedocs.org', + 'readthedocs.com', + 'readthedocs.io', # .org Documentation Sites + 'readthedocs-hosted.com', # .com Documentation Sites + ], + 'tracking': 'N' if dnt_header == '1' else 'T', + }, content_type='application/tracking-status+json') diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 3966ba62b2a..6b00b2d77ef 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -1,7 +1,12 @@ -# -*- coding: utf-8 -*- - """Views pertaining to builds.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import logging import re @@ -16,7 +21,6 @@ from readthedocs.projects.models import Feature, Project from readthedocs.projects.tasks import sync_repository_task - log = logging.getLogger(__name__) @@ -43,14 +47,13 @@ def _build_version(project, slug, already_built=()): version = project.versions.filter(active=True, slug=slug).first() if version and slug not in already_built: log.info( - '(Version build) Building %s:%s', - project.slug, - version.slug, + "(Version build) Building %s:%s", + project.slug, version.slug, ) trigger_build(project=project, version=version, force=True) return slug - log.info('(Version build) Not Building %s', slug) + log.info("(Version build) Not Building %s", slug) return None @@ -67,11 +70,8 @@ def build_branches(project, branch_list): for branch in branch_list: versions = project.versions_from_branch_name(branch) for version in versions: - log.info( - '(Branch Build) Processing %s:%s', - project.slug, - version.slug, - ) + log.info("(Branch Build) Processing %s:%s", + project.slug, version.slug) ret = _build_version(project, version.slug, already_built=to_build) if ret: to_build.add(ret) @@ -95,9 +95,9 @@ def sync_versions(project): try: version_identifier = project.get_default_branch() version = ( - project.versions.filter( - identifier=version_identifier, - ).first() + project.versions + .filter(identifier=version_identifier) + .first() ) if not version: log.info('Unable to sync from %s version', version_identifier) @@ -120,13 +120,10 @@ def get_project_from_url(url): def log_info(project, msg): - log.info( - constants.LOG_TEMPLATE.format( - project=project, - version='', - msg=msg, - ), - ) + log.info(constants.LOG_TEMPLATE + .format(project=project, + version='', + msg=msg)) def _build_url(url, projects, branches): @@ -136,7 +133,7 @@ def _build_url(url, projects, branches): Check each of the ``branches`` to see if they are active and should be built. """ - ret = '' + ret = "" all_built = {} all_not_building = {} @@ -159,19 +156,15 @@ def _build_url(url, projects, branches): for project_slug, built in list(all_built.items()): if built: - msg = '(URL Build) Build Started: {} [{}]'.format( - url, - ' '.join(built), - ) + msg = '(URL Build) Build Started: %s [%s]' % ( + url, ' '.join(built)) log_info(project_slug, msg=msg) ret += msg for project_slug, not_building in list(all_not_building.items()): if not_building: - msg = '(URL Build) Not Building: {} [{}]'.format( - url, - ' '.join(not_building), - ) + msg = '(URL Build) Not Building: %s [%s]' % ( + url, ' '.join(not_building)) log_info(project_slug, msg=msg) ret += msg @@ -218,14 +211,14 @@ def github_build(request): # noqa: D205 log.info( 'GitHub webhook search: url=%s branches=%s', http_search_url, - branches, + branches ) ssh_projects = get_project_from_url(ssh_search_url) if ssh_projects: log.info( 'GitHub webhook search: url=%s branches=%s', ssh_search_url, - branches, + branches ) projects = repo_projects | ssh_projects return _build_url(http_search_url, projects, branches) @@ -300,24 +293,24 @@ def bitbucket_build(request): else: data = json.loads(request.body) - version = 2 if request.META.get('HTTP_USER_AGENT') == 'Bitbucket-Webhooks/2.0' else 1 # yapf: disabled # noqa + version = 2 if request.META.get('HTTP_USER_AGENT') == 'Bitbucket-Webhooks/2.0' else 1 if version == 1: - branches = [ - commit.get('branch', '') for commit in data['commits'] - ] + branches = [commit.get('branch', '') + for commit in data['commits']] repository = data['repository'] if not repository['absolute_url']: return HttpResponse('Invalid request', status=400) - search_url = 'bitbucket.org{}'.format( - repository['absolute_url'].rstrip('/'), + search_url = 'bitbucket.org{0}'.format( + repository['absolute_url'].rstrip('/') ) elif version == 2: changes = data['push']['changes'] - branches = [change['new']['name'] for change in changes] + branches = [change['new']['name'] + for change in changes] if not data['repository']['full_name']: return HttpResponse('Invalid request', status=400) - search_url = 'bitbucket.org/{}'.format( - data['repository']['full_name'], + search_url = 'bitbucket.org/{0}'.format( + data['repository']['full_name'] ) except (TypeError, ValueError, KeyError): log.exception('Invalid Bitbucket webhook payload') @@ -365,12 +358,10 @@ def generic_build(request, project_id_or_slug=None): project = Project.objects.get(slug=project_id_or_slug) except (Project.DoesNotExist, ValueError): log.exception( - '(Incoming Generic Build) Repo not found: %s', - project_id_or_slug, - ) + "(Incoming Generic Build) Repo not found: %s", + project_id_or_slug) return HttpResponseNotFound( - 'Repo not found: %s' % project_id_or_slug, - ) + 'Repo not found: %s' % project_id_or_slug) # This endpoint doesn't require authorization, we shouldn't allow builds to # be triggered from this any longer. Deprecation plan is to selectively # allow access to this endpoint for now. @@ -379,11 +370,8 @@ def generic_build(request, project_id_or_slug=None): if request.method == 'POST': slug = request.POST.get('version_slug', project.default_version) log.info( - '(Incoming Generic Build) %s [%s]', - project.slug, - slug, - ) + "(Incoming Generic Build) %s [%s]", project.slug, slug) _build_version(project, slug) else: - return HttpResponse('You must POST to this resource.') + return HttpResponse("You must POST to this resource.") return redirect('builds_project_list', project.slug) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index a1f65a11ea5..36510440ae3 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ Doc serving from Python. @@ -26,15 +25,18 @@ SERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve. """ +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging import mimetypes import os from functools import wraps from django.conf import settings -from django.http import Http404, HttpResponse, HttpResponseRedirect -from django.shortcuts import get_object_or_404, render -from django.utils.encoding import iri_to_uri +from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.shortcuts import get_object_or_404 +from django.shortcuts import render from django.views.static import serve from readthedocs.builds.models import Version @@ -44,7 +46,6 @@ from readthedocs.projects import constants from readthedocs.projects.models import Project, ProjectRelationship - log = logging.getLogger(__name__) @@ -56,11 +57,8 @@ def map_subproject_slug(view_func): .. warning:: Does not take into account any kind of privacy settings. """ - @wraps(view_func) - def inner_view( # noqa - request, subproject=None, subproject_slug=None, *args, **kwargs, - ): + def inner_view(request, subproject=None, subproject_slug=None, *args, **kwargs): # noqa if subproject is None and subproject_slug: # Try to fetch by subproject alias first, otherwise we might end up # redirected to an unrelated project. @@ -86,11 +84,8 @@ def map_project_slug(view_func): .. warning:: Does not take into account any kind of privacy settings. """ - @wraps(view_func) - def inner_view( # noqa - request, project=None, project_slug=None, *args, **kwargs - ): + def inner_view(request, project=None, project_slug=None, *args, **kwargs): # noqa if project is None: if not project_slug: project_slug = request.slug @@ -115,14 +110,13 @@ def redirect_project_slug(request, project, subproject): # pylint: disable=unus def redirect_page_with_filename(request, project, subproject, filename): # pylint: disable=unused-argument # noqa """Redirect /page/file.html to /en/latest/file.html.""" return HttpResponseRedirect( - resolve(subproject or project, filename=filename), - ) + resolve(subproject or project, filename=filename)) def _serve_401(request, project): res = render(request, '401.html') res.status_code = 401 - log.debug('Unauthorized access to {} documentation'.format(project.slug)) + log.debug('Unauthorized access to {0} documentation'.format(project.slug)) return res @@ -134,24 +128,16 @@ def _serve_file(request, filename, basepath): # Serve from Nginx content_type, encoding = mimetypes.guess_type( - os.path.join(basepath, filename), - ) + os.path.join(basepath, filename)) content_type = content_type or 'application/octet-stream' response = HttpResponse(content_type=content_type) if encoding: response['Content-Encoding'] = encoding try: - iri_path = os.path.join( + response['X-Accel-Redirect'] = os.path.join( basepath[len(settings.SITE_ROOT):], filename, ) - # NGINX does not support non-ASCII characters in the header, so we - # convert the IRI path to URI so it's compatible with what NGINX expects - # as the header value. - # https://github.com/benoitc/gunicorn/issues/1448 - # https://docs.djangoproject.com/en/1.11/ref/unicode/#uri-and-iri-handling - x_accel_redirect = iri_to_uri(iri_path) - response['X-Accel-Redirect'] = x_accel_redirect except UnicodeEncodeError: raise Http404 @@ -161,14 +147,9 @@ def _serve_file(request, filename, basepath): @map_project_slug @map_subproject_slug def serve_docs( - request, - project, - subproject, - lang_slug=None, - version_slug=None, - filename='', -): - """Map existing proj, lang, version, filename views to the file format.""" + request, project, subproject, lang_slug=None, version_slug=None, + filename=''): + """Exists to map existing proj, lang, version, filename views to the file format.""" if not version_slug: version_slug = project.get_default_version() try: @@ -233,51 +214,4 @@ def _serve_symlink_docs(request, project, privacy_level, filename=''): files_tried.append(os.path.join(basepath, filename)) raise Http404( - 'File not found. Tried these files: %s' % ','.join(files_tried), - ) - - -@map_project_slug -def robots_txt(request, project): - """ - Serve custom user's defined ``/robots.txt``. - - If the user added a ``robots.txt`` in the "default version" of the project, - we serve it directly. - """ - # Use the ``robots.txt`` file from the default version configured - version_slug = project.get_default_version() - version = project.versions.get(slug=version_slug) - - no_serve_robots_txt = any([ - # If project is private or, - project.privacy_level == constants.PRIVATE, - # default version is private or, - version.privacy_level == constants.PRIVATE, - # default version is not active or, - not version.active, - # default version is not built - not version.built, - ]) - if no_serve_robots_txt: - # ... we do return a 404 - raise Http404() - - filename = resolve_path( - project, - version_slug=version_slug, - filename='robots.txt', - subdomain=True, # subdomain will make it a "full" path without a URL prefix - ) - - # This breaks path joining, by ignoring the root when given an "absolute" path - if filename[0] == '/': - filename = filename[1:] - - basepath = PublicSymlink(project).project_root - fullpath = os.path.join(basepath, filename) - - if os.path.exists(fullpath): - return HttpResponse(open(fullpath).read(), content_type='text/plain') - - return HttpResponse('User-agent: *\nAllow: /\n', content_type='text/plain') + 'File not found. Tried these files: %s' % ','.join(files_tried)) diff --git a/readthedocs/doc_builder/backends/mkdocs.py b/readthedocs/doc_builder/backends/mkdocs.py index 201bd039368..6119673b8fa 100644 --- a/readthedocs/doc_builder/backends/mkdocs.py +++ b/readthedocs/doc_builder/backends/mkdocs.py @@ -1,10 +1,11 @@ -# -*- coding: utf-8 -*- - """ MkDocs_ backend for building docs. .. _MkDocs: http://www.mkdocs.org/ """ +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import json import logging import os @@ -17,7 +18,6 @@ from readthedocs.doc_builder.exceptions import MkDocsYAMLParseError from readthedocs.projects.models import Feature - log = logging.getLogger(__name__) @@ -44,11 +44,10 @@ class BaseMkdocs(BaseBuilder): DEFAULT_THEME_NAME = 'mkdocs' def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(BaseMkdocs, self).__init__(*args, **kwargs) self.old_artifact_path = os.path.join( self.version.project.checkout_path(self.version.slug), - self.build_dir, - ) + self.build_dir) self.root_path = self.version.project.checkout_path(self.version.slug) self.yaml_file = self.get_yaml_config() @@ -68,13 +67,14 @@ def __init__(self, *args, **kwargs): else: self.DEFAULT_THEME_NAME = 'mkdocs' + def get_yaml_config(self): """Find the ``mkdocs.yml`` file in the project root.""" mkdoc_path = self.config.mkdocs.configuration if not mkdoc_path: mkdoc_path = os.path.join( self.project.checkout_path(self.version.slug), - 'mkdocs.yml', + 'mkdocs.yml' ) if not os.path.exists(mkdoc_path): return None @@ -84,10 +84,12 @@ def load_yaml_config(self): """ Load a YAML config. - :raises: ``MkDocsYAMLParseError`` if failed due to syntax errors. + Raise BuildEnvironmentError if failed due to syntax errors. """ try: - return yaml.safe_load(open(self.yaml_file, 'r'),) + return yaml.safe_load( + open(self.yaml_file, 'r') + ) except IOError: return { 'site_name': self.version.project.name, @@ -96,22 +98,14 @@ def load_yaml_config(self): note = '' if hasattr(exc, 'problem_mark'): mark = exc.problem_mark - note = ' (line %d, column %d)' % ( - mark.line + 1, - mark.column + 1, - ) + note = ' (line %d, column %d)' % (mark.line + 1, mark.column + 1) raise MkDocsYAMLParseError( 'Your mkdocs.yml could not be loaded, ' - 'possibly due to a syntax error{note}'.format(note=note), + 'possibly due to a syntax error{note}'.format(note=note) ) def append_conf(self, **__): - """ - Set mkdocs config values. - - :raises: ``MkDocsYAMLParseError`` if failed due to known type errors - (i.e. expecting a list and a string is found). - """ + """Set mkdocs config values.""" if not self.yaml_file: self.yaml_file = os.path.join(self.root_path, 'mkdocs.yml') @@ -119,27 +113,12 @@ def append_conf(self, **__): # Handle custom docs dirs user_docs_dir = user_config.get('docs_dir') - if not isinstance(user_docs_dir, (type(None), str)): - raise MkDocsYAMLParseError( - MkDocsYAMLParseError.INVALID_DOCS_DIR_CONFIG, - ) - docs_dir = self.docs_dir(docs_dir=user_docs_dir) self.create_index(extension='md') user_config['docs_dir'] = docs_dir # Set mkdocs config values static_url = get_absolute_static_url() - - for config in ('extra_css', 'extra_javascript'): - user_value = user_config.get(config, []) - if not isinstance(user_value, list): - raise MkDocsYAMLParseError( - MkDocsYAMLParseError.INVALID_EXTRA_CONFIG.format( - config=config, - ), - ) - user_config.setdefault('extra_javascript', []).extend([ 'readthedocs-data.js', '%score/js/readthedocs-doc-embed.js' % static_url, @@ -154,13 +133,13 @@ def append_conf(self, **__): # of the mkdocs configuration file. docs_path = os.path.join( os.path.dirname(self.yaml_file), - docs_dir, + docs_dir ) # RTD javascript writing rtd_data = self.generate_rtd_data( docs_dir=os.path.relpath(docs_path, self.root_path), - mkdocs_config=user_config, + mkdocs_config=user_config ) with open(os.path.join(docs_path, 'readthedocs-data.js'), 'w') as f: f.write(rtd_data) @@ -179,7 +158,7 @@ def append_conf(self, **__): # Write the modified mkdocs configuration yaml.safe_dump( user_config, - open(self.yaml_file, 'w'), + open(self.yaml_file, 'w') ) # Write the mkdocs.yml to the build logs @@ -206,21 +185,13 @@ def generate_rtd_data(self, docs_dir, mkdocs_config): 'programming_language': self.version.project.programming_language, 'page': None, 'theme': self.get_theme_name(mkdocs_config), - 'builder': 'mkdocs', + 'builder': "mkdocs", 'docroot': docs_dir, - 'source_suffix': '.md', - 'api_host': getattr( - settings, - 'PUBLIC_API_URL', - 'https://readthedocs.org', - ), + 'source_suffix': ".md", + 'api_host': getattr(settings, 'PUBLIC_API_URL', 'https://readthedocs.org'), 'ad_free': not self.project.show_advertising, 'commit': self.version.project.vcs_repo(self.version.slug).commit, - 'global_analytics_code': getattr( - settings, - 'GLOBAL_ANALYTICS_CODE', - 'UA-17997319-1', - ), + 'global_analytics_code': getattr(settings, 'GLOBAL_ANALYTICS_CODE', 'UA-17997319-1'), 'user_analytics_code': analytics_code, } data_json = json.dumps(readthedocs_data, indent=4) @@ -241,22 +212,21 @@ def build(self): self.python_env.venv_bin(filename='mkdocs'), self.builder, '--clean', - '--site-dir', - self.build_dir, - '--config-file', - self.yaml_file, + '--site-dir', self.build_dir, + '--config-file', self.yaml_file, ] if self.config.mkdocs.fail_on_warning: build_command.append('--strict') cmd_ret = self.run( - *build_command, cwd=checkout_path, + *build_command, + cwd=checkout_path, bin_path=self.python_env.venv_bin() ) return cmd_ret.successful def get_theme_name(self, mkdocs_config): """ - Get the theme configuration in the mkdocs_config. + Get the theme configuration in the mkdocs_config In v0.17.0, the theme configuration switched from two separate configs (both optional) to a nested directive. diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index c5a664835fa..1d280b8a318 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -1,27 +1,30 @@ # -*- coding: utf-8 -*- - """ Sphinx_ backend for building docs. .. _Sphinx: http://www.sphinx-doc.org/ """ +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import codecs +import shutil import logging import os -import shutil import sys import zipfile from glob import glob +import six from django.conf import settings from django.template import loader as template_loader from django.template.loader import render_to_string from readthedocs.builds import utils as version_utils from readthedocs.projects.exceptions import ProjectConfigurationError -from readthedocs.projects.models import Feature from readthedocs.projects.utils import safe_write from readthedocs.restapi.client import api +from readthedocs.projects.models import Feature from ..base import BaseBuilder, restoring_chdir from ..constants import PDF_RE @@ -29,7 +32,6 @@ from ..exceptions import BuildEnvironmentError from ..signals import finalize_sphinx_context_data - log = logging.getLogger(__name__) @@ -38,14 +40,14 @@ class BaseSphinx(BaseBuilder): """The parent for most sphinx builders.""" def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(BaseSphinx, self).__init__(*args, **kwargs) self.config_file = self.config.sphinx.configuration try: if not self.config_file: self.config_file = self.project.conf_file(self.version.slug) self.old_artifact_path = os.path.join( os.path.dirname(self.config_file), - self.sphinx_build_dir, + self.sphinx_build_dir ) except ProjectConfigurationError: docs_dir = self.docs_dir() @@ -58,13 +60,11 @@ def _write_config(self, master_doc='index'): """Create ``conf.py`` if it doesn't exist.""" docs_dir = self.docs_dir() conf_template = render_to_string( - 'sphinx/conf.py.conf', - { + 'sphinx/conf.py.conf', { 'project': self.project, 'version': self.version, 'master_doc': master_doc, - }, - ) + }) conf_file = os.path.join(docs_dir, 'conf.py') safe_write(conf_file, conf_template) @@ -76,28 +76,25 @@ def get_config_params(self): os.path.dirname( os.path.relpath( self.config_file, - self.project.checkout_path(self.version.slug), - ), + self.project.checkout_path(self.version.slug) + ) ), '', ) remote_version = self.version.commit_name github_user, github_repo = version_utils.get_github_username_repo( - url=self.project.repo, - ) + url=self.project.repo) github_version_is_editable = (self.version.type == 'branch') display_github = github_user is not None bitbucket_user, bitbucket_repo = version_utils.get_bitbucket_username_repo( # noqa - url=self.project.repo, - ) + url=self.project.repo) bitbucket_version_is_editable = (self.version.type == 'branch') display_bitbucket = bitbucket_user is not None gitlab_user, gitlab_repo = version_utils.get_gitlab_username_repo( - url=self.project.repo, - ) + url=self.project.repo) gitlab_version_is_editable = (self.version.type == 'branch') display_gitlab = gitlab_user is not None @@ -149,7 +146,7 @@ def get_config_params(self): # Features 'dont_overwrite_sphinx_context': self.project.has_feature( - Feature.DONT_OVERWRITE_SPHINX_CONTEXT, + Feature.DONT_OVERWRITE_SPHINX_CONTEXT ), } @@ -162,25 +159,26 @@ def get_config_params(self): return data def append_conf(self, **__): - """ - Find or create a ``conf.py`` and appends default content. - - The default content is rendered from ``doc_builder/conf.py.tmpl``. - """ + """Find or create a ``conf.py`` with a rendered ``doc_builder/conf.py.tmpl`` appended""" if self.config_file is None: master_doc = self.create_index(extension='rst') self._write_config(master_doc=master_doc) try: self.config_file = ( - self.config_file or self.project.conf_file(self.version.slug) + self.config_file or + self.project.conf_file(self.version.slug) ) outfile = codecs.open(self.config_file, encoding='utf-8', mode='a') except (ProjectConfigurationError, IOError): trace = sys.exc_info()[2] - raise ProjectConfigurationError( - ProjectConfigurationError.NOT_FOUND, - ).with_traceback(trace) + six.reraise( + ProjectConfigurationError, + ProjectConfigurationError( + ProjectConfigurationError.NOT_FOUND + ), + trace + ) # Append config to project conf file tmpl = template_loader.get_template('doc_builder/conf.py.tmpl') @@ -224,7 +222,8 @@ def build(self): self.sphinx_build_dir, ]) cmd_ret = self.run( - *build_command, cwd=os.path.dirname(self.config_file), + *build_command, + cwd=os.path.dirname(self.config_file), bin_path=self.python_env.venv_bin() ) return cmd_ret.successful @@ -235,19 +234,18 @@ class HtmlBuilder(BaseSphinx): sphinx_build_dir = '_build/html' def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(HtmlBuilder, self).__init__(*args, **kwargs) self.sphinx_builder = 'readthedocs' def move(self, **__): - super().move() + super(HtmlBuilder, self).move() # Copy JSON artifacts to its own directory # to keep compatibility with the older builder. json_path = os.path.abspath( - os.path.join(self.old_artifact_path, '..', 'json'), + os.path.join(self.old_artifact_path, '..', 'json') ) json_path_target = self.project.artifact_path( - version=self.version.slug, - type_='sphinx_search', + version=self.version.slug, type_='sphinx_search' ) if os.path.exists(json_path): if os.path.exists(json_path_target): @@ -255,17 +253,19 @@ def move(self, **__): log.info('Copying json on the local filesystem') shutil.copytree( json_path, - json_path_target, + json_path_target ) else: - log.warning('Not moving json because the build dir is unknown.',) + log.warning( + 'Not moving json because the build dir is unknown.' + ) class HtmlDirBuilder(HtmlBuilder): type = 'sphinx_htmldir' def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(HtmlDirBuilder, self).__init__(*args, **kwargs) self.sphinx_builder = 'readthedocsdirhtml' @@ -273,7 +273,7 @@ class SingleHtmlBuilder(HtmlBuilder): type = 'sphinx_singlehtml' def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(SingleHtmlBuilder, self).__init__(*args, **kwargs) self.sphinx_builder = 'readthedocssinglehtml' @@ -304,8 +304,7 @@ def move(self, **__): filename=to_write, arcname=os.path.join( '{}-{}'.format(self.project.slug, self.version.slug), - to_write, - ), + to_write), ) archive.close() @@ -339,7 +338,7 @@ class LatexBuildCommand(BuildCommand): """Ignore LaTeX exit code if there was file output.""" def run(self): - super().run() + super(LatexBuildCommand, self).run() # Force LaTeX exit code to be a little more optimistic. If LaTeX # reports an output file, let's just assume we're fine. if PDF_RE.search(self.output): @@ -351,7 +350,7 @@ class DockerLatexBuildCommand(DockerBuildCommand): """Ignore LaTeX exit code if there was file output.""" def run(self): - super().run() + super(DockerLatexBuildCommand, self).run() # Force LaTeX exit code to be a little more optimistic. If LaTeX # reports an output file, let's just assume we're fine. if PDF_RE.search(self.output): @@ -394,16 +393,11 @@ def build(self): # Run LaTeX -> PDF conversions pdflatex_cmds = [ ['pdflatex', '-interaction=nonstopmode', tex_file] - for tex_file in tex_files - ] # yapf: disable + for tex_file in tex_files] # yapf: disable makeindex_cmds = [ - [ - 'makeindex', '-s', 'python.ist', '{}.idx'.format( - os.path.splitext(os.path.relpath(tex_file, latex_cwd))[0], - ), - ] - for tex_file in tex_files - ] # yapf: disable + ['makeindex', '-s', 'python.ist', '{0}.idx'.format( + os.path.splitext(os.path.relpath(tex_file, latex_cwd))[0])] + for tex_file in tex_files] # yapf: disable if self.build_env.command_class == DockerBuildCommand: latex_class = DockerLatexBuildCommand @@ -412,27 +406,15 @@ def build(self): pdf_commands = [] for cmd in pdflatex_cmds: cmd_ret = self.build_env.run_command_class( - cls=latex_class, - cmd=cmd, - cwd=latex_cwd, - warn_only=True, - ) + cls=latex_class, cmd=cmd, cwd=latex_cwd, warn_only=True) pdf_commands.append(cmd_ret) for cmd in makeindex_cmds: cmd_ret = self.build_env.run_command_class( - cls=latex_class, - cmd=cmd, - cwd=latex_cwd, - warn_only=True, - ) + cls=latex_class, cmd=cmd, cwd=latex_cwd, warn_only=True) pdf_commands.append(cmd_ret) for cmd in pdflatex_cmds: cmd_ret = self.build_env.run_command_class( - cls=latex_class, - cmd=cmd, - cwd=latex_cwd, - warn_only=True, - ) + cls=latex_class, cmd=cmd, cwd=latex_cwd, warn_only=True) pdf_match = PDF_RE.search(cmd_ret.output) if pdf_match: self.pdf_file_name = pdf_match.group(1).strip() @@ -466,9 +448,7 @@ def move(self, **__): from_file = None if from_file: to_file = os.path.join( - self.target, - '{}.pdf'.format(self.project.slug), - ) + self.target, '{}.pdf'.format(self.project.slug)) self.run( 'mv', '-f', diff --git a/readthedocs/doc_builder/base.py b/readthedocs/doc_builder/base.py index 6b143ca8db9..83aac0da617 100644 --- a/readthedocs/doc_builder/base.py +++ b/readthedocs/doc_builder/base.py @@ -1,13 +1,15 @@ # -*- coding: utf-8 -*- - """Base classes for Builders.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging import os import shutil +from builtins import object from functools import wraps - log = logging.getLogger(__name__) @@ -24,7 +26,7 @@ def decorator(*args, **kw): return decorator -class BaseBuilder: +class BaseBuilder(object): """ The Base for all Builders. Defines the API for subclasses. @@ -47,9 +49,7 @@ def __init__(self, build_env, python_env, force=False): self.config = python_env.config if python_env else None self._force = force self.target = self.project.artifact_path( - version=self.version.slug, - type_=self.type, - ) + version=self.version.slug, type_=self.type) def force(self, **__): """An optional step to force a build even when nothing has changed.""" @@ -70,7 +70,7 @@ def move(self, **__): shutil.copytree( self.old_artifact_path, self.target, - ignore=shutil.ignore_patterns(*self.ignore_patterns), + ignore=shutil.ignore_patterns(*self.ignore_patterns) ) else: log.warning('Not moving docs, because the build dir is unknown.') @@ -99,14 +99,10 @@ def create_index(self, extension='md', **__): docs_dir = self.docs_dir() index_filename = os.path.join( - docs_dir, - 'index.{ext}'.format(ext=extension), - ) + docs_dir, 'index.{ext}'.format(ext=extension)) if not os.path.exists(index_filename): readme_filename = os.path.join( - docs_dir, - 'README.{ext}'.format(ext=extension), - ) + docs_dir, 'README.{ext}'.format(ext=extension)) if os.path.exists(readme_filename): return 'README' diff --git a/readthedocs/doc_builder/config.py b/readthedocs/doc_builder/config.py index 9cf7f9d8abb..00a0095bcc6 100644 --- a/readthedocs/doc_builder/config.py +++ b/readthedocs/doc_builder/config.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- - """An API to load config from a readthedocs.yml file.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + from os import path from readthedocs.config import BuildConfigV1, ConfigError, InvalidConfig @@ -31,7 +33,7 @@ def load_yaml_config(version): try: sphinx_configuration = path.join( version.get_conf_py_path(), - 'conf.py', + 'conf.py' ) except ProjectConfigurationError: sphinx_configuration = None @@ -41,6 +43,8 @@ def load_yaml_config(version): 'build': { 'image': img_name, }, + 'output_base': '', + 'name': version.slug, 'defaults': { 'install_project': project.install_project, 'formats': get_default_formats(project), @@ -50,11 +54,12 @@ def load_yaml_config(version): 'sphinx_configuration': sphinx_configuration, 'build_image': project.container_image, 'doctype': project.documentation_type, - }, + } } img_settings = DOCKER_IMAGE_SETTINGS.get(img_name, None) if img_settings: env_config.update(img_settings) + env_config['DOCKER_IMAGE_SETTINGS'] = img_settings try: config = load_config( diff --git a/readthedocs/doc_builder/constants.py b/readthedocs/doc_builder/constants.py index 4f6deeb6174..1cd1a5b1348 100644 --- a/readthedocs/doc_builder/constants.py +++ b/readthedocs/doc_builder/constants.py @@ -1,14 +1,15 @@ # -*- coding: utf-8 -*- - """Doc build constants.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging import os import re from django.conf import settings - log = logging.getLogger(__name__) MKDOCS_TEMPLATE_DIR = os.path.join( @@ -32,9 +33,7 @@ old_config = getattr(settings, 'DOCKER_BUILD_IMAGES', None) if old_config: - log.warning( - 'Old config detected, DOCKER_BUILD_IMAGES->DOCKER_IMAGE_SETTINGS', - ) + log.warning('Old config detected, DOCKER_BUILD_IMAGES->DOCKER_IMAGE_SETTINGS') DOCKER_IMAGE_SETTINGS.update(old_config) DOCKER_LIMITS = {'memory': '200m', 'time': 600} diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index ddf0b471d7d..b8980ec6971 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -2,6 +2,13 @@ """Documentation Builder Environments.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import os import re @@ -11,6 +18,8 @@ import traceback from datetime import datetime +import six +from builtins import object, str from django.conf import settings from django.utils.translation import ugettext_lazy as _ from docker import APIClient @@ -23,7 +32,6 @@ from readthedocs.builds.models import BuildCommandResultMixin from readthedocs.core.utils import slugify from readthedocs.projects.constants import LOG_TEMPLATE -from readthedocs.projects.models import Feature from readthedocs.restapi.client import api as api_v2 from .constants import ( @@ -85,19 +93,9 @@ class BuildCommand(BuildCommandResultMixin): :param description: a more grokable description of the command being run """ - def __init__( - self, - command, - cwd=None, - shell=False, - environment=None, - combine_output=True, - input_data=None, - build_env=None, - bin_path=None, - description=None, - record_as_success=False, - ): + def __init__(self, command, cwd=None, shell=False, environment=None, + combine_output=True, input_data=None, build_env=None, + bin_path=None, description=None, record_as_success=False): self.command = command self.shell = shell if cwd is None: @@ -125,7 +123,7 @@ def __init__( def __str__(self): # TODO do we want to expose the full command here? - output = '' + output = u'' if self.output is not None: output = self.output.encode('utf-8') return '\n'.join([self.get_command(), output]) @@ -180,7 +178,7 @@ def run(self): if self.input_data is not None: cmd_input = self.input_data - if isinstance(cmd_input, str): + if isinstance(cmd_input, six.string_types): cmd_input_bytes = cmd_input.encode('utf-8') else: cmd_input_bytes = cmd_input @@ -312,18 +310,11 @@ def run(self): # nicer. Sometimes the kernel kills the command and Docker doesn't # not use the specific exit code, so we check if the word `Killed` # is in the last 15 lines of the command's output - killed_in_output = 'Killed' in '\n'.join( - self.output.splitlines()[-15:], - ) - if self.exit_code == DOCKER_OOM_EXIT_CODE or ( - self.exit_code == 1 and - killed_in_output - ): - self.output += str( - _( - '\n\nCommand killed due to excessive memory consumption\n', - ), - ) + killed_in_output = 'Killed' in '\n'.join(self.output.splitlines()[-15:]) + if self.exit_code == DOCKER_OOM_EXIT_CODE or (self.exit_code == 1 and killed_in_output): + self.output += str(_( + '\n\nCommand killed due to excessive memory consumption\n' + )) except DockerAPIError: self.exit_code = -1 if self.output is None or not self.output: @@ -341,28 +332,20 @@ def get_wrapped_command(self): install requests<0.8``. This escapes a good majority of those characters. """ - bash_escape_re = re.compile( - r"([\t\ \!\"\#\$\&\'\(\)\*\:\;\<\>\?\@" - r'\[\\\]\^\`\{\|\}\~])', - ) + bash_escape_re = re.compile(r"([\t\ \!\"\#\$\&\'\(\)\*\:\;\<\>\?\@" + r"\[\\\]\^\`\{\|\}\~])") prefix = '' if self.bin_path: - prefix += 'PATH={}:$PATH '.format(self.bin_path) - return ( - "/bin/sh -c 'cd {cwd} && {prefix}{cmd}'".format( - cwd=self.cwd, - prefix=prefix, - cmd=( - ' '.join([ - bash_escape_re.sub(r'\\\1', part) - for part in self.command - ]) - ), - ) - ) + prefix += 'PATH={0}:$PATH '.format(self.bin_path) + return ("/bin/sh -c 'cd {cwd} && {prefix}{cmd}'" + .format( + cwd=self.cwd, + prefix=prefix, + cmd=(' '.join([bash_escape_re.sub(r'\\\1', part) + for part in self.command])))) -class BaseEnvironment: +class BaseEnvironment(object): """ Base environment class. @@ -385,8 +368,7 @@ def run(self, *cmd, **kwargs): def run_command_class( self, cls, cmd, record=None, warn_only=False, - record_as_success=False, **kwargs - ): + record_as_success=False, **kwargs): """ Run command from this environment. @@ -436,19 +418,17 @@ def run_command_class( self.commands.append(build_cmd) if build_cmd.failed: - msg = 'Command {cmd} failed'.format(cmd=build_cmd.get_command()) + msg = u'Command {cmd} failed'.format(cmd=build_cmd.get_command()) if build_cmd.output: - msg += ':\n{out}'.format(out=build_cmd.output) + msg += u':\n{out}'.format(out=build_cmd.output) if warn_only: - log.warning( - LOG_TEMPLATE.format( - project=self.project.slug, - version='latest', - msg=msg, - ), - ) + log.warning(LOG_TEMPLATE.format( + project=self.project.slug, + version='latest', + msg=msg, + )) else: raise BuildEnvironmentWarning(msg) return build_cmd @@ -504,17 +484,9 @@ class BuildEnvironment(BaseEnvironment): MkDocsYAMLParseError, ) - def __init__( - self, - project=None, - version=None, - build=None, - config=None, - record=True, - environment=None, - update_on_success=True, - ): - super().__init__(project, environment) + def __init__(self, project=None, version=None, build=None, config=None, + record=True, environment=None, update_on_success=True): + super(BuildEnvironment, self).__init__(project, environment) self.version = version self.build = build self.config = config @@ -590,39 +562,33 @@ def run(self, *cmd, **kwargs): kwargs.update({ 'build_env': self, }) - return super().run(*cmd, **kwargs) + return super(BuildEnvironment, self).run(*cmd, **kwargs) def run_command_class(self, *cmd, **kwargs): # pylint: disable=arguments-differ kwargs.update({ 'build_env': self, }) - return super().run_command_class(*cmd, **kwargs) + return super(BuildEnvironment, self).run_command_class(*cmd, **kwargs) @property def successful(self): - """Build completed, without top level failures or failing commands.""" - return ( - self.done and self.failure is None and - all(cmd.successful for cmd in self.commands) - ) + """Is build completed, without top level failures or failing commands.""" # noqa + return (self.done and self.failure is None and + all(cmd.successful for cmd in self.commands)) @property def failed(self): """Is build completed, but has top level failure or failing commands.""" - return ( - self.done and ( - self.failure is not None or - any(cmd.failed for cmd in self.commands) - ) - ) + return (self.done and ( + self.failure is not None or + any(cmd.failed for cmd in self.commands) + )) @property def done(self): """Is build in finished state.""" - return ( - self.build is not None and - self.build['state'] == BUILD_STATE_FINISHED - ) + return (self.build is not None and + self.build['state'] == BUILD_STATE_FINISHED) def update_build(self, state=None): """ @@ -668,15 +634,13 @@ def update_build(self, state=None): if self.failure is not None: # Surface a generic error if the class is not a # BuildEnvironmentError - # yapf: disable if not isinstance( - self.failure, - ( - BuildEnvironmentException, - BuildEnvironmentWarning, - ), + self.failure, + ( + BuildEnvironmentException, + BuildEnvironmentWarning, + ), ): - # yapf: enable log.error( 'Build failed with unhandled exception: %s', str(self.failure), @@ -698,7 +662,7 @@ def update_build(self, state=None): # Attempt to stop unicode errors on build reporting for key, val in list(self.build.items()): - if isinstance(val, bytes): + if isinstance(val, six.binary_type): self.build[key] = val.decode('utf-8', 'ignore') # We are selective about when we update the build object here @@ -713,7 +677,7 @@ def update_build(self, state=None): if update_build: try: api_v2.build(self.build['id']).put(self.build) - except HttpClientError: + except HttpClientError as e: log.exception( 'Unable to update build: id=%d', self.build['id'], @@ -752,7 +716,7 @@ class DockerBuildEnvironment(BuildEnvironment): def __init__(self, *args, **kwargs): self.docker_socket = kwargs.pop('docker_socket', DOCKER_SOCKET) - super().__init__(*args, **kwargs) + super(DockerBuildEnvironment, self).__init__(*args, **kwargs) self.client = None self.container = None self.container_name = slugify( @@ -762,18 +726,10 @@ def __init__(self, *args, **kwargs): project_name=self.project.slug, )[:DOCKER_HOSTNAME_MAX_LEN], ) - - # Decide what Docker image to use, based on priorities: - # Use the Docker image set by our feature flag: ``testing`` or, - if self.project.has_feature(Feature.USE_TESTING_BUILD_IMAGE): - self.container_image = 'readthedocs/build:testing' - # the image set by user or, if self.config and self.config.build.image: self.container_image = self.config.build.image - # the image overridden by the project (manually set by an admin). if self.project.container_image: self.container_image = self.project.container_image - if self.project.container_mem_limit: self.container_mem_limit = self.project.container_mem_limit if self.project.container_time_limit: @@ -804,11 +760,10 @@ def __enter__(self): project=self.project.slug, version=self.version.slug, msg=( - 'Removing stale container {}'.format( - self.container_id, - ) + 'Removing stale container {0}' + .format(self.container_id) ), - ), + ) ) client = self.get_client() client.remove_container(self.container_id) @@ -869,7 +824,7 @@ def __exit__(self, exc_type, exc_value, tb): if not all([exc_type, exc_value, tb]): exc_type, exc_value, tb = sys.exc_info() - return super().__exit__(exc_type, exc_value, tb) + return super(DockerBuildEnvironment, self).__exit__(exc_type, exc_value, tb) def get_client(self): """Create Docker client connection.""" @@ -880,7 +835,7 @@ def get_client(self): version=DOCKER_VERSION, ) return self.client - except DockerException: + except DockerException as e: log.exception( LOG_TEMPLATE.format( project=self.project.slug, @@ -978,10 +933,10 @@ def update_build_from_container_state(self): ) elif state.get('Error'): self.failure = BuildEnvironmentError(( - _('Build exited due to unknown error: {0}').format( - state.get('Error'), - ) - ),) + _('Build exited due to unknown error: {0}') + .format(state.get('Error')) + ), + ) def create_container(self): """Create docker container.""" @@ -1006,7 +961,7 @@ def create_container(self): environment=self.environment, ) client.start(container=self.container_id) - except ConnectionError: + except ConnectionError as e: log.exception( LOG_TEMPLATE.format( project=self.project.slug, diff --git a/readthedocs/doc_builder/exceptions.py b/readthedocs/doc_builder/exceptions.py index eaa2858aa6e..ce2ce3d844b 100644 --- a/readthedocs/doc_builder/exceptions.py +++ b/readthedocs/doc_builder/exceptions.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- - """Exceptions raised when building documentation.""" +from __future__ import division, print_function, unicode_literals + from django.utils.translation import ugettext_noop @@ -10,12 +11,9 @@ class BuildEnvironmentException(Exception): status_code = None def __init__(self, message=None, **kwargs): - self.status_code = kwargs.pop( - 'status_code', - None, - ) or self.status_code or 1 + self.status_code = kwargs.pop('status_code', None) or self.status_code or 1 message = message or self.get_default_message() - super().__init__(message, **kwargs) + super(BuildEnvironmentException, self).__init__(message, **kwargs) def get_default_message(self): return self.message @@ -61,13 +59,3 @@ class MkDocsYAMLParseError(BuildEnvironmentError): GENERIC_WITH_PARSE_EXCEPTION = ugettext_noop( 'Problem parsing MkDocs YAML configuration. {exception}', ) - - INVALID_DOCS_DIR_CONFIG = ugettext_noop( - 'The "docs_dir" config from your MkDocs YAML config file has to be a ' - 'string with relative or absolute path.', - ) - - INVALID_EXTRA_CONFIG = ugettext_noop( - 'The "{config}" config from your MkDocs YAML config file has to be a ' - 'a list of relative paths.', - ) diff --git a/readthedocs/doc_builder/loader.py b/readthedocs/doc_builder/loader.py index 016cc6bba9a..0edcaace778 100644 --- a/readthedocs/doc_builder/loader.py +++ b/readthedocs/doc_builder/loader.py @@ -1,26 +1,16 @@ -# -*- coding: utf-8 -*- - """Lookup tables for builders and backends.""" +from __future__ import absolute_import from importlib import import_module from django.conf import settings - # Managers mkdocs = import_module( - getattr( - settings, - 'MKDOCS_BACKEND', - 'readthedocs.doc_builder.backends.mkdocs', - ), -) + getattr(settings, 'MKDOCS_BACKEND', + 'readthedocs.doc_builder.backends.mkdocs')) sphinx = import_module( - getattr( - settings, - 'SPHINX_BACKEND', - 'readthedocs.doc_builder.backends.sphinx', - ), -) + getattr(settings, 'SPHINX_BACKEND', + 'readthedocs.doc_builder.backends.sphinx')) BUILDER_BY_NAME = { # Possible HTML Builders diff --git a/readthedocs/doc_builder/python_environments.py b/readthedocs/doc_builder/python_environments.py index 69628d19c5e..fa9bd9f8bf3 100644 --- a/readthedocs/doc_builder/python_environments.py +++ b/readthedocs/doc_builder/python_environments.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- - """An abstraction over virtualenv and Conda environments.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import copy import itertools import json @@ -9,6 +15,8 @@ import os import shutil +import six +from builtins import object, open from django.conf import settings from readthedocs.doc_builder.config import load_yaml_config @@ -18,11 +26,10 @@ from readthedocs.projects.constants import LOG_TEMPLATE from readthedocs.projects.models import Feature - log = logging.getLogger(__name__) -class PythonEnvironment: +class PythonEnvironment(object): """An isolated environment into which Python packages can be installed.""" @@ -41,29 +48,24 @@ def delete_existing_build_dir(self): # Handle deleting old build dir build_dir = os.path.join( self.venv_path(), - 'build', - ) + 'build') if os.path.exists(build_dir): - log.info( - LOG_TEMPLATE.format( - project=self.project.slug, - version=self.version.slug, - msg='Removing existing build directory', - ), - ) + log.info(LOG_TEMPLATE.format( + project=self.project.slug, + version=self.version.slug, + msg='Removing existing build directory', + )) shutil.rmtree(build_dir) def delete_existing_venv_dir(self): venv_dir = self.venv_path() # Handle deleting old venv dir if os.path.exists(venv_dir): - log.info( - LOG_TEMPLATE.format( - project=self.project.slug, - version=self.version.slug, - msg='Removing existing venv directory', - ), - ) + log.info(LOG_TEMPLATE.format( + project=self.project.slug, + version=self.version.slug, + msg='Removing existing venv directory', + )) shutil.rmtree(venv_dir) def install_package(self): @@ -71,24 +73,23 @@ def install_package(self): getattr(settings, 'USE_PIP_INSTALL', False)): extra_req_param = '' if self.config.python.extra_requirements: - extra_req_param = '[{}]'.format( - ','.join(self.config.python.extra_requirements), + extra_req_param = '[{0}]'.format( + ','.join(self.config.python.extra_requirements) ) self.build_env.run( - self.venv_bin(filename='python'), - '-m', - 'pip', + 'python', + self.venv_bin(filename='pip'), 'install', '--ignore-installed', '--cache-dir', self.project.pip_cache_path, - '.{}'.format(extra_req_param), + '.{0}'.format(extra_req_param), cwd=self.checkout_path, bin_path=self.venv_bin(), ) elif self.config.python.install_with_setup: self.build_env.run( - self.venv_bin(filename='python'), + 'python', 'setup.py', 'install', '--force', @@ -141,9 +142,7 @@ def is_obsolete(self): with open(self.environment_json_path(), 'r') as fpath: environment_conf = json.load(fpath) except (IOError, TypeError, KeyError, ValueError): - log.warning( - 'Unable to read/parse readthedocs-environment.json file', - ) + log.warning('Unable to read/parse readthedocs-environment.json file') # We remove the JSON file here to avoid cycling over time with a # corrupted file. os.remove(self.environment_json_path()) @@ -177,15 +176,7 @@ def is_obsolete(self): ]) def save_environment_json(self): - """ - Save on builders disk data about the environment used to build docs. - - The data is saved as a ``.json`` file with this information on it: - - - python.version - - build.image - - build.hash - """ + """Save on disk Python and build image versions used to create the venv.""" data = { 'python': { 'version': self.config.python_full_version, @@ -204,7 +195,7 @@ def save_environment_json(self): with open(self.environment_json_path(), 'w') as fpath: # Compatibility for Py2 and Py3. ``io.TextIOWrapper`` expects # unicode but ``json.dumps`` returns str in Py2. - fpath.write(str(json.dumps(data))) + fpath.write(six.text_type(json.dumps(data))) class Virtualenv(PythonEnvironment): @@ -238,9 +229,8 @@ def setup_base(self): def install_core_requirements(self): """Install basic Read the Docs requirements into the virtualenv.""" pip_install_cmd = [ - self.venv_bin(filename='python'), - '-m', - 'pip', + 'python', + self.venv_bin(filename='pip'), 'install', '--upgrade', '--cache-dir', @@ -251,7 +241,9 @@ def install_core_requirements(self): # so it is used when installing the other requirements. cmd = pip_install_cmd + ['pip'] self.build_env.run( - *cmd, bin_path=self.venv_bin(), cwd=self.checkout_path + *cmd, + bin_path=self.venv_bin(), + cwd=self.checkout_path ) requirements = [ @@ -283,7 +275,7 @@ def install_core_requirements(self): negative='sphinx<1.8', ), 'sphinx-rtd-theme<0.5', - 'readthedocs-sphinx-ext<0.6', + 'readthedocs-sphinx-ext<0.6' ]) cmd = copy.copy(pip_install_cmd) @@ -304,12 +296,8 @@ def install_user_requirements(self): requirements_file_path = self.config.python.requirements if not requirements_file_path and requirements_file_path != '': builder_class = get_builder_class(self.config.doctype) - docs_dir = ( - builder_class( - build_env=self.build_env, - python_env=self, - ).docs_dir() - ) + docs_dir = (builder_class(build_env=self.build_env, python_env=self) + .docs_dir()) paths = [docs_dir, ''] req_files = ['pip_requirements.txt', 'requirements.txt'] for path, req_file in itertools.product(paths, req_files): @@ -320,9 +308,8 @@ def install_user_requirements(self): if requirements_file_path: args = [ - self.venv_bin(filename='python'), - '-m', - 'pip', + 'python', + self.venv_bin(filename='pip'), 'install', ] if self.project.has_feature(Feature.PIP_ALWAYS_UPGRADE): @@ -358,19 +345,16 @@ def setup_base(self): if os.path.exists(version_path): # Re-create conda directory each time to keep fresh state - log.info( - LOG_TEMPLATE.format( - project=self.project.slug, - version=self.version.slug, - msg='Removing existing conda directory', - ), - ) + log.info(LOG_TEMPLATE.format( + project=self.project.slug, + version=self.version.slug, + msg='Removing existing conda directory', + )) shutil.rmtree(version_path) self.build_env.run( 'conda', 'env', 'create', - '--quiet', '--name', self.version.slug, '--file', @@ -402,7 +386,6 @@ def install_core_requirements(self): 'conda', 'install', '--yes', - '--quiet', '--name', self.version.slug, ] @@ -413,9 +396,8 @@ def install_core_requirements(self): ) pip_cmd = [ - self.venv_bin(filename='python'), - '-m', - 'pip', + 'python', + self.venv_bin(filename='pip'), 'install', '-U', '--cache-dir', diff --git a/readthedocs/doc_builder/signals.py b/readthedocs/doc_builder/signals.py index 5821ddeccf2..419531a5630 100644 --- a/readthedocs/doc_builder/signals.py +++ b/readthedocs/doc_builder/signals.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- +"""Signals for adding custom context data""" -"""Signals for adding custom context data.""" +from __future__ import absolute_import import django.dispatch - finalize_sphinx_context_data = django.dispatch.Signal( - providing_args=['buildenv', 'context', 'response_data'], + providing_args=['buildenv', 'context', 'response_data'] ) diff --git a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl index 0ec2b9c331b..e36cdd16af5 100644 --- a/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl +++ b/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl @@ -13,7 +13,6 @@ # https://github.com/rtfd/readthedocs.org/blob/master/readthedocs/doc_builder/templates/doc_builder/conf.py.tmpl # -from __future__ import absolute_import, division, print_function, unicode_literals import importlib import sys @@ -82,9 +81,9 @@ context = { ("{{ slug }}", "{{ url }}"),{% endfor %} ], 'slug': '{{ project.slug }}', - 'name': '{{ project.name }}', - 'rtd_language': '{{ project.language }}', - 'programming_language': '{{ project.programming_language }}', + 'name': u'{{ project.name }}', + 'rtd_language': u'{{ project.language }}', + 'programming_language': u'{{ project.programming_language }}', 'canonical_url': '{{ project.get_canonical_url }}', 'analytics_code': '{{ project.analytics_code }}', 'single_version': {{ project.single_version }}, diff --git a/readthedocs/doc_builder/templates/doc_builder/data.js.tmpl b/readthedocs/doc_builder/templates/doc_builder/data.js.tmpl index 29ab61b0e65..4dda93914e8 100644 --- a/readthedocs/doc_builder/templates/doc_builder/data.js.tmpl +++ b/readthedocs/doc_builder/templates/doc_builder/data.js.tmpl @@ -6,7 +6,7 @@ var doc_slug = "{{ slug }}"; var page_name = "{{ pagename }}"; var html_theme = "{{ html_theme }}"; -// mkdocs_page_input_path is only defined on the RTD mkdocs theme but it isn't +// mkdocs_page_input_path is only defined on the RTD mkdocs theme but it isn't // available on all pages (e.g. missing in search result) if (typeof mkdocs_page_input_path !== "undefined") { READTHEDOCS_DATA["page"] = mkdocs_page_input_path.substr( diff --git a/readthedocs/gold/__init__.py b/readthedocs/gold/__init__.py index 2ef7166adf3..b26c8ed7c84 100644 --- a/readthedocs/gold/__init__.py +++ b/readthedocs/gold/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ A Django app for Gold Membership. diff --git a/readthedocs/gold/admin.py b/readthedocs/gold/admin.py index 2da5d033444..ecd512b75a8 100644 --- a/readthedocs/gold/admin.py +++ b/readthedocs/gold/admin.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- - """Django admin configuration for the Gold Membership app.""" +from __future__ import absolute_import from django.contrib import admin - from .models import GoldUser diff --git a/readthedocs/gold/apps.py b/readthedocs/gold/apps.py index c54b9d5424e..eda30768d46 100644 --- a/readthedocs/gold/apps.py +++ b/readthedocs/gold/apps.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - """Django app configuration for the Gold Membership app.""" +from __future__ import absolute_import from django.apps import AppConfig diff --git a/readthedocs/gold/forms.py b/readthedocs/gold/forms.py index 84c19fba387..949ab9c4c61 100644 --- a/readthedocs/gold/forms.py +++ b/readthedocs/gold/forms.py @@ -1,8 +1,10 @@ -# -*- coding: utf-8 -*- - """Gold subscription forms.""" +from __future__ import absolute_import + +from builtins import object from django import forms + from django.utils.translation import ugettext_lazy as _ from readthedocs.payments.forms import StripeModelForm, StripeResourceMixin @@ -22,7 +24,7 @@ class GoldSubscriptionForm(StripeResourceMixin, StripeModelForm): :py:class:`StripeResourceMixin` for common operations against the Stripe API. """ - class Meta: + class Meta(object): model = GoldUser fields = ['last_4_card_digits', 'level'] @@ -30,11 +32,9 @@ class Meta: required=True, min_length=4, max_length=4, - widget=forms.HiddenInput( - attrs={ - 'data-bind': 'valueInit: last_4_card_digits, value: last_4_card_digits', - }, - ), + widget=forms.HiddenInput(attrs={ + 'data-bind': 'valueInit: last_4_card_digits, value: last_4_card_digits', + }) ) level = forms.ChoiceField( @@ -44,7 +44,7 @@ class Meta: def clean(self): self.instance.user = self.customer - return super().clean() + return super(GoldSubscriptionForm, self).clean() def validate_stripe(self): subscription = self.get_subscription() @@ -54,8 +54,7 @@ def validate_stripe(self): def get_customer_kwargs(self): data = { - 'description': self.customer.get_full_name() or - self.customer.username, + 'description': self.customer.get_full_name() or self.customer.username, 'email': self.customer.email, 'id': self.instance.stripe_id or None, } @@ -83,7 +82,7 @@ def get_subscription(self): # Add a new subscription subscription = customer.subscriptions.create( plan=self.cleaned_data['level'], - source=self.cleaned_data['stripe_token'], + source=self.cleaned_data['stripe_token'] ) return subscription @@ -92,13 +91,13 @@ def get_subscription(self): class GoldProjectForm(forms.Form): project = forms.ChoiceField( required=True, - help_text='Select a project.', + help_text='Select a project.' ) def __init__(self, active_user, *args, **kwargs): self.user = kwargs.pop('user', None) self.projects = kwargs.pop('projects', None) - super().__init__(*args, **kwargs) + super(GoldProjectForm, self).__init__(*args, **kwargs) self.fields['project'].choices = self.generate_choices(active_user) def generate_choices(self, active_user): @@ -115,11 +114,8 @@ def clean_project(self): return project_slug def clean(self): - cleaned_data = super().clean() + cleaned_data = super(GoldProjectForm, self).clean() if self.projects.count() < self.user.num_supported_projects: return cleaned_data - self.add_error( - None, - 'You already have the max number of supported projects.', - ) + self.add_error(None, 'You already have the max number of supported projects.') diff --git a/readthedocs/gold/migrations/0001_initial.py b/readthedocs/gold/migrations/0001_initial.py index 80a61461da8..baeb6d28361 100644 --- a/readthedocs/gold/migrations/0001_initial.py +++ b/readthedocs/gold/migrations/0001_initial.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations from django.conf import settings -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/gold/migrations/0002_rename_last_4_digits.py b/readthedocs/gold/migrations/0002_rename_last_4_digits.py index 478681e4ad3..2ed345fd3ce 100644 --- a/readthedocs/gold/migrations/0002_rename_last_4_digits.py +++ b/readthedocs/gold/migrations/0002_rename_last_4_digits.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-07-16 15:45 -from django.db import migrations +from __future__ import unicode_literals + +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/gold/migrations/0003_add_missing_model_change_migrations.py b/readthedocs/gold/migrations/0003_add_missing_model_change_migrations.py index f1f9f1dbd5f..2e919ac202e 100644 --- a/readthedocs/gold/migrations/0003_add_missing_model_change_migrations.py +++ b/readthedocs/gold/migrations/0003_add_missing_model_change_migrations.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-31 11:25 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/gold/migrations/0004_add_vat_id.py b/readthedocs/gold/migrations/0004_add_vat_id.py index ee4899eebac..eab1771f1a2 100644 --- a/readthedocs/gold/migrations/0004_add_vat_id.py +++ b/readthedocs/gold/migrations/0004_add_vat_id.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-10-22 07:13 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/gold/models.py b/readthedocs/gold/models.py index c73ec79e39d..b87e0f72345 100644 --- a/readthedocs/gold/models.py +++ b/readthedocs/gold/models.py @@ -1,6 +1,13 @@ # -*- coding: utf-8 -*- """Django models for recurring donations aka Gold Membership.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import math from django.db import models @@ -9,7 +16,6 @@ from readthedocs.projects.models import Project - #: The membership options that are currently available LEVEL_CHOICES = ( ('v1-org-5', '$5/mo'), @@ -56,7 +62,7 @@ class GoldUser(models.Model): business_vat_id = models.CharField(max_length=128, null=True, blank=True) def __str__(self): - return 'Gold Level {} for {}'.format(self.level, self.user) + return 'Gold Level %s for %s' % (self.level, self.user) @property def num_supported_projects(self): diff --git a/readthedocs/gold/signals.py b/readthedocs/gold/signals.py index a5b377290be..a3cef14ca9e 100644 --- a/readthedocs/gold/signals.py +++ b/readthedocs/gold/signals.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Gold model signals.""" +"""Gold model signals""" +from __future__ import absolute_import from django.db.models.signals import pre_delete from django.dispatch import receiver @@ -12,6 +11,6 @@ @receiver(pre_delete, sender=GoldUser) def delete_customer(sender, instance, **__): - """On Gold subscription deletion, remove the customer from Stripe.""" + """On Gold subscription deletion, remove the customer from Stripe""" if sender == GoldUser and instance.stripe_id is not None: utils.delete_customer(instance.stripe_id) diff --git a/readthedocs/gold/templates/gold/projects.html b/readthedocs/gold/templates/gold/projects.html index e832b53ee67..682b7bf3c9f 100644 --- a/readthedocs/gold/templates/gold/projects.html +++ b/readthedocs/gold/templates/gold/projects.html @@ -48,3 +48,4 @@

{% trans "Add a project" %}

{% endblock %} + diff --git a/readthedocs/gold/templates/gold/subscription_form.html b/readthedocs/gold/templates/gold/subscription_form.html index 36300828f88..9dc1ae592dc 100644 --- a/readthedocs/gold/templates/gold/subscription_form.html +++ b/readthedocs/gold/templates/gold/subscription_form.html @@ -33,16 +33,16 @@ {% endblock %} {% block edit_content %} -
-

Read the Docs Gold

+
+

Read the Docs Gold

-

- {% blocktrans trimmed %} - Supporting Read the Docs lets us work more on features that people love. - Your money will go directly to maintenance and development of the - product. - {% endblocktrans %} -

+

+ {% blocktrans trimmed %} + Supporting Read the Docs lets us work more on features that people love. + Your money will go directly to maintenance and development of the + product. + {% endblocktrans %} +

{% blocktrans trimmed %} If you are an individual, @@ -60,86 +60,85 @@

Read the Docs Gold

{% endblocktrans %}

-

{% trans 'Becoming a Gold Member also makes Read the Docs ad-free for as long as you are logged-in.' %}

- -

- {% blocktrans trimmed %} - You can also make one-time donations on our sustainability page. - {% endblocktrans %} -

- - {% if domains.count %} -

Domains

-

- {% blocktrans trimmed %} - We ask that folks who use custom Domains give Read the Docs $5 per domain they are using. - This is optional, but it really does help us maintain the site going forward. - {% endblocktrans %} -

- -

- You are currently using {{ domains.count }} domains: +

{% trans 'Becoming a Gold Member also makes Read the Docs ad-free for as long as you are logged-in.' %}

- -

+

+ {% blocktrans trimmed %} + You can also make one-time donations on our sustainability page. + {% endblocktrans %} +

- {% endif %} + {% if domains.count %} +

Domains

+

+ {% blocktrans trimmed %} + We ask that folks who use custom Domains give Read the Docs $5 per domain they are using. + This is optional, but it really does help us maintain the site going forward. + {% endblocktrans %} +

+ +

+ You are currently using {{ domains.count }} domains: + +

+

+ + {% endif %} + + {% trans "Become a Gold Member" as subscription_title %} + {% if golduser %} + {% trans "Update Your Subscription" as subscription_title %} + {% endif %} +

{{ subscription_title }}

+ +
+ {% csrf_token %} + + {{ form.non_field_errors }} + + {% for field in form.fields_with_cc_group %} + {% if field.is_cc_group %} + + + + + {% else %} + {% include 'core/ko_form_field.html' with field=field %} + {% endif %} + {% endfor %} - {% trans "Become a Gold Member" as subscription_title %} + {% trans "Sign Up" as form_submit_text %} {% if golduser %} - {% trans "Update Your Subscription" as subscription_title %} + {% trans "Update Subscription" as form_submit_text %} {% endif %} -

{{ subscription_title }}

- - - {% csrf_token %} - - {{ form.non_field_errors }} - - {% for field in form.fields_with_cc_group %} - {% if field.is_cc_group %} - - - - - {% else %} - {% include 'core/ko_form_field.html' with field=field %} - {% endif %} - {% endfor %} - - {% trans "Sign Up" as form_submit_text %} - {% if golduser %} - {% trans "Update Subscription" as form_submit_text %} - {% endif %} - + - {% trans "All information is submitted directly to Stripe." %} -
-
+ {% trans "All information is submitted directly to Stripe." %} + {% endblock %} diff --git a/readthedocs/gold/tests/test_forms.py b/readthedocs/gold/tests/test_forms.py index c379bba3b79..acf1c82fe1e 100644 --- a/readthedocs/gold/tests/test_forms.py +++ b/readthedocs/gold/tests/test_forms.py @@ -1,13 +1,13 @@ -# -*- coding: utf-8 -*- -import django_dynamic_fixture as fixture +from __future__ import absolute_import import mock -from django.contrib.auth.models import User +import django_dynamic_fixture as fixture from django.test import TestCase +from django.contrib.auth.models import User from readthedocs.projects.models import Project -from ..forms import GoldSubscriptionForm from ..models import GoldUser +from ..forms import GoldSubscriptionForm class GoldSubscriptionFormTests(TestCase): @@ -32,7 +32,7 @@ def mock_request(self, resp): self.mocks['request'].request = mock.Mock(side_effect=resp) def test_add_subscription(self): - """Valid subscription form.""" + """Valid subscription form""" subscription_list = { 'object': 'list', 'data': [], @@ -44,7 +44,7 @@ def test_add_subscription(self): 'id': 'cus_12345', 'description': self.user.get_full_name(), 'email': self.user.email, - 'subscriptions': subscription_list, + 'subscriptions': subscription_list } subscription_obj = { 'id': 'sub_12345', @@ -56,7 +56,7 @@ def test_add_subscription(self): 'amount': 1000, 'currency': 'usd', 'name': 'Test', - }, + } } self.mock_request([ (customer_obj, ''), @@ -65,14 +65,13 @@ def test_add_subscription(self): ]) # Create user and subscription - subscription_form = GoldSubscriptionForm( - { - 'level': 'v1-org-5', - 'last_4_card_digits': '0000', - 'stripe_token': 'GARYBUSEY', - 'business_vat_id': 'business-vat-id', - }, - customer=self.user, + subscription_form = GoldSubscriptionForm({ + 'level': 'v1-org-5', + 'last_4_card_digits': '0000', + 'stripe_token': 'GARYBUSEY', + 'business_vat_id': 'business-vat-id', + }, + customer=self.user, ) self.assertTrue(subscription_form.is_valid()) subscription = subscription_form.save() @@ -84,28 +83,22 @@ def test_add_subscription(self): self.assertEqual(self.user.gold.first().level, 'v1-org-5') self.mocks['request'].request.assert_has_calls([ - mock.call( - 'post', - '/v1/customers', - {'description': mock.ANY, 'email': mock.ANY, 'business_vat_id': 'business-vat-id'}, - mock.ANY, - ), - mock.call( - 'get', - '/v1/customers/cus_12345/subscriptions', - mock.ANY, - mock.ANY, - ), - mock.call( - 'post', - '/v1/customers/cus_12345/subscriptions', - {'source': mock.ANY, 'plan': 'v1-org-5'}, - mock.ANY, - ), + mock.call('post', + '/v1/customers', + {'description': mock.ANY, 'email': mock.ANY, 'business_vat_id': 'business-vat-id'}, + mock.ANY), + mock.call('get', + '/v1/customers/cus_12345/subscriptions', + mock.ANY, + mock.ANY), + mock.call('post', + '/v1/customers/cus_12345/subscriptions', + {'source': mock.ANY, 'plan': 'v1-org-5'}, + mock.ANY), ]) def test_add_subscription_update_user(self): - """Valid subscription form.""" + """Valid subscription form""" subscription_list = { 'object': 'list', 'data': [], @@ -117,7 +110,7 @@ def test_add_subscription_update_user(self): 'id': 'cus_12345', 'description': self.user.get_full_name(), 'email': self.user.email, - 'subscriptions': subscription_list, + 'subscriptions': subscription_list } subscription_obj = { 'id': 'sub_12345', @@ -129,7 +122,7 @@ def test_add_subscription_update_user(self): 'amount': 1000, 'currency': 'usd', 'name': 'Test', - }, + } } self.mock_request([ (customer_obj, ''), @@ -141,13 +134,11 @@ def test_add_subscription_update_user(self): # Create user and update the current gold subscription golduser = fixture.get(GoldUser, user=self.user, stripe_id='cus_12345') subscription_form = GoldSubscriptionForm( - { - 'level': 'v1-org-5', - 'last_4_card_digits': '0000', - 'stripe_token': 'GARYBUSEY', - }, + {'level': 'v1-org-5', + 'last_4_card_digits': '0000', + 'stripe_token': 'GARYBUSEY'}, customer=self.user, - instance=golduser, + instance=golduser ) self.assertTrue(subscription_form.is_valid()) subscription = subscription_form.save() @@ -158,34 +149,26 @@ def test_add_subscription_update_user(self): self.assertEqual(self.user.gold.first().level, 'v1-org-5') self.mocks['request'].request.assert_has_calls([ - mock.call( - 'get', - '/v1/customers/cus_12345', - {}, - mock.ANY, - ), - mock.call( - 'post', - '/v1/customers/cus_12345', - {'description': mock.ANY, 'email': mock.ANY}, - mock.ANY, - ), - mock.call( - 'get', - '/v1/customers/cus_12345/subscriptions', - mock.ANY, - mock.ANY, - ), - mock.call( - 'post', - '/v1/customers/cus_12345/subscriptions', - {'source': mock.ANY, 'plan': 'v1-org-5'}, - mock.ANY, - ), + mock.call('get', + '/v1/customers/cus_12345', + {}, + mock.ANY), + mock.call('post', + '/v1/customers/cus_12345', + {'description': mock.ANY, 'email': mock.ANY}, + mock.ANY), + mock.call('get', + '/v1/customers/cus_12345/subscriptions', + mock.ANY, + mock.ANY), + mock.call('post', + '/v1/customers/cus_12345/subscriptions', + {'source': mock.ANY, 'plan': 'v1-org-5'}, + mock.ANY), ]) def test_update_subscription_plan(self): - """Update subcription plan.""" + """Update subcription plan""" subscription_obj = { 'id': 'sub_12345', 'object': 'subscription', @@ -196,7 +179,7 @@ def test_update_subscription_plan(self): 'amount': 1000, 'currency': 'usd', 'name': 'Test', - }, + } } subscription_list = { 'object': 'list', @@ -209,7 +192,7 @@ def test_update_subscription_plan(self): 'id': 'cus_12345', 'description': self.user.get_full_name(), 'email': self.user.email, - 'subscriptions': subscription_list, + 'subscriptions': subscription_list } self.mock_request([ (customer_obj, ''), @@ -217,12 +200,10 @@ def test_update_subscription_plan(self): (subscription_obj, ''), ]) subscription_form = GoldSubscriptionForm( - { - 'level': 'v1-org-5', - 'last_4_card_digits': '0000', - 'stripe_token': 'GARYBUSEY', - }, - customer=self.user, + {'level': 'v1-org-5', + 'last_4_card_digits': '0000', + 'stripe_token': 'GARYBUSEY'}, + customer=self.user ) self.assertTrue(subscription_form.is_valid()) subscription = subscription_form.save() @@ -232,22 +213,16 @@ def test_update_subscription_plan(self): self.assertEqual(self.user.gold.first().level, 'v1-org-5') self.mocks['request'].request.assert_has_calls([ - mock.call( - 'post', - '/v1/customers', - {'description': mock.ANY, 'email': mock.ANY}, - mock.ANY, - ), - mock.call( - 'get', - '/v1/customers/cus_12345/subscriptions', - mock.ANY, - mock.ANY, - ), - mock.call( - 'post', - '/v1/subscriptions/sub_12345', - {'source': mock.ANY, 'plan': 'v1-org-5'}, - mock.ANY, - ), + mock.call('post', + '/v1/customers', + {'description': mock.ANY, 'email': mock.ANY}, + mock.ANY), + mock.call('get', + '/v1/customers/cus_12345/subscriptions', + mock.ANY, + mock.ANY), + mock.call('post', + '/v1/subscriptions/sub_12345', + {'source': mock.ANY, 'plan': 'v1-org-5'}, + mock.ANY), ]) diff --git a/readthedocs/gold/tests/test_signals.py b/readthedocs/gold/tests/test_signals.py index f55e0b53be8..62618581c06 100644 --- a/readthedocs/gold/tests/test_signals.py +++ b/readthedocs/gold/tests/test_signals.py @@ -1,10 +1,14 @@ -# -*- coding: utf-8 -*- -import django_dynamic_fixture as fixture +from __future__ import absolute_import import mock -from django.contrib.auth.models import User +import django_dynamic_fixture as fixture from django.test import TestCase +from django.contrib.auth.models import User +from django.db.models.signals import pre_delete + +from readthedocs.projects.models import Project from ..models import GoldUser +from ..signals import delete_customer class GoldSignalTests(TestCase): diff --git a/readthedocs/gold/urls.py b/readthedocs/gold/urls.py index 295e5250d10..dd8b6e354a4 100644 --- a/readthedocs/gold/urls.py +++ b/readthedocs/gold/urls.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Gold subscription URLs.""" +"""Gold subscription URLs""" +from __future__ import absolute_import from django.conf.urls import url from readthedocs.gold import views @@ -10,24 +9,12 @@ urlpatterns = [ url(r'^$', views.DetailGoldSubscription.as_view(), name='gold_detail'), - url( - r'^subscription/$', - views.UpdateGoldSubscription.as_view(), - name='gold_subscription', - ), - url( - r'^cancel/$', - views.DeleteGoldSubscription.as_view(), - name='gold_cancel', - ), + url(r'^subscription/$', views.UpdateGoldSubscription.as_view(), + name='gold_subscription'), + url(r'^cancel/$', views.DeleteGoldSubscription.as_view(), name='gold_cancel'), url(r'^projects/$', views.projects, name='gold_projects'), - url( - ( - r'^projects/remove/(?P{project_slug})/$'.format( - project_slug=PROJECT_SLUG_REGEX, - ) - ), + url((r'^projects/remove/(?P{project_slug})/$' + .format(project_slug=PROJECT_SLUG_REGEX)), views.projects_remove, - name='gold_projects_remove', - ), + name='gold_projects_remove'), ] diff --git a/readthedocs/gold/views.py b/readthedocs/gold/views.py index d5f0d6121c5..a0122d8286c 100644 --- a/readthedocs/gold/views.py +++ b/readthedocs/gold/views.py @@ -1,14 +1,20 @@ # -*- coding: utf-8 -*- - """Gold subscription views.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals +) + from django.conf import settings from django.contrib import messages from django.contrib.auth.decorators import login_required from django.contrib.messages.views import SuccessMessageMixin +from django.urls import reverse, reverse_lazy from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404, render -from django.urls import reverse, reverse_lazy from django.utils.translation import ugettext_lazy as _ from vanilla import DeleteView, DetailView, UpdateView @@ -20,11 +26,8 @@ from .models import GoldUser -class GoldSubscriptionMixin( - SuccessMessageMixin, - StripeMixin, - LoginRequiredMixin, -): +class GoldSubscriptionMixin(SuccessMessageMixin, StripeMixin, + LoginRequiredMixin): """Gold subscription mixin for view classes.""" @@ -40,16 +43,16 @@ def get_object(self): def get_form(self, data=None, files=None, **kwargs): """Pass in copy of POST data to avoid read only QueryDicts.""" kwargs['customer'] = self.request.user - return super().get_form(data, files, **kwargs) + return super(GoldSubscriptionMixin, self).get_form(data, files, **kwargs) def get_success_url(self, **__): return reverse_lazy('gold_detail') def get_template_names(self): - return ('gold/subscription{}.html'.format(self.template_name_suffix)) + return ('gold/subscription{0}.html'.format(self.template_name_suffix)) def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(GoldSubscriptionMixin, self).get_context_data(**kwargs) domains = Domain.objects.filter(project__users=self.request.user) context['domains'] = domains return context @@ -67,7 +70,7 @@ def get(self, request, *args, **kwargs): If there is a gold subscription instance, then we show the normal detail page, otherwise show the registration form """ - resp = super().get(request, *args, **kwargs) + resp = super(DetailGoldSubscription, self).get(request, *args, **kwargs) if self.object is None: return HttpResponseRedirect(reverse('gold_subscription')) return resp @@ -91,7 +94,7 @@ class DeleteGoldSubscription(GoldSubscriptionMixin, DeleteView): def post(self, request, *args, **kwargs): """Add success message to delete post.""" - resp = super().post(request, *args, **kwargs) + resp = super(DeleteGoldSubscription, self).post(request, *args, **kwargs) success_message = self.get_success_message({}) if success_message: messages.success(self.request, success_message) @@ -105,11 +108,7 @@ def projects(request): if request.method == 'POST': form = GoldProjectForm( - active_user=request.user, - data=request.POST, - user=gold_user, - projects=gold_projects, - ) + active_user=request.user, data=request.POST, user=gold_user, projects=gold_projects) if form.is_valid(): to_add = Project.objects.get(slug=form.cleaned_data['project']) gold_user.projects.add(to_add) @@ -122,16 +121,13 @@ def projects(request): form = GoldProjectForm(active_user=request.user) return render( - request, - 'gold/projects.html', - { + request, 'gold/projects.html', { 'form': form, 'gold_user': gold_user, 'publishable': settings.STRIPE_PUBLISHABLE, 'user': request.user, 'projects': gold_projects, - }, - ) + }) @login_required diff --git a/readthedocs/integrations/admin.py b/readthedocs/integrations/admin.py index 1b27ed8f5c8..cbeabea02e6 100644 --- a/readthedocs/integrations/admin.py +++ b/readthedocs/integrations/admin.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - """Integration admin models.""" -from django import urls +from __future__ import absolute_import from django.contrib import admin +from django import urls from django.utils.safestring import mark_safe from pygments.formatters import HtmlFormatter -from .models import HttpExchange, Integration +from .models import Integration, HttpExchange def pretty_json_field(field, description, include_styles=False): @@ -19,13 +18,11 @@ def inner(_, obj): if include_styles: formatter = HtmlFormatter(style='colorful') styles = '' - return mark_safe( - '
{}
{}'.format( - 'float: left;', - obj.formatted_json(field), - styles, - ), - ) + return mark_safe('
{1}
{2}'.format( + 'float: left;', + obj.formatted_json(field), + styles, + )) inner.short_description = description return inner @@ -99,20 +96,16 @@ def exchanges(self, obj): JSONField doesn't do well with fieldsets for whatever reason. This is just to link to the exchanges. """ - url = urls.reverse( - 'admin:{}_{}_changelist'.format( - HttpExchange._meta.app_label, # pylint: disable=protected-access - HttpExchange._meta.model_name, # pylint: disable=protected-access - ), - ) - return mark_safe( - '{} HTTP transactions'.format( - url, - 'integrations', - obj.pk, - obj.exchanges.count(), - ), - ) + url = urls.reverse('admin:{0}_{1}_changelist'.format( + HttpExchange._meta.app_label, # pylint: disable=protected-access + HttpExchange._meta.model_name, # pylint: disable=protected-access + )) + return mark_safe('{3} HTTP transactions'.format( + url, + 'integrations', + obj.pk, + obj.exchanges.count(), + )) exchanges.short_description = 'HTTP exchanges' diff --git a/readthedocs/integrations/migrations/0001_add_http_exchange.py b/readthedocs/integrations/migrations/0001_add_http_exchange.py index c1ee6c0b714..b4440b6eab5 100644 --- a/readthedocs/integrations/migrations/0001_add_http_exchange.py +++ b/readthedocs/integrations/migrations/0001_add_http_exchange.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-03-16 18:30 -import uuid +from __future__ import unicode_literals +from __future__ import absolute_import +from django.db import migrations, models import django.db.models.deletion import jsonfield.fields -from django.db import migrations, models +import uuid class Migration(migrations.Migration): diff --git a/readthedocs/integrations/migrations/0002_add-webhook.py b/readthedocs/integrations/migrations/0002_add-webhook.py index e42ef2a8613..3d061993c7a 100644 --- a/readthedocs/integrations/migrations/0002_add-webhook.py +++ b/readthedocs/integrations/migrations/0002_add-webhook.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-03-29 21:29 +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import migrations, models import django.db.models.deletion import jsonfield.fields -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/integrations/migrations/0003_add_missing_model_change_migrations.py b/readthedocs/integrations/migrations/0003_add_missing_model_change_migrations.py index d1a4314417d..a1356c48fa2 100644 --- a/readthedocs/integrations/migrations/0003_add_missing_model_change_migrations.py +++ b/readthedocs/integrations/migrations/0003_add_missing_model_change_migrations.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-31 11:25 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/integrations/models.py b/readthedocs/integrations/models.py index c562c372983..7514699cef6 100644 --- a/readthedocs/integrations/models.py +++ b/readthedocs/integrations/models.py @@ -2,10 +2,18 @@ """Integration models for external services.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import re import uuid +from builtins import object, str from django.contrib.contenttypes.fields import ( GenericForeignKey, GenericRelation, @@ -65,11 +73,11 @@ def from_exchange(self, req, resp, related_object, payload=None): # headers. HTTP headers are prefixed with `HTTP_`, which we remove, # and because the keys are all uppercase, we'll normalize them to # title case-y hyphen separated values. - request_headers = { - key[5:].title().replace('_', '-'): str(val) + request_headers = dict( + (key[5:].title().replace('_', '-'), str(val)) for (key, val) in list(req.META.items()) - if key.startswith('HTTP_') - } # yapf: disable + if key.startswith('HTTP_'), + ) # yapf: disable request_headers['Content-Type'] = req.content_type # Remove unwanted headers @@ -138,7 +146,7 @@ class HttpExchange(models.Model): objects = HttpExchangeManager() - class Meta: + class Meta(object): ordering = ['-date'] def __str__(self): @@ -183,11 +191,11 @@ class IntegrationQuerySet(models.QuerySet): def _get_subclass(self, integration_type): # Build a mapping of integration_type -> class dynamically - class_map = { - cls.integration_type_id: cls + class_map = dict( + (cls.integration_type_id, cls) for cls in self.model.__subclasses__() - if hasattr(cls, 'integration_type_id') - } # yapf: disable + if hasattr(cls, 'integration_type_id'), + ) # yapf: disable return class_map.get(integration_type) def _get_subclass_replacement(self, original): @@ -207,7 +215,7 @@ def _get_subclass_replacement(self, original): return new def get(self, *args, **kwargs): - original = super().get(*args, **kwargs) + original = super(IntegrationQuerySet, self).get(*args, **kwargs) return self._get_subclass_replacement(original) def subclass(self, instance): @@ -269,8 +277,7 @@ class Integration(models.Model): def __str__(self): return ( _('{0} for {1}') - .format(self.get_integration_type_display(), self.project.name) - ) + .format(self.get_integration_type_display(), self.project.name)) class GitHubWebhook(Integration): @@ -278,7 +285,7 @@ class GitHubWebhook(Integration): integration_type_id = Integration.GITHUB_WEBHOOK has_sync = True - class Meta: + class Meta(object): proxy = True @property @@ -294,7 +301,7 @@ class BitbucketWebhook(Integration): integration_type_id = Integration.BITBUCKET_WEBHOOK has_sync = True - class Meta: + class Meta(object): proxy = True @property @@ -310,7 +317,7 @@ class GitLabWebhook(Integration): integration_type_id = Integration.GITLAB_WEBHOOK has_sync = True - class Meta: + class Meta(object): proxy = True @property @@ -326,7 +333,7 @@ class GenericAPIWebhook(Integration): integration_type_id = Integration.API_WEBHOOK has_sync = False - class Meta: + class Meta(object): proxy = True def save(self, *args, **kwargs): # pylint: disable=arguments-differ @@ -339,7 +346,7 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ if token is None: token = default_token() self.provider_data = {'token': token} - super().save(*args, **kwargs) + super(GenericAPIWebhook, self).save(*args, **kwargs) @property def token(self): diff --git a/readthedocs/integrations/utils.py b/readthedocs/integrations/utils.py index 0939a36a855..978da9c8504 100644 --- a/readthedocs/integrations/utils.py +++ b/readthedocs/integrations/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Integration utility functions.""" diff --git a/readthedocs/notifications/__init__.py b/readthedocs/notifications/__init__.py index 2802d286a1e..c1860cbc8d1 100644 --- a/readthedocs/notifications/__init__.py +++ b/readthedocs/notifications/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Extensions to Django messages to support notifications to users. @@ -20,7 +18,8 @@ __all__ = ( 'Notification', 'SiteNotification', - 'send_notification', + 'send_notification' ) + default_app_config = 'readthedocs.notifications.apps.NotificationsAppConfig' diff --git a/readthedocs/notifications/apps.py b/readthedocs/notifications/apps.py index 60543374e05..38ed93cda31 100644 --- a/readthedocs/notifications/apps.py +++ b/readthedocs/notifications/apps.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- - """Django app configuration for the notifications app.""" +from __future__ import absolute_import from django.apps import AppConfig diff --git a/readthedocs/notifications/backends.py b/readthedocs/notifications/backends.py index 909248f60ca..28529f1f1a6 100644 --- a/readthedocs/notifications/backends.py +++ b/readthedocs/notifications/backends.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ Pluggable backends for the delivery of notifications. @@ -8,6 +7,10 @@ displayed on the site. """ +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + +from builtins import object from django.conf import settings from django.http import HttpRequest from django.utils.module_loading import import_string @@ -29,10 +32,14 @@ def send_notification(request, notification): backends = getattr(settings, 'NOTIFICATION_BACKENDS', []) for cls_name in backends: backend = import_string(cls_name)(request) - backend.send(notification) + # Do not send email notification if defined explicitly + if backend.name == EmailBackend.name and not notification.send_email: + pass + else: + backend.send(notification) -class Backend: +class Backend(object): def __init__(self, request): self.request = request @@ -48,16 +55,11 @@ class EmailBackend(Backend): The content body is first rendered from an on-disk template, then passed into the standard email templates as a string. - - If the notification is set to ``send_email=False``, this backend will exit - early from :py:meth:`send`. """ name = 'email' def send(self, notification): - if not notification.send_email: - return # FIXME: if the level is an ERROR an email is received and sometimes # it's not necessary. This behavior should be clearly documented in the # code @@ -112,6 +114,6 @@ def send(self, notification): backend_name=self.name, source_format=HTML, ), - extra_tags=notification.extra_tags, + extra_tags='', user=notification.user, ) diff --git a/readthedocs/notifications/constants.py b/readthedocs/notifications/constants.py index 640170aff40..d15efa98448 100644 --- a/readthedocs/notifications/constants.py +++ b/readthedocs/notifications/constants.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Notification constants.""" +"""Notification constants""" +from __future__ import absolute_import from messages_extends import constants as message_constants @@ -20,6 +19,7 @@ ERROR: message_constants.ERROR_PERSISTENT, } + # Message levels to save the message into the database and mark as read # immediately after retrieved (one-time shown message) DEBUG_NON_PERSISTENT = 100 diff --git a/readthedocs/notifications/forms.py b/readthedocs/notifications/forms.py index a5ed64e966d..b65c1c15e76 100644 --- a/readthedocs/notifications/forms.py +++ b/readthedocs/notifications/forms.py @@ -1,6 +1,5 @@ -# -*- coding: utf-8 -*- - """HTML forms for sending notifications.""" +from __future__ import absolute_import from django import forms from django.utils.translation import ugettext_lazy as _ @@ -30,14 +29,12 @@ class SendNotificationForm(forms.Form): def __init__(self, *args, **kwargs): self.notification_classes = kwargs.pop('notification_classes', []) - super().__init__(*args, **kwargs) - self.fields['source'].choices = [ - (cls.name, cls.name) - for cls in self.notification_classes - ] + super(SendNotificationForm, self).__init__(*args, **kwargs) + self.fields['source'].choices = [(cls.name, cls.name) for cls + in self.notification_classes] def clean_source(self): """Get the source class from the class name.""" source = self.cleaned_data['source'] - classes = {cls.name: cls for cls in self.notification_classes} + classes = dict((cls.name, cls) for cls in self.notification_classes) return classes.get(source, None) diff --git a/readthedocs/notifications/notification.py b/readthedocs/notifications/notification.py index d6532941f8e..d2300e1f4f0 100644 --- a/readthedocs/notifications/notification.py +++ b/readthedocs/notifications/notification.py @@ -1,23 +1,23 @@ # -*- coding: utf-8 -*- - """Support for templating of notifications.""" +from __future__ import absolute_import +from builtins import object import logging - from django.conf import settings +from django.template import Template, Context +from django.template.loader import render_to_string from django.db import models from django.http import HttpRequest -from django.template import Context, Template -from django.template.loader import render_to_string -from . import constants from .backends import send_notification +from . import constants log = logging.getLogger(__name__) -class Notification: +class Notification(object): """ An unsent notification linked to an object. @@ -35,7 +35,6 @@ class Notification: subject = None user = None send_email = True - extra_tags = '' def __init__(self, context_object, request, user=None): self.object = context_object @@ -53,8 +52,7 @@ def get_context_data(self): self.context_object_name: self.object, 'request': self.request, 'production_uri': '{scheme}://{host}'.format( - scheme='https', - host=settings.PRODUCTION_DOMAIN, + scheme='https', host=settings.PRODUCTION_DOMAIN, ), } @@ -63,13 +61,13 @@ def get_template_names(self, backend_name, source_format=constants.HTML): if self.object and isinstance(self.object, models.Model): meta = self.object._meta # pylint: disable=protected-access names.append( - '{app}/notifications/{name}_{backend}.{source_format}'.format( + '{app}/notifications/{name}_{backend}.{source_format}' + .format( app=meta.app_label, name=self.name or meta.model_name, backend=backend_name, source_format=source_format, - ), - ) + )) return names raise AttributeError() @@ -123,14 +121,8 @@ class SiteNotification(Notification): failure_level = constants.ERROR_NON_PERSISTENT def __init__( - self, - user, - success, - reason=None, - context_object=None, - request=None, - extra_context=None, - ): + self, user, success, reason=None, context_object=None, + request=None, extra_context=None): self.object = context_object self.user = user or request.user @@ -142,10 +134,10 @@ def __init__( self.success = success self.reason = reason self.extra_context = extra_context or {} - super().__init__(context_object, request, user) + super(SiteNotification, self).__init__(context_object, request, user) def get_context_data(self): - context = super().get_context_data() + context = super(SiteNotification, self).get_context_data() context.update(self.extra_context) return context diff --git a/readthedocs/notifications/storages.py b/readthedocs/notifications/storages.py index 31ee884d640..1755db952e0 100644 --- a/readthedocs/notifications/storages.py +++ b/readthedocs/notifications/storages.py @@ -1,22 +1,21 @@ # -*- coding: utf-8 -*- - """Customised storage for notifications.""" +from __future__ import absolute_import from django.contrib.messages.storage.base import Message from django.db.models import Q from django.utils.safestring import mark_safe -from messages_extends.constants import PERSISTENT_MESSAGE_LEVELS -from messages_extends.models import Message as PersistentMessage from messages_extends.storages import FallbackStorage, PersistentStorage - -from .constants import NON_PERSISTENT_MESSAGE_LEVELS - +from messages_extends.models import Message as PersistentMessage +from messages_extends.constants import PERSISTENT_MESSAGE_LEVELS try: from django.utils import timezone except ImportError: from datetime import datetime as timezone +from .constants import NON_PERSISTENT_MESSAGE_LEVELS + class FallbackUniqueStorage(FallbackStorage): @@ -50,7 +49,8 @@ class FallbackUniqueStorage(FallbackStorage): def _get(self, *args, **kwargs): # The database backend for persistent messages doesn't support setting # messages with ``mark_safe``, therefore, we need to do it broadly here. - messages, all_ret = (super()._get(self, *args, **kwargs)) + messages, all_ret = (super(FallbackUniqueStorage, self) + ._get(self, *args, **kwargs)) safe_messages = [] for message in messages: @@ -59,11 +59,9 @@ def _get(self, *args, **kwargs): # process ephemeral messages if message.level in PERSISTENT_MESSAGE_LEVELS + NON_PERSISTENT_MESSAGE_LEVELS: message_pk = message.pk - message = Message( - message.level, - mark_safe(message.message), - message.extra_tags, - ) + message = Message(message.level, + mark_safe(message.message), + message.extra_tags) message.pk = message_pk safe_messages.append(message) return safe_messages, all_ret @@ -71,16 +69,14 @@ def _get(self, *args, **kwargs): def add(self, level, message, extra_tags='', *args, **kwargs): # noqa user = kwargs.get('user') or self.request.user if not user.is_anonymous: - persist_messages = ( - PersistentMessage.objects.filter( - message=message, - user=user, - read=False, - ) - ) + persist_messages = (PersistentMessage.objects + .filter(message=message, + user=user, + read=False)) if persist_messages.exists(): return - super().add(level, message, extra_tags, *args, **kwargs) + super(FallbackUniqueStorage, self).add(level, message, extra_tags, + *args, **kwargs) class NonPersistentStorage(PersistentStorage): @@ -137,7 +133,7 @@ def process_message(self, message, *args, **kwargs): if message.level not in NON_PERSISTENT_MESSAGE_LEVELS: return message - user = kwargs.get('user') or self.get_user() + user = kwargs.get("user") or self.get_user() try: anonymous = user.is_anonymous @@ -145,15 +141,14 @@ def process_message(self, message, *args, **kwargs): anonymous = user.is_anonymous if anonymous: raise NotImplementedError( - 'Persistent message levels cannot be used for anonymous users.', - ) + 'Persistent message levels cannot be used for anonymous users.') message_persistent = PersistentMessage() message_persistent.level = message.level message_persistent.message = message.message message_persistent.extra_tags = message.extra_tags message_persistent.user = user - if 'expires' in kwargs: - message_persistent.expires = kwargs['expires'] + if "expires" in kwargs: + message_persistent.expires = kwargs["expires"] message_persistent.save() return None diff --git a/readthedocs/notifications/urls.py b/readthedocs/notifications/urls.py index 91803e3efef..349c1f293aa 100644 --- a/readthedocs/notifications/urls.py +++ b/readthedocs/notifications/urls.py @@ -1,20 +1,14 @@ -# -*- coding: utf-8 -*- - -"""Renames for messages_extends URLs.""" +"""Renames for messages_extends URLs""" +from __future__ import absolute_import from django.conf.urls import url -from messages_extends.views import message_mark_all_read, message_mark_read + +from messages_extends.views import message_mark_read, message_mark_all_read urlpatterns = [ - url( - r'^dismiss/(?P\d+)/$', - message_mark_read, - name='message_mark_read', - ), - url( - r'^dismiss/all/$', - message_mark_all_read, - name='message_mark_all_read', - ), + url(r'^dismiss/(?P\d+)/$', message_mark_read, + name='message_mark_read'), + url(r'^dismiss/all/$', message_mark_all_read, + name='message_mark_all_read'), ] diff --git a/readthedocs/notifications/views.py b/readthedocs/notifications/views.py index 0ee0495ddb8..0ba04fae0c9 100644 --- a/readthedocs/notifications/views.py +++ b/readthedocs/notifications/views.py @@ -1,9 +1,8 @@ -# -*- coding: utf-8 -*- - """Django views for the notifications app.""" +from __future__ import absolute_import +from django.views.generic import FormView from django.contrib import admin, messages from django.http import HttpResponseRedirect -from django.views.generic import FormView from .forms import SendNotificationForm @@ -15,11 +14,13 @@ class SendNotificationView(FormView): Accepts the following additional parameters: - :param queryset: Queryset to use to determine the users to send emails to - :param action_name: Name of the action to pass to the form template, - determines the action to pass back to the admin view - :param notification_classes: List of :py:class:`Notification` classes to - display in the form + queryset + The queryset to use to determine the users to send emails to + + :cvar action_name: Name of the action to pass to the form template, + determines the action to pass back to the admin view + :cvar notification_classes: List of :py:class:`Notification` classes to + display in the form """ form_class = SendNotificationForm @@ -34,7 +35,7 @@ def get_form_kwargs(self): The admin posts to this view initially, so detect the send button on form post variables. Drop additional fields if we see the send button. """ - kwargs = super().get_form_kwargs() + kwargs = super(SendNotificationView, self).get_form_kwargs() kwargs['notification_classes'] = self.notification_classes if 'send' not in self.request.POST: kwargs.pop('data', None) @@ -43,10 +44,9 @@ def get_form_kwargs(self): def get_initial(self): """Add selected ids to initial form data.""" - initial = super().get_initial() + initial = super(SendNotificationView, self).get_initial() initial['_selected_action'] = self.request.POST.getlist( - admin.ACTION_CHECKBOX_NAME, - ) + admin.ACTION_CHECKBOX_NAME) return initial def form_valid(self, form): @@ -55,17 +55,15 @@ def form_valid(self, form): notification_cls = form.cleaned_data['source'] for obj in self.get_queryset().all(): for recipient in self.get_object_recipients(obj): - notification = notification_cls( - context_object=obj, - request=self.request, - user=recipient, - ) + notification = notification_cls(context_object=obj, + request=self.request, + user=recipient) notification.send() count += 1 if count == 0: - self.message_user('No recipients to send to', level=messages.ERROR) + self.message_user("No recipients to send to", level=messages.ERROR) else: - self.message_user('Queued {} messages'.format(count)) + self.message_user("Queued {0} messages".format(count)) return HttpResponseRedirect(self.request.get_full_path()) def get_object_recipients(self, obj): @@ -91,7 +89,7 @@ def get_queryset(self): def get_context_data(self, **kwargs): """Return queryset in context.""" - context = super().get_context_data(**kwargs) + context = super(SendNotificationView, self).get_context_data(**kwargs) recipients = [] for obj in self.get_queryset().all(): recipients.extend(self.get_object_recipients(obj)) @@ -99,26 +97,14 @@ def get_context_data(self, **kwargs): context['action_name'] = self.action_name return context - def message_user( - self, - message, - level=messages.INFO, - extra_tags='', - fail_silently=False, - ): + def message_user(self, message, level=messages.INFO, extra_tags='', + fail_silently=False): """ - Implementation of. - - :py:meth:`django.contrib.admin.options.ModelAdmin.message_user` + Implementation of :py:meth:`django.contrib.admin.options.ModelAdmin.message_user` Send message through messages framework """ # TODO generalize this or check if implementation in ModelAdmin is # usable here - messages.add_message( - self.request, - level, - message, - extra_tags=extra_tags, - fail_silently=fail_silently, - ) + messages.add_message(self.request, level, message, extra_tags=extra_tags, + fail_silently=fail_silently) diff --git a/readthedocs/oauth/__init__.py b/readthedocs/oauth/__init__.py index 32bb8a9a4a5..510c93a3526 100644 --- a/readthedocs/oauth/__init__.py +++ b/readthedocs/oauth/__init__.py @@ -1,2 +1 @@ -# -*- coding: utf-8 -*- default_app_config = 'readthedocs.oauth.apps.OAuthConfig' diff --git a/readthedocs/oauth/admin.py b/readthedocs/oauth/admin.py index eeee459e6ec..b961e6472c5 100644 --- a/readthedocs/oauth/admin.py +++ b/readthedocs/oauth/admin.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- - """Admin configuration for the OAuth app.""" +from __future__ import absolute_import from django.contrib import admin -from .models import RemoteOrganization, RemoteRepository +from .models import RemoteRepository, RemoteOrganization class RemoteRepositoryAdmin(admin.ModelAdmin): diff --git a/readthedocs/oauth/apps.py b/readthedocs/oauth/apps.py index 1486873907b..b8998b8f458 100644 --- a/readthedocs/oauth/apps.py +++ b/readthedocs/oauth/apps.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- - -"""OAuth app config.""" +"""OAuth app config""" from django.apps import AppConfig diff --git a/readthedocs/oauth/migrations/0001_initial.py b/readthedocs/oauth/migrations/0001_initial.py index 3b3c4b1547b..352af573c87 100644 --- a/readthedocs/oauth/migrations/0001_initial.py +++ b/readthedocs/oauth/migrations/0001_initial.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations from django.conf import settings -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/oauth/migrations/0002_combine_services.py b/readthedocs/oauth/migrations/0002_combine_services.py index 1290dc5d95e..ab053be97f0 100644 --- a/readthedocs/oauth/migrations/0002_combine_services.py +++ b/readthedocs/oauth/migrations/0002_combine_services.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- -import django.core.validators +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations from django.conf import settings -from django.db import migrations, models +import django.core.validators class Migration(migrations.Migration): diff --git a/readthedocs/oauth/migrations/0003_move_github.py b/readthedocs/oauth/migrations/0003_move_github.py index 2b4837f4e1a..40fd4c562e0 100644 --- a/readthedocs/oauth/migrations/0003_move_github.py +++ b/readthedocs/oauth/migrations/0003_move_github.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- -import gc +from __future__ import unicode_literals + +from __future__ import absolute_import import json +import gc import logging -from django.db import migrations - +from django.db import models, migrations log = logging.getLogger(__name__) @@ -23,7 +25,7 @@ def chunks(queryset, chunksize=1000): def forwards_move_repos(apps, schema_editor): - """Moves OAuth repos.""" + """Moves OAuth repos""" db = schema_editor.connection.alias # Organizations @@ -107,7 +109,7 @@ def forwards_move_repos(apps, schema_editor): else: new_repo.clone_url = data.get('clone_url') new_repo.json = json.dumps(data) - except (SyntaxError, ValueError): + except (SyntaxError, ValueError) as e: pass new_repo.save() log.info('Migrated project: %s', project.name) @@ -141,21 +143,21 @@ def forwards_move_repos(apps, schema_editor): new_repo.private = data.get('is_private', False) new_repo.json = json.dumps(data) - clone_urls = {location['name']: location['href'] + clone_urls = dict((location['name'], location['href']) for location - in data.get('links', {}).get('clone', {})} + in data.get('links', {}).get('clone', {})) if new_repo.private: new_repo.clone_url = clone_urls.get('ssh', project.git_url) else: new_repo.clone_url = clone_urls.get('https', project.html_url) - except (SyntaxError, ValueError): + except (SyntaxError, ValueError) as e: pass new_repo.save() log.info('Migrated project: %s', project.name) def reverse_move_repos(apps, schema_editor): - """Drop OAuth repos.""" + """Drop OAuth repos""" db = schema_editor.connection.alias RemoteRepository = apps.get_model('oauth', 'RemoteRepository') RemoteOrganization = apps.get_model('oauth', 'RemoteOrganization') diff --git a/readthedocs/oauth/migrations/0004_drop_github_and_bitbucket_models.py b/readthedocs/oauth/migrations/0004_drop_github_and_bitbucket_models.py index 5b00b8377e5..628891ff795 100644 --- a/readthedocs/oauth/migrations/0004_drop_github_and_bitbucket_models.py +++ b/readthedocs/oauth/migrations/0004_drop_github_and_bitbucket_models.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations def forwards_remove_content_types(apps, schema_editor): @@ -7,10 +10,8 @@ def forwards_remove_content_types(apps, schema_editor): ContentType = apps.get_model('contenttypes', 'ContentType') ContentType.objects.using(db).filter( app_label='oauth', - model__in=[ - 'githubproject', 'githuborganization', - 'bitbucketproject', 'bitbucketteam', - ], + model__in=['githubproject', 'githuborganization', + 'bitbucketproject', 'bitbucketteam'] ).delete() diff --git a/readthedocs/oauth/migrations/0005_add_account_relation.py b/readthedocs/oauth/migrations/0005_add_account_relation.py index c8c466db37f..100bcd71aef 100644 --- a/readthedocs/oauth/migrations/0005_add_account_relation.py +++ b/readthedocs/oauth/migrations/0005_add_account_relation.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/oauth/migrations/0006_move_oauth_source.py b/readthedocs/oauth/migrations/0006_move_oauth_source.py index 8689b134fa2..a19d0be04a7 100644 --- a/readthedocs/oauth/migrations/0006_move_oauth_source.py +++ b/readthedocs/oauth/migrations/0006_move_oauth_source.py @@ -1,9 +1,12 @@ # -*- coding: utf-8 -*- -from django.db import migrations +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations def forwards_move_repo_source(apps, schema_editor): - """Use source field to set repository account.""" + """Use source field to set repository account""" RemoteRepository = apps.get_model('oauth', 'RemoteRepository') SocialAccount = apps.get_model('socialaccount', 'SocialAccount') for account in SocialAccount.objects.all(): @@ -13,7 +16,7 @@ def forwards_move_repo_source(apps, schema_editor): def backwards_move_repo_source(apps, schema_editor): - apps.get_model('oauth', 'RemoteRepository') + RemoteRepository = apps.get_model('oauth', 'RemoteRepository') SocialAccount = apps.get_model('socialaccount', 'SocialAccount') for account in SocialAccount.objects.all(): rows = (account.remote_repositories @@ -21,7 +24,7 @@ def backwards_move_repo_source(apps, schema_editor): def forwards_move_org_source(apps, schema_editor): - """Use source field to set organization account.""" + """Use source field to set organization account""" RemoteOrganization = apps.get_model('oauth', 'RemoteOrganization') SocialAccount = apps.get_model('socialaccount', 'SocialAccount') for account in SocialAccount.objects.all(): @@ -31,8 +34,8 @@ def forwards_move_org_source(apps, schema_editor): def backwards_move_org_source(apps, schema_editor): - """Use source field to set organization account.""" - apps.get_model('oauth', 'RemoteOrganization') + """Use source field to set organization account""" + RemoteOrganization = apps.get_model('oauth', 'RemoteOrganization') SocialAccount = apps.get_model('socialaccount', 'SocialAccount') for account in SocialAccount.objects.all(): rows = (account.remote_organizations @@ -46,12 +49,8 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython( - forwards_move_repo_source, - backwards_move_repo_source, - ), - migrations.RunPython( - forwards_move_org_source, - backwards_move_org_source, - ), + migrations.RunPython(forwards_move_repo_source, + backwards_move_repo_source), + migrations.RunPython(forwards_move_org_source, + backwards_move_org_source), ] diff --git a/readthedocs/oauth/migrations/0007_org_slug_nonunique.py b/readthedocs/oauth/migrations/0007_org_slug_nonunique.py index 97078038491..65f6f4f4f70 100644 --- a/readthedocs/oauth/migrations/0007_org_slug_nonunique.py +++ b/readthedocs/oauth/migrations/0007_org_slug_nonunique.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/oauth/migrations/0008_add-project-relation.py b/readthedocs/oauth/migrations/0008_add-project-relation.py index 070b57e654c..1e2a478e69f 100644 --- a/readthedocs/oauth/migrations/0008_add-project-relation.py +++ b/readthedocs/oauth/migrations/0008_add-project-relation.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-03-22 20:10 -import django.db.models.deletion +from __future__ import unicode_literals + +from __future__ import absolute_import from django.db import migrations, models +import django.db.models.deletion class Migration(migrations.Migration): diff --git a/readthedocs/oauth/migrations/0009_add_missing_model_change_migrations.py b/readthedocs/oauth/migrations/0009_add_missing_model_change_migrations.py index c23743a846a..015c233ac20 100644 --- a/readthedocs/oauth/migrations/0009_add_missing_model_change_migrations.py +++ b/readthedocs/oauth/migrations/0009_add_missing_model_change_migrations.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-31 11:25 +from __future__ import unicode_literals + import django.core.validators from django.db import migrations, models diff --git a/readthedocs/oauth/models.py b/readthedocs/oauth/models.py index 40d224df6b3..b93f71b9faa 100644 --- a/readthedocs/oauth/models.py +++ b/readthedocs/oauth/models.py @@ -1,15 +1,19 @@ # -*- coding: utf-8 -*- - """OAuth service models.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import json +from builtins import object from allauth.socialaccount.models import SocialAccount +from django.conf import settings from django.contrib.auth.models import User +from django.urls import reverse from django.core.validators import URLValidator from django.db import models from django.db.models import Q -from django.urls import reverse from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ @@ -33,17 +37,10 @@ class RemoteOrganization(models.Model): modified_date = models.DateTimeField(_('Modified date'), auto_now=True) users = models.ManyToManyField( - User, - verbose_name=_('Users'), - related_name='oauth_organizations', - ) + User, verbose_name=_('Users'), related_name='oauth_organizations') account = models.ForeignKey( - SocialAccount, - verbose_name=_('Connected account'), - related_name='remote_organizations', - null=True, - blank=True, - ) + SocialAccount, verbose_name=_('Connected account'), + related_name='remote_organizations', null=True, blank=True) active = models.BooleanField(_('Active'), default=False) slug = models.CharField(_('Slug'), max_length=255) @@ -51,11 +48,7 @@ class RemoteOrganization(models.Model): email = models.EmailField(_('Email'), max_length=255, null=True, blank=True) avatar_url = models.URLField(_('Avatar image URL'), null=True, blank=True) url = models.URLField( - _('URL to organization page'), - max_length=200, - null=True, - blank=True, - ) + _('URL to organization page'), max_length=200, null=True, blank=True) json = models.TextField(_('Serialized API response')) @@ -89,24 +82,13 @@ class RemoteRepository(models.Model): # This should now be a OneToOne users = models.ManyToManyField( - User, - verbose_name=_('Users'), - related_name='oauth_repositories', - ) + User, verbose_name=_('Users'), related_name='oauth_repositories') account = models.ForeignKey( - SocialAccount, - verbose_name=_('Connected account'), - related_name='remote_repositories', - null=True, - blank=True, - ) + SocialAccount, verbose_name=_('Connected account'), + related_name='remote_repositories', null=True, blank=True) organization = models.ForeignKey( - RemoteOrganization, - verbose_name=_('Organization'), - related_name='repositories', - null=True, - blank=True, - ) + RemoteOrganization, verbose_name=_('Organization'), + related_name='repositories', null=True, blank=True) active = models.BooleanField(_('Active'), default=False) project = models.OneToOneField( @@ -141,7 +123,7 @@ class RemoteRepository(models.Model): max_length=512, blank=True, validators=[ - URLValidator(schemes=['http', 'https', 'ssh', 'git', 'svn']), + URLValidator(schemes=['http', 'https', 'ssh', 'git', 'svn']) ], ) html_url = models.URLField(_('HTML URL'), null=True, blank=True) @@ -159,7 +141,7 @@ class RemoteRepository(models.Model): objects = RemoteRepositoryQuerySet.as_manager() - class Meta: + class Meta(object): ordering = ['organization__name', 'name'] verbose_name_plural = 'remote repositories' @@ -178,6 +160,7 @@ def get_serialized(self, key=None, default=None): @property def clone_fuzzy_url(self): """Try to match against several permutations of project URL.""" + pass def matches(self, user): """Projects that exist with repository URL already.""" diff --git a/readthedocs/oauth/notifications.py b/readthedocs/oauth/notifications.py index c5c5165aa7f..fc9aefc0c3b 100644 --- a/readthedocs/oauth/notifications.py +++ b/readthedocs/oauth/notifications.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from django.urls import reverse from django.utils.translation import ugettext_lazy as _ from messages_extends.constants import ERROR_PERSISTENT @@ -15,16 +17,12 @@ class AttachWebhookNotification(SiteNotification): context_object_name = 'provider' success_message = _('Webhook successfully added.') failure_message = { - NO_PERMISSIONS: _( - 'Could not add webhook for {{ project.name }}. Make sure you have the correct {{ provider.name }} permissions.', # noqa - ), - NO_ACCOUNTS: _( - 'Could not add webhook for {{ project.name }}. Please connect your {{ provider.name }} account.', # noqa - ), + NO_PERMISSIONS: _('Could not add webhook for {{ project.name }}. Make sure you have the correct {{ provider.name }} permissions.'), # noqa + NO_ACCOUNTS: _('Could not add webhook for {{ project.name }}. Please connect your {{ provider.name }} account.'), # noqa } def get_context_data(self): - context = super().get_context_data() + context = super(AttachWebhookNotification, self).get_context_data() project = self.extra_context.get('project') context.update({ 'url_connect_account': reverse( @@ -43,11 +41,10 @@ class InvalidProjectWebhookNotification(SiteNotification): failure_message = _( "The project {{ project.name }} doesn't have a valid webhook set up, " "commits won't trigger new builds for this project. " - "See the project integrations for more information.", - ) # noqa + "See the project integrations for more information.") # noqa def get_context_data(self): - context = super().get_context_data() + context = super(InvalidProjectWebhookNotification, self).get_context_data() context.update({ 'url_integrations': reverse( 'projects_integrations', diff --git a/readthedocs/oauth/querysets.py b/readthedocs/oauth/querysets.py index e7f20dc184e..d01703eb1f0 100644 --- a/readthedocs/oauth/querysets.py +++ b/readthedocs/oauth/querysets.py @@ -1,6 +1,6 @@ -# -*- coding: utf-8 -*- +"""Managers for OAuth models""" -"""Managers for OAuth models.""" +from __future__ import absolute_import from django.db import models @@ -12,7 +12,7 @@ class RelatedUserQuerySetBase(models.QuerySet): """For models with relations through :py:class:`User`""" def api(self, user=None): - """Return objects for user.""" + """Return objects for user""" if not user.is_authenticated: return self.none() return self.filter(users=user) diff --git a/readthedocs/oauth/services/__init__.py b/readthedocs/oauth/services/__init__.py index a249e15d934..b1b5003b08a 100644 --- a/readthedocs/oauth/services/__init__.py +++ b/readthedocs/oauth/services/__init__.py @@ -1,13 +1,8 @@ # -*- coding: utf-8 -*- - """Conditional classes for OAuth services.""" from __future__ import ( - absolute_import, - division, - print_function, - unicode_literals, -) + absolute_import, division, print_function, unicode_literals) from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.oauth.services import bitbucket, github, gitlab diff --git a/readthedocs/oauth/services/base.py b/readthedocs/oauth/services/base.py index b1f0e7a12c5..93064779ef9 100644 --- a/readthedocs/oauth/services/base.py +++ b/readthedocs/oauth/services/base.py @@ -1,23 +1,25 @@ # -*- coding: utf-8 -*- - """OAuth utility functions.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging from datetime import datetime from allauth.socialaccount.models import SocialAccount from allauth.socialaccount.providers import registry +from builtins import object from django.conf import settings from django.utils import timezone from oauthlib.oauth2.rfc6749.errors import InvalidClientIdError from requests.exceptions import RequestException from requests_oauthlib import OAuth2Session - log = logging.getLogger(__name__) -class Service: +class Service(object): """ Service mapping for local accounts. @@ -116,11 +118,10 @@ def token_updater(self, token): u'expires_at': 1449218652.558185 } """ - def _updater(data): token.token = data['access_token'] token.expires_at = timezone.make_aware( - datetime.fromtimestamp(data['expires_at']), + datetime.fromtimestamp(data['expires_at']) ) token.save() log.info('Updated token %s:', token) diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 817ea98bd75..0ce2f55a21c 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -1,17 +1,16 @@ -# -*- coding: utf-8 -*- - """OAuth utility functions.""" -import json +from __future__ import absolute_import +from builtins import str import logging +import json import re -from allauth.socialaccount.providers.bitbucket_oauth2.views import ( - BitbucketOAuth2Adapter, -) from django.conf import settings from django.urls import reverse from requests.exceptions import RequestException +from allauth.socialaccount.providers.bitbucket_oauth2.views import ( + BitbucketOAuth2Adapter) from readthedocs.builds import utils as build_utils from readthedocs.integrations.models import Integration @@ -42,30 +41,25 @@ def sync_repositories(self): # Get user repos try: repos = self.paginate( - 'https://bitbucket.org/api/2.0/repositories/?role=member', - ) + 'https://bitbucket.org/api/2.0/repositories/?role=member') for repo in repos: self.create_repository(repo) - except (TypeError, ValueError): + except (TypeError, ValueError) as e: log.exception('Error syncing Bitbucket repositories') - raise Exception( - 'Could not sync your Bitbucket repositories, ' - 'try reconnecting your account', - ) + raise Exception('Could not sync your Bitbucket repositories, ' + 'try reconnecting your account') # Because privileges aren't returned with repository data, run query # again for repositories that user has admin role for, and update # existing repositories. try: resp = self.paginate( - 'https://bitbucket.org/api/2.0/repositories/?role=admin', - ) + 'https://bitbucket.org/api/2.0/repositories/?role=admin') repos = ( - RemoteRepository.objects.filter( - users=self.user, - full_name__in=[r['full_name'] for r in resp], - account=self.account, - ) + RemoteRepository.objects + .filter(users=self.user, + full_name__in=[r['full_name'] for r in resp], + account=self.account) ) for repo in repos: repo.admin = True @@ -77,19 +71,17 @@ def sync_teams(self): """Sync Bitbucket teams and team repositories.""" try: teams = self.paginate( - 'https://api.bitbucket.org/2.0/teams/?role=member', + 'https://api.bitbucket.org/2.0/teams/?role=member' ) for team in teams: org = self.create_organization(team) repos = self.paginate(team['links']['repositories']['href']) for repo in repos: self.create_repository(repo, organization=org) - except ValueError: + except ValueError as e: log.exception('Error syncing Bitbucket organizations') - raise Exception( - 'Could not sync your Bitbucket team repositories, ' - 'try reconnecting your account', - ) + raise Exception('Could not sync your Bitbucket team repositories, ' + 'try reconnecting your account') def create_repository(self, fields, privacy=None, organization=None): """ @@ -107,17 +99,17 @@ def create_repository(self, fields, privacy=None, organization=None): :rtype: RemoteRepository """ privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL - if ((privacy == 'private') or - (fields['is_private'] is False and privacy == 'public')): + if ( + (privacy == 'private') or + (fields['is_private'] is False and privacy == 'public') + ): repo, _ = RemoteRepository.objects.get_or_create( full_name=fields['full_name'], account=self.account, ) if repo.organization and repo.organization != organization: - log.debug( - 'Not importing %s because mismatched orgs', - fields['name'], - ) + log.debug('Not importing %s because mismatched orgs', + fields['name']) return None repo.organization = organization @@ -127,13 +119,11 @@ def create_repository(self, fields, privacy=None, organization=None): repo.private = fields['is_private'] # Default to HTTPS, use SSH for private repositories - clone_urls = { - u['name']: u['href'] - for u in fields['links']['clone'] - } + clone_urls = dict((u['name'], u['href']) + for u in fields['links']['clone']) repo.clone_url = self.https_url_pattern.sub( 'https://bitbucket.org/', - clone_urls.get('https'), + clone_urls.get('https') ) repo.ssh_url = clone_urls.get('ssh') if repo.private: @@ -189,18 +179,14 @@ def get_paginated_results(self, response): def get_webhook_data(self, project, integration): """Get webhook JSON data to post to the API.""" return json.dumps({ - 'description': 'Read the Docs ({domain})'.format( - domain=settings.PRODUCTION_DOMAIN, - ), + 'description': 'Read the Docs ({domain})'.format(domain=settings.PRODUCTION_DOMAIN), 'url': 'https://{domain}{path}'.format( domain=settings.PRODUCTION_DOMAIN, path=reverse( 'api_webhook', - kwargs={ - 'project_slug': project.slug, - 'integration_pk': integration.pk, - }, - ), + kwargs={'project_slug': project.slug, + 'integration_pk': integration.pk} + ) ), 'active': True, 'events': ['repo:push'], @@ -225,12 +211,10 @@ def setup_webhook(self, project): resp = None try: resp = session.post( - ( - 'https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/hooks' - .format(owner=owner, repo=repo) - ), + ('https://api.bitbucket.org/2.0/repositories/{owner}/{repo}/hooks' + .format(owner=owner, repo=repo)), data=data, - headers={'content-type': 'application/json'}, + headers={'content-type': 'application/json'} ) if resp.status_code == 201: recv_data = resp.json() @@ -281,7 +265,7 @@ def update_webhook(self, project, integration): resp = session.put( url, data=data, - headers={'content-type': 'application/json'}, + headers={'content-type': 'application/json'} ) if resp.status_code == 200: recv_data = resp.json() diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 099743cd6bb..f9a03c4ff76 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- - """OAuth utility functions.""" -import json +from __future__ import absolute_import +from builtins import str import logging +import json import re -from allauth.socialaccount.models import SocialToken -from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter from django.conf import settings from django.urls import reverse from requests.exceptions import RequestException +from allauth.socialaccount.models import SocialToken +from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter from readthedocs.builds import utils as build_utils from readthedocs.integrations.models import Integration @@ -19,7 +19,6 @@ from ..models import RemoteOrganization, RemoteRepository from .base import Service - log = logging.getLogger(__name__) @@ -42,12 +41,10 @@ def sync_repositories(self): try: for repo in repos: self.create_repository(repo) - except (TypeError, ValueError): + except (TypeError, ValueError) as e: log.exception('Error syncing GitHub repositories') - raise Exception( - 'Could not sync your GitHub repositories, ' - 'try reconnecting your account', - ) + raise Exception('Could not sync your GitHub repositories, ' + 'try reconnecting your account') def sync_organizations(self): """Sync organizations from GitHub API.""" @@ -59,16 +56,14 @@ def sync_organizations(self): # Add repos # TODO ?per_page=100 org_repos = self.paginate( - '{org_url}/repos'.format(org_url=org['url']), + '{org_url}/repos'.format(org_url=org['url']) ) for repo in org_repos: self.create_repository(repo, organization=org_obj) - except (TypeError, ValueError): + except (TypeError, ValueError) as e: log.exception('Error syncing GitHub organizations') - raise Exception( - 'Could not sync your GitHub organizations, ' - 'try reconnecting your account', - ) + raise Exception('Could not sync your GitHub organizations, ' + 'try reconnecting your account') def create_repository(self, fields, privacy=None, organization=None): """ @@ -81,8 +76,10 @@ def create_repository(self, fields, privacy=None, organization=None): :rtype: RemoteRepository """ privacy = privacy or settings.DEFAULT_PRIVACY_LEVEL - if ((privacy == 'private') or - (fields['private'] is False and privacy == 'public')): + if ( + (privacy == 'private') or + (fields['private'] is False and privacy == 'public') + ): try: repo = RemoteRepository.objects.get( full_name=fields['full_name'], @@ -96,10 +93,8 @@ def create_repository(self, fields, privacy=None, organization=None): ) repo.users.add(self.user) if repo.organization and repo.organization != organization: - log.debug( - 'Not importing %s because mismatched orgs', - fields['name'], - ) + log.debug('Not importing %s because mismatched orgs', + fields['name']) return None repo.organization = organization @@ -122,10 +117,8 @@ def create_repository(self, fields, privacy=None, organization=None): repo.save() return repo else: - log.debug( - 'Not importing %s because mismatched type', - fields['name'], - ) + log.debug('Not importing %s because mismatched type', + fields['name']) def create_organization(self, fields): """ @@ -173,11 +166,9 @@ def get_webhook_data(self, project, integration): domain=settings.PRODUCTION_DOMAIN, path=reverse( 'api_webhook', - kwargs={ - 'project_slug': project.slug, - 'integration_pk': integration.pk, - }, - ), + kwargs={'project_slug': project.slug, + 'integration_pk': integration.pk} + ) ), 'content_type': 'json', }, @@ -203,31 +194,19 @@ def setup_webhook(self, project): resp = None try: resp = session.post( - ( - 'https://api.github.com/repos/{owner}/{repo}/hooks' - .format(owner=owner, repo=repo) - ), + ('https://api.github.com/repos/{owner}/{repo}/hooks' + .format(owner=owner, repo=repo)), data=data, - headers={'content-type': 'application/json'}, + headers={'content-type': 'application/json'} ) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: recv_data = resp.json() integration.provider_data = recv_data integration.save() - log.info( - 'GitHub webhook creation successful for project: %s', - project, - ) + log.info('GitHub webhook creation successful for project: %s', + project) return (True, resp) - - if resp.status_code in [401, 403, 404]: - log.info( - 'GitHub project does not exist or user does not have ' - 'permissions: project=%s', - project, - ) - return (False, resp) # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception( @@ -270,7 +249,7 @@ def update_webhook(self, project, integration): resp = session.patch( url, data=data, - headers={'content-type': 'application/json'}, + headers={'content-type': 'application/json'} ) # GitHub will return 200 if already synced if resp.status_code in [200, 201]: @@ -323,8 +302,7 @@ def get_token_for_project(cls, project, force_local=False): for user in project.users.all(): tokens = SocialToken.objects.filter( account__user=user, - app__provider=cls.adapter.provider_id, - ) + app__provider=cls.adapter.provider_id) if tokens.exists(): token = tokens[0].token except Exception: diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 5f94bd51690..b9562617adf 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- - """OAuth utility functions.""" +from __future__ import division, print_function, unicode_literals + import json import logging import re @@ -18,7 +19,6 @@ from ..models import RemoteOrganization, RemoteRepository from .base import Service - try: from urlparse import urljoin, urlparse except ImportError: @@ -41,8 +41,7 @@ class GitLabService(Service): # Just use the network location to determine if it's a GitLab project # because private repos have another base url, eg. git@gitlab.example.com url_pattern = re.compile( - re.escape(urlparse(adapter.provider_base_url).netloc), - ) + re.escape(urlparse(adapter.provider_base_url).netloc)) def _get_repo_id(self, project): # The ID or URL-encoded path of the project @@ -95,8 +94,7 @@ def sync_repositories(self): log.exception('Error syncing GitLab repositories') raise Exception( 'Could not sync your GitLab repositories, try reconnecting ' - 'your account', - ) + 'your account') def sync_organizations(self): orgs = self.paginate( @@ -126,8 +124,7 @@ def sync_organizations(self): log.exception('Error syncing GitLab organizations') raise Exception( 'Could not sync your GitLab organization, try reconnecting ' - 'your account', - ) + 'your account') def is_owned_by(self, owner_id): return self.account.extra_data['id'] == owner_id @@ -352,9 +349,7 @@ def update_webhook(self, project, integration): integration.provider_data = recv_data integration.save() log.info( - 'GitLab webhook update successful for project: %s', - project, - ) + 'GitLab webhook update successful for project: %s', project) return (True, resp) # GitLab returns 404 when the webhook doesn't exist. In this case, @@ -365,9 +360,7 @@ def update_webhook(self, project, integration): # Catch exceptions with request or deserializing JSON except (RequestException, ValueError): log.exception( - 'GitLab webhook update failed for project: %s', - project, - ) + 'GitLab webhook update failed for project: %s', project) else: log.error( 'GitLab webhook update failed for project: %s', diff --git a/readthedocs/oauth/tasks.py b/readthedocs/oauth/tasks.py index 18c0ccb2eca..45b49ceac09 100644 --- a/readthedocs/oauth/tasks.py +++ b/readthedocs/oauth/tasks.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- - """Tasks for OAuth services.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging from allauth.socialaccount.providers import registry as allauth_registry @@ -17,7 +23,6 @@ from .services import registry - log = logging.getLogger(__name__) diff --git a/readthedocs/oauth/utils.py b/readthedocs/oauth/utils.py index b33fc9a6e65..64e7dc7ed07 100644 --- a/readthedocs/oauth/utils.py +++ b/readthedocs/oauth/utils.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- - """Support code for OAuth, including webhook support.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging from django.contrib import messages @@ -9,14 +11,9 @@ from readthedocs.integrations.models import Integration from readthedocs.oauth.services import ( - BitbucketService, - GitHubService, - GitLabService, - registry, -) + BitbucketService, GitHubService, GitLabService, registry) from readthedocs.projects.models import Project - log = logging.getLogger(__name__) SERVICE_MAP = { @@ -55,9 +52,7 @@ def update_webhook(project, integration, request=None): request, _( 'Webhook activation failed. ' - 'Make sure you have the necessary permissions.', - ), - ) + 'Make sure you have the necessary permissions.')) project.has_valid_webhook = False project.save() return False diff --git a/readthedocs/payments/forms.py b/readthedocs/payments/forms.py index 59f9bce97eb..eae6e88dcb6 100644 --- a/readthedocs/payments/forms.py +++ b/readthedocs/payments/forms.py @@ -1,21 +1,21 @@ -# -*- coding: utf-8 -*- - """Payment forms.""" +from __future__ import absolute_import +from builtins import str +from builtins import object import logging +from stripe import Customer, Charge +from stripe.error import InvalidRequestError from django import forms from django.utils.translation import ugettext_lazy as _ -from stripe import Charge, Customer -from stripe.error import InvalidRequestError from .utils import stripe - log = logging.getLogger(__name__) -class StripeResourceMixin: +class StripeResourceMixin(object): """Stripe actions for resources, available as a Form mixin class.""" @@ -38,29 +38,23 @@ def get_customer_kwargs(self): raise NotImplementedError def get_customer(self): - return self.ensure_stripe_resource( - resource=Customer, - attrs=self.get_customer_kwargs(), - ) + return self.ensure_stripe_resource(resource=Customer, + attrs=self.get_customer_kwargs()) def get_subscription_kwargs(self): raise NotImplementedError def get_subscription(self): customer = self.get_customer() - return self.ensure_stripe_resource( - resource=customer.subscriptions, - attrs=self.get_subscription_kwargs(), - ) + return self.ensure_stripe_resource(resource=customer.subscriptions, + attrs=self.get_subscription_kwargs()) def get_charge_kwargs(self): raise NotImplementedError def get_charge(self): - return self.ensure_stripe_resource( - resource=Charge, - attrs=self.get_charge_kwargs(), - ) + return self.ensure_stripe_resource(resource=Charge, + attrs=self.get_charge_kwargs()) class StripeModelForm(forms.ModelForm): @@ -85,62 +79,45 @@ class StripeModelForm(forms.ModelForm): # Stripe token input from Stripe.js stripe_token = forms.CharField( required=False, - widget=forms.HiddenInput( - attrs={ - 'data-bind': 'valueInit: stripe_token', - }, - ), + widget=forms.HiddenInput(attrs={ + 'data-bind': 'valueInit: stripe_token', + }) ) # Fields used for fetching token with javascript, listed as form fields so # that data can survive validation errors cc_number = forms.CharField( label=_('Card number'), - widget=forms.TextInput( - attrs={ - 'data-bind': ( - 'valueInit: cc_number, ' - 'textInput: cc_number, ' - '''css: {'field-error': error_cc_number() != null}''' - ), - }, - ), + widget=forms.TextInput(attrs={ + 'data-bind': ('valueInit: cc_number, ' + 'textInput: cc_number, ' + '''css: {'field-error': error_cc_number() != null}''') + }), max_length=25, - required=False, - ) + required=False) cc_expiry = forms.CharField( label=_('Card expiration'), - widget=forms.TextInput( - attrs={ - 'data-bind': ( - 'valueInit: cc_expiry, ' - 'textInput: cc_expiry, ' - '''css: {'field-error': error_cc_expiry() != null}''' - ), - }, - ), + widget=forms.TextInput(attrs={ + 'data-bind': ('valueInit: cc_expiry, ' + 'textInput: cc_expiry, ' + '''css: {'field-error': error_cc_expiry() != null}''') + }), max_length=10, - required=False, - ) + required=False) cc_cvv = forms.CharField( label=_('Card CVV'), - widget=forms.TextInput( - attrs={ - 'data-bind': ( - 'valueInit: cc_cvv, ' - 'textInput: cc_cvv, ' - '''css: {'field-error': error_cc_cvv() != null}''' - ), - 'autocomplete': 'off', - }, - ), + widget=forms.TextInput(attrs={ + 'data-bind': ('valueInit: cc_cvv, ' + 'textInput: cc_cvv, ' + '''css: {'field-error': error_cc_cvv() != null}'''), + 'autocomplete': 'off', + }), max_length=8, - required=False, - ) + required=False) def __init__(self, *args, **kwargs): self.customer = kwargs.pop('customer', None) - super().__init__(*args, **kwargs) + super(StripeModelForm, self).__init__(*args, **kwargs) def validate_stripe(self): """ @@ -170,7 +147,7 @@ def clean(self): raise any issues as validation errors. This is required because part of Stripe's validation happens on the API call to establish a subscription. """ - cleaned_data = super().clean() + cleaned_data = super(StripeModelForm, self).clean() # Form isn't valid, no need to try to associate a card now if not self.is_valid(): @@ -196,8 +173,7 @@ def clean(self): except stripe.error.StripeError as e: log.exception('There was a problem communicating with Stripe') raise forms.ValidationError( - _('There was a problem communicating with Stripe'), - ) + _('There was a problem communicating with Stripe')) return cleaned_data def clear_card_data(self): @@ -210,14 +186,12 @@ def clear_card_data(self): try: self.data['stripe_token'] = None except AttributeError: - raise AttributeError( - 'Form was passed immutable QueryDict POST data', - ) + raise AttributeError('Form was passed immutable QueryDict POST data') def fields_with_cc_group(self): group = { 'is_cc_group': True, - 'fields': [], + 'fields': [] } for field in self: if field.name in ['cc_number', 'cc_expiry', 'cc_cvv']: diff --git a/readthedocs/payments/mixins.py b/readthedocs/payments/mixins.py index 4bce56d8216..0219da08098 100644 --- a/readthedocs/payments/mixins.py +++ b/readthedocs/payments/mixins.py @@ -1,16 +1,16 @@ -# -*- coding: utf-8 -*- - """Payment view mixin classes.""" +from __future__ import absolute_import +from builtins import object from django.conf import settings -class StripeMixin: +class StripeMixin(object): """Adds Stripe publishable key to the context data.""" def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(StripeMixin, self).get_context_data(**kwargs) context['stripe_publishable'] = settings.STRIPE_PUBLISHABLE return context diff --git a/readthedocs/payments/utils.py b/readthedocs/payments/utils.py index ba1b045a2f8..a65b5b0a8f6 100644 --- a/readthedocs/payments/utils.py +++ b/readthedocs/payments/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Payment utility functions. @@ -7,10 +5,10 @@ :py:class:`readthedocs.payments.forms.StripeResourceMixin`. """ +from __future__ import absolute_import import stripe from django.conf import settings - stripe.api_key = getattr(settings, 'STRIPE_SECRET', None) diff --git a/readthedocs/profiles/urls/private.py b/readthedocs/profiles/urls/private.py index ca6c13428f0..85acbf2d9fa 100644 --- a/readthedocs/profiles/urls/private.py +++ b/readthedocs/profiles/urls/private.py @@ -1,7 +1,12 @@ -# -*- coding: utf-8 -*- - """URL patterns for views to modify user profiles.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from django.conf.urls import url from readthedocs.core.forms import UserProfileForm @@ -10,13 +15,12 @@ urlpatterns = [ url( - r'^edit/', - views.edit_profile, + r'^edit/', views.edit_profile, { 'form_class': UserProfileForm, 'template_name': 'profiles/private/edit_profile.html', }, - name='profiles_profile_edit', + name='profiles_profile_edit' ), url(r'^delete/', views.delete_account, name='delete_account'), url( diff --git a/readthedocs/profiles/urls/public.py b/readthedocs/profiles/urls/public.py index d2cec291bae..2a9c458e6fd 100644 --- a/readthedocs/profiles/urls/public.py +++ b/readthedocs/profiles/urls/public.py @@ -1,17 +1,14 @@ -# -*- coding: utf-8 -*- - """URL patterns to view user profiles.""" +from __future__ import absolute_import from django.conf.urls import url from readthedocs.profiles import views urlpatterns = [ - url( - r'^(?P[+\w@.-]+)/$', + url(r'^(?P[+\w@.-]+)/$', views.profile_detail, {'template_name': 'profiles/public/profile_detail.html'}, - name='profiles_profile_detail', - ), + name='profiles_profile_detail'), ] diff --git a/readthedocs/profiles/views.py b/readthedocs/profiles/views.py index e2e85b22de8..d288f4767bf 100644 --- a/readthedocs/profiles/views.py +++ b/readthedocs/profiles/views.py @@ -1,14 +1,20 @@ # -*- coding: utf-8 -*- - """Views for creating, editing and viewing site-specific user profiles.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from django.contrib import messages from django.contrib.auth import logout from django.contrib.auth.decorators import login_required from django.contrib.auth.models import User +from django.urls import reverse from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404, redirect, render -from django.urls import reverse from django.utils.translation import ugettext_lazy as _ from readthedocs.core.forms import UserAdvertisingForm, UserDeleteForm @@ -16,12 +22,8 @@ @login_required def edit_profile( - request, - form_class, - success_url=None, - template_name='profiles/private/edit_profile.html', - extra_context=None, -): + request, form_class, success_url=None, + template_name='profiles/private/edit_profile.html', extra_context=None): """ Edit the current user's profile. @@ -68,14 +70,10 @@ def edit_profile( if success_url is None: success_url = reverse( 'profiles_profile_detail', - kwargs={'username': request.user.username}, - ) + kwargs={'username': request.user.username}) if request.method == 'POST': form = form_class( - data=request.POST, - files=request.FILES, - instance=profile_obj, - ) + data=request.POST, files=request.FILES, instance=profile_obj) if form.is_valid(): form.save() return HttpResponseRedirect(success_url) @@ -116,12 +114,9 @@ def delete_account(request): def profile_detail( - request, - username, - public_profile_field=None, + request, username, public_profile_field=None, template_name='profiles/public/profile_detail.html', - extra_context=None, -): + extra_context=None): """ Detail view of a user's profile. diff --git a/readthedocs/projects/__init__.py b/readthedocs/projects/__init__.py index 186dc5eb841..ff5ded49b17 100644 --- a/readthedocs/projects/__init__.py +++ b/readthedocs/projects/__init__.py @@ -1,2 +1 @@ -# -*- coding: utf-8 -*- default_app_config = 'readthedocs.projects.apps.ProjectsConfig' diff --git a/readthedocs/projects/admin.py b/readthedocs/projects/admin.py index 73914222ce3..d55954fdbcf 100644 --- a/readthedocs/projects/admin.py +++ b/readthedocs/projects/admin.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- - """Django administration interface for `projects.models`""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from django.contrib import admin, messages from django.contrib.admin.actions import delete_selected from django.utils.translation import ugettext_lazy as _ @@ -24,20 +30,12 @@ ProjectRelationship, WebHook, ) -from .notifications import ( - DeprecatedBuildWebhookNotification, - DeprecatedGitHubWebhookNotification, - ResourceUsageNotification, -) -from .tasks import remove_dirs +from .notifications import ResourceUsageNotification +from .tasks import remove_dir class ProjectSendNotificationView(SendNotificationView): - notification_classes = [ - ResourceUsageNotification, - DeprecatedBuildWebhookNotification, - DeprecatedGitHubWebhookNotification, - ] + notification_classes = [ResourceUsageNotification] def get_object_recipients(self, obj): for owner in obj.users.all(): @@ -103,7 +101,9 @@ class ProjectOwnerBannedFilter(admin.SimpleListFilter): OWNER_BANNED = 'true' def lookups(self, request, model_admin): - return ((self.OWNER_BANNED, _('Yes')),) + return ( + (self.OWNER_BANNED, _('Yes')), + ) def queryset(self, request, queryset): if self.value() == self.OWNER_BANNED: @@ -117,23 +117,13 @@ class ProjectAdmin(GuardedModelAdmin): prepopulated_fields = {'slug': ('name',)} list_display = ('name', 'slug', 'repo', 'repo_type', 'featured') - list_filter = ( - 'repo_type', - 'featured', - 'privacy_level', - 'documentation_type', - 'programming_language', - 'feature__feature_id', - ProjectOwnerBannedFilter, - ) + list_filter = ('repo_type', 'featured', 'privacy_level', + 'documentation_type', 'programming_language', + ProjectOwnerBannedFilter) list_editable = ('featured',) search_fields = ('slug', 'repo') - inlines = [ - ProjectRelationshipInline, - RedirectInline, - VersionInline, - DomainInline, - ] + inlines = [ProjectRelationshipInline, RedirectInline, + VersionInline, DomainInline] readonly_fields = ('feature_flags',) raw_id_fields = ('users', 'main_language_project') actions = ['send_owner_email', 'ban_owner'] @@ -143,7 +133,7 @@ def feature_flags(self, obj): def send_owner_email(self, request, queryset): view = ProjectSendNotificationView.as_view( - action_name='send_owner_email', + action_name='send_owner_email' ) return view(request, queryset=queryset) @@ -160,25 +150,18 @@ def ban_owner(self, request, queryset): total = 0 for project in queryset: if project.users.count() == 1: - count = ( - UserProfile.objects.filter(user__projects=project - ).update(banned=True) - ) # yapf: disabled + count = (UserProfile.objects + .filter(user__projects=project) + .update(banned=True)) total += count else: - messages.add_message( - request, - messages.ERROR, - 'Project has multiple owners: {}'.format(project), - ) + messages.add_message(request, messages.ERROR, + 'Project has multiple owners: {0}'.format(project)) if total == 0: messages.add_message(request, messages.ERROR, 'No users banned') else: - messages.add_message( - request, - messages.INFO, - 'Banned {} user(s)'.format(total), - ) + messages.add_message(request, messages.INFO, + 'Banned {0} user(s)'.format(total)) ban_owner.short_description = 'Ban project owner' @@ -191,19 +174,15 @@ def delete_selected_and_artifacts(self, request, queryset): """ if request.POST.get('post'): for project in queryset: - broadcast( - type='app', - task=remove_dirs, - args=[(project.doc_path,)], - ) + broadcast(type='app', task=remove_dir, args=[project.doc_path]) return delete_selected(self, request, queryset) def get_actions(self, request): - actions = super().get_actions(request) + actions = super(ProjectAdmin, self).get_actions(request) actions['delete_selected'] = ( self.__class__.delete_selected_and_artifacts, 'delete_selected', - delete_selected.short_description, + delete_selected.short_description ) return actions diff --git a/readthedocs/projects/apps.py b/readthedocs/projects/apps.py index 76b3fae1b69..e29afbe49ce 100644 --- a/readthedocs/projects/apps.py +++ b/readthedocs/projects/apps.py @@ -1,6 +1,4 @@ -# -*- coding: utf-8 -*- - -"""Project app config.""" +"""Project app config""" from django.apps import AppConfig diff --git a/readthedocs/projects/backends/views.py b/readthedocs/projects/backends/views.py index ebae2d8b173..2b3d0fa41c5 100644 --- a/readthedocs/projects/backends/views.py +++ b/readthedocs/projects/backends/views.py @@ -1,12 +1,11 @@ -# -*- coding: utf-8 -*- - """ -Project views loaded by configuration settings. +Project views loaded by configuration settings Use these views instead of calling the views directly, in order to allow for settings override of the view class. """ +from __future__ import absolute_import from readthedocs.core.utils.extend import SettingsOverrideObject from readthedocs.projects.views import private diff --git a/readthedocs/projects/constants.py b/readthedocs/projects/constants.py index 26093e84917..7e6d130e93e 100644 --- a/readthedocs/projects/constants.py +++ b/readthedocs/projects/constants.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ Project constants. @@ -7,12 +6,15 @@ theme names and repository types. """ +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import re from django.utils.translation import ugettext_lazy as _ - DOCUMENTATION_CHOICES = ( + ('auto', _('Automatically Choose')), ('sphinx', _('Sphinx Html')), ('mkdocs', _('Mkdocs (Markdown)')), ('sphinx_htmldir', _('Sphinx HtmlDir')), @@ -310,13 +312,10 @@ ] GITHUB_URL = ( 'https://github.com/{user}/{repo}/' - '{action}/{version}{docroot}{path}{source_suffix}' -) + '{action}/{version}{docroot}{path}{source_suffix}') BITBUCKET_URL = ( 'https://bitbucket.org/{user}/{repo}/' - 'src/{version}{docroot}{path}{source_suffix}' -) + 'src/{version}{docroot}{path}{source_suffix}') GITLAB_URL = ( 'https://gitlab.com/{user}/{repo}/' - '{action}/{version}{docroot}{path}{source_suffix}' -) + '{action}/{version}{docroot}{path}{source_suffix}') diff --git a/readthedocs/projects/exceptions.py b/readthedocs/projects/exceptions.py index 85b439400df..41daacdeaaf 100644 --- a/readthedocs/projects/exceptions.py +++ b/readthedocs/projects/exceptions.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- - """Project exceptions.""" +from __future__ import division, print_function, unicode_literals + from django.conf import settings from django.utils.translation import ugettext_noop as _ @@ -14,13 +15,13 @@ class ProjectConfigurationError(BuildEnvironmentError): NOT_FOUND = _( 'A configuration file was not found. ' - 'Make sure you have a conf.py file in your repository.', + 'Make sure you have a conf.py file in your repository.' ) MULTIPLE_CONF_FILES = _( 'We found more than one conf.py and are not sure which one to use. ' 'Please, specify the correct file under the Advanced settings tab ' - "in the project's Admin.", + "in the project's Admin." ) @@ -30,22 +31,22 @@ class RepositoryError(BuildEnvironmentError): PRIVATE_ALLOWED = _( 'There was a problem connecting to your repository, ' - 'ensure that your repository URL is correct.', + 'ensure that your repository URL is correct.' ) PRIVATE_NOT_ALLOWED = _( 'There was a problem connecting to your repository, ' 'ensure that your repository URL is correct and your repository is public. ' - 'Private repositories are not supported.', + 'Private repositories are not supported.' ) - INVALID_SUBMODULES = _('One or more submodule URLs are not valid: {}.',) + INVALID_SUBMODULES = _( + 'One or more submodule URLs are not valid: {}.' + ) DUPLICATED_RESERVED_VERSIONS = _( - 'You can not have two versions with the name latest or stable.', + 'You can not have two versions with the name latest or stable.' ) - FAILED_TO_CHECKOUT = _('Failed to checkout revision: {}') - def get_default_message(self): if settings.ALLOW_PRIVATE_REPOS: return self.PRIVATE_ALLOWED @@ -60,3 +61,5 @@ class ProjectSpamError(Exception): This error is not raised to users, we use this for banning users in the background. """ + + pass diff --git a/readthedocs/projects/feeds.py b/readthedocs/projects/feeds.py index 35ebdd6ac32..b3739f4b005 100644 --- a/readthedocs/projects/feeds.py +++ b/readthedocs/projects/feeds.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Project RSS feeds.""" +"""Project RSS feeds""" +from __future__ import absolute_import from django.contrib.syndication.views import Feed from readthedocs.projects.models import Project @@ -9,11 +8,11 @@ class LatestProjectsFeed(Feed): - """RSS feed for projects that were recently updated.""" + """RSS feed for projects that were recently updated""" - title = 'Recently updated documentation' - link = 'http://readthedocs.org' - description = 'Recently updated documentation on Read the Docs' + title = "Recently updated documentation" + link = "http://readthedocs.org" + description = "Recently updated documentation on Read the Docs" def items(self): return Project.objects.public().order_by('-modified_date')[:10] @@ -27,11 +26,11 @@ def item_description(self, item): class NewProjectsFeed(Feed): - """RSS feed for newly created projects.""" + """RSS feed for newly created projects""" - title = 'Newest documentation' - link = 'http://readthedocs.org' - description = 'Recently created documentation on Read the Docs' + title = "Newest documentation" + link = "http://readthedocs.org" + description = "Recently created documentation on Read the Docs" def items(self): return Project.objects.public().order_by('-pk')[:10] diff --git a/readthedocs/projects/fixtures/test_auth.json b/readthedocs/projects/fixtures/test_auth.json index c0d160196f1..83d7738406e 100644 --- a/readthedocs/projects/fixtures/test_auth.json +++ b/readthedocs/projects/fixtures/test_auth.json @@ -701,4 +701,4 @@ "date_joined": "2014-02-09T19:48:39.934+00:00" } } -] +] \ No newline at end of file diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index aae5e9d278c..5f7f407f1ab 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -1,17 +1,23 @@ # -*- coding: utf-8 -*- - """Project forms.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from random import choice -from re import fullmatch -from urllib.parse import urlparse +from builtins import object from django import forms from django.conf import settings from django.contrib.auth.models import User from django.template.loader import render_to_string from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ +from future.backports.urllib.parse import urlparse from guardian.shortcuts import assign from textclassifier.validators import ClassifierValidator @@ -21,11 +27,11 @@ from readthedocs.integrations.models import Integration from readthedocs.oauth.models import RemoteRepository from readthedocs.projects import constants +from readthedocs.projects.constants import PUBLIC from readthedocs.projects.exceptions import ProjectSpamError from readthedocs.projects.models import ( Domain, EmailHook, - EnvironmentVariable, Feature, Project, ProjectRelationship, @@ -46,17 +52,17 @@ class ProjectForm(forms.ModelForm): def __init__(self, *args, **kwargs): self.user = kwargs.pop('user', None) - super().__init__(*args, **kwargs) + super(ProjectForm, self).__init__(*args, **kwargs) def save(self, commit=True): - project = super().save(commit) + project = super(ProjectForm, self).save(commit) if commit: if self.user and not project.users.filter(pk=self.user.pk).exists(): project.users.add(self.user) return project -class ProjectTriggerBuildMixin: +class ProjectTriggerBuildMixin(object): """ Mixin to trigger build on form save. @@ -67,7 +73,7 @@ class ProjectTriggerBuildMixin: def save(self, commit=True): """Trigger build on commit save.""" - project = super().save(commit) + project = super(ProjectTriggerBuildMixin, self).save(commit) if commit: trigger_build(project=project) return project @@ -84,7 +90,7 @@ class ProjectBasicsForm(ProjectForm): """Form for basic project fields.""" - class Meta: + class Meta(object): model = Project fields = ('name', 'repo', 'repo_type') @@ -95,7 +101,7 @@ class Meta: def __init__(self, *args, **kwargs): show_advanced = kwargs.pop('show_advanced', False) - super().__init__(*args, **kwargs) + super(ProjectBasicsForm, self).__init__(*args, **kwargs) if show_advanced: self.fields['advanced'] = forms.BooleanField( required=False, @@ -106,7 +112,7 @@ def __init__(self, *args, **kwargs): def save(self, commit=True): """Add remote repository relationship to the project instance.""" - instance = super().save(commit) + instance = super(ProjectBasicsForm, self).save(commit) remote_repo = self.cleaned_data.get('remote_repository', None) if remote_repo: if commit: @@ -122,11 +128,12 @@ def clean_name(self): potential_slug = slugify(name) if Project.objects.filter(slug=potential_slug).exists(): raise forms.ValidationError( - _('Invalid project name, a project already exists with that name'), - ) # yapf: disable # noqa + _('Invalid project name, a project already exists with that name')) # yapf: disable # noqa if not potential_slug: # Check the generated slug won't be empty - raise forms.ValidationError(_('Invalid project name'),) + raise forms.ValidationError( + _('Invalid project name'), + ) return name @@ -158,7 +165,7 @@ class ProjectExtraForm(ProjectForm): """Additional project information form.""" - class Meta: + class Meta(object): model = Project fields = ( 'description', @@ -180,9 +187,7 @@ def clean_tags(self): for tag in tags: if len(tag) > 100: raise forms.ValidationError( - _( - 'Length of each tag must be less than or equal to 100 characters.', - ), + _('Length of each tag must be less than or equal to 100 characters.') ) return tags @@ -194,13 +199,11 @@ class ProjectAdvancedForm(ProjectTriggerBuildMixin, ProjectForm): python_interpreter = forms.ChoiceField( choices=constants.PYTHON_CHOICES, initial='python', - help_text=_( - 'The Python interpreter used to create the virtual ' - 'environment.', - ), + help_text=_('The Python interpreter used to create the virtual ' + 'environment.'), ) - class Meta: + class Meta(object): model = Project fields = ( # Standard build edits @@ -224,43 +227,35 @@ class Meta: ) def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(ProjectAdvancedForm, self).__init__(*args, **kwargs) default_choice = (None, '-' * 9) all_versions = self.instance.versions.values_list( - 'identifier', - 'verbose_name', + 'identifier', 'verbose_name' ) self.fields['default_branch'].widget = forms.Select( - choices=[default_choice] + list(all_versions), + choices=[default_choice] + list(all_versions) ) active_versions = self.instance.all_active_versions().values_list( 'slug', 'verbose_name' - ) # yapf: disabled + ) self.fields['default_version'].widget = forms.Select( - choices=active_versions, + choices=active_versions ) def clean_conf_py_file(self): filename = self.cleaned_data.get('conf_py_file', '').strip() if filename and 'conf.py' not in filename: raise forms.ValidationError( - _( - 'Your configuration file is invalid, make sure it contains ' - 'conf.py in it.', - ), - ) # yapf: disable + _('Your configuration file is invalid, make sure it contains ' + 'conf.py in it.')) # yapf: disable return filename -class UpdateProjectForm( - ProjectTriggerBuildMixin, - ProjectBasicsForm, - ProjectExtraForm, -): - - class Meta: +class UpdateProjectForm(ProjectTriggerBuildMixin, ProjectBasicsForm, + ProjectExtraForm): + class Meta(object): model = Project fields = ( # Basics @@ -282,17 +277,17 @@ def clean_language(self): if project: msg = _( 'There is already a "{lang}" translation ' - 'for the {proj} project.', + 'for the {proj} project.' ) if project.translations.filter(language=language).exists(): raise forms.ValidationError( - msg.format(lang=language, proj=project.slug), + msg.format(lang=language, proj=project.slug) ) main_project = project.main_language_project if main_project: if main_project.language == language: raise forms.ValidationError( - msg.format(lang=language, proj=main_project.slug), + msg.format(lang=language, proj=main_project.slug) ) siblings = ( main_project.translations @@ -302,7 +297,7 @@ def clean_language(self): ) if siblings: raise forms.ValidationError( - msg.format(lang=language, proj=main_project.slug), + msg.format(lang=language, proj=main_project.slug) ) return language @@ -313,14 +308,14 @@ class ProjectRelationshipBaseForm(forms.ModelForm): parent = forms.CharField(widget=forms.HiddenInput(), required=False) - class Meta: + class Meta(object): model = ProjectRelationship fields = '__all__' def __init__(self, *args, **kwargs): self.project = kwargs.pop('project') self.user = kwargs.pop('user') - super().__init__(*args, **kwargs) + super(ProjectRelationshipBaseForm, self).__init__(*args, **kwargs) # Don't display the update form with an editable child, as it will be # filtered out from the queryset anyways. if hasattr(self, 'instance') and self.instance.pk is not None: @@ -333,16 +328,14 @@ def clean_parent(self): # This validation error is mostly for testing, users shouldn't see # this in normal circumstances raise forms.ValidationError( - _('Subproject nesting is not supported'), - ) + _('Subproject nesting is not supported')) return self.project def clean_child(self): child = self.cleaned_data['child'] if child == self.project: raise forms.ValidationError( - _('A project can not be a subproject of itself'), - ) + _('A project can not be a subproject of itself')) return child def get_subproject_queryset(self): @@ -356,8 +349,7 @@ def get_subproject_queryset(self): Project.objects.for_admin_user(self.user) .exclude(subprojects__isnull=False) .exclude(superprojects__isnull=False) - .exclude(pk=self.project.pk) - ) + .exclude(pk=self.project.pk)) return queryset @@ -370,11 +362,11 @@ class DualCheckboxWidget(forms.CheckboxInput): """Checkbox with link to the version's built documentation.""" def __init__(self, version, attrs=None, check_test=bool): - super().__init__(attrs, check_test) + super(DualCheckboxWidget, self).__init__(attrs, check_test) self.version = version def render(self, name, value, attrs=None, renderer=None): - checkbox = super().render(name, value, attrs, renderer) + checkbox = super(DualCheckboxWidget, self).render(name, value, attrs, renderer) icon = self.render_icon() return mark_safe('{}{}'.format(checkbox, icon)) @@ -462,14 +454,12 @@ def build_versions_form(project): class BaseUploadHTMLForm(forms.Form): content = forms.FileField(label=_('Zip file of HTML')) - overwrite = forms.BooleanField( - required=False, - label=_('Overwrite existing HTML?'), - ) + overwrite = forms.BooleanField(required=False, + label=_('Overwrite existing HTML?')) def __init__(self, *args, **kwargs): self.request = kwargs.pop('request', None) - super().__init__(*args, **kwargs) + super(BaseUploadHTMLForm, self).__init__(*args, **kwargs) def clean(self): version_slug = self.cleaned_data['version'] @@ -509,15 +499,14 @@ class UserForm(forms.Form): def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) + super(UserForm, self).__init__(*args, **kwargs) def clean_user(self): name = self.cleaned_data['user'] user_qs = User.objects.filter(username=name) if not user_qs.exists(): raise forms.ValidationError( - _('User {name} does not exist').format(name=name), - ) + _('User {name} does not exist').format(name=name)) self.user = user_qs[0] return name @@ -536,13 +525,11 @@ class EmailHookForm(forms.Form): def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) + super(EmailHookForm, self).__init__(*args, **kwargs) def clean_email(self): self.email = EmailHook.objects.get_or_create( - email=self.cleaned_data['email'], - project=self.project, - )[0] + email=self.cleaned_data['email'], project=self.project)[0] return self.email def save(self): @@ -556,13 +543,11 @@ class WebHookForm(forms.ModelForm): def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) + super(WebHookForm, self).__init__(*args, **kwargs) def save(self, commit=True): self.webhook = WebHook.objects.get_or_create( - url=self.cleaned_data['url'], - project=self.project, - )[0] + url=self.cleaned_data['url'], project=self.project)[0] self.project.webhook_notifications.add(self.webhook) return self.project @@ -580,17 +565,15 @@ class TranslationBaseForm(forms.Form): def __init__(self, *args, **kwargs): self.parent = kwargs.pop('parent', None) self.user = kwargs.pop('user') - super().__init__(*args, **kwargs) + super(TranslationBaseForm, self).__init__(*args, **kwargs) self.fields['project'].choices = self.get_choices() def get_choices(self): - return [( - project.slug, - '{project} ({lang})'.format( - project=project.slug, - lang=project.get_language_display(), - ), - ) for project in self.get_translation_queryset().all()] + return [ + (project.slug, '{project} ({lang})'.format( + project=project.slug, lang=project.get_language_display())) + for project in self.get_translation_queryset().all() + ] def clean_project(self): translation_project_slug = self.cleaned_data['project'] @@ -599,35 +582,36 @@ def clean_project(self): if self.parent.main_language_project is not None: msg = 'Project "{project}" is already a translation' raise forms.ValidationError( - (_(msg).format(project=self.parent.slug)), + (_(msg).format(project=self.parent.slug)) ) project_translation_qs = self.get_translation_queryset().filter( - slug=translation_project_slug, + slug=translation_project_slug ) if not project_translation_qs.exists(): msg = 'Project "{project}" does not exist.' raise forms.ValidationError( - (_(msg).format(project=translation_project_slug)), + (_(msg).format(project=translation_project_slug)) ) self.translation = project_translation_qs.first() if self.translation.language == self.parent.language: - msg = ('Both projects can not have the same language ({lang}).') + msg = ( + 'Both projects can not have the same language ({lang}).' + ) raise forms.ValidationError( - _(msg).format(lang=self.parent.get_language_display()), + _(msg).format(lang=self.parent.get_language_display()) ) - - # yapf: disable exists_translation = ( self.parent.translations .filter(language=self.translation.language) .exists() ) - # yapf: enable if exists_translation: - msg = ('This project already has a translation for {lang}.') + msg = ( + 'This project already has a translation for {lang}.' + ) raise forms.ValidationError( - _(msg).format(lang=self.translation.get_language_display()), + _(msg).format(lang=self.translation.get_language_display()) ) is_parent = self.translation.translations.exists() if is_parent: @@ -662,13 +646,13 @@ class RedirectForm(forms.ModelForm): """Form for project redirects.""" - class Meta: + class Meta(object): model = Redirect fields = ['redirect_type', 'from_url', 'to_url'] def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) + super(RedirectForm, self).__init__(*args, **kwargs) def save(self, **_): # pylint: disable=arguments-differ # TODO this should respect the unused argument `commit`. It's not clear @@ -689,13 +673,13 @@ class DomainBaseForm(forms.ModelForm): project = forms.CharField(widget=forms.HiddenInput(), required=False) - class Meta: + class Meta(object): model = Domain exclude = ['machine', 'cname', 'count'] # pylint: disable=modelform-uses-exclude def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) + super(DomainBaseForm, self).__init__(*args, **kwargs) def clean_project(self): return self.project @@ -711,10 +695,11 @@ def clean_domain(self): def clean_canonical(self): canonical = self.cleaned_data['canonical'] _id = self.initial.get('id') - if canonical and Domain.objects.filter(project=self.project, canonical=True).exclude(pk=_id).exists(): # yapf: disabled # noqa + if canonical and Domain.objects.filter( + project=self.project, canonical=True + ).exclude(pk=_id).exists(): raise forms.ValidationError( - _('Only 1 Domain can be canonical at a time.'), - ) + _('Only 1 Domain can be canonical at a time.')) return canonical @@ -732,13 +717,13 @@ class IntegrationForm(forms.ModelForm): project = forms.CharField(widget=forms.HiddenInput(), required=False) - class Meta: + class Meta(object): model = Integration exclude = ['provider_data', 'exchanges'] # pylint: disable=modelform-uses-exclude def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) + super(IntegrationForm, self).__init__(*args, **kwargs) # Alter the integration type choices to only provider webhooks self.fields['integration_type'].choices = Integration.WEBHOOK_INTEGRATIONS # yapf: disable # noqa @@ -747,20 +732,20 @@ def clean_project(self): def save(self, commit=True): self.instance = Integration.objects.subclass(self.instance) - return super().save(commit) + return super(IntegrationForm, self).save(commit) class ProjectAdvertisingForm(forms.ModelForm): """Project promotion opt-out form.""" - class Meta: + class Meta(object): model = Project fields = ['allow_promos'] def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) + super(ProjectAdvertisingForm, self).__init__(*args, **kwargs) class FeatureForm(forms.ModelForm): @@ -775,58 +760,10 @@ class FeatureForm(forms.ModelForm): feature_id = forms.ChoiceField() - class Meta: + class Meta(object): model = Feature fields = ['projects', 'feature_id', 'default_true'] def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(FeatureForm, self).__init__(*args, **kwargs) self.fields['feature_id'].choices = Feature.FEATURES - - -class EnvironmentVariableForm(forms.ModelForm): - - """ - Form to add an EnvironmentVariable to a Project. - - This limits the name of the variable. - """ - - project = forms.CharField(widget=forms.HiddenInput(), required=False) - - class Meta: - model = EnvironmentVariable - fields = ('name', 'value', 'project') - - def __init__(self, *args, **kwargs): - self.project = kwargs.pop('project', None) - super().__init__(*args, **kwargs) - - def clean_project(self): - return self.project - - def clean_name(self): - name = self.cleaned_data['name'] - if name.startswith('__'): - raise forms.ValidationError( - _("Variable name can't start with __ (double underscore)"), - ) - elif name.startswith('READTHEDOCS'): - raise forms.ValidationError( - _("Variable name can't start with READTHEDOCS"), - ) - elif self.project.environmentvariable_set.filter(name=name).exists(): - raise forms.ValidationError( - _( - 'There is already a variable with this name for this project', - ), - ) - elif ' ' in name: - raise forms.ValidationError( - _("Variable name can't contain spaces"), - ) - elif not fullmatch('[a-zA-Z0-9_]+', name): - raise forms.ValidationError( - _('Only letters, numbers and underscore are allowed'), - ) - return name diff --git a/readthedocs/projects/management/commands/import_project_from_live.py b/readthedocs/projects/management/commands/import_project_from_live.py index 2b006f84ecb..95c85778349 100644 --- a/readthedocs/projects/management/commands/import_project_from_live.py +++ b/readthedocs/projects/management/commands/import_project_from_live.py @@ -1,21 +1,19 @@ -# -*- coding: utf-8 -*- +"""Import project command""" -"""Import project command.""" - -import json - -import slumber -from django.contrib.auth.models import User +from __future__ import absolute_import from django.core.management import call_command from django.core.management.base import BaseCommand +import json +import slumber +from django.contrib.auth.models import User from ...models import Project class Command(BaseCommand): """ - Import project from production API. + Import project from production API This is a helper to debug issues with projects on the server more easily locally. It allows you to import projects based on the data that the public @@ -24,8 +22,8 @@ class Command(BaseCommand): help = ( "Retrieves the data of a project from readthedocs.org's API and puts " - 'it into the local database. This is mostly useful for debugging ' - 'issues with projects on the live site.' + "it into the local database. This is mostly useful for debugging " + "issues with projects on the live site." ) def add_arguments(self, parser): @@ -43,11 +41,10 @@ def handle(self, *args, **options): project_data = project_data['objects'][0] except (KeyError, IndexError): self.stderr.write( - 'Cannot find {slug} in API. Response was:\n{response}'.format( + 'Cannot find {slug} in API. Response was:\n{response}' + .format( slug=slug, - response=json.dumps(project_data), - ), - ) + response=json.dumps(project_data))) try: project = Project.objects.get(slug=slug) diff --git a/readthedocs/projects/migrations/0001_initial.py b/readthedocs/projects/migrations/0001_initial.py index 00d2a7915b0..734358ad0b0 100644 --- a/readthedocs/projects/migrations/0001_initial.py +++ b/readthedocs/projects/migrations/0001_initial.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- -import taggit.managers +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations from django.conf import settings -from django.db import migrations, models +import taggit.managers class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0002_add_importedfile_model.py b/readthedocs/projects/migrations/0002_add_importedfile_model.py index cfa6f3b9e63..a03fff529cb 100644 --- a/readthedocs/projects/migrations/0002_add_importedfile_model.py +++ b/readthedocs/projects/migrations/0002_add_importedfile_model.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0003_project_cdn_enabled.py b/readthedocs/projects/migrations/0003_project_cdn_enabled.py index e89cfed99ac..471df331910 100644 --- a/readthedocs/projects/migrations/0003_project_cdn_enabled.py +++ b/readthedocs/projects/migrations/0003_project_cdn_enabled.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0004_add_project_container_image.py b/readthedocs/projects/migrations/0004_add_project_container_image.py index 724e62e45fc..70c969d1be5 100644 --- a/readthedocs/projects/migrations/0004_add_project_container_image.py +++ b/readthedocs/projects/migrations/0004_add_project_container_image.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0005_sync_project_model.py b/readthedocs/projects/migrations/0005_sync_project_model.py index 12537572ce8..75b9e6d5e06 100644 --- a/readthedocs/projects/migrations/0005_sync_project_model.py +++ b/readthedocs/projects/migrations/0005_sync_project_model.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0006_add_domain_models.py b/readthedocs/projects/migrations/0006_add_domain_models.py index e50617a6931..78e05b81e28 100644 --- a/readthedocs/projects/migrations/0006_add_domain_models.py +++ b/readthedocs/projects/migrations/0006_add_domain_models.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0007_migrate_canonical_data.py b/readthedocs/projects/migrations/0007_migrate_canonical_data.py index 633f975fc5c..743d6a145cb 100644 --- a/readthedocs/projects/migrations/0007_migrate_canonical_data.py +++ b/readthedocs/projects/migrations/0007_migrate_canonical_data.py @@ -1,9 +1,13 @@ # -*- coding: utf-8 -*- -from django.db import migrations, transaction +from __future__ import unicode_literals, print_function + +from __future__ import absolute_import +from django.db import migrations +from django.db import transaction def migrate_canonical(apps, schema_editor): - Project = apps.get_model('projects', 'Project') + Project = apps.get_model("projects", "Project") for project in Project.objects.all(): if project.canonical_url: try: @@ -12,11 +16,11 @@ def migrate_canonical(apps, schema_editor): url=project.canonical_url, canonical=True, ) - print('Added {url} to {project}'.format(url=project.canonical_url, project=project.name)) + print(u"Added {url} to {project}".format(url=project.canonical_url, project=project.name)) except Exception as e: print(e) - print('Failed adding {url} to {project}'.format( - url=project.canonical_url, project=project.name, + print(u"Failed adding {url} to {project}".format( + url=project.canonical_url, project=project.name )) @@ -27,5 +31,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(migrate_canonical), + migrations.RunPython(migrate_canonical) ] diff --git a/readthedocs/projects/migrations/0008_add_subproject_alias_prefix.py b/readthedocs/projects/migrations/0008_add_subproject_alias_prefix.py index ec61ecabd15..b3eb933882c 100644 --- a/readthedocs/projects/migrations/0008_add_subproject_alias_prefix.py +++ b/readthedocs/projects/migrations/0008_add_subproject_alias_prefix.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0009_add_domain_field.py b/readthedocs/projects/migrations/0009_add_domain_field.py index 4c910c74e44..20230a4738c 100644 --- a/readthedocs/projects/migrations/0009_add_domain_field.py +++ b/readthedocs/projects/migrations/0009_add_domain_field.py @@ -1,7 +1,11 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals +from __future__ import absolute_import +from django.db import models, migrations import django.contrib.sites.models -from django.db import migrations, models + +import uuid class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0010_migrate_domain_data.py b/readthedocs/projects/migrations/0010_migrate_domain_data.py index 49d3dc07bb9..ef60b2a00d1 100644 --- a/readthedocs/projects/migrations/0010_migrate_domain_data.py +++ b/readthedocs/projects/migrations/0010_migrate_domain_data.py @@ -1,19 +1,19 @@ # -*- coding: utf-8 -*- -from urllib.parse import urlparse +from __future__ import (absolute_import, print_function, unicode_literals) -from django.db import migrations, models +from django.db import models, migrations +from future.backports.urllib.parse import urlparse import readthedocs.projects.validators def migrate_url(apps, schema_editor): - Domain = apps.get_model('projects', 'Domain') + Domain = apps.get_model("projects", "Domain") Domain.objects.filter(count=0).delete() for domain in Domain.objects.all(): if domain.project.superprojects.count() or domain.project.main_language_project: - print('{project} is a subproject or translation. Deleting domain.'.format( - project=domain.project.slug, - )) + print("{project} is a subproject or translation. Deleting domain.".format( + project=domain.project.slug)) domain.delete() continue parsed = urlparse(domain.url) @@ -24,10 +24,10 @@ def migrate_url(apps, schema_editor): try: domain.domain = domain_string domain.save() - print('Added {domain} from {url}'.format(url=domain.url, domain=domain_string)) + print(u"Added {domain} from {url}".format(url=domain.url, domain=domain_string)) except Exception as e: print(e) - print('Failed {domain} from {url}'.format(url=domain.url, domain=domain_string)) + print(u"Failed {domain} from {url}".format(url=domain.url, domain=domain_string)) dms = Domain.objects.filter(domain=domain_string).order_by('-count') if dms.count() > 1: diff --git a/readthedocs/projects/migrations/0011_delete-url.py b/readthedocs/projects/migrations/0011_delete-url.py index 3b01ed32cf9..fcd83c02753 100644 --- a/readthedocs/projects/migrations/0011_delete-url.py +++ b/readthedocs/projects/migrations/0011_delete-url.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0012_proper-name-for-install-project.py b/readthedocs/projects/migrations/0012_proper-name-for-install-project.py index 9f143977ca0..8f5f116269b 100644 --- a/readthedocs/projects/migrations/0012_proper-name-for-install-project.py +++ b/readthedocs/projects/migrations/0012_proper-name-for-install-project.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0013_add-container-limits.py b/readthedocs/projects/migrations/0013_add-container-limits.py index 9a052e00fc1..c2820609037 100644 --- a/readthedocs/projects/migrations/0013_add-container-limits.py +++ b/readthedocs/projects/migrations/0013_add-container-limits.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0014_add-state-tracking.py b/readthedocs/projects/migrations/0014_add-state-tracking.py index d2c34c28e5a..628bf970dce 100644 --- a/readthedocs/projects/migrations/0014_add-state-tracking.py +++ b/readthedocs/projects/migrations/0014_add-state-tracking.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0015_add_project_allow_promos.py b/readthedocs/projects/migrations/0015_add_project_allow_promos.py index 882893160fc..5c50eeac924 100644 --- a/readthedocs/projects/migrations/0015_add_project_allow_promos.py +++ b/readthedocs/projects/migrations/0015_add_project_allow_promos.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0016_build-queue-name.py b/readthedocs/projects/migrations/0016_build-queue-name.py index 0350bae8690..46833ada78c 100644 --- a/readthedocs/projects/migrations/0016_build-queue-name.py +++ b/readthedocs/projects/migrations/0016_build-queue-name.py @@ -1,15 +1,17 @@ # -*- coding: utf-8 -*- -from django.db import migrations +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations def update_build_queue(apps, schema): - """Update project build queue to include the previously implied build- - prefix.""" - Project = apps.get_model('projects', 'Project') + """Update project build queue to include the previously implied build- prefix""" + Project = apps.get_model("projects", "Project") for project in Project.objects.all(): if project.build_queue is not None: if not project.build_queue.startswith('build-'): - project.build_queue = 'build-{}'.format(project.build_queue) + project.build_queue = 'build-{0}'.format(project.build_queue) project.save() diff --git a/readthedocs/projects/migrations/0017_add_domain_https.py b/readthedocs/projects/migrations/0017_add_domain_https.py index 18788581ccf..9bf94eeb5cb 100644 --- a/readthedocs/projects/migrations/0017_add_domain_https.py +++ b/readthedocs/projects/migrations/0017_add_domain_https.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0018_fix-translation-model.py b/readthedocs/projects/migrations/0018_fix-translation-model.py index 2541fb0d36b..bfe283d27cb 100644 --- a/readthedocs/projects/migrations/0018_fix-translation-model.py +++ b/readthedocs/projects/migrations/0018_fix-translation-model.py @@ -1,6 +1,9 @@ # -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations import django.db.models.deletion -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0019_add-features.py b/readthedocs/projects/migrations/0019_add-features.py index 6b7ee7a8bcd..6d1036dd123 100644 --- a/readthedocs/projects/migrations/0019_add-features.py +++ b/readthedocs/projects/migrations/0019_add-features.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-10-27 12:55 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0020_add-api-project-proxy.py b/readthedocs/projects/migrations/0020_add-api-project-proxy.py index 0040581f77b..34eafa4846f 100644 --- a/readthedocs/projects/migrations/0020_add-api-project-proxy.py +++ b/readthedocs/projects/migrations/0020_add-api-project-proxy.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-10-27 12:56 -from django.db import migrations +from __future__ import unicode_literals + +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0021_add-webhook-deprecation-feature.py b/readthedocs/projects/migrations/0021_add-webhook-deprecation-feature.py index 91f27b6d001..84dc7cf923b 100644 --- a/readthedocs/projects/migrations/0021_add-webhook-deprecation-feature.py +++ b/readthedocs/projects/migrations/0021_add-webhook-deprecation-feature.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- -"""Add feature for allowing access to deprecated webhook endpoints.""" +"""Add feature for allowing access to deprecated webhook endpoints""" + +from __future__ import unicode_literals from django.db import migrations @@ -28,5 +30,5 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(forward_add_feature, reverse_add_feature), + migrations.RunPython(forward_add_feature, reverse_add_feature) ] diff --git a/readthedocs/projects/migrations/0022_add-alias-slug.py b/readthedocs/projects/migrations/0022_add-alias-slug.py index 90c434a2752..8439c56e85e 100644 --- a/readthedocs/projects/migrations/0022_add-alias-slug.py +++ b/readthedocs/projects/migrations/0022_add-alias-slug.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-12-21 16:30 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0023_migrate-alias-slug.py b/readthedocs/projects/migrations/0023_migrate-alias-slug.py index 531c3dc332f..4942848b952 100644 --- a/readthedocs/projects/migrations/0023_migrate-alias-slug.py +++ b/readthedocs/projects/migrations/0023_migrate-alias-slug.py @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.12 on 2017-12-21 16:31 -import re +from __future__ import unicode_literals from django.db import migrations +import re + class Migration(migrations.Migration): @@ -12,7 +14,7 @@ def migrate_data(apps, schema_editor): # so that we don't break a bunch of folks URL's. # They will have to change them on update. invalid_chars_re = re.compile('[^-._a-zA-Z0-9]') - ProjectRelationship = apps.get_model('projects', 'ProjectRelationship') + ProjectRelationship = apps.get_model("projects", "ProjectRelationship") for p in ProjectRelationship.objects.all(): if p.alias and invalid_chars_re.match(p.alias): new_alias = invalid_chars_re.sub('', p.alias) @@ -27,5 +29,5 @@ def reverse(apps, schema_editor): ] operations = [ - migrations.RunPython(migrate_data, reverse), + migrations.RunPython(migrate_data, reverse) ] diff --git a/readthedocs/projects/migrations/0024_add-show-version-warning.py b/readthedocs/projects/migrations/0024_add-show-version-warning.py index bfa0b2edb9e..6bc60e4aeeb 100644 --- a/readthedocs/projects/migrations/0024_add-show-version-warning.py +++ b/readthedocs/projects/migrations/0024_add-show-version-warning.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-05-02 01:27 -from django.db import migrations, models +from __future__ import unicode_literals +from django.db import migrations, models import readthedocs.projects.validators diff --git a/readthedocs/projects/migrations/0025_show-version-warning-existing-projects.py b/readthedocs/projects/migrations/0025_show-version-warning-existing-projects.py index e38349fc52f..5d073258caf 100644 --- a/readthedocs/projects/migrations/0025_show-version-warning-existing-projects.py +++ b/readthedocs/projects/migrations/0025_show-version-warning-existing-projects.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-05-07 19:25 +from __future__ import unicode_literals + from django.db import migrations diff --git a/readthedocs/projects/migrations/0026_ad-free-option.py b/readthedocs/projects/migrations/0026_ad-free-option.py index d108f25e190..a32fe5e74b7 100644 --- a/readthedocs/projects/migrations/0026_ad-free-option.py +++ b/readthedocs/projects/migrations/0026_ad-free-option.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-06-29 15:53 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0027_remove_json_with_html_feature.py b/readthedocs/projects/migrations/0027_remove_json_with_html_feature.py index d0fb5d14e93..c5daf0ece54 100644 --- a/readthedocs/projects/migrations/0027_remove_json_with_html_feature.py +++ b/readthedocs/projects/migrations/0027_remove_json_with_html_feature.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.9.13 on 2018-08-22 09:19 +from __future__ import unicode_literals + from django.db import migrations diff --git a/readthedocs/projects/migrations/0028_remove_comments_and_update_old_migration.py b/readthedocs/projects/migrations/0028_remove_comments_and_update_old_migration.py index 4d4fc04a5a8..056e9b8e61a 100644 --- a/readthedocs/projects/migrations/0028_remove_comments_and_update_old_migration.py +++ b/readthedocs/projects/migrations/0028_remove_comments_and_update_old_migration.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-31 10:08 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0029_add_additional_languages.py b/readthedocs/projects/migrations/0029_add_additional_languages.py index 8e9e48d4b6f..b4cfc77535b 100644 --- a/readthedocs/projects/migrations/0029_add_additional_languages.py +++ b/readthedocs/projects/migrations/0029_add_additional_languages.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-01 13:38 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0030_change-max-length-project-slug.py b/readthedocs/projects/migrations/0030_change-max-length-project-slug.py index ee27e9602a9..7e9b48da270 100644 --- a/readthedocs/projects/migrations/0030_change-max-length-project-slug.py +++ b/readthedocs/projects/migrations/0030_change-max-length-project-slug.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-01 20:55 +from __future__ import unicode_literals + from django.db import migrations, models from django.db.models.functions import Length diff --git a/readthedocs/projects/migrations/0031_add_modified_date_importedfile.py b/readthedocs/projects/migrations/0031_add_modified_date_importedfile.py index 617a420c2a3..255da1c003a 100644 --- a/readthedocs/projects/migrations/0031_add_modified_date_importedfile.py +++ b/readthedocs/projects/migrations/0031_add_modified_date_importedfile.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-01 14:37 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0032_increase_webhook_maxsize.py b/readthedocs/projects/migrations/0032_increase_webhook_maxsize.py index 49b231590ed..eed6d3de06a 100644 --- a/readthedocs/projects/migrations/0032_increase_webhook_maxsize.py +++ b/readthedocs/projects/migrations/0032_increase_webhook_maxsize.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-06 23:12 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0033_add_environment_variables.py b/readthedocs/projects/migrations/0033_add_environment_variables.py index 9279fa8b338..de9e3d18e5b 100644 --- a/readthedocs/projects/migrations/0033_add_environment_variables.py +++ b/readthedocs/projects/migrations/0033_add_environment_variables.py @@ -1,8 +1,10 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-12 13:57 +from __future__ import unicode_literals + +from django.db import migrations, models import django.db.models.deletion import django_extensions.db.fields -from django.db import migrations, models class Migration(migrations.Migration): diff --git a/readthedocs/projects/migrations/0034_remove_unused_project_model_fields.py b/readthedocs/projects/migrations/0034_remove_unused_project_model_fields.py index 4ad8b27f8a5..996cdd9c6d3 100644 --- a/readthedocs/projects/migrations/0034_remove_unused_project_model_fields.py +++ b/readthedocs/projects/migrations/0034_remove_unused_project_model_fields.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-11-05 12:20 +from __future__ import unicode_literals + from django.db import migrations diff --git a/readthedocs/projects/migrations/0035_container_time_limit_as_integer.py b/readthedocs/projects/migrations/0035_container_time_limit_as_integer.py index 7256548fd6f..28dab124004 100644 --- a/readthedocs/projects/migrations/0035_container_time_limit_as_integer.py +++ b/readthedocs/projects/migrations/0035_container_time_limit_as_integer.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-12-10 11:19 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/projects/migrations/0036_remove-auto-doctype.py b/readthedocs/projects/migrations/0036_remove-auto-doctype.py deleted file mode 100644 index b0f9ad28165..00000000000 --- a/readthedocs/projects/migrations/0036_remove-auto-doctype.py +++ /dev/null @@ -1,26 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.11.16 on 2018-12-17 17:32 -from django.db import migrations, models - - -def migrate_auto_doctype(apps, schema_editor): - Project = apps.get_model('projects', 'Project') - Project.objects.filter(documentation_type='auto').update( - documentation_type='sphinx', - ) - - -class Migration(migrations.Migration): - - dependencies = [ - ('projects', '0035_container_time_limit_as_integer'), - ] - - operations = [ - migrations.RunPython(migrate_auto_doctype), - migrations.AlterField( - model_name='project', - name='documentation_type', - field=models.CharField(choices=[('sphinx', 'Sphinx Html'), ('mkdocs', 'Mkdocs (Markdown)'), ('sphinx_htmldir', 'Sphinx HtmlDir'), ('sphinx_singlehtml', 'Sphinx Single Page HTML')], default='sphinx', help_text='Type of documentation you are building. More info.', max_length=20, verbose_name='Documentation type'), - ), - ] diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index f39a54c4625..d02ecfc76c0 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -1,21 +1,23 @@ # -*- coding: utf-8 -*- - """Project models.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import fnmatch import logging import os -from urllib.parse import urlparse +from builtins import object # pylint: disable=redefined-builtin from django.conf import settings from django.contrib.auth.models import User -from django.db import models from django.urls import NoReverseMatch, reverse +from django.db import models from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext_lazy as _ from django_extensions.db.models import TimeStampedModel +from future.backports.urllib.parse import urlparse # noqa from guardian.shortcuts import assign -from six.moves import shlex_quote from taggit.managers import TaggableManager from readthedocs.builds.constants import LATEST, STABLE @@ -24,22 +26,15 @@ from readthedocs.projects import constants from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.querysets import ( - ChildRelatedProjectQuerySet, - FeatureQuerySet, - ProjectQuerySet, - RelatedProjectQuerySet, -) + ChildRelatedProjectQuerySet, FeatureQuerySet, ProjectQuerySet, + RelatedProjectQuerySet) from readthedocs.projects.templatetags.projects_tags import sort_version_aware -from readthedocs.projects.validators import ( - validate_domain_name, - validate_repository_url, -) +from readthedocs.projects.validators import validate_domain_name, validate_repository_url from readthedocs.projects.version_handling import determine_stable_version from readthedocs.restapi.client import api from readthedocs.vcs_support.backends import backend_cls from readthedocs.vcs_support.utils import Lock, NonBlockingLock - log = logging.getLogger(__name__) @@ -52,33 +47,21 @@ class ProjectRelationship(models.Model): This is used for subprojects """ - parent = models.ForeignKey( - 'Project', - verbose_name=_('Parent'), - related_name='subprojects', - ) - child = models.ForeignKey( - 'Project', - verbose_name=_('Child'), - related_name='superprojects', - ) - alias = models.SlugField( - _('Alias'), - max_length=255, - null=True, - blank=True, - db_index=False, - ) + parent = models.ForeignKey('Project', verbose_name=_('Parent'), + related_name='subprojects') + child = models.ForeignKey('Project', verbose_name=_('Child'), + related_name='superprojects') + alias = models.SlugField(_('Alias'), max_length=255, null=True, blank=True, db_index=False) objects = ChildRelatedProjectQuerySet.as_manager() def __str__(self): - return '{} -> {}'.format(self.parent, self.child) + return '%s -> %s' % (self.parent, self.child) def save(self, *args, **kwargs): # pylint: disable=arguments-differ if not self.alias: self.alias = self.child.slug - super().save(*args, **kwargs) + super(ProjectRelationship, self).save(*args, **kwargs) # HACK def get_absolute_url(self): @@ -95,202 +78,113 @@ class Project(models.Model): modified_date = models.DateTimeField(_('Modified date'), auto_now=True) # Generally from conf.py - users = models.ManyToManyField( - User, - verbose_name=_('User'), - related_name='projects', - ) + users = models.ManyToManyField(User, verbose_name=_('User'), + related_name='projects') # A DNS label can contain up to 63 characters. name = models.CharField(_('Name'), max_length=63) slug = models.SlugField(_('Slug'), max_length=63, unique=True) - description = models.TextField( - _('Description'), - blank=True, - help_text=_( - 'The reStructuredText ' - 'description of the project', - ), - ) - repo = models.CharField( - _('Repository URL'), - max_length=255, - validators=[validate_repository_url], - help_text=_('Hosted documentation repository URL'), - ) - repo_type = models.CharField( - _('Repository type'), - max_length=10, - choices=constants.REPO_CHOICES, - default='git', - ) - project_url = models.URLField( - _('Project homepage'), - blank=True, - help_text=_('The project\'s homepage'), - ) - canonical_url = models.URLField( - _('Canonical URL'), - blank=True, - help_text=_('URL that documentation is expected to serve from'), - ) + description = models.TextField(_('Description'), blank=True, + help_text=_('The reStructuredText ' + 'description of the project')) + repo = models.CharField(_('Repository URL'), max_length=255, + validators=[validate_repository_url], + help_text=_('Hosted documentation repository URL')) + repo_type = models.CharField(_('Repository type'), max_length=10, + choices=constants.REPO_CHOICES, default='git') + project_url = models.URLField(_('Project homepage'), blank=True, + help_text=_('The project\'s homepage')) + canonical_url = models.URLField(_('Canonical URL'), blank=True, + help_text=_('URL that documentation is expected to serve from')) single_version = models.BooleanField( - _('Single version'), - default=False, - help_text=_( - 'A single version site has no translations and only your ' - '"latest" version, served at the root of the domain. Use ' - 'this with caution, only turn it on if you will never ' - 'have multiple versions of your docs.', - ), - ) + _('Single version'), default=False, + help_text=_('A single version site has no translations and only your ' + '"latest" version, served at the root of the domain. Use ' + 'this with caution, only turn it on if you will never ' + 'have multiple versions of your docs.')) default_version = models.CharField( - _('Default version'), - max_length=255, - default=LATEST, - help_text=_('The version of your project that / redirects to'), - ) + _('Default version'), max_length=255, default=LATEST, + help_text=_('The version of your project that / redirects to')) # In default_branch, None means the backend should choose the # appropriate branch. Eg 'master' for git default_branch = models.CharField( - _('Default branch'), - max_length=255, - default=None, - null=True, - blank=True, - help_text=_( - 'What branch "latest" points to. Leave empty ' - 'to use the default value for your VCS (eg. ' - 'trunk or master).', - ), - ) + _('Default branch'), max_length=255, default=None, null=True, + blank=True, help_text=_('What branch "latest" points to. Leave empty ' + 'to use the default value for your VCS (eg. ' + 'trunk or master).')) requirements_file = models.CharField( - _('Requirements file'), - max_length=255, - default=None, - null=True, - blank=True, - help_text=_( + _('Requirements file'), max_length=255, default=None, null=True, + blank=True, help_text=_( 'A ' 'pip requirements file needed to build your documentation. ' - 'Path from the root of your project.', - ), - ) + 'Path from the root of your project.')) documentation_type = models.CharField( - _('Documentation type'), - max_length=20, - choices=constants.DOCUMENTATION_CHOICES, - default='sphinx', - help_text=_( - 'Type of documentation you are building. More info.', - ), - ) + _('Documentation type'), max_length=20, + choices=constants.DOCUMENTATION_CHOICES, default='sphinx', + help_text=_('Type of documentation you are building. More info.')) # Project features cdn_enabled = models.BooleanField(_('CDN Enabled'), default=False) analytics_code = models.CharField( - _('Analytics code'), - max_length=50, - null=True, - blank=True, - help_text=_( - 'Google Analytics Tracking ID ' - '(ex. UA-22345342-1). ' - 'This may slow down your page loads.', - ), - ) + _('Analytics code'), max_length=50, null=True, blank=True, + help_text=_('Google Analytics Tracking ID ' + '(ex. UA-22345342-1). ' + 'This may slow down your page loads.')) container_image = models.CharField( - _('Alternative container image'), - max_length=64, - null=True, - blank=True, - ) + _('Alternative container image'), max_length=64, null=True, blank=True) container_mem_limit = models.CharField( - _('Container memory limit'), - max_length=10, - null=True, - blank=True, - help_text=_( - 'Memory limit in Docker format ' - '-- example: 512m or 1g', - ), - ) + _('Container memory limit'), max_length=10, null=True, blank=True, + help_text=_('Memory limit in Docker format ' + '-- example: 512m or 1g')) container_time_limit = models.IntegerField( _('Container time limit in seconds'), null=True, blank=True, ) build_queue = models.CharField( - _('Alternate build queue id'), - max_length=32, - null=True, - blank=True, - ) + _('Alternate build queue id'), max_length=32, null=True, blank=True) allow_promos = models.BooleanField( - _('Allow paid advertising'), - default=True, - help_text=_('If unchecked, users will still see community ads.'), - ) + _('Allow paid advertising'), default=True, help_text=_( + 'If unchecked, users will still see community ads.')) ad_free = models.BooleanField( _('Ad-free'), default=False, help_text='If checked, do not show advertising for this project', ) show_version_warning = models.BooleanField( - _('Show version warning'), - default=False, - help_text=_('Show warning banner in non-stable nor latest versions.'), + _('Show version warning'), default=False, + help_text=_('Show warning banner in non-stable nor latest versions.') ) # Sphinx specific build options. enable_epub_build = models.BooleanField( - _('Enable EPUB build'), - default=True, + _('Enable EPUB build'), default=True, help_text=_( - 'Create a EPUB version of your documentation with each build.', - ), - ) + 'Create a EPUB version of your documentation with each build.')) enable_pdf_build = models.BooleanField( - _('Enable PDF build'), - default=True, + _('Enable PDF build'), default=True, help_text=_( - 'Create a PDF version of your documentation with each build.', - ), - ) + 'Create a PDF version of your documentation with each build.')) # Other model data. - path = models.CharField( - _('Path'), - max_length=255, - editable=False, - help_text=_( - 'The directory where ' - 'conf.py lives', - ), - ) + path = models.CharField(_('Path'), max_length=255, editable=False, + help_text=_('The directory where ' + 'conf.py lives')) conf_py_file = models.CharField( - _('Python configuration file'), - max_length=255, - default='', - blank=True, - help_text=_( - 'Path from project root to conf.py file ' - '(ex. docs/conf.py). ' - 'Leave blank if you want us to find it for you.', - ), - ) + _('Python configuration file'), max_length=255, default='', blank=True, + help_text=_('Path from project root to conf.py file ' + '(ex. docs/conf.py). ' + 'Leave blank if you want us to find it for you.')) featured = models.BooleanField(_('Featured'), default=False) skip = models.BooleanField(_('Skip'), default=False) install_project = models.BooleanField( _('Install Project'), - help_text=_( - 'Install your project inside a virtualenv using setup.py ' - 'install', - ), - default=False, + help_text=_('Install your project inside a virtualenv using setup.py ' + 'install'), + default=False ) # This model attribute holds the python interpreter used to create the @@ -300,104 +194,64 @@ class Project(models.Model): max_length=20, choices=constants.PYTHON_CHOICES, default='python', - help_text=_( - 'The Python interpreter used to create the virtual ' - 'environment.', - ), - ) + help_text=_('The Python interpreter used to create the virtual ' + 'environment.')) use_system_packages = models.BooleanField( _('Use system packages'), - help_text=_( - 'Give the virtual environment access to the global ' - 'site-packages dir.', - ), - default=False, + help_text=_('Give the virtual environment access to the global ' + 'site-packages dir.'), + default=False ) privacy_level = models.CharField( - _('Privacy Level'), - max_length=20, - choices=constants.PRIVACY_CHOICES, - default=getattr( - settings, - 'DEFAULT_PRIVACY_LEVEL', - 'public', - ), - help_text=_( - 'Level of privacy that you want on the repository. ' - 'Protected means public but not in listings.', - ), - ) + _('Privacy Level'), max_length=20, choices=constants.PRIVACY_CHOICES, + default=getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public'), + help_text=_('Level of privacy that you want on the repository. ' + 'Protected means public but not in listings.')) version_privacy_level = models.CharField( - _('Version Privacy Level'), - max_length=20, - choices=constants.PRIVACY_CHOICES, - default=getattr( - settings, - 'DEFAULT_PRIVACY_LEVEL', - 'public', - ), - help_text=_( - 'Default level of privacy you want on built ' - 'versions of documentation.', - ), - ) + _('Version Privacy Level'), max_length=20, + choices=constants.PRIVACY_CHOICES, default=getattr( + settings, 'DEFAULT_PRIVACY_LEVEL', 'public'), + help_text=_('Default level of privacy you want on built ' + 'versions of documentation.')) # Subprojects related_projects = models.ManyToManyField( - 'self', - verbose_name=_('Related projects'), - blank=True, - symmetrical=False, - through=ProjectRelationship, - ) + 'self', verbose_name=_('Related projects'), blank=True, + symmetrical=False, through=ProjectRelationship) # Language bits - language = models.CharField( - _('Language'), - max_length=20, - default='en', - help_text=_( - 'The language the project ' - 'documentation is rendered in. ' - "Note: this affects your project's URL.", - ), - choices=constants.LANGUAGES, - ) + language = models.CharField(_('Language'), max_length=20, default='en', + help_text=_('The language the project ' + 'documentation is rendered in. ' + "Note: this affects your project's URL."), + choices=constants.LANGUAGES) programming_language = models.CharField( _('Programming Language'), max_length=20, default='words', help_text=_( - 'The primary programming language the project is written in.', - ), - choices=constants.PROGRAMMING_LANGUAGES, - blank=True, - ) + 'The primary programming language the project is written in.'), + choices=constants.PROGRAMMING_LANGUAGES, blank=True) # A subproject pointed at its main language, so it can be tracked - main_language_project = models.ForeignKey( - 'self', - related_name='translations', - on_delete=models.SET_NULL, - blank=True, - null=True, - ) + main_language_project = models.ForeignKey('self', + related_name='translations', + on_delete=models.SET_NULL, + blank=True, null=True) has_valid_webhook = models.BooleanField( - default=False, - help_text=_('This project has been built with a webhook'), + default=False, help_text=_('This project has been built with a webhook') ) has_valid_clone = models.BooleanField( - default=False, - help_text=_('This project has been successfully cloned'), + default=False, help_text=_('This project has been successfully cloned') ) tags = TaggableManager(blank=True) objects = ProjectQuerySet.as_manager() all_objects = models.Manager() - class Meta: + class Meta(object): ordering = ('slug',) permissions = ( # Translators: Permission around whether a user can view the @@ -416,7 +270,12 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ self.slug = slugify(self.name) if not self.slug: raise Exception(_('Model must have slug')) - super().save(*args, **kwargs) + if self.documentation_type == 'auto': + # This used to determine the type and automatically set the + # documentation type to Sphinx for rST and Mkdocs for markdown. + # It now just forces Sphinx, due to markdown support. + self.documentation_type = 'sphinx' + super(Project, self).save(*args, **kwargs) for owner in self.users.all(): assign('view_project', owner, self) try: @@ -455,10 +314,7 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ try: if not first_save: broadcast( - type='app', - task=tasks.update_static_metadata, - args=[self.pk], - ) + type='app', task=tasks.update_static_metadata, args=[self.pk],) except Exception: log.exception('failed to update static metadata') try: @@ -477,20 +333,12 @@ def get_docs_url(self, version_slug=None, lang_slug=None, private=None): Always use http for now, to avoid content warnings. """ - return resolve( - project=self, - version_slug=version_slug, - language=lang_slug, - private=private, - ) + return resolve(project=self, version_slug=version_slug, language=lang_slug, private=private) def get_builds_url(self): - return reverse( - 'builds_project_list', - kwargs={ - 'project_slug': self.slug, - }, - ) + return reverse('builds_project_list', kwargs={ + 'project_slug': self.slug, + }) def get_canonical_url(self): if getattr(settings, 'DONT_HIT_DB', True): @@ -504,8 +352,11 @@ def get_subproject_urls(self): This is used in search result linking """ if getattr(settings, 'DONT_HIT_DB', True): - return [(proj['slug'], proj['canonical_url']) for proj in - (api.project(self.pk).subprojects().get()['subprojects'])] + return [(proj['slug'], proj['canonical_url']) + for proj in ( + api.project(self.pk) + .subprojects() + .get()['subprojects'])] return [(proj.child.slug, proj.child.get_docs_url()) for proj in self.subprojects.all()] @@ -520,43 +371,29 @@ def get_production_media_path(self, type_, version_slug, include_file=True): :returns: Full path to media file or path """ - if getattr(settings, 'DEFAULT_PRIVACY_LEVEL', - 'public') == 'public' or settings.DEBUG: + if getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') == 'public' or settings.DEBUG: path = os.path.join( - settings.MEDIA_ROOT, - type_, - self.slug, - version_slug, - ) + settings.MEDIA_ROOT, type_, self.slug, version_slug) else: path = os.path.join( - settings.PRODUCTION_MEDIA_ARTIFACTS, - type_, - self.slug, - version_slug, - ) + settings.PRODUCTION_MEDIA_ARTIFACTS, type_, self.slug, version_slug) if include_file: path = os.path.join( - path, - '{}.{}'.format(self.slug, type_.replace('htmlzip', 'zip')), - ) + path, '%s.%s' % (self.slug, type_.replace('htmlzip', 'zip'))) return path def get_production_media_url(self, type_, version_slug, full_path=True): """Get the URL for downloading a specific media file.""" try: - path = reverse( - 'project_download_media', - kwargs={ - 'project_slug': self.slug, - 'type_': type_, - 'version_slug': version_slug, - }, - ) + path = reverse('project_download_media', kwargs={ + 'project_slug': self.slug, + 'type_': type_, + 'version_slug': version_slug, + }) except NoReverseMatch: return '' if full_path: - path = '//{}{}'.format(settings.PRODUCTION_DOMAIN, path) + path = '//%s%s' % (settings.PRODUCTION_DOMAIN, path) return path def subdomain(self): @@ -566,17 +403,11 @@ def subdomain(self): def get_downloads(self): downloads = {} downloads['htmlzip'] = self.get_production_media_url( - 'htmlzip', - self.get_default_version(), - ) + 'htmlzip', self.get_default_version()) downloads['epub'] = self.get_production_media_url( - 'epub', - self.get_default_version(), - ) + 'epub', self.get_default_version()) downloads['pdf'] = self.get_production_media_url( - 'pdf', - self.get_default_version(), - ) + 'pdf', self.get_default_version()) return downloads @property @@ -676,9 +507,7 @@ def conf_file(self, version=LATEST): """Find a ``conf.py`` file in the project checkout.""" if self.conf_py_file: conf_path = os.path.join( - self.checkout_path(version), - self.conf_py_file, - ) + self.checkout_path(version), self.conf_py_file,) if os.path.exists(conf_path): log.info('Inserting conf.py file path from model') @@ -701,10 +530,12 @@ def conf_file(self, version=LATEST): # the `doc` word in the path, we raise an error informing this to the user if len(files) > 1: raise ProjectConfigurationError( - ProjectConfigurationError.MULTIPLE_CONF_FILES, + ProjectConfigurationError.MULTIPLE_CONF_FILES ) - raise ProjectConfigurationError(ProjectConfigurationError.NOT_FOUND) + raise ProjectConfigurationError( + ProjectConfigurationError.NOT_FOUND + ) def conf_dir(self, version=LATEST): conf_file = self.conf_file(version) @@ -730,30 +561,18 @@ def has_aliases(self): def has_pdf(self, version_slug=LATEST): if not self.enable_pdf_build: return False - return os.path.exists( - self.get_production_media_path( - type_='pdf', - version_slug=version_slug, - ) - ) + return os.path.exists(self.get_production_media_path( + type_='pdf', version_slug=version_slug)) def has_epub(self, version_slug=LATEST): if not self.enable_epub_build: return False - return os.path.exists( - self.get_production_media_path( - type_='epub', - version_slug=version_slug, - ) - ) + return os.path.exists(self.get_production_media_path( + type_='epub', version_slug=version_slug)) def has_htmlzip(self, version_slug=LATEST): - return os.path.exists( - self.get_production_media_path( - type_='htmlzip', - version_slug=version_slug, - ) - ) + return os.path.exists(self.get_production_media_path( + type_='htmlzip', version_slug=version_slug)) @property def sponsored(self): @@ -853,10 +672,8 @@ def api_versions(self): def active_versions(self): from readthedocs.builds.models import Version versions = Version.objects.public(project=self, only_active=True) - return ( - versions.filter(built=True, active=True) | - versions.filter(active=True, uploaded=True) - ) + return (versions.filter(built=True, active=True) | + versions.filter(active=True, uploaded=True)) def ordered_active_versions(self, user=None): from readthedocs.builds.models import Version @@ -897,27 +714,23 @@ def update_stable_version(self): current_stable = self.get_stable_version() if current_stable: identifier_updated = ( - new_stable.identifier != current_stable.identifier - ) + new_stable.identifier != current_stable.identifier) if identifier_updated and current_stable.active and current_stable.machine: log.info( 'Update stable version: {project}:{version}'.format( project=self.slug, - version=new_stable.identifier, - ), - ) + version=new_stable.identifier)) current_stable.identifier = new_stable.identifier current_stable.save() return new_stable else: log.info( - 'Creating new stable version: {project}:{version}' - .format(project=self.slug, version=new_stable.identifier), - ) + 'Creating new stable version: {project}:{version}'.format( + project=self.slug, + version=new_stable.identifier)) current_stable = self.versions.create_stable( type=new_stable.type, - identifier=new_stable.identifier, - ) + identifier=new_stable.identifier) return new_stable def versions_from_branch_name(self, branch): @@ -940,8 +753,7 @@ def get_default_version(self): return self.default_version # check if the default_version exists version_qs = self.versions.filter( - slug=self.default_version, - active=True, + slug=self.default_version, active=True ) if version_qs.exists(): return self.default_version @@ -955,9 +767,7 @@ def get_default_branch(self): def add_subproject(self, child, alias=None): subproject, __ = ProjectRelationship.objects.get_or_create( - parent=self, - child=child, - alias=alias, + parent=self, child=child, alias=alias, ) return subproject @@ -990,7 +800,7 @@ def get_feature_value(self, feature, positive, negative): @property def show_advertising(self): """ - Whether this project is ad-free. + Whether this project is ad-free :returns: ``True`` if advertising should be shown and ``False`` otherwise :rtype: bool @@ -1040,18 +850,13 @@ def __init__(self, *args, **kwargs): ad_free = (not kwargs.pop('show_advertising', True)) # These fields only exist on the API return, not on the model, so we'll # remove them to avoid throwing exceptions due to unexpected fields - for key in [ - 'users', - 'resource_uri', - 'absolute_url', - 'downloads', - 'main_language_project', - 'related_projects']: + for key in ['users', 'resource_uri', 'absolute_url', 'downloads', + 'main_language_project', 'related_projects']: try: del kwargs[key] except KeyError: pass - super().__init__(*args, **kwargs) + super(APIProject, self).__init__(*args, **kwargs) # Overwrite the database property with the value from the API self.ad_free = ad_free @@ -1083,17 +888,10 @@ class ImportedFile(models.Model): things like CDN invalidation. """ - project = models.ForeignKey( - 'Project', - verbose_name=_('Project'), - related_name='imported_files', - ) - version = models.ForeignKey( - 'builds.Version', - verbose_name=_('Version'), - related_name='imported_files', - null=True, - ) + project = models.ForeignKey('Project', verbose_name=_('Project'), + related_name='imported_files') + version = models.ForeignKey('builds.Version', verbose_name=_('Version'), + related_name='imported_files', null=True) name = models.CharField(_('Name'), max_length=255) slug = models.SlugField(_('Slug')) path = models.CharField(_('Path'), max_length=255) @@ -1102,21 +900,18 @@ class ImportedFile(models.Model): modified_date = models.DateTimeField(_('Modified date'), auto_now=True) def get_absolute_url(self): - return resolve( - project=self.project, - version_slug=self.version.slug, - filename=self.path, - ) + return resolve(project=self.project, version_slug=self.version.slug, filename=self.path) def __str__(self): - return '{}: {}'.format(self.name, self.project) + return '%s: %s' % (self.name, self.project) class Notification(models.Model): - project = models.ForeignKey(Project, related_name='%(class)s_notifications') + project = models.ForeignKey(Project, + related_name='%(class)s_notifications') objects = RelatedProjectQuerySet.as_manager() - class Meta: + class Meta(object): abstract = True @@ -1130,11 +925,8 @@ def __str__(self): @python_2_unicode_compatible class WebHook(Notification): - url = models.URLField( - max_length=600, - blank=True, - help_text=_('URL to send the webhook to'), - ) + url = models.URLField(max_length=600, blank=True, + help_text=_('URL to send the webhook to')) def __str__(self): return self.url @@ -1146,47 +938,35 @@ class Domain(models.Model): """A custom domain name for a project.""" project = models.ForeignKey(Project, related_name='domains') - domain = models.CharField( - _('Domain'), - unique=True, - max_length=255, - validators=[validate_domain_name], - ) + domain = models.CharField(_('Domain'), unique=True, max_length=255, + validators=[validate_domain_name]) machine = models.BooleanField( - default=False, - help_text=_('This Domain was auto-created'), + default=False, help_text=_('This Domain was auto-created') ) cname = models.BooleanField( - default=False, - help_text=_('This Domain is a CNAME for the project'), + default=False, help_text=_('This Domain is a CNAME for the project') ) canonical = models.BooleanField( default=False, help_text=_( 'This Domain is the primary one where the documentation is ' - 'served from', - ), + 'served from') ) https = models.BooleanField( _('Use HTTPS'), default=False, - help_text=_('Always use HTTPS for this domain'), - ) - count = models.IntegerField( - default=0, - help_text=_('Number of times this domain has been hit'), + help_text=_('Always use HTTPS for this domain') ) + count = models.IntegerField(default=0, help_text=_( + 'Number of times this domain has been hit'),) objects = RelatedProjectQuerySet.as_manager() - class Meta: + class Meta(object): ordering = ('-canonical', '-machine', 'domain') def __str__(self): - return '{domain} pointed at {project}'.format( - domain=self.domain, - project=self.project.name, - ) + return '{domain} pointed at {project}'.format(domain=self.domain, project=self.project.name) def save(self, *args, **kwargs): # pylint: disable=arguments-differ from readthedocs.projects import tasks @@ -1195,21 +975,15 @@ def save(self, *args, **kwargs): # pylint: disable=arguments-differ self.domain = parsed.netloc else: self.domain = parsed.path - super().save(*args, **kwargs) - broadcast( - type='app', - task=tasks.symlink_domain, - args=[self.project.pk, self.pk], - ) + super(Domain, self).save(*args, **kwargs) + broadcast(type='app', task=tasks.symlink_domain, + args=[self.project.pk, self.pk],) def delete(self, *args, **kwargs): # pylint: disable=arguments-differ from readthedocs.projects import tasks - broadcast( - type='app', - task=tasks.symlink_domain, - args=[self.project.pk, self.pk, True], - ) - super().delete(*args, **kwargs) + broadcast(type='app', task=tasks.symlink_domain, + args=[self.project.pk, self.pk, True],) + super(Domain, self).delete(*args, **kwargs) @python_2_unicode_compatible @@ -1240,7 +1014,6 @@ def add_features(sender, **kwargs): ALLOW_V2_CONFIG_FILE = 'allow_v2_config_file' MKDOCS_THEME_RTD = 'mkdocs_theme_rtd' DONT_SHALLOW_CLONE = 'dont_shallow_clone' - USE_TESTING_BUILD_IMAGE = 'use_testing_build_image' FEATURES = ( (USE_SPHINX_LATEST, _('Use latest version of Sphinx')), @@ -1248,34 +1021,13 @@ def add_features(sender, **kwargs): (ALLOW_DEPRECATED_WEBHOOKS, _('Allow deprecated webhook views')), (PIP_ALWAYS_UPGRADE, _('Always run pip install --upgrade')), (SKIP_SUBMODULES, _('Skip git submodule checkout')), - ( - DONT_OVERWRITE_SPHINX_CONTEXT, - _( - 'Do not overwrite context vars in conf.py with Read the Docs context', - ), - ), - ( - ALLOW_V2_CONFIG_FILE, - _( - 'Allow to use the v2 of the configuration file', - ), - ), - ( - MKDOCS_THEME_RTD, - _('Use Read the Docs theme for MkDocs as default theme') - ), - ( - DONT_SHALLOW_CLONE, - _( - 'Do not shallow clone when cloning git repos', - ), - ), - ( - USE_TESTING_BUILD_IMAGE, - _( - 'Use Docker image labelled as `testing` to build the docs', - ), - ), + (DONT_OVERWRITE_SPHINX_CONTEXT, _( + 'Do not overwrite context vars in conf.py with Read the Docs context')), + (ALLOW_V2_CONFIG_FILE, _( + 'Allow to use the v2 of the configuration file')), + (MKDOCS_THEME_RTD, _('Use Read the Docs theme for MkDocs as default theme')), + (DONT_SHALLOW_CLONE, _( + 'Do not shallow clone when cloning git repos')), ) projects = models.ManyToManyField( @@ -1301,7 +1053,9 @@ def add_features(sender, **kwargs): objects = FeatureQuerySet.as_manager() def __str__(self): - return '{} feature'.format(self.get_feature_display(),) + return '{0} feature'.format( + self.get_feature_display(), + ) def get_feature_display(self): """ @@ -1313,7 +1067,6 @@ def get_feature_display(self): return dict(self.FEATURES).get(self.feature_id, self.feature_id) -@python_2_unicode_compatible class EnvironmentVariable(TimeStampedModel, models.Model): name = models.CharField( max_length=128, @@ -1328,10 +1081,3 @@ class EnvironmentVariable(TimeStampedModel, models.Model): on_delete=models.CASCADE, help_text=_('Project where this variable will be used'), ) - - def __str__(self): - return self.name - - def save(self, *args, **kwargs): # pylint: disable=arguments-differ - self.value = shlex_quote(self.value) - return super().save(*args, **kwargs) diff --git a/readthedocs/projects/notifications.py b/readthedocs/projects/notifications.py index 4848b575f18..eafbecf66c3 100644 --- a/readthedocs/projects/notifications.py +++ b/readthedocs/projects/notifications.py @@ -1,9 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Project notifications.""" - -from django.http import HttpRequest +"""Project notifications""" +from __future__ import absolute_import from readthedocs.notifications import Notification from readthedocs.notifications.constants import REQUIREMENT @@ -14,40 +11,3 @@ class ResourceUsageNotification(Notification): context_object_name = 'project' subject = 'Builds for {{ project.name }} are using too many resources' level = REQUIREMENT - - -class DeprecatedViewNotification(Notification): - - """Notification to alert user of a view that is going away.""" - - context_object_name = 'project' - subject = '{{ project.name }} project webhook needs to be updated' - level = REQUIREMENT - - @classmethod - def notify_project_users(cls, projects): - """ - Notify project users of deprecated view. - - :param projects: List of project instances - :type projects: [:py:class:`Project`] - """ - for project in projects: - # Send one notification to each owner of the project - for user in project.users.all(): - notification = cls( - context_object=project, - request=HttpRequest(), - user=user, - ) - notification.send() - - -class DeprecatedGitHubWebhookNotification(DeprecatedViewNotification): - - name = 'deprecated_github_webhook' - - -class DeprecatedBuildWebhookNotification(DeprecatedViewNotification): - - name = 'deprecated_build_webhook' diff --git a/readthedocs/projects/querysets.py b/readthedocs/projects/querysets.py index 3d04667d0ff..3884eb26db6 100644 --- a/readthedocs/projects/querysets.py +++ b/readthedocs/projects/querysets.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- - """Project model QuerySet classes.""" +from __future__ import absolute_import + from django.db import models from django.db.models import Q from guardian.shortcuts import get_objects_for_user @@ -44,9 +45,7 @@ def public(self, user=None): return queryset def protected(self, user=None): - queryset = self.filter( - privacy_level__in=[constants.PUBLIC, constants.PROTECTED], - ) + queryset = self.filter(privacy_level__in=[constants.PUBLIC, constants.PROTECTED]) if user: return self._add_user_repos(queryset, user) return queryset @@ -63,7 +62,6 @@ def is_active(self, project): The check consists on, * the Project shouldn't be marked as skipped. - * any of the project's owners is banned. :param project: project to be checked :type project: readthedocs.projects.models.Project @@ -71,8 +69,7 @@ def is_active(self, project): :returns: whether or not the project is active :rtype: bool """ - any_owner_banned = any(u.profile.banned for u in project.users.all()) - if project.skip or any_owner_banned: + if project.skip: return False return True @@ -94,11 +91,9 @@ class ProjectQuerySet(SettingsOverrideObject): class RelatedProjectQuerySetBase(models.QuerySet): """ - Useful for objects that relate to Project and its permissions. - - Objects get the permissions from the project itself. + A manager for things that relate to Project and need to get their perms from the project. - ..note:: This shouldn't be used as a subclass. + This shouldn't be used as a subclass. """ use_for_related_fields = True @@ -127,10 +122,7 @@ def public(self, user=None, project=None): def protected(self, user=None, project=None): kwargs = { - '%s__privacy_level__in' % self.project_field: [ - constants.PUBLIC, - constants.PROTECTED, - ], + '%s__privacy_level__in' % self.project_field: [constants.PUBLIC, constants.PROTECTED] } queryset = self.filter(**kwargs) if user: @@ -185,5 +177,5 @@ class FeatureQuerySet(models.QuerySet): def for_project(self, project): return self.filter( Q(projects=project) | - Q(default_true=True, add_date__gt=project.pub_date), + Q(default_true=True, add_date__gt=project.pub_date) ).distinct() diff --git a/readthedocs/projects/signals.py b/readthedocs/projects/signals.py index 1d1788ef28a..6ef49f9e67c 100644 --- a/readthedocs/projects/signals.py +++ b/readthedocs/projects/signals.py @@ -1,19 +1,16 @@ # -*- coding: utf-8 -*- +"""Project signals""" -"""Project signals.""" - +from __future__ import absolute_import import django.dispatch -before_vcs = django.dispatch.Signal(providing_args=['version']) -after_vcs = django.dispatch.Signal(providing_args=['version']) - -before_build = django.dispatch.Signal(providing_args=['version']) -after_build = django.dispatch.Signal(providing_args=['version']) +before_vcs = django.dispatch.Signal(providing_args=["version"]) +after_vcs = django.dispatch.Signal(providing_args=["version"]) -project_import = django.dispatch.Signal(providing_args=['project']) +before_build = django.dispatch.Signal(providing_args=["version"]) +after_build = django.dispatch.Signal(providing_args=["version"]) -files_changed = django.dispatch.Signal(providing_args=['project', 'files']) +project_import = django.dispatch.Signal(providing_args=["project"]) -# Used to force verify a domain (eg. for SSL cert issuance) -domain_verify = django.dispatch.Signal(providing_args=['domain']) +files_changed = django.dispatch.Signal(providing_args=["project", "files"]) diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 5f00374aef1..bcc81913753 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- - """ Tasks related to projects. @@ -7,6 +6,13 @@ rebuilding documentation. """ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import datetime import hashlib import json @@ -17,10 +23,11 @@ from collections import Counter, defaultdict import requests +from builtins import str from celery.exceptions import SoftTimeLimitExceeded from django.conf import settings -from django.db.models import Q from django.urls import reverse +from django.db.models import Q from django.utils import timezone from django.utils.translation import ugettext_lazy as _ from slumber.exceptions import HttpClientError @@ -49,11 +56,11 @@ ) from readthedocs.doc_builder.exceptions import ( BuildEnvironmentError, - BuildEnvironmentWarning, BuildTimeoutError, ProjectBuildsSkippedError, VersionLockedError, YAMLParseError, + BuildEnvironmentWarning, ) from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv @@ -72,15 +79,13 @@ after_vcs, before_build, before_vcs, - domain_verify, files_changed, ) - log = logging.getLogger(__name__) -class SyncRepositoryMixin: +class SyncRepositoryMixin(object): """Mixin that handles the VCS sync/update.""" @@ -100,9 +105,9 @@ def get_version(project=None, version_pk=None): if version_pk: version_data = api_v2.version(version_pk).get() else: - version_data = ( - api_v2.version(project.slug).get(slug=LATEST)['objects'][0] - ) + version_data = (api_v2 + .version(project.slug) + .get(slug=LATEST)['objects'][0]) return APIVersion(**version_data) def get_vcs_repo(self): @@ -137,13 +142,11 @@ def sync_repo(self): slug=self.version.slug, identifier=self.version.identifier, ) - log.info( - LOG_TEMPLATE.format( - project=self.project.slug, - version=self.version.slug, - msg=msg, - ), - ) + log.info(LOG_TEMPLATE.format( + project=self.project.slug, + version=self.version.slug, + msg=msg, + )) version_repo = self.get_vcs_repo() version_repo.update() self.sync_versions(version_repo) @@ -155,28 +158,34 @@ def sync_versions(self, version_repo): """ Update tags/branches hitting the API. - It may trigger a new build to the stable version when hittig the - ``sync_versions`` endpoint. + It may trigger a new build to the stable version + when hittig the ``sync_versions`` endpoint. """ version_post_data = {'repo': version_repo.repo_url} if version_repo.supports_tags: - version_post_data['tags'] = [{ - 'identifier': v.identifier, - 'verbose_name': v.verbose_name, - } for v in version_repo.tags] + version_post_data['tags'] = [ + { + 'identifier': v.identifier, + 'verbose_name': v.verbose_name, + } + for v in version_repo.tags + ] if version_repo.supports_branches: - version_post_data['branches'] = [{ - 'identifier': v.identifier, - 'verbose_name': v.verbose_name, - } for v in version_repo.branches] + version_post_data['branches'] = [ + { + 'identifier': v.identifier, + 'verbose_name': v.verbose_name, + } + for v in version_repo.branches + ] self.validate_duplicate_reserved_versions(version_post_data) try: api_v2.project(self.project.pk).sync_versions.post( - version_post_data, + version_post_data ) except HttpClientError: log.exception('Sync Versions Exception') @@ -201,7 +210,7 @@ def validate_duplicate_reserved_versions(self, data): for reserved_name in [STABLE_VERBOSE_NAME, LATEST_VERBOSE_NAME]: if counter[reserved_name] > 1: raise RepositoryError( - RepositoryError.DUPLICATED_RESERVED_VERSIONS, + RepositoryError.DUPLICATED_RESERVED_VERSIONS ) @@ -259,6 +268,7 @@ def run(self, version_pk): # pylint: disable=arguments-differ 'version': self.version.slug, }, }, + ) return False @@ -272,8 +282,8 @@ def run(self, version_pk): # pylint: disable=arguments-differ ProjectBuildsSkippedError, YAMLParseError, BuildTimeoutError, - ProjectBuildsSkippedError, - ), + ProjectBuildsSkippedError + ) ) def update_docs_task(self, project_id, *args, **kwargs): step = UpdateDocsTaskStep(task=self) @@ -295,19 +305,12 @@ class UpdateDocsTaskStep(SyncRepositoryMixin): underlying task. Previously, we were using a custom ``celery.Task`` for this, but this class is only instantiated once -- on startup. The effect was that this instance shared state between workers. + """ - def __init__( - self, - build_env=None, - python_env=None, - config=None, - force=False, - build=None, - project=None, - version=None, - task=None, - ): + def __init__(self, build_env=None, python_env=None, config=None, + force=False, build=None, project=None, + version=None, task=None): self.build_env = build_env self.python_env = python_env self.build_force = force @@ -326,10 +329,8 @@ def __init__( self.setup_env = None # pylint: disable=arguments-differ - def run( - self, pk, version_pk=None, build_pk=None, record=True, docker=None, - force=False, **__ - ): + def run(self, pk, version_pk=None, build_pk=None, record=True, + docker=None, force=False, **__): """ Run a documentation sync n' build. @@ -389,7 +390,7 @@ def run( self.setup_env.failure = BuildEnvironmentError( BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format( build_id=build_pk, - ), + ) ) self.setup_env.update_build(BUILD_STATE_FINISHED) @@ -417,7 +418,7 @@ def run( self.build_env.failure = BuildEnvironmentError( BuildEnvironmentError.GENERIC_WITH_BUILD_ID.format( build_id=build_pk, - ), + ) ) self.build_env.update_build(BUILD_STATE_FINISHED) @@ -456,7 +457,7 @@ def run_setup(self, record=True): raise YAMLParseError( YAMLParseError.GENERIC_WITH_PARSE_EXCEPTION.format( exception=str(e), - ), + ) ) self.save_build_config() @@ -464,15 +465,13 @@ def run_setup(self, record=True): if self.setup_env.failure or self.config is None: msg = 'Failing build because of setup failure: {}'.format( - self.setup_env.failure, - ) - log.info( - LOG_TEMPLATE.format( - project=self.project.slug, - version=self.version.slug, - msg=msg, - ), + self.setup_env.failure ) + log.info(LOG_TEMPLATE.format( + project=self.project.slug, + version=self.version.slug, + msg=msg, + )) # Send notification to users only if the build didn't fail because # of VersionLockedError: this exception occurs when a build is @@ -527,13 +526,11 @@ def run_build(self, docker, record): with self.build_env: python_env_cls = Virtualenv if self.config.conda is not None: - log.info( - LOG_TEMPLATE.format( - project=self.project.slug, - version=self.version.slug, - msg='Using conda', - ), - ) + log.info(LOG_TEMPLATE.format( + project=self.project.slug, + version=self.version.slug, + msg='Using conda', + )) python_env_cls = Conda self.python_env = python_env_cls( version=self.version, @@ -588,14 +585,12 @@ def get_build(build_pk): if build_pk: build = api_v2.build(build_pk).get() private_keys = [ - 'project', - 'version', - 'resource_uri', - 'absolute_uri', + 'project', 'version', 'resource_uri', 'absolute_uri', ] return { key: val - for key, val in build.items() if key not in private_keys + for key, val in build.items() + if key not in private_keys } def setup_vcs(self): @@ -608,13 +603,11 @@ def setup_vcs(self): """ self.setup_env.update_build(state=BUILD_STATE_CLONING) - log.info( - LOG_TEMPLATE.format( - project=self.project.slug, - version=self.version.slug, - msg='Updating docs from VCS', - ), - ) + log.info(LOG_TEMPLATE.format( + project=self.project.slug, + version=self.version.slug, + msg='Updating docs from VCS', + )) try: self.sync_repo() except RepositoryError: @@ -695,14 +688,8 @@ def save_build_config(self): }) self.build['config'] = config - def update_app_instances( - self, - html=False, - localmedia=False, - search=False, - pdf=False, - epub=False, - ): + def update_app_instances(self, html=False, localmedia=False, search=False, + pdf=False, epub=False): """ Update application instances with build artifacts. @@ -813,7 +800,7 @@ def build_docs_html(self): type='app', task=move_files, args=[self.version.pk, socket.gethostname()], - kwargs=dict(html=True), + kwargs=dict(html=True) ) except socket.error: log.exception('move_files task has failed on socket error.') @@ -865,7 +852,7 @@ def build_docs_class(self, builder_class): """ builder = get_builder_class(builder_class)( self.build_env, - python_env=self.python_env, + python_env=self.python_env ) success = builder.build() builder.move() @@ -882,16 +869,8 @@ def is_type_sphinx(self): # Web tasks @app.task(queue='web') -def sync_files( - project_pk, - version_pk, - hostname=None, - html=False, - localmedia=False, - search=False, - pdf=False, - epub=False, -): +def sync_files(project_pk, version_pk, hostname=None, html=False, + localmedia=False, search=False, pdf=False, epub=False): """ Sync build artifacts to application instances. @@ -901,19 +880,19 @@ def sync_files( # Clean up unused artifacts version = Version.objects.get(pk=version_pk) if not pdf: - remove_dirs([ + remove_dir( version.project.get_production_media_path( type_='pdf', version_slug=version.slug, ), - ]) + ) if not epub: - remove_dirs([ + remove_dir( version.project.get_production_media_path( type_='epub', version_slug=version.slug, ), - ]) + ) # Sync files to the web servers move_files( @@ -934,15 +913,8 @@ def sync_files( @app.task(queue='web') -def move_files( - version_pk, - hostname, - html=False, - localmedia=False, - search=False, - pdf=False, - epub=False, -): +def move_files(version_pk, hostname, html=False, localmedia=False, + search=False, pdf=False, epub=False): """ Task to move built documentation to web servers. @@ -965,7 +937,7 @@ def move_files( project=version.project.slug, version=version.slug, msg='Moving files', - ), + ) ) if html: @@ -1042,16 +1014,13 @@ def update_search(version_pk, commit, delete_non_commit_files=True): else: log.debug( 'Unknown documentation type: %s', - version.project.documentation_type, + version.project.documentation_type ) return log_msg = ' '.join([page['path'] for page in page_list]) - log.info( - '(Search Index) Sending Data: %s [%s]', - version.project.slug, - log_msg, - ) + log.info("(Search Index) Sending Data: %s [%s]", version.project.slug, + log_msg) index_search_request( version=version, page_list=page_list, @@ -1095,9 +1064,7 @@ def remove_orphan_symlinks(): """ for symlink in [PublicSymlink, PrivateSymlink]: for domain_path in [symlink.PROJECT_CNAME_ROOT, symlink.CNAME_ROOT]: - valid_cnames = set( - Domain.objects.all().values_list('domain', flat=True), - ) + valid_cnames = set(Domain.objects.all().values_list('domain', flat=True)) orphan_cnames = set(os.listdir(domain_path)) - valid_cnames for cname in orphan_cnames: orphan_domain_path = os.path.join(domain_path, cname) @@ -1142,7 +1109,7 @@ def fileify(version_pk, commit): 'Imported File not being built because no commit ' 'information' ), - ), + ) ) return @@ -1153,7 +1120,7 @@ def fileify(version_pk, commit): project=version.project.slug, version=version.slug, msg='Creating ImportedFiles', - ), + ) ) _manage_imported_files(version, path, commit) else: @@ -1162,7 +1129,7 @@ def fileify(version_pk, commit): project=project.slug, version=version.slug, msg='No ImportedFile files', - ), + ) ) @@ -1177,10 +1144,8 @@ def _manage_imported_files(version, path, commit): changed_files = set() for root, __, filenames in os.walk(path): for filename in filenames: - dirpath = os.path.join( - root.replace(path, '').lstrip('/'), - filename.lstrip('/'), - ) + dirpath = os.path.join(root.replace(path, '').lstrip('/'), + filename.lstrip('/')) full_path = os.path.join(root, filename) md5 = hashlib.md5(open(full_path, 'rb').read()).hexdigest() try: @@ -1200,22 +1165,16 @@ def _manage_imported_files(version, path, commit): obj.commit = commit obj.save() # Delete ImportedFiles from previous versions - ImportedFile.objects.filter( - project=version.project, - version=version, - ).exclude(commit=commit).delete() + ImportedFile.objects.filter(project=version.project, + version=version + ).exclude(commit=commit).delete() changed_files = [ resolve_path( - version.project, - filename=file, - version_slug=version.slug, + version.project, filename=file, version_slug=version.slug, ) for file in changed_files ] - files_changed.send( - sender=Project, - project=version.project, - files=changed_files, - ) + files_changed.send(sender=Project, project=version.project, + files=changed_files) @app.task(queue='web') @@ -1225,10 +1184,7 @@ def send_notifications(version_pk, build_pk): for hook in version.project.webhook_notifications.all(): webhook_notification(version, build, hook.url) - for email in version.project.emailhook_notifications.all().values_list( - 'email', - flat=True, - ): + for email in version.project.emailhook_notifications.all().values_list('email', flat=True): email_notification(version, build, email) @@ -1245,7 +1201,7 @@ def email_notification(version, build, email): project=version.project.slug, version=version.slug, msg='sending email to: %s' % email, - ), + ) ) # We send only what we need from the Django model objects here to avoid @@ -1261,24 +1217,20 @@ def email_notification(version, build, email): 'pk': build.pk, 'error': build.error, }, - 'build_url': 'https://{}{}'.format( + 'build_url': 'https://{0}{1}'.format( getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), build.get_absolute_url(), ), - 'unsub_url': 'https://{}{}'.format( + 'unsub_url': 'https://{0}{1}'.format( getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org'), reverse('projects_notifications', args=[version.project.slug]), ), } if build.commit: - title = _( - 'Failed: {project[name]} ({commit})', - ).format(commit=build.commit[:8], **context) + title = _('Failed: {project[name]} ({commit})').format(commit=build.commit[:8], **context) else: - title = _('Failed: {project[name]} ({version[verbose_name]})').format( - **context - ) + title = _('Failed: {project[name]} ({version[verbose_name]})').format(**context) send_email( email, @@ -1313,7 +1265,7 @@ def webhook_notification(version, build, hook_url): project=project.slug, version='', msg='sending notification to: %s' % hook_url, - ), + ) ) try: requests.post(hook_url, data=data) @@ -1346,7 +1298,7 @@ def update_static_metadata(project_pk, path=None): project=project.slug, version='', msg='Updating static metadata', - ), + ) ) translations = [trans.language for trans in project.translations.all()] languages = set(translations) @@ -1368,25 +1320,34 @@ def update_static_metadata(project_pk, path=None): LOG_TEMPLATE.format( project=project.slug, version='', - msg='Cannot write to metadata.json: {}'.format(e), - ), + msg='Cannot write to metadata.json: {0}'.format(e), + ) ) # Random Tasks @app.task() -def remove_dirs(paths): +def remove_dir(path): """ - Remove artifacts from servers. + Remove a directory on the build/celery server. - This is mainly a wrapper around shutil.rmtree so that we can remove things across - every instance of a type of server (eg. all builds or all webs). + This is mainly a wrapper around shutil.rmtree so that app servers can kill + things on the build server. + """ + log.info('Removing %s', path) + shutil.rmtree(path, ignore_errors=True) + + +@app.task() +def clear_artifacts(paths): + """ + Remove artifacts from the web servers. - :param paths: list containing PATHs where file is on disk + :param paths: list containing PATHs where production media is on disk + (usually ``Version.get_artifact_paths``) """ for path in paths: - log.info('Removing %s', path) - shutil.rmtree(path, ignore_errors=True) + remove_dir(path) @app.task(queue='web') @@ -1414,9 +1375,8 @@ def finish_inactive_builds(): """ time_limit = int(DOCKER_LIMITS['time'] * 1.2) delta = datetime.timedelta(seconds=time_limit) - query = ( - ~Q(state=BUILD_STATE_FINISHED) & Q(date__lte=timezone.now() - delta) - ) + query = (~Q(state=BUILD_STATE_FINISHED) & + Q(date__lte=timezone.now() - delta)) builds_finished = 0 builds = Build.objects.filter(query)[:50] @@ -1436,7 +1396,7 @@ def finish_inactive_builds(): build.error = _( 'This build was terminated due to inactivity. If you ' 'continue to encounter this error, file a support ' - 'request with and reference this build id ({}).'.format(build.pk), + 'request with and reference this build id ({0}).'.format(build.pk), ) build.save() builds_finished += 1 @@ -1445,17 +1405,3 @@ def finish_inactive_builds(): 'Builds marked as "Terminated due inactivity": %s', builds_finished, ) - - -@app.task(queue='web') -def retry_domain_verification(domain_pk): - """ - Trigger domain verification on a domain. - - :param domain_pk: a `Domain` pk to verify - """ - domain = Domain.objects.get(pk=domain_pk) - domain_verify.send( - sender=domain.__class__, - domain=domain, - ) diff --git a/readthedocs/projects/templatetags/projects_tags.py b/readthedocs/projects/templatetags/projects_tags.py index 6a6d6d16a2e..699b1a24900 100644 --- a/readthedocs/projects/templatetags/projects_tags.py +++ b/readthedocs/projects/templatetags/projects_tags.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - -"""Project template tags and filters.""" +"""Project template tags and filters""" +from __future__ import absolute_import from django import template from readthedocs.projects.version_handling import comparable_version @@ -12,15 +11,14 @@ @register.filter def sort_version_aware(versions): - """Takes a list of versions objects and sort them using version schemes.""" + """Takes a list of versions objects and sort them using version schemes""" return sorted( versions, key=lambda version: comparable_version(version.verbose_name), - reverse=True, - ) + reverse=True) @register.filter def is_project_user(user, project): - """Return if user is a member of project.users.""" + """Return if user is a member of project.users""" return user in project.users.all() diff --git a/readthedocs/projects/urls/private.py b/readthedocs/projects/urls/private.py index 75d6324d03b..1cdc294416d 100644 --- a/readthedocs/projects/urls/private.py +++ b/readthedocs/projects/urls/private.py @@ -1,270 +1,173 @@ -# -*- coding: utf-8 -*- - -"""Project URLs for authenticated users.""" +"""Project URLs for authenticated users""" +from __future__ import absolute_import from django.conf.urls import url from readthedocs.constants import pattern_opts -from readthedocs.projects.backends.views import ImportDemoView, ImportWizardView from readthedocs.projects.views import private from readthedocs.projects.views.private import ( - DomainCreate, - DomainDelete, - DomainList, - DomainUpdate, - EnvironmentVariableCreate, - EnvironmentVariableDelete, - EnvironmentVariableDetail, - EnvironmentVariableList, - ImportView, - IntegrationCreate, - IntegrationDelete, - IntegrationDetail, - IntegrationExchangeDetail, - IntegrationList, - IntegrationWebhookSync, - ProjectAdvancedUpdate, - ProjectAdvertisingUpdate, - ProjectDashboard, - ProjectUpdate, -) + ProjectDashboard, ImportView, + ProjectUpdate, ProjectAdvancedUpdate, + DomainList, DomainCreate, DomainDelete, DomainUpdate, + IntegrationList, IntegrationCreate, IntegrationDetail, IntegrationDelete, + IntegrationExchangeDetail, IntegrationWebhookSync, ProjectAdvertisingUpdate) +from readthedocs.projects.backends.views import ImportWizardView, ImportDemoView urlpatterns = [ - url(r'^$', ProjectDashboard.as_view(), name='projects_dashboard'), - url( - r'^import/$', ImportView.as_view(wizard_class=ImportWizardView), - {'wizard': ImportWizardView}, name='projects_import', - ), - url( - r'^import/manual/$', ImportWizardView.as_view(), - name='projects_import_manual', - ), - url( - r'^import/manual/demo/$', ImportDemoView.as_view(), - name='projects_import_demo', - ), - url( - r'^(?P[-\w]+)/$', private.project_manage, - name='projects_manage', - ), - url( - r'^(?P[-\w]+)/edit/$', ProjectUpdate.as_view(), - name='projects_edit', - ), - url( - r'^(?P[-\w]+)/advanced/$', - ProjectAdvancedUpdate.as_view(), name='projects_advanced', - ), - url( - r'^(?P[-\w]+)/version/(?P[^/]+)/delete_html/$', - private.project_version_delete_html, name='project_version_delete_html', - ), - url( - r'^(?P[-\w]+)/version/(?P[^/]+)/$', - private.project_version_detail, name='project_version_detail', - ), - url( - r'^(?P[-\w]+)/versions/$', private.project_versions, - name='projects_versions', - ), - url( - r'^(?P[-\w]+)/delete/$', private.project_delete, - name='projects_delete', - ), - url( - r'^(?P[-\w]+)/users/$', private.project_users, - name='projects_users', - ), - url( - r'^(?P[-\w]+)/users/delete/$', - private.project_users_delete, name='projects_users_delete', - ), - url( - r'^(?P[-\w]+)/notifications/$', - private.project_notifications, name='projects_notifications', - ), - url( - r'^(?P[-\w]+)/notifications/delete/$', - private.project_notifications_delete, name='projects_notification_delete', - ), - url( - r'^(?P[-\w]+)/translations/$', - private.project_translations, name='projects_translations', - ), - url( - r'^(?P[-\w]+)/translations/delete/(?P[-\w]+)/$', # noqa + url(r'^$', + ProjectDashboard.as_view(), + name='projects_dashboard'), + + url(r'^import/$', + ImportView.as_view(wizard_class=ImportWizardView), + {'wizard': ImportWizardView}, + name='projects_import'), + + url(r'^import/manual/$', + ImportWizardView.as_view(), + name='projects_import_manual'), + + url(r'^import/manual/demo/$', + ImportDemoView.as_view(), + name='projects_import_demo'), + + url(r'^(?P[-\w]+)/$', + private.project_manage, + name='projects_manage'), + + url(r'^(?P[-\w]+)/edit/$', + ProjectUpdate.as_view(), + name='projects_edit'), + + url(r'^(?P[-\w]+)/advanced/$', + ProjectAdvancedUpdate.as_view(), + name='projects_advanced'), + + url(r'^(?P[-\w]+)/version/(?P[^/]+)/delete_html/$', + private.project_version_delete_html, + name='project_version_delete_html'), + + url(r'^(?P[-\w]+)/version/(?P[^/]+)/$', + private.project_version_detail, + name='project_version_detail'), + + url(r'^(?P[-\w]+)/versions/$', + private.project_versions, + name='projects_versions'), + + url(r'^(?P[-\w]+)/delete/$', + private.project_delete, + name='projects_delete'), + + url(r'^(?P[-\w]+)/users/$', + private.project_users, + name='projects_users'), + + url(r'^(?P[-\w]+)/users/delete/$', + private.project_users_delete, + name='projects_users_delete'), + + url(r'^(?P[-\w]+)/notifications/$', + private.project_notifications, + name='projects_notifications'), + + url(r'^(?P[-\w]+)/notifications/delete/$', + private.project_notifications_delete, + name='projects_notification_delete'), + + url(r'^(?P[-\w]+)/translations/$', + private.project_translations, + name='projects_translations'), + + url(r'^(?P[-\w]+)/translations/delete/(?P[-\w]+)/$', # noqa private.project_translations_delete, - name='projects_translations_delete', - ), - url( - r'^(?P[-\w]+)/redirects/$', private.project_redirects, - name='projects_redirects', - ), - url( - r'^(?P[-\w]+)/redirects/delete/$', - private.project_redirects_delete, name='projects_redirects_delete', - ), - url( - r'^(?P[-\w]+)/advertising/$', - ProjectAdvertisingUpdate.as_view(), name='projects_advertising', - ), + name='projects_translations_delete'), + + url(r'^(?P[-\w]+)/redirects/$', + private.project_redirects, + name='projects_redirects'), + + url(r'^(?P[-\w]+)/redirects/delete/$', + private.project_redirects_delete, + name='projects_redirects_delete'), + + url(r'^(?P[-\w]+)/advertising/$', + ProjectAdvertisingUpdate.as_view(), + name='projects_advertising'), ] domain_urls = [ - url( - r'^(?P[-\w]+)/domains/$', + url(r'^(?P[-\w]+)/domains/$', DomainList.as_view(), - name='projects_domains', - ), - url( - r'^(?P[-\w]+)/domains/create/$', + name='projects_domains'), + url(r'^(?P[-\w]+)/domains/create/$', DomainCreate.as_view(), - name='projects_domains_create', - ), - url( - r'^(?P[-\w]+)/domains/(?P[-\w]+)/edit/$', + name='projects_domains_create'), + url(r'^(?P[-\w]+)/domains/(?P[-\w]+)/edit/$', DomainUpdate.as_view(), - name='projects_domains_edit', - ), - url( - r'^(?P[-\w]+)/domains/(?P[-\w]+)/delete/$', + name='projects_domains_edit'), + url(r'^(?P[-\w]+)/domains/(?P[-\w]+)/delete/$', DomainDelete.as_view(), - name='projects_domains_delete', - ), + name='projects_domains_delete'), ] urlpatterns += domain_urls integration_urls = [ - url( - r'^(?P{project_slug})/integrations/$'.format( - **pattern_opts - ), + url(r'^(?P{project_slug})/integrations/$'.format(**pattern_opts), IntegrationList.as_view(), - name='projects_integrations', - ), - url( - r'^(?P{project_slug})/integrations/sync/$'.format( - **pattern_opts - ), + name='projects_integrations'), + url(r'^(?P{project_slug})/integrations/sync/$'.format(**pattern_opts), IntegrationWebhookSync.as_view(), - name='projects_integrations_webhooks_sync', - ), - url( - ( - r'^(?P{project_slug})/integrations/create/$'.format( - **pattern_opts - ) - ), + name='projects_integrations_webhooks_sync'), + url((r'^(?P{project_slug})/integrations/create/$' + .format(**pattern_opts)), IntegrationCreate.as_view(), - name='projects_integrations_create', - ), - url( - ( - r'^(?P{project_slug})/' - r'integrations/(?P{integer_pk})/$'.format( - **pattern_opts - ) - ), + name='projects_integrations_create'), + url((r'^(?P{project_slug})/' + r'integrations/(?P{integer_pk})/$' + .format(**pattern_opts)), IntegrationDetail.as_view(), - name='projects_integrations_detail', - ), - url( - ( - r'^(?P{project_slug})/' - r'integrations/(?P{integer_pk})/' - r'exchange/(?P[-\w]+)/$'.format(**pattern_opts) - ), + name='projects_integrations_detail'), + url((r'^(?P{project_slug})/' + r'integrations/(?P{integer_pk})/' + r'exchange/(?P[-\w]+)/$' + .format(**pattern_opts)), IntegrationExchangeDetail.as_view(), - name='projects_integrations_exchanges_detail', - ), - url( - ( - r'^(?P{project_slug})/' - r'integrations/(?P{integer_pk})/sync/$'.format( - **pattern_opts - ) - ), + name='projects_integrations_exchanges_detail'), + url((r'^(?P{project_slug})/' + r'integrations/(?P{integer_pk})/sync/$' + .format(**pattern_opts)), IntegrationWebhookSync.as_view(), - name='projects_integrations_webhooks_sync', - ), - url( - ( - r'^(?P{project_slug})/' - r'integrations/(?P{integer_pk})/delete/$'.format( - **pattern_opts - ) - ), + name='projects_integrations_webhooks_sync'), + url((r'^(?P{project_slug})/' + r'integrations/(?P{integer_pk})/delete/$' + .format(**pattern_opts)), IntegrationDelete.as_view(), - name='projects_integrations_delete', - ), + name='projects_integrations_delete'), ] urlpatterns += integration_urls subproject_urls = [ - url( - r'^(?P{project_slug})/subprojects/$'.format( - **pattern_opts - ), + url(r'^(?P{project_slug})/subprojects/$'.format(**pattern_opts), private.ProjectRelationshipList.as_view(), - name='projects_subprojects', - ), - url( - ( - r'^(?P{project_slug})/subprojects/create/$'.format( - **pattern_opts - ) - ), + name='projects_subprojects'), + url((r'^(?P{project_slug})/subprojects/create/$' + .format(**pattern_opts)), private.ProjectRelationshipCreate.as_view(), - name='projects_subprojects_create', - ), - url( - ( - r'^(?P{project_slug})/' - r'subprojects/(?P{project_slug})/edit/$'.format( - **pattern_opts - ) - ), + name='projects_subprojects_create'), + url((r'^(?P{project_slug})/' + r'subprojects/(?P{project_slug})/edit/$' + .format(**pattern_opts)), private.ProjectRelationshipUpdate.as_view(), - name='projects_subprojects_update', - ), - url( - ( - r'^(?P{project_slug})/' - r'subprojects/(?P{project_slug})/delete/$'.format( - **pattern_opts - ) - ), + name='projects_subprojects_update'), + url((r'^(?P{project_slug})/' + r'subprojects/(?P{project_slug})/delete/$' + .format(**pattern_opts)), private.ProjectRelationshipDelete.as_view(), - name='projects_subprojects_delete', - ), + name='projects_subprojects_delete'), ] urlpatterns += subproject_urls - -environmentvariable_urls = [ - url( - r'^(?P[-\w]+)/environmentvariables/$', - EnvironmentVariableList.as_view(), - name='projects_environmentvariables', - ), - url( - r'^(?P[-\w]+)/environmentvariables/create/$', - EnvironmentVariableCreate.as_view(), - name='projects_environmentvariables_create', - ), - url( - r'^(?P[-\w]+)/environmentvariables/(?P[-\w]+)/$', - EnvironmentVariableDetail.as_view(), - name='projects_environmentvariables_detail', - ), - url( - r'^(?P[-\w]+)/environmentvariables/(?P[-\w]+)/delete/$', - EnvironmentVariableDelete.as_view(), - name='projects_environmentvariables_delete', - ), -] - -urlpatterns += environmentvariable_urls diff --git a/readthedocs/projects/urls/public.py b/readthedocs/projects/urls/public.py index f353714ac19..b46b8105aa4 100644 --- a/readthedocs/projects/urls/public.py +++ b/readthedocs/projects/urls/public.py @@ -1,80 +1,61 @@ -# -*- coding: utf-8 -*- - -"""Project URLS for public users.""" +"""Project URLS for public users""" +from __future__ import absolute_import from django.conf.urls import url +from readthedocs.projects.views import public +from readthedocs.projects.views.public import ProjectIndex, ProjectDetailView + from readthedocs.builds import views as build_views from readthedocs.constants import pattern_opts -from readthedocs.projects.views import public -from readthedocs.projects.views.public import ProjectDetailView, ProjectIndex urlpatterns = [ - url( - r'^$', + url(r'^$', ProjectIndex.as_view(), - name='projects_list', - ), - url( - r'^(?P{project_slug})/$'.format(**pattern_opts), + name='projects_list'), + + url(r'^(?P{project_slug})/$'.format(**pattern_opts), ProjectDetailView.as_view(), - name='projects_detail', - ), - url( - r'^(?P{project_slug})/downloads/$'.format(**pattern_opts), + name='projects_detail'), + + url(r'^(?P{project_slug})/downloads/$'.format(**pattern_opts), public.project_downloads, - name='project_downloads', - ), - url( - ( - r'^(?P{project_slug})/downloads/(?P[-\w]+)/' - r'(?P{version_slug})/$'.format(**pattern_opts) - ), + name='project_downloads'), + + url((r'^(?P{project_slug})/downloads/(?P[-\w]+)/' + r'(?P{version_slug})/$'.format(**pattern_opts)), public.project_download_media, - name='project_download_media', - ), - url( - r'^(?P{project_slug})/badge/$'.format(**pattern_opts), + name='project_download_media'), + + url(r'^(?P{project_slug})/badge/$'.format(**pattern_opts), public.project_badge, - name='project_badge', - ), - url( - ( - r'^(?P{project_slug})/tools/embed/$'.format( - **pattern_opts - ) - ), + name='project_badge'), + + url((r'^(?P{project_slug})/tools/embed/$' + .format(**pattern_opts)), public.project_embed, - name='project_embed', - ), - url( - r'^(?P{project_slug})/search/$'.format(**pattern_opts), + name='project_embed'), + + url(r'^(?P{project_slug})/search/$'.format(**pattern_opts), public.elastic_project_search, - name='elastic_project_search', - ), - url( - ( - r'^(?P{project_slug})/builds/(?P\d+)/$'.format( - **pattern_opts - ) - ), + name='elastic_project_search'), + + url((r'^(?P{project_slug})/builds/(?P\d+)/$' + .format(**pattern_opts)), build_views.BuildDetail.as_view(), - name='builds_detail', - ), - url( - (r'^(?P{project_slug})/builds/$'.format(**pattern_opts)), + name='builds_detail'), + + url((r'^(?P{project_slug})/builds/$' + .format(**pattern_opts)), build_views.BuildList.as_view(), - name='builds_project_list', - ), - url( - r'^(?P{project_slug})/versions/$'.format(**pattern_opts), + name='builds_project_list'), + + url(r'^(?P{project_slug})/versions/$'.format(**pattern_opts), public.project_versions, - name='project_version_list', - ), - url( - r'^tags/(?P[-\w]+)/$', + name='project_version_list'), + + url(r'^tags/(?P[-\w]+)/$', ProjectIndex.as_view(), - name='projects_tag_detail', - ), + name='projects_tag_detail'), ] diff --git a/readthedocs/projects/utils.py b/readthedocs/projects/utils.py index f83a3906a44..840dd17482a 100644 --- a/readthedocs/projects/utils.py +++ b/readthedocs/projects/utils.py @@ -1,13 +1,19 @@ # -*- coding: utf-8 -*- - """Utility functions used by projects.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import os +from builtins import open from django.conf import settings - log = logging.getLogger(__name__) diff --git a/readthedocs/projects/validators.py b/readthedocs/projects/validators.py index dc64b3fc086..01d350a43aa 100644 --- a/readthedocs/projects/validators.py +++ b/readthedocs/projects/validators.py @@ -1,15 +1,15 @@ -# -*- coding: utf-8 -*- - """Validators for projects app.""" +# From https://github.com/django/django/pull/3477/files +from __future__ import absolute_import import re -from urllib.parse import urlparse from django.conf import settings from django.core.exceptions import ValidationError -from django.core.validators import RegexValidator from django.utils.deconstruct import deconstructible from django.utils.translation import ugettext_lazy as _ +from django.core.validators import RegexValidator +from future.backports.urllib.parse import urlparse domain_regex = ( @@ -28,13 +28,13 @@ class DomainNameValidator(RegexValidator): def __init__(self, accept_idna=True, **kwargs): message = kwargs.get('message') self.accept_idna = accept_idna - super().__init__(**kwargs) + super(DomainNameValidator, self).__init__(**kwargs) if not self.accept_idna and message is None: self.message = _('Enter a valid domain name value') def __call__(self, value): try: - super().__call__(value) + super(DomainNameValidator, self).__call__(value) except ValidationError as exc: if not self.accept_idna: raise @@ -44,14 +44,14 @@ def __call__(self, value): idnavalue = value.encode('idna') except UnicodeError: raise exc - super().__call__(idnavalue) + super(DomainNameValidator, self).__call__(idnavalue) validate_domain_name = DomainNameValidator() @deconstructible -class RepositoryURLValidator: +class RepositoryURLValidator(object): disallow_relative_url = True @@ -99,7 +99,7 @@ def __call__(self, value): class SubmoduleURLValidator(RepositoryURLValidator): """ - A URL validator for repository submodules. + A URL validator for repository submodules If a repository has a relative submodule, the URL path is effectively the supermodule's remote ``origin`` URL with the relative path applied. diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 7a730e61fd0..2a7cffa5bdd 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- - """Project version handling.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import unicodedata +import six from packaging.version import InvalidVersion, Version from readthedocs.builds.constants import ( - LATEST_VERBOSE_NAME, - STABLE_VERBOSE_NAME, - TAG, -) + LATEST_VERBOSE_NAME, STABLE_VERBOSE_NAME, TAG) def parse_version_failsafe(version_string): @@ -25,7 +25,7 @@ def parse_version_failsafe(version_string): :rtype: packaging.version.Version """ - if not isinstance(version_string, str): + if not isinstance(version_string, six.text_type): uni_version = version_string.decode('utf-8') else: uni_version = version_string @@ -89,8 +89,7 @@ def sort_versions(version_list): versions, key=lambda version_info: version_info[1], reverse=True, - ), - ) + )) def highest_version(version_list): @@ -118,11 +117,9 @@ def determine_stable_version(version_list): :rtype: readthedocs.builds.models.Version """ versions = sort_versions(version_list) - versions = [ - (version_obj, comparable) - for version_obj, comparable in versions - if not comparable.is_prerelease - ] + versions = [(version_obj, comparable) + for version_obj, comparable in versions + if not comparable.is_prerelease] if versions: # We take preference for tags over branches. If we don't find any tag, diff --git a/readthedocs/projects/views/base.py b/readthedocs/projects/views/base.py index a29d65f663d..2bdb17fdfb2 100644 --- a/readthedocs/projects/views/base.py +++ b/readthedocs/projects/views/base.py @@ -1,31 +1,33 @@ # -*- coding: utf-8 -*- - """Mix-in classes for project views.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging +from builtins import object from datetime import timedelta from django.conf import settings +from django.urls import reverse from django.http import HttpResponseRedirect from django.shortcuts import get_object_or_404 -from django.urls import reverse from django.utils import timezone from ..exceptions import ProjectSpamError from ..models import Project - log = logging.getLogger(__name__) USER_MATURITY_DAYS = getattr(settings, 'USER_MATURITY_DAYS', 7) -class ProjectOnboardMixin: +class ProjectOnboardMixin(object): """Add project onboard context data to project object views.""" def get_context_data(self, **kwargs): """Add onboard context data.""" - context = super().get_context_data(**kwargs) + context = super(ProjectOnboardMixin, self).get_context_data(**kwargs) # If more than 1 project, don't show onboarding at all. This could # change in the future, to onboard each user maybe? if Project.objects.for_admin_user(self.request.user).count() > 1: @@ -49,7 +51,7 @@ def get_context_data(self, **kwargs): # Mixins -class ProjectAdminMixin: +class ProjectAdminMixin(object): """ Mixin class that provides project sublevel objects. @@ -72,12 +74,11 @@ def get_project(self): return None return get_object_or_404( Project.objects.for_admin_user(user=self.request.user), - slug=self.kwargs[self.project_url_field], - ) + slug=self.kwargs[self.project_url_field]) def get_context_data(self, **kwargs): """Add project to context data.""" - context = super().get_context_data(**kwargs) + context = super(ProjectAdminMixin, self).get_context_data(**kwargs) context['project'] = self.get_project() return context @@ -87,7 +88,7 @@ def get_form(self, data=None, files=None, **kwargs): return self.form_class(data, files, **kwargs) -class ProjectSpamMixin: +class ProjectSpamMixin(object): """Protects POST views from spammers.""" @@ -99,7 +100,7 @@ def post(self, request, *args, **kwargs): ) return HttpResponseRedirect(self.get_failure_url()) try: - return super().post(request, *args, **kwargs) + return super(ProjectSpamMixin, self).post(request, *args, **kwargs) except ProjectSpamError: date_maturity = timezone.now() - timedelta(days=USER_MATURITY_DAYS) if request.user.date_joined > date_maturity: diff --git a/readthedocs/projects/views/mixins.py b/readthedocs/projects/views/mixins.py index 670caa21f83..50e03beb475 100644 --- a/readthedocs/projects/views/mixins.py +++ b/readthedocs/projects/views/mixins.py @@ -1,13 +1,13 @@ -# -*- coding: utf-8 -*- - """Mixin classes for project views.""" +from __future__ import absolute_import +from builtins import object from django.shortcuts import get_object_or_404 from readthedocs.projects.models import Project -class ProjectRelationMixin: +class ProjectRelationMixin(object): """ Mixin class for constructing model views for project dashboard. @@ -32,7 +32,7 @@ def get_project(self): return None return get_object_or_404( self.get_project_queryset(), - slug=self.kwargs[self.project_lookup_url_kwarg], + slug=self.kwargs[self.project_lookup_url_kwarg] ) def get_queryset(self): @@ -41,6 +41,6 @@ def get_queryset(self): ) def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(ProjectRelationMixin, self).get_context_data(**kwargs) context[self.project_context_object_name] = self.get_project() return context diff --git a/readthedocs/projects/views/private.py b/readthedocs/projects/views/private.py index 02256089c81..330c22b5b7d 100644 --- a/readthedocs/projects/views/private.py +++ b/readthedocs/projects/views/private.py @@ -2,6 +2,13 @@ """Project views for authenticated users.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging from allauth.socialaccount.models import SocialAccount @@ -36,7 +43,6 @@ from readthedocs.projects.forms import ( DomainForm, EmailHookForm, - EnvironmentVariableForm, IntegrationForm, ProjectAdvancedForm, ProjectAdvertisingForm, @@ -53,7 +59,6 @@ from readthedocs.projects.models import ( Domain, EmailHook, - EnvironmentVariable, Project, ProjectRelationship, WebHook, @@ -61,9 +66,6 @@ from readthedocs.projects.signals import project_import from readthedocs.projects.views.base import ProjectAdminMixin, ProjectSpamMixin -from ..tasks import retry_domain_verification - - log = logging.getLogger(__name__) @@ -82,7 +84,7 @@ def get_queryset(self): return Project.objects.dashboard(self.request.user) def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(ProjectDashboard, self).get_context_data(**kwargs) return context @@ -190,7 +192,7 @@ def project_version_detail(request, project_slug, version_slug): log.info('Removing files for version %s', version.slug) broadcast( type='app', - task=tasks.remove_dirs, + task=tasks.clear_artifacts, args=[version.get_artifact_paths()], ) version.built = False @@ -219,11 +221,7 @@ def project_delete(request, project_slug): ) if request.method == 'POST': - broadcast( - type='app', - task=tasks.remove_dirs, - args=[(project.doc_path,)], - ) + broadcast(type='app', task=tasks.remove_dir, args=[project.doc_path]) project.delete() messages.success(request, _('Project deleted')) project_dashboard = reverse('projects_dashboard') @@ -252,7 +250,7 @@ def get_form_kwargs(self, step=None): def get_template_names(self): """Return template names based on step name.""" - return 'projects/import_{}.html'.format(self.steps.current) + return 'projects/import_{0}.html'.format(self.steps.current) def done(self, form_list, **kwargs): """ @@ -353,7 +351,7 @@ def get(self, request, *args, **kwargs): def get_form_data(self): """Get form data to post to import form.""" return { - 'name': '{}-demo'.format(self.request.user.username), + 'name': '{0}-demo'.format(self.request.user.username), 'repo_type': 'git', 'repo': 'https://github.com/readthedocs/template.git', } @@ -407,7 +405,7 @@ def get(self, request, *args, **kwargs): ) )), # yapf: disable ) - return super().get(request, *args, **kwargs) + return super(ImportView, self).get(request, *args, **kwargs) def post(self, request, *args, **kwargs): initial_data = {} @@ -421,7 +419,7 @@ def post(self, request, *args, **kwargs): return self.wizard_class.as_view(initial_dict=initial_data)(request) def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(ImportView, self).get_context_data(**kwargs) context['view_csrf_token'] = get_token(self.request) context['has_connected_accounts'] = SocialAccount.objects.filter( user=self.request.user, @@ -442,7 +440,10 @@ def get_queryset(self): def get_form(self, data=None, files=None, **kwargs): kwargs['user'] = self.request.user - return super().get_form(data, files, **kwargs) + return super( + ProjectRelationshipMixin, + self, + ).get_form(data, files, **kwargs) def form_valid(self, form): broadcast( @@ -450,7 +451,7 @@ def form_valid(self, form): task=tasks.symlink_subproject, args=[self.get_project().pk], ) - return super().form_valid(form) + return super(ProjectRelationshipMixin, self).form_valid(form) def get_success_url(self): return reverse('projects_subprojects', args=[self.get_project().slug]) @@ -459,7 +460,7 @@ def get_success_url(self): class ProjectRelationshipList(ProjectRelationshipMixin, ListView): def get_context_data(self, **kwargs): - ctx = super().get_context_data(**kwargs) + ctx = super(ProjectRelationshipList, self).get_context_data(**kwargs) ctx['superproject'] = self.project.superprojects.first() return ctx @@ -703,7 +704,7 @@ def project_version_delete_html(request, project_slug, version_slug): version.save() broadcast( type='app', - task=tasks.remove_dirs, + task=tasks.clear_artifacts, args=[version.get_artifact_paths()], ) else: @@ -725,15 +726,7 @@ def get_success_url(self): class DomainList(DomainMixin, ListViewWithForm): - - def get_context_data(self, **kwargs): - ctx = super().get_context_data(**kwargs) - - # Retry validation on all domains if applicable - for domain in ctx['domain_list']: - retry_domain_verification.delay(domain_pk=domain.pk) - - return ctx + pass class DomainCreate(DomainMixin, CreateView): @@ -782,7 +775,7 @@ def get_success_url(self): def get_template_names(self): if self.template_name: return self.template_name - return 'projects/integration{}.html'.format(self.template_name_suffix) + return 'projects/integration{0}.html'.format(self.template_name_suffix) class IntegrationList(IntegrationMixin, ListView): @@ -817,7 +810,7 @@ def get_template_names(self): integration_type = self.get_integration().integration_type suffix = self.SUFFIX_MAP.get(integration_type, integration_type) return ( - 'projects/integration_{}{}.html' + 'projects/integration_{0}{1}.html' .format(suffix, self.template_name_suffix) ) @@ -882,37 +875,3 @@ def get_queryset(self): def get_success_url(self): return reverse('projects_advertising', args=[self.object.slug]) - - -class EnvironmentVariableMixin(ProjectAdminMixin, PrivateViewMixin): - - """Environment Variables to be added when building the Project.""" - - model = EnvironmentVariable - form_class = EnvironmentVariableForm - lookup_url_kwarg = 'environmentvariable_pk' - - def get_success_url(self): - return reverse( - 'projects_environmentvariables', - args=[self.get_project().slug], - ) - - -class EnvironmentVariableList(EnvironmentVariableMixin, ListView): - pass - - -class EnvironmentVariableCreate(EnvironmentVariableMixin, CreateView): - pass - - -class EnvironmentVariableDetail(EnvironmentVariableMixin, DetailView): - pass - - -class EnvironmentVariableDelete(EnvironmentVariableMixin, DeleteView): - - # This removes the delete confirmation - def get(self, request, *args, **kwargs): - return self.http_method_not_allowed(request, *args, **kwargs) diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index e90bbeb3e08..90f5ef978b1 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- - """Public project views.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import json import logging import mimetypes @@ -14,9 +16,9 @@ from django.contrib import messages from django.contrib.auth.models import User from django.core.cache import cache -from django.http import HttpResponse, HttpResponseRedirect -from django.shortcuts import get_object_or_404, render from django.urls import reverse +from django.http import Http404, HttpResponse, HttpResponseRedirect +from django.shortcuts import get_object_or_404, render from django.views.decorators.cache import never_cache from django.views.generic import DetailView, ListView from taggit.models import Tag @@ -24,13 +26,12 @@ from readthedocs.builds.constants import LATEST from readthedocs.builds.models import Version from readthedocs.builds.views import BuildTriggerMixin -from readthedocs.projects.models import Project +from readthedocs.projects.models import ImportedFile, Project from readthedocs.search.indexes import PageIndex from readthedocs.search.views import LOG_TEMPLATE from .base import ProjectOnboardMixin - log = logging.getLogger(__name__) search_log = logging.getLogger(__name__ + '.search') mimetypes.add_type('application/epub+zip', '.epub') @@ -53,9 +54,7 @@ def get_queryset(self): if self.kwargs.get('username'): self.user = get_object_or_404( - User, - username=self.kwargs.get('username'), - ) + User, username=self.kwargs.get('username')) queryset = queryset.filter(user=self.user) else: self.user = None @@ -63,7 +62,7 @@ def get_queryset(self): return queryset def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(ProjectIndex, self).get_context_data(**kwargs) context['person'] = self.user context['tag'] = self.tag return context @@ -83,13 +82,11 @@ def get_queryset(self): return Project.objects.protected(self.request.user) def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) + context = super(ProjectDetailView, self).get_context_data(**kwargs) project = self.get_object() context['versions'] = Version.objects.public( - user=self.request.user, - project=project, - ) + user=self.request.user, project=project) protocol = 'http' if self.request.is_secure(): @@ -97,7 +94,7 @@ def get_context_data(self, **kwargs): version_slug = project.get_default_version() - context['badge_url'] = '{}://{}{}?version={}'.format( + context['badge_url'] = '%s://%s%s?version=%s' % ( protocol, settings.PRODUCTION_DOMAIN, reverse('project_badge', args=[project.slug]), @@ -115,14 +112,8 @@ def get_context_data(self, **kwargs): def project_badge(request, project_slug): """Return a sweet badge for the project.""" style = request.GET.get('style', 'flat') - if style not in ( - 'flat', - 'plastic', - 'flat-square', - 'for-the-badge', - 'social', - ): - style = 'flat' + if style not in ("flat", "plastic", "flat-square", "for-the-badge", "social"): + style = "flat" # Get the local path to the badge files badge_path = os.path.join( @@ -138,15 +129,10 @@ def project_badge(request, project_slug): file_path = badge_path % 'unknown' version = Version.objects.public(request.user).filter( - project__slug=project_slug, - slug=version_slug, - ).first() + project__slug=project_slug, slug=version_slug).first() if version: - last_build = version.builds.filter( - type='html', - state='finished', - ).order_by('-date').first() + last_build = version.builds.filter(type='html', state='finished').order_by('-date').first() if last_build: if last_build.success: file_path = badge_path % 'passing' @@ -160,18 +146,14 @@ def project_badge(request, project_slug): content_type='image/svg+xml', ) except (IOError, OSError): - log.exception( - 'Failed to read local filesystem while serving a docs badge', - ) + log.exception('Failed to read local filesystem while serving a docs badge') return HttpResponse(status=503) def project_downloads(request, project_slug): """A detail view for a project with various dataz.""" project = get_object_or_404( - Project.objects.protected(request.user), - slug=project_slug, - ) + Project.objects.protected(request.user), slug=project_slug) versions = Version.objects.public(user=request.user, project=project) version_data = OrderedDict() for version in versions: @@ -209,21 +191,15 @@ def project_download_media(request, project_slug, type_, version_slug): privacy_level = getattr(settings, 'DEFAULT_PRIVACY_LEVEL', 'public') if privacy_level == 'public' or settings.DEBUG: path = os.path.join( - settings.MEDIA_URL, - type_, - project_slug, - version_slug, - '{}.{}'.format(project_slug, type_.replace('htmlzip', 'zip')), - ) + settings.MEDIA_URL, type_, project_slug, version_slug, + '%s.%s' % (project_slug, type_.replace('htmlzip', 'zip'))) return HttpResponseRedirect(path) # Get relative media path path = ( version.project.get_production_media_path( - type_=type_, - version_slug=version_slug, - ).replace(settings.PRODUCTION_ROOT, '/prod_artifacts') - ) + type_=type_, version_slug=version_slug) + .replace(settings.PRODUCTION_ROOT, '/prod_artifacts')) content_type, encoding = mimetypes.guess_type(path) content_type = content_type or 'application/octet-stream' response = HttpResponse(content_type=content_type) @@ -231,11 +207,8 @@ def project_download_media(request, project_slug, type_, version_slug): response['Content-Encoding'] = encoding response['X-Accel-Redirect'] = path # Include version in filename; this fixes a long-standing bug - filename = '{}-{}.{}'.format( - project_slug, - version_slug, - path.split('.')[-1], - ) + filename = '%s-%s.%s' % ( + project_slug, version_slug, path.split('.')[-1]) response['Content-Disposition'] = 'filename=%s' % filename return response @@ -258,8 +231,7 @@ def elastic_project_search(request, project_slug): version=version_slug or '', language='', msg=query or '', - ), - ) + )) if query: @@ -271,22 +243,22 @@ def elastic_project_search(request, project_slug): {'match': {'title': {'query': query, 'boost': 10}}}, {'match': {'headers': {'query': query, 'boost': 5}}}, {'match': {'content': {'query': query}}}, - ], - }, + ] + } }, 'highlight': { 'fields': { 'title': {}, 'headers': {}, 'content': {}, - }, + } }, 'fields': ['title', 'project', 'version', 'path'], 'filter': { 'and': [ {'term': {'project': project_slug}}, {'term': {'version': version_slug}}, - ], + ] }, 'size': 50, # TODO: Support pagination. } @@ -323,15 +295,10 @@ def project_versions(request, project_slug): Shows the available versions and lets the user choose which ones to build. """ project = get_object_or_404( - Project.objects.protected(request.user), - slug=project_slug, - ) + Project.objects.protected(request.user), slug=project_slug) versions = Version.objects.public( - user=request.user, - project=project, - only_active=False, - ) + user=request.user, project=project, only_active=False) active_versions = versions.filter(active=True) inactive_versions = versions.filter(active=False) @@ -357,9 +324,7 @@ def project_versions(request, project_slug): def project_analytics(request, project_slug): """Have a analytics API placeholder.""" project = get_object_or_404( - Project.objects.protected(request.user), - slug=project_slug, - ) + Project.objects.protected(request.user), slug=project_slug) analytics_cache = cache.get('analytics:%s' % project_slug) if analytics_cache: analytics = json.loads(analytics_cache) @@ -367,10 +332,8 @@ def project_analytics(request, project_slug): try: resp = requests.get( '{host}/api/v1/index/1/heatmap/'.format( - host=settings.GROK_API_HOST, - ), - params={'project': project.slug, 'days': 7, 'compare': True}, - ) + host=settings.GROK_API_HOST), + params={'project': project.slug, 'days': 7, 'compare': True}) analytics = resp.json() cache.set('analytics:%s' % project_slug, resp.content, 1800) except requests.exceptions.RequestException: @@ -381,18 +344,12 @@ def project_analytics(request, project_slug): reversed( sorted( list(analytics['page'].items()), - key=operator.itemgetter(1), - ), - ), - ) + key=operator.itemgetter(1)))) version_list = list( reversed( sorted( list(analytics['version'].items()), - key=operator.itemgetter(1), - ), - ), - ) + key=operator.itemgetter(1)))) else: page_list = [] version_list = [] @@ -418,13 +375,9 @@ def project_analytics(request, project_slug): def project_embed(request, project_slug): """Have a content API placeholder.""" project = get_object_or_404( - Project.objects.protected(request.user), - slug=project_slug, - ) + Project.objects.protected(request.user), slug=project_slug) version = project.versions.get(slug=LATEST) - files = version.imported_files.filter( - name__endswith='.html', - ).order_by('path') + files = version.imported_files.filter(name__endswith='.html').order_by('path') return render( request, diff --git a/readthedocs/redirects/admin.py b/readthedocs/redirects/admin.py index 4ce0239d48f..6bd2d73470d 100644 --- a/readthedocs/redirects/admin.py +++ b/readthedocs/redirects/admin.py @@ -1,9 +1,8 @@ -# -*- coding: utf-8 -*- - """Django admin configuration for the redirects app.""" -from django.contrib import admin +from __future__ import absolute_import +from django.contrib import admin from .models import Redirect diff --git a/readthedocs/redirects/managers.py b/readthedocs/redirects/managers.py index 37e10890cc9..9c0f1bf47fa 100644 --- a/readthedocs/redirects/managers.py +++ b/readthedocs/redirects/managers.py @@ -1,20 +1,15 @@ -# -*- coding: utf-8 -*- - """Manager and queryset for the redirects app.""" +from __future__ import absolute_import from django.db.models import Manager from django.db.models.query import QuerySet class RedirectQuerySet(QuerySet): - def get_redirect_path(self, path, language=None, version_slug=None): for redirect in self.select_related('project'): new_path = redirect.get_redirect_path( - path=path, - language=language, - version_slug=version_slug, - ) + path=path, language=language, version_slug=version_slug) if new_path: return new_path diff --git a/readthedocs/redirects/migrations/0001_initial.py b/readthedocs/redirects/migrations/0001_initial.py index 0bb2fb946eb..010f36342b3 100644 --- a/readthedocs/redirects/migrations/0001_initial.py +++ b/readthedocs/redirects/migrations/0001_initial.py @@ -1,5 +1,8 @@ # -*- coding: utf-8 -*- -from django.db import migrations, models +from __future__ import unicode_literals + +from __future__ import absolute_import +from django.db import models, migrations class Migration(migrations.Migration): diff --git a/readthedocs/redirects/migrations/0002_add_missing_model_change_migrations.py b/readthedocs/redirects/migrations/0002_add_missing_model_change_migrations.py index e1d83010c0a..a837e6fb146 100644 --- a/readthedocs/redirects/migrations/0002_add_missing_model_change_migrations.py +++ b/readthedocs/redirects/migrations/0002_add_missing_model_change_migrations.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- # Generated by Django 1.11.16 on 2018-10-31 11:25 +from __future__ import unicode_literals + from django.db import migrations, models diff --git a/readthedocs/redirects/models.py b/readthedocs/redirects/models.py index 7945ecb5a82..cbd080ca28c 100644 --- a/readthedocs/redirects/models.py +++ b/readthedocs/redirects/models.py @@ -1,23 +1,22 @@ -# -*- coding: utf-8 -*- - """Django models for the redirects app.""" -import logging -import re - +from __future__ import absolute_import +from builtins import object from django.db import models from django.utils.encoding import python_2_unicode_compatible from django.utils.translation import ugettext from django.utils.translation import ugettext_lazy as _ +import logging +import re from readthedocs.core.resolver import resolve_path from readthedocs.projects.models import Project - from .managers import RedirectManager log = logging.getLogger(__name__) + HTTP_STATUS_CHOICES = ( (301, _('301 - Permanent Redirect')), (302, _('302 - Temporary Redirect')), @@ -41,14 +40,12 @@ # make sense for "Prefix Redirects" since the from URL is considered after the # ``/$lang/$version/`` part. Also, there is a feature for the "Exact # Redirects" that should be mentioned here: the usage of ``$rest`` -from_url_helptext = _( - 'Absolute path, excluding the domain. ' - 'Example: /docs/ or /install.html', -) -to_url_helptext = _( - 'Absolute or relative URL. Example: ' - '/tutorial/install.html', -) +from_url_helptext = _('Absolute path, excluding the domain. ' + 'Example: /docs/ or /install.html' + ) +to_url_helptext = _('Absolute or relative URL. Example: ' + '/tutorial/install.html' + ) redirect_type_helptext = _('The type of redirect you wish to use.') @@ -57,40 +54,21 @@ class Redirect(models.Model): """A HTTP redirect associated with a Project.""" - project = models.ForeignKey( - Project, - verbose_name=_('Project'), - related_name='redirects', - ) - - redirect_type = models.CharField( - _('Redirect Type'), - max_length=255, - choices=TYPE_CHOICES, - help_text=redirect_type_helptext, - ) - - from_url = models.CharField( - _('From URL'), - max_length=255, - db_index=True, - help_text=from_url_helptext, - blank=True, - ) - - to_url = models.CharField( - _('To URL'), - max_length=255, - db_index=True, - help_text=to_url_helptext, - blank=True, - ) - - http_status = models.SmallIntegerField( - _('HTTP Status'), - choices=HTTP_STATUS_CHOICES, - default=301, - ) + project = models.ForeignKey(Project, verbose_name=_('Project'), + related_name='redirects') + + redirect_type = models.CharField(_('Redirect Type'), max_length=255, choices=TYPE_CHOICES, + help_text=redirect_type_helptext) + + from_url = models.CharField(_('From URL'), max_length=255, + db_index=True, help_text=from_url_helptext, blank=True) + + to_url = models.CharField(_('To URL'), max_length=255, + db_index=True, help_text=to_url_helptext, blank=True) + + http_status = models.SmallIntegerField(_('HTTP Status'), + choices=HTTP_STATUS_CHOICES, + default=301) status = models.BooleanField(choices=STATUS_CHOICES, default=True) create_dt = models.DateTimeField(auto_now_add=True) @@ -98,7 +76,7 @@ class Redirect(models.Model): objects = RedirectManager() - class Meta: + class Meta(object): verbose_name = _('redirect') verbose_name_plural = _('redirects') ordering = ('-update_dt',) @@ -108,12 +86,10 @@ def __str__(self): if self.redirect_type in ['prefix', 'page', 'exact']: return redirect_text.format( type=self.get_redirect_type_display(), - from_to_url=self.get_from_to_url_display(), + from_to_url=self.get_from_to_url_display() ) - return ugettext( - 'Redirect: {}'.format( - self.get_redirect_type_display(), - ), + return ugettext('Redirect: {}'.format( + self.get_redirect_type_display()) ) def get_from_to_url_display(self): @@ -123,11 +99,11 @@ def get_from_to_url_display(self): if self.redirect_type == 'prefix': to_url = '/{lang}/{version}/'.format( lang=self.project.language, - version=self.project.default_version, + version=self.project.default_version ) return '{from_url} -> {to_url}'.format( from_url=from_url, - to_url=to_url, + to_url=to_url ) return '' @@ -143,19 +119,13 @@ def get_full_path(self, filename, language=None, version_slug=None): return filename return resolve_path( - project=self.project, - language=language, - version_slug=version_slug, - filename=filename, + project=self.project, language=language, + version_slug=version_slug, filename=filename ) def get_redirect_path(self, path, language=None, version_slug=None): - method = getattr( - self, - 'redirect_{type}'.format( - type=self.redirect_type, - ), - ) + method = getattr(self, 'redirect_{type}'.format( + type=self.redirect_type)) return method(path, language=language, version_slug=version_slug) def redirect_prefix(self, path, language=None, version_slug=None): @@ -165,8 +135,7 @@ def redirect_prefix(self, path, language=None, version_slug=None): to = self.get_full_path( filename=cut_path, language=language, - version_slug=version_slug, - ) + version_slug=version_slug) return to def redirect_page(self, path, language=None, version_slug=None): @@ -175,8 +144,7 @@ def redirect_page(self, path, language=None, version_slug=None): to = self.get_full_path( filename=self.to_url.lstrip('/'), language=language, - version_slug=version_slug, - ) + version_slug=version_slug) return to def redirect_exact(self, path, language=None, version_slug=None): @@ -203,8 +171,7 @@ def redirect_sphinx_html(self, path, language=None, version_slug=None): return self.get_full_path( filename=to, language=language, - version_slug=version_slug, - ) + version_slug=version_slug) def redirect_sphinx_htmldir(self, path, language=None, version_slug=None): if path.endswith('.html'): @@ -214,5 +181,4 @@ def redirect_sphinx_htmldir(self, path, language=None, version_slug=None): return self.get_full_path( filename=to, language=language, - version_slug=version_slug, - ) + version_slug=version_slug) diff --git a/readthedocs/redirects/utils.py b/readthedocs/redirects/utils.py index ce1d7514083..1edc628626a 100644 --- a/readthedocs/redirects/utils.py +++ b/readthedocs/redirects/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Redirection view support. @@ -9,11 +7,11 @@ These are not used directly as views; they are instead included into 404 handlers, so that redirects only take effect if no other view matches. """ +from __future__ import absolute_import +from django.http import HttpResponseRedirect import logging import re -from django.http import HttpResponseRedirect - from readthedocs.constants import LANGUAGES_REGEX from readthedocs.projects.models import Project @@ -39,8 +37,7 @@ def project_and_path_from_request(request, path): # docs prefix. match = re.match( r'^/docs/(?P[^/]+)(?P/.*)$', - path, - ) + path) if match: project_slug = match.groupdict()['project_slug'] path = match.groupdict()['path'] @@ -59,8 +56,7 @@ def project_and_path_from_request(request, path): def language_and_version_from_path(path): match = re.match( r'^/(?P%s)/(?P[^/]+)(?P/.*)$' % LANGUAGES_REGEX, - path, - ) + path) if match: language = match.groupdict()['language'] version_slug = match.groupdict()['version_slug'] @@ -80,10 +76,7 @@ def get_redirect_response(request, path): language, version_slug, path = language_and_version_from_path(path) new_path = project.redirects.get_redirect_path( - path=path, - language=language, - version_slug=version_slug, - ) + path=path, language=language, version_slug=version_slug) if new_path is None: return None diff --git a/readthedocs/restapi/client.py b/readthedocs/restapi/client.py index 53428b707fd..83f5b861d83 100644 --- a/readthedocs/restapi/client.py +++ b/readthedocs/restapi/client.py @@ -2,6 +2,13 @@ """Simple client to access our API with Slumber credentials.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import requests @@ -11,7 +18,6 @@ from rest_framework.renderers import JSONRenderer from slumber import API, serialize - log = logging.getLogger(__name__) PRODUCTION_DOMAIN = getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org') @@ -22,7 +28,7 @@ class DrfJsonSerializer(serialize.JsonSerializer): - """Additional serialization help from the DRF renderer.""" + """Additional serialization help from the DRF renderer""" key = 'json-drf' diff --git a/readthedocs/restapi/permissions.py b/readthedocs/restapi/permissions.py index 93d4695a7cc..615872d307e 100644 --- a/readthedocs/restapi/permissions.py +++ b/readthedocs/restapi/permissions.py @@ -1,7 +1,7 @@ -# -*- coding: utf-8 -*- - """Defines access permissions for the API.""" +from __future__ import absolute_import + from rest_framework import permissions from readthedocs.core.permissions import AdminPermission @@ -51,15 +51,12 @@ class APIPermission(permissions.IsAuthenticatedOrReadOnly): """ def has_permission(self, request, view): - has_perm = super().has_permission(request, view) + has_perm = super(APIPermission, self).has_permission(request, view) return has_perm or (request.user and request.user.is_staff) def has_object_permission(self, request, view, obj): - has_perm = super().has_object_permission( - request, - view, - obj, - ) + has_perm = super(APIPermission, self).has_object_permission( + request, view, obj) return has_perm or (request.user and request.user.is_staff) diff --git a/readthedocs/restapi/serializers.py b/readthedocs/restapi/serializers.py index 380f2209368..1d473a9eada 100644 --- a/readthedocs/restapi/serializers.py +++ b/readthedocs/restapi/serializers.py @@ -1,31 +1,27 @@ -# -*- coding: utf-8 -*- - """Defines serializers for each of our models.""" +from __future__ import absolute_import + +from builtins import object + from allauth.socialaccount.models import SocialAccount from rest_framework import serializers from readthedocs.builds.models import Build, BuildCommandResult, Version from readthedocs.oauth.models import RemoteOrganization, RemoteRepository -from readthedocs.projects.models import Domain, Project +from readthedocs.projects.models import Project, Domain class ProjectSerializer(serializers.ModelSerializer): canonical_url = serializers.ReadOnlyField(source='get_docs_url') - class Meta: + class Meta(object): model = Project fields = ( 'id', - 'name', - 'slug', - 'description', - 'language', - 'programming_language', - 'repo', - 'repo_type', - 'default_version', - 'default_branch', + 'name', 'slug', 'description', 'language', + 'programming_language', 'repo', 'repo_type', + 'default_version', 'default_branch', 'documentation_type', 'users', 'canonical_url', @@ -47,12 +43,6 @@ class ProjectAdminSerializer(ProjectSerializer): slug_field='feature_id', ) - def get_environment_variables(self, obj): - return { - variable.name: variable.value - for variable in obj.environmentvariable_set.all() - } - class Meta(ProjectSerializer.Meta): fields = ProjectSerializer.Meta.fields + ( 'enable_epub_build', @@ -80,16 +70,13 @@ class VersionSerializer(serializers.ModelSerializer): project = ProjectSerializer() downloads = serializers.DictField(source='get_downloads', read_only=True) - class Meta: + class Meta(object): model = Version fields = ( 'id', - 'project', - 'slug', - 'identifier', - 'verbose_name', - 'active', - 'built', + 'project', 'slug', + 'identifier', 'verbose_name', + 'active', 'built', 'downloads', 'type', ) @@ -106,7 +93,7 @@ class BuildCommandSerializer(serializers.ModelSerializer): run_time = serializers.ReadOnlyField() - class Meta: + class Meta(object): model = BuildCommandResult exclude = ('') @@ -124,7 +111,7 @@ class BuildSerializer(serializers.ModelSerializer): # https://github.com/dmkoch/django-jsonfield/issues/188#issuecomment-300439829 config = serializers.JSONField(required=False) - class Meta: + class Meta(object): model = Build # `_config` should be excluded to avoid conflicts with `config` exclude = ('builder', '_config') @@ -149,7 +136,7 @@ class SearchIndexSerializer(serializers.Serializer): class DomainSerializer(serializers.ModelSerializer): project = ProjectSerializer() - class Meta: + class Meta(object): model = Domain fields = ( 'id', @@ -163,7 +150,7 @@ class Meta: class RemoteOrganizationSerializer(serializers.ModelSerializer): - class Meta: + class Meta(object): model = RemoteOrganization exclude = ('json', 'email', 'users') @@ -175,7 +162,7 @@ class RemoteRepositorySerializer(serializers.ModelSerializer): organization = RemoteOrganizationSerializer() matches = serializers.SerializerMethodField() - class Meta: + class Meta(object): model = RemoteRepository exclude = ('json', 'users') @@ -197,12 +184,13 @@ class SocialAccountSerializer(serializers.ModelSerializer): avatar_url = serializers.URLField(source='get_avatar_url') provider = ProviderSerializer(source='get_provider') - class Meta: + class Meta(object): model = SocialAccount exclude = ('extra_data',) def get_username(self, obj): return ( - obj.extra_data.get('username') or obj.extra_data.get('login') + obj.extra_data.get('username') or + obj.extra_data.get('login') # FIXME: which one is GitLab? ) diff --git a/readthedocs/restapi/signals.py b/readthedocs/restapi/signals.py index 65509fc551d..6b6d0b3955f 100644 --- a/readthedocs/restapi/signals.py +++ b/readthedocs/restapi/signals.py @@ -1,10 +1,9 @@ -# -*- coding: utf-8 -*- - """We define custom Django signals to trigger when a footer is rendered.""" -import django.dispatch +from __future__ import absolute_import +import django.dispatch footer_response = django.dispatch.Signal( - providing_args=['request', 'context', 'response_data'], + providing_args=["request", "context", "response_data"] ) diff --git a/readthedocs/restapi/urls.py b/readthedocs/restapi/urls.py index 7a0d5c54e2d..c8cdf6cd21e 100644 --- a/readthedocs/restapi/urls.py +++ b/readthedocs/restapi/urls.py @@ -2,6 +2,13 @@ """Define routes between URL paths and views/endpoints.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from django.conf import settings from django.conf.urls import include, url from rest_framework import routers @@ -28,7 +35,6 @@ VersionViewSet, ) - router = routers.DefaultRouter() router.register(r'build', BuildViewSet, basename='build') router.register(r'command', BuildCommandViewSet, basename='buildcommandresult') @@ -95,30 +101,26 @@ integration_urls = [ url( - r'webhook/github/(?P{project_slug})/$'.format( - **pattern_opts - ), + r'webhook/github/(?P{project_slug})/$' + .format(**pattern_opts), integrations.GitHubWebhookView.as_view(), name='api_webhook_github', ), url( - r'webhook/gitlab/(?P{project_slug})/$'.format( - **pattern_opts - ), + r'webhook/gitlab/(?P{project_slug})/$' + .format(**pattern_opts), integrations.GitLabWebhookView.as_view(), name='api_webhook_gitlab', ), url( - r'webhook/bitbucket/(?P{project_slug})/$'.format( - **pattern_opts - ), + r'webhook/bitbucket/(?P{project_slug})/$' + .format(**pattern_opts), integrations.BitbucketWebhookView.as_view(), name='api_webhook_bitbucket', ), url( - r'webhook/generic/(?P{project_slug})/$'.format( - **pattern_opts - ), + r'webhook/generic/(?P{project_slug})/$' + .format(**pattern_opts), integrations.APIWebhookView.as_view(), name='api_webhook_generic', ), diff --git a/readthedocs/restapi/utils.py b/readthedocs/restapi/utils.py index 69c5dcfbc7c..8637cd1779b 100644 --- a/readthedocs/restapi/utils.py +++ b/readthedocs/restapi/utils.py @@ -1,7 +1,13 @@ # -*- coding: utf-8 -*- - """Utility functions that are used by both views and celery tasks.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import hashlib import logging @@ -19,15 +25,13 @@ from readthedocs.builds.models import Version from readthedocs.search.indexes import PageIndex, ProjectIndex, SectionIndex - log = logging.getLogger(__name__) def sync_versions(project, versions, type): # pylint: disable=redefined-builtin """Update the database with the current versions from the repository.""" old_version_values = project.versions.filter(type=type).values_list( - 'verbose_name', - 'identifier', + 'verbose_name', 'identifier' ) old_versions = dict(old_version_values) @@ -45,7 +49,7 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin slug=STABLE, version_id=version_id, verbose_name=version_name, - type_=type, + type_=type ) if created: added.add(created_version.slug) @@ -56,7 +60,7 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin slug=LATEST, version_id=version_id, verbose_name=version_name, - type_=type, + type_=type ) if created: added.add(created_version.slug) @@ -67,13 +71,11 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin else: # Update slug with new identifier Version.objects.filter( - project=project, - verbose_name=version_name, - ).update( - identifier=version_id, - type=type, - machine=False, - ) # noqa + project=project, verbose_name=version_name).update( + identifier=version_id, + type=type, + machine=False, + ) # noqa log.info( '(Sync Versions) Updated Version: [%s=%s] ', @@ -91,7 +93,9 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin added.add(created_version.slug) if not has_user_stable: stable_version = ( - project.versions.filter(slug=STABLE, type=type).first() + project.versions + .filter(slug=STABLE, type=type) + .first() ) if stable_version: # Put back the RTD's stable version @@ -99,7 +103,9 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin stable_version.save() if not has_user_latest: latest_version = ( - project.versions.filter(slug=LATEST, type=type).first() + project.versions + .filter(slug=LATEST, type=type) + .first() ) if latest_version: # Put back the RTD's latest version @@ -114,7 +120,11 @@ def sync_versions(project, versions, type): # pylint: disable=redefined-builtin def set_or_create_version(project, slug, version_id, verbose_name, type_): """Search or create a version and set its machine attribute to false.""" - version = (project.versions.filter(slug=slug).first()) + version = ( + project.versions + .filter(slug=slug) + .first() + ) if version: version.identifier = version_id version.machine = False @@ -136,10 +146,12 @@ def delete_versions(project, version_data): # We use verbose_name for tags # because several tags can point to the same identifier. versions_tags = [ - version['verbose_name'] for version in version_data.get('tags', []) + version['verbose_name'] + for version in version_data.get('tags', []) ] versions_branches = [ - version['identifier'] for version in version_data.get('branches', []) + version['identifier'] + for version in version_data.get('branches', []) ] to_delete_qs = project.versions.all() to_delete_qs = to_delete_qs.exclude( @@ -163,14 +175,8 @@ def delete_versions(project, version_data): def index_search_request( - version, - page_list, - commit, - project_scale, - page_scale, - section=True, - delete=True, -): + version, page_list, commit, project_scale, page_scale, section=True, + delete=True): """ Update search indexes with build output JSON. @@ -200,8 +206,7 @@ def index_search_request( 'url': project.get_absolute_url(), 'tags': None, 'weight': project_scale, - }, - ) + }) page_obj = PageIndex() section_obj = SectionIndex() diff --git a/readthedocs/restapi/views/core_views.py b/readthedocs/restapi/views/core_views.py index 1e4afdd8a7b..08fc9e7d764 100644 --- a/readthedocs/restapi/views/core_views.py +++ b/readthedocs/restapi/views/core_views.py @@ -1,16 +1,17 @@ -# -*- coding: utf-8 -*- - """Utility endpoints relating to canonical urls, embedded content, etc.""" -from django.shortcuts import get_object_or_404 +from __future__ import absolute_import + from rest_framework import decorators, permissions, status from rest_framework.renderers import JSONRenderer from rest_framework.response import Response +from django.shortcuts import get_object_or_404 + from readthedocs.builds.constants import LATEST from readthedocs.builds.models import Version -from readthedocs.core.templatetags.core_tags import make_document_url from readthedocs.projects.models import Project +from readthedocs.core.templatetags.core_tags import make_document_url @decorators.api_view(['GET']) @@ -23,26 +24,18 @@ def docurl(request): Example:: GET https://readthedocs.org/api/v2/docurl/?project=requests&version=latest&doc=index + """ project = request.GET.get('project') version = request.GET.get('version', LATEST) doc = request.GET.get('doc', 'index') if project is None: - return Response( - {'error': 'Need project and doc'}, - status=status.HTTP_400_BAD_REQUEST, - ) + return Response({'error': 'Need project and doc'}, status=status.HTTP_400_BAD_REQUEST) project = get_object_or_404(Project, slug=project) version = get_object_or_404( - Version.objects - .public(request.user, project=project, only_active=False), - slug=version, - ) + Version.objects.public(request.user, project=project, only_active=False), + slug=version) return Response({ - 'url': make_document_url( - project=project, - version=version.slug, - page=doc, - ), + 'url': make_document_url(project=project, version=version.slug, page=doc) }) diff --git a/readthedocs/restapi/views/footer_views.py b/readthedocs/restapi/views/footer_views.py index bf66f5f80b2..f09ceaff527 100644 --- a/readthedocs/restapi/views/footer_views.py +++ b/readthedocs/restapi/views/footer_views.py @@ -1,7 +1,10 @@ # -*- coding: utf-8 -*- - """Endpoint to generate footer HTML.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + +import six from django.conf import settings from django.shortcuts import get_object_or_404 from django.template import loader as template_loader @@ -14,9 +17,7 @@ from readthedocs.builds.models import Version from readthedocs.projects.models import Project from readthedocs.projects.version_handling import ( - highest_version, - parse_version_failsafe, -) + highest_version, parse_version_failsafe) from readthedocs.restapi.signals import footer_response @@ -34,11 +35,10 @@ def get_version_compare_data(project, base_version=None): versions_qs = versions_qs.filter(type=TAG) highest_version_obj, highest_version_comparable = highest_version( - versions_qs, - ) + versions_qs) ret_val = { - 'project': str(highest_version_obj), - 'version': str(highest_version_comparable), + 'project': six.text_type(highest_version_obj), + 'version': six.text_type(highest_version_comparable), 'is_highest': True, } if highest_version_obj: @@ -47,14 +47,12 @@ def get_version_compare_data(project, base_version=None): if base_version and base_version.slug != LATEST: try: base_version_comparable = parse_version_failsafe( - base_version.verbose_name, - ) + base_version.verbose_name) if base_version_comparable: # This is only place where is_highest can get set. All error # cases will be set to True, for non- standard versions. ret_val['is_highest'] = ( - base_version_comparable >= highest_version_comparable - ) + base_version_comparable >= highest_version_comparable) else: ret_val['is_highest'] = True except (Version.DoesNotExist, TypeError): @@ -86,19 +84,13 @@ def footer_html(request): project = get_object_or_404(Project, slug=project_slug) version = get_object_or_404( Version.objects.public( - request.user, - project=project, - only_active=False, - ), - slug__iexact=version_slug, - ) + request.user, project=project, only_active=False), + slug__iexact=version_slug) main_project = project.main_language_project or project if page_slug and page_slug != 'index': - if ( - main_project.documentation_type == 'sphinx_htmldir' or - main_project.documentation_type == 'mkdocs' - ): + if (main_project.documentation_type == 'sphinx_htmldir' or + main_project.documentation_type == 'mkdocs'): path = page_slug + '/' elif main_project.documentation_type == 'sphinx_singlehtml': path = 'index.html#document-' + page_slug diff --git a/readthedocs/restapi/views/integrations.py b/readthedocs/restapi/views/integrations.py index 7f44fb4f11a..9a27e2ef9af 100644 --- a/readthedocs/restapi/views/integrations.py +++ b/readthedocs/restapi/views/integrations.py @@ -1,11 +1,18 @@ # -*- coding: utf-8 -*- - """Endpoints integrating with Github, Bitbucket, and other webhooks.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import logging import re +import six from django.shortcuts import get_object_or_404 from rest_framework import permissions, status from rest_framework.exceptions import NotFound, ParseError @@ -23,7 +30,6 @@ from readthedocs.integrations.utils import normalize_request_payload from readthedocs.projects.models import Project - log = logging.getLogger(__name__) GITHUB_EVENT_HEADER = 'HTTP_X_GITHUB_EVENT' @@ -37,7 +43,7 @@ BITBUCKET_PUSH = 'repo:push' -class WebhookMixin: +class WebhookMixin(object): """Base class for Webhook mixins.""" @@ -69,7 +75,7 @@ def get_project(self, **kwargs): def finalize_response(self, req, *args, **kwargs): """If the project was set on POST, store an HTTP exchange.""" - resp = super().finalize_response(req, *args, **kwargs) + resp = super(WebhookMixin, self).finalize_response(req, *args, **kwargs) if hasattr(self, 'project') and self.project: HttpExchange.objects.from_exchange( req, @@ -127,17 +133,12 @@ def get_response_push(self, project, branches): """ to_build, not_building = build_branches(project, branches) if not_building: - log.info( - 'Skipping project branches: project=%s branches=%s', - project, - branches, - ) + log.info('Skipping project branches: project=%s branches=%s', + project, branches) triggered = True if to_build else False - return { - 'build_triggered': triggered, - 'project': project.slug, - 'versions': list(to_build), - } + return {'build_triggered': triggered, + 'project': project.slug, + 'versions': list(to_build)} def sync_versions(self, project): version = sync_versions(project) @@ -179,7 +180,7 @@ def get_data(self): return json.loads(self.request.data['payload']) except (ValueError, KeyError): pass - return super().get_data() + return super(GitHubWebhookView, self).get_data() def handle_webhook(self): # Get event and trigger other webhook events @@ -188,7 +189,7 @@ def handle_webhook(self): Project, project=self.project, data=self.data, - event=event, + event=event ) # Handle push events and trigger builds if event == GITHUB_PUSH: @@ -244,7 +245,7 @@ def handle_webhook(self): Project, project=self.project, data=self.request.data, - event=event, + event=event ) # Handle push events and trigger builds if event in (GITLAB_PUSH, GITLAB_TAG_PUSH): @@ -305,16 +306,16 @@ def handle_webhook(self): """ Handle BitBucket events for push. - BitBucket doesn't have a separate event for creation/deletion, instead - it sets the new attribute (null if it is a deletion) and the old - attribute (null if it is a creation). + BitBucket doesn't have a separate event for creation/deletion, + instead it sets the new attribute (null if it is a deletion) + and the old attribute (null if it is a creation). """ event = self.request.META.get(BITBUCKET_EVENT_HEADER, BITBUCKET_PUSH) webhook_bitbucket.send( Project, project=self.project, data=self.request.data, - event=event, + event=event ) if event == BITBUCKET_PUSH: try: @@ -349,7 +350,8 @@ class IsAuthenticatedOrHasToken(permissions.IsAuthenticated): """ def has_permission(self, request, view): - has_perm = (super().has_permission(request, view)) + has_perm = (super(IsAuthenticatedOrHasToken, self) + .has_permission(request, view)) return has_perm or 'token' in request.data @@ -378,11 +380,9 @@ def get_project(self, **kwargs): # If the user is not an admin of the project, fall back to token auth if self.request.user.is_authenticated: try: - return ( - Project.objects.for_admin_user( - self.request.user, - ).get(**kwargs) - ) + return (Project.objects + .for_admin_user(self.request.user) + .get(**kwargs)) except Project.DoesNotExist: pass # Recheck project and integration relationship during token auth check @@ -402,9 +402,9 @@ def handle_webhook(self): try: branches = self.request.data.get( 'branches', - [self.project.get_default_branch()], + [self.project.get_default_branch()] ) - if isinstance(branches, str): + if isinstance(branches, six.string_types): branches = [branches] return self.get_response_push(self.project, branches) except TypeError: diff --git a/readthedocs/restapi/views/model_views.py b/readthedocs/restapi/views/model_views.py index 60a634c7b12..3e925031472 100644 --- a/readthedocs/restapi/views/model_views.py +++ b/readthedocs/restapi/views/model_views.py @@ -1,10 +1,13 @@ # -*- coding: utf-8 -*- - """Endpoints for listing Projects, Versions, Builds, etc.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging from allauth.socialaccount.models import SocialAccount +from builtins import str from django.shortcuts import get_object_or_404 from django.template.loader import render_to_string from rest_framework import decorators, permissions, status, viewsets @@ -22,25 +25,12 @@ from .. import utils as api_utils from ..permissions import ( - APIPermission, - APIRestrictedPermission, - IsOwner, - RelatedProjectIsOwner, -) + APIPermission, APIRestrictedPermission, IsOwner, RelatedProjectIsOwner) from ..serializers import ( - BuildAdminSerializer, - BuildCommandSerializer, - BuildSerializer, - DomainSerializer, - ProjectAdminSerializer, - ProjectSerializer, - RemoteOrganizationSerializer, - RemoteRepositorySerializer, - SocialAccountSerializer, - VersionAdminSerializer, - VersionSerializer, -) - + BuildAdminSerializer, BuildCommandSerializer, BuildSerializer, + DomainSerializer, ProjectAdminSerializer, ProjectSerializer, + RemoteOrganizationSerializer, RemoteRepositorySerializer, + SocialAccountSerializer, VersionAdminSerializer, VersionSerializer) log = logging.getLogger(__name__) @@ -62,8 +52,7 @@ def render(self, data, accepted_media_type=None, renderer_context=None): if not response or response.exception: return data.get('detail', '').encode(self.charset) data = render_to_string( - 'restapi/log.txt', - {'build': data}, + 'restapi/log.txt', {'build': data} ) return data.encode(self.charset) @@ -80,10 +69,8 @@ class UserSelectViewSet(viewsets.ModelViewSet): def get_serializer_class(self): try: - if ( - self.request.user.is_staff and - self.admin_serializer_class is not None - ): + if (self.request.user.is_staff and + self.admin_serializer_class is not None): return self.admin_serializer_class except AttributeError: pass @@ -117,9 +104,7 @@ def translations(self, *_, **__): @decorators.action(detail=True) def subprojects(self, request, **kwargs): project = get_object_or_404( - Project.objects.api(request.user), - pk=kwargs['pk'], - ) + Project.objects.api(request.user), pk=kwargs['pk']) rels = project.subprojects.all() children = [rel.child for rel in rels] return Response({ @@ -129,23 +114,16 @@ def subprojects(self, request, **kwargs): @decorators.action(detail=True) def active_versions(self, request, **kwargs): project = get_object_or_404( - Project.objects.api(request.user), - pk=kwargs['pk'], - ) + Project.objects.api(request.user), pk=kwargs['pk']) versions = project.versions.filter(active=True) return Response({ 'versions': VersionSerializer(versions, many=True).data, }) - @decorators.action( - detail=True, - permission_classes=[permissions.IsAdminUser], - ) + @decorators.action(detail=True, permission_classes=[permissions.IsAdminUser]) def token(self, request, **kwargs): project = get_object_or_404( - Project.objects.api(request.user), - pk=kwargs['pk'], - ) + Project.objects.api(request.user), pk=kwargs['pk']) token = GitHubService.get_token_for_project(project, force_local=True) return Response({ 'token': token, @@ -154,16 +132,13 @@ def token(self, request, **kwargs): @decorators.action(detail=True) def canonical_url(self, request, **kwargs): project = get_object_or_404( - Project.objects.api(request.user), - pk=kwargs['pk'], - ) + Project.objects.api(request.user), pk=kwargs['pk']) return Response({ 'url': project.get_docs_url(), }) @decorators.action( - detail=True, - permission_classes=[permissions.IsAdminUser], + detail=True, permission_classes=[permissions.IsAdminUser], methods=['post'], ) def sync_versions(self, request, **kwargs): # noqa: D205 @@ -175,9 +150,7 @@ def sync_versions(self, request, **kwargs): # noqa: D205 :returns: the identifiers for the versions that have been deleted. """ project = get_object_or_404( - Project.objects.api(request.user), - pk=kwargs['pk'], - ) + Project.objects.api(request.user), pk=kwargs['pk']) # If the currently highest non-prerelease version is active, then make # the new latest version active as well. @@ -193,17 +166,11 @@ def sync_versions(self, request, **kwargs): # noqa: D205 added_versions = set() if 'tags' in data: ret_set = api_utils.sync_versions( - project=project, - versions=data['tags'], - type=TAG, - ) + project=project, versions=data['tags'], type=TAG) added_versions.update(ret_set) if 'branches' in data: ret_set = api_utils.sync_versions( - project=project, - versions=data['branches'], - type=BRANCH, - ) + project=project, versions=data['branches'], type=BRANCH) added_versions.update(ret_set) deleted_versions = api_utils.delete_versions(project, data) except Exception as e: @@ -222,16 +189,13 @@ def sync_versions(self, request, **kwargs): # noqa: D205 'Triggering new stable build: {project}:{version}'.format( project=project.slug, version=new_stable.identifier, - ), - ) + )) trigger_build(project=project, version=new_stable) # Marking the tag that is considered the new stable version as # active and building it if it was just added. - if ( - activate_new_stable and - promoted_version.slug in added_versions - ): + if (activate_new_stable and + promoted_version.slug in added_versions): promoted_version.active = True promoted_version.save() trigger_build(project=project, version=promoted_version) @@ -249,14 +213,8 @@ class VersionViewSet(UserSelectViewSet): serializer_class = VersionSerializer admin_serializer_class = VersionAdminSerializer model = Version - filter_fields = ( - 'active', - 'project__slug', - ) # django-filter<2.0.0 - filterset_fields = ( - 'active', - 'project__slug', - ) + filter_fields = ('active', 'project__slug',) # django-filter<2.0.0 + filterset_fields = ('active', 'project__slug',) class BuildViewSetBase(UserSelectViewSet): @@ -311,9 +269,7 @@ def get_queryset(self): self.model.objects.api(self.request.user).filter( account__provider__in=[ service.adapter.provider_id for service in registry - ], - ) - ) + ])) class RemoteRepositoryViewSet(viewsets.ReadOnlyModelViewSet): @@ -339,8 +295,7 @@ def get_queryset(self): query = query.filter( account__provider__in=[ service.adapter.provider_id for service in registry - ], - ) + ]) return query diff --git a/readthedocs/restapi/views/search_views.py b/readthedocs/restapi/views/search_views.py index aa2acc1fdbe..abe36174097 100644 --- a/readthedocs/restapi/views/search_views.py +++ b/readthedocs/restapi/views/search_views.py @@ -1,7 +1,6 @@ -# -*- coding: utf-8 -*- - """Endpoints related to searching through projects, sections, etc.""" +from __future__ import absolute_import import logging from rest_framework import decorators, permissions, status @@ -11,8 +10,8 @@ from readthedocs.builds.constants import LATEST from readthedocs.builds.models import Version from readthedocs.projects.models import Project, ProjectRelationship -from readthedocs.restapi import utils from readthedocs.search.lib import search_file, search_project, search_section +from readthedocs.restapi import utils log = logging.getLogger(__name__) @@ -32,12 +31,8 @@ def index_search(request): page_scale = 1 utils.index_search_request( - version=version, - page_list=data['page_list'], - commit=commit, - project_scale=project_scale, - page_scale=page_scale, - ) + version=version, page_list=data['page_list'], commit=commit, + project_scale=project_scale, page_scale=page_scale) return Response({'indexed': True}) @@ -51,30 +46,20 @@ def search(request): version_slug = request.GET.get('version', LATEST) query = request.GET.get('q', None) if project_slug is None or query is None: - return Response( - {'error': 'Need project and q'}, - status=status.HTTP_400_BAD_REQUEST, - ) + return Response({'error': 'Need project and q'}, + status=status.HTTP_400_BAD_REQUEST) try: project = Project.objects.get(slug=project_slug) except Project.DoesNotExist: - return Response( - {'error': 'Project not found'}, - status=status.HTTP_404_NOT_FOUND, - ) - log.debug('(API Search) %s', query) - results = search_file( - request=request, - project_slug=project_slug, - version_slug=version_slug, - query=query, - ) + return Response({'error': 'Project not found'}, + status=status.HTTP_404_NOT_FOUND) + log.debug("(API Search) %s", query) + results = search_file(request=request, project_slug=project_slug, + version_slug=version_slug, query=query) if results is None: - return Response( - {'error': 'Project not found'}, - status=status.HTTP_404_NOT_FOUND, - ) + return Response({'error': 'Project not found'}, + status=status.HTTP_404_NOT_FOUND) # Supplement result paths with domain information on project hits = results.get('hits', {}).get('hits', []) @@ -88,11 +73,13 @@ def search(request): try: subproject = project.subprojects.get(child__slug=search_project) canonical_url = subproject.child.get_docs_url( - version_slug=search_version, + version_slug=search_version ) except ProjectRelationship.DoesNotExist: pass - results['hits']['hits'][n]['fields']['link'] = (canonical_url + path) + results['hits']['hits'][n]['fields']['link'] = ( + canonical_url + path + ) return Response({'results': results}) @@ -103,11 +90,8 @@ def search(request): def project_search(request): query = request.GET.get('q', None) if query is None: - return Response( - {'error': 'Need project and q'}, - status=status.HTTP_400_BAD_REQUEST, - ) - log.debug('(API Project Search) %s', (query)) + return Response({'error': 'Need project and q'}, status=status.HTTP_400_BAD_REQUEST) + log.debug("(API Project Search) %s", (query)) results = search_project(request=request, query=query) return Response({'results': results}) @@ -151,17 +135,12 @@ def section_search(request): if not query: return Response( {'error': 'Search term required. Use the "q" GET arg to search. '}, - status=status.HTTP_400_BAD_REQUEST, - ) + status=status.HTTP_400_BAD_REQUEST) project_slug = request.GET.get('project', None) version_slug = request.GET.get('version', LATEST) path = request.GET.get('path', None) - log.debug( - '(API Section Search) [%s:%s] %s', - project_slug, - version_slug, - query, - ) + log.debug("(API Section Search) [%s:%s] %s", project_slug, version_slug, + query) results = search_section( request=request, query=query, diff --git a/readthedocs/restapi/views/task_views.py b/readthedocs/restapi/views/task_views.py index 8bf9d3843e4..475fb17bda6 100644 --- a/readthedocs/restapi/views/task_views.py +++ b/readthedocs/restapi/views/task_views.py @@ -1,7 +1,12 @@ -# -*- coding: utf-8 -*- - """Endpoints relating to task/job status, etc.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging from django.urls import reverse @@ -13,14 +18,11 @@ from readthedocs.core.utils.tasks import TaskNoPermission, get_public_task_data from readthedocs.oauth import tasks - log = logging.getLogger(__name__) + SUCCESS_STATES = ('SUCCESS',) -FAILURE_STATES = ( - 'FAILURE', - 'REVOKED', -) +FAILURE_STATES = ('FAILURE', 'REVOKED',) FINISHED_STATES = SUCCESS_STATES + FAILURE_STATES STARTED_STATES = ('RECEIVED', 'STARTED', 'RETRY') + FINISHED_STATES @@ -46,19 +48,24 @@ def get_status_data(task_name, state, data, error=None): def job_status(request, task_id): try: task_name, state, public_data, error = get_public_task_data( - request, - task_id, + request, task_id ) except (TaskNoPermission, ConnectionError): - return Response(get_status_data('unknown', 'PENDING', {}),) - return Response(get_status_data(task_name, state, public_data, error),) + return Response( + get_status_data('unknown', 'PENDING', {}) + ) + return Response( + get_status_data(task_name, state, public_data, error) + ) @decorators.api_view(['POST']) @decorators.permission_classes((permissions.IsAuthenticated,)) @decorators.renderer_classes((JSONRenderer,)) def sync_remote_repositories(request): - result = tasks.sync_remote_repositories.delay(user_id=request.user.id,) + result = tasks.sync_remote_repositories.delay( + user_id=request.user.id + ) task_id = result.task_id return Response({ 'task_id': task_id, diff --git a/readthedocs/rtd_tests/base.py b/readthedocs/rtd_tests/base.py index e7d8f8409e0..f11ec5a49b4 100644 --- a/readthedocs/rtd_tests/base.py +++ b/readthedocs/rtd_tests/base.py @@ -1,18 +1,19 @@ -# -*- coding: utf-8 -*- """Base classes and mixins for unit tests.""" -import logging +from __future__ import absolute_import +from builtins import object import os import shutil +import logging import tempfile from collections import OrderedDict +from mock import patch from django.conf import settings +from django.test import TestCase, RequestFactory from django.contrib.auth.models import AnonymousUser from django.contrib.messages.storage.fallback import FallbackStorage from django.contrib.sessions.middleware import SessionMiddleware -from django.test import RequestFactory, TestCase -from mock import patch - +import six log = logging.getLogger(__name__) @@ -22,7 +23,7 @@ def setUp(self): self.original_DOCROOT = settings.DOCROOT self.cwd = os.path.dirname(__file__) self.build_dir = tempfile.mkdtemp() - log.info('build dir: %s', self.build_dir) + log.info("build dir: %s", self.build_dir) if not os.path.exists(self.build_dir): os.makedirs(self.build_dir) settings.DOCROOT = self.build_dir @@ -41,7 +42,7 @@ class MockBuildTestCase(TestCase): pass -class RequestFactoryTestMixin: +class RequestFactoryTestMixin(object): """ Adds helper methods for testing with :py:class:`RequestFactory` @@ -106,14 +107,14 @@ def post_step(self, step, **kwargs): if not self.url: raise Exception('Missing wizard URL') try: - data = { - '{}-{}'.format(step, k): v + data = dict( + ('{0}-{1}'.format(step, k), v) for (k, v) in list(self.step_data[step].items()) - } + ) except KeyError: pass # Update with prefixed step data - data['{}-current_step'.format(self.wizard_class_slug)] = step + data['{0}-current_step'.format(self.wizard_class_slug)] = step view = self.wizard_class.as_view() req = self.request(self.url, method='post', data=data, **kwargs) resp = view(req) @@ -145,7 +146,7 @@ def assertWizardResponse(self, response, step=None): # noqa response.render() self.assertContains( response, - 'name="{}-current_step"'.format(self.wizard_class_slug), + u'name="{0}-current_step"'.format(self.wizard_class_slug) ) # We use camelCase on purpose here to conform with unittest's naming @@ -169,4 +170,4 @@ def assertWizardFailure(self, response, field, match=None): # noqa self.assertIn(field, response.context_data['wizard']['form'].errors) if match is not None: error = response.context_data['wizard']['form'].errors[field] - self.assertRegex(str(error), match) # noqa + self.assertRegex(six.text_type(error), match) # noqa diff --git a/readthedocs/rtd_tests/files/api.fjson b/readthedocs/rtd_tests/files/api.fjson index 7972e09c87f..0e6077c56f8 100644 --- a/readthedocs/rtd_tests/files/api.fjson +++ b/readthedocs/rtd_tests/files/api.fjson @@ -43,4 +43,4 @@ "title": "Internationalization" }, "metatags": "" -} +} \ No newline at end of file diff --git a/readthedocs/rtd_tests/files/conf.py b/readthedocs/rtd_tests/files/conf.py index 11f872849dd..4007dcfab19 100644 --- a/readthedocs/rtd_tests/files/conf.py +++ b/readthedocs/rtd_tests/files/conf.py @@ -13,7 +13,7 @@ '.md': CommonMarkParser, } master_doc = 'index' -project = 'Pip' +project = u'Pip' copyright = str(datetime.now().year) version = '0.8.1' release = '0.8.1' @@ -23,6 +23,6 @@ html_theme = 'sphinx_rtd_theme' file_insertion_enabled = False latex_documents = [ - ('index', 'pip.tex', 'Pip Documentation', - '', 'manual'), + ('index', 'pip.tex', u'Pip Documentation', + u'', 'manual'), ] diff --git a/readthedocs/rtd_tests/fixtures/sample_repo/source/conf.py b/readthedocs/rtd_tests/fixtures/sample_repo/source/conf.py index e6c4dad0f4b..c6c9fcb64db 100644 --- a/readthedocs/rtd_tests/fixtures/sample_repo/source/conf.py +++ b/readthedocs/rtd_tests/fixtures/sample_repo/source/conf.py @@ -41,8 +41,8 @@ master_doc = 'index' # General information about the project. -project = 'sample' -copyright = '2011, Dan' +project = u'sample' +copyright = u'2011, Dan' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -180,10 +180,8 @@ # Grouping the document tree into LaTeX files. List of tuples (source start # file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ( - 'index', 'sample.tex', 'sample Documentation', - 'Dan', 'manual', - ), + ('index', 'sample.tex', u'sample Documentation', + u'Dan', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -215,8 +213,6 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ( - 'index', 'sample', 'sample Documentation', - ['Dan'], 1, - ), + ('index', 'sample', u'sample Documentation', + [u'Dan'], 1) ] diff --git a/readthedocs/rtd_tests/fixtures/sample_repo/source/index.rst b/readthedocs/rtd_tests/fixtures/sample_repo/source/index.rst index 164c84a69c0..d86e67de5be 100644 --- a/readthedocs/rtd_tests/fixtures/sample_repo/source/index.rst +++ b/readthedocs/rtd_tests/fixtures/sample_repo/source/index.rst @@ -17,3 +17,4 @@ Indices and tables * :ref:`genindex` * :ref:`modindex` * :ref:`search` + diff --git a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml index 3ff7b7fb730..add1eeafe9b 100644 --- a/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml +++ b/readthedocs/rtd_tests/fixtures/spec/v2/schema.yml @@ -123,7 +123,7 @@ submodules: # List of submodules to be ignored # Default: [] exclude: any(list(str()), enum('all'), required=False) - + # Do a recursive clone? # Default: false recursive: bool(required=False) diff --git a/readthedocs/rtd_tests/mocks/environment.py b/readthedocs/rtd_tests/mocks/environment.py index 6928d4cec3c..4b963b769ba 100644 --- a/readthedocs/rtd_tests/mocks/environment.py +++ b/readthedocs/rtd_tests/mocks/environment.py @@ -1,11 +1,12 @@ -# -*- coding: utf-8 -*- # pylint: disable=missing-docstring +from __future__ import absolute_import +from builtins import object import mock -class EnvironmentMockGroup: +class EnvironmentMockGroup(object): - """Mock out necessary environment pieces.""" + """Mock out necessary environment pieces""" def __init__(self): self.patches = { @@ -14,61 +15,43 @@ def __init__(self): 'api': mock.patch('slumber.Resource'), 'api_v2.command': mock.patch( 'readthedocs.doc_builder.environments.api_v2.command', - mock.Mock(**{'get.return_value': {}}), - ), + mock.Mock(**{'get.return_value': {}})), 'api_v2.build': mock.patch( 'readthedocs.doc_builder.environments.api_v2.build', - mock.Mock(**{'get.return_value': {}}), - ), + mock.Mock(**{'get.return_value': {}})), 'api_versions': mock.patch( - 'readthedocs.projects.models.Project.api_versions', - ), + 'readthedocs.projects.models.Project.api_versions'), 'non_blocking_lock': mock.patch( - 'readthedocs.vcs_support.utils.NonBlockingLock.__enter__', - ), + 'readthedocs.vcs_support.utils.NonBlockingLock.__enter__'), 'append_conf': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf', - ), + 'readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf'), 'move': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.BaseSphinx.move', - ), + 'readthedocs.doc_builder.backends.sphinx.BaseSphinx.move'), 'conf_dir': mock.patch( - 'readthedocs.projects.models.Project.conf_dir', - ), + 'readthedocs.projects.models.Project.conf_dir'), 'html_build': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build', - ), + 'readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build'), 'html_move': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.HtmlBuilder.move', - ), + 'readthedocs.doc_builder.backends.sphinx.HtmlBuilder.move'), 'localmedia_build': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.LocalMediaBuilder.build', - ), + 'readthedocs.doc_builder.backends.sphinx.LocalMediaBuilder.build'), 'localmedia_move': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.LocalMediaBuilder.move', - ), + 'readthedocs.doc_builder.backends.sphinx.LocalMediaBuilder.move'), 'pdf_build': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.PdfBuilder.build', - ), + 'readthedocs.doc_builder.backends.sphinx.PdfBuilder.build'), 'pdf_move': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.PdfBuilder.move', - ), + 'readthedocs.doc_builder.backends.sphinx.PdfBuilder.move'), 'epub_build': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.EpubBuilder.build', - ), + 'readthedocs.doc_builder.backends.sphinx.EpubBuilder.build'), 'epub_move': mock.patch( - 'readthedocs.doc_builder.backends.sphinx.EpubBuilder.move', - ), + 'readthedocs.doc_builder.backends.sphinx.EpubBuilder.move'), 'move_mkdocs': mock.patch( - 'readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.move', - ), + 'readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.move'), 'append_conf_mkdocs': mock.patch( - 'readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf', - ), + 'readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf'), 'html_build_mkdocs': mock.patch( - 'readthedocs.doc_builder.backends.mkdocs.MkdocsHTML.build', - ), + 'readthedocs.doc_builder.backends.mkdocs.MkdocsHTML.build'), 'glob': mock.patch('readthedocs.doc_builder.backends.sphinx.glob'), 'docker': mock.patch('readthedocs.doc_builder.environments.APIClient'), @@ -77,7 +60,7 @@ def __init__(self): self.mocks = {} def start(self): - """Create a patch object for class patches.""" + """Create a patch object for class patches""" for patch in self.patches: self.mocks[patch] = self.patches[patch].start() self.mocks['process'].communicate.return_value = ('', '') @@ -95,7 +78,7 @@ def stop(self): pass def configure_mock(self, mock, kwargs): - """Configure object mocks.""" + """Configure object mocks""" self.mocks[mock].configure_mock(**kwargs) def __getattr__(self, name): diff --git a/readthedocs/rtd_tests/mocks/mock_api.py b/readthedocs/rtd_tests/mocks/mock_api.py index 9c6ff251515..84c40d7c4d1 100644 --- a/readthedocs/rtd_tests/mocks/mock_api.py +++ b/readthedocs/rtd_tests/mocks/mock_api.py @@ -1,15 +1,14 @@ -# -*- coding: utf-8 -*- """Mock versions of many API-related classes.""" -import json +from __future__ import absolute_import +from builtins import object from contextlib import contextmanager - +import json import mock - # Mock tastypi API. -class ProjectData: +class ProjectData(object): def get(self): return dict() @@ -19,7 +18,7 @@ def put(self, x=None): def mock_version(repo): """Construct and return a class implementing the Version interface.""" - class MockVersion: + class MockVersion(object): def __init__(self, x=None): pass @@ -72,7 +71,7 @@ def get(self, **kwargs): return MockVersion -class MockApi: +class MockApi(object): def __init__(self, repo): self.version = mock_version(repo) diff --git a/readthedocs/rtd_tests/mocks/paths.py b/readthedocs/rtd_tests/mocks/paths.py index 787eabff923..34fa7e5953f 100644 --- a/readthedocs/rtd_tests/mocks/paths.py +++ b/readthedocs/rtd_tests/mocks/paths.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- """Context managers to patch os.path.exists calls.""" +from __future__ import absolute_import import os import re - import mock diff --git a/readthedocs/rtd_tests/tests/projects/test_admin_actions.py b/readthedocs/rtd_tests/tests/projects/test_admin_actions.py index dd25f4a13b4..54111314cee 100644 --- a/readthedocs/rtd_tests/tests/projects/test_admin_actions.py +++ b/readthedocs/rtd_tests/tests/projects/test_admin_actions.py @@ -1,9 +1,8 @@ -# -*- coding: utf-8 -*- -import django_dynamic_fixture as fixture import mock -from django import urls +import django_dynamic_fixture as fixture from django.contrib.admin.helpers import ACTION_CHECKBOX_NAME from django.contrib.auth.models import User +from django import urls from django.test import TestCase from readthedocs.core.models import UserProfile @@ -35,7 +34,7 @@ def test_project_ban_owner(self): } resp = self.client.post( urls.reverse('admin:projects_project_changelist'), - action_data, + action_data ) self.assertTrue(self.project.users.filter(profile__banned=True).exists()) self.assertFalse(self.project.users.filter(profile__banned=False).exists()) @@ -53,15 +52,15 @@ def test_project_ban_multiple_owners(self): } resp = self.client.post( urls.reverse('admin:projects_project_changelist'), - action_data, + action_data ) self.assertFalse(self.project.users.filter(profile__banned=True).exists()) self.assertEqual(self.project.users.filter(profile__banned=False).count(), 2) @mock.patch('readthedocs.projects.admin.broadcast') def test_project_delete(self, broadcast): - """Test project and artifacts are removed.""" - from readthedocs.projects.tasks import remove_dirs + """Test project and artifacts are removed""" + from readthedocs.projects.tasks import remove_dir action_data = { ACTION_CHECKBOX_NAME: [self.project.pk], 'action': 'delete_selected', @@ -70,11 +69,11 @@ def test_project_delete(self, broadcast): } resp = self.client.post( urls.reverse('admin:projects_project_changelist'), - action_data, + action_data ) self.assertFalse(Project.objects.filter(pk=self.project.pk).exists()) broadcast.assert_has_calls([ mock.call( - type='app', task=remove_dirs, args=[(self.project.doc_path,)], + type='app', task=remove_dir, args=[self.project.doc_path] ), ]) diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index f5dcb1cae06..707a4da79c3 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -1,14 +1,23 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import base64 import datetime import json import mock from allauth.socialaccount.models import SocialAccount +from builtins import str from django.contrib.auth.models import User +from django.urls import reverse from django.http import QueryDict from django.test import TestCase -from django.urls import reverse +from django.utils import six from django_dynamic_fixture import get from rest_framework import status from rest_framework.test import APIClient @@ -34,7 +43,6 @@ ) from readthedocs.restapi.views.task_views import get_status_data - super_auth = base64.b64encode(b'super:test').decode('utf-8') eric_auth = base64.b64encode(b'eric:test').decode('utf-8') @@ -196,7 +204,10 @@ def test_save_same_config_using_patch(self): ) def test_response_building(self): - """The ``view docs`` attr should return a link to the dashboard.""" + """ + The ``view docs`` attr should return a link + to the dashboard. + """ client = APIClient() client.login(username='super', password='test') project = get( @@ -235,7 +246,10 @@ def test_response_building(self): self.assertEqual(build['docs_url'], dashboard_url) def test_response_finished_and_success(self): - """The ``view docs`` attr should return a link to the docs.""" + """ + The ``view docs`` attr should return a link + to the docs. + """ client = APIClient() client.login(username='super', password='test') project = get( @@ -270,7 +284,10 @@ def test_response_finished_and_success(self): self.assertEqual(build['docs_url'], docs_url) def test_response_finished_and_fail(self): - """The ``view docs`` attr should return a link to the dashboard.""" + """ + The ``view docs`` attr should return a link + to the dashboard. + """ client = APIClient() client.login(username='super', password='test') project = get( @@ -342,7 +359,7 @@ def test_update_build_without_permission(self): client.force_authenticate(user=api_user) build = get(Build, project_id=1, version_id=1, state='cloning') resp = client.put( - '/api/v2/build/{}/'.format(build.pk), + '/api/v2/build/{0}/'.format(build.pk), { 'project': 1, 'version': 1, @@ -364,11 +381,11 @@ def test_make_build_protected_fields(self): api_user = get(User, staff=False, password='test') client.force_authenticate(user=api_user) - resp = client.get('/api/v2/build/{}/'.format(build.pk), format='json') + resp = client.get('/api/v2/build/{0}/'.format(build.pk), format='json') self.assertEqual(resp.status_code, 200) client.force_authenticate(user=User.objects.get(username='super')) - resp = client.get('/api/v2/build/{}/'.format(build.pk), format='json') + resp = client.get('/api/v2/build/{0}/'.format(build.pk), format='json') self.assertEqual(resp.status_code, 200) self.assertIn('builder', resp.data) @@ -414,19 +431,19 @@ def test_get_raw_log_success(self): BuildCommandResult, build=build, command='python setup.py install', - output='Installing dependencies...', + output='Installing dependencies...' ) get( BuildCommandResult, build=build, command='git checkout master', - output='Switched to branch "master"', + output='Switched to branch "master"' ) client = APIClient() api_user = get(User, user='test', password='test') client.force_authenticate(user=api_user) - resp = client.get('/api/v2/build/{}.txt'.format(build.pk)) + resp = client.get('/api/v2/build/{0}.txt'.format(build.pk)) self.assertEqual(resp.status_code, 200) self.assertIn('Read the Docs build information', resp.content.decode()) @@ -440,11 +457,11 @@ def test_get_raw_log_success(self): self.assertIn('[rtd-command-info]', resp.content.decode()) self.assertIn( 'python setup.py install\nInstalling dependencies...', - resp.content.decode(), + resp.content.decode() ) self.assertIn( 'git checkout master\nSwitched to branch "master"', - resp.content.decode(), + resp.content.decode() ) def test_get_raw_log_building(self): @@ -464,13 +481,13 @@ def test_get_raw_log_building(self): BuildCommandResult, build=build, command='git checkout master', - output='Switched to branch "master"', + output='Switched to branch "master"' ) client = APIClient() api_user = get(User, user='test', password='test') client.force_authenticate(user=api_user) - resp = client.get('/api/v2/build/{}.txt'.format(build.pk)) + resp = client.get('/api/v2/build/{0}.txt'.format(build.pk)) self.assertEqual(resp.status_code, 200) self.assertIn('Read the Docs build information', resp.content.decode()) @@ -484,17 +501,17 @@ def test_get_raw_log_building(self): self.assertIn('[rtd-command-info]', resp.content.decode()) self.assertIn( 'python setup.py install\nInstalling dependencies...', - resp.content.decode(), + resp.content.decode() ) self.assertIn( 'git checkout master\nSwitched to branch "master"', - resp.content.decode(), + resp.content.decode() ) def test_get_raw_log_failure(self): build = get( Build, project_id=1, version_id=1, - builder='foo', success=False, exit_code=1, + builder='foo', success=False, exit_code=1 ) get( BuildCommandResult, @@ -507,13 +524,13 @@ def test_get_raw_log_failure(self): BuildCommandResult, build=build, command='git checkout master', - output='Switched to branch "master"', + output='Switched to branch "master"' ) client = APIClient() api_user = get(User, user='test', password='test') client.force_authenticate(user=api_user) - resp = client.get('/api/v2/build/{}.txt'.format(build.pk)) + resp = client.get('/api/v2/build/{0}.txt'.format(build.pk)) self.assertEqual(resp.status_code, 200) self.assertIn('Read the Docs build information', resp.content.decode()) @@ -527,11 +544,11 @@ def test_get_raw_log_failure(self): self.assertIn('[rtd-command-info]', resp.content.decode()) self.assertIn( 'python setup.py install\nInstalling dependencies...', - resp.content.decode(), + resp.content.decode() ) self.assertIn( 'git checkout master\nSwitched to branch "master"', - resp.content.decode(), + resp.content.decode() ) def test_get_invalid_raw_log(self): @@ -539,7 +556,7 @@ def test_get_invalid_raw_log(self): api_user = get(User, user='test', password='test') client.force_authenticate(user=api_user) - resp = client.get('/api/v2/build/{}.txt'.format(404)) + resp = client.get('/api/v2/build/{0}.txt'.format(404)) self.assertEqual(resp.status_code, status.HTTP_404_NOT_FOUND) def test_build_filter_by_commit(self): @@ -644,7 +661,8 @@ def test_project_features(self): resp = client.get('/api/v2/project/%s/' % (project.pk)) self.assertEqual(resp.status_code, 200) self.assertIn('features', resp.data) - self.assertCountEqual( + six.assertCountEqual( + self, resp.data['features'], [feature1.feature_id, feature2.feature_id], ) @@ -814,11 +832,11 @@ def setUp(self): self.project = get(Project) self.version = get( Version, slug='master', verbose_name='master', - active=True, project=self.project, + active=True, project=self.project ) self.version_tag = get( Version, slug='v1.0', verbose_name='v1.0', - active=True, project=self.project, + active=True, project=self.project ) self.github_payload = { 'ref': 'master', @@ -850,7 +868,7 @@ def test_webhook_skipped_project(self, trigger_build): self.project.save() response = client.post( - '/api/v2/webhook/github/{}/'.format( + '/api/v2/webhook/github/{0}/'.format( self.project.slug, ), self.github_payload, @@ -865,62 +883,56 @@ def test_github_webhook_for_branches(self, trigger_build): client = APIClient() client.post( - '/api/v2/webhook/github/{}/'.format(self.project.slug), + '/api/v2/webhook/github/{0}/'.format(self.project.slug), {'ref': 'master'}, format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=self.version, project=self.project)], - ) + [mock.call(force=True, version=self.version, project=self.project)]) client.post( - '/api/v2/webhook/github/{}/'.format(self.project.slug), + '/api/v2/webhook/github/{0}/'.format(self.project.slug), {'ref': 'non-existent'}, format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=mock.ANY, project=self.project)], - ) + [mock.call(force=True, version=mock.ANY, project=self.project)]) client.post( - '/api/v2/webhook/github/{}/'.format(self.project.slug), + '/api/v2/webhook/github/{0}/'.format(self.project.slug), {'ref': 'refs/heads/master'}, format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=self.version, project=self.project)], - ) + [mock.call(force=True, version=self.version, project=self.project)]) def test_github_webhook_for_tags(self, trigger_build): """GitHub webhook API.""" client = APIClient() client.post( - '/api/v2/webhook/github/{}/'.format(self.project.slug), + '/api/v2/webhook/github/{0}/'.format(self.project.slug), {'ref': 'v1.0'}, format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=self.version_tag, project=self.project)], - ) + [mock.call(force=True, version=self.version_tag, project=self.project)]) client.post( - '/api/v2/webhook/github/{}/'.format(self.project.slug), + '/api/v2/webhook/github/{0}/'.format(self.project.slug), {'ref': 'refs/heads/non-existent'}, format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=mock.ANY, project=self.project)], - ) + [mock.call(force=True, version=mock.ANY, project=self.project)]) client.post( - '/api/v2/webhook/github/{}/'.format(self.project.slug), + '/api/v2/webhook/github/{0}/'.format(self.project.slug), {'ref': 'refs/tags/v1.0'}, format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=self.version_tag, project=self.project)], - ) + [mock.call(force=True, version=self.version_tag, project=self.project)]) @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_github_create_event(self, sync_repository_task, trigger_build): @@ -974,7 +986,7 @@ def test_github_invalid_webhook(self, trigger_build): """GitHub webhook unhandled event.""" client = APIClient() resp = client.post( - '/api/v2/webhook/github/{}/'.format(self.project.slug), + '/api/v2/webhook/github/{0}/'.format(self.project.slug), {'foo': 'bar'}, format='json', HTTP_X_GITHUB_EVENT='pull_request', @@ -991,7 +1003,7 @@ def test_gitlab_webhook_for_branches(self, trigger_build): format='json', ) trigger_build.assert_called_with( - force=True, version=mock.ANY, project=self.project, + force=True, version=mock.ANY, project=self.project ) trigger_build.reset_mock() @@ -1017,7 +1029,7 @@ def test_gitlab_webhook_for_tags(self, trigger_build): format='json', ) trigger_build.assert_called_with( - force=True, version=self.version_tag, project=self.project, + force=True, version=self.version_tag, project=self.project ) trigger_build.reset_mock() @@ -1030,7 +1042,7 @@ def test_gitlab_webhook_for_tags(self, trigger_build): format='json', ) trigger_build.assert_called_with( - force=True, version=self.version_tag, project=self.project, + force=True, version=self.version_tag, project=self.project ) trigger_build.reset_mock() @@ -1046,8 +1058,7 @@ def test_gitlab_webhook_for_tags(self, trigger_build): @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_gitlab_push_hook_creation( - self, sync_repository_task, trigger_build, - ): + self, sync_repository_task, trigger_build): client = APIClient() self.gitlab_payload.update( before=GITLAB_NULL_HASH, @@ -1068,8 +1079,7 @@ def test_gitlab_push_hook_creation( @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_gitlab_push_hook_deletion( - self, sync_repository_task, trigger_build, - ): + self, sync_repository_task, trigger_build): client = APIClient() self.gitlab_payload.update( before='95790bf891e76fee5e1747ab589903a6a1f80f22', @@ -1090,8 +1100,7 @@ def test_gitlab_push_hook_deletion( @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_gitlab_tag_push_hook_creation( - self, sync_repository_task, trigger_build, - ): + self, sync_repository_task, trigger_build): client = APIClient() self.gitlab_payload.update( object_kind=GITLAB_TAG_PUSH, @@ -1113,8 +1122,7 @@ def test_gitlab_tag_push_hook_creation( @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_gitlab_tag_push_hook_deletion( - self, sync_repository_task, trigger_build, - ): + self, sync_repository_task, trigger_build): client = APIClient() self.gitlab_payload.update( object_kind=GITLAB_TAG_PUSH, @@ -1138,7 +1146,7 @@ def test_gitlab_invalid_webhook(self, trigger_build): """GitLab webhook unhandled event.""" client = APIClient() resp = client.post( - '/api/v2/webhook/gitlab/{}/'.format(self.project.slug), + '/api/v2/webhook/gitlab/{0}/'.format(self.project.slug), {'object_kind': 'pull_request'}, format='json', ) @@ -1154,8 +1162,7 @@ def test_bitbucket_webhook(self, trigger_build): format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=mock.ANY, project=self.project)], - ) + [mock.call(force=True, version=mock.ANY, project=self.project)]) client.post( '/api/v2/webhook/bitbucket/{}/'.format(self.project.slug), { @@ -1171,8 +1178,7 @@ def test_bitbucket_webhook(self, trigger_build): format='json', ) trigger_build.assert_has_calls( - [mock.call(force=True, version=mock.ANY, project=self.project)], - ) + [mock.call(force=True, version=mock.ANY, project=self.project)]) trigger_build_call_count = trigger_build.call_count client.post( @@ -1192,8 +1198,7 @@ def test_bitbucket_webhook(self, trigger_build): @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_bitbucket_push_hook_creation( - self, sync_repository_task, trigger_build, - ): + self, sync_repository_task, trigger_build): client = APIClient() self.bitbucket_payload['push']['changes'][0]['old'] = None resp = client.post( @@ -1211,8 +1216,7 @@ def test_bitbucket_push_hook_creation( @mock.patch('readthedocs.core.views.hooks.sync_repository_task') def test_bitbucket_push_hook_deletion( - self, sync_repository_task, trigger_build, - ): + self, sync_repository_task, trigger_build): client = APIClient() self.bitbucket_payload['push']['changes'][0]['new'] = None resp = client.post( @@ -1232,16 +1236,15 @@ def test_bitbucket_invalid_webhook(self, trigger_build): """Bitbucket webhook unhandled event.""" client = APIClient() resp = client.post( - '/api/v2/webhook/bitbucket/{}/'.format(self.project.slug), - {'foo': 'bar'}, format='json', HTTP_X_EVENT_KEY='pull_request', - ) + '/api/v2/webhook/bitbucket/{0}/'.format(self.project.slug), + {'foo': 'bar'}, format='json', HTTP_X_EVENT_KEY='pull_request') self.assertEqual(resp.status_code, 200) self.assertEqual(resp.data['detail'], 'Unhandled webhook event') def test_generic_api_fails_without_auth(self, trigger_build): client = APIClient() resp = client.post( - '/api/v2/webhook/generic/{}/'.format(self.project.slug), + '/api/v2/webhook/generic/{0}/'.format(self.project.slug), {}, format='json', ) @@ -1259,7 +1262,7 @@ def test_generic_api_respects_token_auth(self, trigger_build): ) self.assertIsNotNone(integration.token) resp = client.post( - '/api/v2/webhook/{}/{}/'.format( + '/api/v2/webhook/{0}/{1}/'.format( self.project.slug, integration.pk, ), @@ -1270,7 +1273,7 @@ def test_generic_api_respects_token_auth(self, trigger_build): self.assertTrue(resp.data['build_triggered']) # Test nonexistent branch resp = client.post( - '/api/v2/webhook/{}/{}/'.format( + '/api/v2/webhook/{0}/{1}/'.format( self.project.slug, integration.pk, ), @@ -1286,7 +1289,7 @@ def test_generic_api_respects_basic_auth(self, trigger_build): self.project.users.add(user) client.force_authenticate(user=user) resp = client.post( - '/api/v2/webhook/generic/{}/'.format(self.project.slug), + '/api/v2/webhook/generic/{0}/'.format(self.project.slug), {}, format='json', ) @@ -1298,11 +1301,10 @@ def test_generic_api_falls_back_to_token_auth(self, trigger_build): user = get(User) client.force_authenticate(user=user) integration = Integration.objects.create( - project=self.project, integration_type=Integration.API_WEBHOOK, - ) + project=self.project, integration_type=Integration.API_WEBHOOK) self.assertIsNotNone(integration.token) resp = client.post( - '/api/v2/webhook/{}/{}/'.format( + '/api/v2/webhook/{0}/{1}/'.format( self.project.slug, integration.pk, ), @@ -1489,8 +1491,9 @@ def test_get_version_by_id(self): ) def test_get_active_versions(self): - """Test the full response of - ``/api/v2/version/?project__slug=pip&active=true``""" + """ + Test the full response of ``/api/v2/version/?project__slug=pip&active=true`` + """ pip = Project.objects.get(slug='pip') data = QueryDict('', mutable=True) @@ -1500,7 +1503,7 @@ def test_get_active_versions(self): }) url = '{base_url}?{querystring}'.format( base_url=reverse('version-list'), - querystring=data.urlencode(), + querystring=data.urlencode() ) resp = self.client.get(url, content_type='application/json') @@ -1514,7 +1517,7 @@ def test_get_active_versions(self): }) url = '{base_url}?{querystring}'.format( base_url=reverse('version-list'), - querystring=data.urlencode(), + querystring=data.urlencode() ) resp = self.client.get(url, content_type='application/json') @@ -1531,13 +1534,11 @@ def test_get_status_data(self): {'data': 'public'}, 'Something bad happened', ) - self.assertEqual( - data, { - 'name': 'public_task_exception', - 'data': {'data': 'public'}, - 'started': True, - 'finished': True, - 'success': False, - 'error': 'Something bad happened', - }, - ) + self.assertEqual(data, { + 'name': 'public_task_exception', + 'data': {'data': 'public'}, + 'started': True, + 'finished': True, + 'success': False, + 'error': 'Something bad happened', + }) diff --git a/readthedocs/rtd_tests/tests/test_api_permissions.py b/readthedocs/rtd_tests/tests/test_api_permissions.py index 7367d8f5d6a..d3666927add 100644 --- a/readthedocs/rtd_tests/tests/test_api_permissions.py +++ b/readthedocs/rtd_tests/tests/test_api_permissions.py @@ -1,8 +1,7 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import from functools import partial -from unittest import TestCase - from mock import Mock +from unittest import TestCase from readthedocs.restapi.permissions import APIRestrictedPermission @@ -18,27 +17,23 @@ def assertAllow(self, handler, method, is_admin, obj=None): if obj is None: self.assertTrue(handler.has_permission( request=self.get_request(method, is_admin=is_admin), - view=None, - )) + view=None)) else: self.assertTrue(handler.has_object_permission( request=self.get_request(method, is_admin=is_admin), view=None, - obj=obj, - )) + obj=obj)) def assertDisallow(self, handler, method, is_admin, obj=None): if obj is None: self.assertFalse(handler.has_permission( request=self.get_request(method, is_admin=is_admin), - view=None, - )) + view=None)) else: self.assertFalse(handler.has_object_permission( request=self.get_request(method, is_admin=is_admin), view=None, - obj=obj, - )) + obj=obj)) def test_non_object_permissions(self): handler = APIRestrictedPermission() diff --git a/readthedocs/rtd_tests/tests/test_api_version_compare.py b/readthedocs/rtd_tests/tests/test_api_version_compare.py index 24b94ad39d6..0daec18a086 100644 --- a/readthedocs/rtd_tests/tests/test_api_version_compare.py +++ b/readthedocs/rtd_tests/tests/test_api_version_compare.py @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import from django.test import TestCase from readthedocs.builds.constants import LATEST diff --git a/readthedocs/rtd_tests/tests/test_backend.py b/readthedocs/rtd_tests/tests/test_backend.py index 5618dd92151..ca0de802c9e 100644 --- a/readthedocs/rtd_tests/tests/test_backend.py +++ b/readthedocs/rtd_tests/tests/test_backend.py @@ -1,10 +1,19 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import os from os.path import exists from tempfile import mkdtemp import django_dynamic_fixture as fixture +import pytest +import six from django.contrib.auth.models import User from mock import Mock, patch @@ -25,15 +34,15 @@ class TestGitBackend(RTDTestCase): def setUp(self): git_repo = make_test_git() - super().setUp() + super(TestGitBackend, self).setUp() self.eric = User(username='eric') self.eric.set_password('test') self.eric.save() self.project = Project.objects.create( - name='Test Project', - repo_type='git', + name="Test Project", + repo_type="git", #Our top-level checkout - repo=git_repo, + repo=git_repo ) self.project.users.add(self.eric) self.dummy_conf = Mock() @@ -72,6 +81,7 @@ def test_git_branches(self, checkout_path): {branch.verbose_name for branch in repo.branches}, ) + @pytest.mark.skipif(six.PY2, reason='Only for python3') @patch('readthedocs.projects.models.Project.checkout_path') def test_git_branches_unicode(self, checkout_path): repo_path = self.project.repo @@ -108,17 +118,6 @@ def test_git_update_and_checkout(self): self.assertEqual(code, 0) self.assertTrue(exists(repo.working_dir)) - def test_git_checkout_invalid_revision(self): - repo = self.project.vcs_repo() - repo.update() - version = 'invalid-revision' - with self.assertRaises(RepositoryError) as e: - repo.checkout(version) - self.assertEqual( - str(e.exception), - RepositoryError.FAILED_TO_CHECKOUT.format(version), - ) - def test_git_tags(self): repo_path = self.project.repo create_git_tag(repo_path, 'v01') @@ -129,7 +128,7 @@ def test_git_tags(self): # so we need to hack the repo path repo.working_dir = repo_path self.assertEqual( - {'v01', 'v02', 'release-ünîø∂é'}, + set(['v01', 'v02', 'release-ünîø∂é']), {vcs.verbose_name for vcs in repo.tags}, ) @@ -179,14 +178,14 @@ def test_check_submodule_urls(self): def test_check_invalid_submodule_urls(self): repo = self.project.vcs_repo() repo.update() - repo.checkout('invalidsubmodule') + r = repo.checkout('invalidsubmodule') with self.assertRaises(RepositoryError) as e: repo.update_submodules(self.dummy_conf) # `invalid` is created in `make_test_git` # it's a url in ssh form. self.assertEqual( str(e.exception), - RepositoryError.INVALID_SUBMODULES.format(['invalid']), + RepositoryError.INVALID_SUBMODULES.format(['invalid']) ) @patch('readthedocs.projects.models.Project.checkout_path') @@ -208,28 +207,28 @@ def test_fetch_clean_tags_and_branches(self, checkout_path): # We still have all branches and tags in the local repo self.assertEqual( - {'v01', 'v02'}, - {vcs.verbose_name for vcs in repo.tags}, + set(['v01', 'v02']), + set(vcs.verbose_name for vcs in repo.tags) ) self.assertEqual( - { + set([ 'invalidsubmodule', 'master', 'submodule', 'newbranch', - }, - {vcs.verbose_name for vcs in repo.branches}, + ]), + set(vcs.verbose_name for vcs in repo.branches) ) repo.update() # We don't have the eliminated branches and tags in the local repo self.assertEqual( - {'v01'}, - {vcs.verbose_name for vcs in repo.tags}, + set(['v01']), + set(vcs.verbose_name for vcs in repo.tags) ) self.assertEqual( - { - 'invalidsubmodule', 'master', 'submodule', - }, - {vcs.verbose_name for vcs in repo.branches}, + set([ + 'invalidsubmodule', 'master', 'submodule' + ]), + set(vcs.verbose_name for vcs in repo.branches) ) @@ -237,7 +236,7 @@ class TestHgBackend(RTDTestCase): def setUp(self): hg_repo = make_test_hg() - super().setUp() + super(TestHgBackend, self).setUp() self.eric = User(username='eric') self.eric.set_password('test') self.eric.save() @@ -245,7 +244,7 @@ def setUp(self): name='Test Project', repo_type='hg', # Our top-level checkout - repo=hg_repo, + repo=hg_repo ) self.project.users.add(self.eric) @@ -268,17 +267,6 @@ def test_update_and_checkout(self): self.assertEqual(code, 0) self.assertTrue(exists(repo.working_dir)) - def test_checkout_invalid_revision(self): - repo = self.project.vcs_repo() - repo.update() - version = 'invalid-revision' - with self.assertRaises(RepositoryError) as e: - repo.checkout(version) - self.assertEqual( - str(e.exception), - RepositoryError.FAILED_TO_CHECKOUT.format(version), - ) - def test_parse_tags(self): data = """\ tip 13575:8e94a1b4e9a4 diff --git a/readthedocs/rtd_tests/tests/test_backend_svn.py b/readthedocs/rtd_tests/tests/test_backend_svn.py index eec70268267..8de9ea776a0 100644 --- a/readthedocs/rtd_tests/tests/test_backend_svn.py +++ b/readthedocs/rtd_tests/tests/test_backend_svn.py @@ -1,14 +1,21 @@ # -*- coding: utf-8 -*- -"""Tests For SVN.""" +"""Tests For SVN""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + +from mock import patch from django_dynamic_fixture import get -from readthedocs.builds.models import Version -from readthedocs.projects.models import Project from readthedocs.rtd_tests.base import RTDTestCase +from readthedocs.projects.models import Project +from readthedocs.builds.models import Version from readthedocs.vcs_support.backends.svn import Backend as SvnBackend - class TestSvnBackend(RTDTestCase): def test_get_url(self): diff --git a/readthedocs/rtd_tests/tests/test_build_config.py b/readthedocs/rtd_tests/tests/test_build_config.py index a0fd290b327..b0c2186d419 100644 --- a/readthedocs/rtd_tests/tests/test_build_config.py +++ b/readthedocs/rtd_tests/tests/test_build_config.py @@ -1,25 +1,26 @@ -# -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from os import path import pytest +import six import yamale -from yamale.validators import DefaultValidators, Validator - from readthedocs.config.tests import utils - +from yamale.validators import DefaultValidators, Validator V2_SCHEMA = path.join( path.dirname(__file__), - '../fixtures/spec/v2/schema.yml', + '../fixtures/spec/v2/schema.yml' ) class PathValidator(Validator): """ - Path validator. + Path validator - Checks if the given value is a string and a existing file. + Checks if the given value is a string and a existing + file. """ tag = 'path' @@ -27,10 +28,10 @@ class PathValidator(Validator): configuration_file = '.' def _is_valid(self, value): - if isinstance(value, str): + if isinstance(value, six.string_types): file_ = path.join( path.dirname(self.configuration_file), - value, + value ) return path.exists(file_) return False @@ -58,7 +59,7 @@ def validate_schema(file): data = yamale.make_data(file) schema = yamale.make_schema( V2_SCHEMA, - validators=validators, + validators=validators ) yamale.validate(schema, data) @@ -84,7 +85,7 @@ def test_invalid_version(tmpdir): assertInvalidConfig( tmpdir, 'version: "latest"', - ['version:', "'latest' not in"], + ['version:', "'latest' not in"] ) @@ -92,7 +93,7 @@ def test_invalid_version_1(tmpdir): assertInvalidConfig( tmpdir, 'version: "1"', - ['version', "'1' not in"], + ['version', "'1' not in"] ) @@ -134,7 +135,7 @@ def test_formats_invalid(tmpdir): assertInvalidConfig( tmpdir, content, - ['formats', "'invalidformat' not in"], + ['formats', "'invalidformat' not in"] ) @@ -164,7 +165,7 @@ def test_conda_invalid(tmpdir): assertInvalidConfig( tmpdir, content, - ['environment.yaml', 'is not a path'], + ['environment.yaml', 'is not a path'] ) @@ -177,7 +178,7 @@ def test_conda_missing_key(tmpdir): assertInvalidConfig( tmpdir, content, - ['conda.environment: Required'], + ['conda.environment: Required'] ) @@ -209,7 +210,7 @@ def test_build_invalid(tmpdir): assertInvalidConfig( tmpdir, content, - ["build.image: '9.0' not in"], + ["build.image: '9.0' not in"] ) @@ -232,7 +233,7 @@ def test_python_version_invalid(tmpdir): assertInvalidConfig( tmpdir, content, - ["version: '4' not in"], + ["version: '4' not in"] ) @@ -265,7 +266,7 @@ def test_python_install_requirements(tmpdir): assertInvalidConfig( tmpdir, content, - ['requirements:', "'23' is not a path"], + ['requirements:', "'23' is not a path"] ) @@ -291,7 +292,7 @@ def test_python_install_invalid(tmpdir): assertInvalidConfig( tmpdir, content, - ["python.install: 'guido' is not a list"], + ["python.install: 'guido' is not a list"] ) @@ -331,15 +332,13 @@ def test_python_install_extra_requirements_empty(tmpdir, value): @pytest.mark.parametrize('pipfile', ['another_docs/', '.', 'project/']) def test_python_install_pipfile(tmpdir, pipfile): - utils.apply_fs( - tmpdir, { - 'another_docs': { - 'Pipfile': '', - }, - 'project': {}, + utils.apply_fs(tmpdir, { + 'another_docs': { 'Pipfile': '', }, - ) + 'project': {}, + 'Pipfile': '', + }) content = ''' version: "2" python: @@ -418,7 +417,7 @@ def test_python_system_packages_invalid(tmpdir, value): assertInvalidConfig( tmpdir, content.format(value=value), - ['is not a bool'], + ['is not a bool'] ) @@ -450,7 +449,7 @@ def test_sphinx_invalid(tmpdir, value): assertInvalidConfig( tmpdir, content, - ['is not a path'], + ['is not a path'] ) @@ -473,7 +472,7 @@ def test_sphinx_fail_on_warning_invalid(tmpdir, value): assertInvalidConfig( tmpdir, content.format(value=value), - ['is not a bool'], + ['is not a bool'] ) @@ -505,7 +504,7 @@ def test_mkdocs_invalid(tmpdir, value): assertInvalidConfig( tmpdir, content, - ['is not a path'], + ['is not a path'] ) @@ -528,7 +527,7 @@ def test_mkdocs_fail_on_warning_invalid(tmpdir, value): assertInvalidConfig( tmpdir, content.format(value=value), - ['is not a bool'], + ['is not a bool'] ) @@ -596,7 +595,7 @@ def test_redirects_invalid(tmpdir): assertInvalidConfig( tmpdir, content, - ['is not a str'], + ['is not a str'] ) diff --git a/readthedocs/rtd_tests/tests/test_build_forms.py b/readthedocs/rtd_tests/tests/test_build_forms.py index 1e901771632..b0ba6890d09 100644 --- a/readthedocs/rtd_tests/tests/test_build_forms.py +++ b/readthedocs/rtd_tests/tests/test_build_forms.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from django.test import TestCase from django_dynamic_fixture import get @@ -28,7 +30,7 @@ def test_default_version_is_active(self): 'active': True, 'privacy_level': PRIVATE, }, - instance=version, + instance=version ) self.assertTrue(form.is_valid()) @@ -46,7 +48,7 @@ def test_default_version_is_inactive(self): 'active': False, 'privacy_level': PRIVATE, }, - instance=version, + instance=version ) self.assertFalse(form.is_valid()) self.assertIn('active', form.errors) diff --git a/readthedocs/rtd_tests/tests/test_build_notifications.py b/readthedocs/rtd_tests/tests/test_build_notifications.py index 81acedb013c..9460589463a 100644 --- a/readthedocs/rtd_tests/tests/test_build_notifications.py +++ b/readthedocs/rtd_tests/tests/test_build_notifications.py @@ -1,15 +1,18 @@ # -*- coding: utf-8 -*- """Notifications sent after build is completed.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import django_dynamic_fixture as fixture from django.core import mail from django.test import TestCase from mock import patch from readthedocs.builds.models import Build, Version -from readthedocs.projects.forms import WebHookForm -from readthedocs.projects.models import EmailHook, Project, WebHook +from readthedocs.projects.models import Project, EmailHook, WebHook from readthedocs.projects.tasks import send_notifications +from readthedocs.projects.forms import WebHookForm class BuildNotificationsTests(TestCase): @@ -71,11 +74,7 @@ def test_webhook_form_url_length(self): self.assertFalse(form.is_valid()) self.assertEqual( form.errors, - { - 'url': - [ - 'Enter a valid URL.', - 'Ensure this value has at most 600 characters (it has 1507).', - ], - }, - ) + {'url': + ['Enter a valid URL.', + 'Ensure this value has at most 600 characters (it has 1507).'] + }) diff --git a/readthedocs/rtd_tests/tests/test_builds.py b/readthedocs/rtd_tests/tests/test_builds.py index afdbe46a3c9..53e222203fe 100644 --- a/readthedocs/rtd_tests/tests/test_builds.py +++ b/readthedocs/rtd_tests/tests/test_builds.py @@ -1,7 +1,12 @@ -# -*- coding: utf-8 -*- -import os +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import mock +import os from django.test import TestCase from django_dynamic_fixture import fixture, get @@ -9,7 +14,7 @@ from readthedocs.doc_builder.config import load_yaml_config from readthedocs.doc_builder.environments import LocalBuildEnvironment from readthedocs.doc_builder.python_environments import Virtualenv -from readthedocs.projects.models import EnvironmentVariable, Project +from readthedocs.projects.models import Project, EnvironmentVariable from readthedocs.projects.tasks import UpdateDocsTaskStep from readthedocs.rtd_tests.tests.test_config_integration import create_load @@ -27,22 +32,18 @@ def tearDown(self): @mock.patch('readthedocs.doc_builder.config.load_config') def test_build(self, load_config): - """Test full build.""" + '''Test full build''' load_config.side_effect = create_load() - project = get( - Project, - slug='project-1', - documentation_type='sphinx', - conf_py_file='test_conf.py', - versions=[fixture()], - ) + project = get(Project, + slug='project-1', + documentation_type='sphinx', + conf_py_file='test_conf.py', + versions=[fixture()]) version = project.versions.all()[0] self.mocks.configure_mock('api_versions', {'return_value': [version]}) - self.mocks.configure_mock( - 'api', { - 'get.return_value': {'downloads': 'no_url_here'}, - }, - ) + self.mocks.configure_mock('api', { + 'get.return_value': {'downloads': "no_url_here"} + }) self.mocks.patches['html_build'].stop() build_env = LocalBuildEnvironment(project=project, version=version, build={}) @@ -50,7 +51,7 @@ def test_build(self, load_config): config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, - version=version, config=config, + version=version, config=config ) task.build_docs() @@ -62,17 +63,15 @@ def test_build(self, load_config): @mock.patch('readthedocs.doc_builder.config.load_config') def test_build_respects_pdf_flag(self, load_config): - """Build output format control.""" + '''Build output format control''' load_config.side_effect = create_load() - project = get( - Project, - slug='project-1', - documentation_type='sphinx', - conf_py_file='test_conf.py', - enable_pdf_build=True, - enable_epub_build=False, - versions=[fixture()], - ) + project = get(Project, + slug='project-1', + documentation_type='sphinx', + conf_py_file='test_conf.py', + enable_pdf_build=True, + enable_epub_build=False, + versions=[fixture()]) version = project.versions.all()[0] build_env = LocalBuildEnvironment(project=project, version=version, build={}) @@ -80,7 +79,7 @@ def test_build_respects_pdf_flag(self, load_config): config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, - version=version, config=config, + version=version, config=config ) task.build_docs() @@ -100,20 +99,20 @@ def test_dont_localmedia_build_pdf_epub_search_in_mkdocs(self, load_config): documentation_type='mkdocs', enable_pdf_build=True, enable_epub_build=True, - versions=[fixture()], + versions=[fixture()] ) version = project.versions.all().first() build_env = LocalBuildEnvironment( project=project, version=version, - build={}, + build={} ) python_env = Virtualenv(version=version, build_env=build_env) config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, - version=version, config=config, + version=version, config=config ) task.build_docs() @@ -127,17 +126,15 @@ def test_dont_localmedia_build_pdf_epub_search_in_mkdocs(self, load_config): @mock.patch('readthedocs.doc_builder.config.load_config') def test_build_respects_epub_flag(self, load_config): - """Test build with epub enabled.""" + '''Test build with epub enabled''' load_config.side_effect = create_load() - project = get( - Project, - slug='project-1', - documentation_type='sphinx', - conf_py_file='test_conf.py', - enable_pdf_build=False, - enable_epub_build=True, - versions=[fixture()], - ) + project = get(Project, + slug='project-1', + documentation_type='sphinx', + conf_py_file='test_conf.py', + enable_pdf_build=False, + enable_epub_build=True, + versions=[fixture()]) version = project.versions.all()[0] build_env = LocalBuildEnvironment(project=project, version=version, build={}) @@ -145,7 +142,7 @@ def test_build_respects_epub_flag(self, load_config): config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, - version=version, config=config, + version=version, config=config ) task.build_docs() @@ -157,17 +154,15 @@ def test_build_respects_epub_flag(self, load_config): @mock.patch('readthedocs.doc_builder.config.load_config') def test_build_respects_yaml(self, load_config): - """Test YAML build options.""" + '''Test YAML build options''' load_config.side_effect = create_load({'formats': ['epub']}) - project = get( - Project, - slug='project-1', - documentation_type='sphinx', - conf_py_file='test_conf.py', - enable_pdf_build=False, - enable_epub_build=False, - versions=[fixture()], - ) + project = get(Project, + slug='project-1', + documentation_type='sphinx', + conf_py_file='test_conf.py', + enable_pdf_build=False, + enable_epub_build=False, + versions=[fixture()]) version = project.versions.all()[0] build_env = LocalBuildEnvironment(project=project, version=version, build={}) @@ -176,7 +171,7 @@ def test_build_respects_yaml(self, load_config): config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, - version=version, config=config, + version=version, config=config ) task.build_docs() @@ -188,21 +183,19 @@ def test_build_respects_yaml(self, load_config): @mock.patch('readthedocs.doc_builder.config.load_config') def test_build_pdf_latex_failures(self, load_config): - """Build failure if latex fails.""" + '''Build failure if latex fails''' load_config.side_effect = create_load() self.mocks.patches['html_build'].stop() self.mocks.patches['pdf_build'].stop() - project = get( - Project, - slug='project-1', - documentation_type='sphinx', - conf_py_file='test_conf.py', - enable_pdf_build=True, - enable_epub_build=False, - versions=[fixture()], - ) + project = get(Project, + slug='project-1', + documentation_type='sphinx', + conf_py_file='test_conf.py', + enable_pdf_build=True, + enable_epub_build=False, + versions=[fixture()]) version = project.versions.all()[0] assert project.conf_dir() == '/tmp/rtd' @@ -211,7 +204,7 @@ def test_build_pdf_latex_failures(self, load_config): config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, - version=version, config=config, + version=version, config=config ) # Mock out the separate calls to Popen using an iterable side_effect @@ -223,13 +216,10 @@ def test_build_pdf_latex_failures(self, load_config): ((b'', b''), 0), # latex ] mock_obj = mock.Mock() - mock_obj.communicate.side_effect = [ - output for (output, status) - in returns - ] + mock_obj.communicate.side_effect = [output for (output, status) + in returns] type(mock_obj).returncode = mock.PropertyMock( - side_effect=[status for (output, status) in returns], - ) + side_effect=[status for (output, status) in returns]) self.mocks.popen.return_value = mock_obj with build_env: @@ -239,21 +229,19 @@ def test_build_pdf_latex_failures(self, load_config): @mock.patch('readthedocs.doc_builder.config.load_config') def test_build_pdf_latex_not_failure(self, load_config): - """Test pass during PDF builds and bad latex failure status code.""" + '''Test pass during PDF builds and bad latex failure status code''' load_config.side_effect = create_load() self.mocks.patches['html_build'].stop() self.mocks.patches['pdf_build'].stop() - project = get( - Project, - slug='project-2', - documentation_type='sphinx', - conf_py_file='test_conf.py', - enable_pdf_build=True, - enable_epub_build=False, - versions=[fixture()], - ) + project = get(Project, + slug='project-2', + documentation_type='sphinx', + conf_py_file='test_conf.py', + enable_pdf_build=True, + enable_epub_build=False, + versions=[fixture()]) version = project.versions.all()[0] assert project.conf_dir() == '/tmp/rtd' @@ -262,7 +250,7 @@ def test_build_pdf_latex_not_failure(self, load_config): config = load_yaml_config(version) task = UpdateDocsTaskStep( build_env=build_env, project=project, python_env=python_env, - version=version, config=config, + version=version, config=config ) # Mock out the separate calls to Popen using an iterable side_effect @@ -274,13 +262,10 @@ def test_build_pdf_latex_not_failure(self, load_config): ((b'', b''), 0), # latex ] mock_obj = mock.Mock() - mock_obj.communicate.side_effect = [ - output for (output, status) - in returns - ] + mock_obj.communicate.side_effect = [output for (output, status) + in returns] type(mock_obj).returncode = mock.PropertyMock( - side_effect=[status for (output, status) in returns], - ) + side_effect=[status for (output, status) in returns]) self.mocks.popen.return_value = mock_obj with build_env: @@ -301,7 +286,7 @@ def test_save_config_in_build_model(self, load_config, api_v2): build = get(Build) version = get(Version, slug='1.8', project=project) task = UpdateDocsTaskStep( - project=project, version=version, build={'id': build.pk}, + project=project, version=version, build={'id': build.pk} ) task.setup_vcs = mock.Mock() task.run_setup() @@ -373,13 +358,13 @@ def test_get_previous_build(self): Build, project=self.project, version=self.version, - config={'version': 1}, + config={'version': 1} ) build_two = get( Build, project=self.project, version=self.version, - config={'version': 2}, + config={'version': 2} ) build_three = get( Build, diff --git a/readthedocs/rtd_tests/tests/test_celery.py b/readthedocs/rtd_tests/tests/test_celery.py index 5ac9fac8883..11a9e31e46f 100644 --- a/readthedocs/rtd_tests/tests/test_celery.py +++ b/readthedocs/rtd_tests/tests/test_celery.py @@ -1,4 +1,6 @@ # -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + import os import shutil from os.path import exists @@ -6,42 +8,35 @@ from django.contrib.auth.models import User from django_dynamic_fixture import get -from mock import MagicMock, patch +from mock import patch, MagicMock from readthedocs.builds.constants import LATEST -from readthedocs.builds.models import Build -from readthedocs.doc_builder.exceptions import VersionLockedError -from readthedocs.projects import tasks from readthedocs.projects.exceptions import RepositoryError +from readthedocs.builds.models import Build from readthedocs.projects.models import Project +from readthedocs.projects import tasks + +from readthedocs.rtd_tests.utils import ( + create_git_branch, create_git_tag, delete_git_branch) +from readthedocs.rtd_tests.utils import make_test_git from readthedocs.rtd_tests.base import RTDTestCase from readthedocs.rtd_tests.mocks.mock_api import mock_api -from readthedocs.rtd_tests.utils import ( - create_git_branch, - create_git_tag, - delete_git_branch, - make_test_git, -) class TestCeleryBuilding(RTDTestCase): - """ - These tests run the build functions directly. - - They don't use celery - """ + """These tests run the build functions directly. They don't use celery""" def setUp(self): repo = make_test_git() self.repo = repo - super().setUp() + super(TestCeleryBuilding, self).setUp() self.eric = User(username='eric') self.eric.set_password('test') self.eric.save() self.project = Project.objects.create( - name='Test Project', - repo_type='git', + name="Test Project", + repo_type="git", # Our top-level checkout repo=repo, ) @@ -49,12 +44,12 @@ def setUp(self): def tearDown(self): shutil.rmtree(self.repo) - super().tearDown() + super(TestCeleryBuilding, self).tearDown() - def test_remove_dirs(self): + def test_remove_dir(self): directory = mkdtemp() self.assertTrue(exists(directory)) - result = tasks.remove_dirs.delay((directory,)) + result = tasks.remove_dir.delay(directory) self.assertTrue(result.successful()) self.assertFalse(exists(directory)) @@ -63,14 +58,14 @@ def test_clear_artifacts(self): directory = self.project.get_production_media_path(type_='pdf', version_slug=version.slug) os.makedirs(directory) self.assertTrue(exists(directory)) - result = tasks.remove_dirs.delay(paths=version.get_artifact_paths()) + result = tasks.clear_artifacts.delay(paths=version.get_artifact_paths()) self.assertTrue(result.successful()) self.assertFalse(exists(directory)) directory = version.project.rtd_build_path(version=version.slug) os.makedirs(directory) self.assertTrue(exists(directory)) - result = tasks.remove_dirs.delay(paths=version.get_artifact_paths()) + result = tasks.clear_artifacts.delay(paths=version.get_artifact_paths()) self.assertTrue(result.successful()) self.assertFalse(exists(directory)) @@ -78,17 +73,14 @@ def test_clear_artifacts(self): @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs', new=MagicMock) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_vcs', new=MagicMock) def test_update_docs(self): - build = get( - Build, project=self.project, - version=self.project.versions.first(), - ) + build = get(Build, project=self.project, + version=self.project.versions.first()) with mock_api(self.repo) as mapi: result = tasks.update_docs_task.delay( self.project.pk, build_pk=build.pk, record=False, - intersphinx=False, - ) + intersphinx=False) self.assertTrue(result.successful()) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_python_environment', new=MagicMock) @@ -98,17 +90,14 @@ def test_update_docs(self): def test_update_docs_unexpected_setup_exception(self, mock_setup_vcs): exc = Exception() mock_setup_vcs.side_effect = exc - build = get( - Build, project=self.project, - version=self.project.versions.first(), - ) + build = get(Build, project=self.project, + version=self.project.versions.first()) with mock_api(self.repo) as mapi: result = tasks.update_docs_task.delay( self.project.pk, build_pk=build.pk, record=False, - intersphinx=False, - ) + intersphinx=False) self.assertTrue(result.successful()) @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_python_environment', new=MagicMock) @@ -118,39 +107,14 @@ def test_update_docs_unexpected_setup_exception(self, mock_setup_vcs): def test_update_docs_unexpected_build_exception(self, mock_build_docs): exc = Exception() mock_build_docs.side_effect = exc - build = get( - Build, project=self.project, - version=self.project.versions.first(), - ) + build = get(Build, project=self.project, + version=self.project.versions.first()) with mock_api(self.repo) as mapi: result = tasks.update_docs_task.delay( self.project.pk, build_pk=build.pk, record=False, - intersphinx=False, - ) - self.assertTrue(result.successful()) - - @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_python_environment', new=MagicMock) - @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.build_docs', new=MagicMock) - @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.send_notifications') - @patch('readthedocs.projects.tasks.UpdateDocsTaskStep.setup_vcs') - def test_no_notification_on_version_locked_error(self, mock_setup_vcs, mock_send_notifications): - mock_setup_vcs.side_effect = VersionLockedError() - - build = get( - Build, project=self.project, - version=self.project.versions.first(), - ) - with mock_api(self.repo) as mapi: - result = tasks.update_docs_task.delay( - self.project.pk, - build_pk=build.pk, - record=False, - intersphinx=False, - ) - - mock_send_notifications.assert_not_called() + intersphinx=False) self.assertTrue(result.successful()) def test_sync_repository(self): @@ -178,7 +142,7 @@ def test_check_duplicate_reserved_version_latest(self, checkout_path, api_v2): sync_repository.sync_repo() self.assertEqual( str(e.exception), - RepositoryError.DUPLICATED_RESERVED_VERSIONS, + RepositoryError.DUPLICATED_RESERVED_VERSIONS ) delete_git_branch(self.repo, 'latest') @@ -204,7 +168,7 @@ def test_check_duplicate_reserved_version_stable(self, checkout_path, api_v2): sync_repository.sync_repo() self.assertEqual( str(e.exception), - RepositoryError.DUPLICATED_RESERVED_VERSIONS, + RepositoryError.DUPLICATED_RESERVED_VERSIONS ) # TODO: Check that we can build properly after @@ -242,11 +206,9 @@ def public_task_exception(): # although the task risen an exception, it's success since we add the # exception into the ``info`` attributes self.assertEqual(result.status, 'SUCCESS') - self.assertEqual( - result.info, { - 'task_name': 'public_task_exception', - 'context': {}, - 'public_data': {}, - 'error': 'Something bad happened', - }, - ) + self.assertEqual(result.info, { + 'task_name': 'public_task_exception', + 'context': {}, + 'public_data': {}, + 'error': 'Something bad happened', + }) diff --git a/readthedocs/rtd_tests/tests/test_config_integration.py b/readthedocs/rtd_tests/tests/test_config_integration.py index e09e82199bb..2b2c6ceb489 100644 --- a/readthedocs/rtd_tests/tests/test_config_integration.py +++ b/readthedocs/rtd_tests/tests/test_config_integration.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import tempfile from os import path @@ -85,12 +88,14 @@ def test_python_supported_versions_default_image_1_0(self, load_config): env_config={ 'allow_v2': mock.ANY, 'build': {'image': 'readthedocs/build:1.0'}, + 'output_base': '', + 'name': mock.ANY, 'defaults': { 'install_project': self.project.install_project, 'formats': [ 'htmlzip', 'epub', - 'pdf', + 'pdf' ], 'use_system_packages': self.project.use_system_packages, 'requirements_file': self.project.requirements_file, @@ -110,10 +115,8 @@ def test_python_supported_versions_image_1_0(self, load_config): self.project.container_image = 'readthedocs/build:1.0' self.project.save() config = load_yaml_config(self.version) - self.assertEqual( - config.get_valid_python_versions(), - [2, 2.7, 3, 3.4], - ) + self.assertEqual(config.get_valid_python_versions(), + [2, 2.7, 3, 3.4]) @mock.patch('readthedocs.doc_builder.config.load_config') def test_python_supported_versions_image_2_0(self, load_config): @@ -121,10 +124,8 @@ def test_python_supported_versions_image_2_0(self, load_config): self.project.container_image = 'readthedocs/build:2.0' self.project.save() config = load_yaml_config(self.version) - self.assertEqual( - config.get_valid_python_versions(), - [2, 2.7, 3, 3.5], - ) + self.assertEqual(config.get_valid_python_versions(), + [2, 2.7, 3, 3.5]) @mock.patch('readthedocs.doc_builder.config.load_config') def test_python_supported_versions_image_latest(self, load_config): @@ -132,10 +133,8 @@ def test_python_supported_versions_image_latest(self, load_config): self.project.container_image = 'readthedocs/build:latest' self.project.save() config = load_yaml_config(self.version) - self.assertEqual( - config.get_valid_python_versions(), - [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6], - ) + self.assertEqual(config.get_valid_python_versions(), + [2, 2.7, 3, 3.3, 3.4, 3.5, 3.6]) @mock.patch('readthedocs.doc_builder.config.load_config') def test_python_default_version(self, load_config): @@ -168,7 +167,7 @@ def test_python_set_python_version_in_config(self, load_config): @mock.patch('readthedocs.doc_builder.config.load_config') def test_python_invalid_version_in_config(self, load_config): load_config.side_effect = create_load({ - 'python': {'version': 2.6}, + 'python': {'version': 2.6} }) self.project.container_image = 'readthedocs/build:2.0' self.project.save() @@ -181,11 +180,11 @@ def test_install_project(self, load_config): config = load_yaml_config(self.version) self.assertEqual( config.python.install_with_pip or config.python.install_with_setup, - False, + False ) load_config.side_effect = create_load({ - 'python': {'setup_py_install': True}, + 'python': {'setup_py_install': True} }) config = load_yaml_config(self.version) self.assertEqual(config.python.install_with_setup, True) @@ -195,16 +194,16 @@ def test_extra_requirements(self, load_config): load_config.side_effect = create_load({ 'python': { 'pip_install': True, - 'extra_requirements': ['tests', 'docs'], - }, + 'extra_requirements': ['tests', 'docs'] + } }) config = load_yaml_config(self.version) self.assertEqual(config.python.extra_requirements, ['tests', 'docs']) load_config.side_effect = create_load({ 'python': { - 'extra_requirements': ['tests', 'docs'], - }, + 'extra_requirements': ['tests', 'docs'] + } }) config = load_yaml_config(self.version) self.assertEqual(config.python.extra_requirements, []) @@ -216,8 +215,8 @@ def test_extra_requirements(self, load_config): load_config.side_effect = create_load({ 'python': { 'setup_py_install': True, - 'extra_requirements': ['tests', 'docs'], - }, + 'extra_requirements': ['tests', 'docs'] + } }) config = load_yaml_config(self.version) self.assertEqual(config.python.extra_requirements, []) @@ -234,7 +233,7 @@ def test_conda_with_cofig(self, checkout_path): { 'conda': { 'file': conda_file, - }, + } }, base_path=base_path, ) @@ -289,7 +288,7 @@ def test_requirements_file_from_yml(self, checkout_path): @pytest.mark.django_db @mock.patch('readthedocs.projects.models.Project.checkout_path') -class TestLoadConfigV2: +class TestLoadConfigV2(object): @pytest.fixture(autouse=True) def create_project(self): @@ -308,11 +307,9 @@ def create_project(self): ) def create_config_file(self, tmpdir, config): - base_path = apply_fs( - tmpdir, { - 'readthedocs.yml': '', - }, - ) + base_path = apply_fs(tmpdir, { + 'readthedocs.yml': '', + }) config.setdefault('version', 2) config_file = path.join(str(base_path), 'readthedocs.yml') yaml.safe_dump(config, open(config_file, 'w')) @@ -320,7 +317,7 @@ def create_config_file(self, tmpdir, config): def get_update_docs_task(self): build_env = LocalBuildEnvironment( - self.project, self.version, record=False, + self.project, self.version, record=False ) update_docs = tasks.UpdateDocsTaskStep( @@ -349,8 +346,7 @@ def test_report_using_invalid_version(self, checkout_path, tmpdir): @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.build') @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.append_conf') def test_build_formats_default_empty( - self, append_conf, html_build, checkout_path, config, tmpdir, - ): + self, append_conf, html_build, checkout_path, config, tmpdir): """ The default value for formats is [], which means no extra formats are build. @@ -362,7 +358,7 @@ def test_build_formats_default_empty( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=update_docs.config, + config=update_docs.config ) update_docs.python_env = python_env outcomes = update_docs.build_docs() @@ -379,9 +375,10 @@ def test_build_formats_default_empty( @patch('readthedocs.doc_builder.backends.sphinx.HtmlBuilder.append_conf') def test_build_formats_only_pdf( self, append_conf, html_build, build_docs_class, - checkout_path, tmpdir, - ): - """Only the pdf format is build.""" + checkout_path, tmpdir): + """ + Only the pdf format is build. + """ checkout_path.return_value = str(tmpdir) self.create_config_file(tmpdir, {'formats': ['pdf']}) @@ -389,7 +386,7 @@ def test_build_formats_only_pdf( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=update_docs.config, + config=update_docs.config ) update_docs.python_env = python_env @@ -413,8 +410,8 @@ def test_conda_environment(self, build_failed, checkout_path, tmpdir): base_path = self.create_config_file( tmpdir, { - 'conda': {'environment': conda_file}, - }, + 'conda': {'environment': conda_file} + } ) update_docs = self.get_update_docs_task() @@ -471,8 +468,8 @@ def test_python_requirements(self, run, checkout_path, tmpdir): base_path = self.create_config_file( tmpdir, { - 'python': {'requirements': requirements_file}, - }, + 'python': {'requirements': requirements_file} + } ) update_docs = self.get_update_docs_task() @@ -481,7 +478,7 @@ def test_python_requirements(self, run, checkout_path, tmpdir): python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env update_docs.python_env.install_user_requirements() @@ -498,8 +495,8 @@ def test_python_requirements_empty(self, run, checkout_path, tmpdir): self.create_config_file( tmpdir, { - 'python': {'requirements': ''}, - }, + 'python': {'requirements': ''} + } ) update_docs = self.get_update_docs_task() @@ -508,7 +505,7 @@ def test_python_requirements_empty(self, run, checkout_path, tmpdir): python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env update_docs.python_env.install_user_requirements() @@ -522,8 +519,8 @@ def test_python_install_setup(self, run, checkout_path, tmpdir): self.create_config_file( tmpdir, { - 'python': {'install': 'setup.py'}, - }, + 'python': {'install': 'setup.py'} + } ) update_docs = self.get_update_docs_task() @@ -532,7 +529,7 @@ def test_python_install_setup(self, run, checkout_path, tmpdir): python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env update_docs.python_env.install_package() @@ -550,8 +547,8 @@ def test_python_install_pip(self, run, checkout_path, tmpdir): self.create_config_file( tmpdir, { - 'python': {'install': 'pip'}, - }, + 'python': {'install': 'pip'} + } ) update_docs = self.get_update_docs_task() @@ -560,7 +557,7 @@ def test_python_install_pip(self, run, checkout_path, tmpdir): python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env update_docs.python_env.install_package() @@ -593,8 +590,8 @@ def test_python_extra_requirements(self, run, checkout_path, tmpdir): 'python': { 'install': 'pip', 'extra_requirements': ['docs'], - }, - }, + } + } ) update_docs = self.get_update_docs_task() @@ -603,7 +600,7 @@ def test_python_extra_requirements(self, run, checkout_path, tmpdir): python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env update_docs.python_env.install_package() @@ -624,8 +621,8 @@ def test_system_packages(self, run, checkout_path, tmpdir): { 'python': { 'system_packages': True, - }, - }, + } + } ) update_docs = self.get_update_docs_task() @@ -634,7 +631,7 @@ def test_system_packages(self, run, checkout_path, tmpdir): python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env update_docs.python_env.setup_base() @@ -644,18 +641,13 @@ def test_system_packages(self, run, checkout_path, tmpdir): assert '--system-site-packages' in args assert config.python.use_system_site_packages - @pytest.mark.parametrize( - 'value,result', - [ - ('html', 'sphinx'), - ('htmldir', 'sphinx_htmldir'), - ('singlehtml', 'sphinx_singlehtml'), - ], - ) + @pytest.mark.parametrize('value,result', + [('html', 'sphinx'), + ('htmldir', 'sphinx_htmldir'), + ('singlehtml', 'sphinx_singlehtml')]) @patch('readthedocs.projects.tasks.get_builder_class') def test_sphinx_builder( - self, get_builder_class, checkout_path, value, result, tmpdir, - ): + self, get_builder_class, checkout_path, value, result, tmpdir): checkout_path.return_value = str(tmpdir) self.create_config_file(tmpdir, {'sphinx': {'builder': value}}) @@ -668,12 +660,10 @@ def test_sphinx_builder( get_builder_class.assert_called_with(result) @pytest.mark.skip( - 'This test is not compatible with the new validation around doctype.', - ) + 'This test is not compatible with the new validation around doctype.') @patch('readthedocs.projects.tasks.get_builder_class') def test_sphinx_builder_default( - self, get_builder_class, checkout_path, tmpdir, - ): + self, get_builder_class, checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) self.create_config_file(tmpdir, {}) @@ -689,8 +679,7 @@ def test_sphinx_builder_default( @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') def test_sphinx_configuration_default( - self, run, append_conf, move, checkout_path, tmpdir, - ): + self, run, append_conf, move, checkout_path, tmpdir): """Should be default to find a conf.py file.""" checkout_path.return_value = str(tmpdir) @@ -704,7 +693,7 @@ def test_sphinx_configuration_default( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env @@ -719,8 +708,7 @@ def test_sphinx_configuration_default( @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') def test_sphinx_configuration_default( - self, run, append_conf, move, checkout_path, tmpdir, - ): + self, run, append_conf, move, checkout_path, tmpdir): """Should be default to find a conf.py file.""" checkout_path.return_value = str(tmpdir) @@ -734,7 +722,7 @@ def test_sphinx_configuration_default( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env @@ -749,24 +737,21 @@ def test_sphinx_configuration_default( @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') def test_sphinx_configuration( - self, run, append_conf, move, checkout_path, tmpdir, - ): + self, run, append_conf, move, checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) - apply_fs( - tmpdir, { + apply_fs(tmpdir, { + 'conf.py': '', + 'docx': { 'conf.py': '', - 'docx': { - 'conf.py': '', - }, }, - ) + }) self.create_config_file( tmpdir, { 'sphinx': { 'configuration': 'docx/conf.py', }, - }, + } ) update_docs = self.get_update_docs_task() @@ -774,7 +759,7 @@ def test_sphinx_configuration( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env @@ -789,16 +774,13 @@ def test_sphinx_configuration( @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.append_conf') @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.run') def test_sphinx_fail_on_warning( - self, run, append_conf, move, checkout_path, tmpdir, - ): + self, run, append_conf, move, checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) - apply_fs( - tmpdir, { - 'docx': { - 'conf.py': '', - }, + apply_fs(tmpdir, { + 'docx': { + 'conf.py': '', }, - ) + }) self.create_config_file( tmpdir, { @@ -806,7 +788,7 @@ def test_sphinx_fail_on_warning( 'configuration': 'docx/conf.py', 'fail_on_warning': True, }, - }, + } ) update_docs = self.get_update_docs_task() @@ -814,7 +796,7 @@ def test_sphinx_fail_on_warning( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env @@ -829,24 +811,21 @@ def test_sphinx_fail_on_warning( @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.run') def test_mkdocs_configuration( - self, run, append_conf, move, checkout_path, tmpdir, - ): + self, run, append_conf, move, checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) - apply_fs( - tmpdir, { + apply_fs(tmpdir, { + 'mkdocs.yml': '', + 'docx': { 'mkdocs.yml': '', - 'docx': { - 'mkdocs.yml': '', - }, }, - ) + }) self.create_config_file( tmpdir, { 'mkdocs': { 'configuration': 'docx/mkdocs.yml', }, - }, + } ) self.project.documentation_type = 'mkdocs' self.project.save() @@ -856,7 +835,7 @@ def test_mkdocs_configuration( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env @@ -872,16 +851,13 @@ def test_mkdocs_configuration( @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.append_conf') @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.run') def test_mkdocs_fail_on_warning( - self, run, append_conf, move, checkout_path, tmpdir, - ): + self, run, append_conf, move, checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) - apply_fs( - tmpdir, { - 'docx': { - 'mkdocs.yml': '', - }, + apply_fs(tmpdir, { + 'docx': { + 'mkdocs.yml': '', }, - ) + }) self.create_config_file( tmpdir, { @@ -889,7 +865,7 @@ def test_mkdocs_fail_on_warning( 'configuration': 'docx/mkdocs.yml', 'fail_on_warning': True, }, - }, + } ) self.project.documentation_type = 'mkdocs' self.project.save() @@ -899,7 +875,7 @@ def test_mkdocs_fail_on_warning( python_env = Virtualenv( version=self.version, build_env=update_docs.build_env, - config=config, + config=config ) update_docs.python_env = python_env @@ -910,17 +886,11 @@ def test_mkdocs_fail_on_warning( append_conf.assert_called_once() move.assert_called_once() - @pytest.mark.parametrize( - 'value,expected', [ - (ALL, ['one', 'two', 'three']), - (['one', 'two'], ['one', 'two']), - ], - ) + @pytest.mark.parametrize('value,expected', [(ALL, ['one', 'two', 'three']), + (['one', 'two'], ['one', 'two'])]) @patch('readthedocs.vcs_support.backends.git.Backend.checkout_submodules') - def test_submodules_include( - self, checkout_submodules, - checkout_path, tmpdir, value, expected, - ): + def test_submodules_include(self, checkout_submodules, + checkout_path, tmpdir, value, expected): checkout_path.return_value = str(tmpdir) self.create_config_file( tmpdir, @@ -928,7 +898,7 @@ def test_submodules_include( 'submodules': { 'include': value, }, - }, + } ) git_repo = make_git_repo(str(tmpdir)) @@ -945,10 +915,8 @@ def test_submodules_include( assert update_docs.config.submodules.recursive is False @patch('readthedocs.vcs_support.backends.git.Backend.checkout_submodules') - def test_submodules_exclude( - self, checkout_submodules, - checkout_path, tmpdir, - ): + def test_submodules_exclude(self, checkout_submodules, + checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) self.create_config_file( tmpdir, @@ -957,7 +925,7 @@ def test_submodules_exclude( 'exclude': ['one'], 'recursive': True, }, - }, + } ) git_repo = make_git_repo(str(tmpdir)) @@ -974,10 +942,8 @@ def test_submodules_exclude( assert update_docs.config.submodules.recursive is True @patch('readthedocs.vcs_support.backends.git.Backend.checkout_submodules') - def test_submodules_exclude_all( - self, checkout_submodules, - checkout_path, tmpdir, - ): + def test_submodules_exclude_all(self, checkout_submodules, + checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) self.create_config_file( tmpdir, @@ -986,7 +952,7 @@ def test_submodules_exclude_all( 'exclude': ALL, 'recursive': True, }, - }, + } ) git_repo = make_git_repo(str(tmpdir)) @@ -1001,15 +967,13 @@ def test_submodules_exclude_all( checkout_submodules.assert_not_called() @patch('readthedocs.vcs_support.backends.git.Backend.checkout_submodules') - def test_submodules_default_exclude_all( - self, checkout_submodules, - checkout_path, tmpdir, - ): + def test_submodules_default_exclude_all(self, checkout_submodules, + checkout_path, tmpdir): checkout_path.return_value = str(tmpdir) self.create_config_file( tmpdir, - {}, + {} ) git_repo = make_git_repo(str(tmpdir)) diff --git a/readthedocs/rtd_tests/tests/test_core_tags.py b/readthedocs/rtd_tests/tests/test_core_tags.py index d119a5d5bbe..851fd40d1b6 100644 --- a/readthedocs/rtd_tests/tests/test_core_tags.py +++ b/readthedocs/rtd_tests/tests/test_core_tags.py @@ -1,18 +1,20 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import mock import pytest + from django.conf import settings from django.test import TestCase from django.test.utils import override_settings +from readthedocs.projects.models import Project from readthedocs.builds.constants import LATEST from readthedocs.core.templatetags import core_tags -from readthedocs.projects.models import Project @override_settings(USE_SUBDOMAIN=False, PRODUCTION_DOMAIN='readthedocs.org') class CoreTagsTests(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ["eric", "test_data"] def setUp(self): url_base = '{scheme}://{domain}/docs/pip{{version}}'.format( @@ -36,7 +38,7 @@ def setUp(self): with mock.patch('readthedocs.projects.models.broadcast'): self.client.login(username='eric', password='test') self.pip = Project.objects.get(slug='pip') - self.pip_fr = Project.objects.create(name='PIP-FR', slug='pip-fr', language='fr', main_language_project=self.pip) + self.pip_fr = Project.objects.create(name="PIP-FR", slug='pip-fr', language='fr', main_language_project=self.pip) def test_project_only(self): proj = Project.objects.get(slug='pip') diff --git a/readthedocs/rtd_tests/tests/test_core_utils.py b/readthedocs/rtd_tests/tests/test_core_utils.py index e57b46c0b17..85e7d735507 100644 --- a/readthedocs/rtd_tests/tests/test_core_utils.py +++ b/readthedocs/rtd_tests/tests/test_core_utils.py @@ -1,13 +1,15 @@ # -*- coding: utf-8 -*- -"""Test core util functions.""" +"""Test core util functions""" +from __future__ import absolute_import import mock -from django.test import TestCase + from django_dynamic_fixture import get +from django.test import TestCase -from readthedocs.builds.models import Version -from readthedocs.core.utils import slugify, trigger_build from readthedocs.projects.models import Project +from readthedocs.builds.models import Version +from readthedocs.core.utils import trigger_build, slugify class CoreUtilTests(TestCase): @@ -30,7 +32,7 @@ def test_trigger_skipped_project(self, update_docs_task): @mock.patch('readthedocs.projects.tasks.update_docs_task') def test_trigger_custom_queue(self, update_docs): - """Use a custom queue when routing the task.""" + """Use a custom queue when routing the task""" self.project.build_queue = 'build03' trigger_build(project=self.project, version=self.version) kwargs = { @@ -56,7 +58,7 @@ def test_trigger_custom_queue(self, update_docs): @mock.patch('readthedocs.projects.tasks.update_docs_task') def test_trigger_build_time_limit(self, update_docs): - """Pass of time limit.""" + """Pass of time limit""" trigger_build(project=self.project, version=self.version) kwargs = { 'version_pk': self.version.pk, @@ -81,7 +83,7 @@ def test_trigger_build_time_limit(self, update_docs): @mock.patch('readthedocs.projects.tasks.update_docs_task') def test_trigger_build_invalid_time_limit(self, update_docs): - """Time limit as string.""" + """Time limit as string""" self.project.container_time_limit = '200s' trigger_build(project=self.project, version=self.version) kwargs = { @@ -107,7 +109,7 @@ def test_trigger_build_invalid_time_limit(self, update_docs): @mock.patch('readthedocs.projects.tasks.update_docs_task') def test_trigger_build_rounded_time_limit(self, update_docs): - """Time limit should round down.""" + """Time limit should round down""" self.project.container_time_limit = 3 trigger_build(project=self.project, version=self.version) kwargs = { @@ -132,24 +134,14 @@ def test_trigger_build_rounded_time_limit(self, update_docs): update_docs.signature().apply_async.assert_called() def test_slugify(self): - """Test additional slugify.""" - self.assertEqual( - slugify('This is a test'), - 'this-is-a-test', - ) - self.assertEqual( - slugify('project_with_underscores-v.1.0'), - 'project-with-underscores-v10', - ) - self.assertEqual( - slugify('project_with_underscores-v.1.0', dns_safe=False), - 'project_with_underscores-v10', - ) - self.assertEqual( - slugify('A title_-_with separated parts'), - 'a-title-with-separated-parts', - ) - self.assertEqual( - slugify('A title_-_with separated parts', dns_safe=False), - 'a-title_-_with-separated-parts', - ) + """Test additional slugify""" + self.assertEqual(slugify('This is a test'), + 'this-is-a-test') + self.assertEqual(slugify('project_with_underscores-v.1.0'), + 'project-with-underscores-v10') + self.assertEqual(slugify('project_with_underscores-v.1.0', dns_safe=False), + 'project_with_underscores-v10') + self.assertEqual(slugify('A title_-_with separated parts'), + 'a-title-with-separated-parts') + self.assertEqual(slugify('A title_-_with separated parts', dns_safe=False), + 'a-title_-_with-separated-parts') diff --git a/readthedocs/rtd_tests/tests/test_doc_builder.py b/readthedocs/rtd_tests/tests/test_doc_builder.py index 57bd6a15460..a384f5d00b7 100644 --- a/readthedocs/rtd_tests/tests/test_doc_builder.py +++ b/readthedocs/rtd_tests/tests/test_doc_builder.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import os import tempfile from collections import namedtuple @@ -15,7 +18,6 @@ from readthedocs.builds.models import Version from readthedocs.doc_builder.backends.mkdocs import MkdocsHTML from readthedocs.doc_builder.backends.sphinx import BaseSphinx -from readthedocs.doc_builder.exceptions import MkDocsYAMLParseError from readthedocs.doc_builder.python_environments import Virtualenv from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Feature, Project @@ -61,12 +63,12 @@ def test_conf_py_path(self, checkout_path, docs_dir): for value, expected in (('conf.py', '/'), ('docs/conf.py', '/docs/')): base_sphinx.config_file = os.path.join( - tmp_dir, value, + tmp_dir, value ) params = base_sphinx.get_config_params() self.assertEqual( params['conf_py_path'], - expected, + expected ) @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.docs_dir') @@ -77,8 +79,7 @@ def test_conf_py_path(self, checkout_path, docs_dir): @patch('readthedocs.projects.models.Project.checkout_path') def test_create_conf_py( self, checkout_path, get_conf_py_path, _, - get_config_params, create_index, docs_dir, - ): + get_config_params, create_index, docs_dir): """ Test for a project without ``conf.py`` file. @@ -117,13 +118,13 @@ def test_create_conf_py( os.path.dirname(__file__), '..', 'files', - 'conf.py', + 'conf.py' ) with open(generated_conf_py) as gf, open(expected_conf_py) as ef: autogenerated_confpy_lines = 28 self.assertEqual( gf.readlines()[:autogenerated_confpy_lines], - ef.readlines()[:autogenerated_confpy_lines], + ef.readlines()[:autogenerated_confpy_lines] ) @patch('readthedocs.doc_builder.backends.sphinx.BaseSphinx.docs_dir') @@ -134,8 +135,7 @@ def test_create_conf_py( @patch('readthedocs.projects.models.Project.checkout_path') def test_multiple_conf_py( self, checkout_path, get_conf_py_path, _, get_config_params, - create_index, docs_dir, - ): + create_index, docs_dir): """ Test for a project with multiple ``conf.py`` files. @@ -306,14 +306,14 @@ def test_append_conf_create_yaml(self, checkout_path, run): config = yaml.safe_load(open(generated_yaml)) self.assertEqual( config['docs_dir'], - os.path.join(tmpdir, 'docs'), + os.path.join(tmpdir, 'docs') ) self.assertEqual( config['extra_css'], [ 'http://readthedocs.org/static/css/badge_only.css', - 'http://readthedocs.org/static/css/readthedocs-doc-embed.css', - ], + 'http://readthedocs.org/static/css/readthedocs-doc-embed.css' + ] ) self.assertEqual( config['extra_javascript'], @@ -321,14 +321,14 @@ def test_append_conf_create_yaml(self, checkout_path, run): 'readthedocs-data.js', 'http://readthedocs.org/static/core/js/readthedocs-doc-embed.js', 'http://readthedocs.org/static/javascript/readthedocs-analytics.js', - ], + ] ) self.assertIsNone( config['google_analytics'], ) self.assertEqual( config['site_name'], - 'mkdocs', + 'mkdocs' ) @patch('readthedocs.doc_builder.base.BaseBuilder.run') @@ -343,7 +343,7 @@ def test_append_conf_existing_yaml_on_root(self, checkout_path, run): 'google_analytics': ['UA-1234-5', 'mkdocs.org'], 'docs_dir': 'docs', }, - open(yaml_file, 'w'), + open(yaml_file, 'w') ) checkout_path.return_value = tmpdir @@ -363,14 +363,14 @@ def test_append_conf_existing_yaml_on_root(self, checkout_path, run): config = yaml.safe_load(open(yaml_file)) self.assertEqual( config['docs_dir'], - 'docs', + 'docs' ) self.assertEqual( config['extra_css'], [ 'http://readthedocs.org/static/css/badge_only.css', - 'http://readthedocs.org/static/css/readthedocs-doc-embed.css', - ], + 'http://readthedocs.org/static/css/readthedocs-doc-embed.css' + ] ) self.assertEqual( config['extra_javascript'], @@ -378,49 +378,16 @@ def test_append_conf_existing_yaml_on_root(self, checkout_path, run): 'readthedocs-data.js', 'http://readthedocs.org/static/core/js/readthedocs-doc-embed.js', 'http://readthedocs.org/static/javascript/readthedocs-analytics.js', - ], + ] ) self.assertIsNone( config['google_analytics'], ) self.assertEqual( config['site_name'], - 'mkdocs', + 'mkdocs' ) - @patch('readthedocs.doc_builder.base.BaseBuilder.run') - @patch('readthedocs.projects.models.Project.checkout_path') - def test_append_conf_existing_yaml_on_root_with_invalid_setting(self, checkout_path, run): - tmpdir = tempfile.mkdtemp() - os.mkdir(os.path.join(tmpdir, 'docs')) - yaml_file = os.path.join(tmpdir, 'mkdocs.yml') - checkout_path.return_value = tmpdir - - python_env = Virtualenv( - version=self.version, - build_env=self.build_env, - config=None, - ) - self.searchbuilder = MkdocsHTML( - build_env=self.build_env, - python_env=python_env, - ) - - # We can't use ``@pytest.mark.parametrize`` on a Django test case - yaml_contents = [ - {'docs_dir': ['docs']}, - {'extra_css': 'a string here'}, - {'extra_javascript': None}, - ] - for content in yaml_contents: - yaml.safe_dump( - content, - open(yaml_file, 'w'), - ) - with self.assertRaises(MkDocsYAMLParseError): - self.searchbuilder.append_conf() - - @patch('readthedocs.doc_builder.base.BaseBuilder.run') @patch('readthedocs.projects.models.Project.checkout_path') def test_dont_override_theme(self, checkout_path, run): @@ -434,7 +401,7 @@ def test_dont_override_theme(self, checkout_path, run): 'site_name': 'mkdocs', 'docs_dir': 'docs', }, - open(yaml_file, 'w'), + open(yaml_file, 'w') ) checkout_path.return_value = tmpdir @@ -454,7 +421,7 @@ def test_dont_override_theme(self, checkout_path, run): config = yaml.safe_load(open(yaml_file)) self.assertEqual( config['theme_dir'], - 'not-readthedocs', + 'not-readthedocs' ) @patch('readthedocs.doc_builder.backends.mkdocs.BaseMkdocs.generate_rtd_data') @@ -469,7 +436,7 @@ def test_write_js_data_docs_dir(self, checkout_path, run, generate_rtd_data): 'site_name': 'mkdocs', 'docs_dir': 'docs', }, - open(yaml_file, 'w'), + open(yaml_file, 'w') ) checkout_path.return_value = tmpdir generate_rtd_data.return_value = '' @@ -487,5 +454,5 @@ def test_write_js_data_docs_dir(self, checkout_path, run, generate_rtd_data): generate_rtd_data.assert_called_with( docs_dir='docs', - mkdocs_config=mock.ANY, + mkdocs_config=mock.ANY ) diff --git a/readthedocs/rtd_tests/tests/test_doc_building.py b/readthedocs/rtd_tests/tests/test_doc_building.py index 25ce88bcb45..6a954de74d6 100644 --- a/readthedocs/rtd_tests/tests/test_doc_building.py +++ b/readthedocs/rtd_tests/tests/test_doc_building.py @@ -5,6 +5,13 @@ * raw subprocess calls like .communicate expects bytes * the Command wrappers encapsulate the bytes and expose unicode """ +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import json import os import re @@ -13,6 +20,7 @@ import mock import pytest +from builtins import str from django.test import TestCase from django_dynamic_fixture import get from docker.errors import APIError as DockerAPIError @@ -37,7 +45,7 @@ DUMMY_BUILD_ID = 123 -SAMPLE_UNICODE = 'HérÉ îß sömê ünïçó∂é' +SAMPLE_UNICODE = u'HérÉ îß sömê ünïçó∂é' SAMPLE_UTF8_BYTES = SAMPLE_UNICODE.encode('utf-8') @@ -58,11 +66,9 @@ def tearDown(self): def test_normal_execution(self): """Normal build in passing state.""" - self.mocks.configure_mock( - 'process', { - 'communicate.return_value': (b'This is okay', ''), - }, - ) + self.mocks.configure_mock('process', { + 'communicate.return_value': (b'This is okay', '') + }) type(self.mocks.process).returncode = PropertyMock(return_value=0) build_env = LocalBuildEnvironment( @@ -77,7 +83,7 @@ def test_normal_execution(self): self.assertTrue(build_env.done) self.assertTrue(build_env.successful) self.assertEqual(len(build_env.commands), 1) - self.assertEqual(build_env.commands[0].output, 'This is okay') + self.assertEqual(build_env.commands[0].output, u'This is okay') # api() is not called anymore, we use api_v2 instead self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called) @@ -97,7 +103,7 @@ def test_normal_execution(self): 'version': self.version.pk, 'success': True, 'project': self.project.pk, - 'setup_error': '', + 'setup_error': u'', 'length': mock.ANY, 'error': '', 'setup': '', @@ -109,11 +115,9 @@ def test_normal_execution(self): def test_command_not_recorded(self): """Normal build in passing state with no command recorded.""" - self.mocks.configure_mock( - 'process', { - 'communicate.return_value': (b'This is okay', ''), - }, - ) + self.mocks.configure_mock('process', { + 'communicate.return_value': (b'This is okay', '') + }) type(self.mocks.process).returncode = PropertyMock(return_value=0) build_env = LocalBuildEnvironment( @@ -148,11 +152,9 @@ def test_command_not_recorded(self): }) def test_record_command_as_success(self): - self.mocks.configure_mock( - 'process', { - 'communicate.return_value': (b'This is okay', ''), - }, - ) + self.mocks.configure_mock('process', { + 'communicate.return_value': (b'This is okay', '') + }) type(self.mocks.process).returncode = PropertyMock(return_value=1) build_env = LocalBuildEnvironment( @@ -167,7 +169,7 @@ def test_record_command_as_success(self): self.assertTrue(build_env.done) self.assertTrue(build_env.successful) self.assertEqual(len(build_env.commands), 1) - self.assertEqual(build_env.commands[0].output, 'This is okay') + self.assertEqual(build_env.commands[0].output, u'This is okay') # api() is not called anymore, we use api_v2 instead self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called) @@ -187,7 +189,7 @@ def test_record_command_as_success(self): 'version': self.version.pk, 'success': True, 'project': self.project.pk, - 'setup_error': '', + 'setup_error': u'', 'length': mock.ANY, 'error': '', 'setup': '', @@ -234,11 +236,9 @@ def test_incremental_state_update_with_no_update(self): def test_failing_execution(self): """Build in failing state.""" - self.mocks.configure_mock( - 'process', { - 'communicate.return_value': (b'This is not okay', ''), - }, - ) + self.mocks.configure_mock('process', { + 'communicate.return_value': (b'This is not okay', '') + }) type(self.mocks.process).returncode = PropertyMock(return_value=1) build_env = LocalBuildEnvironment( @@ -254,7 +254,7 @@ def test_failing_execution(self): self.assertTrue(build_env.done) self.assertTrue(build_env.failed) self.assertEqual(len(build_env.commands), 1) - self.assertEqual(build_env.commands[0].output, 'This is not okay') + self.assertEqual(build_env.commands[0].output, u'This is not okay') # api() is not called anymore, we use api_v2 instead self.assertFalse(self.mocks.api()(DUMMY_BUILD_ID).put.called) @@ -274,7 +274,7 @@ def test_failing_execution(self): 'version': self.version.pk, 'success': False, 'project': self.project.pk, - 'setup_error': '', + 'setup_error': u'', 'length': mock.ANY, 'error': '', 'setup': '', @@ -513,10 +513,8 @@ def test_api_failure(self): 'docker_client', { 'create_container.side_effect': DockerAPIError( 'Failure creating container', response, - 'Failure creating container', - ), - }, - ) + 'Failure creating container') + }) build_env = DockerBuildEnvironment( version=self.version, @@ -539,13 +537,13 @@ def _inner(): 'version': self.version.pk, 'success': False, 'project': self.project.pk, - 'setup_error': '', + 'setup_error': u'', 'exit_code': 1, 'length': mock.ANY, 'error': 'Build environment creation failed', - 'setup': '', - 'output': '', - 'state': 'finished', + 'setup': u'', + 'output': u'', + 'state': u'finished', 'builder': mock.ANY, }) @@ -556,10 +554,8 @@ def test_api_failure_on_docker_memory_limit(self): 'docker_client', { 'exec_create.side_effect': DockerAPIError( 'Failure creating container', response, - 'Failure creating container', - ), - }, - ) + 'Failure creating container'), + }) build_env = DockerBuildEnvironment( version=self.version, @@ -592,23 +588,21 @@ def test_api_failure_on_docker_memory_limit(self): 'version': self.version.pk, 'success': False, 'project': self.project.pk, - 'setup_error': '', + 'setup_error': u'', 'exit_code': -1, 'length': mock.ANY, 'error': '', - 'setup': '', - 'output': '', - 'state': 'finished', + 'setup': u'', + 'output': u'', + 'state': u'finished', 'builder': mock.ANY, }) def test_api_failure_on_error_in_exit(self): response = Mock(status_code=500, reason='Internal Server Error') - self.mocks.configure_mock( - 'docker_client', { - 'kill.side_effect': BuildEnvironmentError('Failed'), - }, - ) + self.mocks.configure_mock('docker_client', { + 'kill.side_effect': BuildEnvironmentError('Failed') + }) build_env = DockerBuildEnvironment( version=self.version, @@ -646,11 +640,9 @@ def test_api_failure_returns_previous_error_on_error_in_exit(self): usable error to show the user. """ response = Mock(status_code=500, reason='Internal Server Error') - self.mocks.configure_mock( - 'docker_client', { - 'kill.side_effect': BuildEnvironmentError('Outer failed'), - }, - ) + self.mocks.configure_mock('docker_client', { + 'kill.side_effect': BuildEnvironmentError('Outer failed') + }) build_env = DockerBuildEnvironment( version=self.version, @@ -687,8 +679,7 @@ def test_command_execution(self): 'exec_create.return_value': {'Id': b'container-foobar'}, 'exec_start.return_value': b'This is the return', 'exec_inspect.return_value': {'ExitCode': 1}, - }, - ) + }) build_env = DockerBuildEnvironment( version=self.version, @@ -701,10 +692,9 @@ def test_command_execution(self): self.mocks.docker_client.exec_create.assert_called_with( container='build-123-project-6-pip', - cmd="/bin/sh -c 'cd /tmp && echo\\ test'", stderr=True, stdout=True, - ) + cmd="/bin/sh -c 'cd /tmp && echo\\ test'", stderr=True, stdout=True) self.assertEqual(build_env.commands[0].exit_code, 1) - self.assertEqual(build_env.commands[0].output, 'This is the return') + self.assertEqual(build_env.commands[0].output, u'This is the return') self.assertEqual(build_env.commands[0].error, None) self.assertTrue(build_env.failed) @@ -743,8 +733,7 @@ def test_command_not_recorded(self): 'exec_create.return_value': {'Id': b'container-foobar'}, 'exec_start.return_value': b'This is the return', 'exec_inspect.return_value': {'ExitCode': 1}, - }, - ) + }) build_env = DockerBuildEnvironment( version=self.version, @@ -757,8 +746,7 @@ def test_command_not_recorded(self): self.mocks.docker_client.exec_create.assert_called_with( container='build-123-project-6-pip', - cmd="/bin/sh -c 'cd /tmp && echo\\ test'", stderr=True, stdout=True, - ) + cmd="/bin/sh -c 'cd /tmp && echo\\ test'", stderr=True, stdout=True) self.assertEqual(len(build_env.commands), 0) self.assertFalse(build_env.failed) @@ -786,8 +774,7 @@ def test_record_command_as_success(self): 'exec_create.return_value': {'Id': b'container-foobar'}, 'exec_start.return_value': b'This is the return', 'exec_inspect.return_value': {'ExitCode': 1}, - }, - ) + }) build_env = DockerBuildEnvironment( version=self.version, @@ -800,10 +787,9 @@ def test_record_command_as_success(self): self.mocks.docker_client.exec_create.assert_called_with( container='build-123-project-6-pip', - cmd="/bin/sh -c 'cd /tmp && echo\\ test'", stderr=True, stdout=True, - ) + cmd="/bin/sh -c 'cd /tmp && echo\\ test'", stderr=True, stdout=True) self.assertEqual(build_env.commands[0].exit_code, 0) - self.assertEqual(build_env.commands[0].output, 'This is the return') + self.assertEqual(build_env.commands[0].output, u'This is the return') self.assertEqual(build_env.commands[0].error, None) self.assertFalse(build_env.failed) @@ -847,9 +833,8 @@ def test_command_execution_cleanup_exception(self): 'Failure killing container', response, 'Failure killing container', - ), - }, - ) + ) + }) build_env = DockerBuildEnvironment( version=self.version, @@ -860,8 +845,7 @@ def test_command_execution_cleanup_exception(self): build_env.run('echo', 'test', cwd='/tmp') self.mocks.docker_client.kill.assert_called_with( - 'build-123-project-6-pip', - ) + 'build-123-project-6-pip') self.assertTrue(build_env.successful) # api() is not called anymore, we use api_v2 instead @@ -883,12 +867,12 @@ def test_command_execution_cleanup_exception(self): 'error': '', 'success': True, 'project': self.project.pk, - 'setup_error': '', + 'setup_error': u'', 'exit_code': 0, 'length': 0, - 'setup': '', - 'output': '', - 'state': 'finished', + 'setup': u'', + 'output': u'', + 'state': u'finished', 'builder': mock.ANY, }) @@ -900,8 +884,7 @@ def test_container_already_exists(self): 'exec_create.return_value': {'Id': b'container-foobar'}, 'exec_start.return_value': b'This is the return', 'exec_inspect.return_value': {'ExitCode': 0}, - }, - ) + }) build_env = DockerBuildEnvironment( version=self.version, @@ -916,8 +899,7 @@ def _inner(): self.assertRaises(BuildEnvironmentError, _inner) self.assertEqual( str(build_env.failure), - 'A build environment is currently running for this version', - ) + 'A build environment is currently running for this version') self.assertEqual(self.mocks.docker_client.exec_create.call_count, 0) self.assertTrue(build_env.failed) @@ -956,8 +938,7 @@ def test_container_timeout(self): 'exec_create.return_value': {'Id': b'container-foobar'}, 'exec_start.return_value': b'This is the return', 'exec_inspect.return_value': {'ExitCode': 0}, - }, - ) + }) build_env = DockerBuildEnvironment( version=self.version, @@ -989,13 +970,13 @@ def test_container_timeout(self): 'version': self.version.pk, 'success': False, 'project': self.project.pk, - 'setup_error': '', + 'setup_error': u'', 'exit_code': 1, 'length': 0, 'error': 'Build exited due to time out', - 'setup': '', - 'output': '', - 'state': 'finished', + 'setup': u'', + 'output': u'', + 'state': u'finished', 'builder': mock.ANY, }) @@ -1059,10 +1040,8 @@ def test_error_output(self): self.assertEqual(cmd.output, 'FOOBAR') self.assertIsNone(cmd.error) # Test non-combined streams - cmd = BuildCommand( - ['/bin/bash', '-c', 'echo -n FOOBAR 1>&2'], - combine_output=False, - ) + cmd = BuildCommand(['/bin/bash', '-c', 'echo -n FOOBAR 1>&2'], + combine_output=False) cmd.run() self.assertEqual(cmd.output, '') self.assertEqual(cmd.error, 'FOOBAR') @@ -1089,8 +1068,7 @@ def test_unicode_output(self, mock_subprocess): cmd.run() self.assertEqual( cmd.output, - 'H\xe9r\xc9 \xee\xdf s\xf6m\xea \xfcn\xef\xe7\xf3\u2202\xe9', - ) + u'H\xe9r\xc9 \xee\xdf s\xf6m\xea \xfcn\xef\xe7\xf3\u2202\xe9') class TestDockerBuildCommand(TestCase): @@ -1106,10 +1084,8 @@ def tearDown(self): def test_wrapped_command(self): """Test shell wrapping for Docker chdir.""" - cmd = DockerBuildCommand( - ['pip', 'install', 'requests'], - cwd='/tmp/foobar', - ) + cmd = DockerBuildCommand(['pip', 'install', 'requests'], + cwd='/tmp/foobar') self.assertEqual( cmd.get_wrapped_command(), "/bin/sh -c 'cd /tmp/foobar && pip install requests'", @@ -1121,11 +1097,9 @@ def test_wrapped_command(self): ) self.assertEqual( cmd.get_wrapped_command(), - ( - '/bin/sh -c ' - "'cd /tmp/foobar && PATH=/tmp/foo:$PATH " - r"python /tmp/foo/pip install Django\>1.7'" - ), + ('/bin/sh -c ' + "'cd /tmp/foobar && PATH=/tmp/foo:$PATH " + r"python /tmp/foo/pip install Django\>1.7'"), ) def test_unicode_output(self): @@ -1135,8 +1109,7 @@ def test_unicode_output(self): 'exec_create.return_value': {'Id': b'container-foobar'}, 'exec_start.return_value': SAMPLE_UTF8_BYTES, 'exec_inspect.return_value': {'ExitCode': 0}, - }, - ) + }) cmd = DockerBuildCommand(['echo', 'test'], cwd='/tmp/foobar') cmd.build_env = Mock() cmd.build_env.get_client.return_value = self.mocks.docker_client @@ -1144,8 +1117,7 @@ def test_unicode_output(self): cmd.run() self.assertEqual( cmd.output, - 'H\xe9r\xc9 \xee\xdf s\xf6m\xea \xfcn\xef\xe7\xf3\u2202\xe9', - ) + u'H\xe9r\xc9 \xee\xdf s\xf6m\xea \xfcn\xef\xe7\xf3\u2202\xe9') self.assertEqual(self.mocks.docker_client.exec_start.call_count, 1) self.assertEqual(self.mocks.docker_client.exec_create.call_count, 1) self.assertEqual(self.mocks.docker_client.exec_inspect.call_count, 1) @@ -1157,8 +1129,7 @@ def test_command_oom_kill(self): 'exec_create.return_value': {'Id': b'container-foobar'}, 'exec_start.return_value': b'Killed\n', 'exec_inspect.return_value': {'ExitCode': 137}, - }, - ) + }) cmd = DockerBuildCommand(['echo', 'test'], cwd='/tmp/foobar') cmd.build_env = Mock() cmd.build_env.get_client.return_value = self.mocks.docker_client @@ -1166,7 +1137,7 @@ def test_command_oom_kill(self): cmd.run() self.assertIn( 'Command killed due to excessive memory consumption\n', - str(cmd.output), + str(cmd.output) ) @@ -1195,9 +1166,8 @@ def setUp(self): ] self.pip_install_args = [ - mock.ANY, # python path - '-m', - 'pip', + 'python', + mock.ANY, # pip path 'install', '--upgrade', '--cache-dir', @@ -1241,7 +1211,7 @@ def test_install_core_requirements_mkdocs(self, checkout_path): checkout_path.return_value = tmpdir python_env = Virtualenv( version=self.version_mkdocs, - build_env=self.build_env_mock, + build_env=self.build_env_mock ) python_env.install_core_requirements() requirements_mkdocs = [ @@ -1272,29 +1242,28 @@ def test_install_user_requirements(self, checkout_path): self.build_env_mock.version = self.version_sphinx python_env = Virtualenv( version=self.version_sphinx, - build_env=self.build_env_mock, + build_env=self.build_env_mock ) checkout_path = python_env.checkout_path docs_requirements = os.path.join( - checkout_path, 'docs', 'requirements.txt', + checkout_path, 'docs', 'requirements.txt' ) root_requirements = os.path.join( - checkout_path, 'requirements.txt', + checkout_path, 'requirements.txt' ) paths = { os.path.join(checkout_path, 'docs'): True, } args = [ - mock.ANY, # python path - '-m', - 'pip', + 'python', + mock.ANY, # pip path 'install', '--exists-action=w', '--cache-dir', mock.ANY, # cache path '-r', - 'requirements_file', + 'requirements_file' ] # One requirements file on the docs/ dir @@ -1359,9 +1328,8 @@ def test_install_core_requirements_sphinx_conda(self, checkout_path): ] args_pip = [ - mock.ANY, # python path - '-m', - 'pip', + 'python', + mock.ANY, # pip path 'install', '-U', '--cache-dir', @@ -1373,7 +1341,6 @@ def test_install_core_requirements_sphinx_conda(self, checkout_path): 'conda', 'install', '--yes', - '--quiet', '--name', self.version_sphinx.slug, ] @@ -1381,7 +1348,7 @@ def test_install_core_requirements_sphinx_conda(self, checkout_path): self.build_env_mock.run.assert_has_calls([ mock.call(*args_conda, cwd=mock.ANY), - mock.call(*args_pip, bin_path=mock.ANY, cwd=mock.ANY), + mock.call(*args_pip, bin_path=mock.ANY, cwd=mock.ANY) ]) @patch('readthedocs.projects.models.Project.checkout_path') @@ -1400,9 +1367,8 @@ def test_install_core_requirements_mkdocs_conda(self, checkout_path): ] args_pip = [ - mock.ANY, # python path - '-m', - 'pip', + 'python', + mock.ANY, # pip path 'install', '-U', '--cache-dir', @@ -1414,7 +1380,6 @@ def test_install_core_requirements_mkdocs_conda(self, checkout_path): 'conda', 'install', '--yes', - '--quiet', '--name', self.version_mkdocs.slug, ] @@ -1422,7 +1387,7 @@ def test_install_core_requirements_mkdocs_conda(self, checkout_path): self.build_env_mock.run.assert_has_calls([ mock.call(*args_conda, cwd=mock.ANY), - mock.call(*args_pip, bin_path=mock.ANY, cwd=mock.ANY), + mock.call(*args_pip, bin_path=mock.ANY, cwd=mock.ANY) ]) @patch('readthedocs.projects.models.Project.checkout_path') @@ -1437,7 +1402,7 @@ def test_install_user_requirements_conda(self, checkout_path): self.build_env_mock.run.assert_not_called() -class AutoWipeEnvironmentBase: +class AutoWipeEnvironmentBase(object): fixtures = ['test_data'] build_env_class = None diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index d3e5ba74989..1236472010a 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -1,23 +1,21 @@ -# -*- coding: utf-8 -*- - -import django_dynamic_fixture as fixture +from __future__ import absolute_import import mock -from django.conf import settings +import django_dynamic_fixture as fixture + from django.contrib.auth.models import User -from django.http import Http404 from django.test import TestCase from django.test.utils import override_settings -from django.urls import reverse -from mock import mock_open, patch +from django.http import Http404 +from django.conf import settings -from readthedocs.core.views.serve import _serve_symlink_docs +from readthedocs.rtd_tests.base import RequestFactoryTestMixin from readthedocs.projects import constants from readthedocs.projects.models import Project -from readthedocs.rtd_tests.base import RequestFactoryTestMixin +from readthedocs.core.views.serve import _serve_symlink_docs @override_settings( - USE_SUBDOMAIN=False, PUBLIC_DOMAIN='public.readthedocs.org', DEBUG=False, + USE_SUBDOMAIN=False, PUBLIC_DOMAIN='public.readthedocs.org', DEBUG=False ) class BaseDocServing(RequestFactoryTestMixin, TestCase): @@ -46,7 +44,7 @@ def test_private_python_media_serving(self): serve_mock.assert_called_with( request, 'en/latest/usage.html', - settings.SITE_ROOT + '/private_web_root/private', + settings.SITE_ROOT + '/private_web_root/private' ) @override_settings(PYTHON_MEDIA=False) @@ -56,17 +54,7 @@ def test_private_nginx_serving(self): r = _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html', privacy_level='private') self.assertEqual(r.status_code, 200) self.assertEqual( - r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html', - ) - - @override_settings(PYTHON_MEDIA=False) - def test_private_nginx_serving_unicode_filename(self): - with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): - request = self.request(self.private_url, user=self.eric) - r = _serve_symlink_docs(request, project=self.private, filename='/en/latest/úñíčódé.html', privacy_level='private') - self.assertEqual(r.status_code, 200) - self.assertEqual( - r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/%C3%BA%C3%B1%C3%AD%C4%8D%C3%B3d%C3%A9.html', + r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' ) @override_settings(PYTHON_MEDIA=False) @@ -77,28 +65,6 @@ def test_private_files_not_found(self): self.assertTrue('private_web_root' in str(exc.exception)) self.assertTrue('public_web_root' not in str(exc.exception)) - @override_settings( - PYTHON_MEDIA=False, - USE_SUBDOMAIN=True, - PUBLIC_DOMAIN='readthedocs.io', - ROOT_URLCONF=settings.SUBDOMAIN_URLCONF, - ) - def test_robots_txt(self): - self.public.versions.update(active=True, built=True) - response = self.client.get( - reverse('robots_txt'), - HTTP_HOST='private.readthedocs.io', - ) - self.assertEqual(response.status_code, 404) - - self.client.force_login(self.eric) - response = self.client.get( - reverse('robots_txt'), - HTTP_HOST='private.readthedocs.io', - ) - # Private projects/versions always return 404 for robots.txt - self.assertEqual(response.status_code, 404) - @override_settings(SERVE_DOCS=[constants.PRIVATE, constants.PUBLIC]) class TestPublicDocs(BaseDocServing): @@ -112,7 +78,7 @@ def test_public_python_media_serving(self): serve_mock.assert_called_with( request, 'en/latest/usage.html', - settings.SITE_ROOT + '/public_web_root/public', + settings.SITE_ROOT + '/public_web_root/public' ) @override_settings(PYTHON_MEDIA=False) @@ -122,7 +88,7 @@ def test_public_nginx_serving(self): r = _serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html', privacy_level='public') self.assertEqual(r.status_code, 200) self.assertEqual( - r._headers['x-accel-redirect'][1], '/public_web_root/public/en/latest/usage.html', + r._headers['x-accel-redirect'][1], '/public_web_root/public/en/latest/usage.html' ) @override_settings(PYTHON_MEDIA=False) @@ -132,40 +98,3 @@ def test_both_files_not_found(self): _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html', privacy_level='public') self.assertTrue('private_web_root' not in str(exc.exception)) self.assertTrue('public_web_root' in str(exc.exception)) - - @override_settings( - PYTHON_MEDIA=False, - USE_SUBDOMAIN=True, - PUBLIC_DOMAIN='readthedocs.io', - ROOT_URLCONF=settings.SUBDOMAIN_URLCONF, - ) - def test_default_robots_txt(self): - self.public.versions.update(active=True, built=True) - response = self.client.get( - reverse('robots_txt'), - HTTP_HOST='public.readthedocs.io', - ) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content, b'User-agent: *\nAllow: /\n') - - @override_settings( - PYTHON_MEDIA=False, - USE_SUBDOMAIN=True, - PUBLIC_DOMAIN='readthedocs.io', - ROOT_URLCONF=settings.SUBDOMAIN_URLCONF, - ) - @patch( - 'builtins.open', - new_callable=mock_open, - read_data='My own robots.txt', - ) - @patch('readthedocs.core.views.serve.os') - def test_custom_robots_txt(self, os_mock, open_mock): - os_mock.path.exists.return_value = True - self.public.versions.update(active=True, built=True) - response = self.client.get( - reverse('robots_txt'), - HTTP_HOST='public.readthedocs.io', - ) - self.assertEqual(response.status_code, 200) - self.assertEqual(response.content, b'My own robots.txt') diff --git a/readthedocs/rtd_tests/tests/test_domains.py b/readthedocs/rtd_tests/tests/test_domains.py index 8c9610ac420..5e487ce2c94 100644 --- a/readthedocs/rtd_tests/tests/test_domains.py +++ b/readthedocs/rtd_tests/tests/test_domains.py @@ -1,16 +1,18 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import import json from django.core.cache import cache from django.test import TestCase from django.test.client import RequestFactory from django.test.utils import override_settings + from django_dynamic_fixture import get from readthedocs.core.middleware import SubdomainMiddleware +from readthedocs.projects.models import Project, Domain from readthedocs.projects.forms import DomainForm -from readthedocs.projects.models import Domain, Project class MiddlewareTests(TestCase): @@ -70,45 +72,33 @@ def setUp(self): self.project = get(Project, slug='kong') def test_https(self): - """Make sure https is an admin-only attribute.""" - form = DomainForm( - {'domain': 'example.com', 'canonical': True}, - project=self.project, - ) + """Make sure https is an admin-only attribute""" + form = DomainForm({'domain': 'example.com', 'canonical': True}, + project=self.project) self.assertTrue(form.is_valid()) domain = form.save() self.assertFalse(domain.https) - form = DomainForm( - { - 'domain': 'example.com', 'canonical': True, - 'https': True, - }, - project=self.project, - ) + form = DomainForm({'domain': 'example.com', 'canonical': True, + 'https': True}, + project=self.project) self.assertFalse(form.is_valid()) def test_canonical_change(self): - """Make sure canonical can be properly changed.""" - form = DomainForm( - {'domain': 'example.com', 'canonical': True}, - project=self.project, - ) + """Make sure canonical can be properly changed""" + form = DomainForm({'domain': 'example.com', 'canonical': True}, + project=self.project) self.assertTrue(form.is_valid()) domain = form.save() self.assertEqual(domain.domain, 'example.com') - form = DomainForm( - {'domain': 'example2.com', 'canonical': True}, - project=self.project, - ) + form = DomainForm({'domain': 'example2.com', 'canonical': True}, + project=self.project) self.assertFalse(form.is_valid()) self.assertEqual(form.errors['canonical'][0], 'Only 1 Domain can be canonical at a time.') - form = DomainForm( - {'domain': 'example2.com', 'canonical': True}, - project=self.project, - instance=domain, - ) + form = DomainForm({'domain': 'example2.com', 'canonical': True}, + project=self.project, + instance=domain) self.assertTrue(form.is_valid()) domain = form.save() self.assertEqual(domain.domain, 'example2.com') diff --git a/readthedocs/rtd_tests/tests/test_extend.py b/readthedocs/rtd_tests/tests/test_extend.py index 74ddbcf8d1f..205fa64e7bc 100644 --- a/readthedocs/rtd_tests/tests/test_extend.py +++ b/readthedocs/rtd_tests/tests/test_extend.py @@ -1,14 +1,13 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import +from builtins import object from django.test import TestCase, override_settings -from readthedocs.core.utils.extend import ( - SettingsOverrideObject, - get_override_class, -) +from readthedocs.core.utils.extend import (SettingsOverrideObject, + get_override_class) # Top level to ensure module name is correct -class FooBase: +class FooBase(object): def bar(self): return 1 @@ -35,7 +34,7 @@ class ExtendTests(TestCase): @override_settings(FOO_OVERRIDE_CLASS=None) def test_no_override(self): - """Test class without override.""" + """Test class without override""" class Foo(SettingsOverrideObject): _default_class = FooBase _override_setting = 'FOO_OVERRIDE_CLASS' @@ -50,7 +49,7 @@ class Foo(SettingsOverrideObject): @override_settings(FOO_OVERRIDE_CLASS=EXTEND_OVERRIDE_PATH) def test_with_basic_override(self): - """Test class override setting defined.""" + """Test class override setting defined""" class Foo(SettingsOverrideObject): _default_class = FooBase _override_setting = 'FOO_OVERRIDE_CLASS' @@ -63,12 +62,10 @@ class Foo(SettingsOverrideObject): override_class = get_override_class(Foo, Foo._default_class) self.assertEqual(override_class, NewFoo) - @override_settings( - FOO_OVERRIDE_CLASS=None, - CLASS_OVERRIDES={ - EXTEND_PATH: EXTEND_OVERRIDE_PATH, - }, - ) + @override_settings(FOO_OVERRIDE_CLASS=None, + CLASS_OVERRIDES={ + EXTEND_PATH: EXTEND_OVERRIDE_PATH, + }) def test_with_advanced_override(self): """Test class with override using `CLASS_OVERRIDES`""" class Foo(SettingsOverrideObject): @@ -83,12 +80,10 @@ class Foo(SettingsOverrideObject): override_class = get_override_class(Foo, Foo._default_class) self.assertEqual(override_class, NewFoo) - @override_settings( - FOO_OVERRIDE_CLASS=None, - CLASS_OVERRIDES={ - EXTEND_PATH: EXTEND_OVERRIDE_PATH, - }, - ) + @override_settings(FOO_OVERRIDE_CLASS=None, + CLASS_OVERRIDES={ + EXTEND_PATH: EXTEND_OVERRIDE_PATH, + }) def test_with_advanced_override_only(self): """Test class with no `override_setting`""" class Foo(SettingsOverrideObject): diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 60f820a7574..ec5ca4d754a 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -1,4 +1,11 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import mock from django.test import TestCase from rest_framework.test import APIRequestFactory, APITestCase diff --git a/readthedocs/rtd_tests/tests/test_gold.py b/readthedocs/rtd_tests/tests/test_gold.py index b52ec2938ae..63f50f8d696 100644 --- a/readthedocs/rtd_tests/tests/test_gold.py +++ b/readthedocs/rtd_tests/tests/test_gold.py @@ -1,9 +1,11 @@ -# -*- coding: utf-8 -*- -from django.test import TestCase +from __future__ import absolute_import + from django.urls import reverse -from django_dynamic_fixture import fixture, get +from django.test import TestCase + +from django_dynamic_fixture import get, fixture -from readthedocs.gold.models import LEVEL_CHOICES, GoldUser +from readthedocs.gold.models import GoldUser, LEVEL_CHOICES from readthedocs.projects.models import Project from readthedocs.rtd_tests.utils import create_user @@ -34,7 +36,7 @@ def test_too_many_projects(self): self.assertEqual(resp.status_code, 302) resp = self.client.post(reverse('gold_projects'), data={'project': self.project2.slug}) self.assertFormError( - resp, form='form', field=None, errors='You already have the max number of supported projects.', + resp, form='form', field=None, errors='You already have the max number of supported projects.' ) self.assertEqual(resp.status_code, 200) self.assertEqual(self.golduser.projects.count(), 1) diff --git a/readthedocs/rtd_tests/tests/test_imported_file.py b/readthedocs/rtd_tests/tests/test_imported_file.py index 187b1af9830..81f5e2a684b 100644 --- a/readthedocs/rtd_tests/tests/test_imported_file.py +++ b/readthedocs/rtd_tests/tests/test_imported_file.py @@ -1,17 +1,15 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import import os - from django.test import TestCase -from readthedocs.projects.models import ImportedFile, Project from readthedocs.projects.tasks import _manage_imported_files - +from readthedocs.projects.models import Project, ImportedFile base_dir = os.path.dirname(os.path.dirname(__file__)) class ImportedFileTests(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ["eric", "test_data"] def setUp(self): self.project = Project.objects.get(slug='pip') diff --git a/readthedocs/rtd_tests/tests/test_integrations.py b/readthedocs/rtd_tests/tests/test_integrations.py index 54e6922a4c6..51006b7e757 100644 --- a/readthedocs/rtd_tests/tests/test_integrations.py +++ b/readthedocs/rtd_tests/tests/test_integrations.py @@ -1,20 +1,22 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from builtins import range import django_dynamic_fixture as fixture -from django.test import TestCase +from django.test import TestCase, RequestFactory +from django.contrib.contenttypes.models import ContentType from rest_framework.test import APIClient +from rest_framework.test import APIRequestFactory +from rest_framework.response import Response from readthedocs.integrations.models import ( - GitHubWebhook, - HttpExchange, - Integration, + HttpExchange, Integration, GitHubWebhook ) from readthedocs.projects.models import Project class HttpExchangeTests(TestCase): - """ - Test HttpExchange model by using existing views. + """Test HttpExchange model by using existing views This doesn't mock out a req/resp cycle, as manually creating these outside views misses a number of attributes on the request object. @@ -24,27 +26,23 @@ def test_exchange_json_request_body(self): client = APIClient() client.login(username='super', password='test') project = fixture.get(Project, main_language_project=None) - integration = fixture.get( - Integration, project=project, - integration_type=Integration.GITHUB_WEBHOOK, - provider_data='', - ) + integration = fixture.get(Integration, project=project, + integration_type=Integration.GITHUB_WEBHOOK, + provider_data='') resp = client.post( - '/api/v2/webhook/github/{}/'.format(project.slug), + '/api/v2/webhook/github/{0}/'.format(project.slug), {'ref': 'exchange_json'}, - format='json', + format='json' ) exchange = HttpExchange.objects.get(integrations=integration) self.assertEqual( exchange.request_body, - '{"ref": "exchange_json"}', + '{"ref": "exchange_json"}' ) self.assertEqual( exchange.request_headers, - { - 'Content-Type': 'application/json; charset=None', - 'Cookie': '', - }, + {u'Content-Type': u'application/json; charset=None', + u'Cookie': u''} ) self.assertEqual( exchange.response_body, @@ -53,37 +51,31 @@ def test_exchange_json_request_body(self): ) self.assertEqual( exchange.response_headers, - { - 'Allow': 'POST, OPTIONS', - 'Content-Type': 'text/html; charset=utf-8', - }, + {u'Allow': u'POST, OPTIONS', + u'Content-Type': u'text/html; charset=utf-8'} ) def test_exchange_form_request_body(self): client = APIClient() client.login(username='super', password='test') project = fixture.get(Project, main_language_project=None) - integration = fixture.get( - Integration, project=project, - integration_type=Integration.GITHUB_WEBHOOK, - provider_data='', - ) + integration = fixture.get(Integration, project=project, + integration_type=Integration.GITHUB_WEBHOOK, + provider_data='') resp = client.post( - '/api/v2/webhook/github/{}/'.format(project.slug), + '/api/v2/webhook/github/{0}/'.format(project.slug), 'payload=%7B%22ref%22%3A+%22exchange_form%22%7D', content_type='application/x-www-form-urlencoded', ) exchange = HttpExchange.objects.get(integrations=integration) self.assertEqual( exchange.request_body, - '{"ref": "exchange_form"}', + '{"ref": "exchange_form"}' ) self.assertEqual( exchange.request_headers, - { - 'Content-Type': 'application/x-www-form-urlencoded', - 'Cookie': '', - }, + {u'Content-Type': u'application/x-www-form-urlencoded', + u'Cookie': u''} ) self.assertEqual( exchange.response_body, @@ -92,63 +84,57 @@ def test_exchange_form_request_body(self): ) self.assertEqual( exchange.response_headers, - { - 'Allow': 'POST, OPTIONS', - 'Content-Type': 'text/html; charset=utf-8', - }, + {u'Allow': u'POST, OPTIONS', + u'Content-Type': u'text/html; charset=utf-8'} ) def test_extraneous_exchanges_deleted_in_correct_order(self): client = APIClient() client.login(username='super', password='test') project = fixture.get(Project, main_language_project=None) - integration = fixture.get( - Integration, project=project, - integration_type=Integration.GITHUB_WEBHOOK, - provider_data='', - ) + integration = fixture.get(Integration, project=project, + integration_type=Integration.GITHUB_WEBHOOK, + provider_data='') self.assertEqual( HttpExchange.objects.filter(integrations=integration).count(), - 0, + 0 ) for _ in range(10): resp = client.post( - '/api/v2/webhook/github/{}/'.format(project.slug), + '/api/v2/webhook/github/{0}/'.format(project.slug), {'ref': 'deleted'}, - format='json', + format='json' ) for _ in range(10): resp = client.post( - '/api/v2/webhook/github/{}/'.format(project.slug), + '/api/v2/webhook/github/{0}/'.format(project.slug), {'ref': 'preserved'}, - format='json', + format='json' ) self.assertEqual( HttpExchange.objects.filter(integrations=integration).count(), - 10, + 10 ) self.assertEqual( HttpExchange.objects.filter( integrations=integration, request_body='{"ref": "preserved"}', ).count(), - 10, + 10 ) def test_request_headers_are_removed(self): client = APIClient() client.login(username='super', password='test') project = fixture.get(Project, main_language_project=None) - integration = fixture.get( - Integration, project=project, - integration_type=Integration.GITHUB_WEBHOOK, - provider_data='', - ) + integration = fixture.get(Integration, project=project, + integration_type=Integration.GITHUB_WEBHOOK, + provider_data='') resp = client.post( - '/api/v2/webhook/github/{}/'.format(project.slug), + '/api/v2/webhook/github/{0}/'.format(project.slug), {'ref': 'exchange_json'}, format='json', HTTP_X_FORWARDED_FOR='1.2.3.4', @@ -158,11 +144,9 @@ def test_request_headers_are_removed(self): exchange = HttpExchange.objects.get(integrations=integration) self.assertEqual( exchange.request_headers, - { - 'Content-Type': 'application/json; charset=None', - 'Cookie': '', - 'X-Foo': 'bar', - }, + {u'Content-Type': u'application/json; charset=None', + u'Cookie': u'', + u'X-Foo': u'bar'} ) @@ -172,7 +156,7 @@ def test_subclass_is_replaced_on_get(self): project = fixture.get(Project, main_language_project=None) integration = Integration.objects.create( project=project, - integration_type=Integration.GITHUB_WEBHOOK, + integration_type=Integration.GITHUB_WEBHOOK ) integration = Integration.objects.get(pk=integration.pk) self.assertIsInstance(integration, GitHubWebhook) @@ -181,7 +165,7 @@ def test_subclass_is_replaced_on_subclass(self): project = fixture.get(Project, main_language_project=None) integration = Integration.objects.create( project=project, - integration_type=Integration.GITHUB_WEBHOOK, + integration_type=Integration.GITHUB_WEBHOOK ) integration = Integration.objects.subclass(integration) self.assertIsInstance(integration, GitHubWebhook) diff --git a/readthedocs/rtd_tests/tests/test_middleware.py b/readthedocs/rtd_tests/tests/test_middleware.py index 4a93274da4a..6954190ec0d 100644 --- a/readthedocs/rtd_tests/tests/test_middleware.py +++ b/readthedocs/rtd_tests/tests/test_middleware.py @@ -1,15 +1,20 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from corsheaders.middleware import CorsMiddleware from django.conf import settings -from django.core.cache import cache from django.http import Http404 from django.test import TestCase from django.test.client import RequestFactory from django.test.utils import override_settings from django.urls.base import get_urlconf, set_urlconf from django_dynamic_fixture import get -from mock import patch from readthedocs.core.middleware import SubdomainMiddleware from readthedocs.projects.models import Domain, Project, ProjectRelationship @@ -26,9 +31,15 @@ def setUp(self): self.middleware = SubdomainMiddleware() self.url = '/' self.owner = create_user(username='owner', password='test') - self.pip = get(Project, slug='pip', users=[self.owner], privacy_level='public') + self.pip = get( + Project, + slug='pip', + users=[self.owner], + privacy_level='public' + ) def test_failey_cname(self): + self.assertFalse(Domain.objects.filter(domain='my.host.com').exists()) request = self.factory.get(self.url, HTTP_HOST='my.host.com') with self.assertRaises(Http404): self.middleware.process_request(request) @@ -74,7 +85,9 @@ def test_restore_urlconf_after_request(self): @override_settings(PRODUCTION_DOMAIN='prod.readthedocs.org') def test_subdomain_different_length(self): - request = self.factory.get(self.url, HTTP_HOST='pip.prod.readthedocs.org') + request = self.factory.get( + self.url, HTTP_HOST='pip.prod.readthedocs.org' + ) self.middleware.process_request(request) self.assertEqual(request.urlconf, self.urlconf_subdomain) self.assertEqual(request.subdomain, True) @@ -95,16 +108,10 @@ def test_domain_object_missing(self): with self.assertRaises(Http404): self.middleware.process_request(request) - def test_proper_cname(self): - cache.get = lambda x: 'my_slug' - request = self.factory.get(self.url, HTTP_HOST='my.valid.homename') - self.middleware.process_request(request) - self.assertEqual(request.urlconf, self.urlconf_subdomain) - self.assertEqual(request.cname, True) - self.assertEqual(request.slug, 'my_slug') - def test_request_header(self): - request = self.factory.get(self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='pip') + request = self.factory.get( + self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='pip' + ) self.middleware.process_request(request) self.assertEqual(request.urlconf, self.urlconf_subdomain) self.assertEqual(request.cname, True) @@ -113,7 +120,7 @@ def test_request_header(self): @override_settings(PRODUCTION_DOMAIN='readthedocs.org') def test_proper_cname_uppercase(self): - cache.get = lambda x: x.split('.')[0] + get(Domain, project=self.pip, domain='pip.random.com') request = self.factory.get(self.url, HTTP_HOST='PIP.RANDOM.COM') self.middleware.process_request(request) self.assertEqual(request.urlconf, self.urlconf_subdomain) @@ -121,20 +128,35 @@ def test_proper_cname_uppercase(self): self.assertEqual(request.slug, 'pip') def test_request_header_uppercase(self): - request = self.factory.get(self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='PIP') + request = self.factory.get( + self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='PIP' + ) self.middleware.process_request(request) self.assertEqual(request.urlconf, self.urlconf_subdomain) self.assertEqual(request.cname, True) self.assertEqual(request.rtdheader, True) self.assertEqual(request.slug, 'pip') - @override_settings(USE_SUBDOMAIN=True) - # no need to do a real dns query so patch cname_to_slug - @patch('readthedocs.core.middleware.cname_to_slug', new=lambda x: 'doesnt') - def test_use_subdomain_on(self): - request = self.factory.get(self.url, HTTP_HOST='doesnt.really.matter') - ret_val = self.middleware.process_request(request) - self.assertIsNone(ret_val, None) + def test_use_subdomain(self): + domain = 'doesnt.exists.org' + get(Domain, project=self.pip, domain=domain) + request = self.factory.get(self.url, HTTP_HOST=domain) + res = self.middleware.process_request(request) + self.assertIsNone(res) + self.assertEqual(request.slug, 'pip') + self.assertTrue(request.domain_object) + + def test_long_bad_subdomain(self): + domain = 'www.pip.readthedocs.org' + request = self.factory.get(self.url, HTTP_HOST=domain) + res = self.middleware.process_request(request) + self.assertEqual(res.status_code, 400) + + def test_long_subdomain(self): + domain = 'some.long.readthedocs.org' + request = self.factory.get(self.url, HTTP_HOST=domain) + res = self.middleware.process_request(request) + self.assertIsNone(res) class TestCORSMiddleware(TestCase): @@ -147,7 +169,7 @@ def setUp(self): self.project = get( Project, slug='pip', users=[self.owner], privacy_level='public', - mail_language_project=None, + mail_language_project=None ) self.subproject = get( Project, @@ -158,7 +180,7 @@ def setUp(self): self.relationship = get( ProjectRelationship, parent=self.project, - child=self.subproject, + child=self.subproject ) self.domain = get(Domain, domain='my.valid.domain', project=self.project) diff --git a/readthedocs/rtd_tests/tests/test_notifications.py b/readthedocs/rtd_tests/tests/test_notifications.py index 1ead82e534a..2213b0d8c7c 100644 --- a/readthedocs/rtd_tests/tests/test_notifications.py +++ b/readthedocs/rtd_tests/tests/test_notifications.py @@ -1,28 +1,18 @@ # -*- coding: utf-8 -*- -"""Notification tests.""" +"""Notification tests""" - -import django_dynamic_fixture as fixture +from __future__ import absolute_import import mock -from django.contrib.auth.models import AnonymousUser, User -from django.http import HttpRequest +import django_dynamic_fixture as fixture from django.test import TestCase from django.test.utils import override_settings +from django.contrib.auth.models import User, AnonymousUser from messages_extends.models import Message as PersistentMessage -from readthedocs.builds.models import Build from readthedocs.notifications import Notification, SiteNotification from readthedocs.notifications.backends import EmailBackend, SiteBackend -from readthedocs.notifications.constants import ( - ERROR, - INFO_NON_PERSISTENT, - WARNING_NON_PERSISTENT, -) -from readthedocs.projects.models import Project -from readthedocs.projects.notifications import ( - DeprecatedBuildWebhookNotification, - DeprecatedGitHubWebhookNotification, -) +from readthedocs.notifications.constants import ERROR, INFO_NON_PERSISTENT, WARNING_NON_PERSISTENT +from readthedocs.builds.models import Build @override_settings( @@ -48,33 +38,21 @@ class TestNotification(Notification): req = mock.MagicMock() notify = TestNotification(context_object=build, request=req) - self.assertEqual( - notify.get_template_names('email'), - ['builds/notifications/foo_email.html'], - ) - self.assertEqual( - notify.get_template_names('site'), - ['builds/notifications/foo_site.html'], - ) - self.assertEqual( - notify.get_subject(), - 'This is {}'.format(build.id), - ) - self.assertEqual( - notify.get_context_data(), - { - 'foo': build, - 'production_uri': 'https://readthedocs.org', - 'request': req, - }, - ) + self.assertEqual(notify.get_template_names('email'), + ['builds/notifications/foo_email.html']) + self.assertEqual(notify.get_template_names('site'), + ['builds/notifications/foo_site.html']) + self.assertEqual(notify.get_subject(), + 'This is {0}'.format(build.id)) + self.assertEqual(notify.get_context_data(), + {'foo': build, + 'production_uri': 'https://readthedocs.org', + 'request': req}) notify.render('site') render_to_string.assert_has_calls([ - mock.call( - context=mock.ANY, - template_name=['builds/notifications/foo_site.html'], - ), + mock.call(context=mock.ANY, + template_name=['builds/notifications/foo_site.html']) ]) @@ -104,10 +82,10 @@ class TestNotification(Notification): request=mock.ANY, template='core/email/common.txt', context={'content': 'Test'}, - subject='This is {}'.format(build.id), + subject=u'This is {}'.format(build.id), template_html='core/email/common.html', recipient=user.email, - ), + ) ]) def test_message_backend(self, render_to_string): @@ -132,7 +110,7 @@ class TestNotification(Notification): self.assertEqual(message.user, user) def test_message_anonymous_user(self, render_to_string): - """Anonymous user still throwns exception on persistent messages.""" + """Anonymous user still throwns exception on persistent messages""" render_to_string.return_value = 'Test' class TestNotification(Notification): @@ -244,43 +222,3 @@ def test_message(self): with mock.patch('readthedocs.notifications.notification.log') as mock_log: self.assertEqual(self.n.get_message(False), '') mock_log.error.assert_called_once() - - -class DeprecatedWebhookEndpointNotificationTests(TestCase): - - def setUp(self): - PersistentMessage.objects.all().delete() - - self.user = fixture.get(User) - self.project = fixture.get(Project, users=[self.user]) - self.request = HttpRequest() - - self.notification = DeprecatedBuildWebhookNotification( - self.project, - self.request, - self.user, - ) - - @mock.patch('readthedocs.notifications.backends.send_email') - def test_dedupliation(self, send_email): - user = fixture.get(User) - project = fixture.get(Project, main_language_project=None) - project.users.add(user) - project.refresh_from_db() - self.assertEqual(project.users.count(), 1) - - self.assertEqual(PersistentMessage.objects.filter(user=user).count(), 0) - DeprecatedGitHubWebhookNotification.notify_project_users([project]) - - # Site and email notification will go out, site message doesn't have - # any reason to deduplicate yet - self.assertEqual(PersistentMessage.objects.filter(user=user).count(), 1) - self.assertTrue(send_email.called) - send_email.reset_mock() - self.assertFalse(send_email.called) - - # Expect the site message to deduplicate, the email won't - DeprecatedGitHubWebhookNotification.notify_project_users([project]) - self.assertEqual(PersistentMessage.objects.filter(user=user).count(), 1) - self.assertTrue(send_email.called) - send_email.reset_mock() diff --git a/readthedocs/rtd_tests/tests/test_oauth.py b/readthedocs/rtd_tests/tests/test_oauth.py index 1ef675cc42c..fedbaf0d47d 100644 --- a/readthedocs/rtd_tests/tests/test_oauth.py +++ b/readthedocs/rtd_tests/tests/test_oauth.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import mock from django.conf import settings from django.contrib.auth.models import User @@ -7,10 +10,7 @@ from readthedocs.oauth.models import RemoteOrganization, RemoteRepository from readthedocs.oauth.services import ( - BitbucketService, - GitHubService, - GitLabService, -) + BitbucketService, GitHubService, GitLabService) from readthedocs.projects import constants from readthedocs.projects.models import Project @@ -39,8 +39,7 @@ def test_make_project_pass(self): 'clone_url': 'https://github.com/testuser/testrepo.git', } repo = self.service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) self.assertIsInstance(repo, RemoteRepository) self.assertEqual(repo.name, 'testrepo') self.assertEqual(repo.full_name, 'testuser/testrepo') @@ -52,11 +51,9 @@ def test_make_project_pass(self): self.assertIn(self.user, repo.users.all()) self.assertEqual(repo.organization, self.org) self.assertEqual( - repo.clone_url, 'https://github.com/testuser/testrepo.git', - ) + repo.clone_url, 'https://github.com/testuser/testrepo.git') self.assertEqual( - repo.ssh_url, 'ssh://git@github.com:testuser/testrepo.git', - ) + repo.ssh_url, 'ssh://git@github.com:testuser/testrepo.git') self.assertEqual(repo.html_url, 'https://github.com/testuser/testrepo') def test_make_project_fail(self): @@ -71,8 +68,7 @@ def test_make_project_fail(self): 'clone_url': '', } github_project = self.service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) self.assertIsNone(github_project) def test_make_organization(self): @@ -109,35 +105,29 @@ def test_multiple_users_same_repo(self): } github_project = self.service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) user2 = User.objects.get(pk=2) service = GitHubService(user=user2, account=None) github_project_2 = service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) self.assertIsInstance(github_project, RemoteRepository) self.assertIsInstance(github_project_2, RemoteRepository) self.assertNotEqual(github_project_2, github_project) github_project_3 = self.service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) github_project_4 = service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) self.assertIsInstance(github_project_3, RemoteRepository) self.assertIsInstance(github_project_4, RemoteRepository) self.assertEqual(github_project, github_project_3) self.assertEqual(github_project_2, github_project_4) github_project_5 = self.service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) github_project_6 = service.create_repository( - repo_json, organization=self.org, privacy=self.privacy, - ) + repo_json, organization=self.org, privacy=self.privacy) self.assertEqual(github_project, github_project_5) self.assertEqual(github_project_2, github_project_6) @@ -271,8 +261,7 @@ def setUp(self): def test_make_project_pass(self): repo = self.service.create_repository( self.repo_response_data, organization=self.org, - privacy=self.privacy, - ) + privacy=self.privacy) self.assertIsInstance(repo, RemoteRepository) self.assertEqual(repo.name, 'tutorials.bitbucket.org') self.assertEqual(repo.full_name, 'tutorials/tutorials.bitbucket.org') @@ -280,30 +269,24 @@ def test_make_project_pass(self): self.assertEqual( repo.avatar_url, ( 'https://bitbucket-assetroot.s3.amazonaws.com/c/photos/2012/Nov/28/' - 'tutorials.bitbucket.org-logo-1456883302-9_avatar.png' - ), - ) + 'tutorials.bitbucket.org-logo-1456883302-9_avatar.png')) self.assertIn(self.user, repo.users.all()) self.assertEqual(repo.organization, self.org) self.assertEqual( repo.clone_url, - 'https://bitbucket.org/tutorials/tutorials.bitbucket.org', - ) + 'https://bitbucket.org/tutorials/tutorials.bitbucket.org') self.assertEqual( repo.ssh_url, - 'ssh://hg@bitbucket.org/tutorials/tutorials.bitbucket.org', - ) + 'ssh://hg@bitbucket.org/tutorials/tutorials.bitbucket.org') self.assertEqual( repo.html_url, - 'https://bitbucket.org/tutorials/tutorials.bitbucket.org', - ) + 'https://bitbucket.org/tutorials/tutorials.bitbucket.org') def test_make_project_fail(self): data = self.repo_response_data.copy() data['is_private'] = True repo = self.service.create_repository( - data, organization=self.org, privacy=self.privacy, - ) + data, organization=self.org, privacy=self.privacy) self.assertIsNone(repo) @override_settings(DEFAULT_PRIVACY_LEVEL='private') @@ -324,9 +307,7 @@ def test_make_organization(self): self.assertEqual( org.avatar_url, ( 'https://bitbucket-assetroot.s3.amazonaws.com/c/photos/2014/Sep/24/' - 'teamsinspace-avatar-3731530358-7_avatar.png' - ), - ) + 'teamsinspace-avatar-3731530358-7_avatar.png')) self.assertEqual(org.url, 'https://bitbucket.org/teamsinspace') def test_import_with_no_token(self): @@ -449,8 +430,7 @@ def test_make_project_pass(self): m.return_value = True repo = self.service.create_repository( self.repo_response_data, organization=self.org, - privacy=self.privacy, - ) + privacy=self.privacy) self.assertIsInstance(repo, RemoteRepository) self.assertEqual(repo.name, 'testrepo') self.assertEqual(repo.full_name, 'testorga / testrepo') @@ -473,15 +453,13 @@ def test_make_project_pass(self): def test_make_private_project_fail(self): repo = self.service.create_repository( self.get_private_repo_data(), organization=self.org, - privacy=self.privacy, - ) + privacy=self.privacy) self.assertIsNone(repo) def test_make_private_project_success(self): repo = self.service.create_repository( self.get_private_repo_data(), organization=self.org, - privacy=constants.PRIVATE, - ) + privacy=constants.PRIVATE) self.assertIsInstance(repo, RemoteRepository) self.assertTrue(repo.private, True) diff --git a/readthedocs/rtd_tests/tests/test_post_commit_hooks.py b/readthedocs/rtd_tests/tests/test_post_commit_hooks.py index 4dd40865e02..86297dbe4b4 100644 --- a/readthedocs/rtd_tests/tests/test_post_commit_hooks.py +++ b/readthedocs/rtd_tests/tests/test_post_commit_hooks.py @@ -1,14 +1,15 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import + import json import logging -from urllib.parse import urlencode import mock from django.test import TestCase from django_dynamic_fixture import get +from future.backports.urllib.parse import urlencode from readthedocs.builds.models import Version -from readthedocs.projects.models import Feature, Project +from readthedocs.projects.models import Project, Feature log = logging.getLogger(__name__) @@ -17,19 +18,15 @@ class BasePostCommitTest(TestCase): def _setup(self): self.rtfd = get( - Project, repo='https://github.com/rtfd/readthedocs.org', slug='read-the-docs', - ) + Project, repo='https://github.com/rtfd/readthedocs.org', slug='read-the-docs') self.rtfd_not_ok = get( - Version, project=self.rtfd, slug='not_ok', identifier='not_ok', active=False, - ) + Version, project=self.rtfd, slug='not_ok', identifier='not_ok', active=False) self.rtfd_awesome = get( - Version, project=self.rtfd, slug='awesome', identifier='awesome', active=True, - ) + Version, project=self.rtfd, slug='awesome', identifier='awesome', active=True) self.pip = get(Project, repo='https://bitbucket.org/pip/pip', repo_type='hg') self.pip_not_ok = get( - Version, project=self.pip, slug='not_ok', identifier='not_ok', active=False, - ) + Version, project=self.pip, slug='not_ok', identifier='not_ok', active=False) self.sphinx = get(Project, repo='https://bitbucket.org/sphinx/sphinx', repo_type='git') self.mocks = [mock.patch('readthedocs.core.views.hooks.trigger_build')] @@ -51,97 +48,91 @@ def _setup(self): class GitLabWebHookTest(BasePostCommitTest): - fixtures = ['eric'] + fixtures = ["eric"] def setUp(self): self._setup() self.payload = { - 'object_kind': 'push', - 'before': '95790bf891e76fee5e1747ab589903a6a1f80f22', - 'after': 'da1560886d4f094c3e6c9ef40349f7d38b5d27d7', - 'ref': 'refs/heads/awesome', - 'checkout_sha': 'da1560886d4f094c3e6c9ef40349f7d38b5d27d7', - 'user_id': 4, - 'user_name': 'John Smith', - 'user_email': 'john@example.com', - 'project_id': 15, - 'project':{ - 'name':'readthedocs', - 'description':'', - 'web_url':'http://example.com/mike/diaspora', - 'avatar_url': None, - 'git_ssh_url':'git@github.com:rtfd/readthedocs.org.git', - 'git_http_url':'http://github.com/rtfd/readthedocs.org.git', - 'namespace':'Mike', - 'visibility_level':0, - 'path_with_namespace':'mike/diaspora', - 'default_branch':'master', - 'homepage':'http://example.com/mike/diaspora', - 'url':'git@github.com/rtfd/readthedocs.org.git', - 'ssh_url':'git@github.com/rtfd/readthedocs.org.git', - 'http_url':'http://github.com/rtfd/readthedocs.org.git', + "object_kind": "push", + "before": "95790bf891e76fee5e1747ab589903a6a1f80f22", + "after": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + "ref": "refs/heads/awesome", + "checkout_sha": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + "user_id": 4, + "user_name": "John Smith", + "user_email": "john@example.com", + "project_id": 15, + "project":{ + "name":"readthedocs", + "description":"", + "web_url":"http://example.com/mike/diaspora", + "avatar_url": None, + "git_ssh_url":"git@github.com:rtfd/readthedocs.org.git", + "git_http_url":"http://github.com/rtfd/readthedocs.org.git", + "namespace":"Mike", + "visibility_level":0, + "path_with_namespace":"mike/diaspora", + "default_branch":"master", + "homepage":"http://example.com/mike/diaspora", + "url":"git@github.com/rtfd/readthedocs.org.git", + "ssh_url":"git@github.com/rtfd/readthedocs.org.git", + "http_url":"http://github.com/rtfd/readthedocs.org.git" }, - 'repository':{ - 'name': 'Diaspora', - 'url': 'git@github.com:rtfd/readthedocs.org.git', - 'description': '', - 'homepage': 'http://github.com/rtfd/readthedocs.org', - 'git_http_url': 'http://github.com/rtfd/readthedocs.org.git', - 'git_ssh_url': 'git@github.com:rtfd/readthedocs.org.git', - 'visibility_level': 0, + "repository":{ + "name": "Diaspora", + "url": "git@github.com:rtfd/readthedocs.org.git", + "description": "", + "homepage": "http://github.com/rtfd/readthedocs.org", + "git_http_url": "http://github.com/rtfd/readthedocs.org.git", + "git_ssh_url": "git@github.com:rtfd/readthedocs.org.git", + "visibility_level": 0 }, - 'commits': [ + "commits": [ { - 'id': 'b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327', - 'message': 'Update Catalan translation to e38cb41.', - 'timestamp': '2011-12-12T14:27:31+02:00', - 'url': 'http://example.com/mike/diaspora/commit/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327', - 'author': { - 'name': 'Jordi Mallach', - 'email': 'jordi@softcatala.org', + "id": "b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", + "message": "Update Catalan translation to e38cb41.", + "timestamp": "2011-12-12T14:27:31+02:00", + "url": "http://example.com/mike/diaspora/commit/b6568db1bc1dcd7f8b4d5a946b0b91f9dacd7327", + "author": { + "name": "Jordi Mallach", + "email": "jordi@softcatala.org" }, - 'added': ['CHANGELOG'], - 'modified': ['app/controller/application.rb'], - 'removed': [], + "added": ["CHANGELOG"], + "modified": ["app/controller/application.rb"], + "removed": [] }, { - 'id': 'da1560886d4f094c3e6c9ef40349f7d38b5d27d7', - 'message': 'fixed readme', - 'timestamp': '2012-01-03T23:36:29+02:00', - 'url': 'http://example.com/mike/diaspora/commit/da1560886d4f094c3e6c9ef40349f7d38b5d27d7', - 'author': { - 'name': 'GitLab dev user', - 'email': 'gitlabdev@dv6700.(none)', + "id": "da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + "message": "fixed readme", + "timestamp": "2012-01-03T23:36:29+02:00", + "url": "http://example.com/mike/diaspora/commit/da1560886d4f094c3e6c9ef40349f7d38b5d27d7", + "author": { + "name": "GitLab dev user", + "email": "gitlabdev@dv6700.(none)" }, - 'added': ['CHANGELOG'], - 'modified': ['app/controller/application.rb'], - 'removed': [], - }, + "added": ["CHANGELOG"], + "modified": ["app/controller/application.rb"], + "removed": [] + } ], - 'total_commits_count': 4, + "total_commits_count": 4 } def test_gitlab_post_commit_hook_builds_branch_docs_if_it_should(self): - """GitLab webhook should only build active versions.""" - r = self.client.post( - '/gitlab/', data=json.dumps(self.payload), - content_type='application/json', - ) + """GitLab webhook should only build active versions""" + r = self.client.post('/gitlab/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: github.com/rtfd/readthedocs.org [awesome]') self.payload['ref'] = 'refs/heads/not_ok' - r = self.client.post( - '/gitlab/', data=json.dumps(self.payload), - content_type='application/json', - ) + r = self.client.post('/gitlab/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) Not Building: github.com/rtfd/readthedocs.org [not_ok]') self.payload['ref'] = 'refs/heads/unknown' - r = self.client.post( - '/gitlab/', data=json.dumps(self.payload), - content_type='application/json', - ) + r = self.client.post('/gitlab/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) No known branches were pushed to.') def test_gitlab_post_commit_knows_default_branches(self): @@ -155,10 +146,8 @@ def test_gitlab_post_commit_knows_default_branches(self): rtd.save() self.payload['ref'] = 'refs/heads/master' - r = self.client.post( - '/gitlab/', data=json.dumps(self.payload), - content_type='application/json', - ) + r = self.client.post('/gitlab/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: github.com/rtfd/readthedocs.org [latest]') rtd.default_branch = old_default @@ -172,7 +161,7 @@ def test_gitlab_request_empty_url(self): self.payload['project']['http_url'] = '' r = self.client.post( '/gitlab/', data=json.dumps(self.payload), - content_type='application/json', + content_type='application/json' ) self.assertEqual(r.status_code, 404) @@ -189,123 +178,110 @@ def test_gitlab_webhook_is_deprecated(self): r = self.client.post( '/gitlab/', data=json.dumps(payload), - content_type='application/json', + content_type='application/json' ) self.assertEqual(r.status_code, 403) class GitHubWebHookTest(BasePostCommitTest): - fixtures = ['eric'] + fixtures = ["eric"] def setUp(self): self._setup() self.payload = { - 'after': '5ad757394b926e5637ffeafe340f952ef48bd270', - 'base_ref': 'refs/heads/master', - 'before': '5b4e453dc913b08642b1d4fb10ed23c9d6e5b129', - 'commits': [ + "after": "5ad757394b926e5637ffeafe340f952ef48bd270", + "base_ref": "refs/heads/master", + "before": "5b4e453dc913b08642b1d4fb10ed23c9d6e5b129", + "commits": [ { - 'added': [], - 'author': { - 'email': 'eric@ericholscher.com', - 'name': 'Eric Holscher', - 'username': 'ericholscher', + "added": [], + "author": { + "email": "eric@ericholscher.com", + "name": "Eric Holscher", + "username": "ericholscher" }, - 'distinct': False, - 'id': '11f229c6a78f5bc8cb173104a3f7a68cdb7eb15a', - 'message': 'Fix it on the front list as well.', - 'modified': [ - 'readthedocs/templates/core/project_list_detailed.html', + "distinct": False, + "id": "11f229c6a78f5bc8cb173104a3f7a68cdb7eb15a", + "message": "Fix it on the front list as well.", + "modified": [ + "readthedocs/templates/core/project_list_detailed.html" ], - 'removed': [], - 'timestamp': '2011-09-12T19:38:55-07:00', - 'url': ( - 'https://github.com/wraithan/readthedocs.org/' - 'commit/11f229c6a78f5bc8cb173104a3f7a68cdb7eb15a' - ), + "removed": [], + "timestamp": "2011-09-12T19:38:55-07:00", + "url": ("https://github.com/wraithan/readthedocs.org/" + "commit/11f229c6a78f5bc8cb173104a3f7a68cdb7eb15a") }, ], - 'compare': ( - 'https://github.com/wraithan/readthedocs.org/compare/' - '5b4e453...5ad7573' - ), - 'created': False, - 'deleted': False, - 'forced': False, - 'pusher': { - 'name': 'none', + "compare": ("https://github.com/wraithan/readthedocs.org/compare/" + "5b4e453...5ad7573"), + "created": False, + "deleted": False, + "forced": False, + "pusher": { + "name": "none" }, - 'ref': 'refs/heads/awesome', - 'repository': { - 'created_at': '2011/09/09 14:20:13 -0700', - 'description': 'source code to readthedocs.org', - 'fork': True, - 'forks': 0, - 'has_downloads': True, - 'has_issues': False, - 'has_wiki': True, - 'homepage': 'http://rtfd.org/', - 'language': 'Python', - 'name': 'readthedocs.org', - 'open_issues': 0, - 'owner': { - 'email': 'XWraithanX@gmail.com', - 'name': 'wraithan', + "ref": "refs/heads/awesome", + "repository": { + "created_at": "2011/09/09 14:20:13 -0700", + "description": "source code to readthedocs.org", + "fork": True, + "forks": 0, + "has_downloads": True, + "has_issues": False, + "has_wiki": True, + "homepage": "http://rtfd.org/", + "language": "Python", + "name": "readthedocs.org", + "open_issues": 0, + "owner": { + "email": "XWraithanX@gmail.com", + "name": "wraithan" }, - 'private': False, - 'pushed_at': '2011/09/12 22:33:34 -0700', - 'size': 140, - 'url': 'https://github.com/rtfd/readthedocs.org', - 'ssh_url': 'git@github.com:rtfd/readthedocs.org.git', - 'watchers': 1, - - }, + "private": False, + "pushed_at": "2011/09/12 22:33:34 -0700", + "size": 140, + "url": "https://github.com/rtfd/readthedocs.org", + "ssh_url": "git@github.com:rtfd/readthedocs.org.git", + "watchers": 1 + + } } def test_post_types(self): - """Ensure various POST formats.""" - r = self.client.post( - '/github/', - data=json.dumps(self.payload), - content_type='application/json', - ) + """Ensure various POST formats""" + r = self.client.post('/github/', + data=json.dumps(self.payload), + content_type='application/json') self.assertEqual(r.status_code, 200) - r = self.client.post( - '/github/', - data=urlencode({'payload': json.dumps(self.payload)}), - content_type='application/x-www-form-urlencoded', - ) + r = self.client.post('/github/', + data=urlencode({'payload': json.dumps(self.payload)}), + content_type='application/x-www-form-urlencoded') self.assertEqual(r.status_code, 200) def test_github_upper_case_repo(self): """ Test the github post commit hook will build properly with upper case repository. - This allows for capitalization differences in post-commit hook URL's. """ payload = self.payload.copy() payload['repository']['url'] = payload['repository']['url'].upper() - r = self.client.post( - '/github/', data=json.dumps(payload), - content_type='application/json', - ) + r = self.client.post('/github/', data=json.dumps(payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: HTTPS://GITHUB.COM/RTFD/READTHEDOCS.ORG [awesome]') self.payload['ref'] = 'refs/heads/not_ok' def test_400_on_no_ref(self): """ GitHub sometimes sends us a post-commit hook without a ref. - - This means we don't know what branch to build, so return a 400. + This means we don't know what branch to build, + so return a 400. """ payload = self.payload.copy() del payload['ref'] - r = self.client.post( - '/github/', data=json.dumps(payload), - content_type='application/json', - ) + r = self.client.post('/github/', data=json.dumps(payload), + content_type='application/json') self.assertEqual(r.status_code, 400) def test_github_request_empty_url(self): @@ -317,7 +293,7 @@ def test_github_request_empty_url(self): self.payload['repository']['ssh_url'] = '' r = self.client.post( '/github/', data=json.dumps(self.payload), - content_type='application/json', + content_type='application/json' ) self.assertEqual(r.status_code, 403) @@ -325,17 +301,16 @@ def test_private_repo_mapping(self): """ Test for private GitHub repo mapping. - Previously we were missing triggering post-commit hooks because we only - compared against the *public* ``github.com/user/repo`` URL. Users can - also enter a ``github.com:user/repo`` URL, which we should support. + Previously we were missing triggering post-commit hooks because + we only compared against the *public* ``github.com/user/repo`` URL. + Users can also enter a ``github.com:user/repo`` URL, + which we should support. """ self.rtfd.repo = 'git@github.com:rtfd/readthedocs.org' self.rtfd.save() payload = self.payload.copy() - r = self.client.post( - '/github/', data=json.dumps(payload), - content_type='application/json', - ) + r = self.client.post('/github/', data=json.dumps(payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: github.com/rtfd/readthedocs.org [awesome]') def test_github_post_commit_hook_builds_branch_docs_if_it_should(self): @@ -344,24 +319,18 @@ def test_github_post_commit_hook_builds_branch_docs_if_it_should(self): versions that are set to be built if the branch they refer to is updated. Otherwise it is no op. """ - r = self.client.post( - '/github/', data=json.dumps(self.payload), - content_type='application/json', - ) + r = self.client.post('/github/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: github.com/rtfd/readthedocs.org [awesome]') self.payload['ref'] = 'refs/heads/not_ok' - r = self.client.post( - '/github/', data=json.dumps(self.payload), - content_type='application/json', - ) + r = self.client.post('/github/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) Not Building: github.com/rtfd/readthedocs.org [not_ok]') self.payload['ref'] = 'refs/heads/unknown' - r = self.client.post( - '/github/', data=json.dumps(self.payload), - content_type='application/json', - ) + r = self.client.post('/github/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) No known branches were pushed to.') def test_github_post_commit_knows_default_branches(self): @@ -375,10 +344,8 @@ def test_github_post_commit_knows_default_branches(self): rtd.save() self.payload['ref'] = 'refs/heads/master' - r = self.client.post( - '/github/', data=json.dumps(self.payload), - content_type='application/json', - ) + r = self.client.post('/github/', data=json.dumps(self.payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: github.com/rtfd/readthedocs.org [latest]') rtd.default_branch = old_default @@ -397,13 +364,13 @@ def test_github_webhook_is_deprecated(self): r = self.client.post( '/github/', data=json.dumps(payload), - content_type='application/json', + content_type='application/json' ) self.assertEqual(r.status_code, 403) class CorePostCommitTest(BasePostCommitTest): - fixtures = ['eric'] + fixtures = ["eric"] def setUp(self): self._setup() @@ -414,10 +381,8 @@ def test_core_commit_hook(self): rtd.save() r = self.client.post('/build/%s' % rtd.pk, {'version_slug': 'master'}) self.assertEqual(r.status_code, 302) - self.assertEqual( - r._headers['location'][1], - '/projects/read-the-docs/builds/', - ) + self.assertEqual(r._headers['location'][1], + '/projects/read-the-docs/builds/') def test_hook_state_tracking(self): rtd = Project.objects.get(slug='read-the-docs') @@ -439,139 +404,123 @@ def setUp(self): self._setup() self.hg_payload = { - 'canon_url': 'https://bitbucket.org', - 'commits': [ + "canon_url": "https://bitbucket.org", + "commits": [ { - 'author': 'marcus', - 'branch': 'default', - 'files': [ + "author": "marcus", + "branch": "default", + "files": [ { - 'file': 'somefile.py', - 'type': 'modified', - }, + "file": "somefile.py", + "type": "modified" + } ], - 'message': 'Added some feature things', - 'node': 'd14d26a93fd2', - 'parents': [ - '1b458191f31a', + "message": "Added some feature things", + "node": "d14d26a93fd2", + "parents": [ + "1b458191f31a" ], - 'raw_author': 'Marcus Bertrand ', - 'raw_node': 'd14d26a93fd28d3166fa81c0cd3b6f339bb95bfe', - 'revision': 3, - 'size': -1, - 'timestamp': '2012-05-30 06:07:03', - 'utctimestamp': '2012-05-30 04:07:03+00:00', - }, + "raw_author": "Marcus Bertrand ", + "raw_node": "d14d26a93fd28d3166fa81c0cd3b6f339bb95bfe", + "revision": 3, + "size": -1, + "timestamp": "2012-05-30 06:07:03", + "utctimestamp": "2012-05-30 04:07:03+00:00" + } ], - 'repository': { - 'absolute_url': '/pip/pip/', - 'fork': False, - 'is_private': True, - 'name': 'Project X', - 'owner': 'marcus', - 'scm': 'hg', - 'slug': 'project-x', - 'website': '', + "repository": { + "absolute_url": "/pip/pip/", + "fork": False, + "is_private": True, + "name": "Project X", + "owner": "marcus", + "scm": "hg", + "slug": "project-x", + "website": "" }, - 'user': 'marcus', + "user": "marcus" } self.git_payload = { - 'canon_url': 'https://bitbucket.org', - 'commits': [ + "canon_url": "https://bitbucket.org", + "commits": [ { - 'author': 'marcus', - 'branch': 'master', - 'files': [ + "author": "marcus", + "branch": "master", + "files": [ { - 'file': 'somefile.py', - 'type': 'modified', - }, + "file": "somefile.py", + "type": "modified" + } ], - 'message': 'Added some more things to somefile.py\n', - 'node': '620ade18607a', - 'parents': [ - '702c70160afc', + "message": "Added some more things to somefile.py\n", + "node": "620ade18607a", + "parents": [ + "702c70160afc" ], - 'raw_author': 'Marcus Bertrand ', - 'raw_node': '620ade18607ac42d872b568bb92acaa9a28620e9', - 'revision': None, - 'size': -1, - 'timestamp': '2012-05-30 05:58:56', - 'utctimestamp': '2012-05-30 03:58:56+00:00', - }, + "raw_author": "Marcus Bertrand ", + "raw_node": "620ade18607ac42d872b568bb92acaa9a28620e9", + "revision": None, + "size": -1, + "timestamp": "2012-05-30 05:58:56", + "utctimestamp": "2012-05-30 03:58:56+00:00" + } ], - 'repository': { - 'absolute_url': '/sphinx/sphinx/', - 'fork': False, - 'is_private': True, - 'name': 'Project X', - 'owner': 'marcus', - 'scm': 'git', - 'slug': 'project-x', - 'website': 'https://atlassian.com/', + "repository": { + "absolute_url": "/sphinx/sphinx/", + "fork": False, + "is_private": True, + "name": "Project X", + "owner": "marcus", + "scm": "git", + "slug": "project-x", + "website": "https://atlassian.com/" }, - 'user': 'marcus', + "user": "marcus" } def test_post_types(self): - """Ensure various POST formats.""" - r = self.client.post( - '/bitbucket/', - data=json.dumps(self.hg_payload), - content_type='application/json', - ) + """Ensure various POST formats""" + r = self.client.post('/bitbucket/', + data=json.dumps(self.hg_payload), + content_type='application/json') self.assertEqual(r.status_code, 200) - r = self.client.post( - '/bitbucket/', - data=urlencode({'payload': json.dumps(self.hg_payload)}), - content_type='application/x-www-form-urlencoded', - ) + r = self.client.post('/bitbucket/', + data=urlencode({'payload': json.dumps(self.hg_payload)}), + content_type='application/x-www-form-urlencoded') self.assertEqual(r.status_code, 200) def test_bitbucket_post_commit(self): - r = self.client.post( - '/bitbucket/', data=json.dumps(self.hg_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.hg_payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: bitbucket.org/pip/pip [latest]') - r = self.client.post( - '/bitbucket/', data=json.dumps(self.git_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.git_payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: bitbucket.org/sphinx/sphinx [latest]') def test_bitbucket_post_commit_empty_commit_list(self): self.hg_payload['commits'] = [] self.git_payload['commits'] = [] - r = self.client.post( - '/bitbucket/', data=json.dumps(self.hg_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.hg_payload), + content_type='application/json') self.assertContains(r, 'Commit/branch not found', status_code=404) - r = self.client.post( - '/bitbucket/', data=json.dumps(self.git_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.git_payload), + content_type='application/json') self.assertContains(r, 'Commit/branch not found', status_code=404) def test_bitbucket_post_commit_non_existent_url(self): self.hg_payload['repository']['absolute_url'] = '/invalid/repository' self.git_payload['repository']['absolute_url'] = '/invalid/repository' - r = self.client.post( - '/bitbucket/', data=json.dumps(self.hg_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.hg_payload), + content_type='application/json') self.assertContains(r, 'Project match not found', status_code=404) - r = self.client.post( - '/bitbucket/', data=json.dumps(self.git_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.git_payload), + content_type='application/json') self.assertContains(r, 'Project match not found', status_code=404) @@ -581,28 +530,22 @@ def test_bitbucket_post_commit_hook_builds_branch_docs_if_it_should(self): versions that are set to be built if the branch they refer to is updated. Otherwise it is no op. """ - r = self.client.post( - '/bitbucket/', data=json.dumps(self.hg_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.hg_payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: bitbucket.org/pip/pip [latest]') self.hg_payload['commits'] = [{ "branch": "not_ok", }] - r = self.client.post( - '/bitbucket/', data=json.dumps(self.hg_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.hg_payload), + content_type='application/json') self.assertContains(r, '(URL Build) Not Building: bitbucket.org/pip/pip [not_ok]') self.hg_payload['commits'] = [{ "branch": "unknown", }] - r = self.client.post( - '/bitbucket/', data=json.dumps(self.hg_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.hg_payload), + content_type='application/json') self.assertContains(r, '(URL Build) No known branches were pushed to.') def test_bitbucket_default_branch(self): @@ -613,16 +556,14 @@ def test_bitbucket_default_branch(self): self.feature.projects.add(self.test_project) self.git_payload['commits'] = [{ - 'branch': 'integration', + "branch": "integration", }] self.git_payload['repository'] = { - 'absolute_url': '/test/project/', + 'absolute_url': '/test/project/' } - r = self.client.post( - '/bitbucket/', data=json.dumps(self.git_payload), - content_type='application/json', - ) + r = self.client.post('/bitbucket/', data=json.dumps(self.git_payload), + content_type='application/json') self.assertContains(r, '(URL Build) Build Started: bitbucket.org/test/project [latest]') def test_bitbucket_request_empty_url(self): @@ -633,7 +574,7 @@ def test_bitbucket_request_empty_url(self): self.git_payload['repository']['absolute_url'] = '' r = self.client.post( '/bitbucket/', data=json.dumps(self.git_payload), - content_type='application/json', + content_type='application/json' ) self.assertEqual(r.status_code, 400) @@ -650,6 +591,6 @@ def test_bitbucket_webhook_is_deprecated(self): r = self.client.post( '/bitbucket/', data=json.dumps(payload), - content_type='application/json', + content_type='application/json' ) self.assertEqual(r.status_code, 403) diff --git a/readthedocs/rtd_tests/tests/test_privacy.py b/readthedocs/rtd_tests/tests/test_privacy.py index 2f55a0fd9e3..8a7359c9055 100644 --- a/readthedocs/rtd_tests/tests/test_privacy.py +++ b/readthedocs/rtd_tests/tests/test_privacy.py @@ -1,18 +1,17 @@ -# -*- coding: utf-8 -*- -import json +from __future__ import absolute_import import logging - +import json import mock -from django.contrib.auth.models import User + from django.test import TestCase from django.test.utils import override_settings +from django.contrib.auth.models import User from readthedocs.builds.constants import LATEST -from readthedocs.builds.models import Build, Version -from readthedocs.projects import tasks -from readthedocs.projects.forms import UpdateProjectForm +from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project - +from readthedocs.projects.forms import UpdateProjectForm +from readthedocs.projects import tasks log = logging.getLogger(__name__) @@ -30,32 +29,27 @@ def setUp(self): tasks.update_docs_task.delay = mock.Mock() - def _create_kong( - self, privacy_level='private', - version_privacy_level='private', - ): + def _create_kong(self, privacy_level='private', + version_privacy_level='private'): self.client.login(username='eric', password='test') log.info( - 'Making kong with privacy: %s and version privacy: %s', + "Making kong with privacy: %s and version privacy: %s", privacy_level, version_privacy_level, ) # Create project via project form, simulate import wizard without magic form = UpdateProjectForm( - data={ - 'repo_type': 'git', - 'repo': 'https://github.com/ericholscher/django-kong', - 'name': 'Django Kong', - 'language': 'en', - 'default_branch': '', - 'project_url': 'http://django-kong.rtfd.org', - 'default_version': LATEST, - 'python_interpreter': 'python', - 'description': 'OOHHH AH AH AH KONG SMASH', - 'documentation_type': 'sphinx', - }, - user=User.objects.get(username='eric'), - ) + data={'repo_type': 'git', + 'repo': 'https://github.com/ericholscher/django-kong', + 'name': 'Django Kong', + 'language': 'en', + 'default_branch': '', + 'project_url': 'http://django-kong.rtfd.org', + 'default_version': LATEST, + 'python_interpreter': 'python', + 'description': 'OOHHH AH AH AH KONG SMASH', + 'documentation_type': 'sphinx'}, + user=User.objects.get(username='eric')) proj = form.save() # Update these directly, no form has all the fields we need proj.privacy_level = privacy_level @@ -135,10 +129,8 @@ def test_private_branch(self): kong = self._create_kong('public', 'private') self.client.login(username='eric', password='test') - Version.objects.create( - project=kong, identifier='test id', - verbose_name='test verbose', privacy_level='private', slug='test-slug', active=True, - ) + Version.objects.create(project=kong, identifier='test id', + verbose_name='test verbose', privacy_level='private', slug='test-slug', active=True) self.assertEqual(Version.objects.count(), 2) self.assertEqual(Version.objects.get(slug='test-slug').privacy_level, 'private') r = self.client.get('/projects/django-kong/') @@ -157,11 +149,9 @@ def test_public_branch(self): kong = self._create_kong('public', 'public') self.client.login(username='eric', password='test') - Version.objects.create( - project=kong, identifier='test id', - verbose_name='test verbose', slug='test-slug', - active=True, built=True, - ) + Version.objects.create(project=kong, identifier='test id', + verbose_name='test verbose', slug='test-slug', + active=True, built=True) self.assertEqual(Version.objects.count(), 2) self.assertEqual(Version.objects.all()[0].privacy_level, 'public') r = self.client.get('/projects/django-kong/') @@ -175,30 +165,22 @@ def test_public_branch(self): def test_public_repo_api(self): self._create_kong('public', 'public') self.client.login(username='eric', password='test') - resp = self.client.get( - 'http://testserver/api/v1/project/django-kong/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/django-kong/", + data={"format": "json"}) self.assertEqual(resp.status_code, 200) - resp = self.client.get( - 'http://testserver/api/v1/project/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/", + data={"format": "json"}) self.assertEqual(resp.status_code, 200) data = json.loads(resp.content) self.assertEqual(data['meta']['total_count'], 1) self.client.login(username='tester', password='test') - resp = self.client.get( - 'http://testserver/api/v1/project/django-kong/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/django-kong/", + data={"format": "json"}) self.assertEqual(resp.status_code, 200) - resp = self.client.get( - 'http://testserver/api/v1/project/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/", + data={"format": "json"}) self.assertEqual(resp.status_code, 200) data = json.loads(resp.content) self.assertEqual(data['meta']['total_count'], 1) @@ -206,29 +188,21 @@ def test_public_repo_api(self): def test_private_repo_api(self): self._create_kong('private', 'private') self.client.login(username='eric', password='test') - resp = self.client.get( - 'http://testserver/api/v1/project/django-kong/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/django-kong/", + data={"format": "json"}) self.assertEqual(resp.status_code, 200) - resp = self.client.get( - 'http://testserver/api/v1/project/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/", + data={"format": "json"}) self.assertEqual(resp.status_code, 200) data = json.loads(resp.content) self.assertEqual(data['meta']['total_count'], 1) self.client.login(username='tester', password='test') - resp = self.client.get( - 'http://testserver/api/v1/project/django-kong/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/django-kong/", + data={"format": "json"}) self.assertEqual(resp.status_code, 404) - resp = self.client.get( - 'http://testserver/api/v1/project/', - data={'format': 'json'}, - ) + resp = self.client.get("http://testserver/api/v1/project/", + data={"format": "json"}) self.assertEqual(resp.status_code, 200) data = json.loads(resp.content) self.assertEqual(data['meta']['total_count'], 0) @@ -237,17 +211,11 @@ def test_private_doc_serving(self): kong = self._create_kong('public', 'private') self.client.login(username='eric', password='test') - Version.objects.create( - project=kong, identifier='test id', - verbose_name='test verbose', privacy_level='private', slug='test-slug', active=True, - ) - self.client.post( - '/dashboard/django-kong/versions/', - { - 'version-test-slug': 'on', - 'privacy-test-slug': 'private', - }, - ) + Version.objects.create(project=kong, identifier='test id', + verbose_name='test verbose', privacy_level='private', slug='test-slug', active=True) + self.client.post('/dashboard/django-kong/versions/', + {'version-test-slug': 'on', + 'privacy-test-slug': 'private'}) r = self.client.get('/docs/django-kong/en/test-slug/') self.client.login(username='eric', password='test') self.assertEqual(r.status_code, 404) @@ -381,10 +349,8 @@ def test_build_filtering(self): kong = self._create_kong('public', 'private') self.client.login(username='eric', password='test') - ver = Version.objects.create( - project=kong, identifier='test id', - verbose_name='test verbose', privacy_level='private', slug='test-slug', active=True, - ) + ver = Version.objects.create(project=kong, identifier='test id', + verbose_name='test verbose', privacy_level='private', slug='test-slug', active=True) r = self.client.get('/projects/django-kong/builds/') self.assertContains(r, 'test-slug') @@ -399,9 +365,11 @@ def test_build_filtering(self): self.assertNotContains(r, 'test-slug') def test_queryset_chaining(self): - """Test that manager methods get set on related querysets.""" + """ + Test that manager methods get set on related querysets. + """ kong = self._create_kong('public', 'private') self.assertEqual( kong.versions.private().get(slug='latest').slug, - 'latest', + 'latest' ) diff --git a/readthedocs/rtd_tests/tests/test_privacy_urls.py b/readthedocs/rtd_tests/tests/test_privacy_urls.py index 9bffbb82558..277c566e22e 100644 --- a/readthedocs/rtd_tests/tests/test_privacy_urls.py +++ b/readthedocs/rtd_tests/tests/test_privacy_urls.py @@ -1,23 +1,25 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import +from __future__ import print_function import re -import mock from allauth.socialaccount.models import SocialAccount +from builtins import object from django.contrib.admindocs.views import extract_views_from_urlpatterns from django.test import TestCase from django.urls import reverse from django_dynamic_fixture import get +import mock from taggit.models import Tag from readthedocs.builds.models import Build, BuildCommandResult from readthedocs.core.utils.tasks import TaskNoPermission from readthedocs.integrations.models import HttpExchange, Integration -from readthedocs.oauth.models import RemoteOrganization, RemoteRepository -from readthedocs.projects.models import Domain, EnvironmentVariable, Project +from readthedocs.projects.models import Project, Domain +from readthedocs.oauth.models import RemoteRepository, RemoteOrganization from readthedocs.rtd_tests.utils import create_user -class URLAccessMixin: +class URLAccessMixin(object): default_kwargs = {} response_data = {} @@ -71,10 +73,8 @@ def assertResponse(self, path, name=None, method=None, data=None, **kwargs): val, ('Attribute mismatch for view {view} ({path}): ' '{key} != {expected} (got {value})' - .format( - view=name, path=path, key=key, expected=val, - value=resp_val, - )), + .format(view=name, path=path, key=key, expected=val, + value=resp_val)) ) return response @@ -93,10 +93,10 @@ def setUp(self): for not_obj in self.context_data: if isinstance(obj, list) or isinstance(obj, set) or isinstance(obj, tuple): self.assertNotIn(not_obj, obj) - print('{} not in {}'.format(not_obj, obj)) + print("%s not in %s" % (not_obj, obj)) else: self.assertNotEqual(not_obj, obj) - print('{} is not {}'.format(not_obj, obj)) + print("%s is not %s" % (not_obj, obj)) def _test_url(self, urlpatterns): deconstructed_urls = extract_views_from_urlpatterns(urlpatterns) @@ -106,8 +106,7 @@ def _test_url(self, urlpatterns): url_ctx = self.get_url_path_ctx() if url_ctx: self.response_data = { - url.format(**url_ctx): data for url, data in self.response_data.items() - } + url.format(**url_ctx): data for url, data in self.response_data.items()} for (view, regex, namespace, name) in deconstructed_urls: request_data = self.request_data.get(name, {}).copy() @@ -126,26 +125,20 @@ def setUp(self): # Previous Fixtures self.owner = create_user(username='owner', password='test') self.tester = create_user(username='tester', password='test') - self.pip = get( - Project, slug='pip', users=[self.owner], - privacy_level='public', main_language_project=None, - ) - self.private = get( - Project, slug='private', privacy_level='private', - main_language_project=None, - ) + self.pip = get(Project, slug='pip', users=[self.owner], + privacy_level='public', main_language_project=None) + self.private = get(Project, slug='private', privacy_level='private', + main_language_project=None) class ProjectMixin(URLAccessMixin): def setUp(self): - super().setUp() + super(ProjectMixin, self).setUp() self.build = get(Build, project=self.pip) self.tag = get(Tag, slug='coolness') - self.subproject = get( - Project, slug='sub', language='ja', - users=[self.owner], main_language_project=None, - ) + self.subproject = get(Project, slug='sub', language='ja', + users=[self.owner], main_language_project=None) self.pip.add_subproject(self.subproject) self.pip.translations.add(self.subproject) self.integration = get(Integration, project=self.pip, provider_data='') @@ -157,7 +150,6 @@ def setUp(self): status_code=200, ) self.domain = get(Domain, url='http://docs.foobar.com', project=self.pip) - self.environment_variable = get(EnvironmentVariable, project=self.pip) self.default_kwargs = { 'project_slug': self.pip.slug, 'subproject_slug': self.subproject.slug, @@ -170,7 +162,6 @@ def setUp(self): 'domain_pk': self.domain.pk, 'integration_pk': self.integration.pk, 'exchange_pk': self.webhook_exchange.pk, - 'environmentvariable_pk': self.environment_variable.pk, } @@ -250,13 +241,11 @@ class PrivateProjectAdminAccessTest(PrivateProjectMixin, TestCase): '/dashboard/pip/integrations/sync/': {'status_code': 405}, '/dashboard/pip/integrations/{integration_id}/sync/': {'status_code': 405}, '/dashboard/pip/integrations/{integration_id}/delete/': {'status_code': 405}, - '/dashboard/pip/environmentvariables/{environmentvariable_id}/delete/': {'status_code': 405}, } def get_url_path_ctx(self): return { 'integration_id': self.integration.id, - 'environmentvariable_id': self.environment_variable.id, } def login(self): @@ -286,7 +275,6 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase): '/dashboard/pip/integrations/sync/': {'status_code': 405}, '/dashboard/pip/integrations/{integration_id}/sync/': {'status_code': 405}, '/dashboard/pip/integrations/{integration_id}/delete/': {'status_code': 405}, - '/dashboard/pip/environmentvariables/{environmentvariable_id}/delete/': {'status_code': 405}, } # Filtered out by queryset on projects that we don't own. @@ -295,7 +283,6 @@ class PrivateProjectUserAccessTest(PrivateProjectMixin, TestCase): def get_url_path_ctx(self): return { 'integration_id': self.integration.id, - 'environmentvariable_id': self.environment_variable.id, } def login(self): @@ -320,7 +307,7 @@ def is_admin(self): class APIMixin(URLAccessMixin): def setUp(self): - super().setUp() + super(APIMixin, self).setUp() self.build = get(Build, project=self.pip) self.build_command_result = get(BuildCommandResult, project=self.pip) self.domain = get(Domain, url='http://docs.foobar.com', project=self.pip) diff --git a/readthedocs/rtd_tests/tests/test_profile_views.py b/readthedocs/rtd_tests/tests/test_profile_views.py index de7c244bf1b..9e3a75a449e 100644 --- a/readthedocs/rtd_tests/tests/test_profile_views.py +++ b/readthedocs/rtd_tests/tests/test_profile_views.py @@ -1,7 +1,8 @@ -# -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from django.contrib.auth.models import User -from django.test import TestCase from django.urls import reverse +from django.test import TestCase from django_dynamic_fixture import get @@ -24,7 +25,7 @@ def test_edit_profile(self): 'first_name': 'Read', 'last_name': 'Docs', 'homepage': 'readthedocs.org', - }, + } ) self.assertTrue(resp.status_code, 200) @@ -46,7 +47,7 @@ def test_edit_profile_with_invalid_values(self): 'first_name': 'a' * 31, 'last_name': 'b' * 31, 'homepage': 'c' * 101, - }, + } ) FORM_ERROR_FORMAT = 'Ensure this value has at most {} characters (it has {}).' @@ -57,20 +58,20 @@ def test_edit_profile_with_invalid_values(self): def test_delete_account(self): resp = self.client.get( - reverse('delete_account'), + reverse('delete_account') ) self.assertEqual(resp.status_code, 200) resp = self.client.post( reverse('delete_account'), data={ 'username': self.user.username, - }, + } ) self.assertEqual(resp.status_code, 302) self.assertEqual(resp['Location'], reverse('homepage')) self.assertFalse( - User.objects.filter(username=self.user.username).exists(), + User.objects.filter(username=self.user.username).exists() ) def test_profile_detail(self): @@ -94,7 +95,7 @@ def test_profile_detail_not_found(self): def test_account_advertising(self): resp = self.client.get( - reverse('account_advertising'), + reverse('account_advertising') ) self.assertEqual(resp.status_code, 200) self.assertTrue(self.user.profile.allow_ads) diff --git a/readthedocs/rtd_tests/tests/test_project.py b/readthedocs/rtd_tests/tests/test_project.py index 4a56bb9395e..679b761a25a 100644 --- a/readthedocs/rtd_tests/tests/test_project.py +++ b/readthedocs/rtd_tests/tests/test_project.py @@ -1,21 +1,20 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import datetime import json from django.contrib.auth.models import User from django.forms.models import model_to_dict from django.test import TestCase -from django.utils import timezone from django_dynamic_fixture import get +from django.utils import timezone from mock import patch from rest_framework.reverse import reverse from readthedocs.builds.constants import ( - BUILD_STATE_CLONING, - BUILD_STATE_FINISHED, - BUILD_STATE_TRIGGERED, - LATEST, -) + BUILD_STATE_CLONING, BUILD_STATE_FINISHED, BUILD_STATE_TRIGGERED, LATEST) from readthedocs.builds.models import Build from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Project @@ -23,7 +22,7 @@ from readthedocs.rtd_tests.mocks.paths import fake_paths_by_regex -class ProjectMixin: +class ProjectMixin(object): fixtures = ['eric', 'test_data'] @@ -104,8 +103,7 @@ def test_conf_file_not_found(self, find_method, full_find_method): full_find_method.return_value = [] with self.assertRaisesMessage( ProjectConfigurationError, - ProjectConfigurationError.NOT_FOUND, - ) as cm: + ProjectConfigurationError.NOT_FOUND) as cm: self.pip.conf_file() @patch('readthedocs.projects.models.Project.find') @@ -117,8 +115,7 @@ def test_multiple_conf_files(self, find_method): ] with self.assertRaisesMessage( ProjectConfigurationError, - ProjectConfigurationError.MULTIPLE_CONF_FILES, - ) as cm: + ProjectConfigurationError.MULTIPLE_CONF_FILES) as cm: self.pip.conf_file() @@ -165,24 +162,23 @@ def test_translation_delete(self): self.assertFalse(Project.objects.filter(pk=project_delete.pk).exists()) self.assertTrue(Project.objects.filter(pk=project_keep.pk).exists()) self.assertIsNone( - Project.objects.get(pk=project_keep.pk).main_language_project, - ) + Project.objects.get(pk=project_keep.pk).main_language_project) def test_user_can_add_own_project_as_translation(self): user_a = User.objects.get(username='eric') project_a = get( Project, users=[user_a], - language='en', main_language_project=None, + language='en', main_language_project=None ) project_b = get( Project, users=[user_a], - language='es', main_language_project=None, + language='es', main_language_project=None ) self.client.login(username=user_a.username, password='test') self.client.post( reverse('projects_translations', args=[project_a.slug]), - data={'project': project_b.slug}, + data={'project': project_b.slug} ) self.assertEqual(project_a.translations.first(), project_b) @@ -194,20 +190,20 @@ def test_user_can_add_project_as_translation_if_is_owner(self): user_a = User.objects.get(username='eric') project_a = get( Project, users=[user_a], - language='es', main_language_project=None, + language='es', main_language_project=None ) user_b = User.objects.get(username='tester') # User A and B are owners of project B project_b = get( Project, users=[user_b, user_a], - language='en', main_language_project=None, + language='en', main_language_project=None ) self.client.login(username=user_a.username, password='test') self.client.post( reverse('projects_translations', args=[project_a.slug]), - data={'project': project_b.slug}, + data={'project': project_b.slug} ) self.assertEqual(project_a.translations.first(), project_b) @@ -217,20 +213,20 @@ def test_user_can_not_add_other_user_project_as_translation(self): user_a = User.objects.get(username='eric') project_a = get( Project, users=[user_a], - language='es', main_language_project=None, + language='es', main_language_project=None ) user_b = User.objects.get(username='tester') project_b = get( Project, users=[user_b], - language='en', main_language_project=None, + language='en', main_language_project=None ) # User A try to add project B as translation of project A self.client.login(username=user_a.username, password='test') resp = self.client.post( reverse('projects_translations', args=[project_a.slug]), - data={'project': project_b.slug}, + data={'project': project_b.slug} ) self.assertContains(resp, 'Select a valid choice') @@ -246,13 +242,13 @@ def test_previous_users_can_list_and_delete_translations_not_owner(self): user_a = User.objects.get(username='eric') project_a = get( Project, users=[user_a], - language='es', main_language_project=None, + language='es', main_language_project=None ) user_b = User.objects.get(username='tester') project_b = get( Project, users=[user_b], - language='en', main_language_project=None, + language='en', main_language_project=None ) project_a.translations.add(project_b) @@ -262,16 +258,16 @@ def test_previous_users_can_list_and_delete_translations_not_owner(self): # Project B is listed under user A translations resp = self.client.get( - reverse('projects_translations', args=[project_a.slug]), + reverse('projects_translations', args=[project_a.slug]) ) self.assertContains(resp, project_b.slug) resp = self.client.post( reverse( 'projects_translations_delete', - args=[project_a.slug, project_b.slug], + args=[project_a.slug, project_b.slug] ), - follow=True, + follow=True ) self.assertEqual(resp.status_code, 200) self.assertNotIn(project_b, project_a.translations.all()) @@ -280,11 +276,11 @@ def test_user_cant_delete_other_user_translations(self): user_a = User.objects.get(username='eric') project_a = get( Project, users=[user_a], - language='es', main_language_project=None, + language='es', main_language_project=None ) project_b = get( Project, users=[user_a], - language='en', main_language_project=None, + language='en', main_language_project=None ) project_a.translations.add(project_b) @@ -293,11 +289,11 @@ def test_user_cant_delete_other_user_translations(self): user_b = User.objects.get(username='tester') project_c = get( Project, users=[user_b], - language='es', main_language_project=None, + language='es', main_language_project=None ) project_d = get( Project, users=[user_b, user_a], - language='en', main_language_project=None, + language='en', main_language_project=None ) project_d.translations.add(project_c) project_d.save() @@ -308,9 +304,9 @@ def test_user_cant_delete_other_user_translations(self): resp = self.client.post( reverse( 'projects_translations_delete', - args=[project_a.slug, project_b.slug], + args=[project_a.slug, project_b.slug] ), - follow=True, + follow=True ) self.assertEqual(resp.status_code, 404) self.assertIn(project_b, project_a.translations.all()) @@ -322,9 +318,9 @@ def test_user_cant_delete_other_user_translations(self): resp = self.client.post( reverse( 'projects_translations_delete', - args=[project_d.slug, project_b.slug], + args=[project_d.slug, project_b.slug] ), - follow=True, + follow=True ) self.assertEqual(resp.status_code, 404) self.assertIn(project_b, project_a.translations.all()) @@ -336,9 +332,9 @@ def test_user_cant_delete_other_user_translations(self): resp = self.client.post( reverse( 'projects_translations_delete', - args=[project_b.slug, project_b.slug], + args=[project_b.slug, project_b.slug] ), - follow=True, + follow=True ) self.assertEqual(resp.status_code, 404) self.assertIn(project_b, project_a.translations.all()) @@ -348,7 +344,7 @@ def test_user_cant_change_lang_to_translation_lang(self): project_a = Project.objects.get(slug='read-the-docs') project_b = get( Project, users=[user_a], - language='es', main_language_project=None, + language='es', main_language_project=None ) project_a.translations.add(project_b) @@ -365,16 +361,16 @@ def test_user_cant_change_lang_to_translation_lang(self): resp = self.client.post( reverse( 'projects_edit', - args=[project_a.slug], + args=[project_a.slug] ), data=data, - follow=True, + follow=True ) self.assertEqual(resp.status_code, 200) self.assertContains( resp, 'There is already a "es" translation ' - 'for the read-the-docs project', + 'for the read-the-docs project' ) def test_user_can_change_project_with_same_lang(self): @@ -382,7 +378,7 @@ def test_user_can_change_project_with_same_lang(self): project_a = Project.objects.get(slug='read-the-docs') project_b = get( Project, users=[user_a], - language='es', main_language_project=None, + language='es', main_language_project=None ) project_a.translations.add(project_b) @@ -399,10 +395,10 @@ def test_user_can_change_project_with_same_lang(self): resp = self.client.post( reverse( 'projects_edit', - args=[project_a.slug], + args=[project_a.slug] ), data=data, - follow=True, + follow=True ) self.assertEqual(resp.status_code, 200) self.assertNotContains(resp, 'There is already a') @@ -433,8 +429,7 @@ def setUp(self): state=BUILD_STATE_TRIGGERED, ) self.build_2.date = ( - timezone.now() - datetime.timedelta(hours=1) - ) + timezone.now() - datetime.timedelta(hours=1)) self.build_2.save() # Build started an hour ago with custom time (2 hours) @@ -444,8 +439,7 @@ def setUp(self): state=BUILD_STATE_TRIGGERED, ) self.build_3.date = ( - timezone.now() - datetime.timedelta(hours=1) - ) + timezone.now() - datetime.timedelta(hours=1)) self.build_3.save() def test_finish_inactive_builds_task(self): diff --git a/readthedocs/rtd_tests/tests/test_project_forms.py b/readthedocs/rtd_tests/tests/test_project_forms.py index cd927eed564..6d358123b65 100644 --- a/readthedocs/rtd_tests/tests/test_project_forms.py +++ b/readthedocs/rtd_tests/tests/test_project_forms.py @@ -1,5 +1,12 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import mock from django.contrib.auth.models import User from django.test import TestCase @@ -18,14 +25,13 @@ ) from readthedocs.projects.exceptions import ProjectSpamError from readthedocs.projects.forms import ( - EnvironmentVariableForm, ProjectAdvancedForm, ProjectBasicsForm, ProjectExtraForm, TranslationForm, UpdateProjectForm, ) -from readthedocs.projects.models import EnvironmentVariable, Project +from readthedocs.projects.models import Project class TestProjectForms(TestCase): @@ -160,7 +166,7 @@ def test_changing_vcs_should_not_change_latest_is_not_none(self): def test_length_of_tags(self): data = { 'documentation_type': 'sphinx', - 'language': 'en', + 'language': 'en' } data['tags'] = '{},{}'.format('a'*50, 'b'*99) form = ProjectExtraForm(data) @@ -169,7 +175,7 @@ def test_length_of_tags(self): data['tags'] = '{},{}'.format('a'*90, 'b'*100) form = ProjectExtraForm(data) self.assertTrue(form.is_valid()) - + data['tags'] = '{},{}'.format('a'*99, 'b'*101) form = ProjectExtraForm(data) self.assertFalse(form.is_valid()) @@ -212,7 +218,7 @@ def setUp(self): slug='public-4', active=False, privacy_level=PUBLIC, - identifier='public/4', + identifier='public/4' ) get( Version, @@ -236,10 +242,10 @@ def test_list_only_active_versions_on_default_version(self): # This version is created automatically by the project on save self.assertTrue(self.project.versions.filter(slug=LATEST).exists()) self.assertEqual( - { + set( slug for slug, _ in form.fields['default_version'].widget.choices - }, + ), {'latest', 'public-1', 'public-2', 'private', 'protected'}, ) @@ -248,13 +254,13 @@ def test_list_all_versions_on_default_branch(self): # This version is created automatically by the project on save self.assertTrue(self.project.versions.filter(slug=LATEST).exists()) self.assertEqual( - { + set( identifier for identifier, _ in form.fields['default_branch'].widget.choices - }, + ), { None, 'master', 'public-1', 'public-2', - 'public-3', 'public/4', 'protected', 'private', + 'public-3', 'public/4', 'protected', 'private' }, ) @@ -275,7 +281,7 @@ def setUp(self): self.project_s_fr = self.get_project( lang='fr', - users=[self.user_b, self.user_a], + users=[self.user_b, self.user_a] ) def get_project(self, lang, users, **kwargs): @@ -300,7 +306,7 @@ def test_list_only_owner_projects(self): ] self.assertEqual( {proj_slug for proj_slug, _ in form.fields['project'].choices}, - {project.slug for project in expected_projects}, + {project.slug for project in expected_projects} ) form = TranslationForm( @@ -315,7 +321,7 @@ def test_list_only_owner_projects(self): ] self.assertEqual( {proj_slug for proj_slug, _ in form.fields['project'].choices}, - {project.slug for project in expected_projects}, + {project.slug for project in expected_projects} ) def test_excludes_existing_translations(self): @@ -336,7 +342,7 @@ def test_excludes_existing_translations(self): ] self.assertEqual( {proj_slug for proj_slug, _ in form.fields['project'].choices}, - {project.slug for project in expected_projects}, + {project.slug for project in expected_projects} ) def test_user_cant_add_other_user_project(self): @@ -348,11 +354,11 @@ def test_user_cant_add_other_user_project(self): self.assertFalse(form.is_valid()) self.assertIn( 'Select a valid choice', - ''.join(form.errors['project']), + ''.join(form.errors['project']) ) self.assertNotIn( self.project_f_ar, - [proj_slug for proj_slug, _ in form.fields['project'].choices], + [proj_slug for proj_slug, _ in form.fields['project'].choices] ) def test_user_cant_add_project_with_same_lang(self): @@ -364,7 +370,7 @@ def test_user_cant_add_project_with_same_lang(self): self.assertFalse(form.is_valid()) self.assertIn( 'Both projects can not have the same language (English).', - ''.join(form.errors['project']), + ''.join(form.errors['project']) ) def test_user_cant_add_project_with_same_lang_of_other_translation(self): @@ -379,7 +385,7 @@ def test_user_cant_add_project_with_same_lang_of_other_translation(self): self.assertFalse(form.is_valid()) self.assertIn( 'This project already has a translation for English.', - ''.join(form.errors['project']), + ''.join(form.errors['project']) ) def test_no_nesting_translation(self): @@ -394,7 +400,7 @@ def test_no_nesting_translation(self): self.assertFalse(form.is_valid()) self.assertIn( 'Select a valid choice', - ''.join(form.errors['project']), + ''.join(form.errors['project']) ) def test_no_nesting_translation_case_2(self): @@ -409,7 +415,7 @@ def test_no_nesting_translation_case_2(self): self.assertFalse(form.is_valid()) self.assertIn( 'A project with existing translations can not', - ''.join(form.errors['project']), + ''.join(form.errors['project']) ) def test_not_already_translation(self): @@ -424,7 +430,7 @@ def test_not_already_translation(self): self.assertFalse(form.is_valid()) self.assertIn( 'is already a translation', - ''.join(form.errors['project']), + ''.join(form.errors['project']) ) def test_cant_change_language_to_translation_lang(self): @@ -438,12 +444,12 @@ def test_cant_change_language_to_translation_lang(self): 'documentation_type': 'sphinx', 'language': 'en', }, - instance=self.project_a_es, + instance=self.project_a_es ) self.assertFalse(form.is_valid()) self.assertIn( 'There is already a "en" translation', - ''.join(form.errors['language']), + ''.join(form.errors['language']) ) # Translation tries to change lang @@ -452,12 +458,12 @@ def test_cant_change_language_to_translation_lang(self): 'documentation_type': 'sphinx', 'language': 'es', }, - instance=self.project_b_en, + instance=self.project_b_en ) self.assertFalse(form.is_valid()) self.assertIn( 'There is already a "es" translation', - ''.join(form.errors['language']), + ''.join(form.errors['language']) ) # Translation tries to change lang @@ -467,12 +473,12 @@ def test_cant_change_language_to_translation_lang(self): 'documentation_type': 'sphinx', 'language': 'br', }, - instance=self.project_b_en, + instance=self.project_b_en ) self.assertFalse(form.is_valid()) self.assertIn( 'There is already a "br" translation', - ''.join(form.errors['language']), + ''.join(form.errors['language']) ) def test_can_change_language_to_self_lang(self): @@ -489,7 +495,7 @@ def test_can_change_language_to_self_lang(self): 'documentation_type': 'sphinx', 'language': 'es', }, - instance=self.project_a_es, + instance=self.project_a_es ) self.assertTrue(form.is_valid()) @@ -502,92 +508,6 @@ def test_can_change_language_to_self_lang(self): 'documentation_type': 'sphinx', 'language': 'en', }, - instance=self.project_b_en, + instance=self.project_b_en ) self.assertTrue(form.is_valid()) - - -class TestProjectEnvironmentVariablesForm(TestCase): - - def setUp(self): - self.project = get(Project) - - def test_use_invalid_names(self): - data = { - 'name': 'VARIABLE WITH SPACES', - 'value': 'string here', - } - form = EnvironmentVariableForm(data, project=self.project) - self.assertFalse(form.is_valid()) - self.assertIn( - "Variable name can't contain spaces", - form.errors['name'], - ) - - data = { - 'name': 'READTHEDOCS__INVALID', - 'value': 'string here', - } - form = EnvironmentVariableForm(data, project=self.project) - self.assertFalse(form.is_valid()) - self.assertIn( - "Variable name can't start with READTHEDOCS", - form.errors['name'], - ) - - data = { - 'name': 'INVALID_CHAR*', - 'value': 'string here', - } - form = EnvironmentVariableForm(data, project=self.project) - self.assertFalse(form.is_valid()) - self.assertIn( - 'Only letters, numbers and underscore are allowed', - form.errors['name'], - ) - - data = { - 'name': '__INVALID', - 'value': 'string here', - } - form = EnvironmentVariableForm(data, project=self.project) - self.assertFalse(form.is_valid()) - self.assertIn( - "Variable name can't start with __ (double underscore)", - form.errors['name'], - ) - - get(EnvironmentVariable, name='EXISTENT_VAR', project=self.project) - data = { - 'name': 'EXISTENT_VAR', - 'value': 'string here', - } - form = EnvironmentVariableForm(data, project=self.project) - self.assertFalse(form.is_valid()) - self.assertIn( - 'There is already a variable with this name for this project', - form.errors['name'], - ) - - def test_create(self): - data = { - 'name': 'MYTOKEN', - 'value': 'string here', - } - form = EnvironmentVariableForm(data, project=self.project) - form.save() - - self.assertEqual(EnvironmentVariable.objects.count(), 1) - self.assertEqual(EnvironmentVariable.objects.first().name, 'MYTOKEN') - self.assertEqual(EnvironmentVariable.objects.first().value, "'string here'") - - data = { - 'name': 'ESCAPED', - 'value': r'string escaped here: #$\1[]{}\|', - } - form = EnvironmentVariableForm(data, project=self.project) - form.save() - - self.assertEqual(EnvironmentVariable.objects.count(), 2) - self.assertEqual(EnvironmentVariable.objects.first().name, 'ESCAPED') - self.assertEqual(EnvironmentVariable.objects.first().value, r"'string escaped here: #$\1[]{}\|'") diff --git a/readthedocs/rtd_tests/tests/test_project_querysets.py b/readthedocs/rtd_tests/tests/test_project_querysets.py index 01f02e986a3..682e54cd9ea 100644 --- a/readthedocs/rtd_tests/tests/test_project_querysets.py +++ b/readthedocs/rtd_tests/tests/test_project_querysets.py @@ -1,15 +1,13 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import + from datetime import timedelta import django_dynamic_fixture as fixture -from django.contrib.auth.models import User from django.test import TestCase -from readthedocs.projects.models import Feature, Project -from readthedocs.projects.querysets import ( - ChildRelatedProjectQuerySet, - ParentRelatedProjectQuerySet, -) +from readthedocs.projects.models import Project, ProjectRelationship, Feature +from readthedocs.projects.querysets import (ParentRelatedProjectQuerySet, + ChildRelatedProjectQuerySet) class ProjectQuerySetTests(TestCase): @@ -24,12 +22,12 @@ def test_subproject_queryset_as_manager_gets_correct_class(self): mgr = ChildRelatedProjectQuerySet.as_manager() self.assertEqual( mgr.__class__.__name__, - 'ManagerFromChildRelatedProjectQuerySetBase', + 'ManagerFromChildRelatedProjectQuerySetBase' ) mgr = ParentRelatedProjectQuerySet.as_manager() self.assertEqual( mgr.__class__.__name__, - 'ManagerFromParentRelatedProjectQuerySetBase', + 'ManagerFromParentRelatedProjectQuerySetBase' ) def test_is_active(self): @@ -39,12 +37,6 @@ def test_is_active(self): project = fixture.get(Project, skip=True) self.assertFalse(Project.objects.is_active(project)) - user = fixture.get(User) - user.profile.banned = True - user.profile.save() - project = fixture.get(Project, skip=False, users=[user]) - self.assertFalse(Project.objects.is_active(project)) - class FeatureQuerySetTests(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_project_symlinks.py b/readthedocs/rtd_tests/tests/test_project_symlinks.py index b3075167eb4..307e4e4516f 100644 --- a/readthedocs/rtd_tests/tests/test_project_symlinks.py +++ b/readthedocs/rtd_tests/tests/test_project_symlinks.py @@ -1,28 +1,25 @@ # -*- coding: utf-8 -*- +from __future__ import absolute_import +from builtins import object import os import shutil import tempfile import mock from django.conf import settings -from django.test import TestCase, override_settings from django.urls import reverse +from django.test import TestCase, override_settings from django_dynamic_fixture import get from readthedocs.builds.models import Version -from readthedocs.core.symlink import PrivateSymlink, PublicSymlink -from readthedocs.projects.models import Domain, Project -from readthedocs.projects.tasks import ( - broadcast_remove_orphan_symlinks, - remove_orphan_symlinks, - symlink_project, -) +from readthedocs.projects.models import Project, Domain +from readthedocs.projects.tasks import broadcast_remove_orphan_symlinks, remove_orphan_symlinks, symlink_project +from readthedocs.core.symlink import PublicSymlink, PrivateSymlink def get_filesystem(path, top_level_path=None): - """ - Recurse into path, return dictionary mapping of path and files. + """Recurse into path, return dictionary mapping of path and files This will return the path `path` as a nested dictionary of path objects. Directories are mapped to dictionary objects, file objects will have a @@ -42,10 +39,8 @@ def get_filesystem(path, top_level_path=None): if os.path.islink(full_path): fs[child] = { 'type': 'link', - 'target': os.path.relpath( - os.path.realpath(full_path), - top_level_path, - ), + 'target': os.path.relpath(os.path.realpath(full_path), + top_level_path) } elif os.path.isfile(full_path): fs[child] = { @@ -79,47 +74,47 @@ def setUp(self): self.mocks = { 'PublicSymlinkBase.CNAME_ROOT': mock.patch( 'readthedocs.core.symlink.PublicSymlinkBase.CNAME_ROOT', - new_callable=mock.PropertyMock, + new_callable=mock.PropertyMock ), 'PublicSymlinkBase.WEB_ROOT': mock.patch( 'readthedocs.core.symlink.PublicSymlinkBase.WEB_ROOT', - new_callable=mock.PropertyMock, + new_callable=mock.PropertyMock ), 'PublicSymlinkBase.PROJECT_CNAME_ROOT': mock.patch( 'readthedocs.core.symlink.PublicSymlinkBase.PROJECT_CNAME_ROOT', - new_callable=mock.PropertyMock, + new_callable=mock.PropertyMock ), 'PrivateSymlinkBase.CNAME_ROOT': mock.patch( 'readthedocs.core.symlink.PrivateSymlinkBase.CNAME_ROOT', - new_callable=mock.PropertyMock, + new_callable=mock.PropertyMock ), 'PrivateSymlinkBase.WEB_ROOT': mock.patch( 'readthedocs.core.symlink.PrivateSymlinkBase.WEB_ROOT', - new_callable=mock.PropertyMock, + new_callable=mock.PropertyMock ), 'PrivateSymlinkBase.PROJECT_CNAME_ROOT': mock.patch( 'readthedocs.core.symlink.PrivateSymlinkBase.PROJECT_CNAME_ROOT', - new_callable=mock.PropertyMock, + new_callable=mock.PropertyMock ), } - self.patches = {key: mock.start() for (key, mock) in list(self.mocks.items())} + self.patches = dict((key, mock.start()) for (key, mock) in list(self.mocks.items())) self.patches['PublicSymlinkBase.CNAME_ROOT'].return_value = os.path.join( - settings.SITE_ROOT, 'public_cname_root', + settings.SITE_ROOT, 'public_cname_root' ) self.patches['PublicSymlinkBase.WEB_ROOT'].return_value = os.path.join( - settings.SITE_ROOT, 'public_web_root', + settings.SITE_ROOT, 'public_web_root' ) self.patches['PublicSymlinkBase.PROJECT_CNAME_ROOT'].return_value = os.path.join( - settings.SITE_ROOT, 'public_cname_project', + settings.SITE_ROOT, 'public_cname_project' ) self.patches['PrivateSymlinkBase.CNAME_ROOT'].return_value = os.path.join( - settings.SITE_ROOT, 'private_cname_root', + settings.SITE_ROOT, 'private_cname_root' ) self.patches['PrivateSymlinkBase.WEB_ROOT'].return_value = os.path.join( - settings.SITE_ROOT, 'private_web_root', + settings.SITE_ROOT, 'private_web_root' ) self.patches['PrivateSymlinkBase.PROJECT_CNAME_ROOT'].return_value = os.path.join( - settings.SITE_ROOT, 'private_cname_project', + settings.SITE_ROOT, 'private_cname_project' ) def tearDown(self): @@ -132,34 +127,30 @@ def assertFilesystem(self, filesystem): self.assertEqual(filesystem, get_filesystem(settings.SITE_ROOT)) -class BaseSymlinkCnames: +class BaseSymlinkCnames(object): def setUp(self): - super().setUp() - self.project = get( - Project, slug='kong', privacy_level=self.privacy, - main_language_project=None, - ) + super(BaseSymlinkCnames, self).setUp() + self.project = get(Project, slug='kong', privacy_level=self.privacy, + main_language_project=None) self.project.versions.update(privacy_level=self.privacy) self.project.save() self.symlink = self.symlink_class(self.project) def test_symlink_cname(self): - self.domain = get( - Domain, project=self.project, domain='woot.com', - url='http://woot.com', cname=True, - ) + self.domain = get(Domain, project=self.project, domain='woot.com', + url='http://woot.com', cname=True) self.symlink.symlink_cnames() filesystem = { 'private_cname_project': { - 'woot.com': {'type': 'link', 'target': 'user_builds/kong'}, + 'woot.com': {'type': 'link', 'target': 'user_builds/kong'} }, 'private_cname_root': { 'woot.com': {'type': 'link', 'target': 'private_web_root/kong'}, }, 'private_web_root': {'kong': {'en': {}}}, 'public_cname_project': { - 'woot.com': {'type': 'link', 'target': 'user_builds/kong'}, + 'woot.com': {'type': 'link', 'target': 'user_builds/kong'} }, 'public_cname_root': { 'woot.com': {'type': 'link', 'target': 'public_web_root/kong'}, @@ -168,8 +159,8 @@ def test_symlink_cname(self): 'kong': {'en': {'latest': { 'type': 'link', 'target': 'user_builds/kong/rtd-builds/latest', - }}}, - }, + }}} + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -179,10 +170,8 @@ def test_symlink_cname(self): self.assertFilesystem(filesystem) def test_symlink_remove_orphan_symlinks(self): - self.domain = get( - Domain, project=self.project, domain='woot.com', - url='http://woot.com', cname=True, - ) + self.domain = get(Domain, project=self.project, domain='woot.com', + url='http://woot.com', cname=True) self.symlink.symlink_cnames() # Editing the Domain and calling save will symlink the new domain and @@ -212,8 +201,8 @@ def test_symlink_remove_orphan_symlinks(self): 'kong': {'en': {'latest': { 'type': 'link', 'target': 'user_builds/kong/rtd-builds/latest', - }}}, - }, + }}} + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -264,22 +253,20 @@ def test_broadcast_remove_orphan_symlinks(self): ) def test_symlink_cname_dont_link_missing_domains(self): - """Domains should be relinked after deletion.""" - self.domain = get( - Domain, project=self.project, domain='woot.com', - url='http://woot.com', cname=True, - ) + """Domains should be relinked after deletion""" + self.domain = get(Domain, project=self.project, domain='woot.com', + url='http://woot.com', cname=True) self.symlink.symlink_cnames() filesystem = { 'private_cname_project': { - 'woot.com': {'type': 'link', 'target': 'user_builds/kong'}, + 'woot.com': {'type': 'link', 'target': 'user_builds/kong'} }, 'private_cname_root': { 'woot.com': {'type': 'link', 'target': 'private_web_root/kong'}, }, 'private_web_root': {'kong': {'en': {}}}, 'public_cname_project': { - 'woot.com': {'type': 'link', 'target': 'user_builds/kong'}, + 'woot.com': {'type': 'link', 'target': 'user_builds/kong'} }, 'public_cname_root': { 'woot.com': {'type': 'link', 'target': 'public_web_root/kong'}, @@ -288,8 +275,8 @@ def test_symlink_cname_dont_link_missing_domains(self): 'kong': {'en': {'latest': { 'type': 'link', 'target': 'user_builds/kong/rtd-builds/latest', - }}}, - }, + }}} + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -316,26 +303,22 @@ class TestPrivateSymlinkCnames(BaseSymlinkCnames, TempSiteRootTestCase): symlink_class = PrivateSymlink -class BaseSubprojects: +class BaseSubprojects(object): def setUp(self): - super().setUp() - self.project = get( - Project, slug='kong', privacy_level=self.privacy, - main_language_project=None, - ) + super(BaseSubprojects, self).setUp() + self.project = get(Project, slug='kong', privacy_level=self.privacy, + main_language_project=None) self.project.versions.update(privacy_level=self.privacy) self.project.save() - self.subproject = get( - Project, slug='sub', privacy_level=self.privacy, - main_language_project=None, - ) + self.subproject = get(Project, slug='sub', privacy_level=self.privacy, + main_language_project=None) self.subproject.versions.update(privacy_level=self.privacy) self.subproject.save() self.symlink = self.symlink_class(self.project) def test_subproject_normal(self): - """Symlink pass adds symlink for subproject.""" + """Symlink pass adds symlink for subproject""" self.project.add_subproject(self.subproject) self.symlink.symlink_subprojects() filesystem = { @@ -357,16 +340,16 @@ def test_subproject_normal(self): 'sub': { 'type': 'link', 'target': 'public_web_root/sub', - }, - }, + } + } }, 'sub': { 'en': {'latest': { 'type': 'link', 'target': 'user_builds/sub/rtd-builds/latest', - }}, - }, - }, + }} + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -377,7 +360,7 @@ def test_subproject_normal(self): self.assertFilesystem(filesystem) def test_subproject_alias(self): - """Symlink pass adds symlink for subproject alias.""" + """Symlink pass adds symlink for subproject alias""" self.project.add_subproject(self.subproject, alias='sweet-alias') self.symlink.symlink_subprojects() filesystem = { @@ -404,15 +387,15 @@ def test_subproject_alias(self): 'type': 'link', 'target': 'public_web_root/sub', }, - }, + } }, 'sub': { 'en': {'latest': { 'type': 'link', 'target': 'user_builds/sub/rtd-builds/latest', - }}, - }, - }, + }} + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -424,7 +407,7 @@ def test_subproject_alias(self): self.assertFilesystem(filesystem) def test_subproject_alias_with_spaces(self): - """Symlink pass adds symlink for subproject alias.""" + """Symlink pass adds symlink for subproject alias""" self.project.add_subproject(self.subproject, alias='Sweet Alias') self.symlink.symlink_subprojects() filesystem = { @@ -451,15 +434,15 @@ def test_subproject_alias_with_spaces(self): 'type': 'link', 'target': 'public_web_root/sub', }, - }, + } }, 'sub': { 'en': {'latest': { 'type': 'link', 'target': 'user_builds/sub/rtd-builds/latest', - }}, - }, - }, + }} + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -471,7 +454,7 @@ def test_subproject_alias_with_spaces(self): self.assertFilesystem(filesystem) def test_remove_subprojects(self): - """Nonexistent subprojects are unlinked.""" + """Nonexistent subprojects are unlinked""" self.project.add_subproject(self.subproject) self.symlink.symlink_subprojects() filesystem = { @@ -493,16 +476,16 @@ def test_remove_subprojects(self): 'sub': { 'type': 'link', 'target': 'public_web_root/sub', - }, - }, + } + } }, 'sub': { 'en': {'latest': { 'type': 'link', 'target': 'user_builds/sub/rtd-builds/latest', - }}, - }, - }, + }} + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -531,37 +514,29 @@ class TestPrivateSubprojects(BaseSubprojects, TempSiteRootTestCase): symlink_class = PrivateSymlink -class BaseSymlinkTranslations: +class BaseSymlinkTranslations(object): def setUp(self): - super().setUp() - self.project = get( - Project, slug='kong', privacy_level=self.privacy, - main_language_project=None, - ) + super(BaseSymlinkTranslations, self).setUp() + self.project = get(Project, slug='kong', privacy_level=self.privacy, + main_language_project=None) self.project.versions.update(privacy_level=self.privacy) self.project.save() - self.translation = get( - Project, slug='pip', language='de', - privacy_level=self.privacy, - main_language_project=None, - ) + self.translation = get(Project, slug='pip', language='de', + privacy_level=self.privacy, + main_language_project=None) self.translation.versions.update(privacy_level=self.privacy) self.translation.save() self.project.translations.add(self.translation) self.symlink = self.symlink_class(self.project) - get( - Version, slug='master', verbose_name='master', active=True, - project=self.project, privacy_level=self.privacy, - ) - get( - Version, slug='master', verbose_name='master', active=True, - project=self.translation, privacy_level=self.privacy, - ) + get(Version, slug='master', verbose_name='master', active=True, + project=self.project, privacy_level=self.privacy) + get(Version, slug='master', verbose_name='master', active=True, + project=self.translation, privacy_level=self.privacy) self.assertIn(self.translation, self.project.translations.all()) def test_symlink_basic(self): - """Test basic scenario, language english, translation german.""" + """Test basic scenario, language english, translation german""" self.symlink.symlink_translations() filesystem = { 'private_cname_project': {}, @@ -599,9 +574,9 @@ def test_symlink_basic(self): 'type': 'link', 'target': 'user_builds/pip/rtd-builds/master', }, - }, - }, - }, + } + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -612,7 +587,7 @@ def test_symlink_basic(self): self.assertFilesystem(filesystem) def test_symlink_non_english(self): - """Test language german, translation english.""" + """Test language german, translation english""" self.project.language = 'de' self.translation.language = 'en' self.project.save() @@ -654,9 +629,9 @@ def test_symlink_non_english(self): 'type': 'link', 'target': 'user_builds/pip/rtd-builds/master', }, - }, - }, - }, + } + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -667,8 +642,7 @@ def test_symlink_non_english(self): self.assertFilesystem(filesystem) def test_symlink_no_english(self): - """ - Test language german, no english. + """Test language german, no english This should symlink the translation to 'en' even though there is no 'en' language in translations or project language @@ -711,9 +685,9 @@ def test_symlink_no_english(self): 'type': 'link', 'target': 'user_builds/pip/rtd-builds/master', }, - }, - }, - }, + } + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -760,9 +734,9 @@ def test_remove_language(self): 'type': 'link', 'target': 'user_builds/pip/rtd-builds/master', }, - }, - }, - }, + } + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -792,14 +766,12 @@ class TestPrivateSymlinkTranslations(BaseSymlinkTranslations, TempSiteRootTestCa symlink_class = PrivateSymlink -class BaseSymlinkSingleVersion: +class BaseSymlinkSingleVersion(object): def setUp(self): - super().setUp() - self.project = get( - Project, slug='kong', privacy_level=self.privacy, - main_language_project=None, - ) + super(BaseSymlinkSingleVersion, self).setUp() + self.project = get(Project, slug='kong', privacy_level=self.privacy, + main_language_project=None) self.project.versions.update(privacy_level=self.privacy) self.project.save() self.version = self.project.versions.get(slug='latest') @@ -822,7 +794,7 @@ def test_symlink_single_version(self): 'type': 'link', 'target': 'user_builds/kong/rtd-builds/latest', }, - }, + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -848,8 +820,8 @@ def test_symlink_single_version_missing(self): 'kong': { 'type': 'link', 'target': 'user_builds/kong/rtd-builds/latest', - }, - }, + } + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -869,21 +841,17 @@ class TestPublicSymlinkSingleVersion(BaseSymlinkSingleVersion, TempSiteRootTestC symlink_class = PrivateSymlink -class BaseSymlinkVersions: +class BaseSymlinkVersions(object): def setUp(self): - super().setUp() - self.project = get( - Project, slug='kong', privacy_level=self.privacy, - main_language_project=None, - ) + super(BaseSymlinkVersions, self).setUp() + self.project = get(Project, slug='kong', privacy_level=self.privacy, + main_language_project=None) self.project.versions.update(privacy_level=self.privacy) self.project.save() - self.stable = get( - Version, slug='stable', verbose_name='stable', - active=True, project=self.project, - privacy_level=self.privacy, - ) + self.stable = get(Version, slug='stable', verbose_name='stable', + active=True, project=self.project, + privacy_level=self.privacy) self.project.versions.update(privacy_level=self.privacy) self.symlink = self.symlink_class(self.project) @@ -910,7 +878,7 @@ def test_symlink_versions(self): }, }, }, - }, + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -940,7 +908,7 @@ def test_removed_versions(self): 'target': 'user_builds/kong/rtd-builds/stable', }, }}, - }, + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -979,7 +947,7 @@ def test_symlink_other_versions(self): 'type': 'link', 'target': 'user_builds/kong/rtd-builds/latest', }}}, - }, + } } if self.privacy == 'private': public_root = filesystem['public_web_root'].copy() @@ -1002,16 +970,12 @@ class TestPrivateSymlinkVersions(BaseSymlinkVersions, TempSiteRootTestCase): class TestPublicSymlinkUnicode(TempSiteRootTestCase): def setUp(self): - super().setUp() - self.project = get( - Project, slug='kong', name='foo-∫', - main_language_project=None, - ) + super(TestPublicSymlinkUnicode, self).setUp() + self.project = get(Project, slug='kong', name=u'foo-∫', + main_language_project=None) self.project.save() - self.stable = get( - Version, slug='foo-a', verbose_name='foo-∂', - active=True, project=self.project, - ) + self.stable = get(Version, slug='foo-a', verbose_name=u'foo-∂', + active=True, project=self.project) self.symlink = PublicSymlink(self.project) def test_symlink_no_error(self): @@ -1071,21 +1035,19 @@ def test_symlink_broadcast_calls_on_project_save(self): class TestPublicPrivateSymlink(TempSiteRootTestCase): def setUp(self): - super().setUp() + super(TestPublicPrivateSymlink, self).setUp() from django.contrib.auth.models import User self.user = get(User) self.project = get( Project, name='project', slug='project', privacy_level='public', - users=[self.user], main_language_project=None, - ) + users=[self.user], main_language_project=None) self.project.versions.update(privacy_level='public') self.project.save() self.subproject = get( Project, name='subproject', slug='subproject', privacy_level='public', - users=[self.user], main_language_project=None, - ) + users=[self.user], main_language_project=None) self.subproject.versions.update(privacy_level='public') self.subproject.save() @@ -1093,8 +1055,8 @@ def test_change_subproject_privacy(self): """ Change subproject's ``privacy_level`` creates proper symlinks. - When the ``privacy_level`` changes in the subprojects, we need to re- - symlink the superproject also to keep in sync its symlink under the + When the ``privacy_level`` changes in the subprojects, we need to + re-symlink the superproject also to keep in sync its symlink under the private/public roots. """ filesystem_before = { @@ -1190,13 +1152,11 @@ def test_change_subproject_privacy(self): self.client.force_login(self.user) self.client.post( - reverse( - 'project_version_detail', - kwargs={ - 'project_slug': self.subproject.slug, - 'version_slug': self.subproject.versions.first().slug, - }, - ), + reverse('project_version_detail', + kwargs={ + 'project_slug': self.subproject.slug, + 'version_slug': self.subproject.versions.first().slug, + }), data={'privacy_level': 'private', 'active': True}, ) @@ -1204,12 +1164,10 @@ def test_change_subproject_privacy(self): self.assertTrue(self.subproject.versions.first().active) self.client.post( - reverse( - 'projects_advanced', - kwargs={ - 'project_slug': self.subproject.slug, - }, - ), + reverse('projects_advanced', + kwargs={ + 'project_slug': self.subproject.slug, + }), data={ # Required defaults 'python_interpreter': 'python', diff --git a/readthedocs/rtd_tests/tests/test_project_views.py b/readthedocs/rtd_tests/tests/test_project_views.py index 0e7225ac65e..c6e4a93f884 100644 --- a/readthedocs/rtd_tests/tests/test_project_views.py +++ b/readthedocs/rtd_tests/tests/test_project_views.py @@ -1,28 +1,28 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import from datetime import timedelta + +from mock import patch +from django.test import TestCase from django.contrib.auth.models import User from django.contrib.messages import constants as message_const -from django.http.response import HttpResponseRedirect -from django.test import TestCase from django.urls import reverse -from django.utils import timezone +from django.http.response import HttpResponseRedirect from django.views.generic.base import ContextMixin +from django.utils import timezone from django_dynamic_fixture import get, new -from mock import patch + +import six from readthedocs.builds.models import Build, Version +from readthedocs.rtd_tests.base import (WizardTestCase, MockBuildTestCase, + RequestFactoryTestMixin) from readthedocs.oauth.models import RemoteRepository -from readthedocs.projects import tasks from readthedocs.projects.exceptions import ProjectSpamError -from readthedocs.projects.models import Domain, Project -from readthedocs.projects.views.mixins import ProjectRelationMixin +from readthedocs.projects.models import Project, Domain from readthedocs.projects.views.private import ImportWizardView -from readthedocs.rtd_tests.base import ( - MockBuildTestCase, - RequestFactoryTestMixin, - WizardTestCase, -) +from readthedocs.projects.views.mixins import ProjectRelationMixin +from readthedocs.projects import tasks @patch('readthedocs.projects.views.private.trigger_build', lambda x: None) @@ -32,7 +32,7 @@ class TestProfileMiddleware(RequestFactoryTestMixin, TestCase): url = '/dashboard/import/manual/' def setUp(self): - super().setUp() + super(TestProfileMiddleware, self).setUp() data = { 'basics': { 'name': 'foobar', @@ -47,12 +47,12 @@ def setUp(self): } self.data = {} for key in data: - self.data.update({('{}-{}'.format(key, k), v) + self.data.update({('{0}-{1}'.format(key, k), v) for (k, v) in list(data[key].items())}) - self.data['{}-current_step'.format(self.wizard_class_slug)] = 'extra' + self.data['{0}-current_step'.format(self.wizard_class_slug)] = 'extra' def test_profile_middleware_no_profile(self): - """User without profile and isn't banned.""" + """User without profile and isn't banned""" req = self.request('/projects/import', method='post', data=self.data) req.user = get(User, profile=None) resp = ImportWizardView.as_view()(req) @@ -61,7 +61,7 @@ def test_profile_middleware_no_profile(self): @patch('readthedocs.projects.views.private.ProjectBasicsForm.clean') def test_profile_middleware_spam(self, form): - """User will be banned.""" + """User will be banned""" form.side_effect = ProjectSpamError req = self.request('/projects/import', method='post', data=self.data) req.user = get(User) @@ -71,7 +71,7 @@ def test_profile_middleware_spam(self, form): self.assertTrue(req.user.profile.banned) def test_profile_middleware_banned(self): - """User is banned.""" + """User is banned""" req = self.request('/projects/import', method='post', data=self.data) req.user = get(User) req.user.profile.banned = True @@ -101,10 +101,10 @@ def tearDown(self): def request(self, *args, **kwargs): kwargs['user'] = self.user - return super().request(*args, **kwargs) + return super(TestBasicsForm, self).request(*args, **kwargs) def test_form_pass(self): - """Only submit the basics.""" + """Only submit the basics""" resp = self.post_step('basics') self.assertIsInstance(resp, HttpResponseRedirect) self.assertEqual(resp.status_code, 302) @@ -136,7 +136,7 @@ def test_remote_repository_is_not_added_for_wrong_user(self): self.assertWizardFailure(resp, 'remote_repository') def test_form_missing(self): - """Submit form with missing data, expect to get failures.""" + """Submit form with missing data, expect to get failures""" self.step_data['basics'] = {'advanced': True} resp = self.post_step('basics') self.assertWizardFailure(resp, 'name') @@ -146,7 +146,7 @@ def test_form_missing(self): class TestAdvancedForm(TestBasicsForm): def setUp(self): - super().setUp() + super(TestAdvancedForm, self).setUp() self.step_data['basics']['advanced'] = True self.step_data['extra'] = { 'description': 'Describe foobar', @@ -156,7 +156,7 @@ def setUp(self): } def test_form_pass(self): - """Test all forms pass validation.""" + """Test all forms pass validation""" resp = self.post_step('basics') self.assertWizardResponse(resp, 'extra') resp = self.post_step('extra', session=list(resp._request.session.items())) @@ -169,16 +169,16 @@ def test_form_pass(self): data = self.step_data['basics'] del data['advanced'] del self.step_data['extra']['tags'] - self.assertCountEqual( + six.assertCountEqual( + self, [tag.name for tag in proj.tags.all()], - ['bar', 'baz', 'foo'], - ) + [u'bar', u'baz', u'foo']) data.update(self.step_data['extra']) for (key, val) in list(data.items()): self.assertEqual(getattr(proj, key), val) def test_form_missing_extra(self): - """Submit extra form with missing data, expect to get failures.""" + """Submit extra form with missing data, expect to get failures""" # Remove extra data to trigger validation errors self.step_data['extra'] = {} @@ -203,12 +203,10 @@ def test_remote_repository_is_added(self): self.assertIsNotNone(proj) self.assertEqual(proj.remote_repository, remote_repo) - @patch( - 'readthedocs.projects.views.private.ProjectExtraForm.clean_description', - create=True, - ) + @patch('readthedocs.projects.views.private.ProjectExtraForm.clean_description', + create=True) def test_form_spam(self, mocked_validator): - """Don't add project on a spammy description.""" + """Don't add project on a spammy description""" self.user.date_joined = timezone.now() - timedelta(days=365) self.user.save() mocked_validator.side_effect = ProjectSpamError @@ -227,12 +225,10 @@ def test_form_spam(self, mocked_validator): proj = Project.objects.get(name='foobar') self.assertFalse(self.user.profile.banned) - @patch( - 'readthedocs.projects.views.private.ProjectExtraForm.clean_description', - create=True, - ) + @patch('readthedocs.projects.views.private.ProjectExtraForm.clean_description', + create=True) def test_form_spam_ban_user(self, mocked_validator): - """Don't add spam and ban new user.""" + """Don't add spam and ban new user""" self.user.date_joined = timezone.now() self.user.save() mocked_validator.side_effect = ProjectSpamError @@ -253,7 +249,7 @@ def test_form_spam_ban_user(self, mocked_validator): class TestImportDemoView(MockBuildTestCase): - """Test project import demo view.""" + """Test project import demo view""" fixtures = ['test_data', 'eric'] @@ -270,7 +266,7 @@ def test_import_demo_pass(self): self.assertEqual(messages[0].level, message_const.SUCCESS) def test_import_demo_already_imported(self): - """Import demo project multiple times, expect failure 2nd post.""" + """Import demo project multiple times, expect failure 2nd post""" self.test_import_demo_pass() project = Project.objects.get(slug='eric-demo') @@ -283,13 +279,11 @@ def test_import_demo_already_imported(self): messages = list(resp_redir.context['messages']) self.assertEqual(messages[0].level, message_const.SUCCESS) - self.assertEqual( - project, - Project.objects.get(slug='eric-demo'), - ) + self.assertEqual(project, + Project.objects.get(slug='eric-demo')) def test_import_demo_another_user_imported(self): - """Import demo project after another user, expect success.""" + """Import demo project after another user, expect success""" self.test_import_demo_pass() project = Project.objects.get(slug='eric-demo') @@ -305,7 +299,7 @@ def test_import_demo_another_user_imported(self): self.assertEqual(messages[0].level, message_const.SUCCESS) def test_import_demo_imported_renamed(self): - """If the demo project is renamed, don't import another.""" + """If the demo project is renamed, don't import another""" self.test_import_demo_pass() project = Project.objects.get(slug='eric-demo') project.name = 'eric-demo-foobar' @@ -319,19 +313,14 @@ def test_import_demo_imported_renamed(self): self.assertEqual(resp_redir.status_code, 200) messages = list(resp_redir.context['messages']) self.assertEqual(messages[0].level, message_const.SUCCESS) - self.assertRegex( - messages[0].message, - r'already imported', - ) + self.assertRegex(messages[0].message, + r'already imported') - self.assertEqual( - project, - Project.objects.get(slug='eric-demo'), - ) + self.assertEqual(project, + Project.objects.get(slug='eric-demo')) def test_import_demo_imported_duplicate(self): - """ - If a project exists with same name, expect a failure importing demo. + """If a project exists with same name, expect a failure importing demo This should be edge case, user would have to import a project (not the demo project), named user-demo, and then manually enter the demo import @@ -350,15 +339,11 @@ def test_import_demo_imported_duplicate(self): self.assertEqual(resp_redir.status_code, 200) messages = list(resp_redir.context['messages']) self.assertEqual(messages[0].level, message_const.ERROR) - self.assertRegex( - messages[0].message, - r'There was a problem', - ) + self.assertRegex(messages[0].message, + r'There was a problem') - self.assertEqual( - project, - Project.objects.get(slug='eric-demo'), - ) + self.assertEqual(project, + Project.objects.get(slug='eric-demo')) class TestPrivateViews(MockBuildTestCase): @@ -394,9 +379,8 @@ def test_delete_project(self): self.assertFalse(Project.objects.filter(slug='pip').exists()) broadcast.assert_called_with( type='app', - task=tasks.remove_dirs, - args=[(project.doc_path,)], - ) + task=tasks.remove_dir, + args=[project.doc_path]) def test_subproject_create(self): project = get(Project, slug='pip', users=[self.user]) @@ -411,8 +395,7 @@ def test_subproject_create(self): broadcast.assert_called_with( type='app', task=tasks.symlink_subproject, - args=[project.pk], - ) + args=[project.pk]) class TestPrivateMixins(MockBuildTestCase): @@ -422,7 +405,7 @@ def setUp(self): self.domain = get(Domain, project=self.project) def test_project_relation(self): - """Class using project relation mixin class.""" + """Class using project relation mixin class""" class FoobarView(ProjectRelationMixin, ContextMixin): model = Domain diff --git a/readthedocs/rtd_tests/tests/test_redirects.py b/readthedocs/rtd_tests/tests/test_redirects.py index 6fbafebf3bc..335c8feba86 100644 --- a/readthedocs/rtd_tests/tests/test_redirects.py +++ b/readthedocs/rtd_tests/tests/test_redirects.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- -import logging - +from __future__ import absolute_import from django.http import Http404 from django.test import TestCase from django.test.utils import override_settings -from django_dynamic_fixture import fixture, get + +from django_dynamic_fixture import get +from django_dynamic_fixture import fixture from mock import patch from readthedocs.builds.constants import LATEST @@ -12,29 +12,28 @@ from readthedocs.projects.models import Project from readthedocs.redirects.models import Redirect +import logging + @override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False, APPEND_SLASH=False) class RedirectTests(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ["eric", "test_data"] def setUp(self): logging.disable(logging.DEBUG) self.client.login(username='eric', password='test') self.client.post( '/dashboard/import/', - { - 'repo_type': 'git', 'name': 'Pip', - 'tags': 'big, fucking, monkey', 'default_branch': '', - 'project_url': 'http://pip.rtfd.org', - 'repo': 'https://github.com/fail/sauce', - 'csrfmiddlewaretoken': '34af7c8a5ba84b84564403a280d9a9be', - 'default_version': LATEST, - 'privacy_level': 'public', - 'version_privacy_level': 'public', - 'description': 'wat', - 'documentation_type': 'sphinx', - }, - ) + {'repo_type': 'git', 'name': 'Pip', + 'tags': 'big, fucking, monkey', 'default_branch': '', + 'project_url': 'http://pip.rtfd.org', + 'repo': 'https://github.com/fail/sauce', + 'csrfmiddlewaretoken': '34af7c8a5ba84b84564403a280d9a9be', + 'default_version': LATEST, + 'privacy_level': 'public', + 'version_privacy_level': 'public', + 'description': 'wat', + 'documentation_type': 'sphinx'}) pip = Project.objects.get(slug='pip') pip.versions.create_latest() @@ -46,17 +45,14 @@ def test_proper_url(self): r = self.client.get('/docs/pip/') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://readthedocs.org/docs/pip/en/latest/', - ) + r['Location'], 'http://readthedocs.org/docs/pip/en/latest/') # Specific Page Redirects def test_proper_page_on_main_site(self): r = self.client.get('/docs/pip/page/test.html') self.assertEqual(r.status_code, 302) - self.assertEqual( - r['Location'], - 'http://readthedocs.org/docs/pip/en/latest/test.html', - ) + self.assertEqual(r['Location'], + 'http://readthedocs.org/docs/pip/en/latest/test.html') # If slug is neither valid lang nor valid version, it should 404. # TODO: This should 404 directly, not redirect first @@ -94,18 +90,15 @@ def test_proper_subdomain(self): r = self.client.get('/', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/') # Specific Page Redirects @override_settings(USE_SUBDOMAIN=True) def test_proper_page_on_subdomain(self): r = self.client.get('/page/test.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) - self.assertEqual( - r['Location'], - 'http://pip.readthedocs.org/en/latest/test.html', - ) + self.assertEqual(r['Location'], + 'http://pip.readthedocs.org/en/latest/test.html') @override_settings(USE_SUBDOMAIN=True) def test_improper_subdomain_filename_only(self): @@ -115,25 +108,22 @@ def test_improper_subdomain_filename_only(self): @override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) class RedirectAppTests(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ["eric", "test_data"] def setUp(self): self.client.login(username='eric', password='test') self.client.post( '/dashboard/import/', - { - 'repo_type': 'git', 'name': 'Pip', - 'tags': 'big, fucking, monkey', 'default_branch': '', - 'project_url': 'http://pip.rtfd.org', - 'repo': 'https://github.com/fail/sauce', - 'csrfmiddlewaretoken': '34af7c8a5ba84b84564403a280d9a9be', - 'default_version': LATEST, - 'privacy_level': 'public', - 'version_privacy_level': 'public', - 'description': 'wat', - 'documentation_type': 'sphinx', - }, - ) + {'repo_type': 'git', 'name': 'Pip', + 'tags': 'big, fucking, monkey', 'default_branch': '', + 'project_url': 'http://pip.rtfd.org', + 'repo': 'https://github.com/fail/sauce', + 'csrfmiddlewaretoken': '34af7c8a5ba84b84564403a280d9a9be', + 'default_version': LATEST, + 'privacy_level': 'public', + 'version_privacy_level': 'public', + 'description': 'wat', + 'documentation_type': 'sphinx'}) self.pip = Project.objects.get(slug='pip') self.pip.versions.create_latest() @@ -155,14 +145,12 @@ def test_redirect_prefix_infinite(self): r = self.client.get('/redirect', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/redirect.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/redirect.html') r = self.client.get('/redirect/', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/redirect/', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/redirect/') r = self.client.get('/en/latest/redirect/', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 404) @@ -170,37 +158,33 @@ def test_redirect_prefix_infinite(self): @override_settings(USE_SUBDOMAIN=True) def test_redirect_root(self): Redirect.objects.create( - project=self.pip, redirect_type='prefix', from_url='/woot/', - ) + project=self.pip, redirect_type='prefix', from_url='/woot/') r = self.client.get('/woot/faq.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/faq.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/faq.html') @override_settings(USE_SUBDOMAIN=True) def test_redirect_page(self): Redirect.objects.create( project=self.pip, redirect_type='page', - from_url='/install.html', to_url='/tutorial/install.html', + from_url='/install.html', to_url='/tutorial/install.html' ) r = self.client.get('/install.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/tutorial/install.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/tutorial/install.html') @override_settings(USE_SUBDOMAIN=True) def test_redirect_exact(self): Redirect.objects.create( project=self.pip, redirect_type='exact', - from_url='/en/latest/install.html', to_url='/en/latest/tutorial/install.html', + from_url='/en/latest/install.html', to_url='/en/latest/tutorial/install.html' ) r = self.client.get('/en/latest/install.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/tutorial/install.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/tutorial/install.html') @override_settings(USE_SUBDOMAIN=True) def test_redirect_exact_with_rest(self): @@ -218,8 +202,7 @@ def test_redirect_exact_with_rest(self): r = self.client.get('/en/latest/guides/install.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/version/guides/install.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/version/guides/install.html') Redirect.objects.create( project=self.pip, redirect_type='exact', @@ -228,8 +211,7 @@ def test_redirect_exact_with_rest(self): r = self.client.get('/es/version/guides/install.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/master/guides/install.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/master/guides/install.html') @override_settings(USE_SUBDOMAIN=True) def test_redirect_inactive_version(self): @@ -254,94 +236,75 @@ def test_redirect_inactive_version(self): r = self.client.get('/en/oldversion/', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/newversion/', - ) + r['Location'], 'http://pip.readthedocs.org/en/newversion/') @override_settings(USE_SUBDOMAIN=True) def test_redirect_keeps_version_number(self): Redirect.objects.create( project=self.pip, redirect_type='page', - from_url='/how_to_install.html', to_url='/install.html', - ) + from_url='/how_to_install.html', to_url='/install.html') with patch('readthedocs.core.views.serve._serve_symlink_docs') as _serve_docs: _serve_docs.side_effect = Http404() - r = self.client.get( - '/en/0.8.1/how_to_install.html', - HTTP_HOST='pip.readthedocs.org', - ) + r = self.client.get('/en/0.8.1/how_to_install.html', + HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( r['Location'], - 'http://pip.readthedocs.org/en/0.8.1/install.html', - ) + 'http://pip.readthedocs.org/en/0.8.1/install.html') @override_settings(USE_SUBDOMAIN=True) def test_redirect_keeps_language(self): Redirect.objects.create( project=self.pip, redirect_type='page', - from_url='/how_to_install.html', to_url='/install.html', - ) + from_url='/how_to_install.html', to_url='/install.html') with patch('readthedocs.core.views.serve._serve_symlink_docs') as _serve_docs: _serve_docs.side_effect = Http404() - r = self.client.get( - '/de/0.8.1/how_to_install.html', - HTTP_HOST='pip.readthedocs.org', - ) + r = self.client.get('/de/0.8.1/how_to_install.html', + HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( r['Location'], - 'http://pip.readthedocs.org/de/0.8.1/install.html', - ) + 'http://pip.readthedocs.org/de/0.8.1/install.html') @override_settings(USE_SUBDOMAIN=True) def test_redirect_recognizes_custom_cname(self): Redirect.objects.create( project=self.pip, redirect_type='page', from_url='/install.html', - to_url='/tutorial/install.html', - ) - r = self.client.get( - '/install.html', - HTTP_HOST='pip.pypa.io', - HTTP_X_RTD_SLUG='pip', - ) + to_url='/tutorial/install.html') + r = self.client.get('/install.html', + HTTP_HOST='pip.pypa.io', + HTTP_X_RTD_SLUG='pip') self.assertEqual(r.status_code, 302) self.assertEqual( r['Location'], - 'http://pip.pypa.io/en/latest/tutorial/install.html', - ) + 'http://pip.pypa.io/en/latest/tutorial/install.html') @override_settings(USE_SUBDOMAIN=True, PYTHON_MEDIA=True) def test_redirect_html(self): Redirect.objects.create( - project=self.pip, redirect_type='sphinx_html', - ) + project=self.pip, redirect_type='sphinx_html') r = self.client.get('/en/latest/faq/', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/faq.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/faq.html') @override_settings(USE_SUBDOMAIN=True, PYTHON_MEDIA=True) def test_redirect_html_index(self): Redirect.objects.create( - project=self.pip, redirect_type='sphinx_html', - ) + project=self.pip, redirect_type='sphinx_html') r = self.client.get('/en/latest/faq/index.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/faq.html', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/faq.html') @override_settings(USE_SUBDOMAIN=True, PYTHON_MEDIA=True) def test_redirect_htmldir(self): Redirect.objects.create( - project=self.pip, redirect_type='sphinx_htmldir', - ) + project=self.pip, redirect_type='sphinx_htmldir') r = self.client.get('/en/latest/faq.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/faq/', - ) + r['Location'], 'http://pip.readthedocs.org/en/latest/faq/') class CustomRedirectTests(TestCase): @@ -376,16 +339,14 @@ def test_redirect_fragment(self): @override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) class RedirectBuildTests(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ["eric", "test_data"] def setUp(self): - self.project = get( - Project, - slug='project-1', - documentation_type='sphinx', - conf_py_file='test_conf.py', - versions=[fixture()], - ) + self.project = get(Project, + slug='project-1', + documentation_type='sphinx', + conf_py_file='test_conf.py', + versions=[fixture()]) self.version = self.project.versions.all()[0] def test_redirect_list(self): @@ -401,46 +362,46 @@ def test_redirect_detail(self): @override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) class GetFullPathTests(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ["eric", "test_data"] def setUp(self): - self.proj = Project.objects.get(slug='read-the-docs') + self.proj = Project.objects.get(slug="read-the-docs") self.redirect = get(Redirect, project=self.proj) def test_http_filenames_return_themselves(self): self.assertEqual( self.redirect.get_full_path('http://rtfd.org'), - 'http://rtfd.org', + 'http://rtfd.org' ) def test_redirects_no_subdomain(self): self.assertEqual( self.redirect.get_full_path('index.html'), - '/docs/read-the-docs/en/latest/', + '/docs/read-the-docs/en/latest/' ) @override_settings( - USE_SUBDOMAIN=True, PRODUCTION_DOMAIN='rtfd.org', + USE_SUBDOMAIN=True, PRODUCTION_DOMAIN='rtfd.org' ) def test_redirects_with_subdomain(self): self.assertEqual( self.redirect.get_full_path('faq.html'), - '/en/latest/faq.html', + '/en/latest/faq.html' ) @override_settings( - USE_SUBDOMAIN=True, PRODUCTION_DOMAIN='rtfd.org', + USE_SUBDOMAIN=True, PRODUCTION_DOMAIN='rtfd.org' ) def test_single_version_with_subdomain(self): self.redirect.project.single_version = True self.assertEqual( self.redirect.get_full_path('faq.html'), - '/faq.html', + '/faq.html' ) def test_single_version_no_subdomain(self): self.redirect.project.single_version = True self.assertEqual( self.redirect.get_full_path('faq.html'), - '/docs/read-the-docs/faq.html', + '/docs/read-the-docs/faq.html' ) diff --git a/readthedocs/rtd_tests/tests/test_repo_parsing.py b/readthedocs/rtd_tests/tests/test_repo_parsing.py index 85ffbbf9997..f946db61e53 100644 --- a/readthedocs/rtd_tests/tests/test_repo_parsing.py +++ b/readthedocs/rtd_tests/tests/test_repo_parsing.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + from django.test import TestCase from readthedocs.projects.models import Project diff --git a/readthedocs/rtd_tests/tests/test_resolver.py b/readthedocs/rtd_tests/tests/test_resolver.py index 81f2f3ef5ab..1ef55d564ca 100644 --- a/readthedocs/rtd_tests/tests/test_resolver.py +++ b/readthedocs/rtd_tests/tests/test_resolver.py @@ -1,13 +1,13 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import django_dynamic_fixture as fixture import mock from django.test import TestCase, override_settings from readthedocs.core.resolver import ( - Resolver, - resolve, - resolve_domain, - resolve_path, + Resolver, resolve, resolve_domain, resolve_path ) from readthedocs.projects.constants import PRIVATE from readthedocs.projects.models import Domain, Project, ProjectRelationship @@ -67,8 +67,7 @@ def test_resolver_filename_index(self): url = resolve_path(project=self.pip, filename='foo/bar/index.html') self.assertEqual(url, '/docs/pip/en/latest/foo/bar/') url = resolve_path( - project=self.pip, filename='foo/index/index.html', - ) + project=self.pip, filename='foo/index/index.html') self.assertEqual(url, '/docs/pip/en/latest/foo/index/') def test_resolver_filename_false_index(self): @@ -76,11 +75,9 @@ def test_resolver_filename_false_index(self): url = resolve_path(project=self.pip, filename='foo/foo_index.html') self.assertEqual(url, '/docs/pip/en/latest/foo/foo_index.html') url = resolve_path( - project=self.pip, filename='foo_index/foo_index.html', - ) + project=self.pip, filename='foo_index/foo_index.html') self.assertEqual( - url, '/docs/pip/en/latest/foo_index/foo_index.html', - ) + url, '/docs/pip/en/latest/foo_index/foo_index.html') def test_resolver_filename_sphinx(self): self.pip.documentation_type = 'sphinx' @@ -176,25 +173,21 @@ def test_resolver_force_single_version(self): self.pip.single_version = False with override_settings(USE_SUBDOMAIN=False): url = resolve_path( - project=self.pip, filename='index.html', single_version=True, - ) + project=self.pip, filename='index.html', single_version=True) self.assertEqual(url, '/docs/pip/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( - project=self.pip, filename='index.html', single_version=True, - ) + project=self.pip, filename='index.html', single_version=True) self.assertEqual(url, '/') def test_resolver_force_domain(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( - project=self.pip, filename='index.html', cname=True, - ) + project=self.pip, filename='index.html', cname=True) self.assertEqual(url, '/en/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( - project=self.pip, filename='index.html', cname=True, - ) + project=self.pip, filename='index.html', cname=True) self.assertEqual(url, '/en/latest/') def test_resolver_force_domain_single_version(self): @@ -202,78 +195,66 @@ def test_resolver_force_domain_single_version(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( project=self.pip, filename='index.html', single_version=True, - cname=True, - ) + cname=True) self.assertEqual(url, '/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( project=self.pip, filename='index.html', single_version=True, - cname=True, - ) + cname=True) self.assertEqual(url, '/') def test_resolver_force_language(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( - project=self.pip, filename='index.html', language='cz', - ) + project=self.pip, filename='index.html', language='cz') self.assertEqual(url, '/docs/pip/cz/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( - project=self.pip, filename='index.html', language='cz', - ) + project=self.pip, filename='index.html', language='cz') self.assertEqual(url, '/cz/latest/') def test_resolver_force_version(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( - project=self.pip, filename='index.html', version_slug='foo', - ) + project=self.pip, filename='index.html', version_slug='foo') self.assertEqual(url, '/docs/pip/en/foo/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( - project=self.pip, filename='index.html', version_slug='foo', - ) + project=self.pip, filename='index.html', version_slug='foo') self.assertEqual(url, '/en/foo/') def test_resolver_force_language_version(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( project=self.pip, filename='index.html', language='cz', - version_slug='foo', - ) + version_slug='foo') self.assertEqual(url, '/docs/pip/cz/foo/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( project=self.pip, filename='index.html', language='cz', - version_slug='foo', - ) + version_slug='foo') self.assertEqual(url, '/cz/foo/') def test_resolver_no_force_translation(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( - project=self.translation, filename='index.html', language='cz', - ) + project=self.translation, filename='index.html', language='cz') self.assertEqual(url, '/docs/pip/ja/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( - project=self.translation, filename='index.html', language='cz', - ) + project=self.translation, filename='index.html', language='cz') self.assertEqual(url, '/ja/latest/') def test_resolver_no_force_translation_with_version(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_path( project=self.translation, filename='index.html', language='cz', - version_slug='foo', - ) + version_slug='foo') self.assertEqual(url, '/docs/pip/ja/foo/') with override_settings(USE_SUBDOMAIN=True): url = resolve_path( project=self.translation, filename='index.html', language='cz', - version_slug='foo', - ) + version_slug='foo') self.assertEqual(url, '/ja/foo/') @@ -293,7 +274,7 @@ def test_project_with_same_translation_and_main_language(self): proj1.save() self.assertEqual( proj1.main_language_project.main_language_project, - proj1, + proj1 ) # This tests that we aren't going to re-recurse back to resolving proj1 @@ -447,8 +428,7 @@ def test_domain_resolver_translation_itself(self): @override_settings( PRODUCTION_DOMAIN='readthedocs.org', - PUBLIC_DOMAIN='public.readthedocs.org', - ) + PUBLIC_DOMAIN='public.readthedocs.org') def test_domain_public(self): with override_settings(USE_SUBDOMAIN=False): url = resolve_domain(project=self.translation) @@ -512,13 +492,11 @@ def test_resolver_subproject(self): with override_settings(USE_SUBDOMAIN=False): url = resolve(project=self.subproject) self.assertEqual( - url, 'http://readthedocs.org/docs/pip/projects/sub/ja/latest/', - ) + url, 'http://readthedocs.org/docs/pip/projects/sub/ja/latest/') with override_settings(USE_SUBDOMAIN=True): url = resolve(project=self.subproject) self.assertEqual( - url, 'http://pip.readthedocs.org/projects/sub/ja/latest/', - ) + url, 'http://pip.readthedocs.org/projects/sub/ja/latest/') @override_settings(PRODUCTION_DOMAIN='readthedocs.org') def test_resolver_translation(self): @@ -598,8 +576,7 @@ def test_resolver_private_version_override(self): @override_settings( PRODUCTION_DOMAIN='readthedocs.org', - PUBLIC_DOMAIN='public.readthedocs.org', - ) + PUBLIC_DOMAIN='public.readthedocs.org') def test_resolver_public_domain_overrides(self): with override_settings(USE_SUBDOMAIN=False): url = resolve(project=self.pip, private=True) @@ -609,12 +586,10 @@ def test_resolver_public_domain_overrides(self): with override_settings(USE_SUBDOMAIN=True): url = resolve(project=self.pip, private=True) self.assertEqual( - url, 'http://pip.public.readthedocs.org/en/latest/', - ) + url, 'http://pip.public.readthedocs.org/en/latest/') url = resolve(project=self.pip, private=False) self.assertEqual( - url, 'http://pip.public.readthedocs.org/en/latest/', - ) + url, 'http://pip.public.readthedocs.org/en/latest/') # Domain overrides PUBLIC_DOMAIN self.domain = fixture.get( @@ -652,7 +627,7 @@ def test_resolver_domain_https(self): self.assertEqual(url, 'http://pip.readthedocs.io/en/latest/') -class ResolverAltSetUp: +class ResolverAltSetUp(object): def setUp(self): with mock.patch('readthedocs.projects.models.broadcast'): @@ -745,24 +720,20 @@ def test_subproject_with_translation_without_custom_domain(self): self.assertEqual( url, 'http://{project.slug}.readthedocs.io/en/latest/'.format( project=self.superproject_en, - ), - ) + )) url = resolve(self.superproject_es, filename='') self.assertEqual( url, 'http://{project.slug}.readthedocs.io/es/latest/'.format( project=self.superproject_en, - ), - ) + )) url = resolve(self.subproject_en, filename='') # yapf: disable self.assertEqual( url, - ( - 'http://{project.slug}.readthedocs.io/projects/' - '{subproject.slug}/en/latest/' - ).format( + ('http://{project.slug}.readthedocs.io/projects/' + '{subproject.slug}/en/latest/').format( project=self.superproject_en, subproject=self.subproject_en, ), @@ -771,10 +742,8 @@ def test_subproject_with_translation_without_custom_domain(self): url = resolve(self.subproject_es, filename='') self.assertEqual( url, - ( - 'http://{project.slug}.readthedocs.io/projects/' - '{subproject.slug}/es/latest/' - ).format( + ('http://{project.slug}.readthedocs.io/projects/' + '{subproject.slug}/es/latest/').format( project=self.superproject_en, subproject=self.subproject_en, ), @@ -801,10 +770,8 @@ def test_subproject_with_translation_with_custom_domain(self): url = resolve(self.subproject_en, filename='') self.assertEqual( url, - ( - 'http://docs.example.com/projects/' - '{subproject.slug}/en/latest/' - ).format( + ('http://docs.example.com/projects/' + '{subproject.slug}/en/latest/').format( subproject=self.subproject_en, ), ) @@ -812,10 +779,8 @@ def test_subproject_with_translation_with_custom_domain(self): url = resolve(self.subproject_es, filename='') self.assertEqual( url, - ( - 'http://docs.example.com/projects/' - '{subproject.slug}/es/latest/' - ).format( + ('http://docs.example.com/projects/' + '{subproject.slug}/es/latest/').format( subproject=self.subproject_en, ), ) diff --git a/readthedocs/rtd_tests/tests/test_restapi_client.py b/readthedocs/rtd_tests/tests/test_restapi_client.py index cf555b74883..88906fdc10f 100644 --- a/readthedocs/rtd_tests/tests/test_restapi_client.py +++ b/readthedocs/rtd_tests/tests/test_restapi_client.py @@ -1,4 +1,7 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, unicode_literals) + from django.test import TestCase from readthedocs.restapi.client import DrfJsonSerializer @@ -6,7 +9,7 @@ class TestDrfJsonSerializer(TestCase): data = { - 'proper': 'json', + 'proper': 'json' } serialized_data = '{"proper":"json"}' diff --git a/readthedocs/rtd_tests/tests/test_search_json_parsing.py b/readthedocs/rtd_tests/tests/test_search_json_parsing.py index 42b0839c4e9..fb91d31b276 100644 --- a/readthedocs/rtd_tests/tests/test_search_json_parsing.py +++ b/readthedocs/rtd_tests/tests/test_search_json_parsing.py @@ -1,11 +1,10 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import import os from django.test import TestCase from readthedocs.search.parse_json import process_file - base_dir = os.path.dirname(os.path.dirname(__file__)) class TestHacks(TestCase): @@ -15,7 +14,7 @@ def test_h2_parsing(self): os.path.join( base_dir, 'files/api.fjson', - ), + ) ) self.assertEqual(data['sections'][1]['id'], 'a-basic-api-client-using-slumber') # Only capture h2's after the first section diff --git a/readthedocs/rtd_tests/tests/test_single_version.py b/readthedocs/rtd_tests/tests/test_single_version.py index e69b80375af..faee5f585f7 100644 --- a/readthedocs/rtd_tests/tests/test_single_version.py +++ b/readthedocs/rtd_tests/tests/test_single_version.py @@ -1,13 +1,14 @@ -# -*- coding: utf-8 -*- -import django_dynamic_fixture as fixture +from __future__ import absolute_import from django.test import TestCase from django.test.utils import override_settings +import django_dynamic_fixture as fixture + from readthedocs.projects.models import Project @override_settings( - USE_SUBDOMAIN=True, PUBLIC_DOMAIN='public.readthedocs.org', SERVE_PUBLIC_DOCS=True, + USE_SUBDOMAIN=True, PUBLIC_DOMAIN='public.readthedocs.org', SERVE_PUBLIC_DOCS=True ) class RedirectSingleVersionTests(TestCase): @@ -16,24 +17,16 @@ def setUp(self): def test_docs_url_generation(self): with override_settings(USE_SUBDOMAIN=False): - self.assertEqual( - self.pip.get_docs_url(), - 'http://readthedocs.org/docs/pip/', - ) + self.assertEqual(self.pip.get_docs_url(), + 'http://readthedocs.org/docs/pip/') with override_settings(USE_SUBDOMAIN=True): - self.assertEqual( - self.pip.get_docs_url(), - 'http://pip.public.readthedocs.org/', - ) + self.assertEqual(self.pip.get_docs_url(), + 'http://pip.public.readthedocs.org/') self.pip.single_version = False with override_settings(USE_SUBDOMAIN=False): - self.assertEqual( - self.pip.get_docs_url(), - 'http://readthedocs.org/docs/pip/en/latest/', - ) + self.assertEqual(self.pip.get_docs_url(), + 'http://readthedocs.org/docs/pip/en/latest/') with override_settings(USE_SUBDOMAIN=True): - self.assertEqual( - self.pip.get_docs_url(), - 'http://pip.public.readthedocs.org/en/latest/', - ) + self.assertEqual(self.pip.get_docs_url(), + 'http://pip.public.readthedocs.org/en/latest/') diff --git a/readthedocs/rtd_tests/tests/test_subprojects.py b/readthedocs/rtd_tests/tests/test_subprojects.py index 255325d7a2f..081bd89a0bb 100644 --- a/readthedocs/rtd_tests/tests/test_subprojects.py +++ b/readthedocs/rtd_tests/tests/test_subprojects.py @@ -1,6 +1,7 @@ -# -*- coding: utf-8 -*- -import django_dynamic_fixture as fixture +from __future__ import absolute_import + import mock +import django_dynamic_fixture as fixture from django.contrib.auth.models import User from django.test import TestCase from django.test.utils import override_settings @@ -18,13 +19,13 @@ def test_empty_child(self): form = ProjectRelationshipForm( {}, project=project, - user=user, + user=user ) form.full_clean() self.assertEqual(len(form.errors['child']), 1) self.assertRegex( form.errors['child'][0], - r'This field is required.', + r'This field is required.' ) def test_nonexistent_child(self): @@ -34,13 +35,13 @@ def test_nonexistent_child(self): form = ProjectRelationshipForm( {'child': 9999}, project=project, - user=user, + user=user ) form.full_clean() self.assertEqual(len(form.errors['child']), 1) self.assertRegex( form.errors['child'][0], - r'Select a valid choice.', + r'Select a valid choice.' ) def test_adding_subproject_fails_when_user_is_not_admin(self): @@ -57,13 +58,13 @@ def test_adding_subproject_fails_when_user_is_not_admin(self): form = ProjectRelationshipForm( {'child': subproject.pk}, project=project, - user=user, + user=user ) form.full_clean() self.assertEqual(len(form.errors['child']), 1) self.assertRegex( form.errors['child'][0], - r'Select a valid choice.', + r'Select a valid choice.' ) def test_adding_subproject_passes_when_user_is_admin(self): @@ -81,14 +82,14 @@ def test_adding_subproject_passes_when_user_is_admin(self): form = ProjectRelationshipForm( {'child': subproject.pk}, project=project, - user=user, + user=user ) form.full_clean() self.assertTrue(form.is_valid()) form.save() self.assertEqual( [r.child for r in project.subprojects.all()], - [subproject], + [subproject] ) def test_subproject_form_cant_create_sub_sub_project(self): @@ -97,7 +98,7 @@ def test_subproject_form_cant_create_sub_sub_project(self): subproject = fixture.get(Project, users=[user]) subsubproject = fixture.get(Project, users=[user]) relation = fixture.get( - ProjectRelationship, parent=project, child=subproject, + ProjectRelationship, parent=project, child=subproject ) self.assertQuerysetEqual( Project.objects.for_admin_user(user), @@ -108,7 +109,7 @@ def test_subproject_form_cant_create_sub_sub_project(self): form = ProjectRelationshipForm( {'child': subsubproject.pk}, project=subproject, - user=user, + user=user ) # The subsubproject is valid here, as far as the child check is # concerned, but the parent check should fail. @@ -120,7 +121,7 @@ def test_subproject_form_cant_create_sub_sub_project(self): self.assertEqual(len(form.errors['parent']), 1) self.assertRegex( form.errors['parent'][0], - r'Subproject nesting is not supported', + r'Subproject nesting is not supported' ) def test_excludes_existing_subprojects(self): @@ -128,7 +129,7 @@ def test_excludes_existing_subprojects(self): project = fixture.get(Project, users=[user]) subproject = fixture.get(Project, users=[user]) relation = fixture.get( - ProjectRelationship, parent=project, child=subproject, + ProjectRelationship, parent=project, child=subproject ) self.assertQuerysetEqual( Project.objects.for_admin_user(user), @@ -139,7 +140,7 @@ def test_excludes_existing_subprojects(self): form = ProjectRelationshipForm( {'child': subproject.pk}, project=project, - user=user, + user=user ) self.assertEqual( [proj_id for (proj_id, __) in form.fields['child'].choices], @@ -153,12 +154,12 @@ def test_exclude_self_project_as_subproject(self): form = ProjectRelationshipForm( {'child': project.pk}, project=project, - user=user, + user=user ) self.assertFalse(form.is_valid()) self.assertNotIn( project.id, - [proj_id for (proj_id, __) in form.fields['child'].choices], + [proj_id for (proj_id, __) in form.fields['child'].choices] ) @@ -170,16 +171,12 @@ def setUp(self): self.owner = create_user(username='owner', password='test') self.tester = create_user(username='tester', password='test') self.pip = fixture.get(Project, slug='pip', users=[self.owner], main_language_project=None) - self.subproject = fixture.get( - Project, slug='sub', language='ja', - users=[ self.owner], - main_language_project=None, - ) - self.translation = fixture.get( - Project, slug='trans', language='ja', - users=[ self.owner], - main_language_project=None, - ) + self.subproject = fixture.get(Project, slug='sub', language='ja', + users=[ self.owner], + main_language_project=None) + self.translation = fixture.get(Project, slug='trans', language='ja', + users=[ self.owner], + main_language_project=None) self.pip.add_subproject(self.subproject) self.pip.translations.add(self.translation) @@ -191,13 +188,13 @@ def setUp(self): @override_settings( PRODUCTION_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False, - ) + ) def test_resolver_subproject_alias(self): resp = self.client.get('/docs/pip/projects/sub_alias/') self.assertEqual(resp.status_code, 302) self.assertEqual( resp._headers['location'][1], - 'http://readthedocs.org/docs/pip/projects/sub_alias/ja/latest/', + 'http://readthedocs.org/docs/pip/projects/sub_alias/ja/latest/' ) @override_settings(USE_SUBDOMAIN=True) @@ -206,5 +203,5 @@ def test_resolver_subproject_subdomain_alias(self): self.assertEqual(resp.status_code, 302) self.assertEqual( resp._headers['location'][1], - 'http://pip.readthedocs.org/projects/sub_alias/ja/latest/', + 'http://pip.readthedocs.org/projects/sub_alias/ja/latest/' ) diff --git a/readthedocs/rtd_tests/tests/test_sync_versions.py b/readthedocs/rtd_tests/tests/test_sync_versions.py index 37339d2a0e5..533ad424729 100644 --- a/readthedocs/rtd_tests/tests/test_sync_versions.py +++ b/readthedocs/rtd_tests/tests/test_sync_versions.py @@ -1,9 +1,17 @@ # -*- coding: utf-8 -*- +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals +) + import json from django.test import TestCase from django.urls import reverse +import pytest from readthedocs.builds.constants import BRANCH, STABLE, TAG from readthedocs.builds.models import Version @@ -169,7 +177,7 @@ def test_delete_version(self): } self.assertTrue( - Version.objects.filter(slug='0.8.3').exists(), + Version.objects.filter(slug='0.8.3').exists() ) self.client.post( @@ -180,7 +188,7 @@ def test_delete_version(self): # There isn't a v0.8.3 self.assertFalse( - Version.objects.filter(slug='0.8.3').exists(), + Version.objects.filter(slug='0.8.3').exists() ) def test_machine_attr_when_user_define_stable_tag_and_delete_it(self): @@ -205,7 +213,7 @@ def test_machine_attr_when_user_define_stable_tag_and_delete_it(self): # 0.8.3 is the current stable self.assertEqual( version8.identifier, - current_stable.identifier, + current_stable.identifier ) self.assertTrue(current_stable.machine) @@ -239,7 +247,7 @@ def test_machine_attr_when_user_define_stable_tag_and_delete_it(self): current_stable = self.pip.get_stable_version() self.assertEqual( '1abc2def3', - current_stable.identifier, + current_stable.identifier ) # Deleting the tag should return the RTD's stable @@ -270,7 +278,7 @@ def test_machine_attr_when_user_define_stable_tag_and_delete_it(self): current_stable = self.pip.get_stable_version() self.assertEqual( '0.8.3', - current_stable.identifier, + current_stable.identifier ) self.assertTrue(current_stable.machine) @@ -317,7 +325,7 @@ def test_machine_attr_when_user_define_stable_tag_and_delete_it_new_project(self current_stable = self.pip.get_stable_version() self.assertEqual( '1abc2def3', - current_stable.identifier, + current_stable.identifier ) # User activates the stable version @@ -352,7 +360,7 @@ def test_machine_attr_when_user_define_stable_tag_and_delete_it_new_project(self current_stable = self.pip.get_stable_version() self.assertEqual( '0.8.3', - current_stable.identifier, + current_stable.identifier ) self.assertTrue(current_stable.machine) @@ -380,7 +388,7 @@ def test_machine_attr_when_user_define_stable_branch_and_delete_it(self): # 0.8.3 is the current stable self.assertEqual( '0.8.3', - current_stable.identifier, + current_stable.identifier ) self.assertTrue(current_stable.machine) @@ -412,7 +420,7 @@ def test_machine_attr_when_user_define_stable_branch_and_delete_it(self): current_stable = self.pip.get_stable_version() self.assertEqual( 'origin/stable', - current_stable.identifier, + current_stable.identifier ) # Deleting the branch should return the RTD's stable @@ -441,16 +449,18 @@ def test_machine_attr_when_user_define_stable_branch_and_delete_it(self): current_stable = self.pip.get_stable_version() self.assertEqual( 'origin/0.8.3', - current_stable.identifier, + current_stable.identifier ) self.assertTrue(current_stable.machine) def test_machine_attr_when_user_define_stable_branch_and_delete_it_new_project(self): - """The user imports a new project with a branch named ``stable``, when - syncing the versions, the RTD's ``stable`` is lost (set to - machine=False) and doesn't update automatically anymore, when the branch - is deleted on the user repository, the RTD's ``stable`` is back (set to - machine=True).""" + """ + The user imports a new project with a branch named ``stable``, + when syncing the versions, the RTD's ``stable`` is lost + (set to machine=False) and doesn't update automatically anymore, + when the branch is deleted on the user repository, the RTD's ``stable`` + is back (set to machine=True). + """ # There isn't a stable version yet self.pip.versions.exclude(slug='master').delete() current_stable = self.pip.get_stable_version() @@ -484,7 +494,7 @@ def test_machine_attr_when_user_define_stable_branch_and_delete_it_new_project(s current_stable = self.pip.get_stable_version() self.assertEqual( 'origin/stable', - current_stable.identifier, + current_stable.identifier ) # User activates the stable version @@ -517,16 +527,18 @@ def test_machine_attr_when_user_define_stable_branch_and_delete_it_new_project(s current_stable = self.pip.get_stable_version() self.assertEqual( 'origin/0.8.3', - current_stable.identifier, + current_stable.identifier ) self.assertTrue(current_stable.machine) def test_machine_attr_when_user_define_latest_tag_and_delete_it(self): - """The user creates a tag named ``latest`` on an existing repo, when - syncing the versions, the RTD's ``latest`` is lost (set to - machine=False) and doesn't update automatically anymore, when the tag is - deleted on the user repository, the RTD's ``latest`` is back (set to - machine=True).""" + """ + The user creates a tag named ``latest`` on an existing repo, + when syncing the versions, the RTD's ``latest`` is lost + (set to machine=False) and doesn't update automatically anymore, + when the tag is deleted on the user repository, the RTD's ``latest`` + is back (set to machine=True). + """ version_post_data = { 'branches': [ { @@ -554,7 +566,7 @@ def test_machine_attr_when_user_define_latest_tag_and_delete_it(self): version_latest = self.pip.versions.get(slug='latest') self.assertEqual( '1abc2def3', - version_latest.identifier, + version_latest.identifier ) # Deleting the tag should return the RTD's latest @@ -565,7 +577,7 @@ def test_machine_attr_when_user_define_latest_tag_and_delete_it(self): 'verbose_name': 'master', }, ], - 'tags': [], + 'tags': [] } resp = self.client.post( @@ -579,16 +591,18 @@ def test_machine_attr_when_user_define_latest_tag_and_delete_it(self): version_latest = self.pip.versions.get(slug='latest') self.assertEqual( 'master', - version_latest.identifier, + version_latest.identifier ) self.assertTrue(version_latest.machine) def test_machine_attr_when_user_define_latest_branch_and_delete_it(self): - """The user creates a branch named ``latest`` on an existing repo, when - syncing the versions, the RTD's ``latest`` is lost (set to - machine=False) and doesn't update automatically anymore, when the branch - is deleted on the user repository, the RTD's ``latest`` is back (set to - machine=True).""" + """ + The user creates a branch named ``latest`` on an existing repo, + when syncing the versions, the RTD's ``latest`` is lost + (set to machine=False) and doesn't update automatically anymore, + when the branch is deleted on the user repository, the RTD's ``latest`` + is back (set to machine=True). + """ version_post_data = { 'branches': [ { @@ -614,7 +628,7 @@ def test_machine_attr_when_user_define_latest_branch_and_delete_it(self): version_latest = self.pip.versions.get(slug='latest') self.assertEqual( 'origin/latest', - version_latest.identifier, + version_latest.identifier ) # Deleting the branch should return the RTD's latest @@ -668,7 +682,7 @@ def test_deletes_version_with_same_identifier(self): # We only have one version with an identifier `1234` self.assertEqual( self.pip.versions.filter(identifier='1234').count(), - 1, + 1 ) # We add a new tag with the same identifier @@ -701,7 +715,7 @@ def test_deletes_version_with_same_identifier(self): # We have two versions with an identifier `1234` self.assertEqual( self.pip.versions.filter(identifier='1234').count(), - 2, + 2 ) # We delete one version with identifier `1234` @@ -730,7 +744,7 @@ def test_deletes_version_with_same_identifier(self): # We have only one version with an identifier `1234` self.assertEqual( self.pip.versions.filter(identifier='1234').count(), - 1, + 1 ) @@ -826,7 +840,7 @@ def test_invalid_version_numbers_are_not_stable(self): 'tags': [ { 'identifier': 'this.is.invalid', - 'verbose_name': 'this.is.invalid', + 'verbose_name': 'this.is.invalid' }, ], } @@ -847,7 +861,7 @@ def test_invalid_version_numbers_are_not_stable(self): }, { 'identifier': 'this.is.invalid', - 'verbose_name': 'this.is.invalid', + 'verbose_name': 'this.is.invalid' }, ], } @@ -897,7 +911,7 @@ def test_update_stable_version(self): 'identifier': '1.0.0', 'verbose_name': '1.0.0', }, - ], + ] } self.client.post( @@ -1109,12 +1123,12 @@ def test_user_defined_stable_version_tag_with_tags(self): self.assertTrue(version_stable.active) self.assertEqual( '1abc2def3', - self.pip.get_stable_version().identifier, + self.pip.get_stable_version().identifier ) # There arent others stable slugs like stable_a other_stable = self.pip.versions.filter( - slug__startswith='stable_', + slug__startswith='stable_' ) self.assertFalse(other_stable.exists()) @@ -1131,10 +1145,10 @@ def test_user_defined_stable_version_tag_with_tags(self): self.assertTrue(version_stable.active) self.assertEqual( '1abc2def3', - self.pip.get_stable_version().identifier, + self.pip.get_stable_version().identifier ) other_stable = self.pip.versions.filter( - slug__startswith='stable_', + slug__startswith='stable_' ) self.assertFalse(other_stable.exists()) @@ -1197,11 +1211,11 @@ def test_user_defined_stable_version_branch_with_tags(self): self.assertTrue(version_stable.active) self.assertEqual( 'origin/stable', - self.pip.get_stable_version().identifier, + self.pip.get_stable_version().identifier ) # There arent others stable slugs like stable_a other_stable = self.pip.versions.filter( - slug__startswith='stable_', + slug__startswith='stable_' ) self.assertFalse(other_stable.exists()) @@ -1218,10 +1232,10 @@ def test_user_defined_stable_version_branch_with_tags(self): self.assertTrue(version_stable.active) self.assertEqual( 'origin/stable', - self.pip.get_stable_version().identifier, + self.pip.get_stable_version().identifier ) other_stable = self.pip.versions.filter( - slug__startswith='stable_', + slug__startswith='stable_' ) self.assertFalse(other_stable.exists()) @@ -1277,12 +1291,12 @@ def test_user_defined_latest_version_tag(self): self.assertTrue(version_latest.active) self.assertEqual( '1abc2def3', - version_latest.identifier, + version_latest.identifier ) # There arent others latest slugs like latest_a other_latest = self.pip.versions.filter( - slug__startswith='latest_', + slug__startswith='latest_' ) self.assertFalse(other_latest.exists()) @@ -1299,10 +1313,10 @@ def test_user_defined_latest_version_tag(self): self.assertTrue(version_latest.active) self.assertEqual( '1abc2def3', - version_latest.identifier, + version_latest.identifier ) other_latest = self.pip.versions.filter( - slug__startswith='latest_', + slug__startswith='latest_' ) self.assertFalse(other_latest.exists()) @@ -1334,12 +1348,12 @@ def test_user_defined_latest_version_branch(self): self.assertTrue(version_latest.active) self.assertEqual( 'origin/latest', - version_latest.identifier, + version_latest.identifier ) # There arent others latest slugs like latest_a other_latest = self.pip.versions.filter( - slug__startswith='latest_', + slug__startswith='latest_' ) self.assertFalse(other_latest.exists()) @@ -1356,9 +1370,9 @@ def test_user_defined_latest_version_branch(self): self.assertTrue(version_latest.active) self.assertEqual( 'origin/latest', - version_latest.identifier, + version_latest.identifier ) other_latest = self.pip.versions.filter( - slug__startswith='latest_', + slug__startswith='latest_' ) self.assertFalse(other_latest.exists()) diff --git a/readthedocs/rtd_tests/tests/test_urls.py b/readthedocs/rtd_tests/tests/test_urls.py index b01c9997648..d70e0be8b71 100644 --- a/readthedocs/rtd_tests/tests/test_urls.py +++ b/readthedocs/rtd_tests/tests/test_urls.py @@ -1,6 +1,7 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from django.urls import reverse, NoReverseMatch from django.test import TestCase -from django.urls import NoReverseMatch, reverse class WipeUrlTests(TestCase): @@ -47,7 +48,7 @@ class TestVersionURLs(TestCase): def test_version_url_with_caps(self): url = reverse( 'project_download_media', - kwargs={'type_': 'pdf', 'version_slug': '1.4.X', 'project_slug': 'django'}, + kwargs={'type_': 'pdf', 'version_slug': u'1.4.X', 'project_slug': u'django'} ) self.assertTrue(url) @@ -57,18 +58,18 @@ class TestProfileDetailURLs(TestCase): def test_profile_detail_url(self): url = reverse( 'profiles_profile_detail', - kwargs={'username': 'foo+bar'}, - ) + kwargs={'username': 'foo+bar'} + ) self.assertEqual(url, '/profiles/foo+bar/') url = reverse( 'profiles_profile_detail', - kwargs={'username': 'abc+def@ghi.jkl'}, - ) + kwargs={'username': 'abc+def@ghi.jkl'} + ) self.assertEqual(url, '/profiles/abc+def@ghi.jkl/') url = reverse( 'profiles_profile_detail', - kwargs={'username': 'abc-def+ghi'}, - ) + kwargs={'username': 'abc-def+ghi'} + ) self.assertEqual(url, '/profiles/abc-def+ghi/') diff --git a/readthedocs/rtd_tests/tests/test_version_commit_name.py b/readthedocs/rtd_tests/tests/test_version_commit_name.py index bf181c3d5da..e9ffd833ee7 100644 --- a/readthedocs/rtd_tests/tests/test_version_commit_name.py +++ b/readthedocs/rtd_tests/tests/test_version_commit_name.py @@ -1,10 +1,15 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import from django.test import TestCase -from django_dynamic_fixture import get, new +from django_dynamic_fixture import get +from django_dynamic_fixture import new -from readthedocs.builds.constants import BRANCH, LATEST, STABLE, TAG +from readthedocs.builds.constants import BRANCH +from readthedocs.builds.constants import LATEST +from readthedocs.builds.constants import STABLE +from readthedocs.builds.constants import TAG from readthedocs.builds.models import Version -from readthedocs.projects.constants import REPO_TYPE_GIT, REPO_TYPE_HG +from readthedocs.projects.constants import REPO_TYPE_GIT +from readthedocs.projects.constants import REPO_TYPE_HG from readthedocs.projects.models import Project @@ -15,60 +20,44 @@ def test_branch_name_unicode_non_ascii(self): self.assertEqual(version.identifier_friendly, unicode_name) def test_branch_name_made_friendly_when_sha(self): - commit_hash = '3d92b728b7d7b842259ac2020c2fa389f13aff0d' - version = new( - Version, identifier=commit_hash, - slug=STABLE, verbose_name=STABLE, type=TAG, - ) + commit_hash = u'3d92b728b7d7b842259ac2020c2fa389f13aff0d' + version = new(Version, identifier=commit_hash, + slug=STABLE, verbose_name=STABLE, type=TAG) # we shorten commit hashes to keep things readable self.assertEqual(version.identifier_friendly, '3d92b728') def test_branch_name(self): - version = new( - Version, identifier='release-2.5.x', - slug='release-2.5.x', verbose_name='release-2.5.x', - type=BRANCH, - ) + version = new(Version, identifier=u'release-2.5.x', + slug=u'release-2.5.x', verbose_name=u'release-2.5.x', + type=BRANCH) self.assertEqual(version.commit_name, 'release-2.5.x') def test_tag_name(self): - version = new( - Version, identifier='10f1b29a2bd2', slug='release-2.5.0', - verbose_name='release-2.5.0', type=TAG, - ) - self.assertEqual(version.commit_name, 'release-2.5.0') + version = new(Version, identifier=u'10f1b29a2bd2', slug=u'release-2.5.0', + verbose_name=u'release-2.5.0', type=TAG) + self.assertEqual(version.commit_name, u'release-2.5.0') def test_branch_with_name_stable(self): - version = new( - Version, identifier='origin/stable', slug=STABLE, - verbose_name='stable', type=BRANCH, - ) - self.assertEqual(version.commit_name, 'stable') + version = new(Version, identifier=u'origin/stable', slug=STABLE, + verbose_name=u'stable', type=BRANCH) + self.assertEqual(version.commit_name, u'stable') def test_stable_version_tag(self): - version = new( - Version, - identifier='3d92b728b7d7b842259ac2020c2fa389f13aff0d', - slug=STABLE, verbose_name=STABLE, type=TAG, - ) - self.assertEqual( - version.commit_name, - '3d92b728b7d7b842259ac2020c2fa389f13aff0d', - ) + version = new(Version, + identifier=u'3d92b728b7d7b842259ac2020c2fa389f13aff0d', + slug=STABLE, verbose_name=STABLE, type=TAG) + self.assertEqual(version.commit_name, + u'3d92b728b7d7b842259ac2020c2fa389f13aff0d') def test_hg_latest_branch(self): hg_project = get(Project, repo_type=REPO_TYPE_HG) - version = new( - Version, identifier='default', slug=LATEST, - verbose_name=LATEST, type=BRANCH, project=hg_project, - ) - self.assertEqual(version.commit_name, 'default') + version = new(Version, identifier=u'default', slug=LATEST, + verbose_name=LATEST, type=BRANCH, project=hg_project) + self.assertEqual(version.commit_name, u'default') def test_git_latest_branch(self): git_project = get(Project, repo_type=REPO_TYPE_GIT) - version = new( - Version, project=git_project, - identifier='origin/master', slug=LATEST, - verbose_name=LATEST, type=BRANCH, - ) - self.assertEqual(version.commit_name, 'master') + version = new(Version, project=git_project, + identifier=u'origin/master', slug=LATEST, + verbose_name=LATEST, type=BRANCH) + self.assertEqual(version.commit_name, u'master') diff --git a/readthedocs/rtd_tests/tests/test_version_config.py b/readthedocs/rtd_tests/tests/test_version_config.py index 2bcd61104de..82286ade4bf 100644 --- a/readthedocs/rtd_tests/tests/test_version_config.py +++ b/readthedocs/rtd_tests/tests/test_version_config.py @@ -1,4 +1,5 @@ -# -*- coding: utf-8 -*- +from __future__ import division, print_function, unicode_literals + from django.test import TestCase from django_dynamic_fixture import get @@ -16,12 +17,12 @@ def test_get_correct_config(self): build_old = Build.objects.create( project=self.project, version=self.version, - config={'version': 1}, + config={'version': 1} ) build_new = Build.objects.create( project=self.project, version=self.version, - config={'version': 2}, + config={'version': 2} ) build_new_error = Build.objects.create( project=self.project, @@ -42,7 +43,7 @@ def test_get_correct_config_when_same_config(self): Build, project=self.project, version=self.version, - config={}, + config={} ) build_old.config = {'version': 1} build_old.save() @@ -51,7 +52,7 @@ def test_get_correct_config_when_same_config(self): Build, project=self.project, version=self.version, - config={}, + config={} ) build_new.config = {'version': 1} build_new.save() diff --git a/readthedocs/rtd_tests/tests/test_version_slug.py b/readthedocs/rtd_tests/tests/test_version_slug.py index a0d4b4b28b8..31cf2ca72fc 100644 --- a/readthedocs/rtd_tests/tests/test_version_slug.py +++ b/readthedocs/rtd_tests/tests/test_version_slug.py @@ -1,10 +1,10 @@ -# -*- coding: utf-8 -*- +from __future__ import absolute_import import re - from django.test import TestCase from readthedocs.builds.models import Version -from readthedocs.builds.version_slug import VERSION_SLUG_REGEX, VersionSlugField +from readthedocs.builds.version_slug import VersionSlugField +from readthedocs.builds.version_slug import VERSION_SLUG_REGEX from readthedocs.projects.models import Project @@ -27,7 +27,7 @@ def test_multiple_words(self): class VersionSlugFieldTests(TestCase): - fixtures = ['eric', 'test_data'] + fixtures = ["eric", "test_data"] def setUp(self): self.pip = Project.objects.get(slug='pip') @@ -35,68 +35,58 @@ def setUp(self): def test_saving(self): version = Version.objects.create( verbose_name='1.0', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, '1.0') def test_normalizing(self): version = Version.objects.create( verbose_name='1%0', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, '1-0') def test_normalizing_slashes(self): version = Version.objects.create( verbose_name='releases/1.0', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, 'releases-1.0') def test_uppercase(self): version = Version.objects.create( verbose_name='SomeString-charclass', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, 'somestring-charclass') def test_placeholder_as_name(self): version = Version.objects.create( verbose_name='-', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, 'unknown') def test_multiple_empty_names(self): version = Version.objects.create( verbose_name='-', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, 'unknown') version = Version.objects.create( verbose_name='-./.-', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, 'unknown_a') def test_uniqueness(self): version = Version.objects.create( verbose_name='1!0', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, '1-0') version = Version.objects.create( verbose_name='1%0', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, '1-0_a') version = Version.objects.create( verbose_name='1?0', - project=self.pip, - ) + project=self.pip) self.assertEqual(version.slug, '1-0_b') def test_uniquifying_suffix(self): diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index 21515ad886b..e68989e1483 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -1,10 +1,16 @@ # -*- coding: utf-8 -*- -from urllib.parse import urlsplit +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) import mock from django.contrib.auth.models import User -from django.test import TestCase from django.urls import reverse +from django.test import TestCase +from django.utils.six.moves.urllib.parse import urlsplit from django_dynamic_fixture import get, new from readthedocs.builds.constants import LATEST @@ -13,7 +19,6 @@ from readthedocs.projects.forms import UpdateProjectForm from readthedocs.projects.models import ImportedFile, Project - class Testmaker(TestCase): def setUp(self): @@ -151,7 +156,7 @@ def test_project_translations(self): def test_project_translations_delete(self): response = self.client.get( - '/dashboard/pip/translations/delete/a-translation/', + '/dashboard/pip/translations/delete/a-translation/' ) self.assertRedirectToLogin(response) @@ -214,13 +219,13 @@ def setUp(self): def test_deny_delete_for_non_project_admins(self): response = self.client.get( - '/dashboard/my-mainproject/subprojects/delete/my-subproject/', + '/dashboard/my-mainproject/subprojects/delete/my-subproject/' ) self.assertEqual(response.status_code, 404) self.assertTrue( self.subproject in - [r.child for r in self.project.subprojects.all()], + [r.child for r in self.project.subprojects.all()] ) def test_admins_can_delete_subprojects(self): @@ -239,7 +244,7 @@ def test_admins_can_delete_subprojects(self): self.assertEqual(response.status_code, 405) self.assertTrue( self.subproject in - [r.child for r in self.project.subprojects.all()], + [r.child for r in self.project.subprojects.all()] ) # Test POST response = self.client.post( @@ -248,11 +253,11 @@ def test_admins_can_delete_subprojects(self): self.assertEqual(response.status_code, 302) self.assertTrue( self.subproject not in - [r.child for r in self.project.subprojects.all()], + [r.child for r in self.project.subprojects.all()] ) def test_project_admins_can_delete_subprojects_that_they_are_not_admin_of( - self, + self ): self.project.users.add(self.user) self.assertFalse(AdminPermission.is_admin(self.user, self.subproject)) @@ -263,7 +268,7 @@ def test_project_admins_can_delete_subprojects_that_they_are_not_admin_of( self.assertEqual(response.status_code, 302) self.assertTrue( self.subproject not in - [r.child for r in self.project.subprojects.all()], + [r.child for r in self.project.subprojects.all()] ) diff --git a/readthedocs/rtd_tests/utils.py b/readthedocs/rtd_tests/utils.py index 79f6eb0bbef..1908ef0ccd5 100644 --- a/readthedocs/rtd_tests/utils.py +++ b/readthedocs/rtd_tests/utils.py @@ -1,6 +1,13 @@ # -*- coding: utf-8 -*- """Utility functions for use in tests.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import subprocess import textwrap @@ -15,12 +22,13 @@ from readthedocs.doc_builder.base import restoring_chdir - log = logging.getLogger(__name__) def get_readthedocs_app_path(): - """Return the absolute path of the ``readthedocs`` app.""" + """ + Return the absolute path of the ``readthedocs`` app. + """ try: import readthedocs @@ -34,7 +42,7 @@ def get_readthedocs_app_path(): def check_output(command, env=None): output = subprocess.Popen( command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=env, + env=env ).communicate()[0] log.info(output) return output @@ -53,7 +61,7 @@ def make_test_git(): # the repo check_output(['git', 'checkout', '-b', 'submodule', 'master'], env=env) add_git_submodule_without_cloning( - directory, 'foobar', 'https://foobar.com/git', + directory, 'foobar', 'https://foobar.com/git' ) check_output(['git', 'add', '.'], env=env) check_output(['git', 'commit', '-m"Add submodule"'], env=env) @@ -61,7 +69,7 @@ def make_test_git(): # Add an invalid submodule URL in the invalidsubmodule branch check_output(['git', 'checkout', '-b', 'invalidsubmodule', 'master'], env=env) add_git_submodule_without_cloning( - directory, 'invalid', 'git@github.com:rtfd/readthedocs.org.git', + directory, 'invalid', 'git@github.com:rtfd/readthedocs.org.git' ) check_output(['git', 'add', '.'], env=env) check_output(['git', 'commit', '-m"Add invalid submodule"'], env=env) @@ -122,11 +130,11 @@ def make_git_repo(directory, name='sample_repo'): check_output(['git', 'init'] + [directory], env=env) check_output( ['git', 'config', 'user.email', 'dev@readthedocs.org'], - env=env, + env=env ) check_output( ['git', 'config', 'user.name', 'Read the Docs'], - env=env, + env=env ) # Set up the actual repository @@ -179,10 +187,8 @@ def delete_git_branch(directory, branch): @restoring_chdir -def create_git_submodule( - directory, submodule, - msg='Add realative submodule', branch='master', -): +def create_git_submodule(directory, submodule, + msg='Add realative submodule', branch='master'): env = environ.copy() env['GIT_DIR'] = pjoin(directory, '.git') chdir(directory) diff --git a/readthedocs/search/indexes.py b/readthedocs/search/indexes.py index 05a2f759ac7..48e4baecc5e 100644 --- a/readthedocs/search/indexes.py +++ b/readthedocs/search/indexes.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Search indexing classes to index into Elasticsearch. @@ -14,14 +12,20 @@ TODO: Handle page removal case in Page. + """ -from django.conf import settings +from __future__ import absolute_import +from builtins import object + from django.utils import timezone + from elasticsearch import Elasticsearch, exceptions from elasticsearch.helpers import bulk_index +from django.conf import settings -class Index: + +class Index(object): """Base class to define some common methods across indexes.""" @@ -37,8 +41,9 @@ def get_settings(self, settings_override=None): """ Returns settings to be passed to ES create_index. - If `settings_override` is provided, this will use `settings_override` to - override the defaults defined here. + If `settings_override` is provided, this will use `settings_override` + to override the defaults defined here. + """ default_settings = { 'number_of_replicas': settings.ES_DEFAULT_NUM_REPLICAS, @@ -62,6 +67,7 @@ def get_analysis(self): define the stopwords for that language. For all languages we've customized we're using the ICU plugin. + """ analyzers = {} filters = {} @@ -86,16 +92,15 @@ def get_analysis(self): } def timestamped_index(self): - return '{}-{}'.format( - self._index, - timezone.now().strftime('%Y%m%d%H%M%S'), - ) + return '{0}-{1}'.format( + self._index, timezone.now().strftime('%Y%m%d%H%M%S')) def create_index(self, index=None): """ Creates index. This uses `get_settings` and `get_mappings` to define the index. + """ index = index or self._index body = { @@ -111,14 +116,8 @@ def put_mapping(self, index=None): index = index or self._index self.es.indices.put_mapping(self._type, self.get_mapping(), index) - def bulk_index( - self, - data, - index=None, - chunk_size=500, - parent=None, - routing=None, - ): + def bulk_index(self, data, index=None, chunk_size=500, parent=None, + routing=None): """ Given a list of documents, uses Elasticsearch bulk indexing. @@ -126,6 +125,7 @@ def bulk_index( `chunk_size` defaults to the elasticsearch lib's default. Override per your document size as needed. + """ index = index or self._index docs = [] @@ -152,7 +152,7 @@ def index_document(self, data, index=None, parent=None, routing=None): 'index': index or self._index, 'doc_type': self._type, 'body': doc, - 'id': doc['id'], + 'id': doc['id'] } if parent: kwargs['parent'] = parent @@ -202,12 +202,8 @@ def update_aliases(self, new_index, delete=True): actions = [] if old_index: - actions.append({ - 'remove': { - 'index': old_index, - 'alias': self._index, - }, - }) + actions.append({'remove': {'index': old_index, + 'alias': self._index}}) actions.append({'add': {'index': new_index, 'alias': self._index}}) self.es.indices.update_aliases(body={'actions': actions}) @@ -217,14 +213,13 @@ def update_aliases(self, new_index, delete=True): self.es.indices.delete(index=old_index) def search(self, body, **kwargs): - return self.es.search( - index=self._index, doc_type=self._type, body=body, **kwargs - ) + return self.es.search(index=self._index, doc_type=self._type, + body=body, **kwargs) class ProjectIndex(Index): - """Search index configuration for Projects.""" + """Search index configuration for Projects""" _type = 'project' @@ -236,10 +231,8 @@ def get_mapping(self): 'properties': { 'id': {'type': 'long'}, 'name': {'type': 'string', 'analyzer': 'default_icu'}, - 'description': { - 'type': 'string', - 'analyzer': 'default_icu', - }, + 'description': {'type': 'string', 'analyzer': 'default_icu'}, + 'slug': {'type': 'string', 'index': 'not_analyzed'}, 'lang': {'type': 'string', 'index': 'not_analyzed'}, 'tags': {'type': 'string', 'index': 'not_analyzed'}, @@ -257,8 +250,8 @@ def get_mapping(self): 'url': {'type': 'string', 'index': 'not_analyzed'}, # Add a weight field to enhance relevancy scoring. 'weight': {'type': 'float'}, - }, - }, + } + } } return mapping @@ -266,16 +259,7 @@ def get_mapping(self): def extract_document(self, data): doc = {} - attrs = ( - 'id', - 'name', - 'slug', - 'description', - 'lang', - 'tags', - 'author', - 'url', - ) + attrs = ('id', 'name', 'slug', 'description', 'lang', 'tags', 'author', 'url') for attr in attrs: doc[attr] = data.get(attr, '') @@ -287,7 +271,7 @@ def extract_document(self, data): class PageIndex(Index): - """Search index configuration for Pages.""" + """Search index configuration for Pages""" _type = 'page' _parent = 'project' @@ -307,13 +291,14 @@ def get_mapping(self): 'path': {'type': 'string', 'index': 'not_analyzed'}, 'taxonomy': {'type': 'string', 'index': 'not_analyzed'}, 'commit': {'type': 'string', 'index': 'not_analyzed'}, + 'title': {'type': 'string', 'analyzer': 'default_icu'}, 'headers': {'type': 'string', 'analyzer': 'default_icu'}, 'content': {'type': 'string', 'analyzer': 'default_icu'}, # Add a weight field to enhance relevancy scoring. 'weight': {'type': 'float'}, - }, - }, + } + } } return mapping @@ -321,17 +306,8 @@ def get_mapping(self): def extract_document(self, data): doc = {} - attrs = ( - 'id', - 'project', - 'title', - 'headers', - 'version', - 'path', - 'content', - 'taxonomy', - 'commit', - ) + attrs = ('id', 'project', 'title', 'headers', 'version', 'path', + 'content', 'taxonomy', 'commit') for attr in attrs: doc[attr] = data.get(attr, '') @@ -343,7 +319,7 @@ def extract_document(self, data): class SectionIndex(Index): - """Search index configuration for Sections.""" + """Search index configuration for Sections""" _type = 'section' _parent = 'page' @@ -374,16 +350,13 @@ def get_mapping(self): 'blocks': { 'type': 'object', 'properties': { - 'code': { - 'type': 'string', - 'analyzer': 'default_icu', - }, - }, + 'code': {'type': 'string', 'analyzer': 'default_icu'} + } }, # Add a weight field to enhance relevancy scoring. 'weight': {'type': 'float'}, - }, - }, + } + } } return mapping @@ -391,16 +364,7 @@ def get_mapping(self): def extract_document(self, data): doc = {} - attrs = ( - 'id', - 'project', - 'title', - 'page_id', - 'version', - 'path', - 'content', - 'commit', - ) + attrs = ('id', 'project', 'title', 'page_id', 'version', 'path', 'content', 'commit') for attr in attrs: doc[attr] = data.get(attr, '') diff --git a/readthedocs/search/lib.py b/readthedocs/search/lib.py index 007a95afdd9..8500a829b03 100644 --- a/readthedocs/search/lib.py +++ b/readthedocs/search/lib.py @@ -1,65 +1,55 @@ -# -*- coding: utf-8 -*- - """Utilities related to searching Elastic.""" +from __future__ import absolute_import +from __future__ import print_function from pprint import pprint from django.conf import settings +from .indexes import PageIndex, ProjectIndex, SectionIndex + from readthedocs.builds.constants import LATEST from readthedocs.projects.models import Project -from readthedocs.search.signals import ( - before_file_search, - before_project_search, - before_section_search, -) - -from .indexes import PageIndex, ProjectIndex, SectionIndex +from readthedocs.search.signals import (before_project_search, + before_file_search, + before_section_search) def search_project(request, query, language=None): """Search index for projects matching query.""" body = { - 'query': { - 'bool': { - 'should': [ - {'match': {'name': {'query': query, 'boost': 10}}}, - {'match': {'description': {'query': query}}}, - ], + "query": { + "bool": { + "should": [ + {"match": {"name": {"query": query, "boost": 10}}}, + {"match": {"description": {"query": query}}}, + ] }, }, - 'facets': { - 'language': { - 'terms': {'field': 'lang'}, + "facets": { + "language": { + "terms": {"field": "lang"}, }, }, - 'highlight': { - 'fields': { - 'name': {}, - 'description': {}, - }, + "highlight": { + "fields": { + "name": {}, + "description": {}, + } }, - 'fields': ['name', 'slug', 'description', 'lang', 'url'], - 'size': 50, # TODO: Support pagination. + "fields": ["name", "slug", "description", "lang", "url"], + "size": 50 # TODO: Support pagination. } if language: - body['facets']['language']['facet_filter'] = { - 'term': {'lang': language}, - } - body['filter'] = {'term': {'lang': language}} + body['facets']['language']['facet_filter'] = {"term": {"lang": language}} + body['filter'] = {"term": {"lang": language}} before_project_search.send(request=request, sender=ProjectIndex, body=body) return ProjectIndex().search(body) -def search_file( - request, - query, - project_slug=None, - version_slug=LATEST, - taxonomy=None, -): +def search_file(request, query, project_slug=None, version_slug=LATEST, taxonomy=None): """ Search index for files matching query. @@ -73,90 +63,78 @@ def search_file( """ kwargs = {} body = { - 'query': { - 'bool': { - 'should': [ - { - 'match_phrase': { - 'title': { - 'query': query, - 'boost': 10, - 'slop': 2, - }, + "query": { + "bool": { + "should": [ + {"match_phrase": { + "title": { + "query": query, + "boost": 10, + "slop": 2, }, - }, - { - 'match_phrase': { - 'headers': { - 'query': query, - 'boost': 5, - 'slop': 3, - }, + }}, + {"match_phrase": { + "headers": { + "query": query, + "boost": 5, + "slop": 3, }, - }, - { - 'match_phrase': { - 'content': { - 'query': query, - 'slop': 5, - }, + }}, + {"match_phrase": { + "content": { + "query": query, + "slop": 5, }, - }, - ], - }, + }}, + ] + } }, - 'facets': { - 'taxonomy': { - 'terms': {'field': 'taxonomy'}, + "facets": { + "taxonomy": { + "terms": {"field": "taxonomy"}, }, - 'project': { - 'terms': {'field': 'project'}, + "project": { + "terms": {"field": "project"}, }, - 'version': { - 'terms': {'field': 'version'}, + "version": { + "terms": {"field": "version"}, }, }, - 'highlight': { - 'fields': { - 'title': {}, - 'headers': {}, - 'content': {}, - }, + "highlight": { + "fields": { + "title": {}, + "headers": {}, + "content": {}, + } }, - 'fields': ['title', 'project', 'version', 'path'], - 'size': 50, # TODO: Support pagination. + "fields": ["title", "project", "version", "path"], + "size": 50 # TODO: Support pagination. } if project_slug or version_slug or taxonomy: - final_filter = {'and': []} + final_filter = {"and": []} if project_slug: try: - project = ( - Project.objects.api(request.user).get(slug=project_slug) - ) + project = (Project.objects + .api(request.user) + .get(slug=project_slug)) project_slugs = [project.slug] # We need to use the obtuse syntax here because the manager # doesn't pass along to ProjectRelationships - project_slugs.extend( - s.slug for s in Project.objects.public( - request.user, - ).filter( - superprojects__parent__slug=project.slug, - ) - ) - final_filter['and'].append({ - 'terms': {'project': project_slugs}, - }) + project_slugs.extend(s.slug for s + in Project.objects.public( + request.user).filter( + superprojects__parent__slug=project.slug)) + final_filter['and'].append({"terms": {"project": project_slugs}}) # Add routing to optimize search by hitting the right shard. # This purposely doesn't apply routing if the project has more # than one parent project. if project.superprojects.exists(): if project.superprojects.count() == 1: - kwargs['routing'] = ( - project.superprojects.first().parent.slug - ) + kwargs['routing'] = (project.superprojects.first() + .parent.slug) else: kwargs['routing'] = project_slug except Project.DoesNotExist: @@ -174,23 +152,18 @@ def search_file( body['facets']['taxonomy']['facet_filter'] = final_filter if settings.DEBUG: - print('Before Signal') + print("Before Signal") pprint(body) before_file_search.send(request=request, sender=PageIndex, body=body) if settings.DEBUG: - print('After Signal') + print("After Signal") pprint(body) return PageIndex().search(body, **kwargs) -def search_section( - request, - query, - project_slug=None, - version_slug=LATEST, - path=None, -): +def search_section(request, query, project_slug=None, version_slug=LATEST, + path=None): """ Search for a section of content. @@ -206,74 +179,70 @@ def search_section( """ kwargs = {} body = { - 'query': { - 'bool': { - 'should': [ - { - 'match_phrase': { - 'title': { - 'query': query, - 'boost': 10, - 'slop': 2, - }, + "query": { + "bool": { + "should": [ + {"match_phrase": { + "title": { + "query": query, + "boost": 10, + "slop": 2, }, - }, - { - 'match_phrase': { - 'content': { - 'query': query, - 'slop': 5, - }, + }}, + {"match_phrase": { + "content": { + "query": query, + "slop": 5, }, - }, - ], - }, + }}, + ] + } }, - 'facets': { - 'project': { - 'terms': {'field': 'project'}, - 'facet_filter': { - 'term': {'version': version_slug}, - }, + "facets": { + "project": { + "terms": {"field": "project"}, + "facet_filter": { + "term": {"version": version_slug}, + } }, }, - 'highlight': { - 'fields': { - 'title': {}, - 'content': {}, - }, + "highlight": { + "fields": { + "title": {}, + "content": {}, + } }, - 'fields': ['title', 'project', 'version', 'path', 'page_id', 'content'], - 'size': 10, # TODO: Support pagination. + "fields": ["title", "project", "version", "path", "page_id", "content"], + "size": 10 # TODO: Support pagination. } if project_slug: body['filter'] = { - 'and': [ - {'term': {'project': project_slug}}, - {'term': {'version': version_slug}}, - ], + "and": [ + {"term": {"project": project_slug}}, + {"term": {"version": version_slug}}, + ] } body['facets']['path'] = { - 'terms': {'field': 'path'}, - 'facet_filter': { - 'term': {'project': project_slug}, - }, + "terms": {"field": "path"}, + "facet_filter": { + "term": {"project": project_slug}, + } }, # Add routing to optimize search by hitting the right shard. kwargs['routing'] = project_slug if path: body['filter'] = { - 'and': [ - {'term': {'path': path}}, - ], + "and": [ + {"term": {"path": path}}, + ] } if path and not project_slug: # Show facets when we only have a path body['facets']['path'] = { - 'terms': {'field': 'path'}, + "terms": {"field": "path"} } before_section_search.send(request=request, sender=PageIndex, body=body) diff --git a/readthedocs/search/parse_json.py b/readthedocs/search/parse_json.py index 194c55ff165..9b19a7e7cb3 100644 --- a/readthedocs/search/parse_json.py +++ b/readthedocs/search/parse_json.py @@ -1,36 +1,31 @@ # -*- coding: utf-8 -*- - """Functions related to converting content into dict/JSON structures.""" +from __future__ import absolute_import + +import logging import codecs import fnmatch import json -import logging import os +from builtins import next, range # pylint: disable=redefined-builtin from pyquery import PyQuery - log = logging.getLogger(__name__) def process_all_json_files(version, build_dir=True): - """Return a list of pages to index.""" + """Return a list of pages to index""" if build_dir: full_path = version.project.full_json_path(version.slug) else: full_path = version.project.get_production_media_path( - type_='json', - version_slug=version.slug, - include_file=False, - ) + type_='json', version_slug=version.slug, include_file=False) html_files = [] for root, _, files in os.walk(full_path): for filename in fnmatch.filter(files, '*.fjson'): - if filename in [ - 'search.fjson', - 'genindex.fjson', - 'py-modindex.fjson']: + if filename in ['search.fjson', 'genindex.fjson', 'py-modindex.fjson']: continue html_files.append(os.path.join(root, filename)) page_list = [] @@ -62,15 +57,15 @@ def generate_sections_from_pyquery(body): h1_section = body('.section > h1') if h1_section: div = h1_section.parent() - h1_title = h1_section.text().replace('¶', '').strip() + h1_title = h1_section.text().replace(u'¶', '').strip() h1_id = div.attr('id') - h1_content = '' + h1_content = "" next_p = body('h1').next() while next_p: if next_p[0].tag == 'div' and 'class' in next_p[0].attrib: if 'section' in next_p[0].attrib['class']: break - h1_content += '\n%s\n' % next_p.html() + h1_content += "\n%s\n" % next_p.html() next_p = next_p.next() if h1_content: yield { @@ -84,7 +79,7 @@ def generate_sections_from_pyquery(body): for num in range(len(section_list)): div = section_list.eq(num).parent() header = section_list.eq(num) - title = header.text().replace('¶', '').strip() + title = header.text().replace(u'¶', '').strip() section_id = div.attr('id') content = div.html() yield { @@ -113,7 +108,7 @@ def process_file(filename): return None if 'body' in data and data['body']: body = PyQuery(data['body']) - body_content = body.text().replace('¶', '') + body_content = body.text().replace(u'¶', '') sections.extend(generate_sections_from_pyquery(body)) else: log.info('Unable to index content for: %s', filename) @@ -124,13 +119,9 @@ def process_file(filename): else: log.info('Unable to index title for: %s', filename) - return { - 'headers': process_headers(data, filename), - 'content': body_content, - 'path': path, - 'title': title, - 'sections': sections, - } + return {'headers': process_headers(data, filename), + 'content': body_content, 'path': path, + 'title': title, 'sections': sections} def recurse_while_none(element): diff --git a/readthedocs/search/signals.py b/readthedocs/search/signals.py index 17cc3649155..6abdf64cce9 100644 --- a/readthedocs/search/signals.py +++ b/readthedocs/search/signals.py @@ -1,9 +1,7 @@ -# -*- coding: utf-8 -*- - """We define custom Django signals to trigger before executing searches.""" +from __future__ import absolute_import import django.dispatch - -before_project_search = django.dispatch.Signal(providing_args=['body']) -before_file_search = django.dispatch.Signal(providing_args=['body']) -before_section_search = django.dispatch.Signal(providing_args=['body']) +before_project_search = django.dispatch.Signal(providing_args=["body"]) +before_file_search = django.dispatch.Signal(providing_args=["body"]) +before_section_search = django.dispatch.Signal(providing_args=["body"]) diff --git a/readthedocs/search/tests/conftest.py b/readthedocs/search/tests/conftest.py index 7b21c7c2137..59961f3a7e2 100644 --- a/readthedocs/search/tests/conftest.py +++ b/readthedocs/search/tests/conftest.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import random import string from random import shuffle @@ -7,14 +6,8 @@ from django_dynamic_fixture import G from readthedocs.projects.models import Project -from readthedocs.search.indexes import ( - Index, - PageIndex, - ProjectIndex, - SectionIndex, -) - -from .dummy_data import ALL_PROJECTS, DUMMY_PAGE_JSON +from readthedocs.search.indexes import Index, ProjectIndex, PageIndex, SectionIndex +from .dummy_data import DUMMY_PAGE_JSON, ALL_PROJECTS @pytest.fixture(autouse=True) diff --git a/readthedocs/search/tests/data/docs/story.json b/readthedocs/search/tests/data/docs/story.json index 10c81a97832..69226b65209 100644 --- a/readthedocs/search/tests/data/docs/story.json +++ b/readthedocs/search/tests/data/docs/story.json @@ -29,4 +29,4 @@ } ], "path": "open-source-philosophy" -} +} \ No newline at end of file diff --git a/readthedocs/search/tests/data/docs/wiping.json b/readthedocs/search/tests/data/docs/wiping.json index bbdbc8860a8..a54889e05fa 100644 --- a/readthedocs/search/tests/data/docs/wiping.json +++ b/readthedocs/search/tests/data/docs/wiping.json @@ -12,4 +12,4 @@ } ], "path": "guides/wipe-environment" -} +} \ No newline at end of file diff --git a/readthedocs/search/tests/data/kuma/docker.json b/readthedocs/search/tests/data/kuma/docker.json index 6e16f7e9784..3f86764073a 100644 --- a/readthedocs/search/tests/data/kuma/docker.json +++ b/readthedocs/search/tests/data/kuma/docker.json @@ -22,4 +22,4 @@ } ], "path": "docker" -} +} \ No newline at end of file diff --git a/readthedocs/search/tests/data/kuma/documentation.json b/readthedocs/search/tests/data/kuma/documentation.json index 8c7b44a42e6..310a01d05c8 100644 --- a/readthedocs/search/tests/data/kuma/documentation.json +++ b/readthedocs/search/tests/data/kuma/documentation.json @@ -18,4 +18,4 @@ } ], "path": "documentation" -} +} \ No newline at end of file diff --git a/readthedocs/search/tests/data/pipeline/installation.json b/readthedocs/search/tests/data/pipeline/installation.json index 22bba4f08fe..30fb78d1d78 100644 --- a/readthedocs/search/tests/data/pipeline/installation.json +++ b/readthedocs/search/tests/data/pipeline/installation.json @@ -30,4 +30,4 @@ } ], "path": "installation" -} +} \ No newline at end of file diff --git a/readthedocs/search/tests/data/pipeline/signals.json b/readthedocs/search/tests/data/pipeline/signals.json index abed6187b3b..3bf3a80537c 100644 --- a/readthedocs/search/tests/data/pipeline/signals.json +++ b/readthedocs/search/tests/data/pipeline/signals.json @@ -24,4 +24,4 @@ } ], "path": "signals" -} +} \ No newline at end of file diff --git a/readthedocs/search/tests/dummy_data.py b/readthedocs/search/tests/dummy_data.py index 2e99d4e7711..fbd4eed1f11 100644 --- a/readthedocs/search/tests/dummy_data.py +++ b/readthedocs/search/tests/dummy_data.py @@ -1,8 +1,6 @@ -# -*- coding: utf-8 -*- import json import os - _DATA_FILES = { 'pipeline': ['installation.json', 'signals.json'], 'kuma': ['documentation.json', 'docker.json'], @@ -16,7 +14,7 @@ def _get_dummy_json(): data = [] for file_name in value: current_path = os.path.abspath(os.path.dirname(__file__)) - path = os.path.join(current_path, 'data', key, file_name) + path = os.path.join(current_path, "data", key, file_name) with open(path) as f: content = json.load(f) data.append(content) diff --git a/readthedocs/search/tests/test_views.py b/readthedocs/search/tests/test_views.py index bfb7058fc05..07444a731fb 100644 --- a/readthedocs/search/tests/test_views.py +++ b/readthedocs/search/tests/test_views.py @@ -14,7 +14,7 @@ @pytest.mark.django_db @pytest.mark.search -class TestElasticSearch: +class TestElasticSearch(object): url = reverse_lazy('search') @@ -35,23 +35,19 @@ def elastic_index(self, mock_parse_json, all_projects, es_index): self._reindex_elasticsearch(es_index=es_index) def test_search_by_project_name(self, client, project): - result, _ = self._get_search_result( - url=self.url, client=client, - search_params={'q': project.name}, - ) + result, _ = self._get_search_result(url=self.url, client=client, + search_params={'q': project.name}) assert project.name.encode('utf-8') in result.text().encode('utf-8') def test_search_project_show_languages(self, client, project, es_index): - """Test that searching project should show all available languages.""" + """Test that searching project should show all available languages""" # Create a project in bn and add it as a translation G(Project, language='bn', name=project.name) self._reindex_elasticsearch(es_index=es_index) - result, page = self._get_search_result( - url=self.url, client=client, - search_params={'q': project.name}, - ) + result, page = self._get_search_result(url=self.url, client=client, + search_params={'q': project.name}) content = page.find('.navigable .language-list') # There should be 2 languages @@ -59,16 +55,14 @@ def test_search_project_show_languages(self, client, project, es_index): assert 'bn' in content.text() def test_search_project_filter_language(self, client, project, es_index): - """Test that searching project filtered according to language.""" + """Test that searching project filtered according to language""" # Create a project in bn and add it as a translation translate = G(Project, language='bn', name=project.name) self._reindex_elasticsearch(es_index=es_index) search_params = {'q': project.name, 'language': 'bn'} - result, page = self._get_search_result( - url=self.url, client=client, - search_params=search_params, - ) + result, page = self._get_search_result(url=self.url, client=client, + search_params=search_params) # There should be only 1 result assert len(result) == 1 @@ -81,27 +75,20 @@ def test_search_project_filter_language(self, client, project, es_index): @pytest.mark.parametrize('data_type', ['content', 'headers', 'title']) @pytest.mark.parametrize('page_num', [0, 1]) def test_search_by_file_content(self, client, project, data_type, page_num): - query = get_search_query_from_project_file( - project_slug=project.slug, page_num=page_num, - data_type=data_type, - ) - - result, _ = self._get_search_result( - url=self.url, client=client, - search_params={'q': query, 'type': 'file'}, - ) + query = get_search_query_from_project_file(project_slug=project.slug, page_num=page_num, + data_type=data_type) + + result, _ = self._get_search_result(url=self.url, client=client, + search_params={'q': query, 'type': 'file'}) assert len(result) == 1 def test_file_search_show_projects(self, client): - """Test that search result page shows list of projects while searching - for files.""" + """Test that search result page shows list of projects while searching for files""" # `Github` word is present both in `kuma` and `pipeline` files # so search with this phrase - result, page = self._get_search_result( - url=self.url, client=client, - search_params={'q': 'GitHub', 'type': 'file'}, - ) + result, page = self._get_search_result(url=self.url, client=client, + search_params={'q': 'GitHub', 'type': 'file'}) # There should be 2 search result assert len(result) == 2 @@ -115,15 +102,13 @@ def test_file_search_show_projects(self, client): assert 'kuma' and 'pipeline' in text def test_file_search_filter_by_project(self, client): - """Test that search result are filtered according to project.""" + """Test that search result are filtered according to project""" # `Github` word is present both in `kuma` and `pipeline` files # so search with this phrase but filter through `kuma` project search_params = {'q': 'GitHub', 'type': 'file', 'project': 'kuma'} - result, page = self._get_search_result( - url=self.url, client=client, - search_params=search_params, - ) + result, page = self._get_search_result(url=self.url, client=client, + search_params=search_params) # There should be 1 search result as we have filtered assert len(result) == 1 @@ -137,11 +122,11 @@ def test_file_search_filter_by_project(self, client): # as the query is present in both projects content = page.find('.navigable .project-list') if len(content) != 2: - pytest.xfail('failing because currently all projects are not showing in project list') + pytest.xfail("failing because currently all projects are not showing in project list") else: assert 'kuma' and 'pipeline' in content.text() - @pytest.mark.xfail(reason='Versions are not showing correctly! Fixme while rewrite!') + @pytest.mark.xfail(reason="Versions are not showing correctly! Fixme while rewrite!") def test_file_search_show_versions(self, client, all_projects, es_index, settings): # override the settings to index all versions settings.INDEX_ONLY_LATEST = False @@ -153,10 +138,8 @@ def test_file_search_show_versions(self, client, all_projects, es_index, setting query = get_search_query_from_project_file(project_slug=project.slug) - result, page = self._get_search_result( - url=self.url, client=client, - search_params={'q': query, 'type': 'file'}, - ) + result, page = self._get_search_result(url=self.url, client=client, + search_params={'q': query, 'type': 'file'}) # There should be only one result because by default # only latest version result should be there @@ -178,7 +161,7 @@ def test_file_search_show_versions(self, client, all_projects, es_index, setting assert sorted(project_versions) == sorted(content_versions) def test_file_search_subprojects(self, client, all_projects, es_index): - """File search should return results from subprojects also.""" + """File search should return results from subprojects also""" project = all_projects[0] subproject = all_projects[1] # Add another project as subproject of the project @@ -188,9 +171,7 @@ def test_file_search_subprojects(self, client, all_projects, es_index): # Now search with subproject content but explicitly filter by the parent project query = get_search_query_from_project_file(project_slug=subproject.slug) search_params = {'q': query, 'type': 'file', 'project': project.slug} - result, page = self._get_search_result( - url=self.url, client=client, - search_params=search_params, - ) + result, page = self._get_search_result(url=self.url, client=client, + search_params=search_params) assert len(result) == 1 diff --git a/readthedocs/search/tests/utils.py b/readthedocs/search/tests/utils.py index 80bf0fadff6..a48ea83dd74 100644 --- a/readthedocs/search/tests/utils.py +++ b/readthedocs/search/tests/utils.py @@ -1,12 +1,9 @@ -# -*- coding: utf-8 -*- from readthedocs.search.tests.dummy_data import DUMMY_PAGE_JSON def get_search_query_from_project_file(project_slug, page_num=0, data_type='title'): - """ - Return search query from the project's page file. - - Query is generated from the value of `data_type` + """Return search query from the project's page file. + Query is generated from the value of `data_type` """ all_pages = DUMMY_PAGE_JSON[project_slug] diff --git a/readthedocs/search/utils.py b/readthedocs/search/utils.py index 85c3c5735a3..a742a341912 100644 --- a/readthedocs/search/utils.py +++ b/readthedocs/search/utils.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- - """Utilities related to reading and generating indexable search content.""" -import codecs -import fnmatch -import json -import logging +from __future__ import absolute_import + import os +import fnmatch import re +import codecs +import logging +import json +from builtins import next, range from pyquery import PyQuery @@ -21,10 +23,7 @@ def process_mkdocs_json(version, build_dir=True): full_path = version.project.full_json_path(version.slug) else: full_path = version.project.get_production_media_path( - type_='json', - version_slug=version.slug, - include_file=False, - ) + type_='json', version_slug=version.slug, include_file=False) html_files = [] for root, _, files in os.walk(full_path): @@ -36,14 +35,8 @@ def process_mkdocs_json(version, build_dir=True): continue relative_path = parse_path_from_file(file_path=filename) html = parse_content_from_file(file_path=filename) - headers = parse_headers_from_file( - documentation_type='mkdocs', - file_path=filename, - ) - sections = parse_sections_from_file( - documentation_type='mkdocs', - file_path=filename, - ) + headers = parse_headers_from_file(documentation_type='mkdocs', file_path=filename) + sections = parse_sections_from_file(documentation_type='mkdocs', file_path=filename) try: title = sections[0]['title'] except IndexError: @@ -68,7 +61,7 @@ def valid_mkdocs_json(file_path): try: with codecs.open(file_path, encoding='utf-8', mode='r') as f: content = f.read() - except IOError: + except IOError as e: log.warning( '(Search Index) Unable to index file: %s', file_path, @@ -81,10 +74,7 @@ def valid_mkdocs_json(file_path): page_json = json.loads(content) for to_check in ['url', 'content']: if to_check not in page_json: - log.warning( - '(Search Index) Unable to index file: %s error: Invalid JSON', - file_path, - ) + log.warning('(Search Index) Unable to index file: %s error: Invalid JSON', file_path) return None return True @@ -95,7 +85,7 @@ def parse_path_from_file(file_path): try: with codecs.open(file_path, encoding='utf-8', mode='r') as f: content = f.read() - except IOError: + except IOError as e: log.warning( '(Search Index) Unable to index file: %s', file_path, @@ -124,7 +114,7 @@ def parse_content_from_file(file_path): try: with codecs.open(file_path, encoding='utf-8', mode='r') as f: content = f.read() - except IOError: + except IOError as e: log.info( '(Search Index) Unable to index file: %s', file_path, @@ -138,10 +128,7 @@ def parse_content_from_file(file_path): content = parse_content(page_content) if not content: - log.info( - '(Search Index) Unable to index file: %s, empty file', - file_path, - ) + log.info('(Search Index) Unable to index file: %s, empty file', file_path) else: log.debug('(Search Index) %s length: %s', file_path, len(content)) return content @@ -165,7 +152,7 @@ def parse_headers_from_file(documentation_type, file_path): try: with codecs.open(file_path, encoding='utf-8', mode='r') as f: content = f.read() - except IOError: + except IOError as e: log.info( '(Search Index) Unable to index file: %s', file_path, @@ -196,7 +183,7 @@ def parse_sections_from_file(documentation_type, file_path): try: with codecs.open(file_path, encoding='utf-8', mode='r') as f: content = f.read() - except IOError: + except IOError as e: log.info( '(Search Index) Unable to index file: %s', file_path, @@ -220,15 +207,15 @@ def parse_sphinx_sections(content): h1_section = body('.section > h1') if h1_section: div = h1_section.parent() - h1_title = h1_section.text().replace('¶', '').strip() + h1_title = h1_section.text().replace(u'¶', '').strip() h1_id = div.attr('id') - h1_content = '' + h1_content = "" next_p = next(body('h1')) # pylint: disable=stop-iteration-return while next_p: if next_p[0].tag == 'div' and 'class' in next_p[0].attrib: if 'section' in next_p[0].attrib['class']: break - h1_content += '\n%s\n' % next_p.html() + h1_content += "\n%s\n" % next_p.html() next_p = next(next_p) # pylint: disable=stop-iteration-return if h1_content: yield { @@ -242,7 +229,7 @@ def parse_sphinx_sections(content): for num in range(len(section_list)): div = section_list.eq(num).parent() header = section_list.eq(num) - title = header.text().replace('¶', '').strip() + title = header.text().replace(u'¶', '').strip() section_id = div.attr('id') content = div.html() yield { @@ -265,14 +252,14 @@ def parse_mkdocs_sections(content): h1 = body('h1') h1_id = h1.attr('id') h1_title = h1.text().strip() - h1_content = '' + h1_content = "" next_p = next(body('h1')) # pylint: disable=stop-iteration-return while next_p: if next_p[0].tag == 'h2': break h1_html = next_p.html() if h1_html: - h1_content += '\n%s\n' % h1_html + h1_content += "\n%s\n" % h1_html next_p = next(next_p) # pylint: disable=stop-iteration-return if h1_content: yield { @@ -287,14 +274,14 @@ def parse_mkdocs_sections(content): h2 = section_list.eq(num) h2_title = h2.text().strip() section_id = h2.attr('id') - h2_content = '' + h2_content = "" next_p = next(body('h2')) # pylint: disable=stop-iteration-return while next_p: if next_p[0].tag == 'h2': break h2_html = next_p.html() if h2_html: - h2_content += '\n%s\n' % h2_html + h2_content += "\n%s\n" % h2_html next_p = next(next_p) # pylint: disable=stop-iteration-return if h2_content: yield { diff --git a/readthedocs/search/views.py b/readthedocs/search/views.py index 8615f183a0f..bac1969e80e 100644 --- a/readthedocs/search/views.py +++ b/readthedocs/search/views.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- - """Search views.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import collections import logging from pprint import pprint @@ -11,9 +13,8 @@ from readthedocs.builds.constants import LATEST from readthedocs.search import lib as search_lib - log = logging.getLogger(__name__) -LOG_TEMPLATE = '(Elastic Search) [{user}:{type}] [{project}:{version}:{language}] {msg}' +LOG_TEMPLATE = u'(Elastic Search) [{user}:{type}] [{project}:{version}:{language}] {msg}' UserInput = collections.namedtuple( 'UserInput', @@ -45,18 +46,11 @@ def elastic_search(request): if user_input.query: if user_input.type == 'project': results = search_lib.search_project( - request, - user_input.query, - language=user_input.language, - ) + request, user_input.query, language=user_input.language) elif user_input.type == 'file': results = search_lib.search_file( - request, - user_input.query, - project_slug=user_input.project, - version_slug=user_input.version, - taxonomy=user_input.taxonomy, - ) + request, user_input.query, project_slug=user_input.project, + version_slug=user_input.version, taxonomy=user_input.taxonomy) if results: # pre and post 1.0 compat @@ -88,8 +82,7 @@ def elastic_search(request): version=user_input.version or '', language=user_input.language or '', msg=user_input.query or '', - ), - ) + )) template_vars = user_input._asdict() template_vars.update({ diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 205dbdd9989..a6a3e867978 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -43,7 +43,7 @@ class CommunityBaseSettings(Settings): PUBLIC_DOMAIN = None PUBLIC_DOMAIN_USES_HTTPS = False USE_SUBDOMAIN = False - PUBLIC_API_URL = 'https://{}'.format(PRODUCTION_DOMAIN) + PUBLIC_API_URL = 'https://{0}'.format(PRODUCTION_DOMAIN) # Email DEFAULT_FROM_EMAIL = 'no-reply@readthedocs.org' diff --git a/readthedocs/settings/dev.py b/readthedocs/settings/dev.py index 8fd9860a0bd..7fa4dafe959 100644 --- a/readthedocs/settings/dev.py +++ b/readthedocs/settings/dev.py @@ -50,7 +50,7 @@ def DATABASES(self): # noqa @property def LOGGING(self): # noqa - avoid pep8 N802 - logging = super().LOGGING + logging = super(CommunityDevSettings, self).LOGGING logging['formatters']['default']['format'] = '[%(asctime)s] ' + self.LOG_FORMAT # Allow Sphinx and other tools to create loggers logging['disable_existing_loggers'] = False diff --git a/readthedocs/templates/404.html b/readthedocs/templates/404.html index a364ec864c5..97330784675 100644 --- a/readthedocs/templates/404.html +++ b/readthedocs/templates/404.html @@ -16,6 +16,24 @@ {% block language-select-form %}{% endblock %} {% block content %} + {% if suggestion %} +
+

You've found something that doesn't exist.

+

{{ suggestion.message }}

+ {% ifequal suggestion.type 'top' %} +

+ Go to the top of the documentation. +

+ {% endifequal %} + {% ifequal suggestion.type 'list' %} + + {% endifequal %} +
+ {% endif %} 
 
         \          SORRY            /
diff --git a/readthedocs/templates/account/email/email_confirmation_message.html b/readthedocs/templates/account/email/email_confirmation_message.html
index a74fa11dfc3..bc37aa55a8e 100644
--- a/readthedocs/templates/account/email/email_confirmation_message.html
+++ b/readthedocs/templates/account/email/email_confirmation_message.html
@@ -15,3 +15,4 @@
       {% trans "If you did not sign up for an account with Read the Docs, you can disregard this email." %}
     

 {% endblock %}
+
diff --git a/readthedocs/templates/core/badge_markup.html b/readthedocs/templates/core/badge_markup.html
index f662e2ed689..f739b43f49d 100644
--- a/readthedocs/templates/core/badge_markup.html
+++ b/readthedocs/templates/core/badge_markup.html
@@ -22,7 +22,7 @@ 

             

                 HTML
             

-                
+                
 
 <a href='{{ site_url }}'>
     <img src='{{ badge_url }}' alt='Documentation Status' />
 </a>
diff --git a/readthedocs/templates/core/build_list_detailed.html b/readthedocs/templates/core/build_list_detailed.html
index ee5bb8c442d..33b3b754a7a 100644
--- a/readthedocs/templates/core/build_list_detailed.html
+++ b/readthedocs/templates/core/build_list_detailed.html
@@ -13,3 +13,4 @@
           {% empty %}
             
  • {% trans "No builds found" %}
    • 
           {% endfor %}
+
diff --git a/readthedocs/templates/core/project_list.html b/readthedocs/templates/core/project_list.html
index 642525ab02a..64018b714e4 100644
--- a/readthedocs/templates/core/project_list.html
+++ b/readthedocs/templates/core/project_list.html
@@ -4,3 +4,4 @@
                         {{ project.name }}
                       
                     {% endfor %}
+
diff --git a/readthedocs/templates/core/project_list_detailed.html b/readthedocs/templates/core/project_list_detailed.html
index 1eb24632650..96e660b3b80 100644
--- a/readthedocs/templates/core/project_list_detailed.html
+++ b/readthedocs/templates/core/project_list_detailed.html
@@ -40,7 +40,7 @@
           {% for version in project.ordered_active_versions reversed %}
               
  • {{ version.slug }}
    • 
           {% endfor %}
-      
+       
     
     {% else %}
     
      
diff --git a/readthedocs/templates/core/widesearchbar.html b/readthedocs/templates/core/widesearchbar.html
index d310e94b5e9..c378d92f586 100644
--- a/readthedocs/templates/core/widesearchbar.html
+++ b/readthedocs/templates/core/widesearchbar.html
@@ -18,4 +18,4 @@ 
      {% trans "Search all the docs" %}
    - + \ No newline at end of file diff --git a/readthedocs/templates/dnt-policy.txt b/readthedocs/templates/dnt-policy.txt index 458a36d033d..ad946d1f80c 100644 --- a/readthedocs/templates/dnt-policy.txt +++ b/readthedocs/templates/dnt-policy.txt @@ -1,4 +1,4 @@ -Do Not Track Compliance Policy +Do Not Track Compliance Policy Version 1.0 @@ -32,23 +32,23 @@ When this domain receives Web requests from a user who enables DNT by actively choosing an opt-out setting in their browser or by installing software that is primarily designed to protect privacy ("DNT User"), we will take the following measures with respect to those users' data, subject to the Exceptions, also -listed below: +listed below: -1. END USER IDENTIFIERS: +1. END USER IDENTIFIERS: a. If a DNT User has logged in to our service, all user identifiers, such as - unique or nearly unique cookies, "supercookies" and fingerprints are - discarded as soon as the HTTP(S) response is issued. + unique or nearly unique cookies, "supercookies" and fingerprints are + discarded as soon as the HTTP(S) response is issued. Data structures which associate user identifiers with accounts may be employed to recognize logged in users per Exception 4 below, but may not be associated with records of the user's activities unless otherwise excepted. - b. If a DNT User is not logged in to our service, we will take steps to ensure - that no user identifiers are transmitted to us at all. + b. If a DNT User is not logged in to our service, we will take steps to ensure + that no user identifiers are transmitted to us at all. -2. LOG RETENTION: +2. LOG RETENTION: a. Logs with DNT Users' identifiers removed (but including IP addresses and User Agent strings) may be retained for a period of 10 days or less, @@ -58,13 +58,13 @@ listed below: and performance problems; and that security and data aggregation systems have time to operate. - b. These logs will not be used for any other purposes. + b. These logs will not be used for any other purposes. -3. OTHER DOMAINS: +3. OTHER DOMAINS: a. If this domain transfers identifiable user data about DNT Users to contractors, affiliates or other parties, or embeds from or posts data to - other domains, we will either: + other domains, we will either: b. ensure that the operators of those domains abide by this policy overall by posting it at /.well-known/dnt-policy.txt via HTTPS on the domains in @@ -75,7 +75,7 @@ listed below: ensure that the recipient's policies and practices require the recipient to respect the policy for our DNT Users' data. - OR + OR obtain a contractual commitment from the recipient to respect this policy for our DNT Users' data. @@ -88,14 +88,14 @@ listed below: c. "Identifiable" means any records which are not Anonymized or otherwise covered by the Exceptions below. -4. PERIODIC REASSERTION OF COMPLIANCE: +4. PERIODIC REASSERTION OF COMPLIANCE: At least once every 12 months, we will take reasonable steps commensurate with the size of our organization and the nature of our service to confirm our ongoing compliance with this document, and we will publicly reassert our compliance. -5. USER NOTIFICATION: +5. USER NOTIFICATION: a. If we are required by law to retain or disclose user identifiers, we will attempt to provide the users with notice (unless we are prohibited or it @@ -105,7 +105,7 @@ listed below: b. We will attempt to provide this notice by email, if the users have given us an email address, and by postal mail if the users have provided a - postal address. + postal address. c. If the users do not challenge the disclosure request, we may be legally required to turn over their information. @@ -120,17 +120,17 @@ EXCEPTIONS Data from DNT Users collected by this domain may be logged or retained only in the following specific situations: -1. CONSENT / "OPT BACK IN" +1. CONSENT / "OPT BACK IN" a. DNT Users are opting out from tracking across the Web. It is possible that for some feature or functionality, we will need to ask a DNT User to - "opt back in" to be tracked by us across the entire Web. + "opt back in" to be tracked by us across the entire Web. b. If we do that, we will take reasonable steps to verify that the users who select this option have genuinely intended to opt back in to tracking. One way to do this is by performing scientifically reasonable user studies with a representative sample of our users, but smaller - organizations can satisfy this requirement by other means. + organizations can satisfy this requirement by other means. c. Where we believe that we have opt back in consent, our server will send a tracking value status header "Tk: C" as described in section 6.2 @@ -138,7 +138,7 @@ the following specific situations: http://www.w3.org/TR/tracking-dnt/#tracking-status-value -2. TRANSACTIONS +2. TRANSACTIONS If a DNT User actively and knowingly enters a transaction with our services (for instance, clicking on a clearly-labeled advertisement, @@ -151,19 +151,19 @@ the following specific situations: item will be shipped. By their nature, some transactions will require data to be retained indefinitely. -3. TECHNICAL AND SECURITY LOGGING: +3. TECHNICAL AND SECURITY LOGGING: a. If, during the processing of the initial request (for unique identifiers) or during the subsequent 10 days (for IP addresses and User Agent strings), we obtain specific information that causes our employees or systems to believe that a request is, or is likely to be, part of a security attack, - spam submission, or fraudulent transaction, then logs of those requests - are not subject to this policy. + spam submission, or fraudulent transaction, then logs of those requests + are not subject to this policy. b. If we encounter technical problems with our site, then, in rare circumstances, we may retain logs for longer than 10 days, if that is necessary to diagnose and fix those problems, but this practice will not be - routinized and we will strive to delete such logs as soon as possible. + routinized and we will strive to delete such logs as soon as possible. 4. AGGREGATION: @@ -179,13 +179,13 @@ the following specific situations: that the dataset, plus any additional information that is in our possession or likely to be available to us, does not allow the reconstruction of reading habits, online or offline activity of groups of - fewer than 5000 individuals or devices. + fewer than 5000 individuals or devices. c. If we generate anonymized datasets under this exception we will publicly document our anonymization methods in sufficient detail to allow outside experts to evaluate the effectiveness of those methods. -5. ERRORS: +5. ERRORS: From time to time, there may be errors by which user data is temporarily logged or retained in violation of this policy. If such errors are diff --git a/readthedocs/templates/error_header.html b/readthedocs/templates/error_header.html index e6cd5b2234f..401ccdf5028 100644 --- a/readthedocs/templates/error_header.html +++ b/readthedocs/templates/error_header.html @@ -14,6 +14,24 @@

    + + + +
    + +
    + + diff --git a/readthedocs/templates/flagging/flag_confirm.html b/readthedocs/templates/flagging/flag_confirm.html index 788b0c32d81..ff584e855da 100644 --- a/readthedocs/templates/flagging/flag_confirm.html +++ b/readthedocs/templates/flagging/flag_confirm.html @@ -16,7 +16,7 @@

    {% trans "Confirm flagging" %}

    {% load flagging %}

    {% trans "What you are saying:" %}

    {{ flag_type.description }}

    - +
    {% csrf_token %}

    {% blocktrans %}Are you sure you want to flag "{{ object }}" as "{{ flag_type.title }}"?{% endblocktrans %}

    diff --git a/readthedocs/templates/flagging/flag_success.html b/readthedocs/templates/flagging/flag_success.html index e8ab7fd04be..d1cdff28e81 100644 --- a/readthedocs/templates/flagging/flag_success.html +++ b/readthedocs/templates/flagging/flag_success.html @@ -14,6 +14,6 @@

    {% trans "Flagging successful" %}

    {% block content %}

    {% blocktrans %}You have successfully flagged "{{ object }}" as "{{ flag_type.title }}"{% endblocktrans %}

    - +

    {% trans "Thank you for contributing to the quality of this site!" %}

    {% endblock %} diff --git a/readthedocs/templates/projects/domain_confirm_delete.html b/readthedocs/templates/projects/domain_confirm_delete.html index c9aeef2ac87..8781f45a85b 100644 --- a/readthedocs/templates/projects/domain_confirm_delete.html +++ b/readthedocs/templates/projects/domain_confirm_delete.html @@ -19,3 +19,5 @@
    {% endblock %} + + diff --git a/readthedocs/templates/projects/domain_form.html b/readthedocs/templates/projects/domain_form.html index ed4688751a9..307e089a39a 100644 --- a/readthedocs/templates/projects/domain_form.html +++ b/readthedocs/templates/projects/domain_form.html @@ -41,3 +41,4 @@

    {% endblock %} + diff --git a/readthedocs/templates/projects/domain_list.html b/readthedocs/templates/projects/domain_list.html index be4290e41ff..eab3abb587f 100644 --- a/readthedocs/templates/projects/domain_list.html +++ b/readthedocs/templates/projects/domain_list.html @@ -33,7 +33,7 @@

    {% trans "Existing Domains" %}

    {% endif %} - +

    {% trans "Add new Domain" %}

    {% csrf_token %} {{ form.as_p }} @@ -42,3 +42,4 @@

    {% trans "Add new Domain" %}

    {% endblock %} + diff --git a/readthedocs/templates/projects/environmentvariable_detail.html b/readthedocs/templates/projects/environmentvariable_detail.html deleted file mode 100644 index 920091b4d1e..00000000000 --- a/readthedocs/templates/projects/environmentvariable_detail.html +++ /dev/null @@ -1,30 +0,0 @@ -{% extends "projects/project_edit_base.html" %} - -{% load i18n %} - -{% block title %}{% trans "Environment Variables" %}{% endblock %} - -{% block nav-dashboard %} class="active"{% endblock %} - -{% block editing-option-edit-environment-variables %}class="active"{% endblock %} - -{% block project-environment-variables-active %}active{% endblock %} -{% block project_edit_content_header %} - {% blocktrans trimmed with name=environmentvariable.name %} - Environment Variable: {{ name }} - {% endblocktrans %} -{% endblock %} - -{% block project_edit_content %} - -

    - {% blocktrans trimmed %} - The value of the environment variable is not shown here for sercurity purposes. - {% endblocktrans %} -

    - -
    - {% csrf_token %} - -
    -{% endblock %} diff --git a/readthedocs/templates/projects/environmentvariable_form.html b/readthedocs/templates/projects/environmentvariable_form.html deleted file mode 100644 index 47c92883134..00000000000 --- a/readthedocs/templates/projects/environmentvariable_form.html +++ /dev/null @@ -1,22 +0,0 @@ -{% extends "projects/project_edit_base.html" %} - -{% load i18n %} - -{% block title %}{% trans "Environment Variables" %}{% endblock %} - -{% block nav-dashboard %} class="active"{% endblock %} - -{% block editing-option-edit-environment-variables %}class="active"{% endblock %} - -{% block project-environment-variables-active %}active{% endblock %} -{% block project_edit_content_header %}{% trans "Environment Variables" %}{% endblock %} - -{% block project_edit_content %} -
    - {% csrf_token %} - {{ form.as_p }} - -
    -{% endblock %} diff --git a/readthedocs/templates/projects/environmentvariable_list.html b/readthedocs/templates/projects/environmentvariable_list.html deleted file mode 100644 index 235b3e6a4ac..00000000000 --- a/readthedocs/templates/projects/environmentvariable_list.html +++ /dev/null @@ -1,47 +0,0 @@ -{% extends "projects/project_edit_base.html" %} - -{% load i18n %} - -{% block title %}{% trans "Environment Variables" %}{% endblock %} - -{% block nav-dashboard %} class="active"{% endblock %} - -{% block editing-option-edit-environment-variables %}class="active"{% endblock %} - -{% block project-environment-variables-active %}active{% endblock %} -{% block project_edit_content_header %}{% trans "Environment Variables" %}{% endblock %} - -{% block project_edit_content %} -

    Environment variables allow you to change the way that your build behaves. Take into account that these environment variables are available to all build steps.

    - -
    - -
    - -
    -
    -
      - {% for environmentvariable in object_list %} -
    • - - {{ environmentvariable.name }} - -
      • - {% empty %} -
    • -

      - {% trans 'No environment variables are currently configured.' %} -

      -
      • - {% endfor %} -
    -
    -
    -{% endblock %} diff --git a/readthedocs/templates/projects/notifications/deprecated_build_webhook_email.html b/readthedocs/templates/projects/notifications/deprecated_build_webhook_email.html deleted file mode 100644 index fb19f176532..00000000000 --- a/readthedocs/templates/projects/notifications/deprecated_build_webhook_email.html +++ /dev/null @@ -1,6 +0,0 @@ -

    Your project, {{ project.name }}, is currently using a legacy incoming webhook to trigger builds on Read the Docs. Effective April 1st, 2019, Read the Docs will no longer accept incoming webhooks through these endpoints.

    - -

    To continue building your Read the Docs project on changes to your repository, you will need to configure a new webhook with your VCS provider. You can find more information on how to configure a new webhook in our documentation, at:

    - -{% comment %}Plain text link because of text version of email{% endcomment %} -

    https://docs.readthedocs.io/en/latest/webhooks.html#webhook-deprecated-endpoints

    diff --git a/readthedocs/templates/projects/notifications/deprecated_build_webhook_site.html b/readthedocs/templates/projects/notifications/deprecated_build_webhook_site.html deleted file mode 100644 index 33c5e27e616..00000000000 --- a/readthedocs/templates/projects/notifications/deprecated_build_webhook_site.html +++ /dev/null @@ -1 +0,0 @@ -Your project, {{ project.name }}, needs to be reconfigured in order to continue building automatically after April 1st, 2019. For more information, see our documentation on webhook integrations. diff --git a/readthedocs/templates/projects/notifications/deprecated_github_webhook_email.html b/readthedocs/templates/projects/notifications/deprecated_github_webhook_email.html deleted file mode 100644 index 7d352390d42..00000000000 --- a/readthedocs/templates/projects/notifications/deprecated_github_webhook_email.html +++ /dev/null @@ -1,8 +0,0 @@ -

    Your project, {{ project.name }}, is currently using GitHub Services to trigger builds on Read the Docs. Effective January 31, 2019, GitHub will no longer process requests using the Services feature, and so Read the Docs will not receive notifications on updates to your repository.

    - -

    To continue building your Read the Docs project on changes to your repository, you will need to add a new webhook on your GitHub repository. You can either connect your GitHub account and configure a GitHub webhook integration, or you can add a generic webhook integration.

    - -

    You can find more information on our webhook intergrations in our documentation, at:

    - -{% comment %}Plain text link because of text version of email{% endcomment %} -

    https://docs.readthedocs.io/en/latest/webhooks.html#webhook-github-services

    diff --git a/readthedocs/templates/projects/notifications/deprecated_github_webhook_site.html b/readthedocs/templates/projects/notifications/deprecated_github_webhook_site.html deleted file mode 100644 index 0832efaf793..00000000000 --- a/readthedocs/templates/projects/notifications/deprecated_github_webhook_site.html +++ /dev/null @@ -1 +0,0 @@ -Your project, {{ project.name }}, needs to be reconfigured in order to continue building automatically after January 31st, 2019. For more information, see our documentation on webhook integrations. diff --git a/readthedocs/templates/projects/project_analytics.html b/readthedocs/templates/projects/project_analytics.html index bf682e90ec5..5ca9ab7d86c 100644 --- a/readthedocs/templates/projects/project_analytics.html +++ b/readthedocs/templates/projects/project_analytics.html @@ -64,7 +64,7 @@

    {% trans "Pages" %}

      {% for page, count in page_list %}
    • - {{ page }} + {{ page }} {{ count }} ({{ analytics.scaled_page|key:page }}%)
      • @@ -77,7 +77,7 @@

      {% trans "Versions" %}

        {% for version, count in version_list %}
      • - {{ version }} + {{ version }} {{ count }} ({{ analytics.scaled_version|key:version }}%)
        • diff --git a/readthedocs/templates/projects/project_edit_base.html b/readthedocs/templates/projects/project_edit_base.html index ca4699010df..cc85e88834a 100644 --- a/readthedocs/templates/projects/project_edit_base.html +++ b/readthedocs/templates/projects/project_edit_base.html @@ -23,7 +23,6 @@
      • {% trans "Translations" %}
      • {% trans "Subprojects" %}
      • {% trans "Integrations" %}
        • -
      • {% trans "Environment Variables" %}
      • {% trans "Notifications" %}
        • {% if USE_PROMOS %}
      • {% trans "Advertising" %}
        • diff --git a/readthedocs/templates/projects/project_version_list.html b/readthedocs/templates/projects/project_version_list.html index f85f3be0de7..a83273fe5e0 100644 --- a/readthedocs/templates/projects/project_version_list.html +++ b/readthedocs/templates/projects/project_version_list.html @@ -113,7 +113,7 @@

        {% trans "Inactive Versions" %}

        {% endblock inactive-versions %} - + {% endfor %}
      diff --git a/readthedocs/templates/search/elastic_search.html b/readthedocs/templates/search/elastic_search.html index 387bf85aa33..b14ad50b20f 100644 --- a/readthedocs/templates/search/elastic_search.html +++ b/readthedocs/templates/search/elastic_search.html @@ -68,7 +68,7 @@
      {% trans 'Version' %}
      {{ name }} {% else %} {{ name }} - {% endif %} + {% endif %} ({{ count }}) @@ -86,7 +86,7 @@
      {% trans 'Taxonomy' %}
      {{ name }} {% else %} {{ name }} - {% endif %} + {% endif %} ({{ count }}) @@ -96,7 +96,7 @@
      {% trans 'Taxonomy' %}
      {% endif %} {% endif %} - + {% block sponsor %}
      Search is sponsored by Elastic, and hosted on Elastic Cloud. @@ -144,7 +144,7 @@

      {% blocktrans with query=query|default:"" %}Results for {{ query }}{% endblo {% for result in results.hits.hits %}

    • - {% if result.fields.name %} + {% if result.fields.name %} {# Project #} {{ result.fields.name }} diff --git a/readthedocs/templates/sphinx/conf.py.conf b/readthedocs/templates/sphinx/conf.py.conf index 520670cca16..43e7b58b290 100644 --- a/readthedocs/templates/sphinx/conf.py.conf +++ b/readthedocs/templates/sphinx/conf.py.conf @@ -13,7 +13,7 @@ source_parsers = { '.md': CommonMarkParser, } master_doc = 'index' -project = '{{ project.name }}' +project = u'{{ project.name }}' copyright = str(datetime.now().year) version = '{{ version.verbose_name }}' release = '{{ version.verbose_name }}' @@ -23,6 +23,6 @@ htmlhelp_basename = '{{ project.slug }}' html_theme = 'sphinx_rtd_theme' file_insertion_enabled = False latex_documents = [ - ('index', '{{ project.slug }}.tex', '{{ project.name }} Documentation', - '{{ project.copyright }}', 'manual'), + ('index', '{{ project.slug }}.tex', u'{{ project.name }} Documentation', + u'{{ project.copyright }}', 'manual'), ] diff --git a/readthedocs/templates/style_catalog.html b/readthedocs/templates/style_catalog.html index 78e2123be3c..68ab9b3e845 100644 --- a/readthedocs/templates/style_catalog.html +++ b/readthedocs/templates/style_catalog.html @@ -8,7 +8,7 @@

      Header 1.

      Header 2.

      Header 3.

      Header 4.

      -
      Header 5.
      +
      Header 5.

      Paragraph. Aside.

      Paragraph with link.

      Paragraph with highlighted text.

      @@ -41,7 +41,7 @@
      Header 5.
    • Ordered list item.
      • -
      +
      @@ -130,18 +130,18 @@

    • Module list item with menu.
    • Module list item with menu and right content. Right-aligned-ish content.
      • - +

    • diff --git a/readthedocs/urls.py b/readthedocs/urls.py index fe238508130..e441bc94459 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -1,29 +1,26 @@ -# -*- coding: utf-8 -*- # pylint: disable=missing-docstring +from __future__ import absolute_import + import os from functools import reduce from operator import add +from django.conf.urls import url, include +from django.contrib import admin from django.conf import settings -from django.conf.urls import include, url from django.conf.urls.static import static -from django.contrib import admin -from django.views.generic.base import RedirectView, TemplateView +from django.views.generic.base import TemplateView, RedirectView from tastypie.api import Api -from readthedocs.api.base import ( - FileResource, - ProjectResource, - UserResource, - VersionResource, -) -from readthedocs.core.urls import core_urls, deprecated_urls, docs_urls +from readthedocs.api.base import (ProjectResource, UserResource, + VersionResource, FileResource) +from readthedocs.core.urls import docs_urls, core_urls, deprecated_urls from readthedocs.core.views import ( HomepageView, SupportView, - do_not_track, server_error_404, server_error_500, + do_not_track, ) from readthedocs.search import views as search_views @@ -43,11 +40,8 @@ url(r'^$', HomepageView.as_view(), name='homepage'), url(r'^support/', SupportView.as_view(), name='support'), url(r'^security/', TemplateView.as_view(template_name='security.html')), - url( - r'^\.well-known/security.txt$', - TemplateView - .as_view(template_name='security.txt', content_type='text/plain'), - ), + url(r'^\.well-known/security.txt$', + TemplateView.as_view(template_name='security.txt', content_type='text/plain')), ] rtd_urls = [ @@ -73,10 +67,7 @@ api_urls = [ url(r'^api/', include(v1_api.urls)), url(r'^api/v2/', include('readthedocs.restapi.urls')), - url( - r'^api-auth/', - include('rest_framework.urls', namespace='rest_framework'), - ), + url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), ] i18n_urls = [ @@ -91,11 +82,8 @@ url(r'^\.well-known/dnt/$', do_not_track), # https://github.com/EFForg/dnt-guide#12-how-to-assert-dnt-compliance - url( - r'^\.well-known/dnt-policy.txt$', - TemplateView - .as_view(template_name='dnt-policy.txt', content_type='text/plain'), - ), + url(r'^\.well-known/dnt-policy.txt$', + TemplateView.as_view(template_name='dnt-policy.txt', content_type='text/plain')), ] debug_urls = [] @@ -105,29 +93,16 @@ document_root=os.path.join(settings.MEDIA_ROOT, build_format), ) debug_urls += [ - url( - 'style-catalog/$', - TemplateView.as_view(template_name='style_catalog.html'), - ), + url('style-catalog/$', + TemplateView.as_view(template_name='style_catalog.html')), # This must come last after the build output files - url( - r'^media/(?P.+)$', - RedirectView.as_view(url=settings.STATIC_URL + '%(remainder)s'), - name='media-redirect', - ), + url(r'^media/(?P.+)$', + RedirectView.as_view(url=settings.STATIC_URL + '%(remainder)s'), name='media-redirect'), ] # Export URLs -groups = [ - basic_urls, - rtd_urls, - project_urls, - api_urls, - core_urls, - i18n_urls, - deprecated_urls, -] +groups = [basic_urls, rtd_urls, project_urls, api_urls, core_urls, i18n_urls, deprecated_urls] if settings.DO_NOT_TRACK_ENABLED: # Include Do Not Track URLs if DNT is supported @@ -142,7 +117,7 @@ if 'readthedocsext.embed' in settings.INSTALLED_APPS: api_urls.insert( 0, - url(r'^api/v1/embed/', include('readthedocsext.embed.urls')), + url(r'^api/v1/embed/', include('readthedocsext.embed.urls')) ) if not getattr(settings, 'USE_SUBDOMAIN', False) or settings.DEBUG: diff --git a/readthedocs/vcs_support/backends/__init__.py b/readthedocs/vcs_support/backends/__init__.py index 0a022a545d1..ff88fa587ec 100644 --- a/readthedocs/vcs_support/backends/__init__.py +++ b/readthedocs/vcs_support/backends/__init__.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Listing of all the VCS backends.""" from __future__ import absolute_import from . import bzr, hg, git, svn diff --git a/readthedocs/vcs_support/backends/bzr.py b/readthedocs/vcs_support/backends/bzr.py index e228ac720d3..5ea817c1156 100644 --- a/readthedocs/vcs_support/backends/bzr.py +++ b/readthedocs/vcs_support/backends/bzr.py @@ -1,10 +1,18 @@ # -*- coding: utf-8 -*- - """Bazaar-related utilities.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import csv import re -from io import StringIO + +from builtins import str # pylint: disable=redefined-builtin +from six import StringIO from readthedocs.projects.exceptions import RepositoryError from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -18,7 +26,7 @@ class Backend(BaseVCS): fallback_branch = '' def update(self): - super().update() + super(Backend, self).update() retcode = self.run('bzr', 'status', record=False)[0] if retcode == 0: return self.up() @@ -82,12 +90,7 @@ def commit(self): return stdout.strip() def checkout(self, identifier=None): - super().checkout() + super(Backend, self).checkout() if not identifier: return self.up() - exit_code, stdout, stderr = self.run('bzr', 'switch', identifier) - if exit_code != 0: - raise RepositoryError( - RepositoryError.FAILED_TO_CHECKOUT.format(identifier), - ) - return exit_code, stdout, stderr + return self.run('bzr', 'switch', identifier) diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index aa3adc6c2bc..f9608799570 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -1,12 +1,19 @@ # -*- coding: utf-8 -*- - """Git-related utilities.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import logging import os import re import git +from builtins import str from django.core.exceptions import ValidationError from git.exc import BadName, InvalidGitRepositoryError @@ -15,7 +22,6 @@ from readthedocs.projects.validators import validate_submodule_url from readthedocs.vcs_support.base import BaseVCS, VCSVersion - log = logging.getLogger(__name__) @@ -30,7 +36,7 @@ class Backend(BaseVCS): repo_depth = 50 def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(Backend, self).__init__(*args, **kwargs) self.token = kwargs.get('token', None) self.repo_url = self._get_clone_url() @@ -40,7 +46,7 @@ def _get_clone_url(self): hacked_url = re.sub('.git$', '', hacked_url) clone_url = 'https://%s' % hacked_url if self.token: - clone_url = 'https://{}@{}'.format(self.token, hacked_url) + clone_url = 'https://%s@%s' % (self.token, hacked_url) return clone_url # Don't edit URL because all hosts aren't the same # else: @@ -52,7 +58,7 @@ def set_remote_url(self, url): def update(self): """Clone or update the repository.""" - super().update() + super(Backend, self).update() if self.repo_exists(): self.set_remote_url(self.repo_url) return self.fetch() @@ -71,12 +77,11 @@ def are_submodules_available(self, config): # TODO remove this after users migrate to a config file from readthedocs.projects.models import Feature submodules_in_config = ( - config.submodules.exclude != ALL or config.submodules.include + config.submodules.exclude != ALL or + config.submodules.include ) - if ( - self.project.has_feature(Feature.SKIP_SUBMODULES) or - not submodules_in_config - ): + if (self.project.has_feature(Feature.SKIP_SUBMODULES) or + not submodules_in_config): return False # Keep compatibility with previous projects @@ -105,7 +110,10 @@ def validate_submodules(self, config): Returns the list of invalid submodules. """ repo = git.Repo(self.working_dir) - submodules = {sub.path: sub for sub in repo.submodules} + submodules = { + sub.path: sub + for sub in repo.submodules + } for sub_path in config.submodules.exclude: path = sub_path.rstrip('/') @@ -161,9 +169,7 @@ def checkout_revision(self, revision=None): code, out, err = self.run('git', 'checkout', '--force', revision) if code != 0: - raise RepositoryError( - RepositoryError.FAILED_TO_CHECKOUT.format(revision), - ) + log.warning("Failed to checkout revision '%s': %s", revision, code) return [code, out, err] def clone(self): @@ -187,7 +193,7 @@ def tags(self): for tag in repo.tags: try: versions.append(VCSVersion(self, str(tag.commit), str(tag))) - except ValueError: + except ValueError as e: # ValueError: Cannot resolve commit as tag TAGNAME points to a # blob object - use the `.object` property instead to access it # This is not a real tag for us, so we skip it @@ -217,14 +223,12 @@ def branches(self): @property def commit(self): - if self.repo_exists(): - _, stdout, _ = self.run('git', 'rev-parse', 'HEAD') - return stdout.strip() - return None + _, stdout, _ = self.run('git', 'rev-parse', 'HEAD') + return stdout.strip() def checkout(self, identifier=None): """Checkout to identifier or latest.""" - super().checkout() + super(Backend, self).checkout() # Find proper identifier if not identifier: identifier = self.default_branch or self.fallback_branch @@ -247,7 +251,7 @@ def update_submodules(self, config): self.checkout_submodules(submodules, config) else: raise RepositoryError( - RepositoryError.INVALID_SUBMODULES.format(submodules), + RepositoryError.INVALID_SUBMODULES.format(submodules) ) def checkout_submodules(self, submodules, config): @@ -287,7 +291,7 @@ def ref_exists(self, ref): @property def env(self): - env = super().env + env = super(Backend, self).env env['GIT_DIR'] = os.path.join(self.working_dir, '.git') # Don't prompt for username, this requires Git 2.3+ env['GIT_TERMINAL_PROMPT'] = '0' diff --git a/readthedocs/vcs_support/backends/hg.py b/readthedocs/vcs_support/backends/hg.py index 0361bfa462c..4a66624f1ae 100644 --- a/readthedocs/vcs_support/backends/hg.py +++ b/readthedocs/vcs_support/backends/hg.py @@ -1,6 +1,12 @@ # -*- coding: utf-8 -*- - """Mercurial-related utilities.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + from readthedocs.projects.exceptions import RepositoryError from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -14,7 +20,7 @@ class Backend(BaseVCS): fallback_branch = 'default' def update(self): - super().update() + super(Backend, self).update() retcode = self.run('hg', 'status', record=False)[0] if retcode == 0: return self.pull() @@ -39,11 +45,7 @@ def clone(self): @property def branches(self): retcode, stdout = self.run( - 'hg', - 'branches', - '--quiet', - record_as_success=True, - )[:2] + 'hg', 'branches', '--quiet', record_as_success=True)[:2] # error (or no tags found) if retcode != 0: return [] @@ -105,17 +107,7 @@ def commit(self): return stdout.strip() def checkout(self, identifier=None): - super().checkout() + super(Backend, self).checkout() if not identifier: identifier = 'tip' - exit_code, stdout, stderr = self.run( - 'hg', - 'update', - '--clean', - identifier, - ) - if exit_code != 0: - raise RepositoryError( - RepositoryError.FAILED_TO_CHECKOUT.format(identifier), - ) - return exit_code, stdout, stderr + return self.run('hg', 'update', '--clean', identifier) diff --git a/readthedocs/vcs_support/backends/svn.py b/readthedocs/vcs_support/backends/svn.py index b1a945aec2c..61e29324281 100644 --- a/readthedocs/vcs_support/backends/svn.py +++ b/readthedocs/vcs_support/backends/svn.py @@ -1,9 +1,17 @@ # -*- coding: utf-8 -*- - """Subversion-related utilities.""" +from __future__ import ( + absolute_import, + division, + print_function, + unicode_literals, +) + import csv -from io import StringIO + +from builtins import str +from six import StringIO # noqa from readthedocs.projects.exceptions import RepositoryError from readthedocs.vcs_support.base import BaseVCS, VCSVersion @@ -17,7 +25,7 @@ class Backend(BaseVCS): fallback_branch = '/trunk/' def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) + super(Backend, self).__init__(*args, **kwargs) if self.repo_url[-1] != '/': self.base_url = self.repo_url self.repo_url += '/' @@ -28,7 +36,7 @@ def __init__(self, *args, **kwargs): self.base_url = self.repo_url def update(self): - super().update() + super(Backend, self).update() # For some reason `svn status` gives me retcode 0 in non-svn # directories that's why I use `svn info` here. retcode, _, _ = self.run('svn', 'info', record=False) @@ -41,13 +49,8 @@ def up(self): if retcode != 0: raise RepositoryError retcode, out, err = self.run( - 'svn', - 'up', - '--accept', - 'theirs-full', - '--trust-server-cert', - '--non-interactive', - ) + 'svn', 'up', '--accept', 'theirs-full', + '--trust-server-cert', '--non-interactive') if retcode != 0: raise RepositoryError return retcode, out, err @@ -65,12 +68,8 @@ def co(self, identifier=None): @property def tags(self): - retcode, stdout = self.run( - 'svn', - 'list', - '%s/tags/' % self.base_url, - record_as_success=True, - )[:2] + retcode, stdout = self.run('svn', 'list', '%s/tags/' + % self.base_url, record_as_success=True)[:2] # error (or no tags found) if retcode != 0: return [] @@ -80,12 +79,12 @@ def parse_tags(self, data): """ Parses output of svn list, eg: - release-1.1/ - release-1.2/ - release-1.3/ - release-1.4/ - release-1.4.1/ - release-1.5/ + release-1.1/ + release-1.2/ + release-1.3/ + release-1.4/ + release-1.4.1/ + release-1.5/ """ # parse the lines into a list of tuples (commit-hash, tag ref name) # StringIO below is expecting Unicode data, so ensure that it gets it. @@ -103,7 +102,7 @@ def commit(self): return stdout.strip() def checkout(self, identifier=None): - super().checkout() + super(Backend, self).checkout() return self.co(identifier) def get_url(self, base_url, identifier): diff --git a/readthedocs/vcs_support/base.py b/readthedocs/vcs_support/base.py index 653629110fc..0e632e3b858 100644 --- a/readthedocs/vcs_support/base.py +++ b/readthedocs/vcs_support/base.py @@ -1,15 +1,18 @@ # -*- coding: utf-8 -*- - """Base classes for VCS backends.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + import logging import os import shutil +from builtins import object log = logging.getLogger(__name__) -class VCSVersion: +class VCSVersion(object): """ Represents a Version (tag or branch) in a VCS. @@ -26,13 +29,11 @@ def __init__(self, repository, identifier, verbose_name): self.verbose_name = verbose_name def __repr__(self): - return ' self.timeout: - log.info( - 'Lock (%s): Force unlock, old lockfile', - self.name, - ) + log.info("Lock (%s): Force unlock, old lockfile", + self.name) os.remove(self.fpath) break - log.info('Lock (%s): Locked, waiting..', self.name) + log.info("Lock (%s): Locked, waiting..", self.name) time.sleep(self.polling_interval) timesince = time.time() - start if timesince > self.timeout: - log.info( - 'Lock (%s): Force unlock, timeout reached', - self.name, - ) + log.info("Lock (%s): Force unlock, timeout reached", + self.name) os.remove(self.fpath) break - log.info( - '%s still locked after %.2f seconds; retry for %.2f' - ' seconds', - self.name, - timesince, - self.timeout, - ) + log.info("%s still locked after %.2f seconds; retry for %.2f" + " seconds", self.name, timesince, self.timeout) open(self.fpath, 'w').close() - log.info('Lock (%s): Lock acquired', self.name) + log.info("Lock (%s): Lock acquired", self.name) def __exit__(self, exc, value, tb): try: - log.info('Lock (%s): Releasing', self.name) + log.info("Lock (%s): Releasing", self.name) os.remove(self.fpath) except OSError as e: # We want to ignore "No such file or directory" and log any other # type of error. if e.errno != errno.ENOENT: log.exception( - 'Lock (%s): Failed to release, ignoring...', + "Lock (%s): Failed to release, ignoring...", self.name, ) -class NonBlockingLock: +class NonBlockingLock(object): """ Acquire a lock in a non-blocking manner. @@ -95,10 +84,7 @@ class NonBlockingLock: """ def __init__(self, project, version, max_lock_age=None): - self.fpath = os.path.join( - project.doc_path, - '%s__rtdlock' % version.slug, - ) + self.fpath = os.path.join(project.doc_path, '%s__rtdlock' % version.slug) self.max_lock_age = max_lock_age self.name = project.slug @@ -107,23 +93,21 @@ def __enter__(self): if path_exists and self.max_lock_age is not None: lock_age = time.time() - os.stat(self.fpath)[stat.ST_MTIME] if lock_age > self.max_lock_age: - log.info( - 'Lock (%s): Force unlock, old lockfile', - self.name, - ) + log.info("Lock (%s): Force unlock, old lockfile", + self.name) os.remove(self.fpath) else: raise LockTimeout( - 'Lock ({}): Lock still active'.format(self.name), - ) + "Lock ({}): Lock still active".format(self.name)) elif path_exists: - raise LockTimeout('Lock ({}): Lock still active'.format(self.name),) + raise LockTimeout( + "Lock ({}): Lock still active".format(self.name)) open(self.fpath, 'w').close() return self def __exit__(self, exc_type, exc_val, exc_tb): try: - log.info('Lock (%s): Releasing', self.name) + log.info("Lock (%s): Releasing", self.name) os.remove(self.fpath) except (IOError, OSError) as e: # We want to ignore "No such file or directory" and log any other diff --git a/readthedocs/worker.py b/readthedocs/worker.py index d038e0062bc..21023e9cdff 100644 --- a/readthedocs/worker.py +++ b/readthedocs/worker.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- +"""Celery worker application instantiation""" -"""Celery worker application instantiation.""" +from __future__ import absolute_import, unicode_literals import os diff --git a/readthedocs/wsgi.py b/readthedocs/wsgi.py index 997fb628506..e1cd6a8ec80 100644 --- a/readthedocs/wsgi.py +++ b/readthedocs/wsgi.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- - -"""WSGI application helper.""" +"""WSGI application helper""" from __future__ import absolute_import import os -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'readthedocs.settings.dev') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "readthedocs.settings.dev") # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION diff --git a/requirements/local-docs-build.txt b/requirements/local-docs-build.txt deleted file mode 100644 index c9ce6ed62a6..00000000000 --- a/requirements/local-docs-build.txt +++ /dev/null @@ -1,24 +0,0 @@ --r pip.txt - -# Base packages -docutils==0.14 -Sphinx==1.8.3 -sphinx_rtd_theme==0.4.2 -sphinx-tabs==1.1.10 -# Required to avoid Transifex error with reserved slug -# https://github.com/sphinx-doc/sphinx-intl/pull/27 -git+https://github.com/agjohnson/sphinx-intl.git@7b5c66bdb30f872b3b1286e371f569c8dcb66de5#egg=sphinx-intl - -Pygments==2.3.1 - -mkdocs==1.0.4 -Markdown==3.0.1 - -# Docs -sphinxcontrib-httpdomain==1.7.0 -sphinx-prompt==1.0.0 - -# commonmark 0.5.5 is the latest version compatible with our docs, the -# newer ones make `tox -e docs` to fail -commonmark==0.5.5 -recommonmark==0.4.0 diff --git a/requirements/onebox.txt b/requirements/onebox.txt new file mode 100644 index 00000000000..b18cb758833 --- /dev/null +++ b/requirements/onebox.txt @@ -0,0 +1,7 @@ +-r pip.txt +gunicorn +#For resizing images +pillow +python-memcached +whoosh +django-redis diff --git a/requirements/pip.txt b/requirements/pip.txt index 61782c00543..ac542b1415d 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -2,8 +2,22 @@ pip==18.1 appdirs==1.4.3 virtualenv==16.2.0 +docutils==0.14 +Sphinx==1.8.3 +sphinx_rtd_theme==0.4.2 +sphinx-tabs==1.1.10 +# Required to avoid Transifex error with reserved slug +# https://github.com/sphinx-doc/sphinx-intl/pull/27 +git+https://github.com/agjohnson/sphinx-intl.git@7b5c66bdb30f872b3b1286e371f569c8dcb66de5#egg=sphinx-intl + +Pygments==2.3.1 + +mkdocs==1.0.4 +Markdown==3.0.1 django==1.11.18 +six==1.12.0 +future==0.17.1 django-tastypie==0.14.2 django-guardian==1.4.9 django-extensions==2.1.4 @@ -27,8 +41,6 @@ requests-toolbelt==0.8.0 slumber==0.7.1 lxml==4.2.5 defusedxml==0.5.0 -pyyaml==3.13 -Pygments==2.3.1 # Basic tools # Redis 3.x has an incompatible change and fails @@ -84,6 +96,15 @@ djangorestframework-jsonp==1.0.2 django-taggit==0.23.0 dj-pagination==2.4.0 +# Docs +sphinxcontrib-httpdomain==1.7.0 + +# commonmark 0.5.5 is the latest version compatible with our docs, the +# newer ones make `tox -e docs` to fail +commonmark==0.5.5 + +recommonmark==0.4.0 + # Version comparison stuff packaging==18.0 diff --git a/requirements/testing.txt b/requirements/testing.txt index 8c9cd8d55e2..77737f87f59 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,5 +1,4 @@ -r pip.txt --r local-docs-build.txt django-dynamic-fixture==2.0.0 pytest==4.0.2 diff --git a/scripts/travis/run_tests.sh b/scripts/travis/run_tests.sh index c5da9d551f7..99c56387033 100755 --- a/scripts/travis/run_tests.sh +++ b/scripts/travis/run_tests.sh @@ -1,4 +1,4 @@ -if ! [[ "$TOXENV" =~ ^(docs|lint|eslint|migrations) ]]; +if ! [[ "$TOXENV" =~ ^(docs|lint|eslint) ]]; then args="'--including-search'" fi diff --git a/setup.cfg b/setup.cfg index 94f19cf1280..9581d8286aa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = readthedocs -version = 2.8.5 +version = 2.8.4 license = MIT description = Read the Docs builds and hosts documentation author = Read the Docs, Inc diff --git a/tasks.py b/tasks.py index c3d52e586c3..02a36534645 100644 --- a/tasks.py +++ b/tasks.py @@ -1,4 +1,6 @@ -"""Read the Docs tasks.""" +""" +Read the Docs tasks +""" from __future__ import division, print_function, unicode_literals @@ -12,28 +14,15 @@ ROOT_PATH = os.path.dirname(__file__) -namespace = Collection() -namespace.add_collection( - Collection( - common.tasks.prepare, - common.tasks.release, - ), - name='deploy', -) +# TODO make these tasks namespaced +# release = Collection(common.tasks.prepare, common.tasks.release) -namespace.add_collection( - Collection( - common.tasks.setup_labels, - ), - name='github', +namespace = Collection( + common.tasks.prepare, + common.tasks.release, + #release=release, ) -namespace.add_collection( - Collection( - common.tasks.upgrade_all_packages, - ), - name='packages', -) # Localization tasks @task diff --git a/tox.ini b/tox.ini index ffac6f78c68..4f96fee1936 100644 --- a/tox.ini +++ b/tox.ini @@ -1,10 +1,11 @@ [tox] minversion=2.9.0 -envlist = py36,lint,docs +envlist = py{27,36},lint,docs skipsdist = True [travis] python = + 2.7: py27 3.6: py36, codecov [testenv]

      • Table headerTable header 2