fixing up tests

Author: safwanrahman

readthedocs/projects/models.py

@@ -13,6 +13,7 @@
 from django.contrib.auth.models import User
 from django.core.urlresolvers import NoReverseMatch, reverse
 from django.db import models
+from django.utils.lru_cache import lru_cache
 from django.utils.encoding import python_2_unicode_compatible
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
@@ -939,11 +940,14 @@ def json_file_path(self):
         file_path = os.path.join(full_json_path, file_path)
         return file_path
 
-    @cached_property
-    def processed_json(self):
+    def get_processed_json(self):
         file_path = self.json_file_path
         return process_file(file_path)
 
+    @cached_property
+    def processed_json(self):
+        return self.get_processed_json()
+
 
 class Notification(models.Model):
     project = models.ForeignKey(Project,

readthedocs/search/documents.py

@@ -1,17 +1,18 @@
-from django.db import models
+from django.conf import settings
 from django_elasticsearch_dsl import DocType, Index, fields
 
 from readthedocs.projects.models import Project, HTMLFile
 from .conf import SEARCH_EXCLUDED_FILE
 
 from readthedocs.search.faceted_search import ProjectSearch, FileSearch
 
-project_index = Index('project')
+project_conf = settings.ES_INDEXES['project']
+project_index = Index(project_conf['name'])
+project_index.settings(**project_conf['settings'])
 
-project_index.settings(
-    number_of_shards=1,
-    number_of_replicas=0
-)
+page_conf = settings.ES_INDEXES['page']
+page_index = Index(page_conf['name'])
+page_index.settings(**page_conf['settings'])
 
 
 @project_index.doc_type
@@ -44,14 +45,6 @@ def faceted_search(cls, query, language=None, using=None, index=None):
         return ProjectSearch(**kwargs)
 
 
-page_index = Index('page')
-
-page_index.settings(
-    number_of_shards=1,
-    number_of_replicas=0
-)
-
-
 @page_index.doc_type
 class PageDocument(DocType):
 

readthedocs/search/tests/conftest.py

@@ -1,20 +1,13 @@
-import random
-import string
+import json
+import os
 from random import shuffle
 
 import pytest
 from django.core.management import call_command
 from django_dynamic_fixture import G
 
-from readthedocs.projects.models import Project
-from readthedocs.search.indexes import Index, ProjectIndex, PageIndex, SectionIndex
-from .dummy_data import DUMMY_PAGE_JSON, ALL_PROJECTS
-
-
-@pytest.fixture(autouse=True)
-def mock_elastic_index(mocker):
-    index_name = ''.join([random.choice(string.ascii_letters) for _ in range(5)])
-    mocker.patch.object(Index, '_index', index_name.lower())
+from readthedocs.projects.models import Project, HTMLFile
+from .dummy_data import ALL_PROJECTS, PROJECT_DATA_FILES
 
 
 @pytest.fixture()
@@ -26,11 +19,23 @@ def es_index():
     call_command('search_index', '--delete', '-f')
 
 
-@pytest.fixture
-def all_projects(es_index):
-    projects = [G(Project, slug=project_slug, name=project_slug) for project_slug in ALL_PROJECTS]
-    shuffle(projects)
-    return projects
+@pytest.fixture(autouse=True)
+def all_projects(es_index, mock_processed_json):
+    projects_list = []
+    for project_slug in ALL_PROJECTS:
+        project = G(Project, slug=project_slug, name=project_slug)
+
+        for file_basename in PROJECT_DATA_FILES[project.slug]:
+            # file_basename in config are without extension so add html extension
+            file_name = file_basename + '.html'
+            version = project.versions.all()[0]
+            f = G(HTMLFile, project=project, version=version, name=file_name)
+            f.save()
+
+        projects_list.append(project)
+
+    shuffle(projects_list)
+    return projects_list
 
 
 @pytest.fixture
@@ -39,16 +44,23 @@ def project(all_projects):
     return all_projects[0]
 
 
-def get_dummy_page_json(version, *args, **kwargs):
-    dummy_page_json = DUMMY_PAGE_JSON
-    project_name = version.project.name
-    return dummy_page_json.get(project_name)
+def get_dummy_processed_json(instance):
+    project_slug = instance.project.slug
+    basename = os.path.splitext(instance.name)[0]
+    file_name = basename + '.json'
+    current_path = os.path.abspath(os.path.dirname(__file__))
+    file_path = os.path.join(current_path, "data", project_slug, file_name)
+
+    if os.path.exists(file_path):
+        with open(file_path) as f:
+            return json.load(f)
 
 
 @pytest.fixture(autouse=True)
-def mock_parse_json(mocker):
+def mock_processed_json(mocker):
 
     # patch the function from `projects.tasks` because it has been point to there
     # http://www.voidspace.org.uk/python/mock/patch.html#where-to-patch
-    mocked_function = mocker.patch('readthedocs.projects.tasks.process_all_json_files')
-    mocked_function.side_effect = get_dummy_page_json
+    mocked_function = mocker.patch.object(HTMLFile, 'get_processed_json', autospec=True)
+    mocked_function.side_effect = get_dummy_processed_json
+

readthedocs/search/tests/data/docs/story.json

@@ -1,5 +1,5 @@
 {
-  "content": "Philosophy\nRead the Docs is Open Source software. We have licensed the code base as MIT, which provides almost no restrictions on the use of the code.\nHowever, as a project there are things that we care about more than others. We built Read the Docs to support  in the Open Source community. The code is open for people to contribute to, so that they may build features into https://readthedocs.org that they want. We also believe sharing the code openly is a valuable learning tool, especially for demonsrating how to collaborate and maintain an enormous website.\nOfficial Support\nThe time of the core developers of Read the Docs is limited. We provide official support for the following things:\nLocal development on the Python code base\nUsage of https://readthedocs.org for Open Source projects\nBug fixes in the code base, as it applies to running it on https://readthedocs.org\nUnsupported\nThere are use cases that we don\u2019t support, because it doesn\u2019t further our goal of promoting  in the Open Source Community.\nWe do not support:\nSpecific usage of Sphinx and Mkdocs, that don\u2019t affect our hosting\nCustom s of Read the Docs at your company\n of Read the Docs on other platforms\nAny  issues outside of the Read the Docs Python Code\nRationale\nRead the Docs was founded to improve  in the Open Source Community. We fully recognize and allow the code to be used for internal installs at companies, but we will not spend our time supporting it. Our time is limited, and we want to spend it on the mission that we set out to originally support.\nIf you feel strongly about installing Read the Docs internal to a company, we will happily link to third party resources on this topic. Please open an issue with a proposal if you want to take on this task.",
+  "content": "ReadtheDocsPhilosophy\nRead the Docs is Open Source software. We have licensed the code base as MIT, which provides almost no restrictions on the use of the code.\nHowever, as a project there are things that we care about more than others. We built Read the Docs to support  in the Open Source community. The code is open for people to contribute to, so that they may build features into https://readthedocs.org that they want. We also believe sharing the code openly is a valuable learning tool, especially for demonsrating how to collaborate and maintain an enormous website.\nOfficial Support\nThe time of the core developers of Read the Docs is limited. We provide official support for the following things:\nLocal development on the Python code base\nUsage of https://readthedocs.org for Open Source projects\nBug fixes in the code base, as it applies to running it on https://readthedocs.org\nUnsupported\nThere are use cases that we don\u2019t support, because it doesn\u2019t further our goal of promoting  in the Open Source Community.\nWe do not support:\nSpecific usage of Sphinx and Mkdocs, that don\u2019t affect our hosting\nCustom s of Read the Docs at your company\n of Read the Docs on other platforms\nAny  issues outside of the Read the Docs Python Code\nRationale\nRead the Docs was founded to improve  in the Open Source Community. We fully recognize and allow the code to be used for internal installs at companies, but we will not spend our time supporting it. Our time is limited, and we want to spend it on the mission that we set out to originally support.\nIf you feel strongly about installing Read the Docs internal to a company, we will happily link to third party resources on this topic. Please open an issue with a proposal if you want to take on this task.",
   "headers": [
     "Official Support",
     "Unsupported",

readthedocs/search/tests/data/docs/wiping.json

@@ -1,5 +1,5 @@
 {
-  "content": "Wiping a Build Environment\nSometimes it happen that your Builds start failing because the build environment where the  is created is stale or broken. This could happen for a couple of different reasons like pip not upgrading a package properly or a corrupted cached Python package.\nIn any of these cases (and many others), the solution could be just wiping out the existing build environment files and allow Read the Docs to create a new fresh one.\nFollow these steps to wipe the build environment:\nGo to Versions\nClick on the Edit button of the version you want to wipe on the right side of the page\nGo to the bottom of the page and click the wipe link, next to the \u201cSave\u201d button\nNote\nBy wiping the  build environment, all the rst, md, and code files associated with it will be removed but not the  already built (HTML and PDF files). Your  will still online after wiping the build environment.\nNow you can re-build the version with a fresh build environment!",
+  "content": "ReadtheDocsWiping a Build Environment\nSometimes it happen that your Builds start failing because the build environment where the  is created is stale or broken. This could happen for a couple of different reasons like pip not upgrading a package properly or a corrupted cached Python package.\nIn any of these cases (and many others), the solution could be just wiping out the existing build environment files and allow Read the Docs to create a new fresh one.\nFollow these steps to wipe the build environment:\nGo to Versions\nClick on the Edit button of the version you want to wipe on the right side of the page\nGo to the bottom of the page and click the wipe link, next to the \u201cSave\u201d button\nNote\nBy wiping the  build environment, all the rst, md, and code files associated with it will be removed but not the  already built (HTML and PDF files). Your  will still online after wiping the build environment.\nNow you can re-build the version with a fresh build environment!",
   "headers": [
     "Wiping a Build Environment"
   ],

readthedocs/search/tests/data/kuma/docker.json

@@ -1,5 +1,5 @@
 {
-  "content": "kuma-Docker Docker is used for development and (soon) for deployment.\nDocker Images\nDocker images are used in development, usually with the local working files mounted in the images to set behaviour.\nImages are built by Jenkins, after tests pass, and are published to quay.io. We try to store the configuration in the environment, so that the published images can be used in deployments by setting environment variables to deployment-specific values.\nHere are some of the images used in the Kuma project:\nkuma\nThe kuma Docker image builds on the kuma_base image, installing a kuma branch and building the assets needed for running as a webservice. The environment can be customized for different deployments.\nThe image can be recreated locally with make build-kuma.\nThe image tagged latest is used by default for development. It can be created locally with make build-kuma VERSION=latest. The  latest image is created from the master branch in Jenkins and published to quay.io.\nkuma_base\nThe kuma_base Docker image contains the OS and libraries (C, Python, and Node.js) that support the kuma project. The kuma image extends this by installing the kuma source and building assets needed for production.\nThe image can be recreated locally with make build-base.\nThe image tagged latest is used by default for development. It can be created localled with make build-base VERSION=latest. The  latest image is created from the master branch in Jenkins and published to quay.io\nkumascript\nThe kumascript Docker image contains the kumascript rendering engine and support files. The environment can be customized for different deployments.\nThe image can be recreated locally with make build-kumascript.\nThe image tagged latest is used by default for development. It can be created locally with make build-kumascript KS_VERSION=latest. The  latest image is created from the master branch in Jenkins and published to quay.io.\nintegration-tests\nThe integration-tests Docker image contains browser-based integration tests that check the functionality of a running Kuma deployment.\nThe image can be recreated locally with docker build -f docker/images/integration-tests/ ., but this is only necessary for image development. Most developer will follow the Client-side testing  to develop and run these integration tests.\nThe image is built and used in Jenkins in the stage-integration-tests and prod-integration-tests pipelines, configured by scripts in the Jenkinsfiles folder. It is not published to quay.io.",
+  "content": "kumadocker Docker is used for development and (soon) for deployment.\nDocker Images\nDocker images are used in development, usually with the local working files mounted in the images to set behaviour.\nImages are built by Jenkins, after tests pass, and are published to quay.io. We try to store the configuration in the environment, so that the published images can be used in deployments by setting environment variables to deployment-specific values.\nHere are some of the images used in the Kuma project:\nkuma\nThe kuma Docker image builds on the kuma_base image, installing a kuma branch and building the assets needed for running as a webservice. The environment can be customized for different deployments.\nThe image can be recreated locally with make build-kuma.\nThe image tagged latest is used by default for development. It can be created locally with make build-kuma VERSION=latest. The  latest image is created from the master branch in Jenkins and published to quay.io.\nkuma_base\nThe kuma_base Docker image contains the OS and libraries (C, Python, and Node.js) that support the kuma project. The kuma image extends this by installing the kuma source and building assets needed for production.\nThe image can be recreated locally with make build-base.\nThe image tagged latest is used by default for development. It can be created localled with make build-base VERSION=latest. The  latest image is created from the master branch in Jenkins and published to quay.io\nkumascript\nThe kumascript Docker image contains the kumascript rendering engine and support files. The environment can be customized for different deployments.\nThe image can be recreated locally with make build-kumascript.\nThe image tagged latest is used by default for development. It can be created locally with make build-kumascript KS_VERSION=latest. The  latest image is created from the master branch in Jenkins and published to quay.io.\nintegration-tests\nThe integration-tests Docker image contains browser-based integration tests that check the functionality of a running Kuma deployment.\nThe image can be recreated locally with docker build -f docker/images/integration-tests/ ., but this is only necessary for image development. Most developer will follow the Client-side testing  to develop and run these integration tests.\nThe image is built and used in Jenkins in the stage-integration-tests and prod-integration-tests pipelines, configured by scripts in the Jenkinsfiles folder. It is not published to quay.io.",
   "headers": [
     "Docker",
     "Docker Images",

readthedocs/search/tests/data/kuma/documentation.json

@@ -1,5 +1,5 @@
 {
-  "content": "kuma-Documentation This documentation is generated and published at Read the Docs whenever the master branch is updated. GitHub can render our .rst documents as ReStructuredText, which is close enough to Sphinx for most code reviews, without features like links between documents.\nIt is occasionally necessary to generate the documentation locally. It is easiest to do this with a virtualenv on the host system, using only to regenerate the MDN Sphinx template. If you are not comfortable with that style of development, it can be done entirely in  using -compose.\nGenerating documentation\nSphinx uses a Makefile in the docs subfolder to build documentation in several formats. MDN only uses the HTML format, and the generated document index is at docs/_build/html/index.html.\nTo generate the documentation in a virtualenv on the host machine, first install the requirements:\npip install -r requirements/docs.txt\nThen switch to the docs folder to use the Makefile:\ncd docs make html python -m webbrowser file://${PWD}/_build/html/index.html\nTo generate the documentation with :\n-compose run --rm --user $(id -u) web sh -c \"\\ virtualenv /tmp/.venvs/docs && \\ . /tmp/.venvs/docs/bin/activate && \\ pip install -r /app/requirements/docs.txt && \\ cd /app/docs && \\ make html\" python -m webbrowser file://${PWD}/docs/_build/html/index.html\nA virtualenv is required, to avoid a pip bug when changing the version of a system-installed package.",
+  "content": "kumadocumentation This documentation is generated and published at Read the Docs whenever the master branch is updated. GitHub can render our .rst documents as ReStructuredText, which is close enough to Sphinx for most code reviews, without features like links between documents.\nIt is occasionally necessary to generate the documentation locally. It is easiest to do this with a virtualenv on the host system, using only to regenerate the MDN Sphinx template. If you are not comfortable with that style of development, it can be done entirely in  using -compose.\nGenerating documentation\nSphinx uses a Makefile in the docs subfolder to build documentation in several formats. MDN only uses the HTML format, and the generated document index is at docs/_build/html/index.html.\nTo generate the documentation in a virtualenv on the host machine, first install the requirements:\npip install -r requirements/docs.txt\nThen switch to the docs folder to use the Makefile:\ncd docs make html python -m webbrowser file://${PWD}/_build/html/index.html\nTo generate the documentation with :\n-compose run --rm --user $(id -u) web sh -c \"\\ virtualenv /tmp/.venvs/docs && \\ . /tmp/.venvs/docs/bin/activate && \\ pip install -r /app/requirements/docs.txt && \\ cd /app/docs && \\ make html\" python -m webbrowser file://${PWD}/docs/_build/html/index.html\nA virtualenv is required, to avoid a pip bug when changing the version of a system-installed package.",
   "headers": [
     "Documentation",
     "Generating documentation"