From 4fe48d172e8661bda3eefa159837111ac80d97f1 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Fri, 22 Apr 2016 12:58:49 -0700 Subject: [PATCH 01/43] Move urls & views into subdirectories --- readthedocs/core/urls/__init__.py | 102 +++ readthedocs/core/urls/single_version.py | 14 + readthedocs/core/urls/subdomain.py | 48 ++ readthedocs/core/views.py | 886 ------------------------ readthedocs/core/views/__init__.py | 327 +++++++++ readthedocs/core/views/hooks.py | 251 +++++++ readthedocs/core/views/serve.py | 252 +++++++ 7 files changed, 994 insertions(+), 886 deletions(-) create mode 100644 readthedocs/core/urls/__init__.py create mode 100644 readthedocs/core/urls/single_version.py create mode 100644 readthedocs/core/urls/subdomain.py delete mode 100644 readthedocs/core/views.py create mode 100644 readthedocs/core/views/__init__.py create mode 100644 readthedocs/core/views/hooks.py create mode 100644 readthedocs/core/views/serve.py diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py new file mode 100644 index 00000000000..d4ab8389e48 --- /dev/null +++ b/readthedocs/core/urls/__init__.py @@ -0,0 +1,102 @@ +from django.conf.urls import url, patterns + +from readthedocs.constants import pattern_opts +from readthedocs.builds.filters import VersionFilter +from readthedocs.projects.feeds import LatestProjectsFeed, NewProjectsFeed +from readthedocs.projects.filters import ProjectFilter + + +docs_urls = patterns( + '', + # For serving docs locally and when nginx isn't + url((r'^docs/(?P{project_slug})/(?P{lang_slug})/' + r'(?P{version_slug})/' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.serve_docs', + name='docs_detail'), + + # Redirect to default version, if only lang_slug is set. + url((r'^docs/(?P{project_slug})/' + r'(?P{lang_slug})/$'.format(**pattern_opts)), + 'readthedocs.core.views.redirect_lang_slug', + name='docs_detail'), + + # Redirect to default version, if only version_slug is set. + url((r'^docs/(?P{project_slug})/' + r'(?P{version_slug})/$'.format(**pattern_opts)), + 'readthedocs.core.views.redirect_version_slug', + name='docs_detail'), + + # Redirect to default version. + url(r'^docs/(?P{project_slug})/$'.format(**pattern_opts), + 'readthedocs.core.views.redirect_project_slug', + name='docs_detail'), + + # Handle /page/ redirects for explicit "latest" version goodness. + url((r'^docs/(?P{project_slug})/page/' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.redirect_page_with_filename', + name='docs_detail'), + + # Handle single version URLs + url((r'^docs/(?P{project_slug})/' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.serve_single_version_docs', + name='docs_detail'), + + # Handle fallbacks + url((r'^user_builds/(?P{project_slug})/rtd-builds/' + r'(?P{version_slug})/' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.server_helpful_404', + name='user_builds_fallback'), + url((r'^user_builds/(?P{project_slug})/translations/' + r'(?P{lang_slug})/(?P{version_slug})/' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.server_helpful_404', + name='user_builds_fallback_translations'), +) + + +core_urls = patterns( + '', + url(r'^github', 'readthedocs.core.views.github_build', name='github_build'), + url(r'^gitlab', 'readthedocs.core.views.gitlab_build', name='gitlab_build'), + url(r'^bitbucket', 'readthedocs.core.views.bitbucket_build', name='bitbucket_build'), + url((r'^build/' + r'(?P{project_slug})'.format(**pattern_opts)), + 'readthedocs.core.views.generic_build', + name='generic_build'), + url(r'^random/(?P{project_slug})'.format(**pattern_opts), + 'readthedocs.core.views.random_page', + name='random_page'), + url(r'^random/$', 'readthedocs.core.views.random_page', name='random_page'), + url(r'^500/$', 'readthedocs.core.views.divide_by_zero', name='divide_by_zero'), + url((r'^wipe/(?P{project_slug})/' + r'(?P{version_slug})/$'.format(**pattern_opts)), + 'readthedocs.core.views.wipe_version', + name='wipe_version'), +) + +deprecated_urls = patterns( + '', + url(r'^filter/version/$', + 'django_filters.views.object_filter', + {'filter_class': VersionFilter, 'template_name': 'filter.html'}, + name='filter_version'), + url(r'^filter/project/$', + 'django_filters.views.object_filter', + {'filter_class': ProjectFilter, 'template_name': 'filter.html'}, + name='filter_project'), + + url(r'^feeds/new/$', + NewProjectsFeed(), + name="new_feed"), + url(r'^feeds/latest/$', + LatestProjectsFeed(), + name="latest_feed"), + url((r'^mlt/(?P{project_slug})/' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.morelikethis', + name='morelikethis'), +) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py new file mode 100644 index 00000000000..857d455034b --- /dev/null +++ b/readthedocs/core/urls/single_version.py @@ -0,0 +1,14 @@ +from django.conf.urls import patterns, url + +urlpatterns = patterns( + '', # base view, flake8 complains if it is on the previous line. + # Handle /docs on RTD domain + url(r'^docs/(?P[-\w]+)/(?P.*)$', + 'readthedocs.core.views.serve_single_version_docs', + name='docs_detail'), + + # Handle subdomains + url(r'^(?P.*)$', + 'readthedocs.core.views.serve_single_version_docs', + name='docs_detail'), +) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py new file mode 100644 index 00000000000..6a1d3e2d66a --- /dev/null +++ b/readthedocs/core/urls/subdomain.py @@ -0,0 +1,48 @@ +from django.conf.urls import url, patterns + +from readthedocs.projects.constants import LANGUAGES_REGEX +from readthedocs.urls import urlpatterns as main_patterns + +handler500 = 'readthedocs.core.views.server_error' +handler404 = 'readthedocs.core.views.server_error_404' + +urlpatterns = patterns( + '', # base view, flake8 complains if it is on the previous line. + url((r'^projects/(?P[\w.-]+)/(?P%s)/' + r'(?P[\w.-]+)/(?P.*)$' % LANGUAGES_REGEX), + 'readthedocs.core.views.subproject_serve_docs', + name='subproject_docs_detail'), + + url(r'^projects/(?P[\w.-]+)', + 'readthedocs.core.views.subproject_serve_docs', + name='subproject_docs_detail'), + + url(r'^projects/$', + 'readthedocs.core.views.subproject_list', + name='subproject_docs_list'), + + url(r'^(?P%s)/(?P[\w.-]+)/(?P.*)$' % LANGUAGES_REGEX, + 'readthedocs.core.views.serve_docs', + name='docs_detail'), + + url(r'^(?P%s)/(?P.*)/$' % LANGUAGES_REGEX, + 'readthedocs.core.views.serve_docs', + {'filename': 'index.html'}, + name='docs_detail'), + + url(r'^page/(?P.*)$', + 'readthedocs.core.views.redirect_page_with_filename', + name='docs_detail'), + + url(r'^(?P%s)/$' % LANGUAGES_REGEX, + 'readthedocs.core.views.redirect_lang_slug', + name='lang_subdomain_handler'), + + url(r'^(?P.*)/$', + 'readthedocs.core.views.redirect_version_slug', + name='version_subdomain_handler'), + + url(r'^$', 'readthedocs.core.views.redirect_project_slug', name='homepage'), +) + +urlpatterns += main_patterns diff --git a/readthedocs/core/views.py b/readthedocs/core/views.py deleted file mode 100644 index 87b235b8f0c..00000000000 --- a/readthedocs/core/views.py +++ /dev/null @@ -1,886 +0,0 @@ -"""Core views, including the main homepage, post-commit build hook, -documentation and header rendering, and server errors. - -""" - -from django.contrib import admin, messages -from django.core.urlresolvers import reverse -from django.conf import settings -from django.http import HttpResponse, HttpResponseRedirect, Http404, HttpResponseNotFound -from django.shortcuts import render_to_response, get_object_or_404, redirect -from django.template import RequestContext -from django.views.decorators.csrf import csrf_exempt -from django.views.static import serve -from django.views.generic import TemplateView, FormView - -from haystack.query import EmptySearchQuerySet -from haystack.query import SearchQuerySet - -from readthedocs.builds.models import Build -from readthedocs.builds.models import Version -from readthedocs.core.forms import FacetedSearchForm, SendEmailForm -from readthedocs.core.utils import trigger_build, broadcast, send_email -from readthedocs.donate.mixins import DonateProgressMixin -from readthedocs.builds.constants import LATEST -from readthedocs.projects import constants -from readthedocs.projects.models import Project, ImportedFile, ProjectRelationship -from readthedocs.projects.tasks import remove_dir, update_imported_docs -from readthedocs.redirects.utils import get_redirect_response - -import json -import mimetypes -import os -import logging -import re - -log = logging.getLogger(__name__) -pc_log = logging.getLogger(__name__ + '.post_commit') - - -class NoProjectException(Exception): - pass - - -class HomepageView(DonateProgressMixin, TemplateView): - - template_name = 'homepage.html' - - def get_context_data(self, **kwargs): - '''Add latest builds and featured projects''' - context = super(HomepageView, self).get_context_data(**kwargs) - latest = [] - latest_builds = ( - Build.objects - .filter( - project__privacy_level=constants.PUBLIC, - success=True, - ) - .order_by('-date') - )[:100] - for build in latest_builds: - if (build.project not in latest and len(latest) < 10): - latest.append(build.project) - context['project_list'] = latest - context['featured_list'] = Project.objects.filter(featured=True) - return context - - -class SupportView(TemplateView): - template_name = 'support.html' - - def get_context_data(self, **kwargs): - context = super(SupportView, self).get_context_data(**kwargs) - support_email = getattr(settings, 'SUPPORT_EMAIL', None) - if not support_email: - support_email = 'support@{domain}'.format( - domain=getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org')) - - context['support_email'] = support_email - return context - - -def random_page(request, project_slug=None): - imported_file = ImportedFile.objects.order_by('?') - if project_slug: - imported_file = imported_file.filter(project__slug=project_slug) - imported_file = imported_file.first() - if imported_file is None: - raise Http404 - url = imported_file.get_absolute_url() - return HttpResponseRedirect(url) - - -def live_builds(request): - builds = Build.objects.filter(state='building')[:5] - websocket_host = getattr(settings, 'WEBSOCKET_HOST', 'localhost:8088') - count = builds.count() - percent = 100 - if count > 1: - percent = 100 / count - return render_to_response('all_builds.html', - {'builds': builds, - 'build_percent': percent, - 'websocket_host': websocket_host}, - context_instance=RequestContext(request)) - - -@csrf_exempt -def wipe_version(request, project_slug, version_slug): - version = get_object_or_404(Version, project__slug=project_slug, - slug=version_slug) - if request.user not in version.project.users.all(): - raise Http404("You must own this project to wipe it.") - - if request.method == 'POST': - del_dirs = [ - os.path.join(version.project.doc_path, 'checkouts', version.slug), - os.path.join(version.project.doc_path, 'envs', version.slug), - os.path.join(version.project.doc_path, 'conda', version.slug), - ] - for del_dir in del_dirs: - broadcast(type='build', task=remove_dir, args=[del_dir]) - return redirect('project_version_list', project_slug) - else: - return render_to_response('wipe_version.html', - context_instance=RequestContext(request)) - - -def _build_version(project, slug, already_built=()): - """ - Where we actually trigger builds for a project and slug. - - All webhook logic should route here to call ``trigger_build``. - """ - default = project.default_branch or (project.vcs_repo().fallback_branch) - if not project.has_valid_webhook: - project.has_valid_webhook = True - project.save() - if slug == default and slug not in already_built: - # short circuit versions that are default - # these will build at "latest", and thus won't be - # active - latest_version = project.versions.get(slug=LATEST) - trigger_build(project=project, version=latest_version, force=True) - pc_log.info(("(Version build) Building %s:%s" - % (project.slug, latest_version.slug))) - if project.versions.exclude(active=False).filter(slug=slug).exists(): - # Handle the case where we want to build the custom branch too - slug_version = project.versions.get(slug=slug) - trigger_build(project=project, version=slug_version, force=True) - pc_log.info(("(Version build) Building %s:%s" - % (project.slug, slug_version.slug))) - return LATEST - elif project.versions.exclude(active=True).filter(slug=slug).exists(): - pc_log.info(("(Version build) Not Building %s" % slug)) - return None - elif slug not in already_built: - version = project.versions.get(slug=slug) - trigger_build(project=project, version=version, force=True) - pc_log.info(("(Version build) Building %s:%s" - % (project.slug, version.slug))) - return slug - else: - pc_log.info(("(Version build) Not Building %s" % slug)) - return None - - -def _build_branches(project, branch_list): - """ - Build the branches for a specific project. - - Returns: - to_build - a list of branches that were built - not_building - a list of branches that we won't build - """ - for branch in branch_list: - versions = project.versions_from_branch_name(branch) - to_build = set() - not_building = set() - for version in versions: - pc_log.info(("(Branch Build) Processing %s:%s" - % (project.slug, version.slug))) - ret = _build_version(project, version.slug, already_built=to_build) - if ret: - to_build.add(ret) - else: - not_building.add(version.slug) - return (to_build, not_building) - - -def get_project_from_url(url): - projects = ( - Project.objects.filter(repo__iendswith=url) | - Project.objects.filter(repo__iendswith=url + '.git')) - return projects - - -def pc_log_info(project, msg): - pc_log.info(constants.LOG_TEMPLATE - .format(project=project, - version='', - msg=msg)) - - -def _build_url(url, projects, branches): - """ - Map a URL onto specific projects to build that are linked to that URL. - - Check each of the ``branches`` to see if they are active and should be built. - """ - ret = "" - all_built = {} - all_not_building = {} - for project in projects: - (built, not_building) = _build_branches(project, branches) - if not built: - # Call update_imported_docs to update tag/branch info - update_imported_docs.delay(project.versions.get(slug=LATEST).pk) - msg = '(URL Build) Syncing versions for %s' % project.slug - pc_log.info(msg) - all_built[project.slug] = built - all_not_building[project.slug] = not_building - - for project_slug, built in all_built.items(): - if built: - msg = '(URL Build) Build Started: %s [%s]' % ( - url, ' '.join(built)) - pc_log_info(project_slug, msg=msg) - ret += msg - - for project_slug, not_building in all_not_building.items(): - if not_building: - msg = '(URL Build) Not Building: %s [%s]' % ( - url, ' '.join(not_building)) - pc_log_info(project_slug, msg=msg) - ret += msg - - if not ret: - ret = '(URL Build) No known branches were pushed to.' - - return HttpResponse(ret) - - -@csrf_exempt -def github_build(request): - """ - A post-commit hook for github. - """ - if request.method == 'POST': - try: - # GitHub RTD integration - obj = json.loads(request.POST['payload']) - except: - # Generic post-commit hook - obj = json.loads(request.body) - repo_url = obj['repository']['url'] - hacked_repo_url = repo_url.replace('http://', '').replace('https://', '') - ssh_url = obj['repository']['ssh_url'] - hacked_ssh_url = ssh_url.replace('git@', '').replace('.git', '') - try: - branch = obj['ref'].replace('refs/heads/', '') - except KeyError: - response = HttpResponse('ref argument required to build branches.') - response.status_code = 400 - return response - - try: - repo_projects = get_project_from_url(hacked_repo_url) - if repo_projects: - pc_log.info("(Incoming GitHub Build) %s [%s]" % (hacked_repo_url, branch)) - ssh_projects = get_project_from_url(hacked_ssh_url) - if ssh_projects: - pc_log.info("(Incoming GitHub Build) %s [%s]" % (hacked_ssh_url, branch)) - projects = repo_projects | ssh_projects - return _build_url(hacked_repo_url, projects, [branch]) - except NoProjectException: - pc_log.error( - "(Incoming GitHub Build) Repo not found: %s" % hacked_repo_url) - return HttpResponseNotFound('Repo not found: %s' % hacked_repo_url) - else: - return HttpResponse("You must POST to this resource.") - - -@csrf_exempt -def gitlab_build(request): - """ - A post-commit hook for GitLab. - """ - if request.method == 'POST': - try: - # GitLab RTD integration - obj = json.loads(request.POST['payload']) - except: - # Generic post-commit hook - obj = json.loads(request.body) - url = obj['repository']['homepage'] - ghetto_url = url.replace('http://', '').replace('https://', '') - branch = obj['ref'].replace('refs/heads/', '') - pc_log.info("(Incoming GitLab Build) %s [%s]" % (ghetto_url, branch)) - projects = get_project_from_url(ghetto_url) - if projects: - return _build_url(ghetto_url, projects, [branch]) - else: - pc_log.error( - "(Incoming GitLab Build) Repo not found: %s" % ghetto_url) - return HttpResponseNotFound('Repo not found: %s' % ghetto_url) - else: - return HttpResponse("You must POST to this resource.") - - -@csrf_exempt -def bitbucket_build(request): - if request.method == 'POST': - payload = request.POST.get('payload') - pc_log.info("(Incoming Bitbucket Build) Raw: %s" % payload) - if not payload: - return HttpResponseNotFound('Invalid Request') - obj = json.loads(payload) - rep = obj['repository'] - branches = [rec.get('branch', '') for rec in obj['commits']] - ghetto_url = "%s%s" % ( - "bitbucket.org", rep['absolute_url'].rstrip('/')) - pc_log.info("(Incoming Bitbucket Build) %s [%s]" % ( - ghetto_url, ' '.join(branches))) - pc_log.info("(Incoming Bitbucket Build) JSON: \n\n%s\n\n" % obj) - projects = get_project_from_url(ghetto_url) - if projects: - return _build_url(ghetto_url, projects, branches) - else: - pc_log.error( - "(Incoming Bitbucket Build) Repo not found: %s" % ghetto_url) - return HttpResponseNotFound('Repo not found: %s' % ghetto_url) - else: - return HttpResponse("You must POST to this resource.") - - -@csrf_exempt -def generic_build(request, project_id_or_slug=None): - try: - project = Project.objects.get(pk=project_id_or_slug) - # Allow slugs too - except (Project.DoesNotExist, ValueError): - try: - project = Project.objects.get(slug=project_id_or_slug) - except (Project.DoesNotExist, ValueError): - pc_log.error( - "(Incoming Generic Build) Repo not found: %s" % ( - project_id_or_slug)) - return HttpResponseNotFound( - 'Repo not found: %s' % project_id_or_slug) - if request.method == 'POST': - slug = request.POST.get('version_slug', project.default_version) - pc_log.info( - "(Incoming Generic Build) %s [%s]" % (project.slug, slug)) - _build_version(project, slug) - else: - return HttpResponse("You must POST to this resource.") - return redirect('builds_project_list', project.slug) - - -def subproject_list(request): - project_slug = request.slug - proj = get_object_or_404(Project, slug=project_slug) - subprojects = [rel.child for rel in proj.subprojects.all()] - return render_to_response( - 'projects/project_list.html', - {'project_list': subprojects}, - context_instance=RequestContext(request) - ) - - -def subproject_serve_docs(request, project_slug, lang_slug=None, - version_slug=None, filename=''): - parent_slug = request.slug - proj = get_object_or_404(Project, slug=project_slug) - subproject_qs = ProjectRelationship.objects.filter( - parent__slug=parent_slug, child__slug=project_slug) - if lang_slug is None or version_slug is None: - # Handle / - version_slug = proj.get_default_version() - url = reverse('subproject_docs_detail', kwargs={ - 'project_slug': project_slug, - 'version_slug': version_slug, - 'lang_slug': proj.language, - 'filename': filename - }) - return HttpResponseRedirect(url) - - if subproject_qs.exists(): - return serve_docs(request, lang_slug, version_slug, filename, - project_slug) - else: - log.info('Subproject lookup failed: %s:%s' % (project_slug, - parent_slug)) - raise Http404("Subproject does not exist") - - -def default_docs_kwargs(request, project_slug=None): - """ - Return kwargs used to reverse lookup a project's default docs URL. - - Determining which URL to redirect to is done based on the kwargs - passed to reverse(serve_docs, kwargs). This function populates - kwargs for the default docs for a project, and sets appropriate keys - depending on whether request is for a subdomain URL, or a non-subdomain - URL. - - """ - if project_slug: - try: - proj = Project.objects.get(slug=project_slug) - except (Project.DoesNotExist, ValueError): - # Try with underscore, for legacy - try: - proj = Project.objects.get(slug=project_slug.replace('-', '_')) - except (Project.DoesNotExist): - proj = None - else: - # If project_slug isn't in URL pattern, it's set in subdomain - # middleware as request.slug. - try: - proj = Project.objects.get(slug=request.slug) - except (Project.DoesNotExist, ValueError): - # Try with underscore, for legacy - try: - proj = Project.objects.get(slug=request.slug.replace('-', '_')) - except (Project.DoesNotExist): - proj = None - if not proj: - raise Http404("Project slug not found") - version_slug = proj.get_default_version() - kwargs = { - 'project_slug': project_slug, - 'version_slug': version_slug, - 'lang_slug': proj.language, - 'filename': '' - } - # Don't include project_slug for subdomains. - # That's how reverse(serve_docs, ...) differentiates subdomain - # views from non-subdomain views. - if project_slug is None: - del kwargs['project_slug'] - return kwargs - - -def redirect_lang_slug(request, lang_slug, project_slug=None): - """Redirect /en/ to /en/latest/.""" - kwargs = default_docs_kwargs(request, project_slug) - kwargs['lang_slug'] = lang_slug - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def redirect_version_slug(request, version_slug, project_slug=None): - """Redirect /latest/ to /en/latest/.""" - kwargs = default_docs_kwargs(request, project_slug) - kwargs['version_slug'] = version_slug - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def redirect_project_slug(request, project_slug=None): - """Redirect / to /en/latest/.""" - kwargs = default_docs_kwargs(request, project_slug) - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def redirect_page_with_filename(request, filename, project_slug=None): - """Redirect /page/file.html to /en/latest/file.html.""" - kwargs = default_docs_kwargs(request, project_slug) - kwargs['filename'] = filename - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def serve_docs(request, lang_slug, version_slug, filename, project_slug=None): - if not project_slug: - project_slug = request.slug - try: - proj = Project.objects.protected(request.user).get(slug=project_slug) - ver = Version.objects.public(request.user).get( - project__slug=project_slug, slug=version_slug) - except (Project.DoesNotExist, Version.DoesNotExist): - proj = None - ver = None - if not proj or not ver: - return server_helpful_404(request, project_slug, lang_slug, version_slug, - filename) - - if ver not in proj.versions.public(request.user, proj, only_active=False): - r = render_to_response('401.html', - context_instance=RequestContext(request)) - r.status_code = 401 - return r - return _serve_docs(request, project=proj, version=ver, filename=filename, - lang_slug=lang_slug, version_slug=version_slug, - project_slug=project_slug) - - -def _serve_docs(request, project, version, filename, lang_slug=None, - version_slug=None, project_slug=None): - '''Actually serve the built documentation files - - This is not called directly, but is wrapped by :py:func:`serve_docs` so that - authentication can be manipulated. - ''' - # Figure out actual file to serve - if not filename: - filename = "index.html" - # This is required because we're forming the filenames outselves instead of - # letting the web server do it. - elif ( - (project.documentation_type == 'sphinx_htmldir' or - project.documentation_type == 'mkdocs') and - "_static" not in filename and - ".css" not in filename and - ".js" not in filename and - ".png" not in filename and - ".jpg" not in filename and - ".svg" not in filename and - "_images" not in filename and - ".html" not in filename and - "font" not in filename and - "inv" not in filename): - filename += "index.html" - else: - filename = filename.rstrip('/') - # Use the old paths if we're on our old location. - # Otherwise use the new language symlinks. - # This can be removed once we have 'en' symlinks for every project. - if lang_slug == project.language: - basepath = project.rtd_build_path(version_slug) - else: - basepath = project.translations_symlink_path(lang_slug) - basepath = os.path.join(basepath, version_slug) - - # Serve file - log.info('Serving %s for %s' % (filename, project)) - if not settings.DEBUG and not getattr(settings, 'PYTHON_MEDIA', False): - fullpath = os.path.join(basepath, filename) - content_type, encoding = mimetypes.guess_type(fullpath) - content_type = content_type or 'application/octet-stream' - response = HttpResponse(content_type=content_type) - if encoding: - response["Content-Encoding"] = encoding - try: - response['X-Accel-Redirect'] = os.path.join(basepath[len(settings.SITE_ROOT):], - filename) - except UnicodeEncodeError: - raise Http404 - - return response - else: - return serve(request, filename, basepath) - - -def serve_single_version_docs(request, filename, project_slug=None): - if not project_slug: - project_slug = request.slug - proj = get_object_or_404(Project, slug=project_slug) - - # This function only handles single version projects - if not proj.single_version: - raise Http404 - - return serve_docs(request, proj.language, proj.default_version, - filename, project_slug) - - -def server_error(request, template_name='500.html'): - """ - A simple 500 handler so we get media - """ - r = render_to_response(template_name, - context_instance=RequestContext(request)) - r.status_code = 500 - return r - - -def server_error_404(request, template_name='404.html'): - """ - A simple 404 handler so we get media - """ - response = get_redirect_response(request, path=request.get_full_path()) - if response: - return response - r = render_to_response(template_name, - context_instance=RequestContext(request)) - r.status_code = 404 - return r - - -def server_helpful_404( - request, project_slug=None, lang_slug=None, version_slug=None, - filename=None, template_name='404.html'): - response = get_redirect_response(request, path=filename) - if response: - return response - pagename = re.sub( - r'/index$', r'', re.sub(r'\.html$', r'', re.sub(r'/$', r'', filename))) - suggestion = get_suggestion( - project_slug, lang_slug, version_slug, pagename, request.user) - r = render_to_response(template_name, - {'suggestion': suggestion}, - context_instance=RequestContext(request)) - r.status_code = 404 - return r - - -def get_suggestion(project_slug, lang_slug, version_slug, pagename, user): - """ - | # | project | version | language | What to show | - | 1 | 0 | 0 | 0 | Error message | - | 2 | 0 | 0 | 1 | Error message (Can't happen) | - | 3 | 0 | 1 | 0 | Error message (Can't happen) | - | 4 | 0 | 1 | 1 | Error message (Can't happen) | - | 5 | 1 | 0 | 0 | A link to top-level page of default version | - | 6 | 1 | 0 | 1 | Available versions on the translation project | - | 7 | 1 | 1 | 0 | Available translations of requested version | - | 8 | 1 | 1 | 1 | A link to top-level page of requested version | - """ - - suggestion = {} - if project_slug: - try: - proj = Project.objects.get(slug=project_slug) - if not lang_slug: - lang_slug = proj.language - try: - ver = Version.objects.get( - project__slug=project_slug, slug=version_slug) - except Version.DoesNotExist: - ver = None - - if ver: # if requested version is available on main project - if lang_slug != proj.language: - try: - translations = proj.translations.filter( - language=lang_slug) - if translations: - ver = Version.objects.get( - project__slug=translations[0].slug, slug=version_slug) - else: - ver = None - except Version.DoesNotExist: - ver = None - # if requested version is available on translation project too - if ver: - # Case #8: Show a link to top-level page of the version - suggestion['type'] = 'top' - suggestion['message'] = "What are you looking for?" - suggestion['href'] = proj.get_docs_url(ver.slug, lang_slug) - # requested version is available but not in requested language - else: - # Case #7: Show available translations of the version - suggestion['type'] = 'list' - suggestion['message'] = ( - "Requested page seems not to be translated in " - "requested language. But it's available in these " - "languages.") - suggestion['list'] = [] - suggestion['list'].append({ - 'label': proj.language, - 'project': proj, - 'version_slug': version_slug, - 'pagename': pagename - }) - for t in proj.translations.all(): - try: - Version.objects.get( - project__slug=t.slug, slug=version_slug) - suggestion['list'].append({ - 'label': t.language, - 'project': t, - 'version_slug': version_slug, - 'pagename': pagename - }) - except Version.DoesNotExist: - pass - else: # requested version does not exist on main project - if lang_slug == proj.language: - trans = proj - else: - translations = proj.translations.filter(language=lang_slug) - trans = translations[0] if translations else None - if trans: # requested language is available - # Case #6: Show available versions of the translation - suggestion['type'] = 'list' - suggestion['message'] = ( - "Requested version seems not to have been built yet. " - "But these versions are available.") - suggestion['list'] = [] - for v in Version.objects.public(user, trans, True): - suggestion['list'].append({ - 'label': v.slug, - 'project': trans, - 'version_slug': v.slug, - 'pagename': pagename - }) - # requested project exists but requested version and language - # are not available. - else: - # Case #5: Show a link to top-level page of default version - # of main project - suggestion['type'] = 'top' - suggestion['message'] = 'What are you looking for??' - suggestion['href'] = proj.get_docs_url() - except Project.DoesNotExist: - # Case #1-4: Show error mssage - suggestion['type'] = 'none' - suggestion[ - 'message'] = "We're sorry, we don't know what you're looking for" - else: - suggestion['type'] = 'none' - suggestion[ - 'message'] = "We're sorry, we don't know what you're looking for" - - return suggestion - - -def divide_by_zero(request): - return 1 / 0 - - -def morelikethis(request, project_slug, filename): - project = get_object_or_404(Project, slug=project_slug) - file = get_object_or_404(ImportedFile, project=project, path=filename) - # sqs = SearchQuerySet().filter(project=project).more_like_this(file)[:5] - sqs = SearchQuerySet().more_like_this(file)[:5] - if len(sqs): - output = [(obj.title, obj.absolute_url) for obj in sqs] - json_response = json.dumps(output) - else: - json_response = {"message": "Not Found"} - jsonp = "%s(%s)" % (request.GET.get('callback'), json_response) - return HttpResponse(jsonp, content_type='text/javascript') - - -class SearchView(TemplateView): - - template_name = "search/base_facet.html" - results = EmptySearchQuerySet() - form_class = FacetedSearchForm - form = None - query = '' - selected_facets = None - selected_facets_list = None - - def get_context_data(self, request, **kwargs): - context = super(SearchView, self).get_context_data(**kwargs) - context['request'] = self.request - # causes solr request #1 - context['facets'] = self.results.facet_counts() - context['form'] = self.form - context['query'] = self.query - context['selected_facets'] = ('&'.join(self.selected_facets) - if self.selected_facets else '') - context['selected_facets_list'] = self.selected_facets_list - context['results'] = self.results - context['count'] = len(self.results) # causes solr request #2 - return context - - def get(self, request, **kwargs): - """ - Performing the search causes three requests to be sent to Solr. - 1. For the facets - 2. For the count (unavoidable, as pagination will cause this anyay) - 3. For the results - """ - self.request = request - self.form = self.build_form() - self.selected_facets = self.get_selected_facets() - self.selected_facets_list = self.get_selected_facets_list() - self.query = self.get_query() - if self.form.is_valid(): - self.results = self.get_results() - context = self.get_context_data(request, **kwargs) - - # For returning results partials for javascript - if request.is_ajax() or request.GET.get('ajax'): - self.template_name = 'search/faceted_results.html' - - return self.render_to_response(context) - - def build_form(self): - """ - Instantiates the form the class should use to process the search query. - """ - data = self.request.GET if len(self.request.GET) else None - return self.form_class(data, facets=('project',)) - - def get_selected_facets_list(self): - return [tuple(s.split(':')) for s in self.selected_facets if s] - - def get_selected_facets(self): - """ - Returns the a list of facetname:value strings - - e.g. [u'project_exact:Read The Docs', u'author_exact:Eric Holscher'] - """ - return self.request.GET.getlist('selected_facets') - - def get_query(self): - """ - Returns the query provided by the user. - Returns an empty string if the query is invalid. - """ - return self.request.GET.get('q') - - def get_results(self): - """ - Fetches the results via the form. - """ - return self.form.search() - - -class SendEmailView(FormView): - - """Form view for sending emails to users from admin pages - - Accepts the following additional parameters: - - queryset - The queryset to use to determine the users to send emails to - """ - - form_class = SendEmailForm - template_name = 'core/send_email_form.html' - - def get_form_kwargs(self): - """Override form kwargs based on input fields - - The admin posts to this view initially, so detect the send button on - form post variables. Drop additional fields if we see the send button. - """ - kwargs = super(SendEmailView, self).get_form_kwargs() - if 'send' not in self.request.POST: - kwargs.pop('data', None) - kwargs.pop('files', None) - return kwargs - - def get_initial(self): - """Add selected ids to initial form data""" - initial = super(SendEmailView, self).get_initial() - initial['_selected_action'] = self.request.POST.getlist( - admin.ACTION_CHECKBOX_NAME) - return initial - - def form_valid(self, form): - """If form is valid, send emails to selected users""" - count = 0 - for user in self.get_queryset().all(): - send_email( - user.email, - subject=form.cleaned_data['subject'], - template='core/email/common.txt', - template_html='core/email/common.html', - context={'user': user, 'content': form.cleaned_data['body']}, - request=self.request, - ) - count += 1 - if count == 0: - self.message_user("No receipients to send to", level=messages.ERROR) - else: - self.message_user("Queued {0} messages".format(count)) - return HttpResponseRedirect(self.request.get_full_path()) - - def get_queryset(self): - return self.kwargs.get('queryset') - - def get_context_data(self, **kwargs): - """Return queryset in context""" - context = super(SendEmailView, self).get_context_data(**kwargs) - context['users'] = self.get_queryset().all() - return context - - def message_user(self, message, level=messages.INFO, extra_tags='', - fail_silently=False): - """Implementation of :py:meth:`django.contrib.admin.options.ModelAdmin.message_user` - - Send message through messages framework - """ - # TODO generalize this or check if implementation in ModelAdmin is - # useable here - messages.add_message(self.request, level, message, extra_tags=extra_tags, - fail_silently=fail_silently) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py new file mode 100644 index 00000000000..4f4ea6a1eff --- /dev/null +++ b/readthedocs/core/views/__init__.py @@ -0,0 +1,327 @@ +""" +Core views, including the main homepage, +documentation and header rendering, and server errors. +""" + +import re +import os +import logging + + +from django.contrib import admin, messages +from django.conf import settings +from django.http import HttpResponseRedirect, Http404 +from django.shortcuts import render_to_response, get_object_or_404, redirect +from django.template import RequestContext +from django.views.decorators.csrf import csrf_exempt +from django.views.generic import TemplateView, FormView + +from readthedocs.builds.models import Build +from readthedocs.builds.models import Version +from readthedocs.core.utils import broadcast, send_email +from readthedocs.core.forms import SendEmailForm +from readthedocs.donate.mixins import DonateProgressMixin +from readthedocs.projects import constants +from readthedocs.projects.models import Project, ImportedFile +from readthedocs.projects.tasks import remove_dir +from readthedocs.redirects.utils import get_redirect_response + +log = logging.getLogger(__name__) +pc_log = logging.getLogger(__name__ + '.post_commit') + + +class NoProjectException(Exception): + pass + + +class HomepageView(DonateProgressMixin, TemplateView): + + template_name = 'homepage.html' + + def get_context_data(self, **kwargs): + '''Add latest builds and featured projects''' + context = super(HomepageView, self).get_context_data(**kwargs) + latest = [] + latest_builds = ( + Build.objects + .filter( + project__privacy_level=constants.PUBLIC, + success=True, + ) + .order_by('-date') + )[:100] + for build in latest_builds: + if (build.project not in latest and len(latest) < 10): + latest.append(build.project) + context['project_list'] = latest + context['featured_list'] = Project.objects.filter(featured=True) + return context + + +class SupportView(TemplateView): + template_name = 'support.html' + + def get_context_data(self, **kwargs): + context = super(SupportView, self).get_context_data(**kwargs) + support_email = getattr(settings, 'SUPPORT_EMAIL', None) + if not support_email: + support_email = 'support@{domain}'.format( + domain=getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org')) + + context['support_email'] = support_email + return context + + +def random_page(request, project_slug=None): + imported_file = ImportedFile.objects.order_by('?') + if project_slug: + imported_file = imported_file.filter(project__slug=project_slug) + imported_file = imported_file.first() + if imported_file is None: + raise Http404 + url = imported_file.get_absolute_url() + return HttpResponseRedirect(url) + + +@csrf_exempt +def wipe_version(request, project_slug, version_slug): + version = get_object_or_404(Version, project__slug=project_slug, + slug=version_slug) + if request.user not in version.project.users.all(): + raise Http404("You must own this project to wipe it.") + + if request.method == 'POST': + del_dirs = [ + os.path.join(version.project.doc_path, 'checkouts', version.slug), + os.path.join(version.project.doc_path, 'envs', version.slug), + os.path.join(version.project.doc_path, 'conda', version.slug), + ] + for del_dir in del_dirs: + broadcast(type='build', task=remove_dir, args=[del_dir]) + return redirect('project_version_list', project_slug) + else: + return render_to_response('wipe_version.html', + context_instance=RequestContext(request)) + + +def divide_by_zero(request): + return 1 / 0 + + +def server_error(request, template_name='500.html'): + """ + A simple 500 handler so we get media + """ + r = render_to_response(template_name, + context_instance=RequestContext(request)) + r.status_code = 500 + return r + + +def server_error_404(request, template_name='404.html'): + """ + A simple 404 handler so we get media + """ + response = get_redirect_response(request, path=request.get_full_path()) + if response: + return response + r = render_to_response(template_name, + context_instance=RequestContext(request)) + r.status_code = 404 + return r + + +def server_helpful_404( + request, project_slug=None, lang_slug=None, version_slug=None, + filename=None, template_name='404.html'): + response = get_redirect_response(request, path=filename) + if response: + return response + pagename = re.sub( + r'/index$', r'', re.sub(r'\.html$', r'', re.sub(r'/$', r'', filename))) + suggestion = get_suggestion( + project_slug, lang_slug, version_slug, pagename, request.user) + r = render_to_response(template_name, + {'suggestion': suggestion}, + context_instance=RequestContext(request)) + r.status_code = 404 + return r + + +def get_suggestion(project_slug, lang_slug, version_slug, pagename, user): + """ + | # | project | version | language | What to show | + | 1 | 0 | 0 | 0 | Error message | + | 2 | 0 | 0 | 1 | Error message (Can't happen) | + | 3 | 0 | 1 | 0 | Error message (Can't happen) | + | 4 | 0 | 1 | 1 | Error message (Can't happen) | + | 5 | 1 | 0 | 0 | A link to top-level page of default version | + | 6 | 1 | 0 | 1 | Available versions on the translation project | + | 7 | 1 | 1 | 0 | Available translations of requested version | + | 8 | 1 | 1 | 1 | A link to top-level page of requested version | + """ + + suggestion = {} + if project_slug: + try: + proj = Project.objects.get(slug=project_slug) + if not lang_slug: + lang_slug = proj.language + try: + ver = Version.objects.get( + project__slug=project_slug, slug=version_slug) + except Version.DoesNotExist: + ver = None + + if ver: # if requested version is available on main project + if lang_slug != proj.language: + try: + translations = proj.translations.filter( + language=lang_slug) + if translations: + ver = Version.objects.get( + project__slug=translations[0].slug, slug=version_slug) + else: + ver = None + except Version.DoesNotExist: + ver = None + # if requested version is available on translation project too + if ver: + # Case #8: Show a link to top-level page of the version + suggestion['type'] = 'top' + suggestion['message'] = "What are you looking for?" + suggestion['href'] = proj.get_docs_url(ver.slug, lang_slug) + # requested version is available but not in requested language + else: + # Case #7: Show available translations of the version + suggestion['type'] = 'list' + suggestion['message'] = ( + "Requested page seems not to be translated in " + "requested language. But it's available in these " + "languages.") + suggestion['list'] = [] + suggestion['list'].append({ + 'label': proj.language, + 'project': proj, + 'version_slug': version_slug, + 'pagename': pagename + }) + for t in proj.translations.all(): + try: + Version.objects.get( + project__slug=t.slug, slug=version_slug) + suggestion['list'].append({ + 'label': t.language, + 'project': t, + 'version_slug': version_slug, + 'pagename': pagename + }) + except Version.DoesNotExist: + pass + else: # requested version does not exist on main project + if lang_slug == proj.language: + trans = proj + else: + translations = proj.translations.filter(language=lang_slug) + trans = translations[0] if translations else None + if trans: # requested language is available + # Case #6: Show available versions of the translation + suggestion['type'] = 'list' + suggestion['message'] = ( + "Requested version seems not to have been built yet. " + "But these versions are available.") + suggestion['list'] = [] + for v in Version.objects.public(user, trans, True): + suggestion['list'].append({ + 'label': v.slug, + 'project': trans, + 'version_slug': v.slug, + 'pagename': pagename + }) + # requested project exists but requested version and language + # are not available. + else: + # Case #5: Show a link to top-level page of default version + # of main project + suggestion['type'] = 'top' + suggestion['message'] = 'What are you looking for??' + suggestion['href'] = proj.get_docs_url() + except Project.DoesNotExist: + # Case #1-4: Show error mssage + suggestion['type'] = 'none' + suggestion[ + 'message'] = "We're sorry, we don't know what you're looking for" + else: + suggestion['type'] = 'none' + suggestion[ + 'message'] = "We're sorry, we don't know what you're looking for" + + return suggestion + + +class SendEmailView(FormView): + + """Form view for sending emails to users from admin pages + Accepts the following additional parameters: + queryset + The queryset to use to determine the users to send emails to + """ + + form_class = SendEmailForm + template_name = 'core/send_email_form.html' + + def get_form_kwargs(self): + """Override form kwargs based on input fields + The admin posts to this view initially, so detect the send button on + form post variables. Drop additional fields if we see the send button. + """ + kwargs = super(SendEmailView, self).get_form_kwargs() + if 'send' not in self.request.POST: + kwargs.pop('data', None) + kwargs.pop('files', None) + return kwargs + + def get_initial(self): + """Add selected ids to initial form data""" + initial = super(SendEmailView, self).get_initial() + initial['_selected_action'] = self.request.POST.getlist( + admin.ACTION_CHECKBOX_NAME) + return initial + + def form_valid(self, form): + """If form is valid, send emails to selected users""" + count = 0 + for user in self.get_queryset().all(): + send_email( + user.email, + subject=form.cleaned_data['subject'], + template='core/email/common.txt', + template_html='core/email/common.html', + context={'user': user, 'content': form.cleaned_data['body']}, + request=self.request, + ) + count += 1 + if count == 0: + self.message_user("No receipients to send to", level=messages.ERROR) + else: + self.message_user("Queued {0} messages".format(count)) + return HttpResponseRedirect(self.request.get_full_path()) + + def get_queryset(self): + return self.kwargs.get('queryset') + + def get_context_data(self, **kwargs): + """Return queryset in context""" + context = super(SendEmailView, self).get_context_data(**kwargs) + context['users'] = self.get_queryset().all() + return context + + def message_user(self, message, level=messages.INFO, extra_tags='', + fail_silently=False): + """Implementation of :py:meth:`django.contrib.admin.options.ModelAdmin.message_user` + Send message through messages framework + """ + # TODO generalize this or check if implementation in ModelAdmin is + # useable here + messages.add_message(self.request, level, message, extra_tags=extra_tags, + fail_silently=fail_silently) diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py new file mode 100644 index 00000000000..4e7439d2c9a --- /dev/null +++ b/readthedocs/core/views/hooks.py @@ -0,0 +1,251 @@ +import json + +from django.http import HttpResponse, HttpResponseNotFound +from django.shortcuts import redirect +from django.views.decorators.csrf import csrf_exempt + +from readthedocs.core.utils import trigger_build +from readthedocs.builds.constants import LATEST +from readthedocs.projects import constants +from readthedocs.projects.models import Project +from readthedocs.projects.tasks import update_imported_docs + +import logging + +log = logging.getLogger(__name__) + + +class NoProjectException(Exception): + pass + + +def _build_version(project, slug, already_built=()): + """ + Where we actually trigger builds for a project and slug. + + All webhook logic should route here to call ``trigger_build``. + """ + default = project.default_branch or (project.vcs_repo().fallback_branch) + if not project.has_valid_webhook: + project.has_valid_webhook = True + project.save() + if slug == default and slug not in already_built: + # short circuit versions that are default + # these will build at "latest", and thus won't be + # active + latest_version = project.versions.get(slug=LATEST) + trigger_build(project=project, version=latest_version, force=True) + log.info(("(Version build) Building %s:%s" + % (project.slug, latest_version.slug))) + if project.versions.exclude(active=False).filter(slug=slug).exists(): + # Handle the case where we want to build the custom branch too + slug_version = project.versions.get(slug=slug) + trigger_build(project=project, version=slug_version, force=True) + log.info(("(Version build) Building %s:%s" + % (project.slug, slug_version.slug))) + return LATEST + elif project.versions.exclude(active=True).filter(slug=slug).exists(): + log.info(("(Version build) Not Building %s" % slug)) + return None + elif slug not in already_built: + version = project.versions.get(slug=slug) + trigger_build(project=project, version=version, force=True) + log.info(("(Version build) Building %s:%s" + % (project.slug, version.slug))) + return slug + else: + log.info(("(Version build) Not Building %s" % slug)) + return None + + +def _build_branches(project, branch_list): + """ + Build the branches for a specific project. + + Returns: + to_build - a list of branches that were built + not_building - a list of branches that we won't build + """ + for branch in branch_list: + versions = project.versions_from_branch_name(branch) + to_build = set() + not_building = set() + for version in versions: + log.info(("(Branch Build) Processing %s:%s" + % (project.slug, version.slug))) + ret = _build_version(project, version.slug, already_built=to_build) + if ret: + to_build.add(ret) + else: + not_building.add(version.slug) + return (to_build, not_building) + + +def get_project_from_url(url): + projects = ( + Project.objects.filter(repo__iendswith=url) | + Project.objects.filter(repo__iendswith=url + '.git')) + return projects + + +def log_info(project, msg): + log.info(constants.LOG_TEMPLATE + .format(project=project, + version='', + msg=msg)) + + +def _build_url(url, projects, branches): + """ + Map a URL onto specific projects to build that are linked to that URL. + + Check each of the ``branches`` to see if they are active and should be built. + """ + ret = "" + all_built = {} + all_not_building = {} + for project in projects: + (built, not_building) = _build_branches(project, branches) + if not built: + # Call update_imported_docs to update tag/branch info + update_imported_docs.delay(project.versions.get(slug=LATEST).pk) + msg = '(URL Build) Syncing versions for %s' % project.slug + log.info(msg) + all_built[project.slug] = built + all_not_building[project.slug] = not_building + + for project_slug, built in all_built.items(): + if built: + msg = '(URL Build) Build Started: %s [%s]' % ( + url, ' '.join(built)) + log_info(project_slug, msg=msg) + ret += msg + + for project_slug, not_building in all_not_building.items(): + if not_building: + msg = '(URL Build) Not Building: %s [%s]' % ( + url, ' '.join(not_building)) + log_info(project_slug, msg=msg) + ret += msg + + if not ret: + ret = '(URL Build) No known branches were pushed to.' + + return HttpResponse(ret) + + +@csrf_exempt +def github_build(request): + """ + A post-commit hook for github. + """ + if request.method == 'POST': + try: + # GitHub RTD integration + obj = json.loads(request.POST['payload']) + except: + # Generic post-commit hook + obj = json.loads(request.body) + repo_url = obj['repository']['url'] + hacked_repo_url = repo_url.replace('http://', '').replace('https://', '') + ssh_url = obj['repository']['ssh_url'] + hacked_ssh_url = ssh_url.replace('git@', '').replace('.git', '') + try: + branch = obj['ref'].replace('refs/heads/', '') + except KeyError: + response = HttpResponse('ref argument required to build branches.') + response.status_code = 400 + return response + + try: + repo_projects = get_project_from_url(hacked_repo_url) + if repo_projects: + log.info("(Incoming GitHub Build) %s [%s]" % (hacked_repo_url, branch)) + ssh_projects = get_project_from_url(hacked_ssh_url) + if ssh_projects: + log.info("(Incoming GitHub Build) %s [%s]" % (hacked_ssh_url, branch)) + projects = repo_projects | ssh_projects + return _build_url(hacked_repo_url, projects, [branch]) + except NoProjectException: + log.error( + "(Incoming GitHub Build) Repo not found: %s" % hacked_repo_url) + return HttpResponseNotFound('Repo not found: %s' % hacked_repo_url) + else: + return HttpResponse("You must POST to this resource.") + + +@csrf_exempt +def gitlab_build(request): + """ + A post-commit hook for GitLab. + """ + if request.method == 'POST': + try: + # GitLab RTD integration + obj = json.loads(request.POST['payload']) + except: + # Generic post-commit hook + obj = json.loads(request.body) + url = obj['repository']['homepage'] + ghetto_url = url.replace('http://', '').replace('https://', '') + branch = obj['ref'].replace('refs/heads/', '') + log.info("(Incoming GitLab Build) %s [%s]" % (ghetto_url, branch)) + projects = get_project_from_url(ghetto_url) + if projects: + return _build_url(ghetto_url, projects, [branch]) + else: + log.error( + "(Incoming GitLab Build) Repo not found: %s" % ghetto_url) + return HttpResponseNotFound('Repo not found: %s' % ghetto_url) + else: + return HttpResponse("You must POST to this resource.") + + +@csrf_exempt +def bitbucket_build(request): + if request.method == 'POST': + payload = request.POST.get('payload') + log.info("(Incoming Bitbucket Build) Raw: %s" % payload) + if not payload: + return HttpResponseNotFound('Invalid Request') + obj = json.loads(payload) + rep = obj['repository'] + branches = [rec.get('branch', '') for rec in obj['commits']] + ghetto_url = "%s%s" % ( + "bitbucket.org", rep['absolute_url'].rstrip('/')) + log.info("(Incoming Bitbucket Build) %s [%s]" % ( + ghetto_url, ' '.join(branches))) + log.info("(Incoming Bitbucket Build) JSON: \n\n%s\n\n" % obj) + projects = get_project_from_url(ghetto_url) + if projects: + return _build_url(ghetto_url, projects, branches) + else: + log.error( + "(Incoming Bitbucket Build) Repo not found: %s" % ghetto_url) + return HttpResponseNotFound('Repo not found: %s' % ghetto_url) + else: + return HttpResponse("You must POST to this resource.") + + +@csrf_exempt +def generic_build(request, project_id_or_slug=None): + try: + project = Project.objects.get(pk=project_id_or_slug) + # Allow slugs too + except (Project.DoesNotExist, ValueError): + try: + project = Project.objects.get(slug=project_id_or_slug) + except (Project.DoesNotExist, ValueError): + log.error( + "(Incoming Generic Build) Repo not found: %s" % ( + project_id_or_slug)) + return HttpResponseNotFound( + 'Repo not found: %s' % project_id_or_slug) + if request.method == 'POST': + slug = request.POST.get('version_slug', project.default_version) + log.info( + "(Incoming Generic Build) %s [%s]" % (project.slug, slug)) + _build_version(project, slug) + else: + return HttpResponse("You must POST to this resource.") + return redirect('builds_project_list', project.slug) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py new file mode 100644 index 00000000000..57a7b40705f --- /dev/null +++ b/readthedocs/core/views/serve.py @@ -0,0 +1,252 @@ +""" +Doc serving from Python. + +In production there are two modes, +* Serving from public symlinks in nginx (readthedocs.org & readthedocs.com) +* Serving from private symlinks in Python (readthedocs.com only) + +In development, we have two modes: +* Serving from public symlinks in Python +* Serving from private symlinks in Python + +This means we should only serve from public symlinks in dev, +and generally default to serving from private symlinks in Python only. + +Privacy +------- + +These views will take into account the version privacy level. + +Settings +-------- + +PYTHON_MEDIA (False) - Set this to True to serve docs & media from Python +SERVE_PUBLIC_DOCS (False) - Set this to True to serve public as well as private docs from Python +""" + +from django.core.urlresolvers import reverse +from django.conf import settings +from django.http import HttpResponse, HttpResponseRedirect, Http404 +from django.shortcuts import render_to_response, get_object_or_404 +from django.template import RequestContext +from django.views.static import serve + +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project, ProjectRelationship + + +import mimetypes +import os +import logging + +log = logging.getLogger(__name__) + + +def subproject_list(request): + project_slug = request.slug + proj = get_object_or_404(Project, slug=project_slug) + subprojects = [rel.child for rel in proj.subprojects.all()] + return render_to_response( + 'projects/project_list.html', + {'project_list': subprojects}, + context_instance=RequestContext(request) + ) + + +def subproject_serve_docs(request, project_slug, lang_slug=None, + version_slug=None, filename=''): + parent_slug = request.slug + proj = get_object_or_404(Project, slug=project_slug) + subproject_qs = ProjectRelationship.objects.filter( + parent__slug=parent_slug, child__slug=project_slug) + if lang_slug is None or version_slug is None: + # Handle / + version_slug = proj.get_default_version() + url = reverse('subproject_docs_detail', kwargs={ + 'project_slug': project_slug, + 'version_slug': version_slug, + 'lang_slug': proj.language, + 'filename': filename + }) + return HttpResponseRedirect(url) + + if subproject_qs.exists(): + return serve_docs(request, lang_slug, version_slug, filename, + project_slug) + else: + log.info('Subproject lookup failed: %s:%s' % (project_slug, + parent_slug)) + raise Http404("Subproject does not exist") + + +def default_docs_kwargs(request, project_slug=None): + """ + Return kwargs used to reverse lookup a project's default docs URL. + + Determining which URL to redirect to is done based on the kwargs + passed to reverse(serve_docs, kwargs). This function populates + kwargs for the default docs for a project, and sets appropriate keys + depending on whether request is for a subdomain URL, or a non-subdomain + URL. + + """ + if project_slug: + try: + proj = Project.objects.get(slug=project_slug) + except (Project.DoesNotExist, ValueError): + # Try with underscore, for legacy + try: + proj = Project.objects.get(slug=project_slug.replace('-', '_')) + except (Project.DoesNotExist): + proj = None + else: + # If project_slug isn't in URL pattern, it's set in subdomain + # middleware as request.slug. + try: + proj = Project.objects.get(slug=request.slug) + except (Project.DoesNotExist, ValueError): + # Try with underscore, for legacy + try: + proj = Project.objects.get(slug=request.slug.replace('-', '_')) + except (Project.DoesNotExist): + proj = None + if not proj: + raise Http404("Project slug not found") + version_slug = proj.get_default_version() + kwargs = { + 'project_slug': project_slug, + 'version_slug': version_slug, + 'lang_slug': proj.language, + 'filename': '' + } + # Don't include project_slug for subdomains. + # That's how reverse(serve_docs, ...) differentiates subdomain + # views from non-subdomain views. + if project_slug is None: + del kwargs['project_slug'] + return kwargs + + +def redirect_lang_slug(request, lang_slug, project_slug=None): + """Redirect /en/ to /en/latest/.""" + kwargs = default_docs_kwargs(request, project_slug) + kwargs['lang_slug'] = lang_slug + url = reverse('docs_detail', kwargs=kwargs) + return HttpResponseRedirect(url) + + +def redirect_version_slug(request, version_slug, project_slug=None): + """Redirect /latest/ to /en/latest/.""" + kwargs = default_docs_kwargs(request, project_slug) + kwargs['version_slug'] = version_slug + url = reverse('docs_detail', kwargs=kwargs) + return HttpResponseRedirect(url) + + +def redirect_project_slug(request, project_slug=None): + """Redirect / to /en/latest/.""" + kwargs = default_docs_kwargs(request, project_slug) + url = reverse('docs_detail', kwargs=kwargs) + return HttpResponseRedirect(url) + + +def redirect_page_with_filename(request, filename, project_slug=None): + """Redirect /page/file.html to /en/latest/file.html.""" + kwargs = default_docs_kwargs(request, project_slug) + kwargs['filename'] = filename + url = reverse('docs_detail', kwargs=kwargs) + return HttpResponseRedirect(url) + + +def serve_docs(request, lang_slug, version_slug, filename, project_slug=None): + if not project_slug: + project_slug = request.slug + try: + proj = Project.objects.protected(request.user).get(slug=project_slug) + ver = Version.objects.public(request.user).get( + project__slug=project_slug, slug=version_slug) + except (Project.DoesNotExist, Version.DoesNotExist): + proj = None + ver = None + if not proj or not ver: + return server_helpful_404(request, project_slug, lang_slug, version_slug, + filename) + + if ver not in proj.versions.public(request.user, proj, only_active=False): + r = render_to_response('401.html', + context_instance=RequestContext(request)) + r.status_code = 401 + return r + return _serve_docs(request, project=proj, version=ver, filename=filename, + lang_slug=lang_slug, version_slug=version_slug, + project_slug=project_slug) + + +def _serve_docs(request, project, version, filename, lang_slug=None, + version_slug=None, project_slug=None): + '''Actually serve the built documentation files + + This is not called directly, but is wrapped by :py:func:`serve_docs` so that + authentication can be manipulated. + ''' + # Figure out actual file to serve + if not filename: + filename = "index.html" + # This is required because we're forming the filenames outselves instead of + # letting the web server do it. + elif ( + (project.documentation_type == 'sphinx_htmldir' or + project.documentation_type == 'mkdocs') and + "_static" not in filename and + ".css" not in filename and + ".js" not in filename and + ".png" not in filename and + ".jpg" not in filename and + ".svg" not in filename and + "_images" not in filename and + ".html" not in filename and + "font" not in filename and + "inv" not in filename): + filename += "index.html" + else: + filename = filename.rstrip('/') + # Use the old paths if we're on our old location. + # Otherwise use the new language symlinks. + # This can be removed once we have 'en' symlinks for every project. + if lang_slug == project.language: + basepath = project.rtd_build_path(version_slug) + else: + basepath = project.translations_symlink_path(lang_slug) + basepath = os.path.join(basepath, version_slug) + + # Serve file + log.info('Serving %s for %s' % (filename, project)) + if not settings.DEBUG and not getattr(settings, 'PYTHON_MEDIA', False): + fullpath = os.path.join(basepath, filename) + content_type, encoding = mimetypes.guess_type(fullpath) + content_type = content_type or 'application/octet-stream' + response = HttpResponse(content_type=content_type) + if encoding: + response["Content-Encoding"] = encoding + try: + response['X-Accel-Redirect'] = os.path.join(basepath[len(settings.SITE_ROOT):], + filename) + except UnicodeEncodeError: + raise Http404 + + return response + else: + return serve(request, filename, basepath) + + +def serve_single_version_docs(request, filename, project_slug=None): + if not project_slug: + project_slug = request.slug + proj = get_object_or_404(Project, slug=project_slug) + + # This function only handles single version projects + if not proj.single_version: + raise Http404 + + return serve_docs(request, proj.language, proj.default_version, + filename, project_slug) From 09188b5cb703415d60022ca15857c85248572439 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Fri, 22 Apr 2016 13:03:01 -0700 Subject: [PATCH 02/43] Update to use fancy URL's and whatnot --- readthedocs/builds/models.py | 1 - readthedocs/core/middleware.py | 11 +- readthedocs/core/resolver.py | 3 +- readthedocs/core/urls/__init__.py | 55 +--- readthedocs/core/urls/single_version.py | 4 +- readthedocs/core/urls/subdomain.py | 36 +-- readthedocs/core/views/serve.py | 274 +++++------------- readthedocs/privacy/backend.py | 4 + .../rtd_tests/tests/test_doc_serving.py | 76 +++++ .../rtd_tests/tests/test_single_version.py | 20 +- readthedocs/settings/base.py | 2 +- 11 files changed, 192 insertions(+), 294 deletions(-) create mode 100644 readthedocs/rtd_tests/tests/test_doc_serving.py diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index a4f63c2f403..e8720f7fe8a 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -18,7 +18,6 @@ from readthedocs.projects.constants import (PRIVACY_CHOICES, GITHUB_URL, GITHUB_REGEXS, BITBUCKET_URL, BITBUCKET_REGEXS, PRIVATE) -from readthedocs.core.resolver import resolve from .constants import (BUILD_STATE, BUILD_TYPES, VERSION_TYPES, LATEST, NON_REPOSITORY_VERSIONS, STABLE, diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index aecdc42e0f0..0311a5a8d6a 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -15,12 +15,12 @@ SUBDOMAIN_URLCONF = getattr( settings, 'SUBDOMAIN_URLCONF', - 'readthedocs.core.subdomain_urls' + 'readthedocs.core.urls.subdomain' ) SINGLE_VERSION_URLCONF = getattr( settings, 'SINGLE_VERSION_URLCONF', - 'readthedocs.core.single_version_urls' + 'readthedocs.core.urls.single_version' ) @@ -34,8 +34,7 @@ def process_request(self, request): path = request.get_full_path() log_kwargs = dict(host=host, path=path) public_domain = getattr(settings, 'PUBLIC_DOMAIN', None) - production_domain = getattr(settings, 'PRODUCTION_DOMAIN', - 'readthedocs.org') + production_domain = getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org') if public_domain is None: public_domain = production_domain @@ -47,12 +46,12 @@ def process_request(self, request): if len(domain_parts) == len(public_domain.split('.')) + 1: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - is_ssl = subdomain.lower() == 'ssl' - if not is_www and not is_ssl and public_domain in host: + if not is_www and public_domain in host: request.subdomain = True request.slug = subdomain request.urlconf = SUBDOMAIN_URLCONF return None + # Serve CNAMEs if (public_domain not in host and production_domain not in host and diff --git a/readthedocs/core/resolver.py b/readthedocs/core/resolver.py index b5e80550c6b..cd4c137c3d4 100644 --- a/readthedocs/core/resolver.py +++ b/readthedocs/core/resolver.py @@ -99,7 +99,8 @@ def base_resolve_path(project_slug, filename, version_slug=None, language=None, def resolve_path(project, filename='', version_slug=None, language=None, single_version=None, subdomain=None, cname=None, private=None): """ Resolve a URL with a subset of fields defined.""" - subdomain = getattr(settings, 'USE_SUBDOMAIN', False) + if subdomain is None: + subdomain = getattr(settings, 'USE_SUBDOMAIN', False) relation = project.superprojects.first() cname = cname or project.domains.filter(canonical=True).first() main_language_project = project.main_language_project diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py index d4ab8389e48..e5851e7fd69 100644 --- a/readthedocs/core/urls/__init__.py +++ b/readthedocs/core/urls/__init__.py @@ -8,65 +8,40 @@ docs_urls = patterns( '', - # For serving docs locally and when nginx isn't - url((r'^docs/(?P{project_slug})/(?P{lang_slug})/' - r'(?P{version_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.serve_docs', - name='docs_detail'), - - # Redirect to default version, if only lang_slug is set. - url((r'^docs/(?P{project_slug})/' - r'(?P{lang_slug})/$'.format(**pattern_opts)), - 'readthedocs.core.views.redirect_lang_slug', - name='docs_detail'), - - # Redirect to default version, if only version_slug is set. - url((r'^docs/(?P{project_slug})/' - r'(?P{version_slug})/$'.format(**pattern_opts)), - 'readthedocs.core.views.redirect_version_slug', - name='docs_detail'), - - # Redirect to default version. - url(r'^docs/(?P{project_slug})/$'.format(**pattern_opts), - 'readthedocs.core.views.redirect_project_slug', - name='docs_detail'), # Handle /page/ redirects for explicit "latest" version goodness. url((r'^docs/(?P{project_slug})/page/' r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.redirect_page_with_filename', + 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), # Handle single version URLs url((r'^docs/(?P{project_slug})/' r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.serve_single_version_docs', + 'readthedocs.core.views.serve.serve_symlink_docs', name='docs_detail'), - # Handle fallbacks - url((r'^user_builds/(?P{project_slug})/rtd-builds/' + # Just for reversing URL's for now + url((r'^docs/(?P{project_slug})/(?P{lang_slug})/' r'(?P{version_slug})/' r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.server_helpful_404', - name='user_builds_fallback'), - url((r'^user_builds/(?P{project_slug})/translations/' - r'(?P{lang_slug})/(?P{version_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.server_helpful_404', - name='user_builds_fallback_translations'), + 'readthedocs.core.views.serve.serve_symlink_docs', + name='docs_detail'), + ) core_urls = patterns( '', - url(r'^github', 'readthedocs.core.views.github_build', name='github_build'), - url(r'^gitlab', 'readthedocs.core.views.gitlab_build', name='gitlab_build'), - url(r'^bitbucket', 'readthedocs.core.views.bitbucket_build', name='bitbucket_build'), + # Hooks + url(r'^github', 'readthedocs.core.views.hooks.github_build', name='github_build'), + url(r'^gitlab', 'readthedocs.core.views.hooks.gitlab_build', name='gitlab_build'), + url(r'^bitbucket', 'readthedocs.core.views.hooks.bitbucket_build', name='bitbucket_build'), url((r'^build/' r'(?P{project_slug})'.format(**pattern_opts)), - 'readthedocs.core.views.generic_build', + 'readthedocs.core.views.hooks.generic_build', name='generic_build'), + # Random other stuff url(r'^random/(?P{project_slug})'.format(**pattern_opts), 'readthedocs.core.views.random_page', name='random_page'), @@ -95,8 +70,4 @@ url(r'^feeds/latest/$', LatestProjectsFeed(), name="latest_feed"), - url((r'^mlt/(?P{project_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.morelikethis', - name='morelikethis'), ) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 857d455034b..5929d06201c 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -4,11 +4,11 @@ '', # base view, flake8 complains if it is on the previous line. # Handle /docs on RTD domain url(r'^docs/(?P[-\w]+)/(?P.*)$', - 'readthedocs.core.views.serve_single_version_docs', + 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), # Handle subdomains url(r'^(?P.*)$', - 'readthedocs.core.views.serve_single_version_docs', + 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), ) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index 6a1d3e2d66a..d4debb46d10 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -1,6 +1,5 @@ from django.conf.urls import url, patterns -from readthedocs.projects.constants import LANGUAGES_REGEX from readthedocs.urls import urlpatterns as main_patterns handler500 = 'readthedocs.core.views.server_error' @@ -8,41 +7,12 @@ urlpatterns = patterns( '', # base view, flake8 complains if it is on the previous line. - url((r'^projects/(?P[\w.-]+)/(?P%s)/' - r'(?P[\w.-]+)/(?P.*)$' % LANGUAGES_REGEX), - 'readthedocs.core.views.subproject_serve_docs', - name='subproject_docs_detail'), - - url(r'^projects/(?P[\w.-]+)', - 'readthedocs.core.views.subproject_serve_docs', - name='subproject_docs_detail'), - - url(r'^projects/$', - 'readthedocs.core.views.subproject_list', - name='subproject_docs_list'), - - url(r'^(?P%s)/(?P[\w.-]+)/(?P.*)$' % LANGUAGES_REGEX, - 'readthedocs.core.views.serve_docs', - name='docs_detail'), - - url(r'^(?P%s)/(?P.*)/$' % LANGUAGES_REGEX, - 'readthedocs.core.views.serve_docs', - {'filename': 'index.html'}, - name='docs_detail'), - url(r'^page/(?P.*)$', - 'readthedocs.core.views.redirect_page_with_filename', + 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), - url(r'^(?P%s)/$' % LANGUAGES_REGEX, - 'readthedocs.core.views.redirect_lang_slug', - name='lang_subdomain_handler'), - - url(r'^(?P.*)/$', - 'readthedocs.core.views.redirect_version_slug', - name='version_subdomain_handler'), - - url(r'^$', 'readthedocs.core.views.redirect_project_slug', name='homepage'), + url(r'^$', 'readthedocs.core.views.serve.redirect_project_slug', name='redirect_project_slug'), + url(r'', 'readthedocs.core.views.serve.serve_symlink_docs', name='serve_symlink_docs'), ) urlpatterns += main_patterns diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 57a7b40705f..7833b3ce3cf 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -24,229 +24,115 @@ SERVE_PUBLIC_DOCS (False) - Set this to True to serve public as well as private docs from Python """ -from django.core.urlresolvers import reverse from django.conf import settings from django.http import HttpResponse, HttpResponseRedirect, Http404 -from django.shortcuts import render_to_response, get_object_or_404 +from django.shortcuts import render_to_response from django.template import RequestContext from django.views.static import serve -from readthedocs.builds.models import Version -from readthedocs.projects.models import Project, ProjectRelationship - +from readthedocs.projects.models import Project +from readthedocs.core.symlink import PrivateSymlink, PublicSymlink +from readthedocs.core.resolver import resolve, resolve_path +from readthedocs.privacy.loader import AdminPermission import mimetypes import os import logging +from functools import wraps log = logging.getLogger(__name__) -def subproject_list(request): - project_slug = request.slug - proj = get_object_or_404(Project, slug=project_slug) - subprojects = [rel.child for rel in proj.subprojects.all()] - return render_to_response( - 'projects/project_list.html', - {'project_list': subprojects}, - context_instance=RequestContext(request) - ) +def map_project_slug(view_func): + """ + A decorator that maps a ``project_slug`` URL param into a Project. + :raises: Http404 if the Project doesn't exist -def subproject_serve_docs(request, project_slug, lang_slug=None, - version_slug=None, filename=''): - parent_slug = request.slug - proj = get_object_or_404(Project, slug=project_slug) - subproject_qs = ProjectRelationship.objects.filter( - parent__slug=parent_slug, child__slug=project_slug) - if lang_slug is None or version_slug is None: - # Handle / - version_slug = proj.get_default_version() - url = reverse('subproject_docs_detail', kwargs={ - 'project_slug': project_slug, - 'version_slug': version_slug, - 'lang_slug': proj.language, - 'filename': filename - }) - return HttpResponseRedirect(url) - - if subproject_qs.exists(): - return serve_docs(request, lang_slug, version_slug, filename, - project_slug) - else: - log.info('Subproject lookup failed: %s:%s' % (project_slug, - parent_slug)) - raise Http404("Subproject does not exist") + .. warning:: Does not take into account any kind of privacy settings. + """ + @wraps(view_func) + def inner_view(request, project=None, project_slug=None, *args, **kwargs): + if project is None: + if not project_slug: + project_slug = request.slug + try: + project = Project.objects.get(slug=project_slug) + except Project.DoesNotExist: + raise Http404 + return view_func(request, project=project, *args, **kwargs) + return inner_view -def default_docs_kwargs(request, project_slug=None): - """ - Return kwargs used to reverse lookup a project's default docs URL. +@map_project_slug +def redirect_project_slug(request, project): + """Handle / -> /en/latest/ directs on subdomains""" + return HttpResponseRedirect(resolve(project)) - Determining which URL to redirect to is done based on the kwargs - passed to reverse(serve_docs, kwargs). This function populates - kwargs for the default docs for a project, and sets appropriate keys - depending on whether request is for a subdomain URL, or a non-subdomain - URL. - """ - if project_slug: - try: - proj = Project.objects.get(slug=project_slug) - except (Project.DoesNotExist, ValueError): - # Try with underscore, for legacy - try: - proj = Project.objects.get(slug=project_slug.replace('-', '_')) - except (Project.DoesNotExist): - proj = None - else: - # If project_slug isn't in URL pattern, it's set in subdomain - # middleware as request.slug. - try: - proj = Project.objects.get(slug=request.slug) - except (Project.DoesNotExist, ValueError): - # Try with underscore, for legacy - try: - proj = Project.objects.get(slug=request.slug.replace('-', '_')) - except (Project.DoesNotExist): - proj = None - if not proj: - raise Http404("Project slug not found") - version_slug = proj.get_default_version() - kwargs = { - 'project_slug': project_slug, - 'version_slug': version_slug, - 'lang_slug': proj.language, - 'filename': '' - } - # Don't include project_slug for subdomains. - # That's how reverse(serve_docs, ...) differentiates subdomain - # views from non-subdomain views. - if project_slug is None: - del kwargs['project_slug'] - return kwargs - - -def redirect_lang_slug(request, lang_slug, project_slug=None): - """Redirect /en/ to /en/latest/.""" - kwargs = default_docs_kwargs(request, project_slug) - kwargs['lang_slug'] = lang_slug - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def redirect_version_slug(request, version_slug, project_slug=None): - """Redirect /latest/ to /en/latest/.""" - kwargs = default_docs_kwargs(request, project_slug) - kwargs['version_slug'] = version_slug - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def redirect_project_slug(request, project_slug=None): - """Redirect / to /en/latest/.""" - kwargs = default_docs_kwargs(request, project_slug) - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def redirect_page_with_filename(request, filename, project_slug=None): +@map_project_slug +def redirect_page_with_filename(request, project, filename): """Redirect /page/file.html to /en/latest/file.html.""" - kwargs = default_docs_kwargs(request, project_slug) - kwargs['filename'] = filename - url = reverse('docs_detail', kwargs=kwargs) - return HttpResponseRedirect(url) - - -def serve_docs(request, lang_slug, version_slug, filename, project_slug=None): - if not project_slug: - project_slug = request.slug - try: - proj = Project.objects.protected(request.user).get(slug=project_slug) - ver = Version.objects.public(request.user).get( - project__slug=project_slug, slug=version_slug) - except (Project.DoesNotExist, Version.DoesNotExist): - proj = None - ver = None - if not proj or not ver: - return server_helpful_404(request, project_slug, lang_slug, version_slug, - filename) - - if ver not in proj.versions.public(request.user, proj, only_active=False): - r = render_to_response('401.html', - context_instance=RequestContext(request)) - r.status_code = 401 - return r - return _serve_docs(request, project=proj, version=ver, filename=filename, - lang_slug=lang_slug, version_slug=version_slug, - project_slug=project_slug) - - -def _serve_docs(request, project, version, filename, lang_slug=None, - version_slug=None, project_slug=None): - '''Actually serve the built documentation files - - This is not called directly, but is wrapped by :py:func:`serve_docs` so that - authentication can be manipulated. - ''' - # Figure out actual file to serve - if not filename: - filename = "index.html" - # This is required because we're forming the filenames outselves instead of - # letting the web server do it. - elif ( - (project.documentation_type == 'sphinx_htmldir' or - project.documentation_type == 'mkdocs') and - "_static" not in filename and - ".css" not in filename and - ".js" not in filename and - ".png" not in filename and - ".jpg" not in filename and - ".svg" not in filename and - "_images" not in filename and - ".html" not in filename and - "font" not in filename and - "inv" not in filename): - filename += "index.html" - else: - filename = filename.rstrip('/') - # Use the old paths if we're on our old location. - # Otherwise use the new language symlinks. - # This can be removed once we have 'en' symlinks for every project. - if lang_slug == project.language: - basepath = project.rtd_build_path(version_slug) - else: - basepath = project.translations_symlink_path(lang_slug) - basepath = os.path.join(basepath, version_slug) + return HttpResponseRedirect(resolve(project, filename=filename)) - # Serve file - log.info('Serving %s for %s' % (filename, project)) - if not settings.DEBUG and not getattr(settings, 'PYTHON_MEDIA', False): + +@map_project_slug +def serve_docs(request, project, lang_slug=None, version_slug=None, filename=''): + filename = resolve_path( + project, version_slug=version_slug, language=lang_slug, filename=filename + ) + return serve_symlink_docs(request, filename=filename, project=project) + + +@map_project_slug +def serve_symlink_docs(request, project, filename=''): + # Handle indexes + if filename == '' or filename[-1] == '/': + filename += 'index.html' + + if settings.DEBUG or getattr(settings, 'SERVE_PUBLIC_DOCS', False): + # Try to serve a public link during dev + public_symlink = PublicSymlink(project) + basepath = public_symlink.project_root fullpath = os.path.join(basepath, filename) + if os.path.exists(fullpath): + return serve(request, filename, basepath) + + if not AdminPermission.is_member(user=request.user, project=project): + # Do basic auth check on the project, but not the version + res = render_to_response('401.html', + context_instance=RequestContext(request)) + res.status_code = 401 + log.error('Unauthorized access to {0} documentation'.format(project.slug)) + return res + + # Handle private + private_symlink = PrivateSymlink(project) + basepath = private_symlink.project_root + fullpath = os.path.join(basepath, filename) + + log.info('Serving %s for %s' % (filename, project)) + + if os.path.exists(fullpath): + raise Http404('Path does not exist: %s' % fullpath) + + # Serve the file from the proper location + if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False): + # Serve from Python + return serve(request, filename, basepath) + else: + # Serve from Nginx content_type, encoding = mimetypes.guess_type(fullpath) content_type = content_type or 'application/octet-stream' response = HttpResponse(content_type=content_type) if encoding: response["Content-Encoding"] = encoding try: - response['X-Accel-Redirect'] = os.path.join(basepath[len(settings.SITE_ROOT):], - filename) + response['X-Accel-Redirect'] = os.path.join( + basepath[len(settings.SITE_ROOT):], + filename + ) except UnicodeEncodeError: raise Http404 return response - else: - return serve(request, filename, basepath) - - -def serve_single_version_docs(request, filename, project_slug=None): - if not project_slug: - project_slug = request.slug - proj = get_object_or_404(Project, slug=project_slug) - - # This function only handles single version projects - if not proj.single_version: - raise Http404 - - return serve_docs(request, proj.language, proj.default_version, - filename, project_slug) diff --git a/readthedocs/privacy/backend.py b/readthedocs/privacy/backend.py index 42be2fbd179..bd4429b1fee 100644 --- a/readthedocs/privacy/backend.py +++ b/readthedocs/privacy/backend.py @@ -288,6 +288,10 @@ class AdminPermission(object): def is_admin(cls, user, project): return user in project.users.all() + @classmethod + def is_member(cls, user, project): + return user in project.users.all() + class AdminNotAuthorized(ValueError): pass diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py new file mode 100644 index 00000000000..2cf00a1f32b --- /dev/null +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -0,0 +1,76 @@ +import django_dynamic_fixture as fixture + +from django.contrib.auth.models import User +from django.test import TestCase +from django.test.utils import override_settings + +from readthedocs.projects.models import Project + + +@override_settings( + USE_SUBDOMAIN=False, PUBLIC_DOMAIN='public.readthedocs.org', DEBUG=False +) +class TestPrivateDocs(TestCase): + + def setUp(self): + self.eric = fixture.get(User, username='eric') + self.eric.set_password('eric') + self.eric.save() + self.public = fixture.get(Project, slug='public', main_language_project=None) + self.private = fixture.get( + Project, slug='private', privacy_level='private', + version_privacy_level='private', users=[self.eric], + ) + + @override_settings( + PYTHON_MEDIA=False, SERVE_PUBLIC_DOCS=False + ) + def test_private_nginx_serving(self): + self.client.login(username='eric', password='eric') + r = self.client.get('/docs/private/en/latest/usage.html') + self.assertEqual(r.status_code, 200) + self.assertEqual( + r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' + ) + + self.client.logout() + r = self.client.get('/docs/private/en/latest/usage.html') + self.assertEqual(r.status_code, 401) + + @override_settings( + PYTHON_MEDIA=False, SERVE_PUBLIC_DOCS=True + ) + def test_public_nginx_doc_serving(self): + self.client.login(username='eric', password='eric') + r = self.client.get('/docs/private/en/latest/usage.html') + self.assertEqual(r.status_code, 200) + self.assertEqual( + r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' + ) + + self.client.logout() + r = self.client.get('/docs/private/en/latest/usage.html') + self.assertEqual(r.status_code, 401) + + @override_settings( + PYTHON_MEDIA=True, SERVE_PUBLIC_DOCS=False + ) + def test_private_python_media_serving(self): + self.client.login(username='eric', password='eric') + r = self.client.get('/docs/private/en/latest/usage.html') + + self.client.logout() + r = self.client.get('/docs/private/en/latest/usage.html') + self.assertEqual(r.status_code, 401) + + @override_settings( + PYTHON_MEDIA=True, SERVE_PUBLIC_DOCS=True + ) + def test_public_python_doc_serving(self): + self.client.login(username='eric', password='eric') + r = self.client.get('/docs/private/en/latest/usage.html') + self.assertEqual(r.status_code, 404) + + self.client.logout() + r = self.client.get('/docs/private/en/latest/usage.html') + self.assertEqual(r.status_code, 401) diff --git a/readthedocs/rtd_tests/tests/test_single_version.py b/readthedocs/rtd_tests/tests/test_single_version.py index 657b7ba1759..bc715421b4f 100644 --- a/readthedocs/rtd_tests/tests/test_single_version.py +++ b/readthedocs/rtd_tests/tests/test_single_version.py @@ -1,25 +1,18 @@ from django.test import TestCase from django.test.utils import override_settings -from readthedocs.builds.constants import LATEST -from readthedocs.builds.models import Version +import django_dynamic_fixture as fixture + from readthedocs.projects.models import Project -@override_settings(USE_SUBDOMAIN=True, PUBLIC_DOMAIN='public.readthedocs.org') +@override_settings( + USE_SUBDOMAIN=True, PUBLIC_DOMAIN='public.readthedocs.org', SERVE_PUBLIC_DOCS=True +) class RedirectSingleVersionTests(TestCase): - fixtures = ["eric", "test_data"] def setUp(self): - self.pip = Project.objects.get(slug='pip') - self.pip.single_version = True - self.pip.save() - - def test_test_case_project_is_single_version(self): - self.assertTrue(Project.objects.get(name='Pip').single_version) - - def test_test_case_version_exists(self): - self.assertTrue(Version.objects.filter(project__name__exact='Pip').get(slug=LATEST)) + self.pip = fixture.get(Project, slug='pip', single_version=True, main_language_project=None) def test_proper_single_version_url_full_with_filename(self): with override_settings(USE_SUBDOMAIN=False): @@ -42,7 +35,6 @@ def test_improper_single_version_url_subdomain(self): self.assertEqual(r.status_code, 404) def test_docs_url_generation(self): - self.pip.single_version = True with override_settings(USE_SUBDOMAIN=False): self.assertEqual(self.pip.get_docs_url(), 'http://public.readthedocs.org/docs/pip/') diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index bf74ca0be9e..9d393ed81ff 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -20,7 +20,7 @@ class CommunityBaseSettings(Settings): # Django settings SITE_ID = 1 ROOT_URLCONF = 'readthedocs.urls' - SUBDOMAIN_URLCONF = 'readthedocs.core.subdomain_urls' + SUBDOMAIN_URLCONF = 'readthedocs.core.urls.subdomain' LOGIN_REDIRECT_URL = '/dashboard/' FORCE_WWW = False SECRET_KEY = 'replace-this-please' # noqa From 9657816b59d42d62ed623cb3be46aa814cf4bdaf Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Fri, 22 Apr 2016 13:52:11 -0700 Subject: [PATCH 03/43] Fix small tests --- readthedocs/rtd_tests/tests/test_middleware.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_middleware.py b/readthedocs/rtd_tests/tests/test_middleware.py index 79bdcb862f6..382d2a5916f 100644 --- a/readthedocs/rtd_tests/tests/test_middleware.py +++ b/readthedocs/rtd_tests/tests/test_middleware.py @@ -41,7 +41,7 @@ def test_failey_cname(self): def test_proper_subdomain(self): request = self.factory.get(self.url, HTTP_HOST='pip.readthedocs.org') self.middleware.process_request(request) - self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls') + self.assertEqual(request.urlconf, 'readthedocs.core.urls.subdomain') self.assertEqual(request.subdomain, True) self.assertEqual(request.slug, 'pip') @@ -49,7 +49,7 @@ def test_proper_subdomain(self): def test_subdomain_different_length(self): request = self.factory.get(self.url, HTTP_HOST='pip.prod.readthedocs.org') self.middleware.process_request(request) - self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls') + self.assertEqual(request.urlconf, 'readthedocs.core.urls.subdomain') self.assertEqual(request.subdomain, True) self.assertEqual(request.slug, 'pip') @@ -58,7 +58,7 @@ def test_domain_object(self): request = self.factory.get(self.url, HTTP_HOST='docs.foobar.com') self.middleware.process_request(request) - self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls') + self.assertEqual(request.urlconf, 'readthedocs.core.urls.subdomain') self.assertEqual(request.domain_object, True) self.assertEqual(request.slug, 'pip') @@ -72,14 +72,14 @@ def test_proper_cname(self): cache.get = lambda x: 'my_slug' request = self.factory.get(self.url, HTTP_HOST='my.valid.homename') self.middleware.process_request(request) - self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls') + self.assertEqual(request.urlconf, 'readthedocs.core.urls.subdomain') self.assertEqual(request.cname, True) self.assertEqual(request.slug, 'my_slug') def test_request_header(self): request = self.factory.get(self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='pip') self.middleware.process_request(request) - self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls') + self.assertEqual(request.urlconf, 'readthedocs.core.urls.subdomain') self.assertEqual(request.cname, True) self.assertEqual(request.rtdheader, True) self.assertEqual(request.slug, 'pip') @@ -89,14 +89,14 @@ def test_proper_cname_uppercase(self): cache.get = lambda x: x.split('.')[0] request = self.factory.get(self.url, HTTP_HOST='PIP.RANDOM.COM') self.middleware.process_request(request) - self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls') + self.assertEqual(request.urlconf, 'readthedocs.core.urls.subdomain') self.assertEqual(request.cname, True) self.assertEqual(request.slug, 'pip') def test_request_header_uppercase(self): request = self.factory.get(self.url, HTTP_HOST='some.random.com', HTTP_X_RTD_SLUG='PIP') self.middleware.process_request(request) - self.assertEqual(request.urlconf, 'readthedocs.core.subdomain_urls') + self.assertEqual(request.urlconf, 'readthedocs.core.urls.subdomain') self.assertEqual(request.cname, True) self.assertEqual(request.rtdheader, True) self.assertEqual(request.slug, 'pip') From ce68cc758a1f452136a2ac91714c72c9eab404ff Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 25 Apr 2016 09:31:40 -0700 Subject: [PATCH 04/43] Futz around with more tests and cleanup --- readthedocs/core/views/serve.py | 70 ++++++----- .../rtd_tests/tests/test_doc_serving.py | 111 ++++++++++++------ 2 files changed, 110 insertions(+), 71 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 7833b3ce3cf..eb7c65b0e4e 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -84,20 +84,53 @@ def serve_docs(request, project, lang_slug=None, version_slug=None, filename='') return serve_symlink_docs(request, filename=filename, project=project) +def _serve_file(request, filename, basepath): + # Serve the file from the proper location + if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False): + # Serve from Python + return serve(request, filename, basepath) + else: + # Serve from Nginx + content_type, encoding = mimetypes.guess_type(os.path.join(basepath, filename)) + content_type = content_type or 'application/octet-stream' + response = HttpResponse(content_type=content_type) + if encoding: + response["Content-Encoding"] = encoding + try: + response['X-Accel-Redirect'] = os.path.join( + basepath[len(settings.SITE_ROOT):], + filename + ) + except UnicodeEncodeError: + raise Http404 + + return response + + @map_project_slug def serve_symlink_docs(request, project, filename=''): + # Handle indexes if filename == '' or filename[-1] == '/': filename += 'index.html' + # This breaks path joining, by ignoring the root when given an "absolute" path + assert filename[0] is not '/' + if settings.DEBUG or getattr(settings, 'SERVE_PUBLIC_DOCS', False): - # Try to serve a public link during dev public_symlink = PublicSymlink(project) basepath = public_symlink.project_root - fullpath = os.path.join(basepath, filename) - if os.path.exists(fullpath): - return serve(request, filename, basepath) + if os.path.exists(os.path.join(basepath, filename)): + return _serve_file(request, filename, basepath) + + # Handle private + private_symlink = PrivateSymlink(project) + basepath = private_symlink.project_root + + if not os.path.exists(os.path.join(basepath, filename)): + raise Http404('Path does not exist: %s' % filename) + # Only check permissions if we are about to serve an existing private file if not AdminPermission.is_member(user=request.user, project=project): # Do basic auth check on the project, but not the version res = render_to_response('401.html', @@ -106,33 +139,6 @@ def serve_symlink_docs(request, project, filename=''): log.error('Unauthorized access to {0} documentation'.format(project.slug)) return res - # Handle private - private_symlink = PrivateSymlink(project) - basepath = private_symlink.project_root - fullpath = os.path.join(basepath, filename) - log.info('Serving %s for %s' % (filename, project)) - if os.path.exists(fullpath): - raise Http404('Path does not exist: %s' % fullpath) - - # Serve the file from the proper location - if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False): - # Serve from Python - return serve(request, filename, basepath) - else: - # Serve from Nginx - content_type, encoding = mimetypes.guess_type(fullpath) - content_type = content_type or 'application/octet-stream' - response = HttpResponse(content_type=content_type) - if encoding: - response["Content-Encoding"] = encoding - try: - response['X-Accel-Redirect'] = os.path.join( - basepath[len(settings.SITE_ROOT):], - filename - ) - except UnicodeEncodeError: - raise Http404 - - return response + return _serve_file(request, filename, basepath) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 2cf00a1f32b..708c751ee6d 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -1,16 +1,19 @@ +import mock import django_dynamic_fixture as fixture from django.contrib.auth.models import User from django.test import TestCase from django.test.utils import override_settings +from readthedocs.rtd_tests.base import RequestFactoryTestMixin from readthedocs.projects.models import Project +from readthedocs.core.views.serve import serve_symlink_docs @override_settings( USE_SUBDOMAIN=False, PUBLIC_DOMAIN='public.readthedocs.org', DEBUG=False ) -class TestPrivateDocs(TestCase): +class BaseDocServing(RequestFactoryTestMixin, TestCase): def setUp(self): self.eric = fixture.get(User, username='eric') @@ -21,56 +24,86 @@ def setUp(self): Project, slug='private', privacy_level='private', version_privacy_level='private', users=[self.eric], ) + self.private_url = '/docs/private/en/latest/usage.html' + self.public_url = '/docs/public/en/latest/usage.html' - @override_settings( - PYTHON_MEDIA=False, SERVE_PUBLIC_DOCS=False - ) - def test_private_nginx_serving(self): - self.client.login(username='eric', password='eric') - r = self.client.get('/docs/private/en/latest/usage.html') - self.assertEqual(r.status_code, 200) - self.assertEqual( - r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' - ) - self.client.logout() - r = self.client.get('/docs/private/en/latest/usage.html') +@override_settings(SERVE_PUBLIC_DOCS=False) +class TestPrivateDocs(BaseDocServing): + + @override_settings(PYTHON_MEDIA=True) + def test_private_python_media_serving(self): + with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: + request = self.request(self.private_url, user=self.eric) + serve_symlink_docs(request, project=self.private, filename='en/latest/usage.html') + serve_mock.assert_called_with( + request, + 'en/latest/usage.html', + '/Users/eric/projects/readthedocs.org/private_web_root/private' + ) + + r = self.client.get(self.private_url) + self.assertEqual(r.status_code, 401) + + r = self.client.get(self.public_url) self.assertEqual(r.status_code, 401) - @override_settings( - PYTHON_MEDIA=False, SERVE_PUBLIC_DOCS=True - ) - def test_public_nginx_doc_serving(self): - self.client.login(username='eric', password='eric') - r = self.client.get('/docs/private/en/latest/usage.html') + @override_settings(PYTHON_MEDIA=False) + def test_private_nginx_serving(self): + request = self.request(self.private_url, user=self.eric) + r = serve_symlink_docs(request, project=self.private, filename='en/latest/usage.html') self.assertEqual(r.status_code, 200) self.assertEqual( r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' ) - self.client.logout() - r = self.client.get('/docs/private/en/latest/usage.html') + r = self.client.get(self.private_url) self.assertEqual(r.status_code, 401) - @override_settings( - PYTHON_MEDIA=True, SERVE_PUBLIC_DOCS=False - ) - def test_private_python_media_serving(self): - self.client.login(username='eric', password='eric') - r = self.client.get('/docs/private/en/latest/usage.html') - - self.client.logout() - r = self.client.get('/docs/private/en/latest/usage.html') + r = self.client.get(self.public_url) self.assertEqual(r.status_code, 401) - @override_settings( - PYTHON_MEDIA=True, SERVE_PUBLIC_DOCS=True - ) - def test_public_python_doc_serving(self): - self.client.login(username='eric', password='eric') - r = self.client.get('/docs/private/en/latest/usage.html') - self.assertEqual(r.status_code, 404) - self.client.logout() - r = self.client.get('/docs/private/en/latest/usage.html') +@override_settings(SERVE_PUBLIC_DOCS=True) +class TestPublicDocs(BaseDocServing): + + @override_settings(PYTHON_MEDIA=True) + def test_public_python_media_serving(self): + with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: + with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): + request = self.request(self.public_url, user=self.eric) + serve_symlink_docs(request, project=self.public, filename='en/latest/usage.html') + serve_mock.assert_called_with( + request, + 'en/latest/usage.html', + '/Users/eric/projects/readthedocs.org/public_web_root/public' + ) + + with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): + r = self.client.get(self.public_url) + self.assertEqual(r.status_code, 200) + + with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: + r = self.client.get(self.public_url) + self.assertEqual(r.status_code, 200) + serve_mock.assert_called_with( + request, + 'en/latest/usage.html', + '/Users/eric/projects/readthedocs.org/public_web_root/public' + ) + + @override_settings(PYTHON_MEDIA=False) + def test_public_nginx_serving(self): + with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): + request = self.request(self.public_url, user=self.eric) + r = serve_symlink_docs(request, project=self.public, filename='en/latest/usage.html') + self.assertEqual(r.status_code, 200) + self.assertEqual( + r._headers['x-accel-redirect'][1], '/public_web_root/public/en/latest/usage.html' + ) + + r = self.client.get(self.public_url) + self.assertEqual(r.status_code, 401) + + r = self.client.get(self.public_url) self.assertEqual(r.status_code, 401) From 64faed20fd3a5672f9b34c4b4565a567eba3deec Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 26 Apr 2016 15:21:16 -0700 Subject: [PATCH 05/43] Remove old files --- readthedocs/core/single_version_urls.py | 14 --- readthedocs/core/subdomain_urls.py | 48 --------- readthedocs/core/urls.py | 102 ------------------- readthedocs/core/views/__init__.py | 128 ------------------------ 4 files changed, 292 deletions(-) delete mode 100644 readthedocs/core/single_version_urls.py delete mode 100644 readthedocs/core/subdomain_urls.py delete mode 100644 readthedocs/core/urls.py diff --git a/readthedocs/core/single_version_urls.py b/readthedocs/core/single_version_urls.py deleted file mode 100644 index 857d455034b..00000000000 --- a/readthedocs/core/single_version_urls.py +++ /dev/null @@ -1,14 +0,0 @@ -from django.conf.urls import patterns, url - -urlpatterns = patterns( - '', # base view, flake8 complains if it is on the previous line. - # Handle /docs on RTD domain - url(r'^docs/(?P[-\w]+)/(?P.*)$', - 'readthedocs.core.views.serve_single_version_docs', - name='docs_detail'), - - # Handle subdomains - url(r'^(?P.*)$', - 'readthedocs.core.views.serve_single_version_docs', - name='docs_detail'), -) diff --git a/readthedocs/core/subdomain_urls.py b/readthedocs/core/subdomain_urls.py deleted file mode 100644 index 6a1d3e2d66a..00000000000 --- a/readthedocs/core/subdomain_urls.py +++ /dev/null @@ -1,48 +0,0 @@ -from django.conf.urls import url, patterns - -from readthedocs.projects.constants import LANGUAGES_REGEX -from readthedocs.urls import urlpatterns as main_patterns - -handler500 = 'readthedocs.core.views.server_error' -handler404 = 'readthedocs.core.views.server_error_404' - -urlpatterns = patterns( - '', # base view, flake8 complains if it is on the previous line. - url((r'^projects/(?P[\w.-]+)/(?P%s)/' - r'(?P[\w.-]+)/(?P.*)$' % LANGUAGES_REGEX), - 'readthedocs.core.views.subproject_serve_docs', - name='subproject_docs_detail'), - - url(r'^projects/(?P[\w.-]+)', - 'readthedocs.core.views.subproject_serve_docs', - name='subproject_docs_detail'), - - url(r'^projects/$', - 'readthedocs.core.views.subproject_list', - name='subproject_docs_list'), - - url(r'^(?P%s)/(?P[\w.-]+)/(?P.*)$' % LANGUAGES_REGEX, - 'readthedocs.core.views.serve_docs', - name='docs_detail'), - - url(r'^(?P%s)/(?P.*)/$' % LANGUAGES_REGEX, - 'readthedocs.core.views.serve_docs', - {'filename': 'index.html'}, - name='docs_detail'), - - url(r'^page/(?P.*)$', - 'readthedocs.core.views.redirect_page_with_filename', - name='docs_detail'), - - url(r'^(?P%s)/$' % LANGUAGES_REGEX, - 'readthedocs.core.views.redirect_lang_slug', - name='lang_subdomain_handler'), - - url(r'^(?P.*)/$', - 'readthedocs.core.views.redirect_version_slug', - name='version_subdomain_handler'), - - url(r'^$', 'readthedocs.core.views.redirect_project_slug', name='homepage'), -) - -urlpatterns += main_patterns diff --git a/readthedocs/core/urls.py b/readthedocs/core/urls.py deleted file mode 100644 index d4ab8389e48..00000000000 --- a/readthedocs/core/urls.py +++ /dev/null @@ -1,102 +0,0 @@ -from django.conf.urls import url, patterns - -from readthedocs.constants import pattern_opts -from readthedocs.builds.filters import VersionFilter -from readthedocs.projects.feeds import LatestProjectsFeed, NewProjectsFeed -from readthedocs.projects.filters import ProjectFilter - - -docs_urls = patterns( - '', - # For serving docs locally and when nginx isn't - url((r'^docs/(?P{project_slug})/(?P{lang_slug})/' - r'(?P{version_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.serve_docs', - name='docs_detail'), - - # Redirect to default version, if only lang_slug is set. - url((r'^docs/(?P{project_slug})/' - r'(?P{lang_slug})/$'.format(**pattern_opts)), - 'readthedocs.core.views.redirect_lang_slug', - name='docs_detail'), - - # Redirect to default version, if only version_slug is set. - url((r'^docs/(?P{project_slug})/' - r'(?P{version_slug})/$'.format(**pattern_opts)), - 'readthedocs.core.views.redirect_version_slug', - name='docs_detail'), - - # Redirect to default version. - url(r'^docs/(?P{project_slug})/$'.format(**pattern_opts), - 'readthedocs.core.views.redirect_project_slug', - name='docs_detail'), - - # Handle /page/ redirects for explicit "latest" version goodness. - url((r'^docs/(?P{project_slug})/page/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.redirect_page_with_filename', - name='docs_detail'), - - # Handle single version URLs - url((r'^docs/(?P{project_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.serve_single_version_docs', - name='docs_detail'), - - # Handle fallbacks - url((r'^user_builds/(?P{project_slug})/rtd-builds/' - r'(?P{version_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.server_helpful_404', - name='user_builds_fallback'), - url((r'^user_builds/(?P{project_slug})/translations/' - r'(?P{lang_slug})/(?P{version_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.server_helpful_404', - name='user_builds_fallback_translations'), -) - - -core_urls = patterns( - '', - url(r'^github', 'readthedocs.core.views.github_build', name='github_build'), - url(r'^gitlab', 'readthedocs.core.views.gitlab_build', name='gitlab_build'), - url(r'^bitbucket', 'readthedocs.core.views.bitbucket_build', name='bitbucket_build'), - url((r'^build/' - r'(?P{project_slug})'.format(**pattern_opts)), - 'readthedocs.core.views.generic_build', - name='generic_build'), - url(r'^random/(?P{project_slug})'.format(**pattern_opts), - 'readthedocs.core.views.random_page', - name='random_page'), - url(r'^random/$', 'readthedocs.core.views.random_page', name='random_page'), - url(r'^500/$', 'readthedocs.core.views.divide_by_zero', name='divide_by_zero'), - url((r'^wipe/(?P{project_slug})/' - r'(?P{version_slug})/$'.format(**pattern_opts)), - 'readthedocs.core.views.wipe_version', - name='wipe_version'), -) - -deprecated_urls = patterns( - '', - url(r'^filter/version/$', - 'django_filters.views.object_filter', - {'filter_class': VersionFilter, 'template_name': 'filter.html'}, - name='filter_version'), - url(r'^filter/project/$', - 'django_filters.views.object_filter', - {'filter_class': ProjectFilter, 'template_name': 'filter.html'}, - name='filter_project'), - - url(r'^feeds/new/$', - NewProjectsFeed(), - name="new_feed"), - url(r'^feeds/latest/$', - LatestProjectsFeed(), - name="latest_feed"), - url((r'^mlt/(?P{project_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.morelikethis', - name='morelikethis'), -) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 4f4ea6a1eff..5cdc48d928c 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -131,134 +131,6 @@ def server_error_404(request, template_name='404.html'): return r -def server_helpful_404( - request, project_slug=None, lang_slug=None, version_slug=None, - filename=None, template_name='404.html'): - response = get_redirect_response(request, path=filename) - if response: - return response - pagename = re.sub( - r'/index$', r'', re.sub(r'\.html$', r'', re.sub(r'/$', r'', filename))) - suggestion = get_suggestion( - project_slug, lang_slug, version_slug, pagename, request.user) - r = render_to_response(template_name, - {'suggestion': suggestion}, - context_instance=RequestContext(request)) - r.status_code = 404 - return r - - -def get_suggestion(project_slug, lang_slug, version_slug, pagename, user): - """ - | # | project | version | language | What to show | - | 1 | 0 | 0 | 0 | Error message | - | 2 | 0 | 0 | 1 | Error message (Can't happen) | - | 3 | 0 | 1 | 0 | Error message (Can't happen) | - | 4 | 0 | 1 | 1 | Error message (Can't happen) | - | 5 | 1 | 0 | 0 | A link to top-level page of default version | - | 6 | 1 | 0 | 1 | Available versions on the translation project | - | 7 | 1 | 1 | 0 | Available translations of requested version | - | 8 | 1 | 1 | 1 | A link to top-level page of requested version | - """ - - suggestion = {} - if project_slug: - try: - proj = Project.objects.get(slug=project_slug) - if not lang_slug: - lang_slug = proj.language - try: - ver = Version.objects.get( - project__slug=project_slug, slug=version_slug) - except Version.DoesNotExist: - ver = None - - if ver: # if requested version is available on main project - if lang_slug != proj.language: - try: - translations = proj.translations.filter( - language=lang_slug) - if translations: - ver = Version.objects.get( - project__slug=translations[0].slug, slug=version_slug) - else: - ver = None - except Version.DoesNotExist: - ver = None - # if requested version is available on translation project too - if ver: - # Case #8: Show a link to top-level page of the version - suggestion['type'] = 'top' - suggestion['message'] = "What are you looking for?" - suggestion['href'] = proj.get_docs_url(ver.slug, lang_slug) - # requested version is available but not in requested language - else: - # Case #7: Show available translations of the version - suggestion['type'] = 'list' - suggestion['message'] = ( - "Requested page seems not to be translated in " - "requested language. But it's available in these " - "languages.") - suggestion['list'] = [] - suggestion['list'].append({ - 'label': proj.language, - 'project': proj, - 'version_slug': version_slug, - 'pagename': pagename - }) - for t in proj.translations.all(): - try: - Version.objects.get( - project__slug=t.slug, slug=version_slug) - suggestion['list'].append({ - 'label': t.language, - 'project': t, - 'version_slug': version_slug, - 'pagename': pagename - }) - except Version.DoesNotExist: - pass - else: # requested version does not exist on main project - if lang_slug == proj.language: - trans = proj - else: - translations = proj.translations.filter(language=lang_slug) - trans = translations[0] if translations else None - if trans: # requested language is available - # Case #6: Show available versions of the translation - suggestion['type'] = 'list' - suggestion['message'] = ( - "Requested version seems not to have been built yet. " - "But these versions are available.") - suggestion['list'] = [] - for v in Version.objects.public(user, trans, True): - suggestion['list'].append({ - 'label': v.slug, - 'project': trans, - 'version_slug': v.slug, - 'pagename': pagename - }) - # requested project exists but requested version and language - # are not available. - else: - # Case #5: Show a link to top-level page of default version - # of main project - suggestion['type'] = 'top' - suggestion['message'] = 'What are you looking for??' - suggestion['href'] = proj.get_docs_url() - except Project.DoesNotExist: - # Case #1-4: Show error mssage - suggestion['type'] = 'none' - suggestion[ - 'message'] = "We're sorry, we don't know what you're looking for" - else: - suggestion['type'] = 'none' - suggestion[ - 'message'] = "We're sorry, we don't know what you're looking for" - - return suggestion - - class SendEmailView(FormView): """Form view for sending emails to users from admin pages From 4f3879924e44a0d5def4f652182c2a2d686939a9 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 26 Apr 2016 15:22:23 -0700 Subject: [PATCH 06/43] Fix url tests --- readthedocs/rtd_tests/tests/test_urls.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_urls.py b/readthedocs/rtd_tests/tests/test_urls.py index 7c14cee643b..0cad35e13bd 100644 --- a/readthedocs/rtd_tests/tests/test_urls.py +++ b/readthedocs/rtd_tests/tests/test_urls.py @@ -9,48 +9,48 @@ class SubdomainUrlTests(TestCase): def test_sub_index(self): - url = reverse(readthedocs.core.views.redirect_project_slug, - urlconf='readthedocs.core.subdomain_urls') + url = reverse(readthedocs.core.views.serve.redirect_project_slug, + urlconf='readthedocs.core.urls.subdomain') self.assertEqual(url, '/') def test_sub_lang_version(self): - url = reverse('docs_detail', urlconf='readthedocs.core.subdomain_urls', + url = reverse('docs_detail', urlconf='readthedocs.core.urls.subdomain', kwargs={'lang_slug': 'en', 'version_slug': LATEST}) self.assertEqual(url, '/en/latest/') def test_sub_lang_version_filename(self): - url = reverse('docs_detail', urlconf='readthedocs.core.subdomain_urls', + url = reverse('docs_detail', urlconf='readthedocs.core.urls.subdomain', args=['en', 'latest', 'index.html']) self.assertEqual(url, '/en/latest/index.html') def test_sub_project_full_path(self): url = reverse('subproject_docs_detail', - urlconf='readthedocs.core.subdomain_urls', + urlconf='readthedocs.core.urls.subdomain', kwargs={'project_slug': 'pyramid', 'lang_slug': 'en', 'version_slug': LATEST, 'filename': 'index.html'}) self.assertEqual(url, '/projects/pyramid/en/latest/index.html') def test_sub_project_slug_only(self): url = reverse('subproject_docs_detail', - urlconf='readthedocs.core.subdomain_urls', + urlconf='readthedocs.core.urls.subdomain', kwargs={'project_slug': 'pyramid'}) self.assertEqual(url, '/projects/pyramid') def test_sub_page(self): url = reverse('docs_detail', - urlconf='readthedocs.core.subdomain_urls', + urlconf='readthedocs.core.urls.subdomain', kwargs={'filename': 'install.html'}) self.assertEqual(url, '/page/install.html') def test_sub_version(self): url = reverse('version_subdomain_handler', - urlconf='readthedocs.core.subdomain_urls', + urlconf='readthedocs.core.urls.subdomain', kwargs={'version_slug': '1.4.1'}) self.assertEqual(url, '/1.4.1/') def test_sub_lang(self): url = reverse('lang_subdomain_handler', - urlconf='readthedocs.core.subdomain_urls', + urlconf='readthedocs.core.urls.subdomain', kwargs={'lang_slug': 'en'}) self.assertEqual(url, '/en/') From 8584c1fbfa2ccd8db027cdf8486733255045cd1f Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 26 Apr 2016 15:41:45 -0700 Subject: [PATCH 07/43] Fix up tests --- readthedocs/core/views/__init__.py | 2 - readthedocs/core/views/serve.py | 43 +++++++----- .../rtd_tests/tests/test_doc_serving.py | 68 ++++++------------- 3 files changed, 45 insertions(+), 68 deletions(-) diff --git a/readthedocs/core/views/__init__.py b/readthedocs/core/views/__init__.py index 5cdc48d928c..f68dee39e79 100644 --- a/readthedocs/core/views/__init__.py +++ b/readthedocs/core/views/__init__.py @@ -3,7 +3,6 @@ documentation and header rendering, and server errors. """ -import re import os import logging @@ -27,7 +26,6 @@ from readthedocs.redirects.utils import get_redirect_response log = logging.getLogger(__name__) -pc_log = logging.getLogger(__name__ + '.post_commit') class NoProjectException(Exception): diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index eb7c65b0e4e..c037455e4cb 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -21,7 +21,7 @@ -------- PYTHON_MEDIA (False) - Set this to True to serve docs & media from Python -SERVE_PUBLIC_DOCS (False) - Set this to True to serve public as well as private docs from Python +SERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve. """ from django.conf import settings @@ -30,6 +30,7 @@ from django.template import RequestContext from django.views.static import serve +from readthedocs.projects import constants from readthedocs.projects.models import Project from readthedocs.core.symlink import PrivateSymlink, PublicSymlink from readthedocs.core.resolver import resolve, resolve_path @@ -115,30 +116,36 @@ def serve_symlink_docs(request, project, filename=''): filename += 'index.html' # This breaks path joining, by ignoring the root when given an "absolute" path - assert filename[0] is not '/' + if filename[0] == '/': + filename = filename[1:] - if settings.DEBUG or getattr(settings, 'SERVE_PUBLIC_DOCS', False): + log.info('Serving %s for %s' % (filename, project)) + + SERVE_DOCS = getattr(settings, 'SERVE_DOCS', [constants.PUBLIC]) + + if settings.DEBUG or constants.PUBLIC in SERVE_DOCS: public_symlink = PublicSymlink(project) basepath = public_symlink.project_root if os.path.exists(os.path.join(basepath, filename)): return _serve_file(request, filename, basepath) - # Handle private - private_symlink = PrivateSymlink(project) - basepath = private_symlink.project_root + if settings.DEBUG or constants.PRIVATE in SERVE_DOCS: - if not os.path.exists(os.path.join(basepath, filename)): - raise Http404('Path does not exist: %s' % filename) + # Handle private + private_symlink = PrivateSymlink(project) + basepath = private_symlink.project_root - # Only check permissions if we are about to serve an existing private file - if not AdminPermission.is_member(user=request.user, project=project): - # Do basic auth check on the project, but not the version - res = render_to_response('401.html', - context_instance=RequestContext(request)) - res.status_code = 401 - log.error('Unauthorized access to {0} documentation'.format(project.slug)) - return res + if os.path.exists(os.path.join(basepath, filename)): - log.info('Serving %s for %s' % (filename, project)) + # Only check permissions if we are about to serve an existing private file + if not AdminPermission.is_member(user=request.user, project=project): + # Do basic auth check on the project, but not the version + res = render_to_response('401.html', + context_instance=RequestContext(request)) + res.status_code = 401 + log.error('Unauthorized access to {0} documentation'.format(project.slug)) + return res + + return _serve_file(request, filename, basepath) - return _serve_file(request, filename, basepath) + raise Http404('No file serving method available.') diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 708c751ee6d..7dd7b6a7967 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -6,6 +6,7 @@ from django.test.utils import override_settings from readthedocs.rtd_tests.base import RequestFactoryTestMixin +from readthedocs.projects import constants from readthedocs.projects.models import Project from readthedocs.core.views.serve import serve_symlink_docs @@ -28,43 +29,33 @@ def setUp(self): self.public_url = '/docs/public/en/latest/usage.html' -@override_settings(SERVE_PUBLIC_DOCS=False) +@override_settings(SERVE_DOCS=[constants.PRIVATE]) class TestPrivateDocs(BaseDocServing): @override_settings(PYTHON_MEDIA=True) def test_private_python_media_serving(self): with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: - request = self.request(self.private_url, user=self.eric) - serve_symlink_docs(request, project=self.private, filename='en/latest/usage.html') - serve_mock.assert_called_with( - request, - 'en/latest/usage.html', - '/Users/eric/projects/readthedocs.org/private_web_root/private' - ) - - r = self.client.get(self.private_url) - self.assertEqual(r.status_code, 401) - - r = self.client.get(self.public_url) - self.assertEqual(r.status_code, 401) + with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): + request = self.request(self.private_url, user=self.eric) + serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + serve_mock.assert_called_with( + request, + 'en/latest/usage.html', + '/Users/eric/projects/readthedocs.org/private_web_root/private' + ) @override_settings(PYTHON_MEDIA=False) def test_private_nginx_serving(self): - request = self.request(self.private_url, user=self.eric) - r = serve_symlink_docs(request, project=self.private, filename='en/latest/usage.html') - self.assertEqual(r.status_code, 200) - self.assertEqual( - r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' - ) - - r = self.client.get(self.private_url) - self.assertEqual(r.status_code, 401) - - r = self.client.get(self.public_url) - self.assertEqual(r.status_code, 401) + with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): + request = self.request(self.private_url, user=self.eric) + r = serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + self.assertEqual(r.status_code, 200) + self.assertEqual( + r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' + ) -@override_settings(SERVE_PUBLIC_DOCS=True) +@override_settings(SERVE_DOCS=[constants.PRIVATE, constants.PUBLIC]) class TestPublicDocs(BaseDocServing): @override_settings(PYTHON_MEDIA=True) @@ -72,38 +63,19 @@ def test_public_python_media_serving(self): with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.public_url, user=self.eric) - serve_symlink_docs(request, project=self.public, filename='en/latest/usage.html') + serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') serve_mock.assert_called_with( request, 'en/latest/usage.html', '/Users/eric/projects/readthedocs.org/public_web_root/public' ) - with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): - r = self.client.get(self.public_url) - self.assertEqual(r.status_code, 200) - - with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: - r = self.client.get(self.public_url) - self.assertEqual(r.status_code, 200) - serve_mock.assert_called_with( - request, - 'en/latest/usage.html', - '/Users/eric/projects/readthedocs.org/public_web_root/public' - ) - @override_settings(PYTHON_MEDIA=False) def test_public_nginx_serving(self): with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.public_url, user=self.eric) - r = serve_symlink_docs(request, project=self.public, filename='en/latest/usage.html') + r = serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') self.assertEqual(r.status_code, 200) self.assertEqual( r._headers['x-accel-redirect'][1], '/public_web_root/public/en/latest/usage.html' ) - - r = self.client.get(self.public_url) - self.assertEqual(r.status_code, 401) - - r = self.client.get(self.public_url) - self.assertEqual(r.status_code, 401) From 0dead7447f0e70ec770829d46f334c6e75d863f6 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 26 Apr 2016 15:54:03 -0700 Subject: [PATCH 08/43] Test 404 handling --- readthedocs/core/views/serve.py | 27 ++++++++++++------- .../rtd_tests/tests/test_doc_serving.py | 17 ++++++++++++ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index c037455e4cb..2ec596c9271 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -108,6 +108,14 @@ def _serve_file(request, filename, basepath): return response +def _serve_401(request, project): + res = render_to_response('401.html', + context_instance=RequestContext(request)) + res.status_code = 401 + log.error('Unauthorized access to {0} documentation'.format(project.slug)) + return res + + @map_project_slug def serve_symlink_docs(request, project, filename=''): @@ -121,13 +129,17 @@ def serve_symlink_docs(request, project, filename=''): log.info('Serving %s for %s' % (filename, project)) - SERVE_DOCS = getattr(settings, 'SERVE_DOCS', [constants.PUBLIC]) + files_tried = [] + + SERVE_DOCS = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE]) if settings.DEBUG or constants.PUBLIC in SERVE_DOCS: public_symlink = PublicSymlink(project) basepath = public_symlink.project_root if os.path.exists(os.path.join(basepath, filename)): return _serve_file(request, filename, basepath) + else: + files_tried.append(os.path.join(basepath, filename)) if settings.DEBUG or constants.PRIVATE in SERVE_DOCS: @@ -137,15 +149,12 @@ def serve_symlink_docs(request, project, filename=''): if os.path.exists(os.path.join(basepath, filename)): - # Only check permissions if we are about to serve an existing private file + # Do basic auth check on the project, but not the version if not AdminPermission.is_member(user=request.user, project=project): - # Do basic auth check on the project, but not the version - res = render_to_response('401.html', - context_instance=RequestContext(request)) - res.status_code = 401 - log.error('Unauthorized access to {0} documentation'.format(project.slug)) - return res + return _serve_401(request, project) return _serve_file(request, filename, basepath) + else: + files_tried.append(os.path.join(basepath, filename)) - raise Http404('No file serving method available.') + raise Http404('File not found. Tried these files: %s' % ','.join(files_tried)) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 7dd7b6a7967..910603405c8 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -4,6 +4,7 @@ from django.contrib.auth.models import User from django.test import TestCase from django.test.utils import override_settings +from django.http import Http404 from readthedocs.rtd_tests.base import RequestFactoryTestMixin from readthedocs.projects import constants @@ -54,6 +55,14 @@ def test_private_nginx_serving(self): r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' ) + @override_settings(PYTHON_MEDIA=False) + def test_private_files_not_found(self): + request = self.request(self.private_url, user=self.eric) + with self.assertRaises(Http404) as exc: + serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + self.assertTrue('private_web_root' in exc.exception.message) + self.assertTrue('public_web_root' not in exc.exception.message) + @override_settings(SERVE_DOCS=[constants.PRIVATE, constants.PUBLIC]) class TestPublicDocs(BaseDocServing): @@ -79,3 +88,11 @@ def test_public_nginx_serving(self): self.assertEqual( r._headers['x-accel-redirect'][1], '/public_web_root/public/en/latest/usage.html' ) + + @override_settings(PYTHON_MEDIA=False) + def test_both_files_not_found(self): + request = self.request(self.private_url, user=self.eric) + with self.assertRaises(Http404) as exc: + serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + self.assertTrue('private_web_root' in exc.exception.message) + self.assertTrue('public_web_root' in exc.exception.message) From 29c957616567ca093eac971b9f6989701ad8b7d6 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 10:44:15 -0700 Subject: [PATCH 09/43] Serve media in subdomains during DEBUG --- readthedocs/core/urls/subdomain.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index d4debb46d10..4e8d58d47b1 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -1,11 +1,15 @@ +from operator import add + from django.conf.urls import url, patterns +from django.conf import settings +from django.conf.urls.static import static from readthedocs.urls import urlpatterns as main_patterns handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' -urlpatterns = patterns( +subdomain_urls = patterns( '', # base view, flake8 complains if it is on the previous line. url(r'^page/(?P.*)$', 'readthedocs.core.views.serve.redirect_page_with_filename', @@ -15,4 +19,11 @@ url(r'', 'readthedocs.core.views.serve.serve_symlink_docs', name='serve_symlink_docs'), ) -urlpatterns += main_patterns +groups = [subdomain_urls] + +if getattr(settings, 'DEBUG', False): + groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) + +groups.append(main_patterns) + +urlpatterns = reduce(add, groups) From 8e6958144c7b193d1389b9c51dbdccda18547c79 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 10:49:19 -0700 Subject: [PATCH 10/43] Add support for *.dev.readthedocs.io --- readthedocs/core/middleware.py | 11 ++++++++--- readthedocs/core/urls/single_version.py | 7 +++++-- readthedocs/core/urls/subdomain.py | 4 +++- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 0311a5a8d6a..3040de76474 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -22,6 +22,11 @@ 'SINGLE_VERSION_URLCONF', 'readthedocs.core.urls.single_version' ) +DEV_URL = getattr( + settings, + 'DEV_URL', + 'dev.readthedocs.io' +) class SubdomainMiddleware(object): @@ -43,10 +48,10 @@ def process_request(self, request): domain_parts = host.split('.') # Serve subdomains - but don't depend on the production domain only having 2 parts - if len(domain_parts) == len(public_domain.split('.')) + 1: + if len(domain_parts) == len(public_domain.split('.')) + 1 or DEV_URL in host: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - if not is_www and public_domain in host: + if not is_www and public_domain in host or DEV_URL in host: request.subdomain = True request.slug = subdomain request.urlconf = SUBDOMAIN_URLCONF @@ -102,7 +107,7 @@ def process_request(self, request): raise Http404(_('Invalid hostname')) # Google was finding crazy www.blah.readthedocs.org domains. # Block these explicitly after trying CNAME logic. - if len(domain_parts) > 3: + if len(domain_parts) > 3 and not settings.DEBUG: # Stop www.fooo.readthedocs.org if domain_parts[0] == 'www': log.debug(LOG_TEMPLATE.format(msg='404ing long domain', **log_kwargs)) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 5929d06201c..85f6b64f11e 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -1,14 +1,17 @@ from django.conf.urls import patterns, url +handler500 = 'readthedocs.core.views.server_error' +handler404 = 'readthedocs.core.views.server_error_404' + urlpatterns = patterns( '', # base view, flake8 complains if it is on the previous line. # Handle /docs on RTD domain url(r'^docs/(?P[-\w]+)/(?P.*)$', - 'readthedocs.core.views.serve.serve_docs', + 'readthedocs.core.views.serve.serve_symlink_docs', name='docs_detail'), # Handle subdomains url(r'^(?P.*)$', - 'readthedocs.core.views.serve.serve_docs', + 'readthedocs.core.views.serve.serve_symlink_docs', name='docs_detail'), ) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index 4e8d58d47b1..56f25e21509 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -16,7 +16,9 @@ name='docs_detail'), url(r'^$', 'readthedocs.core.views.serve.redirect_project_slug', name='redirect_project_slug'), - url(r'', 'readthedocs.core.views.serve.serve_symlink_docs', name='serve_symlink_docs'), + url(r'^(?P.*)$', + 'readthedocs.core.views.serve.serve_symlink_docs', + name='docs_detail'), ) groups = [subdomain_urls] From ecdbeb76ed32aa5b448a821ed58246b44bb10bd4 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 10:55:03 -0700 Subject: [PATCH 11/43] Handle media on subdomains. --- readthedocs/core/urls/single_version.py | 15 +++++++++++++-- readthedocs/core/urls/subdomain.py | 4 ---- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 85f6b64f11e..d2e3d08fb2f 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -1,9 +1,13 @@ -from django.conf.urls import patterns, url +from operator import add + +from django.conf.urls import url, patterns +from django.conf import settings +from django.conf.urls.static import static handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' -urlpatterns = patterns( +single_version_urls = patterns( '', # base view, flake8 complains if it is on the previous line. # Handle /docs on RTD domain url(r'^docs/(?P[-\w]+)/(?P.*)$', @@ -15,3 +19,10 @@ 'readthedocs.core.views.serve.serve_symlink_docs', name='docs_detail'), ) + +groups = [single_version_urls] + +if getattr(settings, 'DEBUG', False): + groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) + +urlpatterns = reduce(add, groups) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index 56f25e21509..b9311416004 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -4,8 +4,6 @@ from django.conf import settings from django.conf.urls.static import static -from readthedocs.urls import urlpatterns as main_patterns - handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' @@ -26,6 +24,4 @@ if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) -groups.append(main_patterns) - urlpatterns = reduce(add, groups) From 3a310d4e1c943fadcaa959654311dbbf843a04b9 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 10:55:12 -0700 Subject: [PATCH 12/43] Add default SERVE_DOCS --- readthedocs/settings/base.py | 1 + 1 file changed, 1 insertion(+) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 9d393ed81ff..fd7cb2b345e 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -266,6 +266,7 @@ def INSTALLED_APPS(self): # noqa REPO_LOCK_SECONDS = 30 ALLOW_PRIVATE_REPOS = False GROK_API_HOST = 'https://api.grokthedocs.com' + SERVE_DOCS = ['public'] # Haystack HAYSTACK_CONNECTIONS = { From f1a2ed504b20fd5f16749d193d0fde884f051a20 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 10:57:15 -0700 Subject: [PATCH 13/43] Fix a few tests --- readthedocs/rtd_tests/tests/test_urls.py | 12 ------------ readthedocs/rtd_tests/tests/test_views.py | 2 -- 2 files changed, 14 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_urls.py b/readthedocs/rtd_tests/tests/test_urls.py index 0cad35e13bd..8014e35c07d 100644 --- a/readthedocs/rtd_tests/tests/test_urls.py +++ b/readthedocs/rtd_tests/tests/test_urls.py @@ -42,18 +42,6 @@ def test_sub_page(self): kwargs={'filename': 'install.html'}) self.assertEqual(url, '/page/install.html') - def test_sub_version(self): - url = reverse('version_subdomain_handler', - urlconf='readthedocs.core.urls.subdomain', - kwargs={'version_slug': '1.4.1'}) - self.assertEqual(url, '/1.4.1/') - - def test_sub_lang(self): - url = reverse('lang_subdomain_handler', - urlconf='readthedocs.core.urls.subdomain', - kwargs={'lang_slug': 'en'}) - self.assertEqual(url, '/en/') - class WipeUrlTests(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index 5e54756fbe2..6e22f30d003 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -44,8 +44,6 @@ def test_imported_docs(self): _ = form.save() _ = Project.objects.get(slug='django-kong') - r = self.client.get('/docs/django-kong/en/latest/', {}) - self.assertEqual(r.status_code, 200) r = self.client.get('/dashboard/django-kong/versions/', {}) self.assertEqual(r.status_code, 200) r = self.client.get('/projects/django-kong/builds/') From 18e3f88460c49d03f47fb3bcc0a44eb8d4c4701b Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 11:05:09 -0700 Subject: [PATCH 14/43] Remove unused tests --- readthedocs/rtd_tests/tests/test_404.py | 59 ------------------------ readthedocs/rtd_tests/tests/test_urls.py | 40 ---------------- 2 files changed, 99 deletions(-) delete mode 100644 readthedocs/rtd_tests/tests/test_404.py diff --git a/readthedocs/rtd_tests/tests/test_404.py b/readthedocs/rtd_tests/tests/test_404.py deleted file mode 100644 index af2f5579e74..00000000000 --- a/readthedocs/rtd_tests/tests/test_404.py +++ /dev/null @@ -1,59 +0,0 @@ -import lxml.html - -from django.test import TestCase - -from readthedocs.projects.models import Project - - -class Testmaker(TestCase): - fixtures = ["eric", "test_data"] - - def setUp(self): - self.client.login(username='eric', password='test') - self.pip = Project.objects.get(slug='pip') - self.latest = self.pip.versions.create_latest() - self.pip_es = Project.objects.create(name="PIP-ES", slug='pip-es', language='es', main_language_project=self.pip) - - def test_project_does_not_exist(self): - # Case 1-4: Project doesn't exist - r = self.client.get('/docs/nonexistent_proj/en/nonexistent_dir/subdir/bogus.html') - self.assertContains(r, '''

We're sorry, we don't know what you're looking for

''', status_code=404, html=False) - - def test_only_project_exist(self): - # Case 5: Project exists but both of version and language are not available - r = self.client.get('/docs/pip/fr/nonexistent_ver/nonexistent_dir/bogus.html', {}) - self.assertContains(r, '

What are you looking for??

', status_code=404, html=False) - - def test_not_built_in_main_language(self): - # Case 6: Project exists and main language is available but the version is not available - r = self.client.get('/docs/pip/en/nonexistent_ver/nonexistent_dir/bogus.html', {}) - self.assertContains(r, '

Requested version seems not to have been built yet.', status_code=404, html=False) - - def test_not_built_in_other_language(self): - # Case 6: Project exists and translation is available but the version is not available - r = self.client.get('/docs/pip/es/nonexistent_ver/nonexistent_dir/bogus.html', {}) - self.assertContains(r, '

Requested version seems not to have been built yet.', status_code=404, html=False) - - def test_not_translated(self): - # Case 7: Project exists and the version is available but not in the language - r = self.client.get('/docs/pip/fr/latest/nonexistent_dir/bogus.html', {}) - self.assertEqual(r.status_code, 200) - self.assertEqual(r['X-Accel-Redirect'], '/user_builds/pip/translations/fr/latest/nonexistent_dir/bogus.html') - r = self.client.get('/user_builds/pip/translations/fr/latest/nonexistent_dir/bogus.html', {}) - self.assertContains(r, '

Requested page seems not to be translated in requested language.', status_code=404, html=False) - - def test_no_dir_or_file_in_main_language(self): - # Case 8: Everything is OK but sub-dir or file doesn't exist - r = self.client.get('/docs/pip/en/latest/nonexistent_dir/bogus.html', {}) - self.assertEqual(r.status_code, 200) - self.assertEqual(r['X-Accel-Redirect'], '/user_builds/pip/rtd-builds/latest/nonexistent_dir/bogus.html') - r = self.client.get('/user_builds/pip/rtd-builds/latest/nonexistent_dir/bogus.html', {}) - self.assertContains(r, '

What are you looking for?

', status_code=404, html=False) - - def test_no_dir_or_file_in_other_language(self): - # Case 8: Everything is OK but sub-dir or file doesn't exist - r = self.client.get('/docs/pip/es/latest/nonexistent_dir/bogus.html', {}) - self.assertEqual(r.status_code, 200) - self.assertEqual(r['X-Accel-Redirect'], '/user_builds/pip/translations/es/latest/nonexistent_dir/bogus.html') - r = self.client.get('/user_builds/pip/translations/es/latest/nonexistent_dir/bogus.html', {}) - self.assertContains(r, '

What are you looking for?

', status_code=404, html=False) diff --git a/readthedocs/rtd_tests/tests/test_urls.py b/readthedocs/rtd_tests/tests/test_urls.py index 8014e35c07d..e5a8b7a5646 100644 --- a/readthedocs/rtd_tests/tests/test_urls.py +++ b/readthedocs/rtd_tests/tests/test_urls.py @@ -2,46 +2,6 @@ from django.core.urlresolvers import NoReverseMatch from django.test import TestCase -from readthedocs.builds.constants import LATEST -import readthedocs.core.views - - -class SubdomainUrlTests(TestCase): - - def test_sub_index(self): - url = reverse(readthedocs.core.views.serve.redirect_project_slug, - urlconf='readthedocs.core.urls.subdomain') - self.assertEqual(url, '/') - - def test_sub_lang_version(self): - url = reverse('docs_detail', urlconf='readthedocs.core.urls.subdomain', - kwargs={'lang_slug': 'en', 'version_slug': LATEST}) - self.assertEqual(url, '/en/latest/') - - def test_sub_lang_version_filename(self): - url = reverse('docs_detail', urlconf='readthedocs.core.urls.subdomain', - args=['en', 'latest', 'index.html']) - self.assertEqual(url, '/en/latest/index.html') - - def test_sub_project_full_path(self): - url = reverse('subproject_docs_detail', - urlconf='readthedocs.core.urls.subdomain', - kwargs={'project_slug': 'pyramid', 'lang_slug': 'en', - 'version_slug': LATEST, 'filename': 'index.html'}) - self.assertEqual(url, '/projects/pyramid/en/latest/index.html') - - def test_sub_project_slug_only(self): - url = reverse('subproject_docs_detail', - urlconf='readthedocs.core.urls.subdomain', - kwargs={'project_slug': 'pyramid'}) - self.assertEqual(url, '/projects/pyramid') - - def test_sub_page(self): - url = reverse('docs_detail', - urlconf='readthedocs.core.urls.subdomain', - kwargs={'filename': 'install.html'}) - self.assertEqual(url, '/page/install.html') - class WipeUrlTests(TestCase): From c6b4327e41359ce465367e5980e8b879ba69cceb Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 11:26:55 -0700 Subject: [PATCH 15/43] More test fixing --- readthedocs/core/middleware.py | 4 +- readthedocs/rtd_tests/tests/test_bookmarks.py | 143 ------------------ readthedocs/rtd_tests/tests/test_core_tags.py | 3 + .../rtd_tests/tests/test_doc_serving.py | 5 +- readthedocs/rtd_tests/tests/test_privacy.py | 2 +- readthedocs/rtd_tests/tests/test_redirects.py | 17 +-- .../rtd_tests/tests/test_single_version.py | 20 --- 7 files changed, 17 insertions(+), 177 deletions(-) delete mode 100644 readthedocs/rtd_tests/tests/test_bookmarks.py diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 3040de76474..f2d9d79da20 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -5,7 +5,7 @@ from django.contrib.sessions.middleware import SessionMiddleware from django.core.cache import cache from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned -from django.http import Http404 +from django.http import Http404, HttpResponseBadRequest from readthedocs.projects.models import Project, Domain @@ -111,7 +111,7 @@ def process_request(self, request): # Stop www.fooo.readthedocs.org if domain_parts[0] == 'www': log.debug(LOG_TEMPLATE.format(msg='404ing long domain', **log_kwargs)) - raise Http404(_('Invalid hostname')) + return HttpResponseBadRequest(_('Invalid hostname')) log.debug(LOG_TEMPLATE.format(msg='Allowing long domain name', **log_kwargs)) # raise Http404(_('Invalid hostname')) # Normal request. diff --git a/readthedocs/rtd_tests/tests/test_bookmarks.py b/readthedocs/rtd_tests/tests/test_bookmarks.py deleted file mode 100644 index e2184e9dd53..00000000000 --- a/readthedocs/rtd_tests/tests/test_bookmarks.py +++ /dev/null @@ -1,143 +0,0 @@ -from django.test import TestCase -from django.core.urlresolvers import reverse -from django.contrib.auth.models import User -import json - -from readthedocs.builds.constants import LATEST -from readthedocs.projects.models import Project -from readthedocs.bookmarks.models import Bookmark - - -class TestBookmarks(TestCase): - fixtures = ['eric', 'test_data'] - - def setUp(self): - self.project = Project.objects.get(pk=1) - self.client.login(username='eric', password='test') - self.user = User.objects.get(pk=1) - self.project.users.add(self.user) - - def __add_bookmark(self): - post_data = { - "project": self.project.slug, - "version": LATEST, - "page": "", - "url": "", - } - - response = self.client.post( - reverse('bookmarks_add'), - data=json.dumps(post_data), - content_type="application/json" - ) - - self.assertEqual(response.status_code, 201) - self.assertContains(response, 'added', status_code=201) - return Bookmark.objects.get(pk=1) - - def test_add_bookmark_denies_get_requests(self): - response = self.client.get(reverse('bookmarks_add')) - self.assertEqual(response.status_code, 405) - self.assertContains(response, 'error', status_code=405) - - def test_add_bookmark(self): - bookmark = self.__add_bookmark() - self.assertEqual(bookmark.user, self.user) - self.assertEqual(bookmark.project.slug, self.project.slug) - self.assertEqual(Bookmark.objects.count(), 1) - - def test_add_bookmark_fails_bad_data(self): - post_data = { - "project": 'fail', "version": 'fail', "page": "", "url": "" - } - response = self.client.post( - reverse('bookmarks_add'), - data=json.dumps(post_data), - content_type="text/javascript" - ) - - self.assertEqual(response.status_code, 400) - self.assertContains(response, 'error', status_code=400) - - def test_delete_bookmark_with_get_renders_confirmation_page(self): - self.__add_bookmark() - response = self.client.get( - reverse('bookmark_remove', kwargs={'bookmark_pk': '1'}) - ) - - self.assertEqual(response.status_code, 200) - self.assertContains(response, 'You sure? O_o') - - def test_delete_bookmark_with_url(self): - self.__add_bookmark() - response = self.client.post( - reverse('bookmark_remove', kwargs={'bookmark_pk': '1'}) - ) - self.assertRedirects(response, reverse('bookmark_list')) - self.assertEqual(Bookmark.objects.count(), 0) - - def test_delete_bookmark_with_json(self): - self.__add_bookmark() - - post_data = { - "project": self.project.slug, - "version": LATEST, - "page": "", - "url": "", - } - - response = self.client.post( - reverse('bookmark_remove_json'), - data=json.dumps(post_data), - content_type="application/json" - ) - - self.assertEqual(response.status_code, 200) - self.assertEqual(Bookmark.objects.count(), 0) - - def test_dont_delete_bookmarks_with_bad_json(self): - self.__add_bookmark() - post_data = {"bad project": self.project.slug} - - response = self.client.post( - reverse('bookmark_remove_json'), - data=json.dumps(post_data), - content_type="application/json" - ) - self.assertEqual(response.status_code, 400) - self.assertEqual(Bookmark.objects.count(), 1) - self.assertContains(response, "Invalid parameters", status_code=400) - - def test_bookmark_exists_true_when_exists(self): - bookmark = self.__add_bookmark() - post_data = { - 'project': bookmark.project.slug, - 'version': bookmark.version.slug, - 'page': bookmark.page - } - - response = self.client.post( - reverse('bookmark_exists'), - data=json.dumps(post_data), - content_type="application/json" - ) - - self.assertEqual(response.status_code, 200) - self.assertTrue(json.loads(response.content)['exists']) - - def test_bookmark_exists_404_when_does_not_exist(self): - response = self.client.post( - reverse('bookmark_exists'), - data=json.dumps( - {'project': 'dont-read-docs', 'version': '', 'page': ''} - ), - content_type="application/json" - ) - - self.assertEqual(response.status_code, 404) - self.assertFalse(json.loads(response.content)['exists']) - - def test_bookmark_exists_forbids_GET(self): - response = self.client.get(reverse('bookmark_exists')) - self.assertEqual(response.status_code, 405) - self.assertNotContains(response, 'exists', status_code=405) diff --git a/readthedocs/rtd_tests/tests/test_core_tags.py b/readthedocs/rtd_tests/tests/test_core_tags.py index 70c0941ad30..2ffb141b02e 100644 --- a/readthedocs/rtd_tests/tests/test_core_tags.py +++ b/readthedocs/rtd_tests/tests/test_core_tags.py @@ -1,11 +1,14 @@ import mock from django.test import TestCase +from django.test.utils import override_settings + from readthedocs.projects.models import Project from readthedocs.builds.constants import LATEST from readthedocs.core.templatetags import core_tags +@override_settings(USE_SUBDOMAIN=False, PUBLIC_DOMAIN='readthedocs.org') class CoreTagsTests(TestCase): fixtures = ["eric", "test_data"] diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 910603405c8..0b57131eedd 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -5,6 +5,7 @@ from django.test import TestCase from django.test.utils import override_settings from django.http import Http404 +from django.conf import settings from readthedocs.rtd_tests.base import RequestFactoryTestMixin from readthedocs.projects import constants @@ -42,7 +43,7 @@ def test_private_python_media_serving(self): serve_mock.assert_called_with( request, 'en/latest/usage.html', - '/Users/eric/projects/readthedocs.org/private_web_root/private' + settings.SITE_ROOT + '/private_web_root/private' ) @override_settings(PYTHON_MEDIA=False) @@ -76,7 +77,7 @@ def test_public_python_media_serving(self): serve_mock.assert_called_with( request, 'en/latest/usage.html', - '/Users/eric/projects/readthedocs.org/public_web_root/public' + settings.SITE_ROOT + '/private_web_root/private' ) @override_settings(PYTHON_MEDIA=False) diff --git a/readthedocs/rtd_tests/tests/test_privacy.py b/readthedocs/rtd_tests/tests/test_privacy.py index 13491272c18..78abd561371 100644 --- a/readthedocs/rtd_tests/tests/test_privacy.py +++ b/readthedocs/rtd_tests/tests/test_privacy.py @@ -203,7 +203,7 @@ def test_private_doc_serving(self): 'privacy-test-slug': 'private'}) r = self.client.get('/docs/django-kong/en/test-slug/') self.client.login(username='eric', password='test') - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, 404) # Make sure it doesn't show up as tester self.client.login(username='tester', password='test') diff --git a/readthedocs/rtd_tests/tests/test_redirects.py b/readthedocs/rtd_tests/tests/test_redirects.py index bda28fdf755..e95612498d4 100644 --- a/readthedocs/rtd_tests/tests/test_redirects.py +++ b/readthedocs/rtd_tests/tests/test_redirects.py @@ -14,6 +14,7 @@ import logging +@override_settings(PUBLIC_DOMAIN='readthedocs.org') class RedirectTests(TestCase): fixtures = ["eric", "test_data"] @@ -40,7 +41,7 @@ def test_proper_url_no_slash(self): # This is triggered by Django, so its a 301, basically just # APPEND_SLASH self.assertEqual(r.status_code, 301) - self.assertEqual(r['Location'], 'http://testserver/docs/pip/') + self.assertEqual(r['Location'], 'http://readthedocs.org/docs/pip/') r = self.client.get(r['Location']) self.assertEqual(r.status_code, 302) r = self.client.get(r['Location']) @@ -50,7 +51,7 @@ def test_proper_url(self): r = self.client.get('/docs/pip/') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://testserver/docs/pip/en/latest/') + r['Location'], 'http://readthedocs.org/docs/pip/en/latest/') r = self.client.get(r['Location']) self.assertEqual(r.status_code, 200) @@ -58,7 +59,7 @@ def test_proper_url_with_lang_slug_only(self): r = self.client.get('/docs/pip/en/') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://testserver/docs/pip/en/latest/') + r['Location'], 'http://readthedocs.org/docs/pip/en/latest/') r = self.client.get(r['Location']) self.assertEqual(r.status_code, 200) @@ -75,7 +76,7 @@ def test_proper_page_on_main_site(self): r = self.client.get('/docs/pip/page/test.html') self.assertEqual(r.status_code, 302) self.assertEqual(r['Location'], - 'http://testserver/docs/pip/en/latest/test.html') + 'http://readthedocs.org/docs/pip/en/latest/test.html') r = self.client.get(r['Location']) self.assertEqual(r.status_code, 200) @@ -83,7 +84,7 @@ def test_proper_url_with_version_slug_only(self): r = self.client.get('/docs/pip/latest/') self.assertEqual(r.status_code, 302) self.assertEqual( - r['Location'], 'http://testserver/docs/pip/en/latest/') + r['Location'], 'http://readthedocs.org/docs/pip/en/latest/') r = self.client.get(r['Location']) self.assertEqual(r.status_code, 200) @@ -91,8 +92,6 @@ def test_proper_url_with_version_slug_only(self): # TODO: This should 404 directly, not redirect first def test_improper_url_with_nonexistent_slug(self): r = self.client.get('/docs/pip/nonexistent/') - self.assertEqual(r.status_code, 302) - r = self.client.get(r['Location']) self.assertEqual(r.status_code, 404) def test_improper_url_filename_only(self): @@ -297,12 +296,12 @@ def setUp(self): def test_redirect_list(self): r = self.client.get('/builds/project-1/') self.assertEqual(r.status_code, 301) - self.assertEqual(r['Location'], 'http://testserver/projects/project-1/builds/') + self.assertEqual(r['Location'], 'http://readthedocs.org/projects/project-1/builds/') def test_redirect_detail(self): r = self.client.get('/builds/project-1/1337/') self.assertEqual(r.status_code, 301) - self.assertEqual(r['Location'], 'http://testserver/projects/project-1/builds/1337/') + self.assertEqual(r['Location'], 'http://readthedocs.org/projects/project-1/builds/1337/') class GetFullPathTests(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_single_version.py b/readthedocs/rtd_tests/tests/test_single_version.py index bc715421b4f..b4b56880815 100644 --- a/readthedocs/rtd_tests/tests/test_single_version.py +++ b/readthedocs/rtd_tests/tests/test_single_version.py @@ -14,26 +14,6 @@ class RedirectSingleVersionTests(TestCase): def setUp(self): self.pip = fixture.get(Project, slug='pip', single_version=True, main_language_project=None) - def test_proper_single_version_url_full_with_filename(self): - with override_settings(USE_SUBDOMAIN=False): - r = self.client.get('/docs/pip/usage.html') - self.assertEqual(r.status_code, 200) - - def test_improper_single_version_url_nonexistent_project(self): - with override_settings(USE_SUBDOMAIN=False): - r = self.client.get('/docs/nonexistent/blah.html') - self.assertEqual(r.status_code, 404) - - def test_proper_single_version_url_subdomain(self): - r = self.client.get('/usage.html', - HTTP_HOST='pip.public.readthedocs.org') - self.assertEqual(r.status_code, 200) - - def test_improper_single_version_url_subdomain(self): - r = self.client.get('/blah.html', - HTTP_HOST='nonexistent.public.readthedocs.org') - self.assertEqual(r.status_code, 404) - def test_docs_url_generation(self): with override_settings(USE_SUBDOMAIN=False): self.assertEqual(self.pip.get_docs_url(), From 99f368233d2f848f52cb27c17a6a5731afb67769 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 11:30:23 -0700 Subject: [PATCH 16/43] Fix linting error --- readthedocs/core/views/serve.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 2ec596c9271..0fc5adb9c38 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -131,9 +131,9 @@ def serve_symlink_docs(request, project, filename=''): files_tried = [] - SERVE_DOCS = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE]) + serve_docs = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE]) - if settings.DEBUG or constants.PUBLIC in SERVE_DOCS: + if settings.DEBUG or constants.PUBLIC in serve_docs: public_symlink = PublicSymlink(project) basepath = public_symlink.project_root if os.path.exists(os.path.join(basepath, filename)): @@ -141,7 +141,7 @@ def serve_symlink_docs(request, project, filename=''): else: files_tried.append(os.path.join(basepath, filename)) - if settings.DEBUG or constants.PRIVATE in SERVE_DOCS: + if settings.DEBUG or constants.PRIVATE in serve_docs: # Handle private private_symlink = PrivateSymlink(project) From 9c1c3ab81e78f4486570c268a965fdad0bfdf945 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 12:17:07 -0700 Subject: [PATCH 17/43] Fix redirects --- readthedocs/core/urls/__init__.py | 5 + readthedocs/redirects/models.py | 35 +----- .../rtd_tests/tests/test_doc_serving.py | 2 +- readthedocs/rtd_tests/tests/test_redirects.py | 105 +++--------------- 4 files changed, 29 insertions(+), 118 deletions(-) diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py index e5851e7fd69..dbbf8a3f9ec 100644 --- a/readthedocs/core/urls/__init__.py +++ b/readthedocs/core/urls/__init__.py @@ -15,6 +15,11 @@ 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), + # Handle single version URLs + url((r'^docs/(?P{project_slug})/$'.format(**pattern_opts)), + 'readthedocs.core.views.serve.redirect_project_slug', + name='docs_detail'), + # Handle single version URLs url((r'^docs/(?P{project_slug})/' r'(?P{filename_slug})$'.format(**pattern_opts)), diff --git a/readthedocs/redirects/models.py b/readthedocs/redirects/models.py index 69289871e50..90aca33f941 100644 --- a/readthedocs/redirects/models.py +++ b/readthedocs/redirects/models.py @@ -1,11 +1,10 @@ -from django.conf import settings -from django.core.urlresolvers import reverse from django.db import models from django.utils.translation import ugettext from django.utils.translation import ugettext_lazy as _ import logging import re +from readthedocs.core.resolver import resolve_path from readthedocs.projects.models import Project from .managers import RedirectManager @@ -89,34 +88,10 @@ def get_full_path(self, filename, language=None, version_slug=None): if re.match('^https?://', filename): return filename - url_kwargs = { - 'project_slug': self.project.slug, - 'filename': filename, - } - - if not self.project.single_version: - if language is None: - language = self.project.language - if version_slug is None: - version_slug = self.project.get_default_version() - else: - versions = self.project.versions.all() - if not versions.filter(slug=version_slug).exists(): - version_slug = self.project.get_default_version() - url_kwargs.update({ - 'lang_slug': language, - 'version_slug': version_slug, - }) - - use_subdomain = getattr(settings, 'USE_SUBDOMAIN', False) - if use_subdomain: - if self.project.single_version: - return "/{filename}".format(**url_kwargs) - else: - return "/{lang_slug}/{version_slug}/{filename}".format( - **url_kwargs) - - return reverse('docs_detail', kwargs=url_kwargs) + return resolve_path( + project=self.project, language=language, + version_slug=version_slug, filename=filename + ) def get_redirect_path(self, path, language=None, version_slug=None): method = getattr(self, 'redirect_{type}'.format( diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 0b57131eedd..c0fc7ff6b5d 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -77,7 +77,7 @@ def test_public_python_media_serving(self): serve_mock.assert_called_with( request, 'en/latest/usage.html', - settings.SITE_ROOT + '/private_web_root/private' + settings.SITE_ROOT + '/public_web_root/private' ) @override_settings(PYTHON_MEDIA=False) diff --git a/readthedocs/rtd_tests/tests/test_redirects.py b/readthedocs/rtd_tests/tests/test_redirects.py index e95612498d4..a8565c9f7f4 100644 --- a/readthedocs/rtd_tests/tests/test_redirects.py +++ b/readthedocs/rtd_tests/tests/test_redirects.py @@ -14,7 +14,7 @@ import logging -@override_settings(PUBLIC_DOMAIN='readthedocs.org') +@override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) class RedirectTests(TestCase): fixtures = ["eric", "test_data"] @@ -41,35 +41,15 @@ def test_proper_url_no_slash(self): # This is triggered by Django, so its a 301, basically just # APPEND_SLASH self.assertEqual(r.status_code, 301) - self.assertEqual(r['Location'], 'http://readthedocs.org/docs/pip/') + self.assertEqual(r['Location'], 'http://testserver/docs/pip/') r = self.client.get(r['Location']) self.assertEqual(r.status_code, 302) - r = self.client.get(r['Location']) - self.assertEqual(r.status_code, 200) def test_proper_url(self): r = self.client.get('/docs/pip/') self.assertEqual(r.status_code, 302) self.assertEqual( r['Location'], 'http://readthedocs.org/docs/pip/en/latest/') - r = self.client.get(r['Location']) - self.assertEqual(r.status_code, 200) - - def test_proper_url_with_lang_slug_only(self): - r = self.client.get('/docs/pip/en/') - self.assertEqual(r.status_code, 302) - self.assertEqual( - r['Location'], 'http://readthedocs.org/docs/pip/en/latest/') - r = self.client.get(r['Location']) - self.assertEqual(r.status_code, 200) - - def test_proper_url_full(self): - r = self.client.get('/docs/pip/en/latest/') - self.assertEqual(r.status_code, 200) - - def test_proper_url_full_with_filename(self): - r = self.client.get('/docs/pip/en/latest/test.html') - self.assertEqual(r.status_code, 200) # Specific Page Redirects def test_proper_page_on_main_site(self): @@ -77,16 +57,6 @@ def test_proper_page_on_main_site(self): self.assertEqual(r.status_code, 302) self.assertEqual(r['Location'], 'http://readthedocs.org/docs/pip/en/latest/test.html') - r = self.client.get(r['Location']) - self.assertEqual(r.status_code, 200) - - def test_proper_url_with_version_slug_only(self): - r = self.client.get('/docs/pip/latest/') - self.assertEqual(r.status_code, 302) - self.assertEqual( - r['Location'], 'http://readthedocs.org/docs/pip/en/latest/') - r = self.client.get(r['Location']) - self.assertEqual(r.status_code, 200) # If slug is neither valid lang nor valid version, it should 404. # TODO: This should 404 directly, not redirect first @@ -126,24 +96,6 @@ def test_proper_subdomain(self): self.assertEqual( r['Location'], 'http://pip.readthedocs.org/en/latest/') - @override_settings(USE_SUBDOMAIN=True) - def test_proper_subdomain_with_lang_slug_only(self): - r = self.client.get('/en/', HTTP_HOST='pip.readthedocs.org') - self.assertEqual(r.status_code, 302) - self.assertEqual( - r['Location'], 'http://pip.readthedocs.org/en/latest/') - - @override_settings(USE_SUBDOMAIN=True) - def test_proper_subdomain_and_url(self): - r = self.client.get('/en/latest/', HTTP_HOST='pip.readthedocs.org') - self.assertEqual(r.status_code, 200) - - @override_settings(USE_SUBDOMAIN=True) - def test_proper_subdomain_and_url_with_filename(self): - r = self.client.get( - '/en/latest/test.html', HTTP_HOST='pip.readthedocs.org') - self.assertEqual(r.status_code, 200) - # Specific Page Redirects @override_settings(USE_SUBDOMAIN=True) def test_proper_page_on_subdomain(self): @@ -152,39 +104,13 @@ def test_proper_page_on_subdomain(self): self.assertEqual(r['Location'], 'http://pip.readthedocs.org/en/latest/test.html') - # When there's only a version slug, the redirect prepends the lang slug - @override_settings(USE_SUBDOMAIN=True) - def test_proper_subdomain_with_version_slug_only(self): - r = self.client.get('/1.4.1/', HTTP_HOST='pip.readthedocs.org') - self.assertEqual(r.status_code, 302) - self.assertEqual(r['Location'], - 'http://pip.readthedocs.org/en/1.4.1/') - @override_settings(USE_SUBDOMAIN=True) def test_improper_subdomain_filename_only(self): r = self.client.get('/test.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 404) -class RedirectUnderscoreTests(TestCase): - fixtures = ["eric", "test_data"] - - def setUp(self): - logging.disable(logging.DEBUG) - self.client.login(username='eric', password='test') - whatup = Project.objects.create( - slug='what_up', name='What Up Underscore') - - # Test _ -> - slug lookup - @override_settings(USE_SUBDOMAIN=True) - def test_underscore_redirect(self): - r = self.client.get('/', - HTTP_HOST='what-up.readthedocs.org') - self.assertEqual(r.status_code, 302) - self.assertEqual( - r['Location'], 'http://what-up.readthedocs.org/en/latest/') - - +@override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) class RedirectAppTests(TestCase): fixtures = ["eric", "test_data"] @@ -217,7 +143,9 @@ def test_redirect_root(self): @override_settings(USE_SUBDOMAIN=True) def test_redirect_page(self): Redirect.objects.create( - project=self.pip, redirect_type='page', from_url='/install.html', to_url='/tutorial/install.html') + project=self.pip, redirect_type='page', + from_url='/install.html', to_url='/tutorial/install.html' + ) r = self.client.get('/install.html', HTTP_HOST='pip.readthedocs.org') self.assertEqual(r.status_code, 302) self.assertEqual( @@ -228,7 +156,7 @@ def test_redirect_keeps_version_number(self): Redirect.objects.create( project=self.pip, redirect_type='page', from_url='/how_to_install.html', to_url='/install.html') - with patch('readthedocs.core.views._serve_docs') as _serve_docs: + with patch('readthedocs.core.views.serve.serve_symlink_docs') as _serve_docs: _serve_docs.side_effect = Http404() r = self.client.get('/en/0.8.1/how_to_install.html', HTTP_HOST='pip.readthedocs.org') @@ -242,7 +170,7 @@ def test_redirect_keeps_language(self): Redirect.objects.create( project=self.pip, redirect_type='page', from_url='/how_to_install.html', to_url='/install.html') - with patch('readthedocs.core.views._serve_docs') as _serve_docs: + with patch('readthedocs.core.views.serve.serve_symlink_docs') as _serve_docs: _serve_docs.side_effect = Http404() r = self.client.get('/de/0.8.1/how_to_install.html', HTTP_HOST='pip.readthedocs.org') @@ -282,28 +210,31 @@ def test_redirect_htmldir(self): self.assertEqual( r['Location'], 'http://pip.readthedocs.org/en/latest/faq/') + +@override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) class RedirectBuildTests(TestCase): fixtures = ["eric", "test_data"] def setUp(self): self.project = get(Project, - slug='project-1', - documentation_type='sphinx', - conf_py_file='test_conf.py', - versions=[fixture()]) + slug='project-1', + documentation_type='sphinx', + conf_py_file='test_conf.py', + versions=[fixture()]) self.version = self.project.versions.all()[0] def test_redirect_list(self): r = self.client.get('/builds/project-1/') self.assertEqual(r.status_code, 301) - self.assertEqual(r['Location'], 'http://readthedocs.org/projects/project-1/builds/') + self.assertEqual(r['Location'], 'http://testserver/projects/project-1/builds/') def test_redirect_detail(self): r = self.client.get('/builds/project-1/1337/') self.assertEqual(r.status_code, 301) - self.assertEqual(r['Location'], 'http://readthedocs.org/projects/project-1/builds/1337/') + self.assertEqual(r['Location'], 'http://testserver/projects/project-1/builds/1337/') +@override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) class GetFullPathTests(TestCase): fixtures = ["eric", "test_data"] @@ -320,7 +251,7 @@ def test_http_filenames_return_themselves(self): def test_redirects_no_subdomain(self): self.assertEqual( self.redirect.get_full_path('index.html'), - '/docs/read-the-docs/en/latest/index.html' + '/docs/read-the-docs/en/latest/' ) @override_settings( From a055eaf3a71eca4600c3b6cb65a3bceaee242e5a Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 12:24:34 -0700 Subject: [PATCH 18/43] Fix silly error --- readthedocs/rtd_tests/tests/test_doc_serving.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index c0fc7ff6b5d..f136b9c39c6 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -77,7 +77,7 @@ def test_public_python_media_serving(self): serve_mock.assert_called_with( request, 'en/latest/usage.html', - settings.SITE_ROOT + '/public_web_root/private' + settings.SITE_ROOT + '/public_web_root/public' ) @override_settings(PYTHON_MEDIA=False) From 90f1d2bb91d62dfb33530266a4e498da3de8902c Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 12:49:21 -0700 Subject: [PATCH 19/43] Include search urls in subdomains/single version --- readthedocs/core/urls/single_version.py | 4 +++- readthedocs/core/urls/subdomain.py | 4 +++- readthedocs/core/views/serve.py | 3 +++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index d2e3d08fb2f..9df6ebdee92 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -4,6 +4,8 @@ from django.conf import settings from django.conf.urls.static import static +from readthedocs.urls import search_urls + handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' @@ -20,7 +22,7 @@ name='docs_detail'), ) -groups = [single_version_urls] +groups = [single_version_urls, search_urls] if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index b9311416004..02bde3224ff 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -4,6 +4,8 @@ from django.conf import settings from django.conf.urls.static import static +from readthedocs.urls import search_urls + handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' @@ -19,7 +21,7 @@ name='docs_detail'), ) -groups = [subdomain_urls] +groups = [subdomain_urls, search_urls] if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 0fc5adb9c38..20e89923803 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -79,6 +79,9 @@ def redirect_page_with_filename(request, project, filename): @map_project_slug def serve_docs(request, project, lang_slug=None, version_slug=None, filename=''): + """ + This exists mainly to map existing proj, lang, version, filename views to the file format. + """ filename = resolve_path( project, version_slug=version_slug, language=lang_slug, filename=filename ) From 6767e58fdbcd2810ce37f722600287de826efaa0 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Mon, 2 May 2016 12:51:36 -0700 Subject: [PATCH 20/43] =?UTF-8?q?Don=E2=80=99t=20include=20any=20URL?= =?UTF-8?q?=E2=80=99s=20in=20sub/single?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- readthedocs/core/urls/single_version.py | 4 +--- readthedocs/core/urls/subdomain.py | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 9df6ebdee92..d2e3d08fb2f 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -4,8 +4,6 @@ from django.conf import settings from django.conf.urls.static import static -from readthedocs.urls import search_urls - handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' @@ -22,7 +20,7 @@ name='docs_detail'), ) -groups = [single_version_urls, search_urls] +groups = [single_version_urls] if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index 02bde3224ff..b9311416004 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -4,8 +4,6 @@ from django.conf import settings from django.conf.urls.static import static -from readthedocs.urls import search_urls - handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' @@ -21,7 +19,7 @@ name='docs_detail'), ) -groups = [subdomain_urls, search_urls] +groups = [subdomain_urls] if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) From faf90d44b97e6ee1aaafa625cc98aa88410bc57b Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 14:16:59 -0700 Subject: [PATCH 21/43] Add small commit to force GH rebuild --- README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.rst b/README.rst index 84bc2e3ae25..390d24ce085 100644 --- a/README.rst +++ b/README.rst @@ -66,3 +66,5 @@ when you push to GitHub. :alt: Documentation Status :scale: 100% :target: https://docs.readthedocs.io/en/latest/?badge=latest + + From 31d9c1cd010aa8ffd061c6deea21e2dd7550e091 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 14:27:06 -0700 Subject: [PATCH 22/43] Respect privacy with doc serving. --- readthedocs/core/resolver.py | 12 ++++++++---- readthedocs/core/urls/__init__.py | 11 ++--------- readthedocs/core/urls/single_version.py | 4 ++-- readthedocs/core/urls/subdomain.py | 9 +++++++-- readthedocs/core/views/serve.py | 24 ++++++++++++++++-------- 5 files changed, 35 insertions(+), 25 deletions(-) diff --git a/readthedocs/core/resolver.py b/readthedocs/core/resolver.py index 3ea2c813bf1..87eed8d590f 100644 --- a/readthedocs/core/resolver.py +++ b/readthedocs/core/resolver.py @@ -51,11 +51,11 @@ class ResolverBase(object): def base_resolve_path(self, project_slug, filename, version_slug=None, language=None, private=False, single_version=None, - subproject_slug=None, subdomain=None, cname=None): + subproject_slug=None, subdomain=None, cname=None, internal=False): """Resolve a with nothing smart, just filling in the blanks""" # Only support `/docs/project' URLs outside our normal environment. Normally # the path should always have a subdomain or CNAME domain - if subdomain or cname or (self._use_subdomain()): + if subdomain or cname or internal or (self._use_subdomain()): url = '/' else: url = '/docs/{project_slug}/' @@ -76,7 +76,7 @@ def base_resolve_path(self, project_slug, filename, version_slug=None, def resolve_path(self, project, filename='', version_slug=None, language=None, single_version=None, subdomain=None, - cname=None, private=None): + cname=None, private=None, internal=False): """Resolve a URL with a subset of fields defined""" relation = project.superprojects.first() cname = cname or project.domains.filter(canonical=True).first() @@ -115,7 +115,8 @@ def resolve_path(self, project, filename='', version_slug=None, single_version=single_version, subproject_slug=subproject_slug, cname=cname, - private=private + private=private, + internal=internal, ) def resolve_domain(self, project, private=None): @@ -181,6 +182,9 @@ def _fix_filename(self, project, filename): This basically means stripping / and .html endings and then re-adding them properly. """ + # Bail out on non-html files + if '.' in filename and '.html' not in filename: + return filename filename = filename.lstrip('/') filename = re.sub('index.html$', '', filename) filename = re.sub('index$', '', filename) diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py index dbbf8a3f9ec..86430309cf9 100644 --- a/readthedocs/core/urls/__init__.py +++ b/readthedocs/core/urls/__init__.py @@ -15,24 +15,17 @@ 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), - # Handle single version URLs + # Redirect root to actual docs url((r'^docs/(?P{project_slug})/$'.format(**pattern_opts)), 'readthedocs.core.views.serve.redirect_project_slug', name='docs_detail'), - # Handle single version URLs - url((r'^docs/(?P{project_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.serve.serve_symlink_docs', - name='docs_detail'), - # Just for reversing URL's for now url((r'^docs/(?P{project_slug})/(?P{lang_slug})/' r'(?P{version_slug})/' r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.serve.serve_symlink_docs', + 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), - ) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index d2e3d08fb2f..1e05ddd2a2d 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -11,12 +11,12 @@ '', # base view, flake8 complains if it is on the previous line. # Handle /docs on RTD domain url(r'^docs/(?P[-\w]+)/(?P.*)$', - 'readthedocs.core.views.serve.serve_symlink_docs', + 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), # Handle subdomains url(r'^(?P.*)$', - 'readthedocs.core.views.serve.serve_symlink_docs', + 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), ) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index b9311416004..605e681fd8a 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -4,6 +4,8 @@ from django.conf import settings from django.conf.urls.static import static +from readthedocs.constants import pattern_opts + handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' @@ -14,8 +16,11 @@ name='docs_detail'), url(r'^$', 'readthedocs.core.views.serve.redirect_project_slug', name='redirect_project_slug'), - url(r'^(?P.*)$', - 'readthedocs.core.views.serve.serve_symlink_docs', + # Just for reversing URL's for now + url((r'^(?P{lang_slug})/' + r'(?P{version_slug})/' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), ) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 20e89923803..e3fa3ac2b97 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -30,6 +30,7 @@ from django.template import RequestContext from django.views.static import serve +from readthedocs.builds.models import Version from readthedocs.projects import constants from readthedocs.projects.models import Project from readthedocs.core.symlink import PrivateSymlink, PublicSymlink @@ -82,10 +83,20 @@ def serve_docs(request, project, lang_slug=None, version_slug=None, filename='') """ This exists mainly to map existing proj, lang, version, filename views to the file format. """ + if not version_slug: + version_slug = project.get_default_version() + try: + version = project.versions.public(request.user).get(slug=version_slug) + except Version.DoesNotExist: + raise Http404("User doesn't have access to this version") filename = resolve_path( - project, version_slug=version_slug, language=lang_slug, filename=filename + project, version_slug=version_slug, language=lang_slug, filename=filename, + internal=True, # internal will make it a "full" path without a URL prefix ) - return serve_symlink_docs(request, filename=filename, project=project) + return _serve_symlink_docs(request, + filename=filename, + project=project, + privacy_level=version.privacy_level) def _serve_file(request, filename, basepath): @@ -119,8 +130,7 @@ def _serve_401(request, project): return res -@map_project_slug -def serve_symlink_docs(request, project, filename=''): +def _serve_symlink_docs(request, project, filename='', privacy_level=constants.PUBLIC): # Handle indexes if filename == '' or filename[-1] == '/': @@ -136,7 +146,7 @@ def serve_symlink_docs(request, project, filename=''): serve_docs = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE]) - if settings.DEBUG or constants.PUBLIC in serve_docs: + if (settings.DEBUG or constants.PUBLIC in serve_docs) and privacy_level != constants.PRIVATE: public_symlink = PublicSymlink(project) basepath = public_symlink.project_root if os.path.exists(os.path.join(basepath, filename)): @@ -144,7 +154,7 @@ def serve_symlink_docs(request, project, filename=''): else: files_tried.append(os.path.join(basepath, filename)) - if settings.DEBUG or constants.PRIVATE in serve_docs: + if (settings.DEBUG or constants.PRIVATE in serve_docs) and privacy_level == constants.PRIVATE: # Handle private private_symlink = PrivateSymlink(project) @@ -152,10 +162,8 @@ def serve_symlink_docs(request, project, filename=''): if os.path.exists(os.path.join(basepath, filename)): - # Do basic auth check on the project, but not the version if not AdminPermission.is_member(user=request.user, project=project): return _serve_401(request, project) - return _serve_file(request, filename, basepath) else: files_tried.append(os.path.join(basepath, filename)) From 2d087970cce8b68f7ed5a0adf6b14b8f9ddaeb2f Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 14:33:26 -0700 Subject: [PATCH 23/43] Little refactor of doc serving code --- readthedocs/core/urls/single_version.py | 1 + readthedocs/core/urls/subdomain.py | 1 + readthedocs/core/views/serve.py | 51 ++++++++++++------------- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 1e05ddd2a2d..314eab05268 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -22,6 +22,7 @@ groups = [single_version_urls] +# Needed to serve media locally if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index 605e681fd8a..d53dbd145e0 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -26,6 +26,7 @@ groups = [subdomain_urls] +# Needed to serve media locally if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index e3fa3ac2b97..958b7b6fefe 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -78,25 +78,12 @@ def redirect_page_with_filename(request, project, filename): return HttpResponseRedirect(resolve(project, filename=filename)) -@map_project_slug -def serve_docs(request, project, lang_slug=None, version_slug=None, filename=''): - """ - This exists mainly to map existing proj, lang, version, filename views to the file format. - """ - if not version_slug: - version_slug = project.get_default_version() - try: - version = project.versions.public(request.user).get(slug=version_slug) - except Version.DoesNotExist: - raise Http404("User doesn't have access to this version") - filename = resolve_path( - project, version_slug=version_slug, language=lang_slug, filename=filename, - internal=True, # internal will make it a "full" path without a URL prefix - ) - return _serve_symlink_docs(request, - filename=filename, - project=project, - privacy_level=version.privacy_level) +def _serve_401(request, project): + res = render_to_response('401.html', + context_instance=RequestContext(request)) + res.status_code = 401 + log.error('Unauthorized access to {0} documentation'.format(project.slug)) + return res def _serve_file(request, filename, basepath): @@ -122,12 +109,25 @@ def _serve_file(request, filename, basepath): return response -def _serve_401(request, project): - res = render_to_response('401.html', - context_instance=RequestContext(request)) - res.status_code = 401 - log.error('Unauthorized access to {0} documentation'.format(project.slug)) - return res +@map_project_slug +def serve_docs(request, project, lang_slug=None, version_slug=None, filename=''): + """ + This exists mainly to map existing proj, lang, version, filename views to the file format. + """ + if not version_slug: + version_slug = project.get_default_version() + try: + version = project.versions.public(request.user).get(slug=version_slug) + except Version.DoesNotExist: + return _serve_401(request, project) + filename = resolve_path( + project, version_slug=version_slug, language=lang_slug, filename=filename, + internal=True, # internal will make it a "full" path without a URL prefix + ) + return _serve_symlink_docs(request, + filename=filename, + project=project, + privacy_level=version.privacy_level) def _serve_symlink_docs(request, project, filename='', privacy_level=constants.PUBLIC): @@ -161,7 +161,6 @@ def _serve_symlink_docs(request, project, filename='', privacy_level=constants.P basepath = private_symlink.project_root if os.path.exists(os.path.join(basepath, filename)): - if not AdminPermission.is_member(user=request.user, project=project): return _serve_401(request, project) return _serve_file(request, filename, basepath) From 4ac2fa9be786b47ed2417cba5f3d0ecd8b9368d9 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 14:36:43 -0700 Subject: [PATCH 24/43] Keep smaller lines --- readthedocs/core/middleware.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index d29e1a6f33a..8fa29ff851d 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -34,7 +34,11 @@ def process_request(self, request): path = request.get_full_path() log_kwargs = dict(host=host, path=path) public_domain = getattr(settings, 'PUBLIC_DOMAIN', None) - production_domain = getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org') + production_domain = getattr( + settings, + 'PRODUCTION_DOMAIN', + 'readthedocs.org' + ) if public_domain is None: public_domain = production_domain From 027f1436123d73ceedb6bb5aff27adbc50f57362 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 14:40:03 -0700 Subject: [PATCH 25/43] Add env to ignore list --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e08a1351cb8..5b848dcb07c 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ .idea .vagrant .tox +.env .rope_project/ .ropeproject/ _build From 4f15dfb7a6a3b4bb7393b1eb5791fa6b03e617ab Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 14:40:54 -0700 Subject: [PATCH 26/43] Remove extra space --- README.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/README.rst b/README.rst index 390d24ce085..98df57833b9 100644 --- a/README.rst +++ b/README.rst @@ -67,4 +67,3 @@ when you push to GitHub. :scale: 100% :target: https://docs.readthedocs.io/en/latest/?badge=latest - From 4e5cc8367d74d1ceb3b25e3d5a05d538a707b031 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 15:46:11 -0700 Subject: [PATCH 27/43] Fix up tests --- readthedocs/core/views/serve.py | 4 +++- readthedocs/rtd_tests/tests/test_privacy.py | 2 +- readthedocs/rtd_tests/tests/test_redirects.py | 21 +++++-------------- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 958b7b6fefe..dfa8fb13405 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -119,7 +119,9 @@ def serve_docs(request, project, lang_slug=None, version_slug=None, filename='') try: version = project.versions.public(request.user).get(slug=version_slug) except Version.DoesNotExist: - return _serve_401(request, project) + if Version.objects.filter(slug=version_slug).exists(): + return _serve_401(request, project) + raise Http404('Version does not exist.') filename = resolve_path( project, version_slug=version_slug, language=lang_slug, filename=filename, internal=True, # internal will make it a "full" path without a URL prefix diff --git a/readthedocs/rtd_tests/tests/test_privacy.py b/readthedocs/rtd_tests/tests/test_privacy.py index 78abd561371..a1f54e6473b 100644 --- a/readthedocs/rtd_tests/tests/test_privacy.py +++ b/readthedocs/rtd_tests/tests/test_privacy.py @@ -208,7 +208,7 @@ def test_private_doc_serving(self): # Make sure it doesn't show up as tester self.client.login(username='tester', password='test') r = self.client.get('/docs/django-kong/en/test-slug/') - self.assertEqual(r.status_code, 404) + self.assertEqual(r.status_code, 401) # Private download tests diff --git a/readthedocs/rtd_tests/tests/test_redirects.py b/readthedocs/rtd_tests/tests/test_redirects.py index a8565c9f7f4..6153232ee94 100644 --- a/readthedocs/rtd_tests/tests/test_redirects.py +++ b/readthedocs/rtd_tests/tests/test_redirects.py @@ -14,7 +14,7 @@ import logging -@override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False) +@override_settings(PUBLIC_DOMAIN='readthedocs.org', USE_SUBDOMAIN=False, APPEND_SLASH=False) class RedirectTests(TestCase): fixtures = ["eric", "test_data"] @@ -38,12 +38,7 @@ def setUp(self): def test_proper_url_no_slash(self): r = self.client.get('/docs/pip') - # This is triggered by Django, so its a 301, basically just - # APPEND_SLASH - self.assertEqual(r.status_code, 301) - self.assertEqual(r['Location'], 'http://testserver/docs/pip/') - r = self.client.get(r['Location']) - self.assertEqual(r.status_code, 302) + self.assertEqual(r.status_code, 404) def test_proper_url(self): r = self.client.get('/docs/pip/') @@ -156,7 +151,7 @@ def test_redirect_keeps_version_number(self): Redirect.objects.create( project=self.pip, redirect_type='page', from_url='/how_to_install.html', to_url='/install.html') - with patch('readthedocs.core.views.serve.serve_symlink_docs') as _serve_docs: + with patch('readthedocs.core.views.serve._serve_symlink_docs') as _serve_docs: _serve_docs.side_effect = Http404() r = self.client.get('/en/0.8.1/how_to_install.html', HTTP_HOST='pip.readthedocs.org') @@ -170,7 +165,7 @@ def test_redirect_keeps_language(self): Redirect.objects.create( project=self.pip, redirect_type='page', from_url='/how_to_install.html', to_url='/install.html') - with patch('readthedocs.core.views.serve.serve_symlink_docs') as _serve_docs: + with patch('readthedocs.core.views.serve._serve_symlink_docs') as _serve_docs: _serve_docs.side_effect = Http404() r = self.client.get('/de/0.8.1/how_to_install.html', HTTP_HOST='pip.readthedocs.org') @@ -277,11 +272,5 @@ def test_single_version_no_subdomain(self): self.redirect.project.single_version = True self.assertEqual( self.redirect.get_full_path('faq.html'), - reverse( - 'docs_detail', - kwargs={ - 'project_slug': self.proj.slug, - 'filename': 'faq.html', - } - ) + '/docs/read-the-docs/faq.html' ) From da003610b9ae2434ccb650ba03edf7f372d45de5 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 15:48:47 -0700 Subject: [PATCH 28/43] Don't overload `internal` with .com --- readthedocs/core/resolver.py | 7 +++---- readthedocs/core/views/serve.py | 3 ++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/readthedocs/core/resolver.py b/readthedocs/core/resolver.py index 87eed8d590f..c5d200c7acc 100644 --- a/readthedocs/core/resolver.py +++ b/readthedocs/core/resolver.py @@ -51,11 +51,11 @@ class ResolverBase(object): def base_resolve_path(self, project_slug, filename, version_slug=None, language=None, private=False, single_version=None, - subproject_slug=None, subdomain=None, cname=None, internal=False): + subproject_slug=None, subdomain=None, cname=None): """Resolve a with nothing smart, just filling in the blanks""" # Only support `/docs/project' URLs outside our normal environment. Normally # the path should always have a subdomain or CNAME domain - if subdomain or cname or internal or (self._use_subdomain()): + if subdomain or cname or (self._use_subdomain()): url = '/' else: url = '/docs/{project_slug}/' @@ -76,7 +76,7 @@ def base_resolve_path(self, project_slug, filename, version_slug=None, def resolve_path(self, project, filename='', version_slug=None, language=None, single_version=None, subdomain=None, - cname=None, private=None, internal=False): + cname=None, private=None): """Resolve a URL with a subset of fields defined""" relation = project.superprojects.first() cname = cname or project.domains.filter(canonical=True).first() @@ -116,7 +116,6 @@ def resolve_path(self, project, filename='', version_slug=None, subproject_slug=subproject_slug, cname=cname, private=private, - internal=internal, ) def resolve_domain(self, project, private=None): diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index dfa8fb13405..b400f28d1f4 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -119,12 +119,13 @@ def serve_docs(request, project, lang_slug=None, version_slug=None, filename='') try: version = project.versions.public(request.user).get(slug=version_slug) except Version.DoesNotExist: + # Properly raise a 404 if the version doesn't exist & a 401 if it does if Version.objects.filter(slug=version_slug).exists(): return _serve_401(request, project) raise Http404('Version does not exist.') filename = resolve_path( project, version_slug=version_slug, language=lang_slug, filename=filename, - internal=True, # internal will make it a "full" path without a URL prefix + subdomain=True, # subdomain will make it a "full" path without a URL prefix ) return _serve_symlink_docs(request, filename=filename, From 73dcab34787a6a971cd4435ac3709fe8b5e39219 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 17:11:54 -0700 Subject: [PATCH 29/43] Support subprojects --- readthedocs/core/resolver.py | 1 + readthedocs/core/urls/__init__.py | 10 +++-- readthedocs/core/urls/single_version.py | 24 +++++++++--- readthedocs/core/urls/subdomain.py | 9 +++-- readthedocs/core/views/serve.py | 38 +++++++++++++++---- .../rtd_tests/tests/test_doc_serving.py | 14 +++---- readthedocs/urls.py | 2 +- 7 files changed, 72 insertions(+), 26 deletions(-) diff --git a/readthedocs/core/resolver.py b/readthedocs/core/resolver.py index c5d200c7acc..19f5a8802f2 100644 --- a/readthedocs/core/resolver.py +++ b/readthedocs/core/resolver.py @@ -116,6 +116,7 @@ def resolve_path(self, project, filename='', version_slug=None, subproject_slug=subproject_slug, cname=cname, private=private, + subdomain=subdomain, ) def resolve_domain(self, project, private=None): diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py index 86430309cf9..f52362a8a1c 100644 --- a/readthedocs/core/urls/__init__.py +++ b/readthedocs/core/urls/__init__.py @@ -16,14 +16,18 @@ name='docs_detail'), # Redirect root to actual docs - url((r'^docs/(?P{project_slug})/$'.format(**pattern_opts)), + url((r'^docs/(?P{project_slug})/' + r'(?:|projects/(?P{project_slug})/)$' + .format(**pattern_opts)), 'readthedocs.core.views.serve.redirect_project_slug', name='docs_detail'), # Just for reversing URL's for now - url((r'^docs/(?P{project_slug})/(?P{lang_slug})/' + url((r'^docs/(?P{project_slug})/' + r'(?:|projects/(?P{project_slug})/)' + r'(?P{lang_slug})/' r'(?P{version_slug})/' - r'(?P{filename_slug})$'.format(**pattern_opts)), + r'(?P{filename_slug})'.format(**pattern_opts)), 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), ) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 314eab05268..44f00373656 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -4,18 +4,20 @@ from django.conf import settings from django.conf.urls.static import static +from readthedocs.constants import pattern_opts + handler500 = 'readthedocs.core.views.server_error' handler404 = 'readthedocs.core.views.server_error_404' single_version_urls = patterns( '', # base view, flake8 complains if it is on the previous line. - # Handle /docs on RTD domain - url(r'^docs/(?P[-\w]+)/(?P.*)$', - 'readthedocs.core.views.serve.serve_docs', + + url(r'^page/(?P.*)$', + 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), - # Handle subdomains - url(r'^(?P.*)$', + url((r'^(?:|projects/(?P{project_slug})/)' + r'(?P{filename_slug})$'.format(**pattern_opts)), 'readthedocs.core.views.serve.serve_docs', name='docs_detail'), ) @@ -26,4 +28,16 @@ if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) +if not getattr(settings, 'USE_SUBDOMAIN', False) or settings.DEBUG: + docs_url = patterns( + '', + url((r'^docs/(?P[-\w]+)/' + r'(?:|projects/(?P{project_slug})/)' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.serve.serve_docs', + name='docs_detail') + ) + groups.insert(1, docs_url) + + urlpatterns = reduce(add, groups) diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index d53dbd145e0..cc9237bfe30 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -15,9 +15,12 @@ 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), - url(r'^$', 'readthedocs.core.views.serve.redirect_project_slug', name='redirect_project_slug'), - # Just for reversing URL's for now - url((r'^(?P{lang_slug})/' + url((r'^(?:|projects/(?P{project_slug})/)$').format(**pattern_opts), + 'readthedocs.core.views.serve.redirect_project_slug', + name='redirect_project_slug'), + + url((r'^(?:|projects/(?P{project_slug})/)' + r'(?P{lang_slug})/' r'(?P{version_slug})/' r'(?P{filename_slug})$'.format(**pattern_opts)), 'readthedocs.core.views.serve.serve_docs', diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index b400f28d1f4..d4816656231 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -45,6 +45,25 @@ log = logging.getLogger(__name__) +def map_subproject_slug(view_func): + """ + A decorator that maps a ``subproject_slug`` URL param into a Project. + + :raises: Http404 if the Project doesn't exist + + .. warning:: Does not take into account any kind of privacy settings. + """ + @wraps(view_func) + def inner_view(request, subproject=None, subproject_slug=None, *args, **kwargs): + if subproject is None and subproject_slug: + try: + subproject = Project.objects.get(slug=subproject_slug) + except Project.DoesNotExist: + raise Http404 + return view_func(request, subproject=subproject, *args, **kwargs) + return inner_view + + def map_project_slug(view_func): """ A decorator that maps a ``project_slug`` URL param into a Project. @@ -67,15 +86,16 @@ def inner_view(request, project=None, project_slug=None, *args, **kwargs): @map_project_slug -def redirect_project_slug(request, project): +@map_subproject_slug +def redirect_project_slug(request, project, subproject): """Handle / -> /en/latest/ directs on subdomains""" - return HttpResponseRedirect(resolve(project)) + return HttpResponseRedirect(resolve(subproject or project)) @map_project_slug -def redirect_page_with_filename(request, project, filename): +def redirect_page_with_filename(request, project, subproject, filename): """Redirect /page/file.html to /en/latest/file.html.""" - return HttpResponseRedirect(resolve(project, filename=filename)) + return HttpResponseRedirect(resolve(subproject or project, filename=filename)) def _serve_401(request, project): @@ -110,7 +130,9 @@ def _serve_file(request, filename, basepath): @map_project_slug -def serve_docs(request, project, lang_slug=None, version_slug=None, filename=''): +@map_subproject_slug +def serve_docs(request, project, subproject, + lang_slug=None, version_slug=None, filename=''): """ This exists mainly to map existing proj, lang, version, filename views to the file format. """ @@ -120,11 +142,12 @@ def serve_docs(request, project, lang_slug=None, version_slug=None, filename='') version = project.versions.public(request.user).get(slug=version_slug) except Version.DoesNotExist: # Properly raise a 404 if the version doesn't exist & a 401 if it does - if Version.objects.filter(slug=version_slug).exists(): + if project.versions.filter(slug=version_slug).exists(): return _serve_401(request, project) raise Http404('Version does not exist.') filename = resolve_path( - project, version_slug=version_slug, language=lang_slug, filename=filename, + subproject or project, # Resolve the subproject if it exists + version_slug=version_slug, language=lang_slug, filename=filename, subdomain=True, # subdomain will make it a "full" path without a URL prefix ) return _serve_symlink_docs(request, @@ -133,6 +156,7 @@ def serve_docs(request, project, lang_slug=None, version_slug=None, filename='') privacy_level=version.privacy_level) +@map_project_slug def _serve_symlink_docs(request, project, filename='', privacy_level=constants.PUBLIC): # Handle indexes diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index f136b9c39c6..c3f86c11b0a 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -10,7 +10,7 @@ from readthedocs.rtd_tests.base import RequestFactoryTestMixin from readthedocs.projects import constants from readthedocs.projects.models import Project -from readthedocs.core.views.serve import serve_symlink_docs +from readthedocs.core.views.serve import _serve_symlink_docs @override_settings( @@ -39,7 +39,7 @@ def test_private_python_media_serving(self): with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.private_url, user=self.eric) - serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') serve_mock.assert_called_with( request, 'en/latest/usage.html', @@ -50,7 +50,7 @@ def test_private_python_media_serving(self): def test_private_nginx_serving(self): with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.private_url, user=self.eric) - r = serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + r = _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') self.assertEqual(r.status_code, 200) self.assertEqual( r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' @@ -60,7 +60,7 @@ def test_private_nginx_serving(self): def test_private_files_not_found(self): request = self.request(self.private_url, user=self.eric) with self.assertRaises(Http404) as exc: - serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') self.assertTrue('private_web_root' in exc.exception.message) self.assertTrue('public_web_root' not in exc.exception.message) @@ -73,7 +73,7 @@ def test_public_python_media_serving(self): with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.public_url, user=self.eric) - serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') serve_mock.assert_called_with( request, 'en/latest/usage.html', @@ -84,7 +84,7 @@ def test_public_python_media_serving(self): def test_public_nginx_serving(self): with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.public_url, user=self.eric) - r = serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') + r = _serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') self.assertEqual(r.status_code, 200) self.assertEqual( r._headers['x-accel-redirect'][1], '/public_web_root/public/en/latest/usage.html' @@ -94,6 +94,6 @@ def test_public_nginx_serving(self): def test_both_files_not_found(self): request = self.request(self.private_url, user=self.eric) with self.assertRaises(Http404) as exc: - serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') self.assertTrue('private_web_root' in exc.exception.message) self.assertTrue('public_web_root' in exc.exception.message) diff --git a/readthedocs/urls.py b/readthedocs/urls.py index 2f393e49647..3077b1146e0 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -88,7 +88,7 @@ groups = [basic_urls, rtd_urls, project_urls, api_urls, core_urls, i18n_urls, money_urls, deprecated_urls] -if not getattr(settings, 'USE_SUBDOMAIN', False): +if not getattr(settings, 'USE_SUBDOMAIN', False) or settings.DEBUG: groups.insert(0, docs_urls) if getattr(settings, 'ALLOW_ADMIN', True): groups.append(admin_urls) From 097a4103790dfee027f11a63b3957d0418aa3869 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 17:16:26 -0700 Subject: [PATCH 30/43] Fully port subproject URL's --- readthedocs/core/urls/single_version.py | 4 +++- readthedocs/core/urls/subdomain.py | 3 ++- readthedocs/core/views/serve.py | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 44f00373656..d08f22127a2 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -12,7 +12,8 @@ single_version_urls = patterns( '', # base view, flake8 complains if it is on the previous line. - url(r'^page/(?P.*)$', + url(r'^(?:|projects/(?P{project_slug})/)' + r'page/(?P.*)$', 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), @@ -28,6 +29,7 @@ if getattr(settings, 'DEBUG', False): groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) +# Allow `/docs/` URL's when not using subdomains or during local dev if not getattr(settings, 'USE_SUBDOMAIN', False) or settings.DEBUG: docs_url = patterns( '', diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index cc9237bfe30..f7207443a5d 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -11,7 +11,8 @@ subdomain_urls = patterns( '', # base view, flake8 complains if it is on the previous line. - url(r'^page/(?P.*)$', + url(r'^(?:|projects/(?P{project_slug})/)' + r'^page/(?P.*)$', 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index d4816656231..d7e28cbe28d 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -93,6 +93,7 @@ def redirect_project_slug(request, project, subproject): @map_project_slug +@map_subproject_slug def redirect_page_with_filename(request, project, subproject, filename): """Redirect /page/file.html to /en/latest/file.html.""" return HttpResponseRedirect(resolve(subproject or project, filename=filename)) From f2ee67420276afdac41681e559771de5f986a0da Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 17:24:14 -0700 Subject: [PATCH 31/43] Clean up tests --- readthedocs/core/urls/single_version.py | 2 +- readthedocs/core/urls/subdomain.py | 2 +- readthedocs/projects/models.py | 3 +-- readthedocs/rtd_tests/tests/test_doc_serving.py | 8 ++++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index d08f22127a2..521dff74cc5 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -13,7 +13,7 @@ '', # base view, flake8 complains if it is on the previous line. url(r'^(?:|projects/(?P{project_slug})/)' - r'page/(?P.*)$', + r'page/(?P.*)$'.format(**pattern_opts), 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py index f7207443a5d..f3532c6d01a 100644 --- a/readthedocs/core/urls/subdomain.py +++ b/readthedocs/core/urls/subdomain.py @@ -12,7 +12,7 @@ subdomain_urls = patterns( '', # base view, flake8 complains if it is on the previous line. url(r'^(?:|projects/(?P{project_slug})/)' - r'^page/(?P.*)$', + r'page/(?P.*)$'.format(**pattern_opts), 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 446d82988db..7916726d9ff 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -847,8 +847,7 @@ class ImportedFile(models.Model): @models.permalink def get_absolute_url(self): - return ('docs_detail', [self.project.slug, self.project.language, - self.version.slug, self.path]) + return resolve(project=self.project, version_slug=self.version.slug, filename=self.path) def __unicode__(self): return '%s: %s' % (self.name, self.project) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index c3f86c11b0a..41b77276d56 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -39,7 +39,7 @@ def test_private_python_media_serving(self): with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.private_url, user=self.eric) - _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html', privacy_level='private') serve_mock.assert_called_with( request, 'en/latest/usage.html', @@ -50,7 +50,7 @@ def test_private_python_media_serving(self): def test_private_nginx_serving(self): with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.private_url, user=self.eric) - r = _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + r = _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html', privacy_level='private') self.assertEqual(r.status_code, 200) self.assertEqual( r._headers['x-accel-redirect'][1], '/private_web_root/private/en/latest/usage.html' @@ -60,7 +60,7 @@ def test_private_nginx_serving(self): def test_private_files_not_found(self): request = self.request(self.private_url, user=self.eric) with self.assertRaises(Http404) as exc: - _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html', privacy_level='private') self.assertTrue('private_web_root' in exc.exception.message) self.assertTrue('public_web_root' not in exc.exception.message) @@ -95,5 +95,5 @@ def test_both_files_not_found(self): request = self.request(self.private_url, user=self.eric) with self.assertRaises(Http404) as exc: _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') - self.assertTrue('private_web_root' in exc.exception.message) + self.assertTrue('private_web_root' not in exc.exception.message) self.assertTrue('public_web_root' in exc.exception.message) From 900d456aa3a05ad26408e46913786cc826bcf8a1 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 17:31:42 -0700 Subject: [PATCH 32/43] Fix linting --- readthedocs/core/urls/__init__.py | 3 +-- readthedocs/core/urls/single_version.py | 12 ++++++------ 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py index f52362a8a1c..b6d848d4f7c 100644 --- a/readthedocs/core/urls/__init__.py +++ b/readthedocs/core/urls/__init__.py @@ -17,8 +17,7 @@ # Redirect root to actual docs url((r'^docs/(?P{project_slug})/' - r'(?:|projects/(?P{project_slug})/)$' - .format(**pattern_opts)), + r'(?:|projects/(?P{project_slug})/)$'.format(**pattern_opts)), 'readthedocs.core.views.serve.redirect_project_slug', name='docs_detail'), diff --git a/readthedocs/core/urls/single_version.py b/readthedocs/core/urls/single_version.py index 521dff74cc5..91df361d8b5 100644 --- a/readthedocs/core/urls/single_version.py +++ b/readthedocs/core/urls/single_version.py @@ -30,14 +30,14 @@ groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)) # Allow `/docs/` URL's when not using subdomains or during local dev -if not getattr(settings, 'USE_SUBDOMAIN', False) or settings.DEBUG: +if not getattr(settings, 'USE_SUBDOMAIN', False) or settings.DEBUG: docs_url = patterns( - '', + '', url((r'^docs/(?P[-\w]+)/' - r'(?:|projects/(?P{project_slug})/)' - r'(?P{filename_slug})$'.format(**pattern_opts)), - 'readthedocs.core.views.serve.serve_docs', - name='docs_detail') + r'(?:|projects/(?P{project_slug})/)' + r'(?P{filename_slug})$'.format(**pattern_opts)), + 'readthedocs.core.views.serve.serve_docs', + name='docs_detail') ) groups.insert(1, docs_url) From bf76ec762c04b437394510faf8e7ccfbdb15db23 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Tue, 3 May 2016 17:36:51 -0700 Subject: [PATCH 33/43] Fix random tests --- readthedocs/projects/models.py | 1 - readthedocs/rtd_tests/tests/test_views.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 7916726d9ff..b7e5b0e9e56 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -845,7 +845,6 @@ class ImportedFile(models.Model): md5 = models.CharField(_('MD5 checksum'), max_length=255) commit = models.CharField(_('Commit'), max_length=255) - @models.permalink def get_absolute_url(self): return resolve(project=self.project, version_slug=self.version.slug, filename=self.path) diff --git a/readthedocs/rtd_tests/tests/test_views.py b/readthedocs/rtd_tests/tests/test_views.py index 6e22f30d003..6bc5fe8fc4d 100644 --- a/readthedocs/rtd_tests/tests/test_views.py +++ b/readthedocs/rtd_tests/tests/test_views.py @@ -179,7 +179,7 @@ def test_random_page_view_redirects(self): def test_takes_project_slug(self): response = self.client.get('/random/pip/') self.assertEqual(response.status_code, 302) - self.assertTrue('/pip/' in response['Location']) + self.assertTrue('pip' in response['Location']) def test_404_for_unknown_project(self): response = self.client.get('/random/not-existent/') From 5c80b3a0b91fadc90d19eeeb4449795492e69e5f Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 08:46:03 -0700 Subject: [PATCH 34/43] CLean up lying comments and require privacy --- readthedocs/core/urls/__init__.py | 3 --- readthedocs/core/views/serve.py | 2 +- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/readthedocs/core/urls/__init__.py b/readthedocs/core/urls/__init__.py index b6d848d4f7c..e60334c8c19 100644 --- a/readthedocs/core/urls/__init__.py +++ b/readthedocs/core/urls/__init__.py @@ -9,19 +9,16 @@ docs_urls = patterns( '', - # Handle /page/ redirects for explicit "latest" version goodness. url((r'^docs/(?P{project_slug})/page/' r'(?P{filename_slug})$'.format(**pattern_opts)), 'readthedocs.core.views.serve.redirect_page_with_filename', name='docs_detail'), - # Redirect root to actual docs url((r'^docs/(?P{project_slug})/' r'(?:|projects/(?P{project_slug})/)$'.format(**pattern_opts)), 'readthedocs.core.views.serve.redirect_project_slug', name='docs_detail'), - # Just for reversing URL's for now url((r'^docs/(?P{project_slug})/' r'(?:|projects/(?P{project_slug})/)' r'(?P{lang_slug})/' diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index d7e28cbe28d..31b71f1c17b 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -158,7 +158,7 @@ def serve_docs(request, project, subproject, @map_project_slug -def _serve_symlink_docs(request, project, filename='', privacy_level=constants.PUBLIC): +def _serve_symlink_docs(request, project, privacy_level, filename=''): # Handle indexes if filename == '' or filename[-1] == '/': From 54ae5e4f1b31daa52b2f78d8782e18720b545eed Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 08:52:59 -0700 Subject: [PATCH 35/43] Pass privacy_level explicitly --- readthedocs/rtd_tests/tests/test_doc_serving.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/rtd_tests/tests/test_doc_serving.py b/readthedocs/rtd_tests/tests/test_doc_serving.py index 41b77276d56..cb80bbf8ef5 100644 --- a/readthedocs/rtd_tests/tests/test_doc_serving.py +++ b/readthedocs/rtd_tests/tests/test_doc_serving.py @@ -73,7 +73,7 @@ def test_public_python_media_serving(self): with mock.patch('readthedocs.core.views.serve.serve') as serve_mock: with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.public_url, user=self.eric) - _serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html', privacy_level='public') serve_mock.assert_called_with( request, 'en/latest/usage.html', @@ -84,7 +84,7 @@ def test_public_python_media_serving(self): def test_public_nginx_serving(self): with mock.patch('readthedocs.core.views.serve.os.path.exists', return_value=True): request = self.request(self.public_url, user=self.eric) - r = _serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html') + r = _serve_symlink_docs(request, project=self.public, filename='/en/latest/usage.html', privacy_level='public') self.assertEqual(r.status_code, 200) self.assertEqual( r._headers['x-accel-redirect'][1], '/public_web_root/public/en/latest/usage.html' @@ -94,6 +94,6 @@ def test_public_nginx_serving(self): def test_both_files_not_found(self): request = self.request(self.private_url, user=self.eric) with self.assertRaises(Http404) as exc: - _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html') + _serve_symlink_docs(request, project=self.private, filename='/en/latest/usage.html', privacy_level='public') self.assertTrue('private_web_root' not in exc.exception.message) self.assertTrue('public_web_root' in exc.exception.message) From 828448c55748d99cc417a16e78123bcd0df8afd1 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 09:21:39 -0700 Subject: [PATCH 36/43] Fix linting --- tox.ini | 1 + 1 file changed, 1 insertion(+) diff --git a/tox.ini b/tox.ini index a33d6f57993..ca6207e34a1 100644 --- a/tox.ini +++ b/tox.ini @@ -26,6 +26,7 @@ deps = pylint<1.5 prospector pylint-django<0.7 + pyflakes<1.2.0 commands = prospector \ --profile-path={toxinidir} \ From c81f118f4dbb8792cf498c22f7d25913352a7fb1 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 11:50:37 -0700 Subject: [PATCH 37/43] Make is_member work on .com as well --- readthedocs/core/middleware.py | 6 +++++- readthedocs/core/views/serve.py | 2 +- readthedocs/privacy/backend.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 9f207a032a6..6784a39898b 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -44,6 +44,7 @@ def process_request(self, request): if public_domain is None: public_domain = production_domain if ':' in host: + full_host = host host = host.split(':')[0] domain_parts = host.split('.') @@ -51,7 +52,10 @@ def process_request(self, request): if len(domain_parts) == len(public_domain.split('.')) + 1 or DEV_URL in host: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - if not is_www and public_domain in host or DEV_URL in host: + if not is_www and public_domain in host or DEV_URL in host or ( + # Support ports during local dev + public_domain in host or public_domain in full_host + ): request.subdomain = True request.slug = subdomain request.urlconf = SUBDOMAIN_URLCONF diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 31b71f1c17b..aefb9f24156 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -189,7 +189,7 @@ def _serve_symlink_docs(request, project, privacy_level, filename=''): basepath = private_symlink.project_root if os.path.exists(os.path.join(basepath, filename)): - if not AdminPermission.is_member(user=request.user, project=project): + if not AdminPermission.is_member(user=request.user, obj=project): return _serve_401(request, project) return _serve_file(request, filename, basepath) else: diff --git a/readthedocs/privacy/backend.py b/readthedocs/privacy/backend.py index bd4429b1fee..4948405b71b 100644 --- a/readthedocs/privacy/backend.py +++ b/readthedocs/privacy/backend.py @@ -289,8 +289,8 @@ def is_admin(cls, user, project): return user in project.users.all() @classmethod - def is_member(cls, user, project): - return user in project.users.all() + def is_member(cls, user, obj): + return user in obj.users.all() class AdminNotAuthorized(ValueError): From 1e1474c45b02eb75d8c79637ebdfb006450d87e5 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 11:59:22 -0700 Subject: [PATCH 38/43] Fix test --- readthedocs/core/middleware.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 6784a39898b..ee12ffa0347 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -35,7 +35,7 @@ def process_request(self, request): if not getattr(settings, 'USE_SUBDOMAIN', False): return None - host = request.get_host().lower() + full_host = host = request.get_host().lower() path = request.get_full_path() log_kwargs = dict(host=host, path=path) public_domain = getattr(settings, 'PUBLIC_DOMAIN', None) @@ -44,7 +44,6 @@ def process_request(self, request): if public_domain is None: public_domain = production_domain if ':' in host: - full_host = host host = host.split(':')[0] domain_parts = host.split('.') From ff3c5ae989e0c5dc12207f7e09b9123337de665a Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 12:07:17 -0700 Subject: [PATCH 39/43] Use DEV_DOMAIN instead of DEV_URL --- readthedocs/core/middleware.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index ee12ffa0347..adccdb29f1a 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -22,9 +22,9 @@ 'SINGLE_VERSION_URLCONF', 'readthedocs.core.urls.single_version' ) -DEV_URL = getattr( +DEV_DOMAIN = getattr( settings, - 'DEV_URL', + 'DEV_DOMAIN', 'dev.readthedocs.io' ) @@ -48,10 +48,10 @@ def process_request(self, request): domain_parts = host.split('.') # Serve subdomains - but don't depend on the production domain only having 2 parts - if len(domain_parts) == len(public_domain.split('.')) + 1 or DEV_URL in host: + if len(domain_parts) == len(public_domain.split('.')) + 1 or DEV_DOMAIN in host: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - if not is_www and public_domain in host or DEV_URL in host or ( + if not is_www and public_domain in host or DEV_DOMAIN in host or ( # Support ports during local dev public_domain in host or public_domain in full_host ): From f27a37ba302442b342308e4b4883cd0b6f0b2c87 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 12:16:13 -0700 Subject: [PATCH 40/43] Remove DEV_DOMAIN --- readthedocs/core/middleware.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index adccdb29f1a..42d99544d77 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -22,11 +22,6 @@ 'SINGLE_VERSION_URLCONF', 'readthedocs.core.urls.single_version' ) -DEV_DOMAIN = getattr( - settings, - 'DEV_DOMAIN', - 'dev.readthedocs.io' -) class SubdomainMiddleware(object): @@ -48,10 +43,10 @@ def process_request(self, request): domain_parts = host.split('.') # Serve subdomains - but don't depend on the production domain only having 2 parts - if len(domain_parts) == len(public_domain.split('.')) + 1 or DEV_DOMAIN in host: + if len(domain_parts) == len(public_domain.split('.')) + 1: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - if not is_www and public_domain in host or DEV_DOMAIN in host or ( + if not is_www and public_domain in host or ( # Support ports during local dev public_domain in host or public_domain in full_host ): From fe2c12d3ee8030e6a14442f2485c8c35620f236d Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 12:18:12 -0700 Subject: [PATCH 41/43] Fix logic here --- readthedocs/core/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 42d99544d77..c54a9b6a218 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -46,7 +46,7 @@ def process_request(self, request): if len(domain_parts) == len(public_domain.split('.')) + 1: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - if not is_www and public_domain in host or ( + if not is_www or ( # Support ports during local dev public_domain in host or public_domain in full_host ): From 8bae094a86293656d1b88bdb9fa7a29021abf925 Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 12:29:46 -0700 Subject: [PATCH 42/43] Fix and/or logic --- readthedocs/core/middleware.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index c54a9b6a218..10cf22cc619 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -46,7 +46,7 @@ def process_request(self, request): if len(domain_parts) == len(public_domain.split('.')) + 1: subdomain = domain_parts[0] is_www = subdomain.lower() == 'www' - if not is_www or ( + if not is_www and ( # Support ports during local dev public_domain in host or public_domain in full_host ): From 2f5a2688729313321a866c15a4f45f360ded9a6d Mon Sep 17 00:00:00 2001 From: Eric Holscher Date: Wed, 4 May 2016 12:33:24 -0700 Subject: [PATCH 43/43] Fix linting on production_domain again --- readthedocs/core/middleware.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/readthedocs/core/middleware.py b/readthedocs/core/middleware.py index 10cf22cc619..b0cd70d7c71 100644 --- a/readthedocs/core/middleware.py +++ b/readthedocs/core/middleware.py @@ -34,7 +34,11 @@ def process_request(self, request): path = request.get_full_path() log_kwargs = dict(host=host, path=path) public_domain = getattr(settings, 'PUBLIC_DOMAIN', None) - production_domain = getattr(settings, 'PRODUCTION_DOMAIN', 'readthedocs.org') + production_domain = getattr( + settings, + 'PRODUCTION_DOMAIN', + 'readthedocs.org' + ) if public_domain is None: public_domain = production_domain