Skip to content

Alter direct calls to redis using intermediate redis settings #1801

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 30, 2015
Merged

Alter direct calls to redis using intermediate redis settings #1801

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8394965
Alter direct calls to redis, use cache backend instead
agjohnson Oct 30, 2015
1eacfdc
Use broker url for redis client configuration
agjohnson Oct 30, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions readthedocs/api/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from django.conf import settings
from django.conf.urls import url
from django.shortcuts import get_object_or_404
from django.core.cache import cache

from tastypie import fields
from tastypie.authorization import DjangoAuthorization
Expand All @@ -25,9 +26,6 @@
log = logging.getLogger(__name__)


redis_client = redis.Redis(**settings.REDIS)


class ProjectResource(ModelResource, SearchMixin):
users = fields.ToManyField('readthedocs.api.base.UserResource', 'users')

Expand Down Expand Up @@ -223,7 +221,11 @@ def get_anchor(self, request, **kwargs):
self.throttle_check(request)

query = request.GET.get('q', '')
redis_data = redis_client.keys("*redirects:v4*%s*" % query)
try:
redis_client = cache.get_client(None)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Didn't know get_client was a thing. It doesn't appear to be documented?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems similar to https://docs.djangoproject.com/en/1.8/topics/cache/#django.core.cache.get_cache -- which is deprecated.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Seems like it should just be using django.core.cache.cache. Seems this code would be dependent on implementation of the redis backend and redis being used as a cache.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's exposed from redis_cache. If the cache backend isn't redis, this expects to catch the AttributeError that will be thrown

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, that makes sense. Guess the code depends on redis as it is :)

redis_data = redis_client.keys("*redirects:v4*%s*" % query)
except (AttributeError, redis.exceptions.ConnectionError):
redis_data = []
# -2 because http:
urls = [''.join(data.split(':')[6:]) for data in redis_data
if 'http://' in data]
Expand Down
2 changes: 1 addition & 1 deletion readthedocs/builds/views.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ def get_context_data(self, **kwargs):
context['versions'] = Version.objects.public(user=self.request.user, project=self.project)

try:
redis = Redis(**settings.REDIS)
redis = Redis.from_url(settings.BROKER_URL)
context['queue_length'] = redis.llen('celery')
except ConnectionError:
context['queue_length'] = None
Expand Down
30 changes: 17 additions & 13 deletions readthedocs/projects/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,9 @@
import logging
from httplib2 import Http

from django.conf import settings
import redis
from django.conf import settings
from django.core.cache import cache


log = logging.getLogger(__name__)
Expand Down Expand Up @@ -149,18 +150,21 @@ def purge_version(version, mainsite=False, subdomain=False, cname=False):
log.info("Purging %s on readthedocs.org", root_url)
h.request(to_purge, method="PURGE", headers=headers)
if cname:
redis_conn = redis.Redis(**settings.REDIS)
for cnamed in redis_conn.smembers('rtd_slug:v1:%s'
% version.project.slug):
headers = {'Host': cnamed}
url = "/en/%s/*" % version.slug
to_purge = "http://%s%s" % (server, url)
log.info("Purging %s on %s", url, cnamed)
h.request(to_purge, method="PURGE", headers=headers)
root_url = "/"
to_purge = "http://%s%s" % (server, root_url)
log.info("Purging %s on %s", root_url, cnamed)
h.request(to_purge, method="PURGE", headers=headers)
try:
redis_client = cache.get_client(None)
for cnamed in redis_client.smembers('rtd_slug:v1:%s'
% version.project.slug):
headers = {'Host': cnamed}
url = "/en/%s/*" % version.slug
to_purge = "http://%s%s" % (server, url)
log.info("Purging %s on %s", url, cnamed)
h.request(to_purge, method="PURGE", headers=headers)
root_url = "/"
to_purge = "http://%s%s" % (server, root_url)
log.info("Purging %s on %s", root_url, cnamed)
h.request(to_purge, method="PURGE", headers=headers)
except (AttributeError, redis.exceptions.ConnectionError):
pass


class DictObj(object):
Expand Down
5 changes: 0 additions & 5 deletions readthedocs/settings/onebox.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,6 @@
}
}

REDIS = {
'host': 'localhost',
'port': 6379,
'db': 0,
}
BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
CELERY_ALWAYS_EAGER = False
Expand Down
6 changes: 0 additions & 6 deletions readthedocs/settings/sqlite.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,6 @@
}
}

REDIS = {
'host': 'localhost',
'port': 6379,
'db': 0,
}

BROKER_URL = 'redis://localhost:6379/0'
CELERY_RESULT_BACKEND = 'redis://localhost:6379/0'
# CELERY_ALWAYS_EAGER = False
Expand Down
2 changes: 1 addition & 1 deletion requirements/deploy.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
psycopg2==2.4.6
gunicorn==19.1.0
pysolr==2.0.13
django-redis-cache==0.9.5
django-redis-cache==1.6.3
django-mailgun==0.2.2

hiredis==0.1.2
Expand Down
2 changes: 1 addition & 1 deletion requirements/pip.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ slumber==0.6.0
lxml==3.3.5

# Basic tools
redis==2.7.1
redis==2.10.3
celery==3.1.18
django-celery==3.1.16
django-allauth==0.21.0
Expand Down