Skip to content

Commit 3c7f166

Browse files
ericholscherericholscher
committed
Add Syncer backend
1 parent cc01f3f commit 3c7f166

File tree

7 files changed

+76
-91
lines changed

7 files changed

+76
-91
lines changed

‎readthedocs/core/utils.py

Lines changed: 2 additions & 81 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import getpass
22
import logging
33
import os
4-
import shutil
54

65
from urlparse import urlparse
76

@@ -11,86 +10,6 @@
1110

1211
SYNC_USER = getattr(settings, 'SYNC_USER', getpass.getuser())
1312

14-
def copy(path, target, file=False):
15-
"""
16-
A better copy command that works with files or directories.
17-
18-
Respects the ``MULTIPLE_APP_SERVERS`` setting when copying.
19-
"""
20-
MULTIPLE_APP_SERVERS = getattr(settings, 'MULTIPLE_APP_SERVERS', [])
21-
if MULTIPLE_APP_SERVERS:
22-
log.info("Remote Copy %s to %s" % (path, target))
23-
for server in MULTIPLE_APP_SERVERS:
24-
mkdir_cmd = ("ssh %s@%s mkdir -p %s" % (SYNC_USER, server, target))
25-
ret = os.system(mkdir_cmd)
26-
if ret != 0:
27-
log.error("COPY ERROR to app servers:")
28-
log.error(mkdir_cmd)
29-
30-
if file:
31-
slash = ""
32-
else:
33-
slash = "/"
34-
# Add a slash when copying directories
35-
sync_cmd = ("rsync -e 'ssh -T' -av --delete %s%s %s@%s:%s"
36-
% (path, slash, SYNC_USER, server, target))
37-
ret = os.system(sync_cmd)
38-
if ret != 0:
39-
log.error("COPY ERROR to app servers.")
40-
log.error(sync_cmd)
41-
else:
42-
log.info("Local Copy %s to %s" % (path, target))
43-
if file:
44-
if os.path.exists(target):
45-
os.remove(target)
46-
shutil.copy2(path, target)
47-
else:
48-
if os.path.exists(target):
49-
shutil.rmtree(target)
50-
shutil.copytree(path, target)
51-
52-
def copy_to_app_servers(full_build_path, target, mkdir=True):
53-
"""
54-
A helper to copy a directory across app servers
55-
"""
56-
log.info("Copying %s to %s" % (full_build_path, target))
57-
for server in getattr(settings, 'MULTIPLE_APP_SERVERS', []):
58-
mkdir_cmd = ("ssh %s@%s mkdir -p %s" % (SYNC_USER, server, target))
59-
ret = os.system(mkdir_cmd)
60-
if ret != 0:
61-
log.error("COPY ERROR to app servers:")
62-
log.error(mkdir_cmd)
63-
64-
sync_cmd = ("rsync -e 'ssh -T' -av --delete %s/ %s@%s:%s"
65-
% (full_build_path, SYNC_USER, server, target))
66-
ret = os.system(sync_cmd)
67-
if ret != 0:
68-
log.error("COPY ERROR to app servers.")
69-
log.error(sync_cmd)
70-
71-
72-
def copy_file_to_app_servers(from_file, to_file):
73-
"""
74-
A helper to copy a single file across app servers
75-
"""
76-
log.info("Copying %s to %s" % (from_file, to_file))
77-
to_path = os.path.dirname(to_file)
78-
for server in getattr(settings, 'MULTIPLE_APP_SERVERS', []):
79-
mkdir_cmd = ("ssh %s@%s mkdir -p %s" % (SYNC_USER, server, to_path))
80-
ret = os.system(mkdir_cmd)
81-
if ret != 0:
82-
log.error("COPY ERROR to app servers.")
83-
log.error(mkdir_cmd)
84-
85-
sync_cmd = ("rsync -e 'ssh -T' -av --delete %s %s@%s:%s" % (from_file,
86-
SYNC_USER,
87-
server,
88-
to_file))
89-
ret = os.system(sync_cmd)
90-
if ret != 0:
91-
log.error("COPY ERROR to app servers.")
92-
log.error(sync_cmd)
93-
9413

9514
def run_on_app_servers(command):
9615
"""
@@ -108,6 +27,7 @@ def run_on_app_servers(command):
10827
ret = os.system(command)
10928
return ret
11029

30+
11131
def make_latest(project):
11232
"""
11333
Useful for correcting versions with no latest, using the database.
@@ -137,6 +57,7 @@ def clean_url(url):
13757
scheme, netloc = "http", parsed.path
13858
return netloc
13959

60+
14061
def cname_to_slug(host):
14162
from dns import resolver
14263
answer = [ans for ans in resolver.query(host, 'CNAME')][0]

‎readthedocs/privacy/backends/__init__.py

Whitespace-only changes.

‎readthedocs/privacy/backends/managers.py

Whitespace-only changes.

‎readthedocs/privacy/backends/permissions.py

Whitespace-only changes.

‎readthedocs/privacy/backends/syncers.py

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import getpass
2+
import logging
3+
import os
4+
import shutil
5+
6+
from django.conf import settings
7+
8+
log = logging.getLogger(__name__)
9+
10+
11+
class LocalSyncer(object):
12+
13+
@classmethod
14+
def copy(path, target, file=False):
15+
"""
16+
A copy command that works with files or directories.
17+
"""
18+
log.info("Local Copy %s to %s" % (path, target))
19+
if file:
20+
if os.path.exists(target):
21+
os.remove(target)
22+
shutil.copy2(path, target)
23+
else:
24+
if os.path.exists(target):
25+
shutil.rmtree(target)
26+
shutil.copytree(path, target)
27+
28+
29+
class RemoteSyncer(object):
30+
31+
@classmethod
32+
def copy(path, target, file=False):
33+
"""
34+
A better copy command that works with files or directories.
35+
36+
Respects the ``MULTIPLE_APP_SERVERS`` setting when copying.
37+
"""
38+
SYNC_USER = getattr(settings, 'SYNC_USER', getpass.getuser())
39+
MULTIPLE_APP_SERVERS = getattr(settings, 'MULTIPLE_APP_SERVERS', [])
40+
if MULTIPLE_APP_SERVERS:
41+
log.info("Remote Copy %s to %s" % (path, target))
42+
for server in MULTIPLE_APP_SERVERS:
43+
mkdir_cmd = ("ssh %s@%s mkdir -p %s" % (SYNC_USER, server, target))
44+
ret = os.system(mkdir_cmd)
45+
if ret != 0:
46+
log.error("COPY ERROR to app servers:")
47+
log.error(mkdir_cmd)
48+
if file:
49+
slash = ""
50+
else:
51+
slash = "/"
52+
# Add a slash when copying directories
53+
sync_cmd = "rsync -e 'ssh -T' -av --delete {path}{slash} {user}@{server}:{target}".format(
54+
path=path,
55+
slash=slash,
56+
user=SYNC_USER,
57+
server=server,
58+
target=target,
59+
)
60+
ret = os.system(sync_cmd)
61+
if ret != 0:
62+
log.error("COPY ERROR to app servers.")
63+
log.error(sync_cmd)

‎readthedocs/privacy/loader.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,6 @@
77

88
# Permissions
99
AdminPermission = import_by_path(getattr(settings, 'ADMIN_PERMISSION', 'privacy.backend.AdminPermission'))
10+
11+
# Syncers
12+
Syncer = import_by_path(getattr(settings, 'FILE_SYNCER', 'privacy.backends.syncers.LocalSyncer'))

‎readthedocs/projects/tasks.py

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,8 @@
2424
make_api_version, make_api_project)
2525
from projects.constants import LOG_TEMPLATE
2626
from projects import symlinks
27+
from privacy.loader import Syncer
2728
from tastyapi import api, apiv2
28-
from core.utils import (copy_to_app_servers, copy_file_to_app_servers,
29-
run_on_app_servers)
30-
from core import utils as core_utils
3129
from search.parse_json import process_all_json_files
3230
from search.utils import process_mkdocs_json
3331
from vcs_support import utils as vcs_support_utils
@@ -138,39 +136,39 @@ def move_files(version, results):
138136
from_path = version.project.artifact_path(
139137
version=version.slug, type=version.project.documentation_type)
140138
target = version.project.rtd_build_path(version.slug)
141-
core_utils.copy(from_path, target)
139+
Syncer.copy(from_path, target)
142140

143141
if 'sphinx' in version.project.documentation_type:
144142
if 'localmedia' in results and results['localmedia'][0] == 0:
145143
from_path = version.project.artifact_path(
146144
version=version.slug, type='sphinx_localmedia')
147145
to_path = version.project.get_production_media_path(type='htmlzip', version_slug=version.slug, include_file=False)
148-
core_utils.copy(from_path, to_path)
146+
Syncer.copy(from_path, to_path)
149147
if 'search' in results and results['search'][0] == 0:
150148
from_path = version.project.artifact_path(
151149
version=version.slug, type='sphinx_search')
152150
to_path = version.project.get_production_media_path(type='json', version_slug=version.slug, include_file=False)
153-
core_utils.copy(from_path, to_path)
151+
Syncer.copy(from_path, to_path)
154152
# Always move PDF's because the return code lies.
155153
if 'pdf' in results:
156154
try:
157155
from_path = version.project.artifact_path(
158156
version=version.slug, type='sphinx_pdf')
159157
to_path = version.project.get_production_media_path(type='pdf', version_slug=version.slug, include_file=False)
160-
core_utils.copy(from_path, to_path)
158+
Syncer.copy(from_path, to_path)
161159
except:
162160
pass
163161
if 'epub' in results and results['epub'][0] == 0:
164162
from_path = version.project.artifact_path(
165163
version=version.slug, type='sphinx_epub')
166164
to_path = version.project.get_production_media_path(type='epub', version_slug=version.slug, include_file=False)
167-
core_utils.copy(from_path, to_path)
165+
Syncer.copy(from_path, to_path)
168166

169167
if 'mkdocs' in version.project.documentation_type:
170168
if 'search' in results and results['search'][0] == 0:
171169
from_path = version.project.artifact_path(version=version.slug, type='mkdocs_json')
172170
to_path = version.project.get_production_media_path(type='json', version_slug=version.slug, include_file=False)
173-
core_utils.copy(from_path, to_path)
171+
Syncer.copy(from_path, to_path)
174172

175173

176174
def run_docker(version):
@@ -604,7 +602,7 @@ def update_static_metadata(project_pk):
604602
fh = open(path, 'w')
605603
json.dump(metadata, fh)
606604
fh.close()
607-
copy_file_to_app_servers(path, path)
605+
Syncer.copy(path, path, file=True)
608606
except (AttributeError, IOError) as e:
609607
log.debug(LOG_TEMPLATE.format(
610608
project=project.slug,

0 commit comments

Comments
 (0)