Skip to content

Add --jobs to allow for some parallelism. #45

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Jul 21, 2018
Merged

Add --jobs to allow for some parallelism. #45

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
b9f2bed
Don't rely on git failing on non-git folders (it may find a repo, low…
JulienPalard Jul 21, 2018
49bb7f8
argparse givs us a float.
JulienPalard Jul 21, 2018
00ea79c
Bump sphinx version: fixes (again) PDF for french.
JulienPalard Jul 21, 2018
cf5689c
Add --jobs to allow for some parallelism.
JulienPalard Mar 24, 2018
1e5f328
jobs: Dropping chdir (process-wide).
JulienPalard Mar 24, 2018
24fb8be
Cleaner make capture.
JulienPalard Mar 25, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 60 additions & 34 deletions build_docs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@

"""

from concurrent.futures import ThreadPoolExecutor, wait, ALL_COMPLETED
from datetime import datetime
import logging
import os
import subprocess
Expand Down Expand Up @@ -88,13 +90,19 @@ def _file_unchanged(old, new):
return True


def shell_out(cmd, shell=False):
def shell_out(cmd, shell=False, logfile=None):
logging.debug("Running command %r", cmd)
try:
return subprocess.check_output(cmd, shell=shell,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
output = subprocess.check_output(cmd, shell=shell,
stdin=subprocess.PIPE,
stderr=subprocess.STDOUT,
universal_newlines=True)
if logfile:
with open(logfile, 'a+') as log:
log.write("#" + str(datetime.now()))
log.write(output)
log.write("\n\n")
return output
except subprocess.CalledProcessError as e:
logging.debug("Command failed with output %r", e.output)
raise
Expand Down Expand Up @@ -123,10 +131,12 @@ def git_clone(repository, directory, branch=None):
"""
logging.info("Updating repository %s in %s", repository, directory)
try:
if not os.path.isdir(os.path.join(directory, '.git')):
raise AssertionError("Not a git repository.")
if branch:
shell_out(['git', '-C', directory, 'checkout', branch])
shell_out(['git', '-C', directory, 'pull', '--ff-only'])
except subprocess.CalledProcessError:
except (subprocess.CalledProcessError, AssertionError):
if os.path.exists(directory):
shutil.rmtree(directory)
logging.info("Cloning %s into %s", repository, directory)
Expand Down Expand Up @@ -174,9 +184,8 @@ def build_one(version, git_branch, isdev, quick, venv, build_root, www_root,
log_directory='/var/log/docsbuild/', language=None):
if not language:
language = 'en'
checkout = os.path.join(build_root, 'python{version}{lang}'.format(
version=str(version).replace('.', ''),
lang=language if language != 'en' else ''))
checkout = os.path.join(
build_root, str(version), 'cpython-{lang}'.format(lang=language))
logging.info("Build start for version: %s, language: %s",
str(version), language)
sphinxopts = SPHINXOPTS[language].copy()
Expand All @@ -194,7 +203,7 @@ def build_one(version, git_branch, isdev, quick, venv, build_root, www_root,
os.chmod(language_dir, 0o775)
target = os.path.join(language_dir, str(version))
gettext_language_tag = pep_545_tag_to_gettext_tag(language)
locale_dirs = os.path.join(build_root, 'locale')
locale_dirs = os.path.join(build_root, str(version), 'locale')
locale_clone_dir = os.path.join(
locale_dirs, gettext_language_tag, 'LC_MESSAGES')
locale_repo = 'https://github.com/python/python-docs-{}.git'.format(
Expand All @@ -217,34 +226,39 @@ def build_one(version, git_branch, isdev, quick, venv, build_root, www_root,
logging.warning("Can't change group of %s: %s", target, str(err))
git_clone('https://github.com/python/cpython.git',
checkout, git_branch)
os.chdir(checkout)
maketarget = "autobuild-" + ("dev" if isdev else "stable") + ("-html" if quick else "")
logging.info("Running make %s", maketarget)
logname = "{}-{}.log".format(os.path.basename(checkout), language)
logname = "{}.log".format(os.path.basename(checkout))
python = os.path.join(venv, "bin/python")
sphinxbuild = os.path.join(venv, "bin/sphinx-build")
blurb = os.path.join(venv, "bin/blurb")
shell_out(
"cd Doc; make PYTHON=%s SPHINXBUILD=%s BLURB=%s VENVDIR=%s SPHINXOPTS='%s' %s >> %s 2>&1" %
(python, sphinxbuild, blurb, venv, ' '.join(sphinxopts), maketarget,
os.path.join(log_directory, logname)), shell=True)
["make",
"-C", os.path.join(checkout, 'Doc'),
"PYTHON=" + python,
"SPHINXBUILD=" + sphinxbuild,
"BLURB=" + blurb,
"VENVDIR=" + venv,
"SPHINXOPTS=" + ' '.join(sphinxopts),
maketarget],
logfile=os.path.join(log_directory, logname))
shell_out(['chgrp', '-R', group, log_directory])
changed = changed_files(os.path.join(checkout, "Doc/build/html"), target)
logging.info("Copying HTML files to %s", target)
shell_out(['chown', '-R', ':' + group, 'Doc/build/html/'])
shell_out(['chmod', '-R', 'o+r', 'Doc/build/html/'])
shell_out(['find', 'Doc/build/html/', '-type', 'd',
shell_out(['chown', '-R', ':' + group, os.path.join(checkout, 'Doc/build/html/')])
shell_out(['chmod', '-R', 'o+r', os.path.join(checkout, 'Doc/build/html/')])
shell_out(['find', os.path.join(checkout, 'Doc/build/html/'), '-type', 'd',
'-exec', 'chmod', 'o+x', '{}', ';'])
if quick:
shell_out(['rsync', '-a', 'Doc/build/html/', target])
shell_out(['rsync', '-a', os.path.join(checkout, 'Doc/build/html/'), target])
else:
shell_out(['rsync', '-a', '--delete-delay', 'Doc/build/html/', target])
shell_out(['rsync', '-a', '--delete-delay', os.path.join(checkout, 'Doc/build/html/'), target])
if not quick:
logging.debug("Copying dist files")
shell_out(['chown', '-R', ':' + group, 'Doc/dist/'])
shell_out(['chmod', '-R', 'o+r', 'Doc/dist/'])
shell_out(['mkdir', '-m', 'o+rx', '-p', target + '/archives'])
shell_out(['chown', ':' + group, target + '/archives'])
shell_out(['chown', '-R', ':' + group, os.path.join(checkout, 'Doc/dist/')])
shell_out(['chmod', '-R', 'o+r', os.path.join('Doc/dist/')])
shell_out(['mkdir', '-m', 'o+rx', '-p', os.path.join(target, 'archives')])
shell_out(['chown', ':' + group, os.path.join(target, 'archives')])
shell_out("cp -a Doc/dist/* %s/archives" % target, shell=True)
changed.append("archives/")
for fn in os.listdir(os.path.join(target, "archives")):
Expand Down Expand Up @@ -317,6 +331,12 @@ def parse_args():
help="Language translation, as a PEP 545 language tag like"
" 'fr' or 'pt-br'.",
metavar='fr')
parser.add_argument(
"--jobs", "-j",
type=int,
default=4,
help="Specifies the number of jobs (languages, versions) "
"to run simultaneously.")
return parser.parse_args()


Expand All @@ -338,23 +358,29 @@ def main():
logging.root.setLevel(logging.DEBUG)
venv = os.path.join(args.build_root, "venv")
if args.branch:
branches_to_do = [(args.branch, args.branch, args.devel)]
branches_to_do = [(args.branch, str(args.branch), args.devel)]
else:
branches_to_do = BRANCHES
if not args.languages:
# Allow "--languages" to build all languages (as if not given)
# instead of none. "--languages en" builds *no* translation,
# as "en" is the untranslated one.
args.languages = LANGUAGES
for version, git_branch, devel in branches_to_do:
for language in args.languages:
try:
build_one(version, git_branch, devel, args.quick, venv,
args.build_root, args.www_root,
args.skip_cache_invalidation, args.group,
args.log_directory, language)
except Exception:
logging.exception("docs build raised exception")
with ThreadPoolExecutor(max_workers=args.jobs) as executor:
futures = []
for version, git_branch, devel in branches_to_do:
for language in args.languages:
futures.append((version, language, executor.submit(
build_one, version, git_branch, devel, args.quick, venv,
args.build_root, args.www_root,
args.skip_cache_invalidation, args.group,
args.log_directory, language)))
wait([future[2] for future in futures], return_when=ALL_COMPLETED)
for version, language, future in futures:
if future.exception():
logging.error("Exception while building %s version %s: %s",
language, version,
future.exception())


if __name__ == '__main__':
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,6 @@ pytz==2018.4
requests==2.19.1
six==1.11.0
snowballstemmer==1.2.1
Sphinx==1.7.5
Sphinx==1.7.6
sphinxcontrib-websupport==1.1.0
urllib3==1.23