Merge remote-tracking branch 'upstream/master' into pandas-rosy-new

Author: RjLi13

.gitignore

@@ -101,14 +101,14 @@ asv_bench/pandas/
 # Documentation generated files #
 #################################
 doc/source/generated
-doc/source/api/generated
+doc/source/user_guide/styled.xlsx
+doc/source/reference/api
 doc/source/_static
 doc/source/vbench
 doc/source/vbench.rst
 doc/source/index.rst
 doc/build/html/index.html
 # Windows specific leftover:
 doc/tmp.sv
-doc/source/styled.xlsx
 env/
 doc/source/savefig/

asv_bench/benchmarks/categoricals.py

@@ -223,12 +223,19 @@ class CategoricalSlicing(object):
 
     def setup(self, index):
         N = 10**6
-        values = list('a' * N + 'b' * N + 'c' * N)
-        indices = {
-            'monotonic_incr': pd.Categorical(values),
-            'monotonic_decr': pd.Categorical(reversed(values)),
-            'non_monotonic': pd.Categorical(list('abc' * N))}
-        self.data = indices[index]
+        categories = ['a', 'b', 'c']
+        values = [0] * N + [1] * N + [2] * N
+        if index == 'monotonic_incr':
+            self.data = pd.Categorical.from_codes(values,
+                                                  categories=categories)
+        elif index == 'monotonic_decr':
+            self.data = pd.Categorical.from_codes(list(reversed(values)),
+                                                  categories=categories)
+        elif index == 'non_monotonic':
+            self.data = pd.Categorical.from_codes([0, 1, 2] * N,
+                                                  categories=categories)
+        else:
+            raise ValueError('Invalid index param: {}'.format(index))
 
         self.scalar = 10000
         self.list = list(range(10000))

asv_bench/benchmarks/ctors.py

@@ -72,7 +72,7 @@ class SeriesDtypesConstructors(object):
 
     def setup(self):
         N = 10**4
-        self.arr = np.random.randn(N, N)
+        self.arr = np.random.randn(N)
         self.arr_str = np.array(['foo', 'bar', 'baz'], dtype=object)
         self.s = Series([Timestamp('20110101'), Timestamp('20120101'),
                          Timestamp('20130101')] * N * 10)

asv_bench/benchmarks/index_object.py

@@ -138,7 +138,8 @@ def setup(self, dtype):
         self.sorted = self.idx.sort_values()
         half = N // 2
         self.non_unique = self.idx[:half].append(self.idx[:half])
-        self.non_unique_sorted = self.sorted[:half].append(self.sorted[:half])
+        self.non_unique_sorted = (self.sorted[:half].append(self.sorted[:half])
+                                  .sort_values())
         self.key = self.sorted[N // 4]
 
     def time_boolean_array(self, dtype):

doc/make.py

@@ -15,15 +15,18 @@
 import sys
 import os
 import shutil
+import csv
 import subprocess
 import argparse
 import webbrowser
+import docutils
+import docutils.parsers.rst
 
 
 DOC_PATH = os.path.dirname(os.path.abspath(__file__))
 SOURCE_PATH = os.path.join(DOC_PATH, 'source')
 BUILD_PATH = os.path.join(DOC_PATH, 'build')
-BUILD_DIRS = ['doctrees', 'html', 'latex', 'plots', '_static', '_templates']
+REDIRECTS_FILE = os.path.join(DOC_PATH, 'redirects.csv')
 
 
 class DocBuilder:
@@ -50,7 +53,7 @@ def __init__(self, num_jobs=0, include_api=True, single_doc=None,
         if single_doc and single_doc.endswith('.rst'):
             self.single_doc_html = os.path.splitext(single_doc)[0] + '.html'
         elif single_doc:
-            self.single_doc_html = 'api/generated/pandas.{}.html'.format(
+            self.single_doc_html = 'reference/api/pandas.{}.html'.format(
                 single_doc)
 
     def _process_single_doc(self, single_doc):
@@ -60,7 +63,7 @@ def _process_single_doc(self, single_doc):
 
         For example, categorial.rst or pandas.DataFrame.head. For the latter,
         return the corresponding file path
-        (e.g. generated/pandas.DataFrame.head.rst).
+        (e.g. reference/api/pandas.DataFrame.head.rst).
         """
         base_name, extension = os.path.splitext(single_doc)
         if extension in ('.rst', '.ipynb'):
@@ -118,8 +121,6 @@ def _sphinx_build(self, kind):
             raise ValueError('kind must be html or latex, '
                              'not {}'.format(kind))
 
-        self.clean()
-
         cmd = ['sphinx-build', '-b', kind]
         if self.num_jobs:
             cmd += ['-j', str(self.num_jobs)]
@@ -139,6 +140,77 @@ def _open_browser(self, single_doc_html):
                            single_doc_html)
         webbrowser.open(url, new=2)
 
+    def _get_page_title(self, page):
+        """
+        Open the rst file `page` and extract its title.
+        """
+        fname = os.path.join(SOURCE_PATH, '{}.rst'.format(page))
+        option_parser = docutils.frontend.OptionParser(
+            components=(docutils.parsers.rst.Parser,))
+        doc = docutils.utils.new_document(
+            '<doc>',
+            option_parser.get_default_values())
+        with open(fname) as f:
+            data = f.read()
+
+        parser = docutils.parsers.rst.Parser()
+        # do not generate any warning when parsing the rst
+        with open(os.devnull, 'a') as f:
+            doc.reporter.stream = f
+            parser.parse(data, doc)
+
+        section = next(node for node in doc.children
+                       if isinstance(node, docutils.nodes.section))
+        title = next(node for node in section.children
+                     if isinstance(node, docutils.nodes.title))
+
+        return title.astext()
+
+    def _add_redirects(self):
+        """
+        Create in the build directory an html file with a redirect,
+        for every row in REDIRECTS_FILE.
+        """
+        html = '''
+        <html>
+            <head>
+                <meta http-equiv="refresh" content="0;URL={url}"/>
+            </head>
+            <body>
+                <p>
+                    The page has been moved to <a href="{url}">{title}</a>
+                </p>
+            </body>
+        <html>
+        '''
+        with open(REDIRECTS_FILE) as mapping_fd:
+            reader = csv.reader(mapping_fd)
+            for row in reader:
+                if not row or row[0].strip().startswith('#'):
+                    continue
+
+                path = os.path.join(BUILD_PATH,
+                                    'html',
+                                    *row[0].split('/')) + '.html'
+
+                try:
+                    title = self._get_page_title(row[1])
+                except Exception:
+                    # the file can be an ipynb and not an rst, or docutils
+                    # may not be able to read the rst because it has some
+                    # sphinx specific stuff
+                    title = 'this page'
+
+                if os.path.exists(path):
+                    raise RuntimeError((
+                        'Redirection would overwrite an existing file: '
+                        '{}').format(path))
+
+                with open(path, 'w') as moved_page_fd:
+                    moved_page_fd.write(
+                        html.format(url='{}.html'.format(row[1]),
+                                    title=title))
+
     def html(self):
         """
         Build HTML documentation.
@@ -150,6 +222,8 @@ def html(self):
 
         if self.single_doc_html is not None:
             self._open_browser(self.single_doc_html)
+        else:
+            self._add_redirects()
         return ret_code
 
     def latex(self, force=False):
@@ -184,7 +258,7 @@ def clean():
         Clean documentation generated files.
         """
         shutil.rmtree(BUILD_PATH, ignore_errors=True)
-        shutil.rmtree(os.path.join(SOURCE_PATH, 'api', 'generated'),
+        shutil.rmtree(os.path.join(SOURCE_PATH, 'reference', 'api'),
                       ignore_errors=True)
 
     def zip_html(self):