CLN: Lint for lists instead of generators in built-in Python functions

Author: mroeschke

ci/lint.sh

@@ -84,6 +84,19 @@ if [ "$LINT" ]; then
     fi
     echo "Check for invalid testing DONE"
 
+    echo "Check for use of lists in built-in Python functions"
+
+    # Example: Avoid `any([i for i in some_iterator])` in favor of `any(i for i in some_iterator)`
+    #
+    # Check the following functions:
+    # any(), all(), sum(), max(), min(), list(), dict(), set(), frozenset(), tuple(), str.join()
+    grep -R --include="*.py*" -E "[^_](any|all|sum|max|min|list|dict|set|frozenset|tuple|join)\(\[.* for .* in .*\]\)"
+
+    if [ $? = "0" ]; then
+        RET=1
+    fi
+    echo "Check for use of lists in built-in Python functions DONE"
+
 else
     echo "NOT Linting"
 fi

pandas/_libs/parsers.pyx

@@ -770,7 +770,7 @@ cdef class TextReader:
                     msg = self.orig_header
                     if isinstance(msg, list):
                         msg = "[%s], len of %d," % (
-                            ','.join([ str(m) for m in msg ]), len(msg))
+                            ','.join(str(m) for m in msg), len(msg))
                     raise ParserError(
                         'Passed header=%s but only %d lines in file'
                         % (msg, self.parser.lines))
@@ -2227,7 +2227,7 @@ def _concatenate_chunks(list chunks):
     for name in names:
         arrs = [chunk.pop(name) for chunk in chunks]
         # Check each arr for consistent types.
-        dtypes = set([a.dtype for a in arrs])
+        dtypes = set(a.dtype for a in arrs)
         if len(dtypes) > 1:
             common_type = np.find_common_type(dtypes, [])
             if common_type == np.object:

pandas/_libs/src/inference.pyx

@@ -1309,7 +1309,7 @@ def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
 
     # we try to coerce datetime w/tz but must all have the same tz
     if seen.datetimetz_:
-        if len(set([getattr(val, 'tzinfo', None) for val in objects])) == 1:
+        if len(set(getattr(val, 'tzinfo', None) for val in objects)) == 1:
             from pandas import DatetimeIndex
             return DatetimeIndex(objects)
         seen.object_ = 1

pandas/_libs/tslibs/resolution.pyx

@@ -218,7 +218,7 @@ class Resolution(object):
         'U': 'N',
         'N': None}
 
-    _str_reso_map = dict([(v, k) for k, v in _reso_str_map.items()])
+    _str_reso_map = dict((v, k) for k, v in _reso_str_map.items())
 
     _reso_freq_map = {
         'year': 'A',
@@ -232,8 +232,7 @@ class Resolution(object):
         'microsecond': 'U',
         'nanosecond': 'N'}
 
-    _freq_reso_map = dict([(v, k)
-                           for k, v in _reso_freq_map.items()])
+    _freq_reso_map = dict((v, k) for k, v in _reso_freq_map.items())
 
     @classmethod
     def get_str(cls, reso):

pandas/_libs/tslibs/strptime.pyx

@@ -568,7 +568,7 @@ class TimeRE(dict):
                 break
         else:
             return ''
-        regex = '|'.join([re.escape(stuff) for stuff in to_convert])
+        regex = '|'.join(re.escape(stuff) for stuff in to_convert)
         regex = '(?P<%s>%s' % (directive, regex)
         return '%s)' % regex
 

pandas/_version.py

@@ -141,11 +141,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         if verbose:
             print("keywords are unexpanded, not using")
         raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    refs = set(r.strip() for r in refnames.strip("()").split(","))
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
     TAG = "tag: "
-    tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)])
+    tags = set(r[len(TAG):] for r in refs if r.startswith(TAG))
     if not tags:
         # Either we're using git < 1.8.3, or there really are no tags. We use
         # a heuristic: assume all version tags have a digit. The old git %d
@@ -154,7 +154,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
         # "stabilization", as well as "HEAD" and "master".
-        tags = set([r for r in refs if re.search(r'\d', r)])
+        tags = set(r for r in refs if re.search(r'\d', r))
         if verbose:
             print("discarding '{}', no digits".format(",".join(refs - tags)))
     if verbose:

pandas/core/common.py

@@ -347,7 +347,7 @@ def map_indices_py(arr):
     Returns a dictionary with (element, index) pairs for each element in the
     given array/list
     """
-    return dict([(x, i) for i, x in enumerate(arr)])
+    return dict((x, i) for i, x in enumerate(arr))
 
 
 def union(*seqs):

pandas/core/dtypes/concat.py

@@ -459,7 +459,7 @@ def _concat_datetimetz(to_concat, name=None):
     it is used in DatetimeIndex.append also
     """
     # do not pass tz to set because tzlocal cannot be hashed
-    if len(set([str(x.dtype) for x in to_concat])) != 1:
+    if len(set(str(x.dtype) for x in to_concat)) != 1:
         raise ValueError('to_concat must have the same tz')
     tz = to_concat[0].tz
     # no need to localize because internal repr will not be changed

pandas/core/frame.py

@@ -3895,7 +3895,7 @@ def f(col):
                     return self._constructor_sliced(r, index=new_index,
                                                     dtype=r.dtype)
 
-                result = dict([(col, f(col)) for col in this])
+                result = dict((col, f(col)) for col in this)
 
             # non-unique
             else:
@@ -3906,9 +3906,7 @@ def f(i):
                     return self._constructor_sliced(r, index=new_index,
                                                     dtype=r.dtype)
 
-                result = dict([
-                    (i, f(i)) for i, col in enumerate(this.columns)
-                ])
+                result = dict((i, f(i)) for i, col in enumerate(this.columns))
                 result = self._constructor(result, index=new_index, copy=False)
                 result.columns = new_columns
                 return result
@@ -3986,7 +3984,7 @@ def _compare_frame_evaluate(self, other, func, str_rep, try_cast=True):
         if self.columns.is_unique:
 
             def _compare(a, b):
-                return dict([(col, func(a[col], b[col])) for col in a.columns])
+                return dict((col, func(a[col], b[col])) for col in a.columns)
 
             new_data = expressions.evaluate(_compare, str_rep, self, other)
             return self._constructor(data=new_data, index=self.index,
@@ -3995,8 +3993,8 @@ def _compare(a, b):
         else:
 
             def _compare(a, b):
-                return dict([(i, func(a.iloc[:, i], b.iloc[:, i]))
-                             for i, col in enumerate(a.columns)])
+                return dict((i, func(a.iloc[:, i], b.iloc[:, i]))
+                            for i, col in enumerate(a.columns))
 
             new_data = expressions.evaluate(_compare, str_rep, self, other)
             result = self._constructor(data=new_data, index=self.index,

pandas/core/generic.py

@@ -279,21 +279,21 @@ def set_axis(a, i):
 
     def _construct_axes_dict(self, axes=None, **kwargs):
         """Return an axes dictionary for myself."""
-        d = dict([(a, self._get_axis(a)) for a in (axes or self._AXIS_ORDERS)])
+        d = dict((a, self._get_axis(a)) for a in (axes or self._AXIS_ORDERS))
         d.update(kwargs)
         return d
 
     @staticmethod
     def _construct_axes_dict_from(self, axes, **kwargs):
         """Return an axes dictionary for the passed axes."""
-        d = dict([(a, ax) for a, ax in zip(self._AXIS_ORDERS, axes)])
+        d = dict((a, ax) for a, ax in zip(self._AXIS_ORDERS, axes))
         d.update(kwargs)
         return d
 
     def _construct_axes_dict_for_slice(self, axes=None, **kwargs):
         """Return an axes dictionary for myself."""
-        d = dict([(self._AXIS_SLICEMAP[a], self._get_axis(a))
-                  for a in (axes or self._AXIS_ORDERS)])
+        d = dict((self._AXIS_SLICEMAP[a], self._get_axis(a))
+                 for a in (axes or self._AXIS_ORDERS))
         d.update(kwargs)
         return d
 
@@ -329,7 +329,7 @@ def _construct_axes_from_arguments(self, args, kwargs, require_all=False):
                         raise TypeError("not enough/duplicate arguments "
                                         "specified!")
 
-        axes = dict([(a, kwargs.pop(a, None)) for a in self._AXIS_ORDERS])
+        axes = dict((a, kwargs.pop(a, None)) for a in self._AXIS_ORDERS)
         return axes, kwargs
 
     @classmethod
@@ -586,10 +586,10 @@ def transpose(self, *args, **kwargs):
         # construct the args
         axes, kwargs = self._construct_axes_from_arguments(args, kwargs,
                                                            require_all=True)
-        axes_names = tuple([self._get_axis_name(axes[a])
-                            for a in self._AXIS_ORDERS])
-        axes_numbers = tuple([self._get_axis_number(axes[a])
-                              for a in self._AXIS_ORDERS])
+        axes_names = tuple(self._get_axis_name(axes[a])
+                           for a in self._AXIS_ORDERS)
+        axes_numbers = tuple(self._get_axis_number(axes[a])
+                             for a in self._AXIS_ORDERS)
 
         # we must have unique axes
         if len(axes) != len(set(axes)):
@@ -699,8 +699,8 @@ def squeeze(self, axis=None):
                 (self._get_axis_number(axis),))
         try:
             return self.iloc[
-                tuple([0 if i in axis and len(a) == 1 else slice(None)
-                       for i, a in enumerate(self.axes)])]
+                tuple(0 if i in axis and len(a) == 1 else slice(None)
+                      for i, a in enumerate(self.axes))]
         except Exception:
             return self
 
@@ -4277,8 +4277,8 @@ def fillna(self, value=None, method=None, axis=None, inplace=False,
             elif self.ndim == 3:
 
                 # fill in 2d chunks
-                result = dict([(col, s.fillna(method=method, value=value))
-                               for col, s in self.iteritems()])
+                result = dict((col, s.fillna(method=method, value=value))
+                              for col, s in self.iteritems())
                 new_obj = self._constructor.\
                     from_dict(result).__finalize__(self)
                 new_data = new_obj._data

pandas/core/groupby.py

@@ -471,7 +471,7 @@ def get_converter(s):
                     raise ValueError(msg)
 
             converters = [get_converter(s) for s in index_sample]
-            names = [tuple([f(n) for f, n in zip(converters, name)])
+            names = [tuple(f(n) for f, n in zip(converters, name))
                      for name in names]
 
         else:

pandas/core/indexes/api.py

@@ -101,7 +101,7 @@ def conv(i):
 
 
 def _sanitize_and_check(indexes):
-    kinds = list(set([type(index) for index in indexes]))
+    kinds = list(set(type(index) for index in indexes))
 
     if list in kinds:
         if len(kinds) > 1:
@@ -122,8 +122,8 @@ def _get_consensus_names(indexes):
 
     # find the non-none names, need to tupleify to make
     # the set hashable, then reverse on return
-    consensus_names = set([tuple(i.names) for i in indexes
-                           if com._any_not_none(*i.names)])
+    consensus_names = set(tuple(i.names) for i in indexes
+                          if com._any_not_none(*i.names))
     if len(consensus_names) == 1:
         return list(list(consensus_names)[0])
     return [None] * indexes[0].nlevels

pandas/core/indexes/base.py

@@ -732,7 +732,7 @@ def _coerce_to_ndarray(cls, data):
 
     def _get_attributes_dict(self):
         """ return an attributes dict for my class """
-        return dict([(k, getattr(self, k, None)) for k in self._attributes])
+        return dict((k, getattr(self, k, None)) for k in self._attributes)
 
     def view(self, cls=None):
 
@@ -838,7 +838,7 @@ def __unicode__(self):
         space = self._format_space()
 
         prepr = (u(",%s") %
-                 space).join([u("%s=%s") % (k, v) for k, v in attrs])
+                 space).join(u("%s=%s") % (k, v) for k, v in attrs)
 
         # no data provided, just attributes
         if data is None:
@@ -1781,7 +1781,7 @@ def append(self, other):
             if not isinstance(obj, Index):
                 raise TypeError('all inputs must be Index')
 
-        names = set([obj.name for obj in to_concat])
+        names = set(obj.name for obj in to_concat)
         name = None if len(names) > 1 else self.name
 
         return self._concat(to_concat, name)

pandas/core/indexes/interval.py

@@ -1002,7 +1002,7 @@ def _concat_same_dtype(self, to_concat, name):
         assert that we all have the same .closed
         we allow a 0-len index here as well
         """
-        if not len(set([i.closed for i in to_concat if len(i)])) == 1:
+        if not len(set(i.closed for i in to_concat if len(i))) == 1:
             msg = ('can only append two IntervalIndex objects '
                    'that are closed on the same side')
             raise ValueError(msg)

pandas/core/indexing.py

@@ -2055,7 +2055,7 @@ def get_indexer(_i, _idx):
         return (axes[_i].get_loc(_idx['key']) if isinstance(_idx, dict) else
                 _idx)
 
-    return tuple([get_indexer(_i, _idx) for _i, _idx in enumerate(indexer)])
+    return tuple(get_indexer(_i, _idx) for _i, _idx in enumerate(indexer))
 
 
 def maybe_convert_indices(indices, n):

pandas/core/internals.py

@@ -240,7 +240,7 @@ def __unicode__(self):
 
         else:
 
-            shape = ' x '.join([pprint_thing(s) for s in self.shape])
+            shape = ' x '.join(pprint_thing(s) for s in self.shape)
             result = '%s: %s, %s, dtype: %s' % (name, pprint_thing(
                 self.mgr_locs.indexer), shape, self.dtype)
 
@@ -3365,7 +3365,7 @@ def reduction(self, f, axis=0, consolidate=True, transposed=False,
             blocks.append(block)
 
         # note that some DatetimeTZ, Categorical are always ndim==1
-        ndim = set([b.ndim for b in blocks])
+        ndim = set(b.ndim for b in blocks)
 
         if 2 in ndim:
 
@@ -4858,7 +4858,7 @@ def _merge_blocks(blocks, dtype=None, _can_consolidate=True):
     if _can_consolidate:
 
         if dtype is None:
-            if len(set([b.dtype for b in blocks])) != 1:
+            if len(set(b.dtype for b in blocks)) != 1:
                 raise AssertionError("_merge_blocks are invalid!")
             dtype = blocks[0].dtype
 

pandas/core/panel.py

@@ -1080,7 +1080,7 @@ def _apply_1d(self, func, axis):
         for i in range(np.prod(shape)):
 
             # construct the object
-            pts = tuple([p[i] for p in points])
+            pts = tuple(p[i] for p in points)
             indexer.put(indlist, slice_indexer)
 
             obj = Series(values[tuple(indexer)], index=slice_axis, name=pts)
@@ -1417,10 +1417,10 @@ def _extract_axes(self, data, axes, **kwargs):
     @staticmethod
     def _extract_axes_for_slice(self, axes):
         """ return the slice dictionary for these axes """
-        return dict([(self._AXIS_SLICEMAP[i], a)
-                     for i, a in zip(
-                         self._AXIS_ORDERS[self._AXIS_LEN - len(axes):],
-                         axes)])
+        return dict((self._AXIS_SLICEMAP[i], a)
+                    for i, a in zip(
+                        self._AXIS_ORDERS[self._AXIS_LEN - len(axes):],
+                        axes))
 
     @staticmethod
     def _prep_ndarray(self, values, copy=True):
@@ -1468,8 +1468,8 @@ def _homogenize_dict(self, frames, intersect=True, dtype=None):
                 adj_frames[k] = v
 
         axes = self._AXIS_ORDERS[1:]
-        axes_dict = dict([(a, ax) for a, ax in zip(axes, self._extract_axes(
-            self, adj_frames, axes, intersect=intersect))])
+        axes_dict = dict((a, ax) for a, ax in zip(axes, self._extract_axes(
+            self, adj_frames, axes, intersect=intersect)))
 
         reindex_dict = dict(
             [(self._AXIS_SLICEMAP[a], axes_dict[a]) for a in axes])

pandas/core/panelnd.py

@@ -105,7 +105,7 @@ def _combine_with_constructor(self, other, func):
             new_axes.append(getattr(self, a).union(getattr(other, a)))
 
         # reindex: could check that everything's the same size, but forget it
-        d = dict([(a, ax) for a, ax in zip(self._AXIS_ORDERS, new_axes)])
+        d = dict((a, ax) for a, ax in zip(self._AXIS_ORDERS, new_axes))
         d['copy'] = False
         this = self.reindex(**d)
         other = other.reindex(**d)

pandas/core/reshape/concat.py

@@ -568,7 +568,7 @@ def _make_concat_multiindex(indexes, keys, levels=None, names=None):
             names = list(names)
         else:
             # make sure that all of the passed indices have the same nlevels
-            if not len(set([idx.nlevels for idx in indexes])) == 1:
+            if not len(set(idx.nlevels for idx in indexes)) == 1:
                 raise AssertionError("Cannot concat indices that do"
                                      " not have the same number of levels")
 

pandas/core/sparse/frame.py

@@ -173,7 +173,7 @@ def _init_matrix(self, data, index, columns, dtype=None):
         """ Init self from ndarray or list of lists """
         data = _prep_ndarray(data, copy=False)
         index, columns = self._prep_index(data, index, columns)
-        data = dict([(idx, data[:, i]) for i, idx in enumerate(columns)])
+        data = dict((idx, data[:, i]) for i, idx in enumerate(columns))
         return self._init_dict(data, index, columns, dtype)
 
     def _init_spmatrix(self, data, index, columns, dtype=None,

pandas/io/clipboards.py

@@ -53,7 +53,7 @@ def read_clipboard(sep='\s+', **kwargs):  # pragma: no cover
     # 0  1  2
     # 1  3  4
 
-    counts = set([x.lstrip().count('\t') for x in lines])
+    counts = set(x.lstrip().count('\t') for x in lines)
     if len(lines) > 1 and len(counts) == 1 and counts.pop() != 0:
         sep = '\t'