refactored ods datetime parser, added parse col selection filter, was not done when copying method from _parse_excel

Author: davidovitch

pandas/io/excel.py

@@ -520,36 +520,42 @@ def _parse_ods(self, sheetname=0, header=0, skiprows=None, skip_footer=0,
         sheet = self.book.sheets[sheetname]
 
         data = []
-
+        should_parse = {}
         for i in range(sheet.nrows()):
             row = []
             for j, cell in enumerate(sheet.row(i)):
-                if isinstance(cell.value, float):
-                    value = cell.value
-                    if convert_float:
-                        # GH5394 - Excel and ODS 'numbers' are always floats
-                        # it's a minimal perf hit and less suprising
-                        # FIXME: this goes wrong when int(cell.value) returns
-                        # a long (>1e18)
-                        val = int(cell.value)
-                        if val == cell.value:
-                            value = val
-                elif isinstance(cell.value, compat.string_types):
-                    typ = cell.value_type
-#                    if typ == 'string':
-#                        value = cell.value
-                    if typ == 'date' or typ == 'time':
-                        value = self._parse_datetime(cell)
-                    else:
+
+                if parse_cols is not None and j not in should_parse:
+                    should_parse[j] = self._should_parse(j, parse_cols)
+
+                if parse_cols is None or should_parse[j]:
+
+                    if isinstance(cell.value, float):
                         value = cell.value
-                elif isinstance(cell.value, bool):
-                    value = cell.value
-#                elif isinstance(cell.value, type(None)):
-#                    value = np.nan
-                else:
-                    value = None
+                        if convert_float:
+                            # GH5394 - Excel and ODS 'numbers' are always floats
+                            # it's a minimal perf hit and less suprising
+                            # FIXME: this goes wrong when int(cell.value) returns
+                            # a long (>1e18)
+                            val = int(cell.value)
+                            if val == cell.value:
+                                value = val
+                    elif isinstance(cell.value, compat.string_types):
+                        typ = cell.value_type
+#                        if typ == 'string':
+#                            value = cell.value
+                        if typ == 'date' or typ == 'time':
+                            value = self._parse_datetime(cell)
+                        else:
+                            value = cell.value
+                    elif isinstance(cell.value, bool):
+                        value = cell.value
+#                    elif isinstance(cell.value, type(None)):
+#                        value = np.nan
+                    else:
+                        value = np.nan
 
-                row.append(value)
+                    row.append(value)
 
             data.append(row)
 
@@ -568,36 +574,56 @@ def _parse_ods(self, sheetname=0, header=0, skiprows=None, skip_footer=0,
 
     def _parse_datetime(self, cell):
         """Parse the date or time from on ods cell to a datetime object.
-        See which formats are returned:
-        https://pythonhosted.org/ezodf/tableobjects.html#cell-class
+        Formats returned by ezodf are documented here:
+        https://pythonhosted.org/ezodf/tableobjects.html#cell-class.
+
+        Date string value formats: 'yyyy-mm-dd' or 'yyyy-mm-ddThh:mm:ss'
+
+        Time string value format: 'PThhHmmMss,ffffS'
         """
-        if cell.value_type == 'time' and cell.formula is not None:
+        def _value2date(value):
             try:
-                value = datetime.datetime.strptime(cell.formula,
-                                                   'of:=TIME(%H;%M;%S)')
-            except ValueError:
-                # hours can be more then 23
-                hours = int(cell.value[2:].split('H')[0])
-                minutes = int(cell.value[2:].split('M')[0][-2:])
-                seconds = int(cell.value[2:].split('M')[1][:-1])
-                if hours > 23:
+                return datetime.datetime.strptime(value, '%Y-%m-%d')
+            except ValueError:#, TypeError):
+                return datetime.datetime.strptime(value, '%Y-%m-%dT%H:%M:%S')
+
+        # Technically it is not necessary to try to derive the date/time
+        # value from the formula field. The date/time can also be obtained
+        # from the cell value field. Consequently the approach over the formula
+        # field could be considered as too verbose and not necessary.
+        if cell.formula is not None:
+            if cell.value_type == 'time':
+                try:
+                    value = datetime.datetime.strptime(cell.formula,
+                                                       'of:=TIME(%H;%M;%S)')
+                except ValueError:
+                    # hours can >23, so create timedelta. Format: 'PT%HH%MM%SS'
+                    hours = int(cell.value[2:].split('H')[0])
+                    minutes = int(cell.value[2:].split('M')[0][-2:])
+                    # seconds can also contain micro seconds as a float
+                    seconds = float(cell.value[2:].split('M')[1][:-1])
+                    # TODO: now timedelta objects will be mixed with time
                     value = datetime.timedelta(hours=hours, minutes=minutes,
                                                seconds=seconds)
-                else:
-                    # TODO: should return a time object, not datetime?
-                    value = datetime.datetime.strptime(cell.value,
-                                                       'PT%HH%MM%SS')
-                    # TODO: this does not cover all scenario's
-                    # TODO: now timedelta objects will be mixed with normal time
-        elif cell.value_type == 'date' and cell.formula is not None:
+            elif cell.value_type == 'date':
+                try:
+                    # formula is not by definition the date formula
+                    value = datetime.datetime.strptime(cell.formula,
+                                                       'of:=DATE(%Y;%m;%d)')
+                except ValueError:
+                    value = _value2date(cell.value)
+            else:
+                value = None
+        elif cell.value_type == 'date':
+            value = _value2date(cell.value)
+        elif cell.value_type == 'time':
             try:
-                value = datetime.datetime.strptime(cell.formula,
-                                                   'of:=DATE(%Y;%m;%d)')
-            except (ValueError, TypeError):
-                # TODO: parsing other scenerio's
-                value = cell.value
+                value = datetime.datetime.strptime(cell.value, 'PT%HH%MM%S.%fS')
+            except ValueError:
+                value = datetime.datetime.strptime(cell.value, 'PT%HH%MM%SS')
         else:
             value = None
+
         return value
 
     def _print_ods_cellinfo(self, cell):