ENH: Allow poorly formatted stata files to be read

Add a fall back decode path that allows improperly formatted Stata
files written in 118 format but using latin-1 encoded strings to be
read

closes pandas-dev#25960

Author: bashtage

doc/source/whatsnew/v0.25.0.rst

@@ -355,6 +355,7 @@ I/O
 - Improved performance in :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` when converting columns that have missing values (:issue:`25772`)
 - Bug in :func:`read_hdf` not properly closing store after a ``KeyError`` is raised (:issue:`25766`)
 - Bug in ``read_csv`` which would not raise ``ValueError`` if a column index in ``usecols`` was out of bounds (:issue:`25623`)
+- Improved :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` to read incorrectly formatted 118 format files saved by Stata (:issue:`25960`)
 
 Plotting
 ^^^^^^^^

pandas/io/stata.py

@@ -1327,12 +1327,20 @@ def _calcsize(self, fmt):
 
     def _decode(self, s):
         s = s.partition(b"\0")[0]
-        return s.decode('utf-8')
+        try:
+            return s.decode('utf-8')
+        except UnicodeDecodeError:
+            # GH 25960
+            return s.decode('latin-1')
 
     def _null_terminate(self, s):
         # have bytes not strings, so must decode
         s = s.partition(b"\0")[0]
-        return s.decode(self._encoding)
+        try:
+            return s.decode(self._encoding)
+        except UnicodeDecodeError:
+            # GH 25960
+            return s.decode('latin-1')
 
     def _read_value_labels(self):
         if self._value_labels_read:

pandas/tests/io/data/stata1_encoding_118.dta

@@ -66,6 +66,8 @@ def setup_method(self, datapath):
         self.dta4_117 = os.path.join(self.dirpath, 'stata4_117.dta')
 
         self.dta_encoding = os.path.join(self.dirpath, 'stata1_encoding.dta')
+        self.dta_encoding_118 = os.path.join(self.dirpath,
+                                             'stata1_encoding_118.dta')
 
         self.csv14 = os.path.join(self.dirpath, 'stata5.csv')
         self.dta14_113 = os.path.join(self.dirpath, 'stata5_113.dta')
@@ -1608,3 +1610,9 @@ def test_strl_latin1(self):
                     val = gso.split(b'\x00')[-2]
                     size = gso[gso.find(b'\x82') + 1]
                     assert len(val) == size - 1
+
+    def test_encoding_latin1_118(self):
+        # GH 25960
+        encoded = read_stata(self.dta_encoding_118)
+        expected = pd.DataFrame([['Düsseldorf']] * 151, columns=['kreis1849'])
+        tm.assert_frame_equal(encoded, expected)