COMPAT: reading json with lines=True from s3, xref #17200

ci/requirements-2.7.pip

@@ -8,3 +8,4 @@ py
 PyCrypto
 mock
 ipython
+moto

ci/requirements-2.7.run

@@ -17,4 +17,4 @@ psycopg2
 patsy
 pymysql=0.6.3
 jinja2=2.8
-xarray=0.8.0
+xarray=0.8.0

ci/requirements-2.7_SLOW.run

@@ -16,4 +16,4 @@ s3fs
 psycopg2
 pymysql
 html5lib
-beautiful-soup
+beautiful-soup

ci/requirements-3.5.pip

@@ -1,2 +1,3 @@
 xarray==0.9.1
 pandas-gbq
+moto

ci/requirements-3.5.run

@@ -17,4 +17,4 @@ pymysql
 psycopg2
 s3fs
 beautifulsoup4
-ipython
+ipython

ci/requirements-3.5_OSX.run

@@ -13,4 +13,4 @@ jinja2
 bottleneck
 xarray
 s3fs
-beautifulsoup4
+beautifulsoup4

ci/requirements-3.6.pip

@@ -1 +1,2 @@
 brotlipy
+moto

ci/requirements-3.6.run

@@ -22,4 +22,4 @@ fastparquet
 beautifulsoup4
 s3fs
 xarray
-ipython
+ipython

ci/requirements-3.6_LOCALE.run

@@ -19,4 +19,4 @@ psycopg2
 beautifulsoup4
 s3fs
 xarray
-ipython
+ipython

ci/requirements-3.6_LOCALE_SLOW.run

@@ -19,4 +19,4 @@ psycopg2
 beautifulsoup4
 s3fs
 xarray
-ipython
+ipython

ci/requirements_all.txt

@@ -26,3 +26,4 @@ sqlalchemy
 bottleneck
 pymysql
 Jinja2
+s3fs

ci/requirements_dev.txt

@@ -5,4 +5,5 @@ cython
 pytest>=3.1.0
 pytest-cov
 flake8
+s3fs
 moto

pandas/io/common.py

@@ -206,7 +206,7 @@ def get_filepath_or_buffer(filepath_or_buffer, encoding=None,
     if _is_s3_url(filepath_or_buffer):
         from pandas.io import s3
         return s3.get_filepath_or_buffer(filepath_or_buffer,
-                                         encoding=encoding,
+                                         encoding=encoding, 
                                          compression=compression)
 
     if isinstance(filepath_or_buffer, (compat.string_types,

pandas/io/json/json.py

@@ -340,12 +340,21 @@ def read_json(path_or_buf=None, orient=None, typ='frame', dtype=True,
             json = filepath_or_buffer
     elif hasattr(filepath_or_buffer, 'read'):
         json = filepath_or_buffer.read()
+
     else:
         json = filepath_or_buffer
 
     if lines:
         # If given a json lines file, we break the string into lines, add
         # commas and put it in a json list to make a valid json object.
+
+        """
+            Handle encoded bytes arrays in PY3 and bytes objects from certain
+            readables before using StringIO.
+        """
+        if isinstance(json, bytes):
+            json = json.decode('utf-8')
+
         lines = list(StringIO(json.strip()))
         json = '[' + ','.join(lines) + ']'
 

pandas/tests/io/json/test_pandas.py

@@ -4,11 +4,11 @@
 from pandas.compat import (range, lrange, StringIO,
                            OrderedDict, is_platform_32bit)
 import os
-
 import numpy as np
 from pandas import (Series, DataFrame, DatetimeIndex, Timestamp,
                     read_json, compat)
 from datetime import timedelta
+moto = pytest.importorskip("moto")
 import pandas as pd
 
 from pandas.util.testing import (assert_almost_equal, assert_frame_equal,
@@ -985,12 +985,46 @@ def test_tz_range_is_utc(self):
         df = DataFrame({'DT': dti})
         assert dumps(df, iso_dates=True) == dfexp
 
-    def test_read_jsonl(self):
+    def test_read_inline_jsonl(self):
         # GH9180
         result = read_json('{"a": 1, "b": 2}\n{"b":2, "a" :1}\n', lines=True)
         expected = DataFrame([[1, 2], [1, 2]], columns=['a', 'b'])
         assert_frame_equal(result, expected)
 
+    @pytest.yield_fixture(scope="function")
+    def testbucket_conn(self):
+        """ Fixture for test_read_s3_jsonl"""
+        boto3 = pytest.importorskip('boto3')
+        moto.mock_s3().start()   # Start and stop mocking only once, surrounding the test run
+                                 # to ensure single context is kept.
+
+        conn = boto3.client('s3')
+        conn.create_bucket(Bucket='testbucket')
+        yield conn
+
+        moto.mock_s3().stop()
+
+    def test_read_s3_jsonl(self, testbucket_conn):
+        # GH17200
+        testbucket_conn.put_object(Body=b'{"a": 1, "b": 2}\n{"b":2, "a" :1}\n', Key='items.jsonl', Bucket='testbucket')
+
+        result = read_json('s3://testbucket/items.jsonl', lines=True)
+        expected = DataFrame([[1, 2], [1, 2]], columns=['a', 'b'])
+        assert_frame_equal(result, expected)
+
+    def test_read_local_jsonl(self):
+        # GH17200
+        fname = "./tmp_items.jsonl"
+        try:
+            with open(fname, "w") as infile:
+                infile.write('{"a": 1, "b": 2}\n{"b":2, "a" :1}\n')
+            result = read_json(fname, lines=True)
+            expected = DataFrame([[1, 2], [1, 2]], columns=['a', 'b'])
+            assert_frame_equal(result, expected)
+        finally:
+            import os
+            os.remove(fname)
+
     def test_read_jsonl_unicode_chars(self):
         # GH15132: non-ascii unicode characters
         # \u201d == RIGHT DOUBLE QUOTATION MARK