From 62a6b09bb6ce8fc091241165a4b8abfcc84d1c37 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Thu, 7 May 2020 09:35:57 -0700 Subject: [PATCH 1/8] CI: check_freq=False --- pandas/_testing.py | 9 ++++++++- pandas/tests/io/pytables/test_store.py | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pandas/_testing.py b/pandas/_testing.py index 8fbdcb89dafca..e343c278faf6d 100644 --- a/pandas/_testing.py +++ b/pandas/_testing.py @@ -1104,6 +1104,7 @@ def assert_series_equal( check_datetimelike_compat=False, check_categorical=True, check_category_order=True, + check_freq=True, obj="Series", ): """ @@ -1140,6 +1141,8 @@ def assert_series_equal( Whether to compare internal Categorical exactly. check_category_order : bool, default True Whether to compare category order of internal Categoricals. + check_freq : bool, default True + Whether to check the `freq` attribute on a DatetimeIndex or TimedeltaIndex. .. versionadded:: 1.0.2 obj : str, default 'Series' @@ -1171,7 +1174,7 @@ def assert_series_equal( check_categorical=check_categorical, obj=f"{obj}.index", ) - if isinstance(left.index, (pd.DatetimeIndex, pd.TimedeltaIndex)): + if check_freq and isinstance(left.index, (pd.DatetimeIndex, pd.TimedeltaIndex)): lidx = left.index ridx = right.index assert lidx.freq == ridx.freq, (lidx.freq, ridx.freq) @@ -1274,6 +1277,7 @@ def assert_frame_equal( check_datetimelike_compat=False, check_categorical=True, check_like=False, + check_freq=True, obj="DataFrame", ): """ @@ -1327,6 +1331,8 @@ def assert_frame_equal( If True, ignore the order of index & columns. Note: index labels must match their respective rows (same as in columns) - same labels must be with the same data. + check_freq : bool, default True + Whether to check the `freq` attribute on a DatetimeIndex or TimedeltaIndex. obj : str, default 'DataFrame' Specify object name being compared, internally used to show appropriate assertion message. @@ -1433,6 +1439,7 @@ def assert_frame_equal( check_names=check_names, check_datetimelike_compat=check_datetimelike_compat, check_categorical=check_categorical, + check_freq=check_freq, obj=f'{obj}.iloc[:, {i}] (column name="{col}")', ) diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index 299ae2f41d676..ebf9758d44029 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -1581,7 +1581,10 @@ def check_col(key, name, size): result = store.select("df_dc", ["B > 0", "C > 0", "string == foo"]) expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (df_dc.string == "foo")] - tm.assert_frame_equal(result, expected, check_index_type=False) + tm.assert_frame_equal( + result, expected, check_index_type=False, check_freq=False + ) + # FIXME: 2020-05-07 freq check randomly fails in the CI with ensure_clean_store(setup_path) as store: # doc example part 2 From 5b65bd58bb746430e3c6e647043e85359db4d7c8 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 8 May 2020 10:01:01 -0700 Subject: [PATCH 2/8] versionadded --- pandas/_testing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pandas/_testing.py b/pandas/_testing.py index e343c278faf6d..5a44c5dc5e0af 100644 --- a/pandas/_testing.py +++ b/pandas/_testing.py @@ -1141,10 +1141,12 @@ def assert_series_equal( Whether to compare internal Categorical exactly. check_category_order : bool, default True Whether to compare category order of internal Categoricals. + + .. versionadded:: 1.0.2 check_freq : bool, default True Whether to check the `freq` attribute on a DatetimeIndex or TimedeltaIndex. - .. versionadded:: 1.0.2 + .. versionadded:: 1.1.0 obj : str, default 'Series' Specify object name being compared, internally used to show appropriate assertion message. @@ -1333,6 +1335,8 @@ def assert_frame_equal( (same as in columns) - same labels must be with the same data. check_freq : bool, default True Whether to check the `freq` attribute on a DatetimeIndex or TimedeltaIndex. + + .. versionadded:: 1.1.0 obj : str, default 'DataFrame' Specify object name being compared, internally used to show appropriate assertion message. From bd1cf1849f33974b862c280e8d52879ac6b612b1 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 8 May 2020 15:54:48 -0700 Subject: [PATCH 3/8] CI: troubleshoot Travis --- pandas/tests/io/parser/test_network.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index 000fc605dd5b1..4da6069315ad3 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -172,17 +172,19 @@ def test_read_s3_fails(self): def test_write_s3_csv_fails(self, tips_df): # GH 32486 # Attempting to write to an invalid S3 path should raise - with pytest.raises( - FileNotFoundError, match="The specified bucket does not exist" - ): + import botocore # 2020-05-08 Travis builds want NoSuchBucket + + err = (FileNotFoundError, botocore.errorfactory.NoSuchBucket) + with pytest.raises(err, match="The specified bucket does not exist"): tips_df.to_csv("s3://an_s3_bucket_data_doesnt_exit/not_real.csv") @td.skip_if_no("pyarrow") def test_write_s3_parquet_fails(self, tips_df): # GH 27679 - with pytest.raises( - FileNotFoundError, match="The specified bucket does not exist" - ): + import botocore # 2020-05-08 Travis builds want NoSuchBucket + + err = (FileNotFoundError, botocore.errorfactory.NoSuchBucket) + with pytest.raises(err, match="The specified bucket does not exist"): tips_df.to_parquet("s3://an_s3_bucket_data_doesnt_exit/not_real.parquet") def test_read_csv_handles_boto_s3_object(self, s3_resource, tips_file): From bbc0b221e5b85805c8733dd8099649d4bbdf170e Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 8 May 2020 16:38:48 -0700 Subject: [PATCH 4/8] compat for older botocore --- pandas/tests/io/parser/test_network.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index 4da6069315ad3..aac2c6d3e4875 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -172,18 +172,27 @@ def test_read_s3_fails(self): def test_write_s3_csv_fails(self, tips_df): # GH 32486 # Attempting to write to an invalid S3 path should raise - import botocore # 2020-05-08 Travis builds want NoSuchBucket - err = (FileNotFoundError, botocore.errorfactory.NoSuchBucket) + import botocore + + err = FileNotFoundError + if hasattr(botocore.errorfactory, "NoSuchBucket"): + # 2020-05-08 Travis builds are faiing because it wants NoSuchBucket + # instead of FileNotBoundError + err = botocore.errorfactory.NoSuchBucket with pytest.raises(err, match="The specified bucket does not exist"): tips_df.to_csv("s3://an_s3_bucket_data_doesnt_exit/not_real.csv") @td.skip_if_no("pyarrow") def test_write_s3_parquet_fails(self, tips_df): # GH 27679 - import botocore # 2020-05-08 Travis builds want NoSuchBucket + import botocore - err = (FileNotFoundError, botocore.errorfactory.NoSuchBucket) + err = FileNotFoundError + if hasattr(botocore.errorfactory, "NoSuchBucket"): + # 2020-05-08 Travis builds are faiing because it wants NoSuchBucket + # instead of FileNotBoundError + err = botocore.errorfactory.NoSuchBucket with pytest.raises(err, match="The specified bucket does not exist"): tips_df.to_parquet("s3://an_s3_bucket_data_doesnt_exit/not_real.parquet") From cd8400608886d3ebb400f4a8f974489609dfea74 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 8 May 2020 17:53:58 -0700 Subject: [PATCH 5/8] troubleshoot Travis failure --- pandas/tests/io/parser/test_network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index aac2c6d3e4875..e482684da01a5 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -173,7 +173,7 @@ def test_write_s3_csv_fails(self, tips_df): # GH 32486 # Attempting to write to an invalid S3 path should raise - import botocore + import botocore.errorfactory err = FileNotFoundError if hasattr(botocore.errorfactory, "NoSuchBucket"): @@ -186,7 +186,7 @@ def test_write_s3_csv_fails(self, tips_df): @td.skip_if_no("pyarrow") def test_write_s3_parquet_fails(self, tips_df): # GH 27679 - import botocore + import botocore.errorfactory err = FileNotFoundError if hasattr(botocore.errorfactory, "NoSuchBucket"): From 968d4078e441963d78c3cfa607005018d3819b35 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 8 May 2020 18:31:34 -0700 Subject: [PATCH 6/8] troubleshoot --- pandas/tests/io/parser/test_network.py | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index e482684da01a5..411ffa8726e27 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -173,26 +173,20 @@ def test_write_s3_csv_fails(self, tips_df): # GH 32486 # Attempting to write to an invalid S3 path should raise - import botocore.errorfactory + import botocore.utils - err = FileNotFoundError - if hasattr(botocore.errorfactory, "NoSuchBucket"): - # 2020-05-08 Travis builds are faiing because it wants NoSuchBucket - # instead of FileNotBoundError - err = botocore.errorfactory.NoSuchBucket + # See https://github.com/boto/boto3/issues/1262 on catching boto erros + err = (FileNotFoundError, botocore.utils.ClientError) with pytest.raises(err, match="The specified bucket does not exist"): tips_df.to_csv("s3://an_s3_bucket_data_doesnt_exit/not_real.csv") @td.skip_if_no("pyarrow") def test_write_s3_parquet_fails(self, tips_df): # GH 27679 - import botocore.errorfactory + import botocore.utils - err = FileNotFoundError - if hasattr(botocore.errorfactory, "NoSuchBucket"): - # 2020-05-08 Travis builds are faiing because it wants NoSuchBucket - # instead of FileNotBoundError - err = botocore.errorfactory.NoSuchBucket + # See https://github.com/boto/boto3/issues/1262 on catching boto erros + err = (FileNotFoundError, botocore.utils.ClientError) with pytest.raises(err, match="The specified bucket does not exist"): tips_df.to_parquet("s3://an_s3_bucket_data_doesnt_exit/not_real.parquet") From 7eca867a90f436c7bf7de32e560f7b4e4d7ede1f Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Fri, 8 May 2020 20:12:22 -0700 Subject: [PATCH 7/8] troubleshoot CI --- pandas/tests/io/pytables/test_store.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pandas/tests/io/pytables/test_store.py b/pandas/tests/io/pytables/test_store.py index ebf9758d44029..c937f9ac42818 100644 --- a/pandas/tests/io/pytables/test_store.py +++ b/pandas/tests/io/pytables/test_store.py @@ -1552,12 +1552,16 @@ def check_col(key, name, size): & (df_new.A > 0) & (df_new.B < 0) ] - tm.assert_frame_equal(result, expected, check_index_type=False) + tm.assert_frame_equal( + result, expected, check_index_type=False, check_freq=False + ) # yield an empty frame result = store.select("df", "string='foo' and string2='cool'") expected = df_new[(df_new.string == "foo") & (df_new.string2 == "cool")] - tm.assert_frame_equal(result, expected, check_index_type=False) + tm.assert_frame_equal( + result, expected, check_index_type=False, check_freq=False + ) with ensure_clean_store(setup_path) as store: # doc example @@ -1577,7 +1581,9 @@ def check_col(key, name, size): result = store.select("df_dc", "B>0") expected = df_dc[df_dc.B > 0] - tm.assert_frame_equal(result, expected, check_index_type=False) + tm.assert_frame_equal( + result, expected, check_index_type=False, check_freq=False + ) result = store.select("df_dc", ["B > 0", "C > 0", "string == foo"]) expected = df_dc[(df_dc.B > 0) & (df_dc.C > 0) & (df_dc.string == "foo")] From f6179102e8eda00b3ae6097b0ba9567ec1c701d7 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 9 May 2020 07:10:39 -0700 Subject: [PATCH 8/8] revert --- pandas/tests/io/parser/test_network.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/pandas/tests/io/parser/test_network.py b/pandas/tests/io/parser/test_network.py index 411ffa8726e27..000fc605dd5b1 100644 --- a/pandas/tests/io/parser/test_network.py +++ b/pandas/tests/io/parser/test_network.py @@ -172,22 +172,17 @@ def test_read_s3_fails(self): def test_write_s3_csv_fails(self, tips_df): # GH 32486 # Attempting to write to an invalid S3 path should raise - - import botocore.utils - - # See https://github.com/boto/boto3/issues/1262 on catching boto erros - err = (FileNotFoundError, botocore.utils.ClientError) - with pytest.raises(err, match="The specified bucket does not exist"): + with pytest.raises( + FileNotFoundError, match="The specified bucket does not exist" + ): tips_df.to_csv("s3://an_s3_bucket_data_doesnt_exit/not_real.csv") @td.skip_if_no("pyarrow") def test_write_s3_parquet_fails(self, tips_df): # GH 27679 - import botocore.utils - - # See https://github.com/boto/boto3/issues/1262 on catching boto erros - err = (FileNotFoundError, botocore.utils.ClientError) - with pytest.raises(err, match="The specified bucket does not exist"): + with pytest.raises( + FileNotFoundError, match="The specified bucket does not exist" + ): tips_df.to_parquet("s3://an_s3_bucket_data_doesnt_exit/not_real.parquet") def test_read_csv_handles_boto_s3_object(self, s3_resource, tips_file):