TST: Refactor and improve tests for Pandas 1.5

Author: sanand0

gramex/config.py

@@ -835,6 +835,11 @@ def used_kwargs(method, kwargs, ignore_keywords=False):
     If the method uses ``**kwargs`` (keywords), it uses all keys. To ignore this
     and return only named arguments, use ``ignore_keywords=True``.
     '''
+    # In Pandas 1.5, DataFrame.to_csv and DataFrame.to_excel are wrapped with @deprecate_kwargs.
+    # We dive deeper to detect the actual keywords. __wrapped__ is provided by functools.wraps
+    # https://docs.python.org/3/library/functools.html
+    while hasattr(method, '__wrapped__'):
+        method = method.__wrapped__
     argspec = inspect.getfullargspec(method)
     # If method uses **kwargs, return all kwargs (unless you ignore **kwargs)
     if argspec.varkw and not ignore_keywords:

gramex/data.py

@@ -1851,7 +1851,7 @@ def _filter_mongodb(
     if len(data) > 0:
         import bson
 
-        for col, val in data.iloc[0].iteritems():
+        for col, val in data.iloc[0].items():
             if type(val) in {bson.objectid.ObjectId}:
                 data[col] = data[col].map(str)
 

gramex/gramex.yaml

@@ -269,17 +269,21 @@ app:
     # Login parameters
     login_url: /login/ # URL used to log in
 
+  # Configure how sessions work. See https://gramener.com/gramex/guide/auth/#session-data
   session:
-    cookie: sid # Name of the session ID cookie
-    type: json # Type of store to use: hdf5, json or memory
-    flush:
-      5 # Write store to disk periodically (in seconds)
-      # For type:json, use 5 seconds. It does not save to disk automatically
-      # For type:hdf5 or sqlite, 60 seconds or so is fine
-    purge: 3600 # Purge every hour
-    path: $GRAMEXDATA/session.json # Path to the store (ignored for memory)
-    expiry: 31 # Session cookies expiry in days
-    httponly: true # Only HTTP access, no JS access
+    # Save in a cookie called sid:
+    cookie: sid
+    # Save in a JSON store
+    type: json
+    path: $GRAMEXDATA/session.json
+    # Flush every 5 seconds
+    flush: 5
+    # Clear expired sessions every hour
+    purge: 3600
+    # Cookies expire after 31 days
+    expiry: 31
+    # Browsers cannot use JS to access session cookie. Only HTTP access allowed, for security
+    httponly: true
 
 # The storelocations: section defines where Gramex stores its data.
 storelocations:

gramex/handlers/authhandler.py

@@ -172,7 +172,7 @@ def set_user(self, user, id):
                 user.update(
                     {
                         key: val
-                        for key, val in users.iloc[0].iteritems()
+                        for key, val in users.iloc[0].items()
                         if not gramex.data.pd.isnull(val)
                     }
                 )

gramex/handlers/mlhandler.py

@@ -139,7 +139,7 @@ def _parse_multipart_form_data(self):
         return pd.concat(dfs, axis=0)
 
     def _parse_application_json(self):
-        return pd.read_json(self.request.body)
+        return pd.read_json(self.request.body.decode('utf-8'))
 
     def _parse_data(self, _cache=True, append=False):
         header = self.request.headers.get('Content-Type', '').split(';')[0]

gramex/install.py

@@ -178,8 +178,11 @@ def _ensure_remove(remove, path, exc_info):
     def _ensure_remove(func, path, exc_info):
         '''onerror callback for rmtree that tries hard to delete files'''
         if issubclass(exc_info[0], WindowsError):
-            import winerror
-
+            winerror = AttrDict(
+                ERROR_PATH_NOT_FOUND=3,
+                ERROR_ACCESS_DENIED=5,
+                ERROR_SHARING_VIOLATION=32,
+            )
             # Delete read-only files
             # https://bugs.python.org/issue19643
             # https://bugs.python.org/msg218021

testlib/test_cache_module.py

@@ -333,8 +333,6 @@ def test_save(self):
             'html': {'index': False, 'escape': False, 'ignore_keyword': 1},
             'hdf': {'index': False, 'key': 'data', 'format': 'fixed', 'ignore_keyword': 1},
             'json': {'orient': 'records', 'ignore_keyword': 1},
-            # In Pandas 1.5, .to_csv() and .to_xlsx() accept **kwargs, but raise for invalid ones.
-            # TODO: Fix gramex.cache.save or gramex.config.used_kwargs to ignore invalid keywords.
             'csv': {'index': False, 'ignore_keyword': 1},
             'xlsx': {'index': False, 'sheet_name': 'Sheet1', 'ignore_keyword': 1},
             # 'stata': dict(index=False),   # cannot test since it doesn't support unicode

testlib/test_pptgen2.py

@@ -899,7 +899,7 @@ def test_table(self, slides=9):
         for row_offset, shape_name in ((1, 'Table 1'), (0, 'Table 2')):
             table = self.get_shape(prs.slides[0].shapes, shape_name).table
             for i, (index, row) in enumerate(data.iterrows()):
-                for j, (column, val) in enumerate(row.iteritems()):
+                for j, (column, val) in enumerate(row.items()):
                     cell = table.rows[i + row_offset].cells[j]
                     eq_(cell.text, '{}'.format(val))
         # Test table header

testlib/test_store.py

@@ -6,10 +6,10 @@
 from nose.tools import eq_, ok_
 from nose.plugins.skip import SkipTest
 import gramex.cache
-from gramex.cache import JSONStore, SQLiteStore, HDF5Store, RedisStore
+from gramex.cache import JSONStore, SQLiteStore, RedisStore
 
 # It must be possible to import from basehandler for backward-compatibility
-from gramex.handlers.basehandler import JSONStore, SQLiteStore, HDF5Store, RedisStore  # noqa
+from gramex.handlers.basehandler import JSONStore, SQLiteStore, RedisStore  # noqa
 from gramex.handlers.basehandler import BaseMixin
 from gramex.config import variables
 from . import tests_dir
@@ -138,16 +138,6 @@ def load(self):
         return {self.store._escape(key): val for key, val in self.store.store.items()}
 
 
-class TestHDF5Store(TestJSONStore):
-    store_class = HDF5Store
-    store_file = 'data.h5'
-
-    def load(self):
-        return {
-            key.replace('\t', '/'): json.loads(val[()]) for key, val in self.store.store.items()
-        }
-
-
 class TestRedisStore(TestJSONStore):
     @classmethod
     def setupClass(cls):

tests/requirements.txt

@@ -16,8 +16,9 @@ mkdocs                # For documentation
 mkdocstrings          # For documentation
 mkdocstrings[python]  # For documentation
 
-markdown              # Markdown is part of gramex/release.json, but re-install since
-                      # importlib_metadata version conflicts with makdocs
+# Markdown is part of gramex/release.json, but see if we need need to
+# re-install since importlib_metadata version conflicts with mkdocs
+# markdown
 
 transformers          # For MLHandler
 datasets              # For MLHandler

tests/test_capturehandler.py

@@ -23,6 +23,7 @@
 from gramex.handlers import Capture
 from . import TestGramex, server
 
+_timeout = 15
 _captures = {}
 paths = {'phantomjs': which('phantomjs'), 'node': which('node')}
 
@@ -215,7 +216,7 @@ class TestCaptureHandlerChrome(TestCaptureHandler):
 
     @classmethod
     def setupClass(cls):
-        cls.capture = get_capture('chrome', port=9412, engine='chrome', timeout=15)
+        cls.capture = get_capture('chrome', port=9412, engine='chrome', timeout=_timeout)
         cls.folder = os.path.dirname(os.path.abspath(__file__))
 
     @staticmethod
@@ -339,14 +340,17 @@ def test_headers(self):
         ok_(user['role'] in text)
 
     def err(self, code, **params):
-        return self.fetch(self.src, code=code, params=params)
+        return self.fetch(self.src, code=code, params=params, timeout=_timeout)
 
     def test_errors(self):
         # nonexistent URLs should capture the 404 page and return a screenshot
         self.err(code=200, url='/nonexistent')
-        self.err(code=500, url='http://nonexistent')
+        # Invalid capture formats report HTTP 400
         self.err(code=400, url=self.url, ext='nonexistent')
+        # Invalid selectors report HTTP 500
         self.err(code=500, url=self.url, selector='nonexistent', ext='png')
+        # Invalid domains report HTTP 500
+        self.err(code=500, url='http://nonexistent')
 
     def test_pdf_header_footer(self):
         result = self.fetch(

tests/test_mlhandler.py

@@ -51,13 +51,13 @@ def tearDownClass(cls):
                 shutil.rmtree(path)
 
     def test_default_sentiment(self):
-        """Ensure that the default model predicts something."""
+        # Ensure that the default model predicts something
         resp = self.get("/sentiment?text=This is bad.&text=This is good.", timeout=60)
         self.assertEqual(resp.json(), ["NEGATIVE", "POSITIVE"])
 
     @skipUnless(os.environ.get("GRAMEX_ML_TESTS"), "Set GRAMEX_ML_TESTS=1 to run slow ML tests")
     def test_train_sentiment(self):
-        """Train with some vague sentences."""
+        # Train with some vague sentences
         df = pd.read_json("https://bit.ly/3NesHFs")
         resp = self.get(
             "/sentiment?_action=train&target_col=label",
@@ -69,7 +69,7 @@ def test_train_sentiment(self):
         self.assertGreaterEqual(resp.json()["score"], 0.9)
 
     def test_default_ner(self):
-        """Ensure that the default model predicts something."""
+        # Ensure that the default model predicts something
         labels = self.get("/ner?text=Narendra Modi is the PM of India.", timeout=300).json()
         ents = [[(r["word"], r["entity_group"]) for r in label] for label in labels]
         self.assertIn(("Narendra Modi", "PER"), ents[0])
@@ -661,7 +661,7 @@ def test_single_line_train_fetch_model(self):
         self.assertIsInstance(model["DecisionTreeClassifier"], DecisionTreeClassifier)
 
     def test_template(self):
-        """Check if viewing the template works fine."""
+        # Check if viewing the template works fine
         r = self.get("/mlhandler")
         self.assertEqual(r.status_code, OK)
         # Try getting predictions
@@ -772,7 +772,7 @@ def tearDownClass(cls):
             shutil.rmtree(titanic)
 
     def test_default(self):
-        """An empty GET request returns all predictions."""
+        # An empty GET request returns all predictions
         y_true = gramex.cache.open(
             op.join(folder, "..", "testlib", "iris.csv"),
             usecols=["species"],
@@ -820,7 +820,7 @@ def test_filters(self):
         self.assertEqual(set(y_pred), {'versicolor', 'virginica'})
 
     def test_from_mlhandler(self):
-        """Check that a model created through MLHandler works."""
+        # Check that a model created through MLHandler works
         # Get results from the MLHandler
         df = gramex.cache.open('titanic.csv', rel=True)
         resp = self.get(