Merge pull request #7 from michaelosthege/windows-test

Fix the stackoverflow problem

Author: MarcoGorelli

pymc3/distributions/posterior_predictive.py

@@ -42,7 +42,7 @@
 )
 from ..exceptions import IncorrectArgumentsError
 from ..vartypes import theano_constant
-from ..util import dataset_to_point_dict, chains_and_samples, get_var_name
+from ..util import dataset_to_point_list, chains_and_samples, get_var_name
 
 # Failing tests:
 #    test_mixture_random_shape::test_mixture_random_shape
@@ -209,10 +209,10 @@ def fast_sample_posterior_predictive(
 
     if isinstance(trace, InferenceData):
         nchains, ndraws = chains_and_samples(trace)
-        trace = dataset_to_point_dict(trace.posterior)
+        trace = dataset_to_point_list(trace.posterior)
     elif isinstance(trace, Dataset):
         nchains, ndraws = chains_and_samples(trace)
-        trace = dataset_to_point_dict(trace)
+        trace = dataset_to_point_list(trace)
     elif isinstance(trace, MultiTrace):
         nchains = trace.nchains
         ndraws = len(trace)

pymc3/memoize.py

@@ -13,7 +13,7 @@
 #   limitations under the License.
 
 import functools
-import pickle
+import dill
 import collections
 from .util import biwrap
 
@@ -23,7 +23,16 @@
 @biwrap
 def memoize(obj, bound=False):
     """
-    An expensive memoizer that works with unhashables
+    Decorator to apply memoization to expensive functions.
+    It uses a custom `hashable` helper function to hash typically unhashable Python objects.
+
+    Parameters
+    ----------
+    obj : callable
+        the function to apply the caching to
+    bound : bool
+        indicates if the [obj] is a bound method (self as first argument)
+        For bound methods, the cache is kept in a `_cache` attribute on [self].
     """
     # this is declared not to be a bound method, so just attach new attr to obj
     if not bound:
@@ -40,7 +49,7 @@ def memoizer(*args, **kwargs):
             key = (hashable(args[1:]), hashable(kwargs))
             if not hasattr(args[0], "_cache"):
                 setattr(args[0], "_cache", collections.defaultdict(dict))
-                # do not add to cache regestry
+                # do not add to cache registry
             cache = getattr(args[0], "_cache")[obj.__name__]
         if key not in cache:
             cache[key] = obj(*args, **kwargs)
@@ -75,19 +84,26 @@ def __setstate__(self, state):
         self.__dict__.update(state)
 
 
-def hashable(a):
+def hashable(a) -> int:
     """
-    Turn some unhashable objects into hashable ones.
+    Hashes many kinds of objects, including some that are unhashable through the builtin `hash` function.
+    Lists and tuples are hashed based on their elements.
     """
     if isinstance(a, dict):
-        return hashable(tuple((hashable(a1), hashable(a2)) for a1, a2 in a.items()))
+        # first hash the keys and values with hashable
+        # then hash the tuple of int-tuples with the builtin
+        return hash(tuple((hashable(k), hashable(v)) for k, v in a.items()))
+    if isinstance(a, (tuple, list)):
+        # lists are mutable and not hashable by default
+        # for memoization, we need the hash to depend on the items
+        return hash(tuple(map(hashable, a)))
     try:
         return hash(a)
     except TypeError:
         pass
     # Not hashable >>>
     try:
-        return hash(pickle.dumps(a))
+        return hash(dill.dumps(a))
     except Exception:
         if hasattr(a, "__dict__"):
             return hashable(a.__dict__)

pymc3/sampling.py

@@ -56,7 +56,7 @@
     get_untransformed_name,
     is_transformed_name,
     get_default_varnames,
-    dataset_to_point_dict,
+    dataset_to_point_list,
     chains_and_samples,
 )
 from .vartypes import discrete_types
@@ -1642,9 +1642,9 @@ def sample_posterior_predictive(
 
     _trace: Union[MultiTrace, PointList]
     if isinstance(trace, InferenceData):
-        _trace = dataset_to_point_dict(trace.posterior)
+        _trace = dataset_to_point_list(trace.posterior)
     elif isinstance(trace, xarray.Dataset):
-        _trace = dataset_to_point_dict(trace)
+        _trace = dataset_to_point_list(trace)
     else:
         _trace = trace
 
@@ -1780,10 +1780,10 @@ def sample_posterior_predictive_w(
         n_samples = [
             trace.posterior.sizes["chain"] * trace.posterior.sizes["draw"] for trace in traces
         ]
-        traces = [dataset_to_point_dict(trace.posterior) for trace in traces]
+        traces = [dataset_to_point_list(trace.posterior) for trace in traces]
     elif isinstance(traces[0], xarray.Dataset):
         n_samples = [trace.sizes["chain"] * trace.sizes["draw"] for trace in traces]
-        traces = [dataset_to_point_dict(trace) for trace in traces]
+        traces = [dataset_to_point_list(trace) for trace in traces]
     else:
         n_samples = [len(i) * i.nchains for i in traces]
 

pymc3/tests/test_memo.py

@@ -11,21 +11,59 @@
 #   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 #   See the License for the specific language governing permissions and
 #   limitations under the License.
+import numpy as np
+import pymc3 as pm
 
-from pymc3.memoize import memoize
+from pymc3 import memoize
 
 
-def getmemo():
-    @memoize
-    def f(a, b=("a")):
-        return str(a) + str(b)
+def test_memo():
+    def fun(inputs, suffix="_a"):
+        return str(inputs) + str(suffix)
 
-    return f
+    inputs = ["i1", "i2"]
+    assert fun(inputs) == "['i1', 'i2']_a"
+    assert fun(inputs, "_b") == "['i1', 'i2']_b"
 
+    funmem = memoize.memoize(fun)
+    assert hasattr(fun, "cache")
+    assert isinstance(fun.cache, dict)
+    assert len(fun.cache) == 0
+
+    # call the memoized function with a list input
+    # and check the size of the cache!
+    assert funmem(inputs) == "['i1', 'i2']_a"
+    assert funmem(inputs) == "['i1', 'i2']_a"
+    assert len(fun.cache) == 1
+    assert funmem(inputs, "_b") == "['i1', 'i2']_b"
+    assert funmem(inputs, "_b") == "['i1', 'i2']_b"
+    assert len(fun.cache) == 2
+
+    # add items to the inputs list (the list instance remains identical !!)
+    inputs.append("i3")
+    assert funmem(inputs) == "['i1', 'i2', 'i3']_a"
+    assert funmem(inputs) == "['i1', 'i2', 'i3']_a"
+    assert len(fun.cache) == 3
 
-def test_memo():
-    f = getmemo()
 
-    assert f("x", ["y", "z"]) == "x['y', 'z']"
-    assert f("x", ["a", "z"]) == "x['a', 'z']"
-    assert f("x", ["y", "z"]) == "x['y', 'z']"
+def test_hashing_of_rv_tuples():
+    obs = np.random.normal(-1, 0.1, size=10)
+    with pm.Model() as pmodel:
+        mu = pm.Normal("mu", 0, 1)
+        sd = pm.Gamma("sd", 1, 2)
+        dd = pm.DensityDist(
+            "dd",
+            pm.Normal.dist(mu, sd).logp,
+            random=pm.Normal.dist(mu, sd).random,
+            observed=obs,
+        )
+        print()
+        for freerv in [mu, sd, dd] + pmodel.free_RVs:
+            for structure in [
+                freerv,
+                dict(alpha=freerv, omega=None),
+                [freerv, []],
+                (freerv, []),
+            ]:
+                print(f"testing hashing of: {structure}")
+                assert isinstance(memoize.hashable(structure), int)

pymc3/tests/test_sampling.py

@@ -17,7 +17,6 @@
 from typing import Tuple
 import numpy as np
 import unittest.mock as mock
-import sys
 
 import numpy.testing as npt
 import arviz as az
@@ -930,9 +929,7 @@ def test_shared(self):
 
         assert gen2["y"].shape == (draws, n2)
 
-    @pytest.mark.skipif(sys.platform.startswith("win"), reason="Stackoverflow error on Windows")
     def test_density_dist(self):
-
         obs = np.random.normal(-1, 0.1, size=10)
         with pm.Model():
             mu = pm.Normal("mu", 0, 1)

pymc3/util.py

@@ -15,6 +15,7 @@
 import re
 import functools
 from typing import List, Dict, Tuple, Union
+import warnings
 
 import numpy as np
 import xarray
@@ -258,6 +259,14 @@ def enhanced(*args, **kwargs):
 # FIXME: this function is poorly named, because it returns a LIST of
 # points, not a dictionary of points.
 def dataset_to_point_dict(ds: xarray.Dataset) -> List[Dict[str, np.ndarray]]:
+    warnings.warn(
+        "dataset_to_point_dict was renamed to dataset_to_point_list and will be removed!.",
+        DeprecationWarning,
+    )
+    return dataset_to_point_list(ds)
+
+
+def dataset_to_point_list(ds: xarray.Dataset) -> List[Dict[str, np.ndarray]]:
     # grab posterior samples for each variable
     _samples: Dict[str, np.ndarray] = {vn: ds[vn].values for vn in ds.keys()}
     # make dicts