but fix devoted to #2109 but for VI

Author: ferrine

pymc3/tests/test_variational_inference.py

@@ -143,12 +143,12 @@ def test_optimizer_with_full_data(self):
             with Model():
                 mu_ = Normal('mu', mu=mu0, sd=sd0, testval=0)
                 Normal('x', mu=mu_, sd=sd, observed=data)
-                inf = self.inference()
+                inf = self.inference(start={})
                 inf.fit(10)
                 approx = inf.fit(self.NITER,
                                  obj_optimizer=self.optimizer,
                                  callbacks=
-                                 [pm.callbacks.CheckParametersConvergence()])
+                                 [pm.callbacks.CheckParametersConvergence()],)
                 trace = approx.sample(10000)
             np.testing.assert_allclose(np.mean(trace['mu']), mu_post, rtol=0.1)
             np.testing.assert_allclose(np.std(trace['mu']), np.sqrt(1. / d), rtol=0.4)
@@ -342,14 +342,15 @@ def test_init_from_noize(self):
     [
         ('undefined', dict(), KeyError),
         (1, dict(), TypeError),
-        (_advi, dict(), None),
+        (_advi, dict(start={}), None),
         (_fullrank_advi, dict(), None),
         (_svgd, dict(), None),
         ('advi', dict(), None),
         ('advi->fullrank_advi', dict(frac=.1), None),
         ('advi->fullrank_advi', dict(frac=1), ValueError),
         ('fullrank_advi', dict(), None),
         ('svgd', dict(), None),
+        ('svgd', dict(start={}), None),
         ('svgd', dict(local_rv={_model.free_RVs[0]: (0, 1)}), ValueError)
     ]
 )

pymc3/variational/approximations.py

@@ -64,7 +64,14 @@ def cov(self):
         return tt.diag(rho2sd(self.rho)**2)
 
     def create_shared_params(self, **kwargs):
-        start = self.gbij.map(kwargs.get('start', self.model.test_point))
+        start = kwargs.get('start')
+        if start is None:
+            start = self.model.test_point
+        else:
+            start_ = self.model.test_point.copy()
+            pm.sampling._update_start_vals(start_, start, self.model)
+            start = start_
+        start = self.gbij.map(start)
         return {'mu': theano.shared(
                     pm.floatX(start),
                     'mu'),
@@ -125,11 +132,13 @@ class FullRank(Approximation):
         Sticking the Landing: A Simple Reduced-Variance Gradient for ADVI
         approximateinference.org/accepted/RoederEtAl2016.pdf
     """
-    def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, gpu_compat=False, seed=None):
+    def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1,
+                 gpu_compat=False, seed=None, **kwargs):
         super(FullRank, self).__init__(
             local_rv=local_rv, model=model,
             cost_part_grad_scale=cost_part_grad_scale,
-            seed=seed
+            seed=seed,
+            **kwargs
         )
         self.gpu_compat = gpu_compat
 
@@ -161,7 +170,14 @@ def tril_index_matrix(self):
         return tril_index_matrix
 
     def create_shared_params(self, **kwargs):
-        start = self.gbij.map(kwargs.get('start', self.model.test_point))
+        start = kwargs.get('start')
+        if start is None:
+            start = self.model.test_point
+        else:
+            start_ = self.model.test_point.copy()
+            pm.sampling._update_start_vals(start_, start, self.model)
+            start = start_
+        start = self.gbij.map(start)
         n = self.global_size
         L_tril = (
             np.eye(n)
@@ -254,8 +270,11 @@ class Empirical(Approximation):
     ...     trace = sample(1000, step=step)
     ...     histogram = Empirical(trace[100:])
     """
-    def __init__(self, trace, local_rv=None, model=None, seed=None):
-        super(Empirical, self).__init__(local_rv=local_rv, model=model, trace=trace, seed=seed)
+    def __init__(self, trace, local_rv=None, model=None, seed=None, **kwargs):
+        super(Empirical, self).__init__(
+            local_rv=local_rv, model=model, trace=trace, seed=seed,
+            **kwargs
+        )
 
     def check_model(self, model, **kwargs):
         trace = kwargs.get('trace')
@@ -355,6 +374,10 @@ def from_noise(cls, size, jitter=.01, local_rv=None, start=None, model=None, see
         hist = cls(None, local_rv=local_rv, model=model, seed=seed)
         if start is None:
             start = hist.model.test_point
+        else:
+            start_ = hist.model.test_point.copy()
+            pm.sampling._update_start_vals(start_, start, hist.model)
+            start = start_
         start = hist.gbij.map(start)
         # Initialize particles
         x0 = np.tile(start, (size, 1))

pymc3/variational/inference.py

@@ -306,7 +306,9 @@ class ADVI(Inference):
         Yuhuai Wu, David Duvenaud, 2016) for details
     seed : None or int
         leave None to use package global RandomStream or other
-        valid value to create instance specific one
+        valid value to create instance specific one    
+    start : Point
+        starting point for inference
 
     References
     ----------
@@ -321,10 +323,12 @@ class ADVI(Inference):
     - Kingma, D. P., & Welling, M. (2014).
       Auto-Encoding Variational Bayes. stat, 1050, 1.
     """
-    def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, seed=None):
+    def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1,
+                 seed=None, start=None):
         super(ADVI, self).__init__(
             KL, MeanField, None,
-            local_rv=local_rv, model=model, cost_part_grad_scale=cost_part_grad_scale, seed=seed)
+            local_rv=local_rv, model=model, cost_part_grad_scale=cost_part_grad_scale,
+            seed=seed, start=start)
 
     @classmethod
     def from_mean_field(cls, mean_field):
@@ -372,6 +376,8 @@ class FullRankADVI(Inference):
     seed : None or int
         leave None to use package global RandomStream or other
         valid value to create instance specific one
+    start : Point
+        starting point for inference
 
     References
     ----------
@@ -386,11 +392,12 @@ class FullRankADVI(Inference):
     - Kingma, D. P., & Welling, M. (2014).
       Auto-Encoding Variational Bayes. stat, 1050, 1.
     """
-    def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1, gpu_compat=False, seed=None):
+    def __init__(self, local_rv=None, model=None, cost_part_grad_scale=1,
+                 gpu_compat=False, seed=None, start=None):
         super(FullRankADVI, self).__init__(
             KL, FullRank, None,
             local_rv=local_rv, model=model, cost_part_grad_scale=cost_part_grad_scale,
-            gpu_compat=gpu_compat, seed=seed)
+            gpu_compat=gpu_compat, seed=seed, start=start)
 
     @classmethod
     def from_full_rank(cls, full_rank):
@@ -497,6 +504,8 @@ class SVGD(Inference):
     seed : None or int
         leave None to use package global RandomStream or other
         valid value to create instance specific one
+    start : Point
+        starting point for inference
 
     References
     ----------
@@ -515,7 +524,7 @@ def __init__(self, n_particles=100, jitter=.01, model=None, kernel=test_function
             model=model, seed=seed)
 
 
-def fit(n=10000, local_rv=None, method='advi', model=None, seed=None, **kwargs):
+def fit(n=10000, local_rv=None, method='advi', model=None, seed=None, start=None, **kwargs):
     """
     Handy shortcut for using inference methods in functional way
 
@@ -536,7 +545,8 @@ def fit(n=10000, local_rv=None, method='advi', model=None, seed=None, **kwargs):
     seed : None or int
         leave None to use package global RandomStream or other
         valid value to create instance specific one
-
+    start : Point
+        starting point for inference
     Returns
     -------
     Approximation
@@ -554,7 +564,7 @@ def fit(n=10000, local_rv=None, method='advi', model=None, seed=None, **kwargs):
             raise ValueError('frac should be in (0, 1)')
         n1 = int(n * frac)
         n2 = n-n1
-        inference = ADVI(local_rv=local_rv, model=model, seed=seed)
+        inference = ADVI(local_rv=local_rv, model=model, seed=seed, start=start)
         logger.info('fitting advi ...')
         inference.fit(n1, **kwargs)
         inference = FullRankADVI.from_advi(inference)
@@ -564,7 +574,8 @@ def fit(n=10000, local_rv=None, method='advi', model=None, seed=None, **kwargs):
     elif isinstance(method, str):
         try:
             inference = _select[method.lower()](
-                local_rv=local_rv, model=model, seed=seed
+                local_rv=local_rv, model=model, seed=seed,
+                start=start
             )
         except KeyError:
             raise KeyError('method should be one of %s '