BUG: fix WLS r-squared to match statsmodels post 10/2012

Author: wesm

RELEASE.rst

@@ -31,6 +31,10 @@ pandas 0.9.1
 
   - Improve HTML display of DataFrame objects with hierarchical columns
 
+**Bug fixes**
+
+  - Make WLS r-squared match statsmodels 0.5.0 fixed value
+
 pandas 0.9.0
 ============
 

pandas/core/frame.py

@@ -1647,7 +1647,7 @@ def icol(self, i):
                 return self.reindex(columns=label)
 
             values = self._data.iget(i)
-            return Series(values, index=self.index, name=label)
+            return Series.from_array(values, index=self.index, name=label)
 
     def _ixs(self, i, axis=0):
         if axis == 0:

pandas/stats/ols.py

@@ -970,8 +970,10 @@ def _resid_stats(self):
         sst = []
         sse = []
 
+        Yreg = self._y
         Y = self._y_trans
         X = self._x_trans
+        weights = self._weights
 
         dates = self._index
         window = self._window
@@ -989,8 +991,16 @@ def _resid_stats(self):
 
             resid = Y_slice - np.dot(X_slice, beta)
 
+            if weights is not None:
+                Y_slice = _y_converter(Yreg.truncate(before=prior_date,
+                                                     after=date))
+                weights_slice = weights.truncate(prior_date, date)
+                demeaned = Y_slice - np.average(Y_slice, weights=weights_slice)
+                SS_total = (weights_slice*demeaned**2).sum()
+            else:
+                SS_total = ((Y_slice - Y_slice.mean()) ** 2).sum()
+
             SS_err = (resid ** 2).sum()
-            SS_total = ((Y_slice - Y_slice.mean()) ** 2).sum()
             SST_uncentered = (Y_slice ** 2).sum()
 
             sse.append(SS_err)

pandas/stats/tests/test_ols.py

@@ -81,6 +81,10 @@ def testOLSWithDatasets(self):
         # self.checkDataSet(datasets.ccard.load(), 39, 49) # one col in X all 0s
 
     def testWLS(self):
+        # WLS centered SS changed (fixed) in 0.5.0
+        if sm.version.version < '0.5.0':
+            raise nose.SkipTest
+
         X = DataFrame(np.random.randn(30, 4), columns=['A', 'B', 'C', 'D'])
         Y = Series(np.random.randn(30))
         weights = X.std(1)

scripts/count_code.sh

@@ -1 +1 @@
-cloc pandas --force-lang=Python,pyx --not-match-f="tseries.c|sandbox.c|engines.c|sparse.c|generated.c|plib.c"
+cloc pandas --force-lang=Python,pyx --not-match-f="parser.c|tseries.c|sandbox.c|engines.c|sparse.c|generated.c|plib.c"