diff --git a/doc/source/release.rst b/doc/source/release.rst
index 444f25e8662bd..33e421fa55960 100644
--- a/doc/source/release.rst
+++ b/doc/source/release.rst
@@ -449,6 +449,8 @@ Bug Fixes
   - Fixed a bug in ``convert_objects`` for > 2 ndims (:issue:`4937`)
   - Fixed a bug in DataFrame/Panel cache insertion and subsequent indexing (:issue:`4939`)
   - Fixed string methods for ``FrozenNDArray`` and ``FrozenList`` (:issue:`4929`)
+  - Fixed a bug with setting invalid or out-of-range values in indexing
+    enlargement scenarios (:issue:`4940`)
 
 pandas 0.12.0
 -------------
diff --git a/pandas/core/indexing.py b/pandas/core/indexing.py
index 3779e78599bde..cb738df6966da 100644
--- a/pandas/core/indexing.py
+++ b/pandas/core/indexing.py
@@ -750,6 +750,13 @@ def _convert_to_indexer(self, obj, axis=0, is_setter=False):
         is_int_index = _is_integer_index(labels)
 
         if com.is_integer(obj) and not is_int_index:
+
+            # if we are setting and its not a valid location
+            # its an insert which fails by definition
+            if is_setter:
+                if obj >= len(self.obj) and not isinstance(labels, MultiIndex):
+                    raise ValueError("cannot set by positional indexing with enlargement")
+
             return obj
 
         try:
@@ -1340,7 +1347,12 @@ def _safe_append_to_index(index, key):
     try:
         return index.insert(len(index), key)
     except:
-        return Index(np.concatenate([index.asobject.values,np.array([key])]))
+
+        # raise here as this is basically an unsafe operation and we want
+        # it to be obvious that you are doing something wrong
+
+        raise ValueError("unsafe appending to index of "
+                         "type {0} with a key {1}".format(index.__class__.__name__,key))
 
 def _maybe_convert_indices(indices, n):
     """ if we have negative indicies, translate to postive here
diff --git a/pandas/sparse/tests/test_sparse.py b/pandas/sparse/tests/test_sparse.py
index e0a9d2b937225..45543547fd64b 100644
--- a/pandas/sparse/tests/test_sparse.py
+++ b/pandas/sparse/tests/test_sparse.py
@@ -1078,6 +1078,12 @@ def test_icol(self):
 
     def test_set_value(self):
 
+        # this is invalid because it is not a valid type for this index
+        self.assertRaises(ValueError, self.frame.set_value, 'foobar', 'B', 1.5)
+
+        res = self.frame
+        res.index = res.index.astype(object)
+
         res = self.frame.set_value('foobar', 'B', 1.5)
         self.assert_(res is not self.frame)
         self.assert_(res.index[-1] == 'foobar')
diff --git a/pandas/tests/test_indexing.py b/pandas/tests/test_indexing.py
index 267cc8605366c..ced4cbdc4dc36 100644
--- a/pandas/tests/test_indexing.py
+++ b/pandas/tests/test_indexing.py
@@ -1480,6 +1480,30 @@ def test_series_partial_set(self):
         result = ser.iloc[[1,1,0,0]]
         assert_series_equal(result, expected)
 
+    def test_partial_set_invalid(self):
+
+        # GH 4940
+        # allow only setting of 'valid' values
+
+        df = tm.makeTimeDataFrame()
+
+        def f():
+            df.loc[100.0, :] = df.ix[0]
+        self.assertRaises(ValueError, f)
+        def f():
+            df.loc[100,:] = df.ix[0]
+        self.assertRaises(ValueError, f)
+        def f():
+            df.loc['a',:] = df.ix[0]
+        self.assertRaises(ValueError, f)
+
+        def f():
+            df.ix[100.0, :] = df.ix[0]
+        self.assertRaises(ValueError, f)
+        def f():
+            df.ix[100,:] = df.ix[0]
+        self.assertRaises(ValueError, f)
+
     def test_cache_updating(self):
         # GH 4939, make sure to update the cache on setitem