diff --git a/setup.py b/setup.py
index 618cad331c25a..01d41ea5241f2 100755
--- a/setup.py
+++ b/setup.py
@@ -545,12 +545,10 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
     return pjoin("pandas", subdir, name + suffix)
 
 
-common_include = ["pandas/_libs/src/klib", "pandas/_libs/src"]
-ts_include = ["pandas/_libs/tslibs/src", "pandas/_libs/tslibs"]
-
-
 lib_depends = ["pandas/_libs/src/parse_helper.h"]
 
+klib_include = ["pandas/_libs/src/klib"]
+
 np_datetime_headers = [
     "pandas/_libs/tslibs/src/datetime/np_datetime.h",
     "pandas/_libs/tslibs/src/datetime/np_datetime_strings.h",
@@ -564,36 +562,42 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
 
 
 ext_data = {
-    "_libs.algos": {"pyxfile": "_libs/algos", "depends": _pxi_dep["algos"]},
+    "_libs.algos": {
+        "pyxfile": "_libs/algos",
+        "include": klib_include,
+        "depends": _pxi_dep["algos"],
+    },
     "_libs.groupby": {"pyxfile": "_libs/groupby"},
-    "_libs.hashing": {"pyxfile": "_libs/hashing", "include": [], "depends": []},
+    "_libs.hashing": {"pyxfile": "_libs/hashing", "depends": []},
     "_libs.hashtable": {
         "pyxfile": "_libs/hashtable",
+        "include": klib_include,
         "depends": (["pandas/_libs/src/klib/khash_python.h"] + _pxi_dep["hashtable"]),
     },
     "_libs.index": {
         "pyxfile": "_libs/index",
-        "include": common_include + ts_include,
+        "include": klib_include,
         "depends": _pxi_dep["index"],
         "sources": np_datetime_sources,
     },
     "_libs.indexing": {"pyxfile": "_libs/indexing"},
     "_libs.internals": {"pyxfile": "_libs/internals"},
-    "_libs.interval": {"pyxfile": "_libs/interval", "depends": _pxi_dep["interval"]},
-    "_libs.join": {"pyxfile": "_libs/join"},
+    "_libs.interval": {
+        "pyxfile": "_libs/interval",
+        "include": klib_include,
+        "depends": _pxi_dep["interval"],
+    },
+    "_libs.join": {"pyxfile": "_libs/join", "include": klib_include},
     "_libs.lib": {
         "pyxfile": "_libs/lib",
-        "include": common_include + ts_include,
         "depends": lib_depends + tseries_depends,
+        "include": klib_include,  # due to tokenizer import
         "sources": ["pandas/_libs/src/parser/tokenizer.c"],
     },
-    "_libs.missing": {
-        "pyxfile": "_libs/missing",
-        "include": common_include + ts_include,
-        "depends": tseries_depends,
-    },
+    "_libs.missing": {"pyxfile": "_libs/missing", "depends": tseries_depends},
     "_libs.parsers": {
         "pyxfile": "_libs/parsers",
+        "include": klib_include + ["pandas/_libs/src"],
         "depends": [
             "pandas/_libs/src/parser/tokenizer.h",
             "pandas/_libs/src/parser/io.h",
@@ -605,87 +609,76 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
     },
     "_libs.reduction": {"pyxfile": "_libs/reduction"},
     "_libs.ops": {"pyxfile": "_libs/ops"},
-    "_libs.properties": {"pyxfile": "_libs/properties", "include": []},
+    "_libs.properties": {"pyxfile": "_libs/properties"},
     "_libs.reshape": {"pyxfile": "_libs/reshape", "depends": []},
     "_libs.sparse": {"pyxfile": "_libs/sparse", "depends": _pxi_dep["sparse"]},
     "_libs.tslib": {
         "pyxfile": "_libs/tslib",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.c_timestamp": {
         "pyxfile": "_libs/tslibs/c_timestamp",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
-    "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar", "include": []},
+    "_libs.tslibs.ccalendar": {"pyxfile": "_libs/tslibs/ccalendar"},
     "_libs.tslibs.conversion": {
         "pyxfile": "_libs/tslibs/conversion",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.fields": {
         "pyxfile": "_libs/tslibs/fields",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
-    "_libs.tslibs.frequencies": {"pyxfile": "_libs/tslibs/frequencies", "include": []},
-    "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype", "include": []},
+    "_libs.tslibs.frequencies": {"pyxfile": "_libs/tslibs/frequencies"},
+    "_libs.tslibs.nattype": {"pyxfile": "_libs/tslibs/nattype"},
     "_libs.tslibs.np_datetime": {
         "pyxfile": "_libs/tslibs/np_datetime",
-        "include": ts_include,
         "depends": np_datetime_headers,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.offsets": {
         "pyxfile": "_libs/tslibs/offsets",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.parsing": {
         "pyxfile": "_libs/tslibs/parsing",
+        "include": klib_include,
         "depends": ["pandas/_libs/src/parser/tokenizer.h"],
         "sources": ["pandas/_libs/src/parser/tokenizer.c"],
     },
     "_libs.tslibs.period": {
         "pyxfile": "_libs/tslibs/period",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.resolution": {
         "pyxfile": "_libs/tslibs/resolution",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.strptime": {
         "pyxfile": "_libs/tslibs/strptime",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.timedeltas": {
         "pyxfile": "_libs/tslibs/timedeltas",
-        "include": ts_include,
         "depends": np_datetime_headers,
         "sources": np_datetime_sources,
     },
     "_libs.tslibs.timestamps": {
         "pyxfile": "_libs/tslibs/timestamps",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
-    "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones", "include": []},
+    "_libs.tslibs.timezones": {"pyxfile": "_libs/tslibs/timezones"},
     "_libs.tslibs.tzconversion": {
         "pyxfile": "_libs/tslibs/tzconversion",
-        "include": ts_include,
         "depends": tseries_depends,
         "sources": np_datetime_sources,
     },
@@ -709,7 +702,7 @@ def srcpath(name=None, suffix=".pyx", subdir="src"):
 
     sources.extend(data.get("sources", []))
 
-    include = data.get("include", common_include)
+    include = data.get("include")
 
     obj = Extension(
         "pandas.{name}".format(name=name),