Skip to content

CLN: Cython Py2/3 Compatible Imports #23940

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 18 commits into from
Dec 26, 2018
Merged

CLN: Cython Py2/3 Compatible Imports #23940

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
0c84dcb
Updated setup
WillAyd Nov 27, 2018
0e90b55
Conversion to absolute imports
WillAyd Nov 27, 2018
62bee57
Missed imports, division, unicode compat
WillAyd Nov 27, 2018
43c8098
Missing import resolution for strptime
WillAyd Nov 27, 2018
385e2da
Timestamp import failure
WillAyd Nov 27, 2018
836f3f2
Merge remote-tracking branch 'upstream/master' into cython-py3-compat
WillAyd Nov 28, 2018
7522818
Reverted non-import changes
WillAyd Nov 28, 2018
ea818aa
Reverted floor div changes
WillAyd Nov 28, 2018
1d6d196
LINT fixup
WillAyd Dec 1, 2018
4d0437f
Merge remote-tracking branch 'upstream/master' into cython-py3-compat
WillAyd Dec 1, 2018
c78d1a9
Merge remote-tracking branch 'upstream/master' into cython-py3-compat
WillAyd Dec 3, 2018
5d09285
Relative imports in tslibs folder
WillAyd Dec 3, 2018
db2acb4
Missed relative imports
WillAyd Dec 3, 2018
f25c0d0
Back to absolute
WillAyd Dec 4, 2018
9034cba
Merge remote-tracking branch 'upstream/master' into cython-py3-compat
WillAyd Dec 4, 2018
c3bce2f
Merge remote-tracking branch 'upstream/master' into cython-py3-compat
WillAyd Dec 6, 2018
877b3d5
Merge remote-tracking branch 'upstream/master' into cython-py3-compat
WillAyd Dec 25, 2018
b908e69
Merge remote-tracking branch 'upstream/master' into cython-py3-compat
WillAyd Dec 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pandas/_libs/algos.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from util cimport numeric
from pandas._libs.util cimport numeric


cdef inline Py_ssize_t swap(numeric *a, numeric *b) nogil:
Expand Down
13 changes: 6 additions & 7 deletions pandas/_libs/algos.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,15 +19,14 @@ from numpy cimport (ndarray,
cnp.import_array()


cimport util
from util cimport numeric, get_nat
cimport pandas._libs.util as util
from pandas._libs.util cimport numeric, get_nat

from khash cimport (khiter_t,
kh_destroy_int64, kh_put_int64,
kh_init_int64, kh_int64_t,
kh_resize_int64, kh_get_int64)
from pandas._libs.khash cimport (
khiter_t, kh_destroy_int64, kh_put_int64, kh_init_int64, kh_int64_t,
kh_resize_int64, kh_get_int64)

import missing
import pandas._libs.missing as missing

cdef float64_t FP_ERR = 1e-13

Expand Down
16 changes: 9 additions & 7 deletions pandas/_libs/groupby.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,13 @@ from numpy cimport (ndarray,
cnp.import_array()


from util cimport numeric, get_nat
from pandas._libs.util cimport numeric, get_nat

from algos cimport (swap, TiebreakEnumType, TIEBREAK_AVERAGE, TIEBREAK_MIN,
TIEBREAK_MAX, TIEBREAK_FIRST, TIEBREAK_DENSE)
from algos import take_2d_axis1_float64_float64, groupsort_indexer, tiebreakers
from pandas._libs.algos cimport (swap, TiebreakEnumType, TIEBREAK_AVERAGE,
TIEBREAK_MIN, TIEBREAK_MAX, TIEBREAK_FIRST,
TIEBREAK_DENSE)
from pandas._libs.algos import (take_2d_axis1_float64_float64,
groupsort_indexer, tiebreakers)

cdef int64_t NPY_NAT = get_nat()

Expand Down Expand Up @@ -54,10 +56,10 @@ cdef inline float64_t median_linear(float64_t* a, int n) nogil:
n -= na_count

if n % 2:
result = kth_smallest_c( a, n / 2, n)
result = kth_smallest_c( a, n // 2, n)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Space after open paren

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure if it changes with language-level, but in cython / defaults to cdivision, which behaves like python floor-division (... mostly. its corner cases are bizarre). The cython docs says there is a non-trivial perf impact of disabling cdivision. It might be worth checking how it treats floordiv.

Then again, this isn't an op we're doing zillions of times, so not a huge deal.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If you try to just use / here compilation will fail as you try to assign a double to an int

else:
result = (kth_smallest_c(a, n / 2, n) +
kth_smallest_c(a, n / 2 - 1, n)) / 2
result = (kth_smallest_c(a, n // 2, n) +
kth_smallest_c(a, n // 2 - 1, n)) / 2

if na_count:
free(a)
Expand Down
2 changes: 1 addition & 1 deletion pandas/_libs/hashing.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import numpy as np
from numpy cimport uint8_t, uint32_t, uint64_t, import_array
import_array()

from util cimport is_nan
from pandas._libs.util cimport is_nan

DEF cROUNDS = 2
DEF dROUNDS = 4
Expand Down
5 changes: 3 additions & 2 deletions pandas/_libs/hashtable.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
from khash cimport (kh_int64_t, kh_uint64_t, kh_float64_t, kh_pymap_t,
kh_str_t, uint64_t, int64_t, float64_t)
from pandas._libs.khash cimport (
kh_int64_t, kh_uint64_t, kh_float64_t, kh_pymap_t, kh_str_t, uint64_t,
int64_t, float64_t)
from numpy cimport ndarray

# prototypes for sharing
Expand Down
6 changes: 3 additions & 3 deletions pandas/_libs/hashtable.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ cdef extern from "numpy/npy_math.h":
float64_t NAN "NPY_NAN"


from khash cimport (
from pandas._libs.khash cimport (
khiter_t,

kh_str_t, kh_init_str, kh_put_str, kh_exist_str,
Expand All @@ -37,9 +37,9 @@ from khash cimport (
kh_put_pymap, kh_resize_pymap)


cimport util
cimport pandas._libs.util as util

from missing cimport checknull
from pandas._libs.missing cimport checknull


cdef int64_t NPY_NAT = util.get_nat()
Expand Down
6 changes: 3 additions & 3 deletions pandas/_libs/index.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,11 @@ from numpy cimport (ndarray, intp_t,
cnp.import_array()


cimport util
cimport pandas._libs.util as util

from tslibs.conversion cimport maybe_datetimelike_to_i8
from pandas._libs.tslibs.conversion cimport maybe_datetimelike_to_i8

from hashtable cimport HashTable
from pandas._libs.hashtable cimport HashTable

from pandas._libs import algos, hashtable as _hash
from pandas._libs.tslibs import Timestamp, Timedelta, period as periodlib
Expand Down
2 changes: 1 addition & 1 deletion pandas/_libs/internals.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ cdef extern from "compat_helper.h":
Py_ssize_t *slicelength) except -1


from algos import ensure_int64
from pandas._libs.algos import ensure_int64


cdef class BlockPlacement:
Expand Down
8 changes: 4 additions & 4 deletions pandas/_libs/interval.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,13 @@ from numpy cimport (
cnp.import_array()


cimport util
cimport pandas._libs.util as util
util.import_array()

from hashtable cimport Int64Vector, Int64VectorData
from pandas._libs.hashtable cimport Int64Vector, Int64VectorData

from tslibs import Timestamp
from tslibs.timezones cimport tz_compare
from pandas._libs.tslibs import Timestamp
from pandas._libs.tslibs.timezones cimport tz_compare


_VALID_CLOSED = frozenset(['left', 'right', 'both', 'neither'])
Expand Down
2 changes: 1 addition & 1 deletion pandas/_libs/join.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -656,7 +656,7 @@ outer_join_indexer_uint64 = outer_join_indexer["uint64_t"]
# asof_join_by
# ----------------------------------------------------------------------

from hashtable cimport (
from pandas._libs.hashtable cimport (
HashTable, PyObjectHashTable, UInt64HashTable, Int64HashTable)

ctypedef fused asof_t:
Expand Down
25 changes: 13 additions & 12 deletions pandas/_libs/lib.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,18 +49,19 @@ cdef extern from "numpy/arrayobject.h":
cdef extern from "src/parse_helper.h":
int floatify(object, float64_t *result, int *maybe_int) except -1

cimport util
from util cimport is_nan, UINT64_MAX, INT64_MAX, INT64_MIN

from tslib import array_to_datetime
from tslibs.nattype cimport NPY_NAT
from tslibs.nattype import NaT
from tslibs.conversion cimport convert_to_tsobject
from tslibs.timedeltas cimport convert_to_timedelta64
from tslibs.timezones cimport get_timezone, tz_compare

from missing cimport (checknull, isnaobj,
is_null_datetime64, is_null_timedelta64, is_null_period)
cimport pandas._libs.util as util
from pandas._libs.util cimport is_nan, UINT64_MAX, INT64_MAX, INT64_MIN

from pandas._libs.tslib import array_to_datetime
from pandas._libs.tslibs.nattype cimport NPY_NAT
from pandas._libs.tslibs.nattype import NaT
from pandas._libs.tslibs.conversion cimport convert_to_tsobject
from pandas._libs.tslibs.timedeltas cimport convert_to_timedelta64
from pandas._libs.tslibs.timezones cimport get_timezone, tz_compare

from pandas._libs.missing cimport (
checknull, isnaobj, is_null_datetime64, is_null_timedelta64, is_null_period
)


# constants that will be compared to potentially arbitrarily large
Expand Down
9 changes: 5 additions & 4 deletions pandas/_libs/missing.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ cimport numpy as cnp
from numpy cimport ndarray, int64_t, uint8_t, float64_t
cnp.import_array()

cimport util
cimport pandas._libs.util as util

from tslibs.np_datetime cimport get_timedelta64_value, get_datetime64_value
from tslibs.nattype cimport checknull_with_nat
from tslibs.nattype import NaT
from pandas._libs.tslibs.np_datetime cimport (
get_timedelta64_value, get_datetime64_value)
from pandas._libs.tslibs.nattype cimport checknull_with_nat
from pandas._libs.tslibs.nattype import NaT

cdef float64_t INF = <float64_t>np.inf
cdef float64_t NEGINF = -INF
Expand Down
4 changes: 2 additions & 2 deletions pandas/_libs/ops.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,9 @@ from numpy cimport ndarray, uint8_t, import_array
import_array()


from util cimport UINT8_MAX, is_nan
from pandas._libs.util cimport UINT8_MAX, is_nan

from missing cimport checknull
from pandas._libs.missing cimport checknull


@cython.wraparound(False)
Expand Down
6 changes: 3 additions & 3 deletions pandas/_libs/parsers.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,10 +32,10 @@ cimport numpy as cnp
from numpy cimport ndarray, uint8_t, uint64_t, int64_t, float64_t
cnp.import_array()

from util cimport UINT64_MAX, INT64_MAX, INT64_MIN
import lib
from pandas._libs.util cimport UINT64_MAX, INT64_MAX, INT64_MIN
import pandas._libs.lib as lib

from khash cimport (
from pandas._libs.khash cimport (
khiter_t,
kh_str_t, kh_init_str, kh_put_str, kh_exist_str,
kh_get_str, kh_destroy_str,
Expand Down
4 changes: 2 additions & 2 deletions pandas/_libs/reduction.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ from numpy cimport (ndarray,
flatiter)
cnp.import_array()

cimport util
from lib import maybe_convert_objects
cimport pandas._libs.util as util
from pandas._libs.lib import maybe_convert_objects


cdef _get_result_array(object obj, Py_ssize_t size, Py_ssize_t cnt):
Expand Down
30 changes: 15 additions & 15 deletions pandas/_libs/tslib.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,36 +17,36 @@ cnp.import_array()

import pytz

from util cimport (is_integer_object, is_float_object, is_string_object,
is_datetime64_object)
from pandas._libs.util cimport (
is_integer_object, is_float_object, is_string_object, is_datetime64_object)


from tslibs.np_datetime cimport (check_dts_bounds,
from pandas._libs.tslibs.np_datetime cimport (check_dts_bounds,
npy_datetimestruct,
_string_to_dts,
dt64_to_dtstruct, dtstruct_to_dt64,
pydatetime_to_dt64, pydate_to_dt64,
get_datetime64_value)
from tslibs.np_datetime import OutOfBoundsDatetime
from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime

from tslibs.parsing import parse_datetime_string
from pandas._libs.tslibs.parsing import parse_datetime_string

from tslibs.timedeltas cimport cast_from_unit
from tslibs.timezones cimport is_utc, is_tzlocal, get_dst_info
from tslibs.timezones import UTC
from tslibs.conversion cimport (tz_convert_single, _TSObject,
from pandas._libs.tslibs.timedeltas cimport cast_from_unit
from pandas._libs.tslibs.timezones cimport is_utc, is_tzlocal, get_dst_info
from pandas._libs.tslibs.timezones import UTC
from pandas._libs.tslibs.conversion cimport (tz_convert_single, _TSObject,
convert_datetime_to_tsobject,
get_datetime64_nanos,
tz_convert_utc_to_tzlocal)

# many modules still look for NaT and iNaT here despite them not being needed
from tslibs.nattype import nat_strings, NaT, iNaT # noqa:F821
from tslibs.nattype cimport checknull_with_nat, NPY_NAT
from pandas._libs.tslibs.nattype import nat_strings, NaT, iNaT # noqa:F821
from pandas._libs.tslibs.nattype cimport checknull_with_nat, NPY_NAT

from tslibs.offsets cimport to_offset
from pandas._libs.tslibs.offsets cimport to_offset

from tslibs.timestamps cimport create_timestamp_from_ts
from tslibs.timestamps import Timestamp
from pandas._libs.tslibs.timestamps cimport create_timestamp_from_ts
from pandas._libs.tslibs.timestamps import Timestamp


cdef bint PY2 = str == bytes
Expand Down Expand Up @@ -277,7 +277,7 @@ def format_array_from_datetime(ndarray[int64_t] values, object tz=None,
dts.sec)

if show_ns:
ns = dts.ps / 1000
ns = dts.ps // 1000
res += '.%.9d' % (ns + 1000 * dts.us)
elif show_us:
res += '.%.6d' % dts.us
Expand Down
2 changes: 1 addition & 1 deletion pandas/_libs/tslibs/ccalendar.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ cpdef int32_t get_week_of_year(int year, int month, int day) nogil:
# estimate
woy = (doy - 1) - dow + 3
if woy >= 0:
woy = woy / 7 + 1
woy = woy // 7 + 1

# verify
if woy < 0:
Expand Down
2 changes: 1 addition & 1 deletion pandas/_libs/tslibs/conversion.pxd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ from cpython.datetime cimport datetime, tzinfo

from numpy cimport int64_t, int32_t

from np_datetime cimport npy_datetimestruct
from pandas._libs.tslibs.np_datetime cimport npy_datetimestruct


cdef class _TSObject:
Expand Down
48 changes: 23 additions & 25 deletions pandas/_libs/tslibs/conversion.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,30 +17,28 @@ from cpython.datetime cimport (datetime, tzinfo,
PyDateTime_CheckExact, PyDateTime_IMPORT)
PyDateTime_IMPORT

from ccalendar import DAY_SECONDS, HOUR_SECONDS

from np_datetime cimport (check_dts_bounds,
npy_datetimestruct,
pandas_datetime_to_datetimestruct, _string_to_dts,
npy_datetime,
dt64_to_dtstruct, dtstruct_to_dt64,
get_datetime64_unit, get_datetime64_value,
pydatetime_to_dt64, NPY_DATETIMEUNIT, NPY_FR_ns)
from np_datetime import OutOfBoundsDatetime

from util cimport (is_string_object,
is_datetime64_object,
is_integer_object, is_float_object)

from timedeltas cimport cast_from_unit
from timezones cimport (is_utc, is_tzlocal, is_fixed_offset,
get_utcoffset, get_dst_info,
get_timezone, maybe_get_tz, tz_compare)
from timezones import UTC
from parsing import parse_datetime_string

from nattype import nat_strings, NaT
from nattype cimport NPY_NAT, checknull_with_nat
from pandas._libs.tslibs.ccalendar import DAY_SECONDS, HOUR_SECONDS

from pandas._libs.tslibs.np_datetime cimport (
check_dts_bounds, npy_datetimestruct,
pandas_datetime_to_datetimestruct, _string_to_dts,
npy_datetime, dt64_to_dtstruct, dtstruct_to_dt64,
get_datetime64_unit, get_datetime64_value,
pydatetime_to_dt64, NPY_DATETIMEUNIT, NPY_FR_ns)
from pandas._libs.tslibs.np_datetime import OutOfBoundsDatetime

from pandas._libs.tslibs.util cimport (
is_string_object, is_datetime64_object, is_integer_object, is_float_object)

from pandas._libs.tslibs.timedeltas cimport cast_from_unit
from pandas._libs.tslibs.timezones cimport (
is_utc, is_tzlocal, is_fixed_offset, get_utcoffset, get_dst_info,
get_timezone, maybe_get_tz, tz_compare)
from pandas._libs.tslibs.timezones import UTC
from pandas._libs.tslibs.parsing import parse_datetime_string

from pandas._libs.tslibs.nattype import nat_strings, NaT
from pandas._libs.tslibs.nattype cimport NPY_NAT, checknull_with_nat

# ----------------------------------------------------------------------
# Constants
Expand Down Expand Up @@ -455,7 +453,7 @@ cdef _TSObject convert_str_to_tsobject(object ts, object tz, object unit,
obj.dts.hour, obj.dts.min, obj.dts.sec,
obj.dts.us, obj.tzinfo)
obj = convert_datetime_to_tsobject(dt, tz,
nanos=obj.dts.ps / 1000)
nanos=obj.dts.ps // 1000)
return obj

else:
Expand Down
Loading