Skip to content

PERF: maybe_convert_numeric speedup #16104

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Apr 25, 2017
Merged

PERF: maybe_convert_numeric speedup #16104

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
5e6343e
inference speedup
chris-b1 Apr 23, 2017
6a01fb5
fix casting, add fastpath
chris-b1 Apr 24, 2017
e28ea41
1 -> 0
chris-b1 Apr 24, 2017
4734f16
handle nan
chris-b1 Apr 24, 2017
ce519b7
expand error catching
chris-b1 Apr 24, 2017
9757099
lint, float unboxing
chris-b1 Apr 24, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 19 additions & 14 deletions pandas/_libs/src/inference.pyx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ cdef class Seen(object):
encountered when trying to perform type conversions.
"""

cdef public:
cdef:
bint int_ # seen_int
bint bool_ # seen_bool
bint null_ # seen_null
Expand Down Expand Up @@ -185,7 +185,7 @@ cdef class Seen(object):
self.null_ = 1
self.float_ = 1

def saw_int(self, val):
cdef saw_int(self, object val):
"""
Set flags indicating that an integer value was encountered.

Expand All @@ -196,7 +196,7 @@ cdef class Seen(object):
"""
self.int_ = 1
self.sint_ = self.sint_ or (val < 0)
self.uint_ = self.uint_ or (val > iINT64_MAX)
self.uint_ = self.uint_ or (val > oINT64_MAX)

@property
def numeric_(self):
Expand Down Expand Up @@ -908,11 +908,15 @@ cpdef bint is_interval_array(ndarray[object] values):
cdef extern from "parse_helper.h":
inline int floatify(object, double *result, int *maybe_int) except -1

cdef int64_t iINT64_MAX = <int64_t> INT64_MAX
cdef int64_t iINT64_MIN = <int64_t> INT64_MIN
cdef uint64_t iUINT64_MAX = <uint64_t> UINT64_MAX
# constants that will be compared to potentially arbitrarily large
# python int
cdef object oINT64_MAX = <object> INT64_MAX
cdef object oINT64_MIN = <object> INT64_MIN
cdef object oUINT64_MAX = <object> UINT64_MAX


@cython.boundscheck(False)
@cython.wraparound(False)
def maybe_convert_numeric(ndarray[object] values, set na_values,
bint convert_empty=True, bint coerce_numeric=False):
"""
Expand Down Expand Up @@ -970,13 +974,12 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
elif util.is_integer_object(val):
floats[i] = complexes[i] = val

as_int = int(val)
seen.saw_int(as_int)
seen.saw_int(val)

if as_int >= 0:
uints[i] = as_int
if as_int <= iINT64_MAX:
ints[i] = as_int
if val >= 0:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you might be able to

cdef ival <int> val
if ival >= 0:
     ...
if ival <= iINT64_max:
    ..

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can't do exactly this because val could be larger than fits into a c int

uints[i] = val
if val <= oINT64_MAX:
ints[i] = val
elif util.is_bool_object(val):
floats[i] = uints[i] = ints[i] = bools[i] = val
seen.bool_ = True
Expand Down Expand Up @@ -1017,12 +1020,12 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
seen.saw_int(as_int)

if not (seen.float_ or as_int in na_values):
if as_int < iINT64_MIN or as_int > iUINT64_MAX:
if as_int < oINT64_MIN or as_int > oUINT64_MAX:
raise ValueError('Integer out of range.')

if as_int >= 0:
uints[i] = as_int
if as_int <= iINT64_MAX:
if as_int <= oINT64_MAX:
ints[i] = as_int
else:
seen.float_ = True
Expand Down Expand Up @@ -1053,6 +1056,8 @@ def maybe_convert_numeric(ndarray[object] values, set na_values,
return ints


@cython.boundscheck(False)
@cython.wraparound(False)
def maybe_convert_objects(ndarray[object] objects, bint try_float=0,
bint safe=0, bint convert_datetime=0,
bint convert_timedelta=0):
Expand Down