Skip to content

Commit c8b4fde

Browse files
committed
Work around nanoarrow bug
1 parent 0cb78cb commit c8b4fde

File tree

1 file changed

+34
-2
lines changed

1 file changed

+34
-2
lines changed

pandas/_libs/arrow_string_accumulations.cc

+34-2
Original file line numberDiff line numberDiff line change
@@ -36,9 +36,25 @@ static auto CumSum(struct ArrowArrayStream *array_stream,
3636
bool seen_na = false;
3737
std::stringstream ss{};
3838

39+
// TODO: we can simplify this further if we just iterate on the array
40+
// and not the array view, but there is an upstream bug in nanoarrow
41+
// that prevents that
42+
// https://github.com/apache/arrow-nanoarrow/issues/701
43+
nanoarrow::UniqueArrayView array_view{};
44+
nanoarrow::UniqueSchema schema{};
45+
NANOARROW_THROW_NOT_OK(
46+
ArrowArrayStreamGetSchema(array_stream, schema.get(), nullptr));
47+
3948
nanoarrow::ViewArrayStream array_stream_view(array_stream);
4049
for (const auto &array : array_stream_view) {
41-
for (const auto &sv : nanoarrow::ViewArrayAsBytes<OffsetSize>(&array)) {
50+
array_view.reset();
51+
NANOARROW_THROW_NOT_OK(
52+
ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), nullptr));
53+
NANOARROW_THROW_NOT_OK(
54+
ArrowArrayViewSetArray(array_view.get(), &array, nullptr));
55+
56+
for (const auto &sv :
57+
nanoarrow::ViewArrayAsBytes<OffsetSize>(array_view.get())) {
4258
if ((!sv || seen_na) && !skipna) {
4359
seen_na = true;
4460
ArrowArrayAppendNull(out, 1);
@@ -68,9 +84,25 @@ static auto CumMinOrMax(struct ArrowArrayStream *array_stream,
6884
bool seen_na = false;
6985
std::optional<std::string> current_str{};
7086

87+
// TODO: we can simplify this further if we just iterate on the array
88+
// and not the array view, but there is an upstream bug in nanoarrow
89+
// that prevents that
90+
// https://github.com/apache/arrow-nanoarrow/issues/701
91+
nanoarrow::UniqueArrayView array_view{};
92+
nanoarrow::UniqueSchema schema{};
93+
NANOARROW_THROW_NOT_OK(
94+
ArrowArrayStreamGetSchema(array_stream, schema.get(), nullptr));
95+
7196
nanoarrow::ViewArrayStream array_stream_view(array_stream);
7297
for (const auto &array : array_stream_view) {
73-
for (const auto &sv : nanoarrow::ViewArrayAsBytes<OffsetSize>(&array)) {
98+
array_view.reset();
99+
NANOARROW_THROW_NOT_OK(
100+
ArrowArrayViewInitFromSchema(array_view.get(), schema.get(), nullptr));
101+
NANOARROW_THROW_NOT_OK(
102+
ArrowArrayViewSetArray(array_view.get(), &array, nullptr));
103+
104+
for (const auto &sv :
105+
nanoarrow::ViewArrayAsBytes<OffsetSize>(array_view.get())) {
74106
if ((!sv || seen_na) && !skipna) {
75107
seen_na = true;
76108
ArrowArrayAppendNull(out, 1);

0 commit comments

Comments
 (0)