gh-121485: Always use 64-bit integers for integers bits count (GH-121486)

Use 64-bit integers instead of platform specific size_t or Py_ssize_t
to represent the number of bits in Python integer.

Author: serhiy-storchaka

Include/cpython/longobject.h

@@ -71,10 +71,10 @@ PyAPI_FUNC(int) _PyLong_Sign(PyObject *v);
    absolute value of a long.  For example, this returns 1 for 1 and -1, 2
    for 2 and -2, and 2 for 3 and -3.  It returns 0 for 0.
    v must not be NULL, and must be a normalized long.
-   (size_t)-1 is returned and OverflowError set if the true result doesn't
+   (uint64_t)-1 is returned and OverflowError set if the true result doesn't
    fit in a size_t.
 */
-PyAPI_FUNC(size_t) _PyLong_NumBits(PyObject *v);
+PyAPI_FUNC(uint64_t) _PyLong_NumBits(PyObject *v);
 
 /* _PyLong_FromByteArray:  View the n unsigned bytes as a binary integer in
    base 256, and return a Python int with the same numeric value.

Include/internal/pycore_long.h

@@ -86,7 +86,7 @@ static inline PyObject* _PyLong_FromUnsignedChar(unsigned char i)
 // OverflowError and returns -1.0 for x, 0 for e.
 //
 // Export for 'math' shared extension
-PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, Py_ssize_t *e);
+PyAPI_DATA(double) _PyLong_Frexp(PyLongObject *a, int64_t *e);
 
 extern PyObject* _PyLong_FromBytes(const char *, Py_ssize_t, int);
 
@@ -105,10 +105,10 @@ PyAPI_DATA(PyObject*) _PyLong_DivmodNear(PyObject *, PyObject *);
 PyAPI_DATA(PyObject*) _PyLong_Format(PyObject *obj, int base);
 
 // Export for 'math' shared extension
-PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, size_t);
+PyAPI_DATA(PyObject*) _PyLong_Rshift(PyObject *, uint64_t);
 
 // Export for 'math' shared extension
-PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, size_t);
+PyAPI_DATA(PyObject*) _PyLong_Lshift(PyObject *, uint64_t);
 
 PyAPI_FUNC(PyObject*) _PyLong_Add(PyLongObject *left, PyLongObject *right);
 PyAPI_FUNC(PyObject*) _PyLong_Multiply(PyLongObject *left, PyLongObject *right);

Lib/test/test_capi/test_long.py

@@ -340,6 +340,18 @@ def test_long_aspid(self):
     def test_long_aspid_limited(self):
         self._test_long_aspid(_testlimitedcapi.pylong_aspid)
 
+    @support.bigmemtest(2**32, memuse=0.35)
+    def test_long_asnativebytes_huge(self, size):
+        asnativebytes = _testcapi.pylong_asnativebytes
+        v = 1 << size
+        buffer = bytearray(size * 2 // 15 + 10)
+        r = asnativebytes(v, buffer, 0, -1)
+        self.assertEqual(r, size // 8 + 1)
+        self.assertEqual(buffer.count(0), len(buffer))
+        r = asnativebytes(v, buffer, len(buffer), -1)
+        self.assertEqual(r, size // 8 + 1)
+        self.assertEqual(buffer.count(0), len(buffer) - 1)
+
     def test_long_asnativebytes(self):
         import math
         from _testcapi import (

Lib/test/test_long.py

@@ -473,6 +473,12 @@ def test_float_conversion(self):
             self.check_float_conversion(value)
             self.check_float_conversion(-value)
 
+    @support.requires_IEEE_754
+    @support.bigmemtest(2**32, memuse=0.2)
+    def test_float_conversion_huge_integer(self, size):
+        v = 1 << size
+        self.assertRaises(OverflowError, float, v)
+
     def test_float_overflow(self):
         for x in -2.0, -1.0, 0.0, 1.0, 2.0:
             self.assertEqual(float(int(x)), x)
@@ -614,6 +620,56 @@ def __lt__(self, other):
                     eq(x > y, Rcmp > 0)
                     eq(x >= y, Rcmp >= 0)
 
+    @support.requires_IEEE_754
+    @support.bigmemtest(2**32, memuse=0.2)
+    def test_mixed_compares_huge_integer(self, size):
+        v = 1 << size
+        f = sys.float_info.max
+        self.assertIs(f == v, False)
+        self.assertIs(f != v, True)
+        self.assertIs(f < v, True)
+        self.assertIs(f <= v, True)
+        self.assertIs(f > v, False)
+        self.assertIs(f >= v, False)
+        f = float('inf')
+        self.assertIs(f == v, False)
+        self.assertIs(f != v, True)
+        self.assertIs(f < v, False)
+        self.assertIs(f <= v, False)
+        self.assertIs(f > v, True)
+        self.assertIs(f >= v, True)
+        f = float('nan')
+        self.assertIs(f == v, False)
+        self.assertIs(f != v, True)
+        self.assertIs(f < v, False)
+        self.assertIs(f <= v, False)
+        self.assertIs(f > v, False)
+        self.assertIs(f >= v, False)
+
+        del v
+        v = (-1) << size
+        f = -sys.float_info.max
+        self.assertIs(f == v, False)
+        self.assertIs(f != v, True)
+        self.assertIs(f < v, False)
+        self.assertIs(f <= v, False)
+        self.assertIs(f > v, True)
+        self.assertIs(f >= v, True)
+        f = float('-inf')
+        self.assertIs(f == v, False)
+        self.assertIs(f != v, True)
+        self.assertIs(f < v, True)
+        self.assertIs(f <= v, True)
+        self.assertIs(f > v, False)
+        self.assertIs(f >= v, False)
+        f = float('nan')
+        self.assertIs(f == v, False)
+        self.assertIs(f != v, True)
+        self.assertIs(f < v, False)
+        self.assertIs(f <= v, False)
+        self.assertIs(f > v, False)
+        self.assertIs(f >= v, False)
+
     def test__format__(self):
         self.assertEqual(format(123456789, 'd'), '123456789')
         self.assertEqual(format(123456789, 'd'), '123456789')
@@ -933,9 +989,12 @@ def test_huge_lshift_of_zero(self):
         self.assertEqual(0 << (sys.maxsize + 1), 0)
 
     @support.cpython_only
-    @support.bigmemtest(sys.maxsize + 1000, memuse=2/15 * 2, dry_run=False)
+    @support.bigmemtest(2**32, memuse=0.2)
     def test_huge_lshift(self, size):
-        self.assertEqual(1 << (sys.maxsize + 1000), 1 << 1000 << sys.maxsize)
+        v = 5 << size
+        self.assertEqual(v.bit_length(), size + 3)
+        self.assertEqual(v.bit_count(), 2)
+        self.assertEqual(v >> size, 5)
 
     def test_huge_rshift(self):
         huge_shift = 1 << 1000
@@ -947,11 +1006,13 @@ def test_huge_rshift(self):
         self.assertEqual(-2**128 >> huge_shift, -1)
 
     @support.cpython_only
-    @support.bigmemtest(sys.maxsize + 500, memuse=2/15, dry_run=False)
+    @support.bigmemtest(2**32, memuse=0.2)
     def test_huge_rshift_of_huge(self, size):
-        huge = ((1 << 500) + 11) << sys.maxsize
-        self.assertEqual(huge >> (sys.maxsize + 1), (1 << 499) + 5)
-        self.assertEqual(huge >> (sys.maxsize + 1000), 0)
+        huge = ((1 << 500) + 11) << size
+        self.assertEqual(huge.bit_length(), size + 501)
+        self.assertEqual(huge.bit_count(), 4)
+        self.assertEqual(huge >> (size + 1), (1 << 499) + 5)
+        self.assertEqual(huge >> (size + 1000), 0)
 
     def test_small_rshift(self):
         self.assertEqual(42 >> 1, 21)

Lib/test/test_math.py

@@ -1120,6 +1120,15 @@ def __index__(self):
                 with self.assertRaises(TypeError):
                     math.isqrt(value)
 
+    @support.bigmemtest(2**32, memuse=0.85)
+    def test_isqrt_huge(self, size):
+        if size & 1:
+            size += 1
+        v = 1 << size
+        w = math.isqrt(v)
+        self.assertEqual(w.bit_length(), size // 2 + 1)
+        self.assertEqual(w.bit_count(), 1)
+
     def test_lcm(self):
         lcm = math.lcm
         self.assertEqual(lcm(0, 0), 0)
@@ -1261,6 +1270,13 @@ def testLog10(self):
         self.assertEqual(math.log(INF), INF)
         self.assertTrue(math.isnan(math.log10(NAN)))
 
+    @support.bigmemtest(2**32, memuse=0.2)
+    def test_log_huge_integer(self, size):
+        v = 1 << size
+        self.assertAlmostEqual(math.log2(v), size)
+        self.assertAlmostEqual(math.log(v), size * 0.6931471805599453)
+        self.assertAlmostEqual(math.log10(v), size * 0.3010299956639812)
+
     def testSumProd(self):
         sumprod = math.sumprod
         Decimal = decimal.Decimal

Misc/NEWS.d/next/Library/2024-07-09-12-23-32.gh-issue-121486.Iultjh.rst

@@ -0,0 +1,3 @@
+:mod:`math` functions :func:`~math.isqrt`, :func:`~math.log`, :func:`~math.log2` and
+:func:`~math.log10` now support integers larger than ``2**2**32`` on 32-bit
+platforms.

Modules/_pickle.c

@@ -2140,7 +2140,7 @@ save_long(PicklerObject *self, PyObject *obj)
 
     if (self->proto >= 2) {
         /* Linear-time pickling. */
-        size_t nbits;
+        uint64_t nbits;
         size_t nbytes;
         unsigned char *pdata;
         char header[5];
@@ -2155,7 +2155,7 @@ save_long(PicklerObject *self, PyObject *obj)
             return 0;
         }
         nbits = _PyLong_NumBits(obj);
-        if (nbits == (size_t)-1 && PyErr_Occurred())
+        if (nbits == (uint64_t)-1 && PyErr_Occurred())
             goto error;
         /* How many bytes do we need?  There are nbits >> 3 full
          * bytes of data, and nbits & 7 leftover bits.  If there
@@ -2171,7 +2171,7 @@ save_long(PicklerObject *self, PyObject *obj)
          * for in advance, though, so we always grab an extra
          * byte at the start, and cut it back later if possible.
          */
-        nbytes = (nbits >> 3) + 1;
+        nbytes = (size_t)((nbits >> 3) + 1);
         if (nbytes > 0x7fffffffL) {
             PyErr_SetString(PyExc_OverflowError,
                             "int too large to pickle");

Modules/_randommodule.c

@@ -295,7 +295,8 @@ random_seed(RandomObject *self, PyObject *arg)
     int result = -1;  /* guilty until proved innocent */
     PyObject *n = NULL;
     uint32_t *key = NULL;
-    size_t bits, keyused;
+    uint64_t bits;
+    size_t keyused;
     int res;
 
     if (arg == NULL || arg == Py_None) {
@@ -334,11 +335,11 @@ random_seed(RandomObject *self, PyObject *arg)
 
     /* Now split n into 32-bit chunks, from the right. */
     bits = _PyLong_NumBits(n);
-    if (bits == (size_t)-1 && PyErr_Occurred())
+    if (bits == (uint64_t)-1 && PyErr_Occurred())
         goto Done;
 
     /* Figure out how many 32-bit chunks this gives us. */
-    keyused = bits == 0 ? 1 : (bits - 1) / 32 + 1;
+    keyused = bits == 0 ? 1 : (size_t)((bits - 1) / 32 + 1);
 
     /* Convert seed to byte sequence. */
     key = (uint32_t *)PyMem_Malloc((size_t)4 * keyused);

Modules/_testinternalcapi.c

@@ -1853,7 +1853,7 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module)
 {
     struct triple {
         long input;
-        size_t nbits;
+        uint64_t nbits;
         int sign;
     } testcases[] = {{0, 0, 0},
                      {1L, 1, 1},
@@ -1873,7 +1873,7 @@ _testinternalcapi_test_long_numbits_impl(PyObject *module)
     size_t i;
 
     for (i = 0; i < Py_ARRAY_LENGTH(testcases); ++i) {
-        size_t nbits;
+        uint64_t nbits;
         int sign;
         PyObject *plong;
 

Modules/mathmodule.c

@@ -1657,7 +1657,7 @@ math_isqrt(PyObject *module, PyObject *n)
 /*[clinic end generated code: output=35a6f7f980beab26 input=5b6e7ae4fa6c43d6]*/
 {
     int a_too_large, c_bit_length;
-    size_t c, d;
+    uint64_t c, d;
     uint64_t m;
     uint32_t u;
     PyObject *a = NULL, *b;
@@ -1680,7 +1680,7 @@ math_isqrt(PyObject *module, PyObject *n)
 
     /* c = (n.bit_length() - 1) // 2 */
     c = _PyLong_NumBits(n);
-    if (c == (size_t)(-1)) {
+    if (c == (uint64_t)(-1)) {
         goto error;
     }
     c = (c - 1U) / 2U;
@@ -1727,7 +1727,7 @@ math_isqrt(PyObject *module, PyObject *n)
 
     for (int s = c_bit_length - 6; s >= 0; --s) {
         PyObject *q;
-        size_t e = d;
+        uint64_t e = d;
 
         d = c >> s;
 
@@ -2185,7 +2185,7 @@ loghelper(PyObject* arg, double (*func)(double))
     /* If it is int, do it ourselves. */
     if (PyLong_Check(arg)) {
         double x, result;
-        Py_ssize_t e;
+        int64_t e;
 
         /* Negative or zero inputs give a ValueError. */
         if (!_PyLong_IsPositive((PyLongObject *)arg)) {

Objects/floatobject.c

@@ -399,7 +399,6 @@ float_richcompare(PyObject *v, PyObject *w, int op)
     else if (PyLong_Check(w)) {
         int vsign = i == 0.0 ? 0 : i < 0.0 ? -1 : 1;
         int wsign = _PyLong_Sign(w);
-        size_t nbits;
         int exponent;
 
         if (vsign != wsign) {
@@ -412,20 +411,25 @@ float_richcompare(PyObject *v, PyObject *w, int op)
         }
         /* The signs are the same. */
         /* Convert w to a double if it fits.  In particular, 0 fits. */
-        nbits = _PyLong_NumBits(w);
-        if (nbits == (size_t)-1 && PyErr_Occurred()) {
-            /* This long is so large that size_t isn't big enough
-             * to hold the # of bits.  Replace with little doubles
+        uint64_t nbits64 = _PyLong_NumBits(w);
+        if (nbits64 > (unsigned int)DBL_MAX_EXP) {
+            /* This Python integer is larger than any finite C double.
+             * Replace with little doubles
              * that give the same outcome -- w is so large that
              * its magnitude must exceed the magnitude of any
              * finite float.
              */
-            PyErr_Clear();
+            if (nbits64 == (uint64_t)-1 && PyErr_Occurred()) {
+                /* This Python integer is so large that uint64_t isn't
+                 * big enough to hold the # of bits. */
+                PyErr_Clear();
+            }
             i = (double)vsign;
             assert(wsign != 0);
             j = wsign * 2.0;
             goto Compare;
         }
+        int nbits = (int)nbits64;
         if (nbits <= 48) {
             j = PyLong_AsDouble(w);
             /* It's impossible that <= 48 bits overflowed. */
@@ -449,12 +453,12 @@ float_richcompare(PyObject *v, PyObject *w, int op)
         /* exponent is the # of bits in v before the radix point;
          * we know that nbits (the # of bits in w) > 48 at this point
          */
-        if (exponent < 0 || (size_t)exponent < nbits) {
+        if (exponent < nbits) {
             i = 1.0;
             j = 2.0;
             goto Compare;
         }
-        if ((size_t)exponent > nbits) {
+        if (exponent > nbits) {
             i = 2.0;
             j = 1.0;
             goto Compare;