Skip to content

Commit db07696

Browse files
authored
Merge pull request #3472 from Zac-HD/better-ftz-error
2 parents 35c228c + e8c8932 commit db07696

File tree

5 files changed

+178
-5
lines changed

5 files changed

+178
-5
lines changed

hypothesis-python/.coveragerc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
[run]
22
branch = True
33
omit =
4+
**/_hypothesis_ftz_detector.py
45
**/_hypothesis_pytestplugin.py
56
**/extra/array_api.py
67
**/extra/cli.py

hypothesis-python/RELEASE.rst

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
RELEASE_TYPE: patch
2+
3+
This patch improves the error message when Hypothesis detects "flush to zero"
4+
mode for floating-point: we now report which package(s) enabled this, which
5+
can make debugging much easier. See :issue:`3458` for details.

hypothesis-python/setup.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ def local_file(name):
128128
"Topic :: Software Development :: Testing",
129129
"Typing :: Typed",
130130
],
131-
py_modules=["_hypothesis_pytestplugin"],
131+
py_modules=["_hypothesis_pytestplugin", "_hypothesis_ftz_detector"],
132132
entry_points={
133133
"pytest11": ["hypothesispytest = _hypothesis_pytestplugin"],
134134
"console_scripts": ["hypothesis = hypothesis.extra.cli:main"],
Lines changed: 149 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
# This file is part of Hypothesis, which may be found at
2+
# https://github.com/HypothesisWorks/hypothesis/
3+
#
4+
# Copyright the Hypothesis Authors.
5+
# Individual contributors are listed in AUTHORS.rst and the git log.
6+
#
7+
# This Source Code Form is subject to the terms of the Mozilla Public License,
8+
# v. 2.0. If a copy of the MPL was not distributed with this file, You can
9+
# obtain one at https://mozilla.org/MPL/2.0/.
10+
11+
"""
12+
This is a toolkit for determining which module set the "flush to zero" flag.
13+
14+
For details, see the docstring and comments in `identify_ftz_culprit()`. This module
15+
is defined outside the main Hypothesis namespace so that we can avoid triggering
16+
import of Hypothesis itself from each subprocess which must import the worker function.
17+
"""
18+
19+
import importlib
20+
import sys
21+
22+
KNOWN_EVER_CULPRITS = (
23+
# https://moyix.blogspot.com/2022/09/someones-been-messing-with-my-subnormals.html
24+
# fmt: off
25+
"archive-pdf-tools", "bgfx-python", "bicleaner-ai-glove", "BTrees", "cadbiom",
26+
"ctranslate2", "dyNET", "dyNET38", "gevent", "glove-python-binary", "higra",
27+
"hybridq", "ikomia", "ioh", "jij-cimod", "lavavu", "lavavu-osmesa", "MulticoreTSNE",
28+
"neural-compressor", "nwhy", "openjij", "openturns", "perfmetrics", "pHashPy",
29+
"pyace-lite", "pyapr", "pycompadre", "pycompadre-serial", "PyKEP", "pykep",
30+
"pylimer-tools", "pyqubo", "pyscf", "PyTAT", "python-prtree", "qiskit-aer",
31+
"qiskit-aer-gpu", "RelStorage", "sail-ml", "segmentation", "sente", "sinr",
32+
"snapml", "superman", "symengine", "systran-align", "texture-tool", "tsne-mp",
33+
"xcsf",
34+
# fmt: on
35+
)
36+
37+
38+
def flush_to_zero():
39+
# If this subnormal number compares equal to zero we have a problem
40+
return 2.0**-1073 == 0
41+
42+
43+
def run_in_process(fn, *args):
44+
import multiprocessing as mp
45+
46+
mp.set_start_method("spawn", force=True)
47+
q = mp.Queue()
48+
p = mp.Process(target=target, args=(q, fn, *args))
49+
p.start()
50+
retval = q.get()
51+
p.join()
52+
return retval
53+
54+
55+
def target(q, fn, *args):
56+
q.put(fn(*args))
57+
58+
59+
def always_imported_modules():
60+
return flush_to_zero(), set(sys.modules)
61+
62+
63+
def modules_imported_by(mod):
64+
"""Return the set of modules imported transitively by mod."""
65+
before = set(sys.modules)
66+
try:
67+
importlib.import_module(mod)
68+
except Exception:
69+
return None, set()
70+
imports = set(sys.modules) - before
71+
return flush_to_zero(), imports
72+
73+
74+
# We don't want to redo all the expensive process-spawning checks when we've already
75+
# done them, so we cache known-good packages and a known-FTZ result if we have one.
76+
KNOWN_FTZ = None
77+
CHECKED_CACHE = set()
78+
79+
80+
def identify_ftz_culprits():
81+
"""Find the modules in sys.modules which cause "mod" to be imported."""
82+
# If we've run this function before, return the same result.
83+
global KNOWN_FTZ
84+
if KNOWN_FTZ:
85+
return KNOWN_FTZ
86+
# Start by determining our baseline: the FTZ and sys.modules state in a fresh
87+
# process which has only imported this module and nothing else.
88+
always_enables_ftz, always_imports = run_in_process(always_imported_modules)
89+
if always_enables_ftz:
90+
raise RuntimeError("Python is always in FTZ mode, even without imports!")
91+
CHECKED_CACHE.update(always_imports)
92+
93+
# Next, we'll search through sys.modules looking for a package (or packages) such
94+
# that importing them in a new process sets the FTZ state. As a heuristic, we'll
95+
# start with packages known to have ever enabled FTZ, then top-level packages as
96+
# a way to eliminate large fractions of the search space relatively quickly.
97+
def key(name):
98+
"""Prefer known-FTZ modules, then top-level packages, then alphabetical."""
99+
return (name not in KNOWN_EVER_CULPRITS, name.count("."), name)
100+
101+
# We'll track the set of modules to be checked, and those which do trigger FTZ.
102+
candidates = set(sys.modules) - CHECKED_CACHE
103+
triggering_modules = {}
104+
while candidates:
105+
mod = min(candidates, key=key)
106+
candidates.discard(mod)
107+
enables_ftz, imports = run_in_process(modules_imported_by, mod)
108+
imports -= CHECKED_CACHE
109+
if enables_ftz:
110+
triggering_modules[mod] = imports
111+
candidates &= imports
112+
else:
113+
candidates -= imports
114+
CHECKED_CACHE.update(imports)
115+
116+
# We only want to report the 'top level' packages which enable FTZ - for example,
117+
# if the enabling code is in `a.b`, and `a` in turn imports `a.b`, we prefer to
118+
# report `a`. On the other hand, if `a` does _not_ import `a.b`, as is the case
119+
# for `hypothesis.extra.*` modules, then `a` will not be in `triggering_modules`
120+
# and we'll report `a.b` here instead.
121+
prefixes = tuple(n + "." for n in triggering_modules)
122+
result = {k for k in triggering_modules if not k.startswith(prefixes)}
123+
124+
# Suppose that `bar` enables FTZ, and `foo` imports `bar`. At this point we're
125+
# tracking both, but only want to report the latter.
126+
for a in sorted(result):
127+
for b in sorted(result):
128+
if a in triggering_modules[b] and b not in triggering_modules[a]:
129+
result.discard(b)
130+
131+
# There may be a cyclic dependency which that didn't handle, or simply two
132+
# separate modules which both enable FTZ. We already gave up comprehensive
133+
# reporting for speed above (`candidates &= imports`), so we'll also buy
134+
# simpler reporting by arbitrarily selecting the alphabetically first package.
135+
KNOWN_FTZ = min(result) # Cache the result - it's likely this will trigger again!
136+
return KNOWN_FTZ
137+
138+
139+
if __name__ == "__main__":
140+
# This would be really really annoying to write automated tests for, so I've
141+
# done some manual exploratory testing: `pip install grequests gevent==21.12.0`,
142+
# and call print() as desired to observe behavior.
143+
import grequests # noqa
144+
145+
# To test without skipping to a known answer, uncomment the following line and
146+
# change the last element of key from `name` to `-len(name)` so that we check
147+
# grequests before gevent.
148+
## KNOWN_EVER_CULPRITS = [c for c in KNOWN_EVER_CULPRITS if c != "gevent"]
149+
print(identify_ftz_culprits())

hypothesis-python/src/hypothesis/strategies/_internal/numbers.py

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -450,13 +450,31 @@ def floats(
450450
# Erroring out here ensures that the database contents are interpreted
451451
# consistently - which matters for such a foundational strategy, even if it's
452452
# not always true for all user-composed strategies further up the stack.
453+
from _hypothesis_ftz_detector import identify_ftz_culprits
454+
455+
try:
456+
ftz_pkg = identify_ftz_culprits()
457+
except Exception:
458+
ftz_pkg = None
459+
if ftz_pkg:
460+
ftz_msg = (
461+
f"This seems to be because the `{ftz_pkg}` package was compiled with "
462+
f"-ffast-math or a similar option, which sets global processor state "
463+
f"- see https://simonbyrne.github.io/notes/fastmath/ for details. "
464+
f"If you don't know why {ftz_pkg} is installed, `pipdeptree -rp "
465+
f"{ftz_pkg}` will show which packages depend on it."
466+
)
467+
else:
468+
ftz_msg = (
469+
"This is usually because something was compiled with -ffast-math "
470+
"or a similar option, which sets global processor state. See "
471+
"https://simonbyrne.github.io/notes/fastmath/ for a more detailed "
472+
"writeup - and good luck!"
473+
)
453474
raise FloatingPointError(
454475
f"Got allow_subnormal={allow_subnormal!r}, but we can't represent "
455476
f"subnormal floats right now, in violation of the IEEE-754 floating-point "
456-
f"specification. This is usually because something was compiled with "
457-
f"-ffast-math or a similar option, which sets global processor state. "
458-
f"See https://simonbyrne.github.io/notes/fastmath/ for a more detailed "
459-
f"writeup - and good luck!"
477+
f"specification. {ftz_msg}"
460478
)
461479

462480
min_arg, max_arg = min_value, max_value

0 commit comments

Comments
 (0)