|
| 1 | +# This file is part of Hypothesis, which may be found at |
| 2 | +# https://github.com/HypothesisWorks/hypothesis/ |
| 3 | +# |
| 4 | +# Copyright the Hypothesis Authors. |
| 5 | +# Individual contributors are listed in AUTHORS.rst and the git log. |
| 6 | +# |
| 7 | +# This Source Code Form is subject to the terms of the Mozilla Public License, |
| 8 | +# v. 2.0. If a copy of the MPL was not distributed with this file, You can |
| 9 | +# obtain one at https://mozilla.org/MPL/2.0/. |
| 10 | + |
| 11 | +""" |
| 12 | +This is a toolkit for determining which module set the "flush to zero" flag. |
| 13 | +
|
| 14 | +For details, see the docstring and comments in `identify_ftz_culprit()`. This module |
| 15 | +is defined outside the main Hypothesis namespace so that we can avoid triggering |
| 16 | +import of Hypothesis itself from each subprocess which must import the worker function. |
| 17 | +""" |
| 18 | + |
| 19 | +import importlib |
| 20 | +import sys |
| 21 | + |
| 22 | +KNOWN_EVER_CULPRITS = ( |
| 23 | + # https://moyix.blogspot.com/2022/09/someones-been-messing-with-my-subnormals.html |
| 24 | + # fmt: off |
| 25 | + "archive-pdf-tools", "bgfx-python", "bicleaner-ai-glove", "BTrees", "cadbiom", |
| 26 | + "ctranslate2", "dyNET", "dyNET38", "gevent", "glove-python-binary", "higra", |
| 27 | + "hybridq", "ikomia", "ioh", "jij-cimod", "lavavu", "lavavu-osmesa", "MulticoreTSNE", |
| 28 | + "neural-compressor", "nwhy", "openjij", "openturns", "perfmetrics", "pHashPy", |
| 29 | + "pyace-lite", "pyapr", "pycompadre", "pycompadre-serial", "PyKEP", "pykep", |
| 30 | + "pylimer-tools", "pyqubo", "pyscf", "PyTAT", "python-prtree", "qiskit-aer", |
| 31 | + "qiskit-aer-gpu", "RelStorage", "sail-ml", "segmentation", "sente", "sinr", |
| 32 | + "snapml", "superman", "symengine", "systran-align", "texture-tool", "tsne-mp", |
| 33 | + "xcsf", |
| 34 | + # fmt: on |
| 35 | +) |
| 36 | + |
| 37 | + |
| 38 | +def flush_to_zero(): |
| 39 | + # If this subnormal number compares equal to zero we have a problem |
| 40 | + return 2.0**-1073 == 0 |
| 41 | + |
| 42 | + |
| 43 | +def run_in_process(fn, *args): |
| 44 | + import multiprocessing as mp |
| 45 | + |
| 46 | + mp.set_start_method("spawn", force=True) |
| 47 | + q = mp.Queue() |
| 48 | + p = mp.Process(target=target, args=(q, fn, *args)) |
| 49 | + p.start() |
| 50 | + retval = q.get() |
| 51 | + p.join() |
| 52 | + return retval |
| 53 | + |
| 54 | + |
| 55 | +def target(q, fn, *args): |
| 56 | + q.put(fn(*args)) |
| 57 | + |
| 58 | + |
| 59 | +def always_imported_modules(): |
| 60 | + return flush_to_zero(), set(sys.modules) |
| 61 | + |
| 62 | + |
| 63 | +def modules_imported_by(mod): |
| 64 | + """Return the set of modules imported transitively by mod.""" |
| 65 | + before = set(sys.modules) |
| 66 | + try: |
| 67 | + importlib.import_module(mod) |
| 68 | + except Exception: |
| 69 | + return None, set() |
| 70 | + imports = set(sys.modules) - before |
| 71 | + return flush_to_zero(), imports |
| 72 | + |
| 73 | + |
| 74 | +# We don't want to redo all the expensive process-spawning checks when we've already |
| 75 | +# done them, so we cache known-good packages and a known-FTZ result if we have one. |
| 76 | +KNOWN_FTZ = None |
| 77 | +CHECKED_CACHE = set() |
| 78 | + |
| 79 | + |
| 80 | +def identify_ftz_culprits(): |
| 81 | + """Find the modules in sys.modules which cause "mod" to be imported.""" |
| 82 | + # If we've run this function before, return the same result. |
| 83 | + global KNOWN_FTZ |
| 84 | + if KNOWN_FTZ: |
| 85 | + return KNOWN_FTZ |
| 86 | + # Start by determining our baseline: the FTZ and sys.modules state in a fresh |
| 87 | + # process which has only imported this module and nothing else. |
| 88 | + always_enables_ftz, always_imports = run_in_process(always_imported_modules) |
| 89 | + if always_enables_ftz: |
| 90 | + raise RuntimeError("Python is always in FTZ mode, even without imports!") |
| 91 | + CHECKED_CACHE.update(always_imports) |
| 92 | + |
| 93 | + # Next, we'll search through sys.modules looking for a package (or packages) such |
| 94 | + # that importing them in a new process sets the FTZ state. As a heuristic, we'll |
| 95 | + # start with packages known to have ever enabled FTZ, then top-level packages as |
| 96 | + # a way to eliminate large fractions of the search space relatively quickly. |
| 97 | + def key(name): |
| 98 | + """Prefer known-FTZ modules, then top-level packages, then alphabetical.""" |
| 99 | + return (name not in KNOWN_EVER_CULPRITS, name.count("."), name) |
| 100 | + |
| 101 | + # We'll track the set of modules to be checked, and those which do trigger FTZ. |
| 102 | + candidates = set(sys.modules) - CHECKED_CACHE |
| 103 | + triggering_modules = {} |
| 104 | + while candidates: |
| 105 | + mod = min(candidates, key=key) |
| 106 | + candidates.discard(mod) |
| 107 | + enables_ftz, imports = run_in_process(modules_imported_by, mod) |
| 108 | + imports -= CHECKED_CACHE |
| 109 | + if enables_ftz: |
| 110 | + triggering_modules[mod] = imports |
| 111 | + candidates &= imports |
| 112 | + else: |
| 113 | + candidates -= imports |
| 114 | + CHECKED_CACHE.update(imports) |
| 115 | + |
| 116 | + # We only want to report the 'top level' packages which enable FTZ - for example, |
| 117 | + # if the enabling code is in `a.b`, and `a` in turn imports `a.b`, we prefer to |
| 118 | + # report `a`. On the other hand, if `a` does _not_ import `a.b`, as is the case |
| 119 | + # for `hypothesis.extra.*` modules, then `a` will not be in `triggering_modules` |
| 120 | + # and we'll report `a.b` here instead. |
| 121 | + prefixes = tuple(n + "." for n in triggering_modules) |
| 122 | + result = {k for k in triggering_modules if not k.startswith(prefixes)} |
| 123 | + |
| 124 | + # Suppose that `bar` enables FTZ, and `foo` imports `bar`. At this point we're |
| 125 | + # tracking both, but only want to report the latter. |
| 126 | + for a in sorted(result): |
| 127 | + for b in sorted(result): |
| 128 | + if a in triggering_modules[b] and b not in triggering_modules[a]: |
| 129 | + result.discard(b) |
| 130 | + |
| 131 | + # There may be a cyclic dependency which that didn't handle, or simply two |
| 132 | + # separate modules which both enable FTZ. We already gave up comprehensive |
| 133 | + # reporting for speed above (`candidates &= imports`), so we'll also buy |
| 134 | + # simpler reporting by arbitrarily selecting the alphabetically first package. |
| 135 | + KNOWN_FTZ = min(result) # Cache the result - it's likely this will trigger again! |
| 136 | + return KNOWN_FTZ |
| 137 | + |
| 138 | + |
| 139 | +if __name__ == "__main__": |
| 140 | + # This would be really really annoying to write automated tests for, so I've |
| 141 | + # done some manual exploratory testing: `pip install grequests gevent==21.12.0`, |
| 142 | + # and call print() as desired to observe behavior. |
| 143 | + import grequests # noqa |
| 144 | + |
| 145 | + # To test without skipping to a known answer, uncomment the following line and |
| 146 | + # change the last element of key from `name` to `-len(name)` so that we check |
| 147 | + # grequests before gevent. |
| 148 | + ## KNOWN_EVER_CULPRITS = [c for c in KNOWN_EVER_CULPRITS if c != "gevent"] |
| 149 | + print(identify_ftz_culprits()) |
0 commit comments