Skip to content

No virtual modules & No scoped loops for unknown collectors. #747

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jan 16, 2024
Merged

No virtual modules & No scoped loops for unknown collectors. #747

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2301f95
[fix] pytest-asyncio no longer uses virtual modules to install dynami…
seifertm Jan 14, 2024
d437496
[fix] Avoid trying to install scoped event loops for unknown test col…
seifertm Jan 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/source/reference/changelog.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ Changelog
0.23.4 (UNRELEASED)
===================
- pytest-asyncio no longer imports additional, unrelated packages during test collection `#729 <https://github.com/pytest-dev/pytest-asyncio/issues/729>`_
- Addresses further issues that caused an internal pytest error during test collection

Known issues
------------
Expand Down
142 changes: 32 additions & 110 deletions pytest_asyncio/plugin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,13 +4,10 @@
import enum
import functools
import inspect
import os
import socket
import sys
import warnings
from asyncio import AbstractEventLoopPolicy
from pathlib import Path
from tempfile import NamedTemporaryFile
from textwrap import dedent
from typing import (
Any,
Expand All @@ -31,7 +28,6 @@
)

import pytest
from _pytest.pathlib import visit
from pytest import (
Class,
Collector,
Expand Down Expand Up @@ -612,7 +608,33 @@ def scoped_event_loop(
# know it exists. We work around this by attaching the fixture function to the
# collected Python object, where it will be picked up by pytest.Class.collect()
# or pytest.Module.collect(), respectively
if type(collector) is Module:
if type(collector) is Package:

def _patched_collect():
# When collector is a Package, collector.obj is the package's
# __init__.py. Accessing the __init__.py to attach the fixture function
# may trigger additional module imports or change the order of imports,
# which leads to a number of problems.
# see https://github.com/pytest-dev/pytest-asyncio/issues/729
# Moreover, Package.obj has been removed in pytest 8.
# Therefore, pytest-asyncio attaches the packages-scoped event loop
# fixture to the first collected module in that package.
package_scoped_loop_added = False
for subcollector in collector.__original_collect():
if (
not package_scoped_loop_added
and isinstance(subcollector, Module)
and getattr(subcollector, "obj", None)
):
subcollector.obj.__pytest_asyncio_package_scoped_event_loop = (
scoped_event_loop
)
package_scoped_loop_added = True
yield subcollector

collector.__original_collect = collector.collect
collector.collect = _patched_collect
elif isinstance(collector, Module):
# Accessing Module.obj triggers a module import executing module-level
# statements. A module-level pytest.skip statement raises the "Skipped"
# OutcomeException or a Collector.CollectError, if the "allow_module_level"
Expand All @@ -621,115 +643,15 @@ def scoped_event_loop(
# Therefore, we monkey patch Module.collect to add the scoped fixture to the
# module before it runs the actual collection.
def _patched_collect():
collector.obj.__pytest_asyncio_scoped_event_loop = scoped_event_loop
# If the collected module is a DoctestTextfile, collector.obj is None
if collector.obj is not None:
collector.obj.__pytest_asyncio_scoped_event_loop = scoped_event_loop
return collector.__original_collect()

collector.__original_collect = collector.collect
collector.collect = _patched_collect
elif type(collector) is Package:
if not collector.funcnamefilter(collector.name):
return

def _patched_collect():
# pytest.Package collects all files and sub-packages. Pytest 8 changes
# this logic to only collect a single directory. Sub-packages are then
# collected by a separate Package collector. Therefore, this logic can be
# dropped, once we move to pytest 8.
collector_dir = Path(collector.path.parent)
for direntry in visit(str(collector_dir), recurse=collector._recurse):
if not direntry.name == "__init__.py":
# No need to register a package-scoped fixture, if we aren't
# collecting a (sub-)package
continue
pkgdir = Path(direntry.path).parent
pkg_nodeid = str(pkgdir.relative_to(collector_dir))
if pkg_nodeid == ".":
pkg_nodeid = ""
# Pytest's fixture matching algorithm compares a fixture's baseid with
# an Item's nodeid to determine whether a fixture is available for a
# specific Item. Package.nodeid ends with __init__.py, so the
# fixture's baseid will also end with __init__.py and prevents
# the fixture from being matched to test items in the package.
# Furthermore, Package also collects any sub-packages, which means
# the ID of the scoped event loop for the package must change for
# each sub-package.
# As the fixture matching is purely based on string comparison, we
# can assemble a path based on the root package path
# (i.e. Package.path.parent) and the sub-package path
# (i.e. Path(direntry.path).parent)). This makes the fixture visible
# to all items in the package.
# see also https://github.com/pytest-dev/pytest/issues/11662#issuecomment-1879310072 # noqa
# Possibly related to https://github.com/pytest-dev/pytest/issues/4085
fixture_id = f"{pkg_nodeid}/__init__.py::<event_loop>".lstrip("/")
# When collector is a Package, collector.obj is the package's
# __init__.py. Accessing the __init__.py to attach the fixture function
# may trigger additional module imports or change the order of imports,
# which leads to a number of problems.
# see https://github.com/pytest-dev/pytest-asyncio/issues/729
# Moreover, Package.obj has been removed in pytest 8.
# Therefore, pytest-asyncio creates a temporary Python module inside the
# collected package. The sole purpose of that module is to house a
# fixture function for the pacakge-scoped event loop fixture. Once the
# fixture has been evaluated by pytest, the temporary module
# can be removed.
with NamedTemporaryFile(
dir=pkgdir,
prefix="pytest_asyncio_virtual_module_",
suffix=".py",
delete=False, # Required for Windows compatibility
) as virtual_module_file:
virtual_module = Module.from_parent(
collector, path=Path(virtual_module_file.name)
)
virtual_module_file.write(
dedent(
f"""\
# This is a temporary file created by pytest-asyncio
# If you see this file, a pytest run has crashed and
# wasn't able to clean up the file in time.
# You can safely remove this file.
import asyncio
import pytest
from pytest_asyncio.plugin \
import _temporary_event_loop_policy
@pytest.fixture(
scope="{collector_scope}",
name="{fixture_id}",
)
def scoped_event_loop(
*args,
event_loop_policy,
):
new_loop_policy = event_loop_policy
with _temporary_event_loop_policy(new_loop_policy):
loop = asyncio.new_event_loop()
loop.__pytest_asyncio = True
asyncio.set_event_loop(loop)
yield loop
loop.close()
"""
).encode()
)
virtual_module_file.flush()
fixturemanager = collector.config.pluginmanager.get_plugin(
"funcmanage"
)
# Collect the fixtures in the virtual module with the node ID of
# the current sub-package to ensure correct fixture matching.
# see also https://github.com/pytest-dev/pytest/issues/11662#issuecomment-1879310072 # noqa
fixturemanager.parsefactories(virtual_module.obj, nodeid=pkg_nodeid)
yield virtual_module
os.unlink(virtual_module_file.name)
yield from collector.__original_collect()

collector.__original_collect = collector.collect
collector.collect = _patched_collect
else:
pyobject = collector.obj
# If the collected module is a DoctestTextfile, collector.obj is None
if pyobject is None:
return
pyobject.__pytest_asyncio_scoped_event_loop = scoped_event_loop
elif isinstance(collector, Class):
collector.obj.__pytest_asyncio_scoped_event_loop = scoped_event_loop


def _removesuffix(s: str, suffix: str) -> str:
Expand Down