perf: some quick refactoring to test #1527

Author: nedbat

lab/benchmark.py

@@ -11,7 +11,7 @@
 import time
 from pathlib import Path
 
-from typing import Dict, Iterable, Iterator, List, Optional, Tuple
+from typing import Any, Dict, Iterable, Iterator, List, Optional, Tuple
 
 
 class ShellSession:
@@ -116,17 +116,21 @@ class ProjectToTest:
 
     # Where can we clone the project from?
     git_url: Optional[str] = None
+    slug: Optional[str] = None
 
     def __init__(self):
-        if self.git_url:
-            self.slug = self.git_url.split("/")[-1]
-            self.dir = Path(self.slug)
+        if not self.slug:
+            if self.git_url:
+                self.slug = self.git_url.split("/")[-1]
 
-    def get_source(self, shell):
-        """Get the source of the project."""
+    def make_dir(self):
+        self.dir = Path(f"work_{self.slug}")
         if self.dir.exists():
             rmrf(self.dir)
-        shell.run_command(f"git clone {self.git_url}")
+
+    def get_source(self, shell):
+        """Get the source of the project."""
+        shell.run_command(f"git clone {self.git_url} {self.dir}")
 
     def prep_environment(self, env):
         """Prepare the environment to run the test suite.
@@ -135,20 +139,41 @@ def prep_environment(self, env):
         """
         pass
 
+    def tweak_coverage_settings(self, settings: Iterable[Tuple[str, Any]]) -> Iterator[None]:
+        """Tweak the coverage settings.
+
+        NOTE: This is not properly factored, and is only used by ToxProject now!!!
+        """
+        pass
+
     def run_no_coverage(self, env):
         """Run the test suite with no coverage measurement."""
         pass
 
-    def run_with_coverage(self, env, pip_args, cov_options):
+    def run_with_coverage(self, env, pip_args, cov_tweaks):
         """Run the test suite with coverage measurement."""
         pass
 
 
+class EmptyProject(ProjectToTest):
+    """A dummy project for testing other parts of this code."""
+    def __init__(self, slug: str="empty", fake_durations: Iterable[float]=(1.23,)):
+        self.slug = slug
+        self.durations = iter(itertools.cycle(fake_durations))
+
+    def get_source(self, shell):
+        pass
+
+    def run_with_coverage(self, env, pip_args, cov_tweaks):
+        """Run the test suite with coverage measurement."""
+        return next(self.durations)
+
+
 class ToxProject(ProjectToTest):
     """A project using tox to run the test suite."""
 
     def prep_environment(self, env):
-        env.shell.run_command(f"{env.python} -m pip install tox")
+        env.shell.run_command(f"{env.python} -m pip install 'tox<4'")
         self.run_tox(env, env.pyver.toxenv, "--notest")
 
     def run_tox(self, env, toxenv, toxargs=""):
@@ -159,26 +184,30 @@ def run_tox(self, env, toxenv, toxargs=""):
     def run_no_coverage(self, env):
         return self.run_tox(env, env.pyver.toxenv, "--skip-pkg-install")
 
-    def run_with_coverage(self, env, pip_args, cov_options):
-        assert not cov_options, f"ToxProject.run_with_coverage can't take cov_options={cov_options!r}"
+    def run_with_coverage(self, env, pip_args, cov_tweaks):
         self.run_tox(env, env.pyver.toxenv, "--notest")
         env.shell.run_command(
             f".tox/{env.pyver.toxenv}/bin/python -m pip install {pip_args}"
         )
-        return self.run_tox(env, env.pyver.toxenv, "--skip-pkg-install")
+        with self.tweak_coverage_settings(cov_tweaks):
+            self.pre_check(env)     # NOTE: Not properly factored, and only used from here.
+            duration = self.run_tox(env, env.pyver.toxenv, "--skip-pkg-install")
+            self.post_check(env)    # NOTE: Not properly factored, and only used from here.
+        return duration
 
 
 class ProjectPytestHtml(ToxProject):
     """pytest-dev/pytest-html"""
 
     git_url = "https://github.com/pytest-dev/pytest-html"
 
-    def run_with_coverage(self, env, pip_args, cov_options):
+    def run_with_coverage(self, env, pip_args, cov_tweaks):
+        raise Exception("This doesn't work because options changed to tweaks")
         covenv = env.pyver.toxenv + "-cov"
         self.run_tox(env, covenv, "--notest")
         env.shell.run_command(f".tox/{covenv}/bin/python -m pip install {pip_args}")
-        if cov_options:
-            replace = ("# reference: https", f"[run]\n{cov_options}\n#")
+        if cov_tweaks:
+            replace = ("# reference: https", f"[run]\n{cov_tweaks}\n#")
         else:
             replace = ("", "")
         with file_replace(Path(".coveragerc"), *replace):
@@ -206,6 +235,34 @@ class ProjectAttrs(ToxProject):
 
     git_url = "https://github.com/python-attrs/attrs"
 
+    def tweak_coverage_settings(self, tweaks: Iterable[Tuple[str, Any]]) -> Iterator[None]:
+        return tweak_toml_coverage_settings("pyproject.toml", tweaks)
+
+    def pre_check(self, env):
+        env.shell.run_command("cat pyproject.toml")
+
+    def post_check(self, env):
+        env.shell.run_command("ls -al")
+
+
+def tweak_toml_coverage_settings(toml_file: str, tweaks: Iterable[Tuple[str, Any]]) -> Iterator[None]:
+    if tweaks:
+        toml_inserts = []
+        for name, value in tweaks:
+            if isinstance(value, bool):
+                toml_inserts.append(f"{name} = {str(value).lower()}")
+            elif isinstance(value, str):
+                toml_inserts.append(f"{name} = '{value}'")
+            else:
+                raise Exception(f"Can't tweak toml setting: {name} = {value!r}")
+        header = "[tool.coverage.run]\n"
+        insert = header + "\n".join(toml_inserts) + "\n"
+    else:
+        header = insert = ""
+    return file_replace(Path(toml_file), header, insert)
+
+
+
 
 class AdHocProject(ProjectToTest):
     """A standalone program to run locally."""
@@ -232,7 +289,7 @@ def run_no_coverage(self, env):
             env.shell.run_command(f"{env.python} {self.python_file}")
         return env.shell.last_duration
 
-    def run_with_coverage(self, env, pip_args, cov_options):
+    def run_with_coverage(self, env, pip_args, cov_tweaks):
         env.shell.run_command(f"{env.python} -m pip install {pip_args}")
         with change_dir(self.cur_dir):
             env.shell.run_command(
@@ -265,15 +322,13 @@ class PyVersion:
     # The tox environment to run this Python
     toxenv: str
 
-
 class Python(PyVersion):
     """A version of CPython to use."""
 
     def __init__(self, major, minor):
         self.command = self.slug = f"python{major}.{minor}"
         self.toxenv = f"py{major}{minor}"
 
-
 class PyPy(PyVersion):
     """A version of PyPy to use."""
 
@@ -288,6 +343,7 @@ def __init__(self, path, slug):
         self.slug = slug
         self.toxenv = None
 
+
 @dataclasses.dataclass
 class Coverage:
     """A version of coverage.py to use, maybe None."""
@@ -296,33 +352,33 @@ class Coverage:
     # Arguments for "pip install ..."
     pip_args: Optional[str] = None
     # Tweaks to the .coveragerc file
-    options: Optional[str] = None
+    tweaks: Optional[Iterable[Tuple[str, Any]]] = None
 
 class CoveragePR(Coverage):
     """A version of coverage.py from a pull request."""
-    def __init__(self, number, options=None):
+    def __init__(self, number, tweaks=None):
         super().__init__(
             slug=f"#{number}",
             pip_args=f"git+https://github.com/nedbat/coveragepy.git@refs/pull/{number}/merge",
-            options=options,
+            tweaks=tweaks,
         )
 
 class CoverageCommit(Coverage):
     """A version of coverage.py from a specific commit."""
-    def __init__(self, sha, options=None):
+    def __init__(self, sha, tweaks=None):
         super().__init__(
             slug=sha,
             pip_args=f"git+https://github.com/nedbat/coveragepy.git@{sha}",
-            options=options,
+            tweaks=tweaks,
         )
 
 class CoverageSource(Coverage):
     """The coverage.py in a working tree."""
-    def __init__(self, directory, options=None):
+    def __init__(self, directory, tweaks=None):
         super().__init__(
             slug="source",
             pip_args=directory,
-            options=options,
+            tweaks=tweaks,
         )
 
 
@@ -337,6 +393,8 @@ class Env:
 
 ResultData = Dict[Tuple[str, str, str], float]
 
+DIMENSION_NAMES = ["proj", "pyver", "cov"]
+
 class Experiment:
     """A particular time experiment to run."""
 
@@ -353,9 +411,18 @@ def __init__(
 
     def run(self, num_runs: int = 3) -> None:
         results = []
+        total_runs = (
+            len(self.projects) *
+            len(self.py_versions) *
+            len(self.cov_versions) *
+            num_runs
+        )
+        total_run_nums = iter(itertools.count(start=1))
+
         for proj in self.projects:
             print(f"Testing with {proj.slug}")
             with ShellSession(f"output_{proj.slug}.log") as shell:
+                proj.make_dir()
                 proj.get_source(shell)
 
                 for pyver in self.py_versions:
@@ -366,20 +433,23 @@ def run(self, num_runs: int = 3) -> None:
                     shell.run_command(f"{python} -V")
                     env = Env(pyver, python, shell)
 
-                    with change_dir(Path(proj.slug)):
+                    with change_dir(proj.dir):
                         print(f"Prepping for {proj.slug} {pyver.slug}")
                         proj.prep_environment(env)
                         for cov_ver in self.cov_versions:
                             durations = []
                             for run_num in range(num_runs):
+                                total_run_num = next(total_run_nums)
                                 print(
-                                    f"Running tests, cov={cov_ver.slug}, {run_num+1} of {num_runs}"
+                                    f"Running tests, cov={cov_ver.slug}, " +
+                                    f"{run_num+1} of {num_runs}, " +
+                                    f"total {total_run_num}/{total_runs}"
                                 )
                                 if cov_ver.pip_args is None:
                                     dur = proj.run_no_coverage(env)
                                 else:
                                     dur = proj.run_with_coverage(
-                                        env, cov_ver.pip_args, cov_ver.options,
+                                        env, cov_ver.pip_args, cov_ver.tweaks,
                                     )
                                 print(f"Tests took {dur:.3f}s")
                                 durations.append(dur)
@@ -411,7 +481,7 @@ def show_results(
 
         table_axes = [dimensions[rowname] for rowname in rows]
         data_order = [*rows, column]
-        remap = [data_order.index(datum) for datum in ["proj", "pyver", "cov"]]
+        remap = [data_order.index(datum) for datum in DIMENSION_NAMES]
 
         WIDTH = 20
         def as_table_row(vals):
@@ -445,8 +515,12 @@ def as_table_row(vals):
 PERF_DIR = Path("/tmp/covperf")
 
 def run_experiment(
-    py_versions: List[PyVersion], cov_versions: List[Coverage], projects: List[ProjectToTest],
-    rows: List[str], column: str, ratios: Iterable[Tuple[str, str, str]] = (),
+    py_versions: List[PyVersion],
+    cov_versions: List[Coverage],
+    projects: List[ProjectToTest],
+    rows: List[str],
+    column: str,
+    ratios: Iterable[Tuple[str, str, str]] = (),
 ):
     slugs = [v.slug for v in py_versions + cov_versions + projects]
     if len(set(slugs)) != len(slugs):
@@ -456,6 +530,8 @@ def run_experiment(
     ratio_slugs = [rslug for ratio in ratios for rslug in ratio[1:]]
     if any(rslug not in slugs for rslug in ratio_slugs):
         raise Exception(f"Ratio slug doesn't match a slug: {ratio_slugs}, {slugs}")
+    if set(rows + [column]) != set(DIMENSION_NAMES):
+        raise Exception(f"All of these must be in rows or column: {', '.join(DIMENSION_NAMES)}")
 
     print(f"Removing and re-making {PERF_DIR}")
     rmrf(PERF_DIR)
@@ -466,7 +542,7 @@ def run_experiment(
         exp.show_results(rows=rows, column=column, ratios=ratios)
 
 
-if 1:
+if 0:
     run_experiment(
         py_versions=[
             #Python(3, 11),
@@ -489,3 +565,30 @@ def run_experiment(
             ("94231 vs 3.10", "94231", "v3.10.5"),
         ],
     )
+
+
+if 1:
+    run_experiment(
+        py_versions=[
+            Python(3, 11),
+        ],
+        cov_versions=[
+            Coverage("701", "coverage==7.0.1"),
+            Coverage("701.dynctx", "coverage==7.0.1", [("dynamic_context", "test_function")]),
+            Coverage("702", "coverage==7.0.2"),
+            Coverage("702.dynctx", "coverage==7.0.2", [("dynamic_context", "test_function")]),
+        ],
+        projects=[
+            #EmptyProject("empty", [1.2, 3.4]),
+            #EmptyProject("dummy", [6.9, 7.1]),
+            #ProjectDateutil(),
+            ProjectAttrs(),
+        ],
+        rows=["proj", "pyver"],
+        column="cov",
+        ratios=[
+            (".2 vs .1", "702", "701"),
+            (".1 dynctx cost", "701.dynctx", "701"),
+            (".2 dynctx cost", "702.dynctx", "702"),
+        ],
+    )