@@ -142,7 +142,9 @@ def prep_environment(self, env):
142
142
"""
143
143
pass
144
144
145
- def tweak_coverage_settings (self , settings : Iterable [Tuple [str , Any ]]) -> Iterator [None ]:
145
+ def tweak_coverage_settings (
146
+ self , settings : Iterable [Tuple [str , Any ]]
147
+ ) -> Iterator [None ]:
146
148
"""Tweak the coverage settings.
147
149
148
150
NOTE: This is not properly factored, and is only used by ToxProject now!!!
@@ -160,6 +162,7 @@ def run_with_coverage(self, env, pip_args, cov_tweaks):
160
162
161
163
class EmptyProject (ProjectToTest ):
162
164
"""A dummy project for testing other parts of this code."""
165
+
163
166
def __init__ (self , slug : str = "empty" , fake_durations : Iterable [float ] = (1.23 ,)):
164
167
self .slug = slug
165
168
self .durations = iter (itertools .cycle (fake_durations ))
@@ -193,9 +196,11 @@ def run_with_coverage(self, env, pip_args, cov_tweaks):
193
196
f".tox/{ env .pyver .toxenv } /bin/python -m pip install { pip_args } "
194
197
)
195
198
with self .tweak_coverage_settings (cov_tweaks ):
196
- self .pre_check (env ) # NOTE: Not properly factored, and only used from here.
199
+ self .pre_check (env ) # NOTE: Not properly factored, and only used from here.
197
200
duration = self .run_tox (env , env .pyver .toxenv , "--skip-pkg-install" )
198
- self .post_check (env ) # NOTE: Not properly factored, and only used from here.
201
+ self .post_check (
202
+ env
203
+ ) # NOTE: Not properly factored, and only used from here.
199
204
return duration
200
205
201
206
@@ -238,7 +243,9 @@ class ProjectAttrs(ToxProject):
238
243
239
244
git_url = "https://github.com/python-attrs/attrs"
240
245
241
- def tweak_coverage_settings (self , tweaks : Iterable [Tuple [str , Any ]]) -> Iterator [None ]:
246
+ def tweak_coverage_settings (
247
+ self , tweaks : Iterable [Tuple [str , Any ]]
248
+ ) -> Iterator [None ]:
242
249
return tweak_toml_coverage_settings ("pyproject.toml" , tweaks )
243
250
244
251
def pre_check (self , env ):
@@ -248,7 +255,9 @@ def post_check(self, env):
248
255
env .shell .run_command ("ls -al" )
249
256
250
257
251
- def tweak_toml_coverage_settings (toml_file : str , tweaks : Iterable [Tuple [str , Any ]]) -> Iterator [None ]:
258
+ def tweak_toml_coverage_settings (
259
+ toml_file : str , tweaks : Iterable [Tuple [str , Any ]]
260
+ ) -> Iterator [None ]:
252
261
if tweaks :
253
262
toml_inserts = []
254
263
for name , value in tweaks :
@@ -265,8 +274,6 @@ def tweak_toml_coverage_settings(toml_file: str, tweaks: Iterable[Tuple[str, Any
265
274
return file_replace (Path (toml_file ), header , insert )
266
275
267
276
268
-
269
-
270
277
class AdHocProject (ProjectToTest ):
271
278
"""A standalone program to run locally."""
272
279
@@ -295,9 +302,7 @@ def run_no_coverage(self, env):
295
302
def run_with_coverage (self , env , pip_args , cov_tweaks ):
296
303
env .shell .run_command (f"{ env .python } -m pip install { pip_args } " )
297
304
with change_dir (self .cur_dir ):
298
- env .shell .run_command (
299
- f"{ env .python } -m coverage run { self .python_file } "
300
- )
305
+ env .shell .run_command (f"{ env .python } -m coverage run { self .python_file } " )
301
306
return env .shell .last_duration
302
307
303
308
@@ -308,13 +313,15 @@ class SlipcoverBenchmark(AdHocProject):
308
313
Clone https://github.com/plasma-umass/slipcover to /src/slipcover
309
314
310
315
"""
316
+
311
317
def __init__ (self , python_file ):
312
318
super ().__init__ (
313
319
python_file = f"/src/slipcover/benchmarks/{ python_file } " ,
314
320
cur_dir = "/src/slipcover" ,
315
321
pip_args = "six pyperf" ,
316
322
)
317
323
324
+
318
325
class PyVersion :
319
326
"""A version of Python to use."""
320
327
@@ -325,22 +332,26 @@ class PyVersion:
325
332
# The tox environment to run this Python
326
333
toxenv : str
327
334
335
+
328
336
class Python (PyVersion ):
329
337
"""A version of CPython to use."""
330
338
331
339
def __init__ (self , major , minor ):
332
340
self .command = self .slug = f"python{ major } .{ minor } "
333
341
self .toxenv = f"py{ major } { minor } "
334
342
343
+
335
344
class PyPy (PyVersion ):
336
345
"""A version of PyPy to use."""
337
346
338
347
def __init__ (self , major , minor ):
339
348
self .command = self .slug = f"pypy{ major } .{ minor } "
340
349
self .toxenv = f"pypy{ major } { minor } "
341
350
351
+
342
352
class AdHocPython (PyVersion ):
343
353
"""A custom build of Python to use."""
354
+
344
355
def __init__ (self , path , slug ):
345
356
self .command = f"{ path } /bin/python3"
346
357
self .slug = slug
@@ -350,33 +361,40 @@ def __init__(self, path, slug):
350
361
@dataclasses .dataclass
351
362
class Coverage :
352
363
"""A version of coverage.py to use, maybe None."""
364
+
353
365
# Short word for messages, directories, etc
354
366
slug : str
355
367
# Arguments for "pip install ..."
356
368
pip_args : Optional [str ] = None
357
369
# Tweaks to the .coveragerc file
358
370
tweaks : Optional [Iterable [Tuple [str , Any ]]] = None
359
371
372
+
360
373
class CoveragePR (Coverage ):
361
374
"""A version of coverage.py from a pull request."""
375
+
362
376
def __init__ (self , number , tweaks = None ):
363
377
super ().__init__ (
364
378
slug = f"#{ number } " ,
365
379
pip_args = f"git+https://github.com/nedbat/coveragepy.git@refs/pull/{ number } /merge" ,
366
380
tweaks = tweaks ,
367
381
)
368
382
383
+
369
384
class CoverageCommit (Coverage ):
370
385
"""A version of coverage.py from a specific commit."""
386
+
371
387
def __init__ (self , sha , tweaks = None ):
372
388
super ().__init__ (
373
389
slug = sha ,
374
390
pip_args = f"git+https://github.com/nedbat/coveragepy.git@{ sha } " ,
375
391
tweaks = tweaks ,
376
392
)
377
393
394
+
378
395
class CoverageSource (Coverage ):
379
396
"""The coverage.py in a working tree."""
397
+
380
398
def __init__ (self , directory , tweaks = None ):
381
399
super ().__init__ (
382
400
slug = "source" ,
@@ -398,6 +416,7 @@ class Env:
398
416
399
417
DIMENSION_NAMES = ["proj" , "pyver" , "cov" ]
400
418
419
+
401
420
class Experiment :
402
421
"""A particular time experiment to run."""
403
422
@@ -415,10 +434,10 @@ def __init__(
415
434
def run (self , num_runs : int = 3 ) -> None :
416
435
results = []
417
436
total_runs = (
418
- len (self .projects ) *
419
- len (self .py_versions ) *
420
- len (self .cov_versions ) *
421
- num_runs
437
+ len (self .projects )
438
+ * len (self .py_versions )
439
+ * len (self .cov_versions )
440
+ * num_runs
422
441
)
423
442
total_run_nums = iter (itertools .count (start = 1 ))
424
443
@@ -444,15 +463,17 @@ def run(self, num_runs: int = 3) -> None:
444
463
for run_num in range (num_runs ):
445
464
total_run_num = next (total_run_nums )
446
465
print (
447
- f"Running tests, cov={ cov_ver .slug } , " +
448
- f"{ run_num + 1 } of { num_runs } , " +
449
- f"total { total_run_num } /{ total_runs } "
466
+ f"Running tests, cov={ cov_ver .slug } , "
467
+ + f"{ run_num + 1 } of { num_runs } , "
468
+ + f"total { total_run_num } /{ total_runs } "
450
469
)
451
470
if cov_ver .pip_args is None :
452
471
dur = proj .run_no_coverage (env )
453
472
else :
454
473
dur = proj .run_with_coverage (
455
- env , cov_ver .pip_args , cov_ver .tweaks ,
474
+ env ,
475
+ cov_ver .pip_args ,
476
+ cov_ver .tweaks ,
456
477
)
457
478
print (f"Tests took { dur :.3f} s" )
458
479
durations .append (dur )
@@ -487,6 +508,7 @@ def show_results(
487
508
remap = [data_order .index (datum ) for datum in DIMENSION_NAMES ]
488
509
489
510
WIDTH = 20
511
+
490
512
def as_table_row (vals ):
491
513
return "| " + " | " .join (v .ljust (WIDTH ) for v in vals ) + " |"
492
514
@@ -506,7 +528,7 @@ def as_table_row(vals):
506
528
for col in dimensions [column ]:
507
529
key = (* tup , col )
508
530
key = tuple (key [i ] for i in remap )
509
- result_time = self .result_data [key ] # type: ignore
531
+ result_time = self .result_data [key ] # type: ignore
510
532
row .append (f"{ result_time :.3f} s" )
511
533
col_data [col ] = result_time
512
534
for _ , num , denom in ratios :
@@ -517,6 +539,7 @@ def as_table_row(vals):
517
539
518
540
PERF_DIR = Path ("/tmp/covperf" )
519
541
542
+
520
543
def run_experiment (
521
544
py_versions : List [PyVersion ],
522
545
cov_versions : List [Coverage ],
@@ -534,21 +557,25 @@ def run_experiment(
534
557
if any (rslug not in slugs for rslug in ratio_slugs ):
535
558
raise Exception (f"Ratio slug doesn't match a slug: { ratio_slugs } , { slugs } " )
536
559
if set (rows + [column ]) != set (DIMENSION_NAMES ):
537
- raise Exception (f"All of these must be in rows or column: { ', ' .join (DIMENSION_NAMES )} " )
560
+ raise Exception (
561
+ f"All of these must be in rows or column: { ', ' .join (DIMENSION_NAMES )} "
562
+ )
538
563
539
564
print (f"Removing and re-making { PERF_DIR } " )
540
565
rmrf (PERF_DIR )
541
566
542
567
with change_dir (PERF_DIR ):
543
- exp = Experiment (py_versions = py_versions , cov_versions = cov_versions , projects = projects )
568
+ exp = Experiment (
569
+ py_versions = py_versions , cov_versions = cov_versions , projects = projects
570
+ )
544
571
exp .run (num_runs = int (sys .argv [1 ]))
545
572
exp .show_results (rows = rows , column = column , ratios = ratios )
546
573
547
574
548
575
if 0 :
549
576
run_experiment (
550
577
py_versions = [
551
- #Python(3, 11),
578
+ # Python(3, 11),
552
579
AdHocPython ("/usr/local/cpython/v3.10.5" , "v3.10.5" ),
553
580
AdHocPython ("/usr/local/cpython/v3.11.0b3" , "v3.11.0b3" ),
554
581
AdHocPython ("/usr/local/cpython/94231" , "94231" ),
@@ -578,14 +605,18 @@ def run_experiment(
578
605
],
579
606
cov_versions = [
580
607
Coverage ("701" , "coverage==7.0.1" ),
581
- Coverage ("701.dynctx" , "coverage==7.0.1" , [("dynamic_context" , "test_function" )]),
608
+ Coverage (
609
+ "701.dynctx" , "coverage==7.0.1" , [("dynamic_context" , "test_function" )]
610
+ ),
582
611
Coverage ("702" , "coverage==7.0.2" ),
583
- Coverage ("702.dynctx" , "coverage==7.0.2" , [("dynamic_context" , "test_function" )]),
612
+ Coverage (
613
+ "702.dynctx" , "coverage==7.0.2" , [("dynamic_context" , "test_function" )]
614
+ ),
584
615
],
585
616
projects = [
586
- #EmptyProject("empty", [1.2, 3.4]),
587
- #EmptyProject("dummy", [6.9, 7.1]),
588
- #ProjectDateutil(),
617
+ # EmptyProject("empty", [1.2, 3.4]),
618
+ # EmptyProject("dummy", [6.9, 7.1]),
619
+ # ProjectDateutil(),
589
620
ProjectAttrs (),
590
621
],
591
622
rows = ["proj" , "pyver" ],
0 commit comments