gh-98903: Test suite fails with exit code 4 if no tests ran

Lib/test/libregrtest/main.py

@@ -28,6 +28,11 @@
 # Must be smaller than buildbot "1200 seconds without output" limit.
 EXIT_TIMEOUT = 120.0
 
+EXITCODE_BAD_TEST = 2
+EXITCODE_INTERRUPTED = 130
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+
 
 class Regrtest:
  """Execute a test suite.
@@ -493,15 +498,18 @@ def display_header(self):
  print("== encodings: locale=%s, FS=%s"
  % (locale.getencoding(), sys.getfilesystemencoding()))
 
+ def no_tests_run(self):
+ return not any((self.good, self.bad, self.skipped, self.interrupted,
+ self.environment_changed))
+
  def get_tests_result(self):
  result = []
  if self.bad:
  result.append("FAILURE")
  elif self.ns.fail_env_changed and self.environment_changed:
  result.append("ENV CHANGED")
- elif not any((self.good, self.bad, self.skipped, self.interrupted,
- self.environment_changed)):
- result.append("NO TEST RUN")
+ elif self.no_tests_run():
+ result.append("NO TESTS RAN")
 
  if self.interrupted:
  result.append("INTERRUPTED")
@@ -750,11 +758,13 @@ def _main(self, tests, kwargs):
  self.save_xml_result()
 
  if self.bad:
- sys.exit(2)
+ sys.exit(EXITCODE_BAD_TEST)
  if self.interrupted:
- sys.exit(130)
+ sys.exit(EXITCODE_INTERRUPTED)
  if self.ns.fail_env_changed and self.environment_changed:
- sys.exit(3)
+ sys.exit(EXITCODE_ENV_CHANGED)
+ if self.no_tests_run():
+ sys.exit(EXITCODE_NO_TESTS_RAN)
  sys.exit(0)
 
 

Lib/test/test_regrtest.py

@@ -28,6 +28,11 @@
 ROOT_DIR = os.path.abspath(os.path.normpath(ROOT_DIR))
 LOG_PREFIX = r'[0-9]+:[0-9]+:[0-9]+ (?:load avg: [0-9]+\.[0-9]{2} )?'
 
+EXITCODE_BAD_TEST = 2
+EXITCODE_ENV_CHANGED = 3
+EXITCODE_NO_TESTS_RAN = 4
+EXITCODE_INTERRUPTED = 130
+
 TEST_INTERRUPTED = textwrap.dedent("""
  from signal import SIGINT, raise_signal
  try:
@@ -497,7 +502,7 @@ def list_regex(line_format, tests):
  result.append('INTERRUPTED')
  if not any((good, result, failed, interrupted, skipped,
  env_changed, fail_env_changed)):
- result.append("NO TEST RUN")
+ result.append("NO TESTS RAN")
  elif not result:
  result.append('SUCCESS')
  result = ', '.join(result)
@@ -707,7 +712,7 @@ def test_failing(self):
  test_failing = self.create_test('failing', code=code)
  tests = [test_ok, test_failing]
 
- output = self.run_tests(*tests, exitcode=2)
+ output = self.run_tests(*tests, exitcode=EXITCODE_BAD_TEST)
  self.check_executed_tests(output, tests, failed=test_failing)
 
  def test_resources(self):
@@ -748,13 +753,14 @@ def test_random(self):
  test = self.create_test('random', code)
 
  # first run to get the output with the random seed
- output = self.run_tests('-r', test)
+ output = self.run_tests('-r', test, exitcode=EXITCODE_NO_TESTS_RAN)
  randseed = self.parse_random_seed(output)
  match = self.regex_search(r'TESTRANDOM: ([0-9]+)', output)
  test_random = int(match.group(1))
 
  # try to reproduce with the random seed
- output = self.run_tests('-r', '--randseed=%s' % randseed, test)
+ output = self.run_tests('-r', '--randseed=%s' % randseed, test,
+ exitcode=EXITCODE_NO_TESTS_RAN)
  randseed2 = self.parse_random_seed(output)
  self.assertEqual(randseed2, randseed)
 
@@ -813,7 +819,7 @@ def test_fromfile(self):
  def test_interrupted(self):
  code = TEST_INTERRUPTED
  test = self.create_test('sigint', code=code)
- output = self.run_tests(test, exitcode=130)
+ output = self.run_tests(test, exitcode=EXITCODE_INTERRUPTED)
  self.check_executed_tests(output, test, omitted=test,
  interrupted=True)
 
@@ -838,7 +844,7 @@ def test_slowest_interrupted(self):
  args = ("--slowest", "-j2", test)
  else:
  args = ("--slowest", test)
- output = self.run_tests(*args, exitcode=130)
+ output = self.run_tests(*args, exitcode=EXITCODE_INTERRUPTED)
  self.check_executed_tests(output, test,
  omitted=test, interrupted=True)
 
@@ -878,7 +884,7 @@ def test_run(self):
  builtins.__dict__['RUN'] = 1
  """)
  test = self.create_test('forever', code=code)
- output = self.run_tests('--forever', test, exitcode=2)
+ output = self.run_tests('--forever', test, exitcode=EXITCODE_BAD_TEST)
  self.check_executed_tests(output, [test]*3, failed=test)
 
  def check_leak(self, code, what):
@@ -887,7 +893,7 @@ def check_leak(self, code, what):
  filename = 'reflog.txt'
  self.addCleanup(os_helper.unlink, filename)
  output = self.run_tests('--huntrleaks', '3:3:', test,
- exitcode=2,
+ exitcode=EXITCODE_BAD_TEST,
  stderr=subprocess.STDOUT)
  self.check_executed_tests(output, [test], failed=test)
 
@@ -969,7 +975,7 @@ def test_crashed(self):
  crash_test = self.create_test(name="crash", code=code)
 
  tests = [crash_test]
- output = self.run_tests("-j2", *tests, exitcode=2)
+ output = self.run_tests("-j2", *tests, exitcode=EXITCODE_BAD_TEST)
  self.check_executed_tests(output, tests, failed=crash_test,
  randomize=True)
 
@@ -1069,7 +1075,8 @@ def test_env_changed(self):
  self.check_executed_tests(output, [testname], env_changed=testname)
 
  # fail with --fail-env-changed
- output = self.run_tests("--fail-env-changed", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
  self.check_executed_tests(output, [testname], env_changed=testname,
  fail_env_changed=True)
 
@@ -1088,7 +1095,7 @@ def test_fail_always(self):
  """)
  testname = self.create_test(code=code)
 
- output = self.run_tests("-w", testname, exitcode=2)
+ output = self.run_tests("-w", testname, exitcode=EXITCODE_BAD_TEST)
  self.check_executed_tests(output, [testname],
  failed=testname, rerun={testname: "test_fail_always"})
 
@@ -1123,7 +1130,8 @@ def test_bug(self):
  """)
  testname = self.create_test(code=code)
 
- output = self.run_tests(testname, "-m", "nosuchtest", exitcode=0)
+ output = self.run_tests(testname, "-m", "nosuchtest",
+ exitcode=EXITCODE_NO_TESTS_RAN)
  self.check_executed_tests(output, [testname], no_test_ran=testname)
 
  def test_no_tests_ran_skip(self):
@@ -1136,7 +1144,7 @@ def test_skipped(self):
  """)
  testname = self.create_test(code=code)
 
- output = self.run_tests(testname, exitcode=0)
+ output = self.run_tests(testname)
  self.check_executed_tests(output, [testname])
 
  def test_no_tests_ran_multiple_tests_nonexistent(self):
@@ -1150,7 +1158,8 @@ def test_bug(self):
  testname = self.create_test(code=code)
  testname2 = self.create_test(code=code)
 
- output = self.run_tests(testname, testname2, "-m", "nosuchtest", exitcode=0)
+ output = self.run_tests(testname, testname2, "-m", "nosuchtest",
+ exitcode=EXITCODE_NO_TESTS_RAN)
  self.check_executed_tests(output, [testname, testname2],
  no_test_ran=[testname, testname2])
 
@@ -1198,7 +1207,8 @@ def test_garbage(self):
  """)
  testname = self.create_test(code=code)
 
- output = self.run_tests("--fail-env-changed", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
  self.check_executed_tests(output, [testname],
  env_changed=[testname],
  fail_env_changed=True)
@@ -1224,7 +1234,8 @@ def test_sleep(self):
  """)
  testname = self.create_test(code=code)
 
- output = self.run_tests("-j2", "--timeout=1.0", testname, exitcode=2)
+ output = self.run_tests("-j2", "--timeout=1.0", testname,
+ exitcode=EXITCODE_BAD_TEST)
  self.check_executed_tests(output, [testname],
  failed=testname)
  self.assertRegex(output,
@@ -1256,7 +1267,8 @@ def test_unraisable_exc(self):
  """)
  testname = self.create_test(code=code)
 
- output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", "-v", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
  self.check_executed_tests(output, [testname],
  env_changed=[testname],
  fail_env_changed=True)
@@ -1287,7 +1299,8 @@ def test_threading_excepthook(self):
  """)
  testname = self.create_test(code=code)
 
- output = self.run_tests("--fail-env-changed", "-v", testname, exitcode=3)
+ output = self.run_tests("--fail-env-changed", "-v", testname,
+ exitcode=EXITCODE_ENV_CHANGED)
  self.check_executed_tests(output, [testname],
  env_changed=[testname],
  fail_env_changed=True)
@@ -1328,7 +1341,7 @@ def test_print_warning(self):
  for option in ("-v", "-W"):
  with self.subTest(option=option):
  cmd = ["--fail-env-changed", option, testname]
- output = self.run_tests(*cmd, exitcode=3)
+ output = self.run_tests(*cmd, exitcode=EXITCODE_ENV_CHANGED)
  self.check_executed_tests(output, [testname],
  env_changed=[testname],
  fail_env_changed=True)
@@ -1373,7 +1386,8 @@ def test_leak_tmp_file(self):
  """)
  testnames = [self.create_test(code=code) for _ in range(3)]
 
- output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames, exitcode=3)
+ output = self.run_tests("--fail-env-changed", "-v", "-j2", *testnames,
+ exitcode=EXITCODE_ENV_CHANGED)
  self.check_executed_tests(output, testnames,
  env_changed=testnames,
  fail_env_changed=True,

Misc/NEWS.d/next/Tests/2022-10-31-14-47-49.gh-issue-98903.7KinCV.rst

@@ -0,0 +1,2 @@
+The Python test suite now fails wit exit code 4 if no tests ran. It should
+help detecting typos in test names and test methods.