tools: add more options to track flaky tests

Refs: #43929 (comment) PR-URL: #43954 Reviewed-By: Matteo Collina <[email protected]> Reviewed-By: Tobias Nießen <[email protected]> Reviewed-By: Feng Yu <[email protected]>

Author: aduh95

.github/workflows/build-tarball.yml

@@ -29,7 +29,7 @@ concurrency:
 
 env:
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read
@@ -94,4 +94,4 @@ jobs:
  - name: Test
 run: |
  cd $TAR_DIR
- make run-ci -j2 V=1 TEST_CI_ARGS="-p dots"
+ make run-ci -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 9"

.github/workflows/build-windows.yml

@@ -24,7 +24,7 @@ concurrency:
 
 env:
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read

.github/workflows/coverage-linux.yml

@@ -27,7 +27,7 @@ concurrency:
 
 env:
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read
@@ -53,7 +53,7 @@ jobs:
 # TODO(bcoe): fix the couple tests that fail with the inspector enabled.
 # The cause is most likely coverage's use of the inspector.
  - name: Test
-run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0
+run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots --measure-flakiness 9" || exit 0
  - name: Report JS
 run: npx c8 report --check-coverage
 env:

.github/workflows/coverage-windows.yml

@@ -29,7 +29,7 @@ concurrency:
 
 env:
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read

.github/workflows/doc.yml

@@ -40,4 +40,4 @@ jobs:
 name: docs
 path: out/doc
  - name: Test
-run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions"
+run: NODE=$(command -v node) make test-doc-ci TEST_CI_ARGS="-p actions --measure-flakiness 9"

.github/workflows/test-asan.yml

@@ -31,7 +31,7 @@ concurrency:
 env:
 ASAN_OPTIONS: intercept_tls_get_addr=0
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read
@@ -58,4 +58,4 @@ jobs:
  - name: Build
 run: make build-ci -j2 V=1
  - name: Test
-run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300"
+run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions -t 300 --measure-flakiness 9"

.github/workflows/test-internet.yml

@@ -22,7 +22,7 @@ concurrency:
 
 env:
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read

.github/workflows/test-linux.yml

@@ -24,7 +24,7 @@ concurrency:
 
 env:
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read
@@ -46,4 +46,4 @@ jobs:
  - name: Build
 run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
  - name: Test
-run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
+run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9"

.github/workflows/test-macos.yml

@@ -30,7 +30,7 @@ concurrency:
 
 env:
 PYTHON_VERSION: '3.10'
-FLAKY_TESTS: dontcare
+FLAKY_TESTS: keep_retrying
 
 permissions:
 contents: read
@@ -60,4 +60,4 @@ jobs:
  - name: Build
 run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn"
  - name: Test
-run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions"
+run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions --measure-flakiness 9"

tools/test.py

@@ -96,10 +96,11 @@ def get_module(name, path):
 
 classProgressIndicator(object):
 
-def__init__(self, cases, flaky_tests_mode):
+def__init__(self, cases, flaky_tests_mode, measure_flakiness):
 self.cases=cases
 self.serial_id=0
 self.flaky_tests_mode=flaky_tests_mode
+self.measure_flakiness=measure_flakiness
 self.parallel_queue=Queue(len(cases))
 self.sequential_queue=Queue(len(cases))
 forcaseincases:
@@ -211,10 +212,22 @@ def RunSingle(self, parallel, thread_id):
 ifoutput.UnexpectedOutput():
 ifFLAKYinoutput.test.outcomesandself.flaky_tests_mode==DONTCARE:
 self.flaky_failed.append(output)
+elifFLAKYinoutput.test.outcomesandself.flaky_tests_mode==KEEP_RETRYING:
+for_inrange(99):
+ifnotcase.Run().UnexpectedOutput():
+self.flaky_failed.append(output)
+break
+else:
+# If after 100 tries, the test is not passing, it's not flaky.
+self.failed.append(output)
 else:
 self.failed.append(output)
 ifoutput.HasCrashed():
 self.crashed+=1
+ifself.measure_flakiness:
+outputs= [case.Run() for_inrange(self.measure_flakiness)]
+# +1s are there because the test already failed once at this point.
+print(f" failed {len([iforiinoutputsifi.UnexpectedOutput()]) +1} out of {self.measure_flakiness+1}")
 else:
 self.succeeded+=1
 self.remaining-=1
@@ -436,8 +449,8 @@ def Done(self):
 
 classCompactProgressIndicator(ProgressIndicator):
 
-def__init__(self, cases, flaky_tests_mode, templates):
-super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode)
+def__init__(self, cases, flaky_tests_mode, measure_flakiness, templates):
+super(CompactProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness)
 self.templates=templates
 self.last_status_length=0
 self.start_time=time.time()
@@ -492,29 +505,29 @@ def PrintProgress(self, name):
 
 classColorProgressIndicator(CompactProgressIndicator):
 
-def__init__(self, cases, flaky_tests_mode):
+def__init__(self, cases, flaky_tests_mode, measure_flakiness):
 templates={
 'status_line': "[%(mins)02i:%(secs)02i|\033[34m%%%(remaining) 4d\033[0m|\033[32m+%(passed) 4d\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s",
 'stdout': "\033[1m%s\033[0m",
 'stderr': "\033[31m%s\033[0m",
  }
-super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
+super(ColorProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
 
 defClearLine(self, last_line_length):
 print("\033[1K\r", end='')
 
 
 classMonochromeProgressIndicator(CompactProgressIndicator):
 
-def__init__(self, cases, flaky_tests_mode):
+def__init__(self, cases, flaky_tests_mode, measure_flakiness):
 templates={
 'status_line': "[%(mins)02i:%(secs)02i|%%%(remaining) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s",
 'stdout': '%s',
 'stderr': '%s',
 'clear': lambdalast_line_length: ("\r"+ (" "*last_line_length) +"\r"),
 'max_length': 78
  }
-super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, templates)
+super(MonochromeProgressIndicator, self).__init__(cases, flaky_tests_mode, measure_flakiness, templates)
 
 defClearLine(self, last_line_length):
 print(("\r"+ (" "*last_line_length) +"\r"), end='')
@@ -948,8 +961,8 @@ def GetTimeout(self, mode, section=''):
 timeout=timeout*6
 returntimeout
 
-defRunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
-progress=PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode)
+defRunTestCases(cases_to_run, progress, tasks, flaky_tests_mode, measure_flakiness):
+progress=PROGRESS_INDICATORS[progress](cases_to_run, flaky_tests_mode, measure_flakiness)
 returnprogress.Run(tasks)
 
 # -------------------------------------------
@@ -967,6 +980,7 @@ def RunTestCases(cases_to_run, progress, tasks, flaky_tests_mode):
 SLOW='slow'
 FLAKY='flaky'
 DONTCARE='dontcare'
+KEEP_RETRYING='keep_retrying'
 
 classExpression(object):
 pass
@@ -1355,8 +1369,11 @@ def BuildOptions():
 result.add_option("--cat", help="Print the source of the tests",
 default=False, action="store_true")
 result.add_option("--flaky-tests",
-help="Regard tests marked as flaky (run|skip|dontcare)",
+help="Regard tests marked as flaky (run|skip|dontcare|keep_retrying)",
 default="run")
+result.add_option("--measure-flakiness",
+help="When a test fails, re-run it x number of times",
+default=0, type="int")
 result.add_option("--skip-tests",
 help="Tests that should not be executed (comma-separated)",
 default="")
@@ -1433,7 +1450,7 @@ def ProcessOptions(options):
 # -j and ignoring -J, which is the opposite of what we used to do before -J
 # became a legacy no-op.
 print('Warning: Legacy -J option is ignored. Using the -j option.')
-ifoptions.flaky_testsnotin [RUN, SKIP, DONTCARE]:
+ifoptions.flaky_testsnotin [RUN, SKIP, DONTCARE, KEEP_RETRYING]:
 print("Unknown flaky-tests mode %s"%options.flaky_tests)
 returnFalse
 returnTrue
@@ -1733,7 +1750,7 @@ def should_keep(case):
 else:
 try:
 start=time.time()
-ifRunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests):
+ifRunTestCases(cases_to_run, options.progress, options.j, options.flaky_tests, options.measure_flakiness):
 result=0
 else:
 result=1