test_runner: add support for --test-coverage-include-all

doc/api/cli.md

@@ -2857,6 +2857,21 @@ This option may be specified multiple times to include multiple glob patterns.
 If both `--test-coverage-exclude` and `--test-coverage-include` are provided,
 files must meet **both** criteria to be included in the coverage report.
 
+### `--test-coverage-include-all`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Includes source files that were never loaded by the test run in the coverage
+report, where they are reported as having zero coverage.
+
+Candidate files are searched for in the current working directory, and are
+subject to the same `--test-coverage-include` and `--test-coverage-exclude`
+filtering as the rest of the report.
+
 ### `--test-coverage-lines=threshold`
 
 <!-- YAML
@@ -3900,6 +3915,7 @@ one is included in the list below.
 * `--test-coverage-branches`
 * `--test-coverage-exclude`
 * `--test-coverage-functions`
+* `--test-coverage-include-all`
 * `--test-coverage-include`
 * `--test-coverage-lines`
 * `--test-global-setup`

doc/api/test.md

@@ -1782,6 +1782,13 @@ changes:
     If both `coverageExcludeGlobs` and `coverageIncludeGlobs` are provided,
     files must meet **both** criteria to be included in the coverage report.
     **Default:** `undefined`.
+  * `coverageIncludeAll` {boolean} Includes source files that were never loaded by
+    the test run in the coverage report, where they are reported as having zero
+    coverage. Candidate files are searched for in `cwd`, and are subject to the
+    same `coverageIncludeGlobs` and `coverageExcludeGlobs` filtering as the rest
+    of the report. This property is only applicable when `coverage` was set to
+    `true`.
+    **Default:** `false`.
   * `lineCoverage` {number} Require a minimum percent of covered lines. If code
     coverage does not reach the threshold specified, the process will exit with code `1`.
     **Default:** `0`.

doc/node.1

@@ -1400,6 +1400,13 @@ This option may be specified multiple times to include multiple glob patterns.
 If both \fB--test-coverage-exclude\fR and \fB--test-coverage-include\fR are provided,
 files must meet \fBboth\fR criteria to be included in the coverage report.
 .
+.It Fl -test-coverage-include-all
+Includes source files that were never loaded by the test run in the coverage
+report, where they are reported as having zero coverage.
+Candidate files are searched for in the current working directory, and are
+subject to the same \fB--test-coverage-include\fR and \fB--test-coverage-exclude\fR
+filtering as the rest of the report.
+.
 .It Fl -test-coverage-lines Ns = Ns Ar threshold
 Require a minimum percent of covered lines. If code coverage does not reach
 the threshold specified, the process will exit with code \fB1\fR.
@@ -2125,6 +2132,8 @@ one is included in the list below.
 .It
 \fB--test-coverage-functions\fR
 .It
+\fB--test-coverage-include-all\fR
+.It
 \fB--test-coverage-include\fR
 .It
 \fB--test-coverage-lines\fR

lib/internal/test_runner/coverage.js

@@ -20,6 +20,7 @@ const {
 } = primordials;
 const {
   copyFileSync,
+  globSync,
   mkdirSync,
   mkdtempSync,
   opendirSync,
@@ -29,7 +30,7 @@ const {
 const { setupCoverageHooks } = require('internal/util');
 const { tmpdir } = require('os');
 const { join, resolve, relative } = require('path');
-const { fileURLToPath, URL } = require('internal/url');
+const { fileURLToPath, pathToFileURL, URL } = require('internal/url');
 const { kMappings, SourceMap } = require('internal/source_map/source_map');
 const {
   codes: {
@@ -48,6 +49,7 @@ const kLineSplitRegex = /(?<=\r?\n)/u;
 const kStatusRegex = /\/\* node:coverage (?<status>enable|disable) \*\//;
 const kTypeOnlyImportRegex = /^\s*import\s+type\b/u;
 const kTypeScriptSourceRegex = /\.(?:cts|mts|ts)$/u;
+const kSourceFileGlob = '**/*.{cjs,cts,js,mjs,mts,ts}';
 
 let stripTypeScriptTypesForCoverage;
 
@@ -407,6 +409,10 @@ class TestCoverage {
         this.mergeCoverage(result, this.mapCoverageWithSourceMap(coverage));
       }
 
+      if (this.options.coverageIncludeAll) {
+        this.#addUntestedFileCoverage(result);
+      }
+
       return ArrayFrom(result.values());
     } finally {
       if (dir) {
@@ -415,6 +421,48 @@ class TestCoverage {
     }
   }
 
+  #addUntestedFileCoverage(merged) {
+    const files = globSync(kSourceFileGlob, {
+      __proto__: null,
+      cwd: this.options.cwd,
+      // Skip node_modules/, since `shouldSkipFileCoverage` would skip it anyway
+      exclude: (name) => name === 'node_modules',
+    });
+
+    for (let i = 0; i < files.length; ++i) {
+      const url = pathToFileURL(resolve(this.options.cwd, files[i])).href;
+
+      if (merged.has(url) || this.shouldSkipFileCoverage(url)) {
+        continue;
+      }
+
+      this.markTypeScriptOnlyLines(url);
+      const lines = this.getLines(url);
+
+      if (!lines || lines.length === 0) {
+        continue;
+      }
+
+      const lastLine = lines[lines.length - 1];
+
+      merged.set(url, {
+        __proto__: null,
+        url,
+        functions: [{
+          __proto__: null,
+          functionName: '',
+          isBlockCoverage: false,
+          ranges: [{
+            __proto__: null,
+            startOffset: 0,
+            endOffset: lastLine.startOffset + lastLine.src.length,
+            count: 0,
+          }],
+        }],
+      });
+    }
+  }
+
 
   mapCoverageWithSourceMap(coverage) {
     const { result } = coverage;

lib/internal/test_runner/runner.js

@@ -723,6 +723,7 @@ function run(options = kEmptyObject) {
     only,
     globPatterns,
     coverage = false,
+    coverageIncludeAll = false,
     lineCoverage = 0,
     branchCoverage = 0,
     functionCoverage = 0,
@@ -882,6 +883,7 @@ function run(options = kEmptyObject) {
 
   validateOneOf(isolation, 'options.isolation', ['process', 'none']);
   validateBoolean(coverage, 'options.coverage');
+  validateBoolean(coverageIncludeAll, 'options.coverageIncludeAll');
   if (coverageExcludeGlobs != null) {
     if (!ArrayIsArray(coverageExcludeGlobs)) {
       coverageExcludeGlobs = [coverageExcludeGlobs];
@@ -923,6 +925,7 @@ function run(options = kEmptyObject) {
     ...parseCommandLine(),
     setup,  // This line can be removed when parseCommandLine() is removed here.
     coverage,
+    coverageIncludeAll,
     coverageExcludeGlobs,
     coverageIncludeGlobs,
     rerunFailuresFilePath,

lib/internal/test_runner/utils.js

@@ -245,6 +245,7 @@ function parseCommandLine() {
 
   const isTestRunner = getOptionValue('--test');
   const coverage = getOptionValue('--experimental-test-coverage');
+  const coverageIncludeAll = getOptionValue('--test-coverage-include-all');
   const forceExit = getOptionValue('--test-force-exit');
   const sourceMaps = getOptionValue('--enable-source-maps');
   const updateSnapshots = getOptionValue('--test-update-snapshots');
@@ -415,6 +416,7 @@ function parseCommandLine() {
     isTestRunner,
     concurrency,
     coverage,
+    coverageIncludeAll,
     coverageExcludeGlobs,
     coverageIncludeGlobs,
     destinations,

src/node_options.cc

@@ -1029,6 +1029,13 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             &EnvironmentOptions::coverage_include_pattern,
             kAllowedInEnvvar,
             OptionNamespaces::kTestRunnerNamespace);
+  AddOption("--test-coverage-include-all",
+            "include source files that were never loaded in the coverage "
+            "report",
+            &EnvironmentOptions::coverage_include_all,
+            kAllowedInEnvvar,
+            false,
+            OptionNamespaces::kTestRunnerNamespace);
   AddOption("--test-coverage-exclude",
             "exclude files from coverage report that match this glob pattern",
             &EnvironmentOptions::coverage_exclude_pattern,

src/node_options.h

@@ -221,6 +221,7 @@ class EnvironmentOptions : public Options {
   std::vector<std::string> test_skip_pattern;
   std::vector<std::string> experimental_test_tag_filter;
   std::vector<std::string> coverage_include_pattern;
+  bool coverage_include_all = false;
   std::vector<std::string> coverage_exclude_pattern;
   bool throw_deprecation = false;
   bool trace_deprecation = false;

test/fixtures/test-runner/coverage-include-all/covered.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function covered() {
+  return 'covered';
+};

test/fixtures/test-runner/coverage-include-all/data.json

@@ -0,0 +1,3 @@
+{
+  "not": "a source file"
+}

test/fixtures/test-runner/coverage-include-all/index.test.js

@@ -0,0 +1,9 @@
+'use strict';
+
+const assert = require('node:assert');
+const test = require('node:test');
+const covered = require('./covered');
+
+test('covered source is executed', () => {
+  assert.strictEqual(covered(), 'covered');
+});

test/fixtures/test-runner/coverage-include-all/nested/deep.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function deep() {
+  return 'deep';
+};

test/fixtures/test-runner/coverage-include-all/untested.js

@@ -0,0 +1,5 @@
+'use strict';
+
+module.exports = function untested() {
+  return 'untested';
+};

test/parallel/test-runner-coverage.js

@@ -528,6 +528,76 @@ test('coverage with included and excluded files', skipIfNoInspector, () => {
   assert(!findCoverageFileForPid(result.pid));
 });
 
+test('coverage does not include untested files by default', skipIfNoInspector, () => {
+  const fixtureDir = fixtures.path('test-runner', 'coverage-include-all');
+  const fixture = fixtures.path('test-runner', 'coverage-include-all', 'index.test.js');
+  const args = [
+    '--test', '--experimental-test-coverage', '--test-coverage-exclude=**/*.test.js',
+    '--test-reporter', 'tap', fixture,
+  ];
+  const result = spawnSync(process.execPath, args, { cwd: fixtureDir });
+  const report = [
+    '# start of coverage report',
+    '# -----------------------------------------------------------',
+    '# file       | line % | branch % | funcs % | uncovered lines',
+    '# -----------------------------------------------------------',
+    '# covered.js | 100.00 |   100.00 |  100.00 | ',
+    '# -----------------------------------------------------------',
+    '# all files  | 100.00 |   100.00 |  100.00 | ',
+    '# -----------------------------------------------------------',
+    '# end of coverage report',
+  ].join('\n');
+
+  assertIncludesReport(result, report);
+  assert.strictEqual(result.status, 0);
+  assert(!findCoverageFileForPid(result.pid));
+});
+
+test('coverage includes untested files', skipIfNoInspector, () => {
+  const fixtureDir = fixtures.path('test-runner', 'coverage-include-all');
+  const fixture = fixtures.path('test-runner', 'coverage-include-all', 'index.test.js');
+  const args = [
+    '--test', '--experimental-test-coverage', '--test-coverage-include-all',
+    '--test-coverage-exclude=**/*.test.js', '--test-reporter', 'tap', fixture,
+  ];
+  const result = spawnSync(process.execPath, args, { cwd: fixtureDir });
+
+  const report = [
+    '# start of coverage report',
+    '# -------------------------------------------------------------',
+    '# file         | line % | branch % | funcs % | uncovered lines',
+    '# -------------------------------------------------------------',
+    '# covered.js   | 100.00 |   100.00 |  100.00 | ',
+    '# nested       |        |          |         | ',
+    '#  deep.js     |   0.00 |   100.00 |  100.00 | 1-5',
+    '# untested.js  |   0.00 |   100.00 |  100.00 | 1-5',
+    '# -------------------------------------------------------------',
+    '# all files    |  33.33 |   100.00 |  100.00 | ',
+    '# -------------------------------------------------------------',
+    '# end of coverage report',
+  ].join('\n');
+
+  assertIncludesReport(result, report);
+  assert.strictEqual(result.status, 0);
+  assert(!findCoverageFileForPid(result.pid));
+});
+
+test('untested files honor coverage include and exclude globs', skipIfNoInspector, () => {
+  const fixtureDir = fixtures.path('test-runner', 'coverage-include-all');
+  const fixture = fixtures.path('test-runner', 'coverage-include-all', 'index.test.js');
+  const args = [
+    '--test', '--experimental-test-coverage', '--test-coverage-include-all',
+    '--test-coverage-exclude=nested/**', '--test-reporter', 'tap', fixture,
+  ];
+  const result = spawnSync(process.execPath, args, { cwd: fixtureDir });
+  const stdout = result.stdout.toString();
+
+  assert.strictEqual(result.status, 0);
+  assert.match(stdout, /^# untested\.js/m);
+  assert.doesNotMatch(stdout, /deep\.js/);
+  assert(!findCoverageFileForPid(result.pid));
+});
+
 test('correctly prints the coverage report of files contained in parent directories', skipIfNoInspector, () => {
   let report = [
     '# start of coverage report',