.gitignore

@@ -1 +1,2 @@
 node_modules/
+test/**/*.js.node.{default|strict}.{fail,pass}

.npmignore

@@ -0,0 +1,2 @@
+node_modules/
+test/

.travis.yml

@@ -0,0 +1,9 @@
+os:
+  - osx
+  - linux
+language: node_js
+node_js:
+  - 10
+  - 11
+install: npm install
+script: npm test

README.md

@@ -1,26 +1,87 @@
-## Test262-Harness
-An experimental Node-based test262 harness. Once this harness has stabilized, I plan to push to include it by default in official test262.
+# Test262-Harness
 
-Requires Node 6 or above.
+[![Travis Build Status](https://travis-ci.org/bterlson/test262-harness.svg?branch=master)](https://travis-ci.org/bterlson/test262-harness)
 
-## Quick Start
-1. `git clone https://github.com/tc39/test262.git --depth 1`
-2. `npm install -g test262-harness`
-3. `test262-harness 'test262\test\**\*.js'`
+## Getting Started
+
+### For Development of Test262-Harness
+
+For development of Test262-harness, test authoring & execution:
+
+```
+git clone https://github.com/tc39/test262.git --depth 1
+cd test262
+```
+
+### For Test Authoring & Test Execution: 
+
+For test authoring & execution:
+```
+npm install -g test262-harness
+```
 
 Run `test262-harness --help` for details on the various configuration options.
 
+### Examples
+
+Running tests in hypothetical JavaScript engine "X": 
+
+```
+cd test262;
+test262-harness --host-type=X --host-path=`which X` test/**/*.js
+```
+
+
+
+
 ## Options
 
-| Name    | Action      |
-|------------|---------------|
-| --hostType | Type of host to run tests in. See [eshost's supported hosts](https://github.com/bterlson/eshost#supported-hosts) for available options.
-| --hostPath | Path to the host executable.
-| --hostArgs | Any additional arguments to pass to the host when invoking it (eg. --harmony, --es6all, etc). 
-| -t, --threads | Run this many tests in parallel. Note that the browser runners don't work great with t > 1.
-| -r, --reporter | Selects test case result format. Currently either `json` or `simple`. Default `simple`.
-|--test262Dir | Optional. Root test262 directory and is used to locate the includes directory.
-|--includesDir | Includes directory. By default inferred from test262Dir or else detected by walking upward from the first test found.
-|--prelude | Path to a file to include before every test (useful for testing polyfills for example)
+| Option Name | Description | Required | Default |
+| -- | -- | -- | -- |
+| `-h`, `--help` | Show help & examples | n/a | n/a |
+| `-v`, `--version` | Print the current version of test262-harness | n/a | n/a |
+| `--host-type` | Type of host to run tests in. See [eshost's supported hosts](https://github.com/bterlson/eshost#supported-hosts) for available options. | No | `node`
+| `--host-path` | Path to the host executable. | Yes, if `host-type` is specified | `process.execPath`
+| `--host-args` | Any additional arguments to pass to the host when invoking it (eg. `--harmony`, `--es6all`, etc). | No | n/a |
+| `-t`, `--threads` | Run this many tests in parallel. Note that the browser runners don't work great with t > 1. | No | 1 |
+| `-r`, `--reporter` | Format of data written to standard output. Currently either `json` or `simple`. | No |  `simple` |
+| `--features` (deprecated) | Comma-separated list of [`features`](https://github.com/tc39/test262/blob/master/features.txt) to filter for. Example: `--features="BigInt,Atomics"`. | No | n/a |
+| `--features-include` | Comma-separated list of [`features`](https://github.com/tc39/test262/blob/master/features.txt) to filter for _inclusion_. Example: `--features-include="BigInt,Atomics"` would run only tests that include the features `BigInt` and `Atomics` | No | n/a |
+| `--features-exclude` | Comma-separated list of [`features`](https://github.com/tc39/test262/blob/master/features.txt) to filter for _exclusion_. Example: `--features-exclude="BigInt,Atomics"` would run only tests that do not include the features `BigInt` and `Atomics`. **NOTE** exclusions are resolved _after_ inclusions. | No | n/a |
+| `--reporter-keys` | Comma-separated list of keys to include in output of `json` reporter. | No | n/a |
+| `--test262-dir` | Root test262 directory and is used to locate the includes directory. | No | Relative to test files |
+| `--includes-dir` | Includes directory. | No | Inferred from `test262-dir` or else detected by walking upward from the first test found. |
+| `--tempDir` | Directory that `eshost` will create its temp files in (does not affect location of files created by `--save-compiled-tests` and `--save-only-failed` | No | OS Temp Dir |
+| `--prelude` | Path to a file to include before every test (useful for testing polyfills for example); supports multiple `--prelude` parameters | No | n/a |
+| `--timeout` | Set a custom test timeout in milliseconds | No | `10000` |
+| `--transformer` | Path to module which exports a code transformer function | No | n/a |
+| `--preprocessor` | Path to module which exports a map function that operates on each Test262Test object before it executed. | No | n/a |
+| `--accept-version` | Execute tests from a version of Test262 that differs from the versions supported by this utility. This may cause the utility to report invalid test results. | No | Inferred from `test262Dir/package.json` |
+| `--save-compiled-tests` | Write the compiled version of `path/to/test.js` as `path/to/test.js.<host-type>.<default\|strict>.<pass\|fail>` so that it can be easily re-run under that host. Run `test262-harness --help` for examples. | No | n/a
+| `--save-only-failed` | Only save the compiled version of the test if it failed, to help easily repro failed tests (implies `--save-compiled-tests`). | No | n/a
+| `--error-for-failures` | Return a non-zero exit code if one or more tests fail. | No | n/a
+
+
+### Preprocessor
+
+The `--preprocessor` feature allows  a module that exports a map function that operates on each [Test262Test object](https://github.com/bocoup/test262-stream#usage) (ie. the object emitted by Test262-Stream) before their execution.
+
+#### `test.result = Result Object`
+
+In some cases, a preprocessor may want to signal to Test262-Harness that a certain result has already been reached, and that it must not further evaluate the test. To create this signal, the preprocessor creates a [`result`](https://github.com/bterlson/eshost#result-object) property on the Test262Test object, which will indicate to Test262-Harness that it must not evaluate the test, but instead return the value of the [`result`](https://github.com/bterlson/eshost#result-object) property as though the test had been executed. For example, a preprocessor may attempt to transpile the value of `test.contents`—which may fail! In the case of failure, the preprocessor can create a [`result`](https://github.com/bterlson/eshost#result-object) property whose value is a [`Result Object`](https://github.com/bterlson/eshost#result-object). This will skip the code evaluation and report the given result object. 
 
+```js
+module.exports = function(test) {
+  try {
+    test.contents = babel.transform(test.contents, options).code;
+  } catch (error) {
+    test.result = {
+      stderr: `${error.name}: ${error.message}\n`,
+      stdout: '',
+      error
+    };
+  }
 
+  return test;
+};
+```

bin/run.js

@@ -2,51 +2,89 @@
 
 // Copyright (C) 2014, Microsoft Corporation. All rights reserved.
 // This code is governed by the BSD License found in the LICENSE file.
-const compile = require('test262-compiler');
+const DEFAULT_TEST_TIMEOUT = 10000;
+
 const fs = require('fs');
-const Path = require('path');
-const globber = require('../lib/globber.js');
-const argv = require('../lib/cli.js').argv;
-const validator = require('../lib/validator.js');
-const Rx = require('rx');
+const path = require('path');
 const util = require('util');
-const resultsEmitter = require('../lib/resultsEmitter.js');
-const agentPool = require('../lib/agentPool.js');
+
+const { zip } = require('rxjs');
+const { flatMap, filter, map } = require('rxjs/operators');
+
+const AgentPool = require('../lib/agent-pool.js');
+const TestStream = require('../lib/test-stream');
+const ResultsEmitter = require('../lib/results-emitter.js');
+const cli = require('../lib/cli.js');
 const test262Finder = require('../lib/findTest262.js');
-const scenariosForTest = require('../lib/scenarios.js');
+const validator = require('../lib/validator.js');
+
+const argv = cli.argv;
 
 // test262 directory (used to locate includes unless overridden with includesDir)
 let test262Dir = argv.test262Dir;
 // where to load includes from (usually a subdirectory of test262dir)
 let includesDir = argv.includesDir;
 
+let tempDir = argv.tempDir;
+let acceptVersion = argv.acceptVersion;
+
+// print version of test262-harness
+if (argv.version) {
+  printVersion();
+  return;
+}
+
 // initialize reporter by attempting to load lib/reporters/${reporter}
 // defaults to 'simple'
 let reporter;
-if (fs.existsSync(Path.join(__dirname, '../lib/reporters', argv.reporter + '.js'))) {
-  reporter = require('../lib/reporters/' + argv.reporter + '.js');
+let reporterOpts = {};
+if (fs.existsSync(path.join(__dirname, '../lib/reporters', `${argv.reporter}.js`))) {
+  reporter = require(`../lib/reporters/${argv.reporter}.js`);
 } else {
   console.error(`Reporter ${argv.reporter} not found.`);
-  process.exit(1);
+  process.exitCode = 1;
+  return;
+}
+
+if (argv.reporterKeys) {
+  reporterOpts.reporterKeys = argv.reporterKeys.split(',');
+}
+
+// Using argv.saveOnlyFailed implies argv.saveCompiledTests
+if (argv.saveOnlyFailed && !argv.saveCompiledTests) {
+  argv.saveCompiledTests = true;
+}
+
+if (argv.saveCompiledTests) {
+  reporterOpts.saveCompiledTests = argv.saveCompiledTests;
+  if (argv.saveOnlyFailed) {
+    reporterOpts.saveOnlyFailed = argv.saveOnlyFailed;
+  }
 }
 
 // load preload contents
-let preludeContents;
+let preludeContents = '';
 if (argv.prelude) {
-  preludeContents = fs.readFileSync(argv.prelude, 'utf8');
-} else {
-  preludeContents = '';
+  if (!Array.isArray(argv.prelude)) {
+    argv.prelude = [argv.prelude];
+  }
+  preludeContents = argv.prelude.map(prelude => fs.readFileSync(prelude, 'utf8')).join('\n');
 }
 
 // Select hostType and hostPath. hostType defaults to 'node'.
 // If using default hostType, hostPath defaults to the current node executable location.
-let hostType, hostPath;
+let hostType;
+let hostPath;
+let featuresInclude;
+let featuresExclude;
+
 if (argv.hostType) {
   hostType = argv.hostType;
 
   if (!argv.hostPath) {
     console.error('Missing host path. Pass --hostPath with a path to the host executable you want to test.');
-    process.exit(1);
+    process.exitCode = 1;
+    return;
   }
 
   hostPath = argv.hostPath;
@@ -60,42 +98,131 @@ if (argv.hostType) {
   }
 }
 
+if (hostType) {
+  reporterOpts.hostType = hostType;
+}
+
+let timeout = argv.timeout || DEFAULT_TEST_TIMEOUT;
+let transform, transformer;
+let preprocessor;
+
+if (argv.transformer) {
+  const transformerPath = path.isAbsolute(argv.transformer) ?
+    argv.transformer : path.join(process.cwd(), argv.transformer)
+
+  transform = transformer = require(transformerPath);
+}
+
+if (argv.preprocessor) {
+  const preprocessorPath = path.isAbsolute(argv.preprocessor) ?
+    argv.preprocessor : path.join(process.cwd(), argv.preprocessor)
+
+  preprocessor = require(preprocessorPath);
+}
+
+if (argv.features || argv.featuresInclude) {
+  featuresInclude = (argv.features || argv.featuresInclude).split(',').map(feature => feature.trim());
+}
+
+if (argv.featuresExclude) {
+  featuresExclude = argv.featuresExclude.split(',').map(feature => feature.trim());
+}
+
+// Show help if no arguments provided
+if (!argv._.length) {
+  cli.showHelp();
+  process.exitCode = 1;
+  return;
+}
+
 // Test Pipeline
-const pool = agentPool(Number(argv.threads), hostType, argv.hostArgs, hostPath);
-const paths = globber(argv._);
-if (!includesDir && !test262Dir) {
-  test262Dir = test262Finder(paths.fileEvents[0]);
-}
-const files = paths.map(pathToTestFile);
-const tests = files.map(compileFile);
-const scenarios = tests.flatMap(scenariosForTest);
-const pairs = Rx.Observable.zip(pool, scenarios);
-const rawResults = pairs.flatMap(pool.runTest).tapOnCompleted(() => pool.destroy());;
-const results = rawResults.map(function (test) {
-  test.result = validator(test);
-  return test;
-});
-const resultEmitter = resultsEmitter(results);
-reporter(resultEmitter);
+const pool = new AgentPool(
+  Number(argv.threads), hostType, argv.hostArgs, hostPath, { tempDir, timeout, transform }
+);
+
+if (!test262Dir) {
+  test262Dir = test262Finder(argv._[0]);
+}
+reporterOpts.test262Dir = test262Dir;
+
+const remove = path.relative(process.cwd(), test262Dir);
+
+argv._ = argv._.map(p => path.relative(remove, p));
+
+let test262Version;
+try {
+  test262Version = JSON.parse(
+    fs.readFileSync(path.join(test262Dir, 'package.json'))
+  ).version;
+} catch (err) {
+  console.error(`Unable to detect version of test262: ${err}`);
+  process.exitCode = 1;
+  return;
+}
+
+if (!acceptVersion) {
+  acceptVersion = test262Version;
+}
+
+const stream = new TestStream(test262Dir, includesDir, acceptVersion, argv._);
+
+let tests = stream.pipe(filter(filterByFeatureInclude)).pipe(filter(filterByFeatureExclude)).pipe(map(insertPrelude));
+
+if (preprocessor) {
+  tests = tests.pipe(filter(preprocessor));
+}
+
+const results = zip(pool, tests).pipe(
+  flatMap(pair => {
+    return pool.runTest(pair);
+  })
+).pipe(
+  map(test => {
+    test.result = validator(test);
+    return test;
+  })
+);
+
+const emitter = new ResultsEmitter(results);
+
+if (argv.errorForFailures) {
+  emitter.on('fail', function () {
+    process.exitCode = 1;
+  });
+}
+
+reporter(emitter, reporterOpts);
 
 function printVersion() {
-    var p = require(path.resolve(__dirname, "..", "package.json"));
-    console.log("test262-harness v" + p.version);
+  const p = require(path.resolve(__dirname, '..', 'package.json'));
+  console.log(`v${p.version}`);
 }
 
-function pathToTestFile(path) {
-  return { file: path, contents: fs.readFileSync(path, 'utf-8')};
+function insertPrelude(test) {
+  const index = test.insertionIndex;
+  if (index === -1) {
+    return test;
+  }
+
+  if (preludeContents) {
+    test.contents = test.contents.slice(0, index) +
+      preludeContents +
+      test.contents.slice(index);
+  }
+
+  return test;
 }
 
-const endFrontmatterRe = /---\*\/\r?\n/g;
-function compileFile(test) {
-  const match = endFrontmatterRe.exec(test.contents);
-  if (match) {
-    test.contents = test.contents.slice(0, endFrontmatterRe.lastIndex)
-                    + preludeContents
-                    + test.contents.slice(endFrontmatterRe.lastIndex);
-  } else {
-    test.contents = preludeContents + test.contents;
+function filterByFeatureInclude(test) {
+  if (!featuresInclude) {
+    return true;
   }
-  return compile(test, { test262Dir: test262Dir, includesDir: includesDir });
-}
+  return featuresInclude.some(feature => (test.attrs.features || []).includes(feature));
+}
+
+function filterByFeatureExclude(test) {
+  if (!featuresExclude) {
+    return true;
+  }
+  return !featuresExclude.some(feature => (test.attrs.features || []).includes(feature));
+}