rule-tester.js 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680
  1. /**
  2. * @fileoverview Mocha test wrapper
  3. * @author Ilya Volodin
  4. */
  5. "use strict";
  6. /* global describe, it */
  7. /*
  8. * This is a wrapper around mocha to allow for DRY unittests for eslint
  9. * Format:
  10. * RuleTester.run("{ruleName}", {
  11. * valid: [
  12. * "{code}",
  13. * { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings} }
  14. * ],
  15. * invalid: [
  16. * { code: "{code}", errors: {numErrors} },
  17. * { code: "{code}", errors: ["{errorMessage}"] },
  18. * { code: "{code}", options: {options}, globals: {globals}, parser: "{parser}", settings: {settings}, errors: [{ message: "{errorMessage}", type: "{errorNodeType}"}] }
  19. * ]
  20. * });
  21. *
  22. * Variables:
  23. * {code} - String that represents the code to be tested
  24. * {options} - Arguments that are passed to the configurable rules.
  25. * {globals} - An object representing a list of variables that are
  26. * registered as globals
  27. * {parser} - String representing the parser to use
  28. * {settings} - An object representing global settings for all rules
  29. * {numErrors} - If failing case doesn't need to check error message,
  30. * this integer will specify how many errors should be
  31. * received
  32. * {errorMessage} - Message that is returned by the rule on failure
  33. * {errorNodeType} - AST node type that is returned by they rule as
  34. * a cause of the failure.
  35. */
  36. //------------------------------------------------------------------------------
  37. // Requirements
  38. //------------------------------------------------------------------------------
  39. const
  40. assert = require("assert"),
  41. path = require("path"),
  42. util = require("util"),
  43. lodash = require("lodash"),
  44. { getRuleOptionsSchema, validate } = require("../shared/config-validator"),
  45. { Linter, SourceCodeFixer, interpolate } = require("../linter");
  46. const ajv = require("../shared/ajv")({ strictDefaults: true });
  47. //------------------------------------------------------------------------------
  48. // Private Members
  49. //------------------------------------------------------------------------------
  50. /*
  51. * testerDefaultConfig must not be modified as it allows to reset the tester to
  52. * the initial default configuration
  53. */
  54. const testerDefaultConfig = { rules: {} };
  55. let defaultConfig = { rules: {} };
  56. /*
  57. * List every parameters possible on a test case that are not related to eslint
  58. * configuration
  59. */
  60. const RuleTesterParameters = [
  61. "code",
  62. "filename",
  63. "options",
  64. "errors",
  65. "output"
  66. ];
  67. const hasOwnProperty = Function.call.bind(Object.hasOwnProperty);
  68. /**
  69. * Clones a given value deeply.
  70. * Note: This ignores `parent` property.
  71. * @param {any} x A value to clone.
  72. * @returns {any} A cloned value.
  73. */
  74. function cloneDeeplyExcludesParent(x) {
  75. if (typeof x === "object" && x !== null) {
  76. if (Array.isArray(x)) {
  77. return x.map(cloneDeeplyExcludesParent);
  78. }
  79. const retv = {};
  80. for (const key in x) {
  81. if (key !== "parent" && hasOwnProperty(x, key)) {
  82. retv[key] = cloneDeeplyExcludesParent(x[key]);
  83. }
  84. }
  85. return retv;
  86. }
  87. return x;
  88. }
  89. /**
  90. * Freezes a given value deeply.
  91. * @param {any} x A value to freeze.
  92. * @returns {void}
  93. */
  94. function freezeDeeply(x) {
  95. if (typeof x === "object" && x !== null) {
  96. if (Array.isArray(x)) {
  97. x.forEach(freezeDeeply);
  98. } else {
  99. for (const key in x) {
  100. if (key !== "parent" && hasOwnProperty(x, key)) {
  101. freezeDeeply(x[key]);
  102. }
  103. }
  104. }
  105. Object.freeze(x);
  106. }
  107. }
  108. /**
  109. * Replace control characters by `\u00xx` form.
  110. * @param {string} text The text to sanitize.
  111. * @returns {string} The sanitized text.
  112. */
  113. function sanitize(text) {
  114. return text.replace(
  115. /[\u0000-\u0009|\u000b-\u001a]/gu, // eslint-disable-line no-control-regex
  116. c => `\\u${c.codePointAt(0).toString(16).padStart(4, "0")}`
  117. );
  118. }
  119. //------------------------------------------------------------------------------
  120. // Public Interface
  121. //------------------------------------------------------------------------------
  122. // default separators for testing
  123. const DESCRIBE = Symbol("describe");
  124. const IT = Symbol("it");
  125. /**
  126. * This is `it` default handler if `it` don't exist.
  127. * @this {Mocha}
  128. * @param {string} text The description of the test case.
  129. * @param {Function} method The logic of the test case.
  130. * @returns {any} Returned value of `method`.
  131. */
  132. function itDefaultHandler(text, method) {
  133. try {
  134. return method.call(this);
  135. } catch (err) {
  136. if (err instanceof assert.AssertionError) {
  137. err.message += ` (${util.inspect(err.actual)} ${err.operator} ${util.inspect(err.expected)})`;
  138. }
  139. throw err;
  140. }
  141. }
  142. /**
  143. * This is `describe` default handler if `describe` don't exist.
  144. * @this {Mocha}
  145. * @param {string} text The description of the test case.
  146. * @param {Function} method The logic of the test case.
  147. * @returns {any} Returned value of `method`.
  148. */
  149. function describeDefaultHandler(text, method) {
  150. return method.call(this);
  151. }
  152. class RuleTester {
  153. /**
  154. * Creates a new instance of RuleTester.
  155. * @param {Object} [testerConfig] Optional, extra configuration for the tester
  156. */
  157. constructor(testerConfig) {
  158. /**
  159. * The configuration to use for this tester. Combination of the tester
  160. * configuration and the default configuration.
  161. * @type {Object}
  162. */
  163. this.testerConfig = lodash.merge(
  164. // we have to clone because merge uses the first argument for recipient
  165. lodash.cloneDeep(defaultConfig),
  166. testerConfig,
  167. { rules: { "rule-tester/validate-ast": "error" } }
  168. );
  169. /**
  170. * Rule definitions to define before tests.
  171. * @type {Object}
  172. */
  173. this.rules = {};
  174. this.linter = new Linter();
  175. }
  176. /**
  177. * Set the configuration to use for all future tests
  178. * @param {Object} config the configuration to use.
  179. * @returns {void}
  180. */
  181. static setDefaultConfig(config) {
  182. if (typeof config !== "object") {
  183. throw new TypeError("RuleTester.setDefaultConfig: config must be an object");
  184. }
  185. defaultConfig = config;
  186. // Make sure the rules object exists since it is assumed to exist later
  187. defaultConfig.rules = defaultConfig.rules || {};
  188. }
  189. /**
  190. * Get the current configuration used for all tests
  191. * @returns {Object} the current configuration
  192. */
  193. static getDefaultConfig() {
  194. return defaultConfig;
  195. }
  196. /**
  197. * Reset the configuration to the initial configuration of the tester removing
  198. * any changes made until now.
  199. * @returns {void}
  200. */
  201. static resetDefaultConfig() {
  202. defaultConfig = lodash.cloneDeep(testerDefaultConfig);
  203. }
  204. /*
  205. * If people use `mocha test.js --watch` command, `describe` and `it` function
  206. * instances are different for each execution. So `describe` and `it` should get fresh instance
  207. * always.
  208. */
  209. static get describe() {
  210. return (
  211. this[DESCRIBE] ||
  212. (typeof describe === "function" ? describe : describeDefaultHandler)
  213. );
  214. }
  215. static set describe(value) {
  216. this[DESCRIBE] = value;
  217. }
  218. static get it() {
  219. return (
  220. this[IT] ||
  221. (typeof it === "function" ? it : itDefaultHandler)
  222. );
  223. }
  224. static set it(value) {
  225. this[IT] = value;
  226. }
  227. /**
  228. * Define a rule for one particular run of tests.
  229. * @param {string} name The name of the rule to define.
  230. * @param {Function} rule The rule definition.
  231. * @returns {void}
  232. */
  233. defineRule(name, rule) {
  234. this.rules[name] = rule;
  235. }
  236. /**
  237. * Adds a new rule test to execute.
  238. * @param {string} ruleName The name of the rule to run.
  239. * @param {Function} rule The rule to test.
  240. * @param {Object} test The collection of tests to run.
  241. * @returns {void}
  242. */
  243. run(ruleName, rule, test) {
  244. const testerConfig = this.testerConfig,
  245. requiredScenarios = ["valid", "invalid"],
  246. scenarioErrors = [],
  247. linter = this.linter;
  248. if (lodash.isNil(test) || typeof test !== "object") {
  249. throw new TypeError(`Test Scenarios for rule ${ruleName} : Could not find test scenario object`);
  250. }
  251. requiredScenarios.forEach(scenarioType => {
  252. if (lodash.isNil(test[scenarioType])) {
  253. scenarioErrors.push(`Could not find any ${scenarioType} test scenarios`);
  254. }
  255. });
  256. if (scenarioErrors.length > 0) {
  257. throw new Error([
  258. `Test Scenarios for rule ${ruleName} is invalid:`
  259. ].concat(scenarioErrors).join("\n"));
  260. }
  261. linter.defineRule(ruleName, Object.assign({}, rule, {
  262. // Create a wrapper rule that freezes the `context` properties.
  263. create(context) {
  264. freezeDeeply(context.options);
  265. freezeDeeply(context.settings);
  266. freezeDeeply(context.parserOptions);
  267. return (typeof rule === "function" ? rule : rule.create)(context);
  268. }
  269. }));
  270. linter.defineRules(this.rules);
  271. /**
  272. * Run the rule for the given item
  273. * @param {string|Object} item Item to run the rule against
  274. * @returns {Object} Eslint run result
  275. * @private
  276. */
  277. function runRuleForItem(item) {
  278. let config = lodash.cloneDeep(testerConfig),
  279. code, filename, output, beforeAST, afterAST;
  280. if (typeof item === "string") {
  281. code = item;
  282. } else {
  283. code = item.code;
  284. /*
  285. * Assumes everything on the item is a config except for the
  286. * parameters used by this tester
  287. */
  288. const itemConfig = lodash.omit(item, RuleTesterParameters);
  289. /*
  290. * Create the config object from the tester config and this item
  291. * specific configurations.
  292. */
  293. config = lodash.merge(
  294. config,
  295. itemConfig
  296. );
  297. }
  298. if (item.filename) {
  299. filename = item.filename;
  300. }
  301. if (hasOwnProperty(item, "options")) {
  302. assert(Array.isArray(item.options), "options must be an array");
  303. config.rules[ruleName] = [1].concat(item.options);
  304. } else {
  305. config.rules[ruleName] = 1;
  306. }
  307. const schema = getRuleOptionsSchema(rule);
  308. /*
  309. * Setup AST getters.
  310. * The goal is to check whether or not AST was modified when
  311. * running the rule under test.
  312. */
  313. linter.defineRule("rule-tester/validate-ast", () => ({
  314. Program(node) {
  315. beforeAST = cloneDeeplyExcludesParent(node);
  316. },
  317. "Program:exit"(node) {
  318. afterAST = node;
  319. }
  320. }));
  321. if (typeof config.parser === "string") {
  322. assert(path.isAbsolute(config.parser), "Parsers provided as strings to RuleTester must be absolute paths");
  323. linter.defineParser(config.parser, require(config.parser));
  324. }
  325. if (schema) {
  326. ajv.validateSchema(schema);
  327. if (ajv.errors) {
  328. const errors = ajv.errors.map(error => {
  329. const field = error.dataPath[0] === "." ? error.dataPath.slice(1) : error.dataPath;
  330. return `\t${field}: ${error.message}`;
  331. }).join("\n");
  332. throw new Error([`Schema for rule ${ruleName} is invalid:`, errors]);
  333. }
  334. /*
  335. * `ajv.validateSchema` checks for errors in the structure of the schema (by comparing the schema against a "meta-schema"),
  336. * and it reports those errors individually. However, there are other types of schema errors that only occur when compiling
  337. * the schema (e.g. using invalid defaults in a schema), and only one of these errors can be reported at a time. As a result,
  338. * the schema is compiled here separately from checking for `validateSchema` errors.
  339. */
  340. try {
  341. ajv.compile(schema);
  342. } catch (err) {
  343. throw new Error(`Schema for rule ${ruleName} is invalid: ${err.message}`);
  344. }
  345. }
  346. validate(config, "rule-tester", id => (id === ruleName ? rule : null));
  347. // Verify the code.
  348. const messages = linter.verify(code, config, filename);
  349. // Ignore syntax errors for backward compatibility if `errors` is a number.
  350. if (typeof item.errors !== "number") {
  351. const errorMessage = messages.find(m => m.fatal);
  352. assert(!errorMessage, `A fatal parsing error occurred: ${errorMessage && errorMessage.message}`);
  353. }
  354. // Verify if autofix makes a syntax error or not.
  355. if (messages.some(m => m.fix)) {
  356. output = SourceCodeFixer.applyFixes(code, messages).output;
  357. const errorMessageInFix = linter.verify(output, config, filename).find(m => m.fatal);
  358. assert(!errorMessageInFix, `A fatal parsing error occurred in autofix: ${errorMessageInFix && errorMessageInFix.message}`);
  359. } else {
  360. output = code;
  361. }
  362. return {
  363. messages,
  364. output,
  365. beforeAST,
  366. afterAST: cloneDeeplyExcludesParent(afterAST)
  367. };
  368. }
  369. /**
  370. * Check if the AST was changed
  371. * @param {ASTNode} beforeAST AST node before running
  372. * @param {ASTNode} afterAST AST node after running
  373. * @returns {void}
  374. * @private
  375. */
  376. function assertASTDidntChange(beforeAST, afterAST) {
  377. if (!lodash.isEqual(beforeAST, afterAST)) {
  378. assert.fail("Rule should not modify AST.");
  379. }
  380. }
  381. /**
  382. * Check if the template is valid or not
  383. * all valid cases go through this
  384. * @param {string|Object} item Item to run the rule against
  385. * @returns {void}
  386. * @private
  387. */
  388. function testValidTemplate(item) {
  389. const result = runRuleForItem(item);
  390. const messages = result.messages;
  391. assert.strictEqual(messages.length, 0, util.format("Should have no errors but had %d: %s",
  392. messages.length, util.inspect(messages)));
  393. assertASTDidntChange(result.beforeAST, result.afterAST);
  394. }
  395. /**
  396. * Asserts that the message matches its expected value. If the expected
  397. * value is a regular expression, it is checked against the actual
  398. * value.
  399. * @param {string} actual Actual value
  400. * @param {string|RegExp} expected Expected value
  401. * @returns {void}
  402. * @private
  403. */
  404. function assertMessageMatches(actual, expected) {
  405. if (expected instanceof RegExp) {
  406. // assert.js doesn't have a built-in RegExp match function
  407. assert.ok(
  408. expected.test(actual),
  409. `Expected '${actual}' to match ${expected}`
  410. );
  411. } else {
  412. assert.strictEqual(actual, expected);
  413. }
  414. }
  415. /**
  416. * Check if the template is invalid or not
  417. * all invalid cases go through this.
  418. * @param {string|Object} item Item to run the rule against
  419. * @returns {void}
  420. * @private
  421. */
  422. function testInvalidTemplate(item) {
  423. assert.ok(item.errors || item.errors === 0,
  424. `Did not specify errors for an invalid test of ${ruleName}`);
  425. const result = runRuleForItem(item);
  426. const messages = result.messages;
  427. if (typeof item.errors === "number") {
  428. assert.strictEqual(messages.length, item.errors, util.format("Should have %d error%s but had %d: %s",
  429. item.errors, item.errors === 1 ? "" : "s", messages.length, util.inspect(messages)));
  430. } else {
  431. assert.strictEqual(
  432. messages.length, item.errors.length,
  433. util.format(
  434. "Should have %d error%s but had %d: %s",
  435. item.errors.length, item.errors.length === 1 ? "" : "s", messages.length, util.inspect(messages)
  436. )
  437. );
  438. const hasMessageOfThisRule = messages.some(m => m.ruleId === ruleName);
  439. for (let i = 0, l = item.errors.length; i < l; i++) {
  440. const error = item.errors[i];
  441. const message = messages[i];
  442. assert(hasMessageOfThisRule, "Error rule name should be the same as the name of the rule being tested");
  443. if (typeof error === "string" || error instanceof RegExp) {
  444. // Just an error message.
  445. assertMessageMatches(message.message, error);
  446. } else if (typeof error === "object") {
  447. /*
  448. * Error object.
  449. * This may have a message, messageId, data, node type, line, and/or
  450. * column.
  451. */
  452. if (hasOwnProperty(error, "message")) {
  453. assert.ok(!hasOwnProperty(error, "messageId"), "Error should not specify both 'message' and a 'messageId'.");
  454. assert.ok(!hasOwnProperty(error, "data"), "Error should not specify both 'data' and 'message'.");
  455. assertMessageMatches(message.message, error.message);
  456. } else if (hasOwnProperty(error, "messageId")) {
  457. assert.ok(
  458. hasOwnProperty(rule, "meta") && hasOwnProperty(rule.meta, "messages"),
  459. "Error can not use 'messageId' if rule under test doesn't define 'meta.messages'."
  460. );
  461. if (!hasOwnProperty(rule.meta.messages, error.messageId)) {
  462. const friendlyIDList = `[${Object.keys(rule.meta.messages).map(key => `'${key}'`).join(", ")}]`;
  463. assert(false, `Invalid messageId '${error.messageId}'. Expected one of ${friendlyIDList}.`);
  464. }
  465. assert.strictEqual(
  466. message.messageId,
  467. error.messageId,
  468. `messageId '${message.messageId}' does not match expected messageId '${error.messageId}'.`
  469. );
  470. if (hasOwnProperty(error, "data")) {
  471. /*
  472. * if data was provided, then directly compare the returned message to a synthetic
  473. * interpolated message using the same message ID and data provided in the test.
  474. * See https://github.com/eslint/eslint/issues/9890 for context.
  475. */
  476. const unformattedOriginalMessage = rule.meta.messages[error.messageId];
  477. const rehydratedMessage = interpolate(unformattedOriginalMessage, error.data);
  478. assert.strictEqual(
  479. message.message,
  480. rehydratedMessage,
  481. `Hydrated message "${rehydratedMessage}" does not match "${message.message}"`
  482. );
  483. }
  484. }
  485. assert.ok(
  486. hasOwnProperty(error, "data") ? hasOwnProperty(error, "messageId") : true,
  487. "Error must specify 'messageId' if 'data' is used."
  488. );
  489. if (error.type) {
  490. assert.strictEqual(message.nodeType, error.type, `Error type should be ${error.type}, found ${message.nodeType}`);
  491. }
  492. if (hasOwnProperty(error, "line")) {
  493. assert.strictEqual(message.line, error.line, `Error line should be ${error.line}`);
  494. }
  495. if (hasOwnProperty(error, "column")) {
  496. assert.strictEqual(message.column, error.column, `Error column should be ${error.column}`);
  497. }
  498. if (hasOwnProperty(error, "endLine")) {
  499. assert.strictEqual(message.endLine, error.endLine, `Error endLine should be ${error.endLine}`);
  500. }
  501. if (hasOwnProperty(error, "endColumn")) {
  502. assert.strictEqual(message.endColumn, error.endColumn, `Error endColumn should be ${error.endColumn}`);
  503. }
  504. if (hasOwnProperty(error, "suggestions")) {
  505. // Support asserting there are no suggestions
  506. if (!error.suggestions || (Array.isArray(error.suggestions) && error.suggestions.length === 0)) {
  507. if (Array.isArray(message.suggestions) && message.suggestions.length > 0) {
  508. assert.fail(`Error should have no suggestions on error with message: "${message.message}"`);
  509. }
  510. } else {
  511. assert.strictEqual(Array.isArray(message.suggestions), true, `Error should have an array of suggestions. Instead received "${message.suggestions}" on error with message: "${message.message}"`);
  512. assert.strictEqual(message.suggestions.length, error.suggestions.length, `Error should have ${error.suggestions.length} suggestions. Instead found ${message.suggestions.length} suggestions`);
  513. error.suggestions.forEach((expectedSuggestion, index) => {
  514. const actualSuggestion = message.suggestions[index];
  515. /**
  516. * Tests equality of a suggestion key if that key is defined in the expected output.
  517. * @param {string} key Key to validate from the suggestion object
  518. * @returns {void}
  519. */
  520. function assertSuggestionKeyEquals(key) {
  521. if (hasOwnProperty(expectedSuggestion, key)) {
  522. assert.deepStrictEqual(actualSuggestion[key], expectedSuggestion[key], `Error suggestion at index: ${index} should have desc of: "${actualSuggestion[key]}"`);
  523. }
  524. }
  525. assertSuggestionKeyEquals("desc");
  526. assertSuggestionKeyEquals("messageId");
  527. if (hasOwnProperty(expectedSuggestion, "output")) {
  528. const codeWithAppliedSuggestion = SourceCodeFixer.applyFixes(item.code, [actualSuggestion]).output;
  529. assert.strictEqual(codeWithAppliedSuggestion, expectedSuggestion.output, `Expected the applied suggestion fix to match the test suggestion output for suggestion at index: ${index} on error with message: "${message.message}"`);
  530. }
  531. });
  532. }
  533. }
  534. } else {
  535. // Message was an unexpected type
  536. assert.fail(`Error should be a string, object, or RegExp, but found (${util.inspect(message)})`);
  537. }
  538. }
  539. }
  540. if (hasOwnProperty(item, "output")) {
  541. if (item.output === null) {
  542. assert.strictEqual(
  543. result.output,
  544. item.code,
  545. "Expected no autofixes to be suggested"
  546. );
  547. } else {
  548. assert.strictEqual(result.output, item.output, "Output is incorrect.");
  549. }
  550. }
  551. assertASTDidntChange(result.beforeAST, result.afterAST);
  552. }
  553. /*
  554. * This creates a mocha test suite and pipes all supplied info through
  555. * one of the templates above.
  556. */
  557. RuleTester.describe(ruleName, () => {
  558. RuleTester.describe("valid", () => {
  559. test.valid.forEach(valid => {
  560. RuleTester.it(sanitize(typeof valid === "object" ? valid.code : valid), () => {
  561. testValidTemplate(valid);
  562. });
  563. });
  564. });
  565. RuleTester.describe("invalid", () => {
  566. test.invalid.forEach(invalid => {
  567. RuleTester.it(sanitize(invalid.code), () => {
  568. testInvalidTemplate(invalid);
  569. });
  570. });
  571. });
  572. });
  573. }
  574. }
  575. RuleTester[DESCRIBE] = RuleTester[IT] = null;
  576. module.exports = RuleTester;