CachedInputFileSystem.js 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664
  1. /*
  2. MIT License http://www.opensource.org/licenses/mit-license.php
  3. Author Tobias Koppers @sokra
  4. */
  5. "use strict";
  6. const nextTick = require("process").nextTick;
  7. /** @typedef {import("./Resolver").FileSystem} FileSystem */
  8. /** @typedef {import("./Resolver").PathLike} PathLike */
  9. /** @typedef {import("./Resolver").PathOrFileDescriptor} PathOrFileDescriptor */
  10. /** @typedef {import("./Resolver").SyncFileSystem} SyncFileSystem */
  11. /** @typedef {FileSystem & SyncFileSystem} BaseFileSystem */
  12. /**
  13. * @template T
  14. * @typedef {import("./Resolver").FileSystemCallback<T>} FileSystemCallback<T>
  15. */
  16. /**
  17. * @param {string} path path
  18. * @returns {string} dirname
  19. */
  20. const dirname = path => {
  21. let idx = path.length - 1;
  22. while (idx >= 0) {
  23. const c = path.charCodeAt(idx);
  24. // slash or backslash
  25. if (c === 47 || c === 92) break;
  26. idx--;
  27. }
  28. if (idx < 0) return "";
  29. return path.slice(0, idx);
  30. };
  31. /**
  32. * @template T
  33. * @param {FileSystemCallback<T>[]} callbacks callbacks
  34. * @param {Error | null} err error
  35. * @param {T} result result
  36. */
  37. const runCallbacks = (callbacks, err, result) => {
  38. if (callbacks.length === 1) {
  39. callbacks[0](err, result);
  40. callbacks.length = 0;
  41. return;
  42. }
  43. let error;
  44. for (const callback of callbacks) {
  45. try {
  46. callback(err, result);
  47. } catch (e) {
  48. if (!error) error = e;
  49. }
  50. }
  51. callbacks.length = 0;
  52. if (error) throw error;
  53. };
  54. class OperationMergerBackend {
  55. /**
  56. * @param {Function | undefined} provider async method in filesystem
  57. * @param {Function | undefined} syncProvider sync method in filesystem
  58. * @param {BaseFileSystem} providerContext call context for the provider methods
  59. */
  60. constructor(provider, syncProvider, providerContext) {
  61. this._provider = provider;
  62. this._syncProvider = syncProvider;
  63. this._providerContext = providerContext;
  64. this._activeAsyncOperations = new Map();
  65. this.provide = this._provider
  66. ? /**
  67. * @param {PathLike | PathOrFileDescriptor} path path
  68. * @param {object | FileSystemCallback<any> | undefined} options options
  69. * @param {FileSystemCallback<any>=} callback callback
  70. * @returns {any} result
  71. */
  72. (path, options, callback) => {
  73. if (typeof options === "function") {
  74. callback = /** @type {FileSystemCallback<any>} */ (options);
  75. options = undefined;
  76. }
  77. if (
  78. typeof path !== "string" &&
  79. !Buffer.isBuffer(path) &&
  80. !(path instanceof URL) &&
  81. typeof path !== "number"
  82. ) {
  83. /** @type {Function} */
  84. (callback)(
  85. new TypeError("path must be a string, Buffer, URL or number")
  86. );
  87. return;
  88. }
  89. if (options) {
  90. return /** @type {Function} */ (this._provider).call(
  91. this._providerContext,
  92. path,
  93. options,
  94. callback
  95. );
  96. }
  97. let callbacks = this._activeAsyncOperations.get(path);
  98. if (callbacks) {
  99. callbacks.push(callback);
  100. return;
  101. }
  102. this._activeAsyncOperations.set(path, (callbacks = [callback]));
  103. /** @type {Function} */
  104. (provider)(
  105. path,
  106. /**
  107. * @param {Error} err error
  108. * @param {any} result result
  109. */
  110. (err, result) => {
  111. this._activeAsyncOperations.delete(path);
  112. runCallbacks(callbacks, err, result);
  113. }
  114. );
  115. }
  116. : null;
  117. this.provideSync = this._syncProvider
  118. ? /**
  119. * @param {PathLike | PathOrFileDescriptor} path path
  120. * @param {object=} options options
  121. * @returns {any} result
  122. */
  123. (path, options) => {
  124. return /** @type {Function} */ (this._syncProvider).call(
  125. this._providerContext,
  126. path,
  127. options
  128. );
  129. }
  130. : null;
  131. }
  132. purge() {}
  133. purgeParent() {}
  134. }
  135. /*
  136. IDLE:
  137. insert data: goto SYNC
  138. SYNC:
  139. before provide: run ticks
  140. event loop tick: goto ASYNC_ACTIVE
  141. ASYNC:
  142. timeout: run tick, goto ASYNC_PASSIVE
  143. ASYNC_PASSIVE:
  144. before provide: run ticks
  145. IDLE --[insert data]--> SYNC --[event loop tick]--> ASYNC_ACTIVE --[interval tick]-> ASYNC_PASSIVE
  146. ^ |
  147. +---------[insert data]-------+
  148. */
  149. const STORAGE_MODE_IDLE = 0;
  150. const STORAGE_MODE_SYNC = 1;
  151. const STORAGE_MODE_ASYNC = 2;
  152. /**
  153. * @callback Provide
  154. * @param {PathLike | PathOrFileDescriptor} path path
  155. * @param {any} options options
  156. * @param {FileSystemCallback<any>} callback callback
  157. * @returns {void}
  158. */
  159. class CacheBackend {
  160. /**
  161. * @param {number} duration max cache duration of items
  162. * @param {function | undefined} provider async method
  163. * @param {function | undefined} syncProvider sync method
  164. * @param {BaseFileSystem} providerContext call context for the provider methods
  165. */
  166. constructor(duration, provider, syncProvider, providerContext) {
  167. this._duration = duration;
  168. this._provider = provider;
  169. this._syncProvider = syncProvider;
  170. this._providerContext = providerContext;
  171. /** @type {Map<string, FileSystemCallback<any>[]>} */
  172. this._activeAsyncOperations = new Map();
  173. /** @type {Map<string, { err: Error | null, result?: any, level: Set<string> }>} */
  174. this._data = new Map();
  175. /** @type {Set<string>[]} */
  176. this._levels = [];
  177. for (let i = 0; i < 10; i++) this._levels.push(new Set());
  178. for (let i = 5000; i < duration; i += 500) this._levels.push(new Set());
  179. this._currentLevel = 0;
  180. this._tickInterval = Math.floor(duration / this._levels.length);
  181. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC} */
  182. this._mode = STORAGE_MODE_IDLE;
  183. /** @type {NodeJS.Timeout | undefined} */
  184. this._timeout = undefined;
  185. /** @type {number | undefined} */
  186. this._nextDecay = undefined;
  187. // @ts-ignore
  188. this.provide = provider ? this.provide.bind(this) : null;
  189. // @ts-ignore
  190. this.provideSync = syncProvider ? this.provideSync.bind(this) : null;
  191. }
  192. /**
  193. * @param {PathLike | PathOrFileDescriptor} path path
  194. * @param {any} options options
  195. * @param {FileSystemCallback<any>} callback callback
  196. * @returns {void}
  197. */
  198. provide(path, options, callback) {
  199. if (typeof options === "function") {
  200. callback = options;
  201. options = undefined;
  202. }
  203. if (
  204. typeof path !== "string" &&
  205. !Buffer.isBuffer(path) &&
  206. !(path instanceof URL) &&
  207. typeof path !== "number"
  208. ) {
  209. callback(new TypeError("path must be a string, Buffer, URL or number"));
  210. return;
  211. }
  212. const strPath = typeof path !== "string" ? path.toString() : path;
  213. if (options) {
  214. return /** @type {Function} */ (this._provider).call(
  215. this._providerContext,
  216. path,
  217. options,
  218. callback
  219. );
  220. }
  221. // When in sync mode we can move to async mode
  222. if (this._mode === STORAGE_MODE_SYNC) {
  223. this._enterAsyncMode();
  224. }
  225. // Check in cache
  226. let cacheEntry = this._data.get(strPath);
  227. if (cacheEntry !== undefined) {
  228. if (cacheEntry.err) return nextTick(callback, cacheEntry.err);
  229. return nextTick(callback, null, cacheEntry.result);
  230. }
  231. // Check if there is already the same operation running
  232. let callbacks = this._activeAsyncOperations.get(strPath);
  233. if (callbacks !== undefined) {
  234. callbacks.push(callback);
  235. return;
  236. }
  237. this._activeAsyncOperations.set(strPath, (callbacks = [callback]));
  238. // Run the operation
  239. /** @type {Function} */
  240. (this._provider).call(
  241. this._providerContext,
  242. path,
  243. /**
  244. * @param {Error | null} err error
  245. * @param {any} [result] result
  246. */
  247. (err, result) => {
  248. this._activeAsyncOperations.delete(strPath);
  249. this._storeResult(strPath, err, result);
  250. // Enter async mode if not yet done
  251. this._enterAsyncMode();
  252. runCallbacks(
  253. /** @type {FileSystemCallback<any>[]} */ (callbacks),
  254. err,
  255. result
  256. );
  257. }
  258. );
  259. }
  260. /**
  261. * @param {PathLike | PathOrFileDescriptor} path path
  262. * @param {any} options options
  263. * @returns {any} result
  264. */
  265. provideSync(path, options) {
  266. if (
  267. typeof path !== "string" &&
  268. !Buffer.isBuffer(path) &&
  269. !(path instanceof URL) &&
  270. typeof path !== "number"
  271. ) {
  272. throw new TypeError("path must be a string");
  273. }
  274. const strPath = typeof path !== "string" ? path.toString() : path;
  275. if (options) {
  276. return /** @type {Function} */ (this._syncProvider).call(
  277. this._providerContext,
  278. path,
  279. options
  280. );
  281. }
  282. // In sync mode we may have to decay some cache items
  283. if (this._mode === STORAGE_MODE_SYNC) {
  284. this._runDecays();
  285. }
  286. // Check in cache
  287. let cacheEntry = this._data.get(strPath);
  288. if (cacheEntry !== undefined) {
  289. if (cacheEntry.err) throw cacheEntry.err;
  290. return cacheEntry.result;
  291. }
  292. // Get all active async operations
  293. // This sync operation will also complete them
  294. const callbacks = this._activeAsyncOperations.get(strPath);
  295. this._activeAsyncOperations.delete(strPath);
  296. // Run the operation
  297. // When in idle mode, we will enter sync mode
  298. let result;
  299. try {
  300. result = /** @type {Function} */ (this._syncProvider).call(
  301. this._providerContext,
  302. path
  303. );
  304. } catch (err) {
  305. this._storeResult(strPath, /** @type {Error} */ (err), undefined);
  306. this._enterSyncModeWhenIdle();
  307. if (callbacks) {
  308. runCallbacks(callbacks, /** @type {Error} */ (err), undefined);
  309. }
  310. throw err;
  311. }
  312. this._storeResult(strPath, null, result);
  313. this._enterSyncModeWhenIdle();
  314. if (callbacks) {
  315. runCallbacks(callbacks, null, result);
  316. }
  317. return result;
  318. }
  319. /**
  320. * @param {string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>} [what] what to purge
  321. */
  322. purge(what) {
  323. if (!what) {
  324. if (this._mode !== STORAGE_MODE_IDLE) {
  325. this._data.clear();
  326. for (const level of this._levels) {
  327. level.clear();
  328. }
  329. this._enterIdleMode();
  330. }
  331. } else if (
  332. typeof what === "string" ||
  333. Buffer.isBuffer(what) ||
  334. what instanceof URL ||
  335. typeof what === "number"
  336. ) {
  337. const strWhat = typeof what !== "string" ? what.toString() : what;
  338. for (let [key, data] of this._data) {
  339. if (key.startsWith(strWhat)) {
  340. this._data.delete(key);
  341. data.level.delete(key);
  342. }
  343. }
  344. if (this._data.size === 0) {
  345. this._enterIdleMode();
  346. }
  347. } else {
  348. for (let [key, data] of this._data) {
  349. for (const item of what) {
  350. const strItem = typeof item !== "string" ? item.toString() : item;
  351. if (key.startsWith(strItem)) {
  352. this._data.delete(key);
  353. data.level.delete(key);
  354. break;
  355. }
  356. }
  357. }
  358. if (this._data.size === 0) {
  359. this._enterIdleMode();
  360. }
  361. }
  362. }
  363. /**
  364. * @param {string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>} [what] what to purge
  365. */
  366. purgeParent(what) {
  367. if (!what) {
  368. this.purge();
  369. } else if (
  370. typeof what === "string" ||
  371. Buffer.isBuffer(what) ||
  372. what instanceof URL ||
  373. typeof what === "number"
  374. ) {
  375. const strWhat = typeof what !== "string" ? what.toString() : what;
  376. this.purge(dirname(strWhat));
  377. } else {
  378. const set = new Set();
  379. for (const item of what) {
  380. const strItem = typeof item !== "string" ? item.toString() : item;
  381. set.add(dirname(strItem));
  382. }
  383. this.purge(set);
  384. }
  385. }
  386. /**
  387. * @param {string} path path
  388. * @param {Error | null} err error
  389. * @param {any} result result
  390. */
  391. _storeResult(path, err, result) {
  392. if (this._data.has(path)) return;
  393. const level = this._levels[this._currentLevel];
  394. this._data.set(path, { err, result, level });
  395. level.add(path);
  396. }
  397. _decayLevel() {
  398. const nextLevel = (this._currentLevel + 1) % this._levels.length;
  399. const decay = this._levels[nextLevel];
  400. this._currentLevel = nextLevel;
  401. for (let item of decay) {
  402. this._data.delete(item);
  403. }
  404. decay.clear();
  405. if (this._data.size === 0) {
  406. this._enterIdleMode();
  407. } else {
  408. /** @type {number} */
  409. (this._nextDecay) += this._tickInterval;
  410. }
  411. }
  412. _runDecays() {
  413. while (
  414. /** @type {number} */ (this._nextDecay) <= Date.now() &&
  415. this._mode !== STORAGE_MODE_IDLE
  416. ) {
  417. this._decayLevel();
  418. }
  419. }
  420. _enterAsyncMode() {
  421. let timeout = 0;
  422. switch (this._mode) {
  423. case STORAGE_MODE_ASYNC:
  424. return;
  425. case STORAGE_MODE_IDLE:
  426. this._nextDecay = Date.now() + this._tickInterval;
  427. timeout = this._tickInterval;
  428. break;
  429. case STORAGE_MODE_SYNC:
  430. this._runDecays();
  431. // _runDecays may change the mode
  432. if (
  433. /** @type {STORAGE_MODE_IDLE | STORAGE_MODE_SYNC | STORAGE_MODE_ASYNC}*/
  434. (this._mode) === STORAGE_MODE_IDLE
  435. )
  436. return;
  437. timeout = Math.max(
  438. 0,
  439. /** @type {number} */ (this._nextDecay) - Date.now()
  440. );
  441. break;
  442. }
  443. this._mode = STORAGE_MODE_ASYNC;
  444. const ref = setTimeout(() => {
  445. this._mode = STORAGE_MODE_SYNC;
  446. this._runDecays();
  447. }, timeout);
  448. if (ref.unref) ref.unref();
  449. this._timeout = ref;
  450. }
  451. _enterSyncModeWhenIdle() {
  452. if (this._mode === STORAGE_MODE_IDLE) {
  453. this._mode = STORAGE_MODE_SYNC;
  454. this._nextDecay = Date.now() + this._tickInterval;
  455. }
  456. }
  457. _enterIdleMode() {
  458. this._mode = STORAGE_MODE_IDLE;
  459. this._nextDecay = undefined;
  460. if (this._timeout) clearTimeout(this._timeout);
  461. }
  462. }
  463. /**
  464. * @template {function} Provider
  465. * @template {function} AsyncProvider
  466. * @template FileSystem
  467. * @param {number} duration duration in ms files are cached
  468. * @param {Provider | undefined} provider provider
  469. * @param {AsyncProvider | undefined} syncProvider sync provider
  470. * @param {BaseFileSystem} providerContext provider context
  471. * @returns {OperationMergerBackend | CacheBackend} backend
  472. */
  473. const createBackend = (duration, provider, syncProvider, providerContext) => {
  474. if (duration > 0) {
  475. return new CacheBackend(duration, provider, syncProvider, providerContext);
  476. }
  477. return new OperationMergerBackend(provider, syncProvider, providerContext);
  478. };
  479. module.exports = class CachedInputFileSystem {
  480. /**
  481. * @param {BaseFileSystem} fileSystem file system
  482. * @param {number} duration duration in ms files are cached
  483. */
  484. constructor(fileSystem, duration) {
  485. this.fileSystem = fileSystem;
  486. this._lstatBackend = createBackend(
  487. duration,
  488. this.fileSystem.lstat,
  489. this.fileSystem.lstatSync,
  490. this.fileSystem
  491. );
  492. const lstat = this._lstatBackend.provide;
  493. this.lstat = /** @type {FileSystem["lstat"]} */ (lstat);
  494. const lstatSync = this._lstatBackend.provideSync;
  495. this.lstatSync = /** @type {SyncFileSystem["lstatSync"]} */ (lstatSync);
  496. this._statBackend = createBackend(
  497. duration,
  498. this.fileSystem.stat,
  499. this.fileSystem.statSync,
  500. this.fileSystem
  501. );
  502. const stat = this._statBackend.provide;
  503. this.stat = /** @type {FileSystem["stat"]} */ (stat);
  504. const statSync = this._statBackend.provideSync;
  505. this.statSync = /** @type {SyncFileSystem["statSync"]} */ (statSync);
  506. this._readdirBackend = createBackend(
  507. duration,
  508. this.fileSystem.readdir,
  509. this.fileSystem.readdirSync,
  510. this.fileSystem
  511. );
  512. const readdir = this._readdirBackend.provide;
  513. this.readdir = /** @type {FileSystem["readdir"]} */ (readdir);
  514. const readdirSync = this._readdirBackend.provideSync;
  515. this.readdirSync = /** @type {SyncFileSystem["readdirSync"]} */ (
  516. readdirSync
  517. );
  518. this._readFileBackend = createBackend(
  519. duration,
  520. this.fileSystem.readFile,
  521. this.fileSystem.readFileSync,
  522. this.fileSystem
  523. );
  524. const readFile = this._readFileBackend.provide;
  525. this.readFile = /** @type {FileSystem["readFile"]} */ (readFile);
  526. const readFileSync = this._readFileBackend.provideSync;
  527. this.readFileSync = /** @type {SyncFileSystem["readFileSync"]} */ (
  528. readFileSync
  529. );
  530. this._readJsonBackend = createBackend(
  531. duration,
  532. // prettier-ignore
  533. this.fileSystem.readJson ||
  534. (this.readFile &&
  535. (
  536. /**
  537. * @param {string} path path
  538. * @param {FileSystemCallback<any>} callback
  539. */
  540. (path, callback) => {
  541. this.readFile(path, (err, buffer) => {
  542. if (err) return callback(err);
  543. if (!buffer || buffer.length === 0)
  544. return callback(new Error("No file content"));
  545. let data;
  546. try {
  547. data = JSON.parse(buffer.toString("utf-8"));
  548. } catch (e) {
  549. return callback(/** @type {Error} */ (e));
  550. }
  551. callback(null, data);
  552. });
  553. })
  554. ),
  555. // prettier-ignore
  556. this.fileSystem.readJsonSync ||
  557. (this.readFileSync &&
  558. (
  559. /**
  560. * @param {string} path path
  561. * @returns {any} result
  562. */
  563. (path) => {
  564. const buffer = this.readFileSync(path);
  565. const data = JSON.parse(buffer.toString("utf-8"));
  566. return data;
  567. }
  568. )),
  569. this.fileSystem
  570. );
  571. const readJson = this._readJsonBackend.provide;
  572. this.readJson = /** @type {FileSystem["readJson"]} */ (readJson);
  573. const readJsonSync = this._readJsonBackend.provideSync;
  574. this.readJsonSync = /** @type {SyncFileSystem["readJsonSync"]} */ (
  575. readJsonSync
  576. );
  577. this._readlinkBackend = createBackend(
  578. duration,
  579. this.fileSystem.readlink,
  580. this.fileSystem.readlinkSync,
  581. this.fileSystem
  582. );
  583. const readlink = this._readlinkBackend.provide;
  584. this.readlink = /** @type {FileSystem["readlink"]} */ (readlink);
  585. const readlinkSync = this._readlinkBackend.provideSync;
  586. this.readlinkSync = /** @type {SyncFileSystem["readlinkSync"]} */ (
  587. readlinkSync
  588. );
  589. this._realpathBackend = createBackend(
  590. duration,
  591. this.fileSystem.realpath,
  592. this.fileSystem.realpathSync,
  593. this.fileSystem
  594. );
  595. const realpath = this._realpathBackend.provide;
  596. this.realpath = /** @type {FileSystem["realpath"]} */ (realpath);
  597. const realpathSync = this._realpathBackend.provideSync;
  598. this.realpathSync = /** @type {SyncFileSystem["realpathSync"]} */ (
  599. realpathSync
  600. );
  601. }
  602. /**
  603. * @param {string | Buffer | URL | number | (string | URL | Buffer | number)[] | Set<string | URL | Buffer | number>} [what] what to purge
  604. */
  605. purge(what) {
  606. this._statBackend.purge(what);
  607. this._lstatBackend.purge(what);
  608. this._readdirBackend.purgeParent(what);
  609. this._readFileBackend.purge(what);
  610. this._readlinkBackend.purge(what);
  611. this._readJsonBackend.purge(what);
  612. this._realpathBackend.purge(what);
  613. }
  614. };