diff --git a/benchmark/fs/bench-watch-recursive.js b/benchmark/fs/bench-watch-recursive.js new file mode 100644 index 000000000000..45ed0871718e --- /dev/null +++ b/benchmark/fs/bench-watch-recursive.js @@ -0,0 +1,23 @@ +'use strict'; + +// Setting up (and tearing down) a recursive fs.watch() on a directory tree. +// On Linux and other platforms without a native recursive watcher this is +// implemented in JavaScript on top of per-directory watchers. + +const common = require('../common'); +const fs = require('fs'); +const path = require('path'); + +const bench = common.createBenchmark(main, { + n: [5], + dir: ['lib', 'test/fixtures'], +}); + +function main({ n, dir }) { + const fullPath = path.resolve(__dirname, '../../', dir); + bench.start(); + for (let i = 0; i < n; i++) { + fs.watch(fullPath, { recursive: true }).close(); + } + bench.end(n); +} diff --git a/lib/internal/fs/recursive_watch.js b/lib/internal/fs/recursive_watch.js index 2f71aeda9f79..c193247f03cd 100644 --- a/lib/internal/fs/recursive_watch.js +++ b/lib/internal/fs/recursive_watch.js @@ -17,7 +17,7 @@ const { }, } = require('internal/errors'); const { getValidatedPath } = require('internal/fs/utils'); -const { createIgnoreMatcher, kFSWatchStart, StatWatcher } = require('internal/fs/watchers'); +const { createIgnoreMatcher, kFSWatchStart } = require('internal/fs/watchers'); const { kEmptyObject } = require('internal/util'); const { validateBoolean, validateAbortSignal, validateIgnoreOption } = require('internal/validators'); const { @@ -40,11 +40,12 @@ let kResistStopPropagation; class FSWatcher extends EventEmitter { #options = null; #closed = false; - #files = new SafeMap(); + // Every path below the root that has been reported (or existed at start). + #entries = new SafeSet(); + // One fs.watch() per directory, plus one per symbolic link. #watchers = new SafeMap(); - #symbolicFiles = new SafeSet(); + #symbolicLinks = new SafeSet(); #rootPath = pathResolve(); - #watchingFile = false; #ignoreMatcher = null; constructor(options = kEmptyObject) { @@ -94,129 +95,147 @@ class FSWatcher extends EventEmitter { this.#closed = true; - for (const file of this.#files.keys()) { - this.#watchers.get(file)?.close(); - this.#watchers.delete(file); + for (const watcher of this.#watchers.values()) { + watcher.close(); } - - this.#files.clear(); - this.#symbolicFiles.clear(); + this.#watchers.clear(); + this.#entries.clear(); + this.#symbolicLinks.clear(); this.emit('close'); } - #unwatchFiles(file) { - this.#symbolicFiles.delete(file); + #emit(eventType, file) { + this.emit('change', eventType, pathRelative(this.#rootPath, file)); + } + #forget(file) { const childPrefix = file + pathSep; - for (const filename of this.#files.keys()) { - if (filename === file || - StringPrototypeStartsWith(filename, childPrefix)) { - this.#files.delete(filename); - this.#watchers.get(filename)?.close(); - this.#watchers.delete(filename); + for (const entry of this.#entries) { + if (entry === file || StringPrototypeStartsWith(entry, childPrefix)) { + this.#entries.delete(entry); + this.#symbolicLinks.delete(entry); + const watcher = this.#watchers.get(entry); + if (watcher !== undefined) { + watcher.close(); + this.#watchers.delete(entry); + } } } } - #watchFolder(folder) { - const { readdirSync } = lazyLoadFsSync(); + #watch(file, onChange) { + if (this.#closed || this.#watchers.has(file)) { + return; + } + const { watch } = lazyLoadFsSync(); + const watcher = watch(file, { persistent: this.#options.persistent }, onChange); + this.#watchers.set(file, watcher); + } + // Registers the entries of `folder` that are not known yet (emitting + // 'rename' for them unless this is the initial scan) and arms one watcher + // for the directory; #addEntry() descends into subdirectories. + #scanFolder(folder, initial) { + const { readdirSync } = lazyLoadFsSync(); + let entries; try { - const files = readdirSync(folder, { - withFileTypes: true, - }); - - for (const file of files) { - if (this.#closed) { - break; - } + entries = readdirSync(folder, { withFileTypes: true }); + } catch (error) { + if (error.code !== 'ENOENT') { + this.emit('error', error); + } + return; + } - const f = pathJoin(folder, file.name); - const relativePath = pathRelative(this.#rootPath, f); + this.#watch(folder, (eventType, filename) => this.#onFolderEvent(folder, filename)); - // Skip watching ignored paths entirely to avoid kernel resource pressure - if (this.#ignoreMatcher?.(relativePath)) { - continue; - } + for (const entry of entries) { + if (this.#closed) { + break; + } + const file = pathJoin(folder, entry.name); + if (!this.#entries.has(file) && !this.#ignoreMatcher?.(pathRelative(this.#rootPath, file))) { + this.#addEntry(file, entry, initial); + } + } + } - if (!this.#files.has(f)) { - this.emit('change', 'rename', relativePath); - - if (file.isSymbolicLink()) { - this.#symbolicFiles.add(f); - } - - try { - this.#watchFile(f); - if (file.isDirectory() && !file.isSymbolicLink()) { - this.#watchFolder(f); - } - } catch (err) { - // Ignore ENOENT - if (err.code !== 'ENOENT') { - throw err; - } - } + // `entry` is the Dirent or the lstat() Stats of `file`. + #addEntry(file, entry, initial) { + this.#entries.add(file); + if (!initial) { + this.#emit('rename', file); + } + if (entry.isSymbolicLink()) { + // The link target is watched so that changes behind the link surface + // as a 'rename' of the link, as they always have on this code path. + this.#symbolicLinks.add(file); + try { + this.#watch(file, () => this.#emit('rename', file)); + } catch (err) { + if (err.code !== 'ENOENT') { + throw err; } } - } catch (error) { - if (error.code !== 'ENOENT') { - this.emit('error', error); - } + } else if (entry.isDirectory()) { + this.#scanFolder(file, initial); } } - #watchFile(file) { + #onFolderEvent(folder, filename) { if (this.#closed) { return; } - - const { watch, statSync } = lazyLoadFsSync(); - - if (this.#files.has(file)) { + const { lstatSync, statSync } = lazyLoadFsSync(); + if (filename == null) { + this.#scanFolder(folder, false); return; } + // Events about the watched directory itself are reported under its own + // name; those take the "unknown entry" path and are resolved by the parent. + const file = pathJoin(folder, filename); - { - const existingStat = statSync(file); - this.#files.set(file, existingStat); + if (!this.#entries.has(file)) { + if (this.#ignoreMatcher?.(pathRelative(this.#rootPath, file))) { + return; + } + const entry = lstatSync(file, { throwIfNoEntry: false }); + if (entry !== undefined) { + this.#addEntry(file, entry, false); + } else if (folder === this.#rootPath && statSync(folder, { throwIfNoEntry: false }) === undefined) { + this.#emit('rename', folder); + this.#forget(folder); + } + return; } - const watcher = watch(file, { - persistent: this.#options.persistent, - }, (eventType, filename) => { - const existingStat = this.#files.get(file); - let currentStats; + const stats = statSync(file, { throwIfNoEntry: false }); + if (stats === undefined) { + this.#emit('rename', file); + this.#forget(file); + } else if (this.#symbolicLinks.has(file)) { + this.#emit('rename', file); + } else if (stats.isDirectory()) { + this.#scanFolder(file, false); + } else { + this.#emit('change', file); + } + } - try { - currentStats = statSync(file); - this.#files.set(file, currentStats); - } catch { - // This happens if the file was removed + #watchRootFile(file) { + const { statSync } = lazyLoadFsSync(); + this.#entries.add(file); + this.#watch(file, () => { + if (this.#closed) { + return; } - - if (currentStats === undefined || (currentStats.birthtimeMs === 0 && existingStat.birthtimeMs !== 0)) { - // The file is now deleted - this.#files.delete(file); - this.#watchers.delete(file); - watcher.close(); - this.emit('change', 'rename', pathRelative(this.#rootPath, file)); - this.#unwatchFiles(file); - } else if (file === this.#rootPath && this.#watchingFile) { - // This case will only be triggered when watching a file with fs.watch - this.emit('change', 'change', pathBasename(file)); - } else if (this.#symbolicFiles.has(file)) { - // Stats from watchFile does not return correct value for currentStats.isSymbolicLink() - // Since it is only valid when using fs.lstat(). Therefore, check the existing symbolic files. - this.emit('change', 'rename', pathRelative(this.#rootPath, file)); - } else if (currentStats.isDirectory()) { - this.#watchFolder(file); + if (statSync(file, { throwIfNoEntry: false }) === undefined) { + this.#emit('rename', file); + this.#forget(file); } else { - // Watching a directory will trigger a change event for child files) - this.emit('change', 'change', pathRelative(this.#rootPath, file)); + this.emit('change', 'change', pathBasename(file)); } }); - this.#watchers.set(file, watcher); } [kFSWatchStart](filename) { @@ -227,11 +246,11 @@ class FSWatcher extends EventEmitter { this.#rootPath = filename; this.#closed = false; - this.#watchingFile = file.isFile(); - this.#watchFile(filename); if (file.isDirectory()) { - this.#watchFolder(filename); + this.#scanFolder(filename, true); + } else { + this.#watchRootFile(filename); } } catch (error) { if (!this.#options.throwIfNoEntry && error.code === 'ENOENT') { @@ -243,19 +262,15 @@ class FSWatcher extends EventEmitter { } ref() { - this.#files.forEach((file) => { - if (file instanceof StatWatcher) { - file.ref(); - } - }); + for (const watcher of this.#watchers.values()) { + watcher.ref(); + } } unref() { - this.#files.forEach((file) => { - if (file instanceof StatWatcher) { - file.unref(); - } - }); + for (const watcher of this.#watchers.values()) { + watcher.unref(); + } } [SymbolAsyncIterator]() { diff --git a/test/parallel/test-fs-watch-recursive-linux-directory-watchers.js b/test/parallel/test-fs-watch-recursive-linux-directory-watchers.js new file mode 100644 index 000000000000..bb68893f2d40 --- /dev/null +++ b/test/parallel/test-fs-watch-recursive-linux-directory-watchers.js @@ -0,0 +1,108 @@ +'use strict'; +const common = require('../common'); + +// This tests the JavaScript recursive fs.watch() implementation (used where +// there is no native one, e.g. Linux): handle count, event reporting, ref/unref. + +if (!common.isLinux) + common.skip('the recursive watcher is native on this platform'); + +const assert = require('assert'); +const fs = require('fs'); +const path = require('path'); +const { setTimeout: wait } = require('timers/promises'); +const tmpdir = require('../common/tmpdir'); + +tmpdir.refresh(); +const delay = () => wait(common.platformTimeout(100)); + +function countFSEventHandles() { + return process.getActiveResourcesInfo().filter((type) => type === 'FSEventWrap').length; +} + +// Collects events until `done(events)` returns true, then closes the watcher. +function watchUntil(target, done) { + return new Promise((resolve) => { + const events = []; + const watcher = fs.watch(target, { recursive: true }); + watcher.on('change', (eventType, filename) => { + events.push(`${eventType} ${filename}`); + if (done(events)) { + watcher.close(); + resolve(events); + } + }); + }); +} + +(async () => { + const root = tmpdir.resolve('root'); + fs.mkdirSync(path.join(root, 'sub'), { recursive: true }); + for (let i = 0; i < 20; i++) { + fs.writeFileSync(path.join(root, `file-${i}.txt`), 'x'); + fs.writeFileSync(path.join(root, 'sub', `file-${i}.txt`), 'x'); + } + fs.symlinkSync(path.join(root, 'sub', 'file-1.txt'), path.join(root, 'link')); + + { + // One handle per directory (root, sub) plus one for the symbolic link. + const before = countFSEventHandles(); + const watcher = fs.watch(root, { recursive: true }); + assert.strictEqual(countFSEventHandles() - before, 3); + watcher.unref(); + watcher.ref(); + watcher.close(); + } + + { + // A file replaced by rename() keeps being reported. + const file = path.join(root, 'sub', 'file-0.txt'); + const changed = `change ${path.join('sub', 'file-0.txt')}`; + const events = watchUntil(root, (seen) => seen.filter((e) => e === changed).length === 2); + await delay(); + fs.writeFileSync(`${file}.tmp`, 'replaced'); + fs.renameSync(`${file}.tmp`, file); + await delay(); + fs.appendFileSync(file, ' and modified'); + await events; + } + + { + // A write behind a symbolic link is reported as a rename of the link, and + // touching a known subdirectory itself does not throw or emit. + const events = watchUntil(root, (seen) => seen.includes('rename link')); + await delay(); + fs.chmodSync(path.join(root, 'sub'), 0o775); + fs.appendFileSync(path.join(root, 'sub', 'file-1.txt'), 'more'); + assert.ok(!(await events).some((e) => e.endsWith(' sub'))); + } + + { + // Removing the watched root directory is reported. + const doomed = tmpdir.resolve('doomed'); + fs.mkdirSync(doomed); + fs.writeFileSync(path.join(doomed, 'inside'), 'x'); + const events = watchUntil(doomed, (seen) => seen.includes('rename ')); + await delay(); + fs.rmSync(doomed, { recursive: true }); + await events; + } + + { + // So is removing a watched root that is a file (with an empty filename, + // as before). + const lone = tmpdir.resolve('lone.txt'); + fs.writeFileSync(lone, 'x'); + const events = watchUntil(lone, (seen) => seen.includes('rename ')); + await delay(); + fs.rmSync(lone); + await events; + } + + { + const watcher = fs.watch(root, { recursive: true }); + watcher.unref(); + // The process exits although this watcher is never closed. + process.on('exit', () => assert.ok(watcher)); + } +})().then(common.mustCall());