From aafa50cc0d503456355338d323c8088c65af02d5 Mon Sep 17 00:00:00 2001 From: hugoer Date: Fri, 4 Sep 2026 12:16:43 +0200 Subject: [PATCH 1/3] feat(utils): slug the URL path into output filenames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buildFilename keyed output names on hostname alone, so every page of one host collided and the _NN tiebreaker carried no page identity. runBatch records results in completion order rather than input order, so _NN did not even correspond to a URL's position in the list — the number a page received varied between runs of the same command. urlSlug derives a filename-safe slug from the path. Query strings and fragments are dropped, so tracking parameters do not produce a different filename for the same page. A root path yields an empty slug, which leaves single-page runs and origin-scoped crux/crux-history byte-identical to before. Two details that are not obvious: Percent-encoding is decoded and Unicode letters are kept, rather than slugging with [a-z0-9]. `pathname` percent-encodes non-ASCII, so an ASCII-only rule emits the UTF-8 bytes as text — /es/zapatos-de-niño becomes es-zapatos-de-ni-c3-b1o, and a non-Latin path becomes hex carrying no page identity at all. The cap is 80 bytes, not 80 characters, and truncation keeps the path's tail. 80 CJK characters is 240 bytes, which would exceed the 255-byte filename limit on its own. Keeping the tail matters because URLs are hierarchical: the shared part is the prefix, so keeping the head would collapse exactly the deep-category pages most likely to be audited together. A 6-char hash of the full path is appended once truncated, which makes the name stable across runs where _NN is not. _NN is retained as the last-resort tiebreaker. urlSlug is exported from web-perf-cli/utils so a consumer can predict a path. --- lib/utils.js | 66 +++++++++++++++++++++++++++++++- lib/utils.test.js | 91 ++++++++++++++++++++++++++++++++++++++++++++ types/lib/utils.d.ts | 18 +++++++++ 3 files changed, 173 insertions(+), 2 deletions(-) diff --git a/lib/utils.js b/lib/utils.js index d2b396a..bdd545c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,4 +1,5 @@ /* eslint-disable no-await-in-loop */ +const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); @@ -23,10 +24,70 @@ function formatDate() { return `${date}-${hours}${mins}${secs}`; } +// Filenames are keyed on hostname, so every page of one site used to collide and the `_NN` +// tiebreaker carried no page identity. A path slug restores it. Capped in BYTES rather than +// characters: the slug keeps Unicode letters, and 80 CJK characters is 240 bytes, which alone +// would blow the 255-byte filename limit every mainstream filesystem enforces. +const SLUG_MAX_BYTES = 80; +const SLUG_HASH_LENGTH = 6; + +/** + * Builds the filename-safe slug for a URL's path, so two pages of one host produce two + * distinguishable filenames. + * + * Query strings and fragments are dropped, which keeps tracking parameters (`utm_source`, + * `gclid`) from producing a different filename for the same page. A root path yields an empty + * slug, so single-page and origin-scoped runs keep the filenames they have always had. + * + * Truncation keeps the END of the path and appends a short hash. URLs are hierarchical, so the + * shared part is the prefix and the discriminating part is the suffix — keeping the head would + * collapse exactly the deep-category pages most likely to be audited together. The hash makes a + * truncated slug stable across runs, which the `_NN` fallback is not: `runBatch` records results + * in completion order, so the number a given URL receives varies run to run. + * + * @param {string} url - absolute URL + * @returns {string} the slug, or '' for a root path + */ +function urlSlug(url) { + const raw = new URL(url).pathname; + let decoded; + try { + decoded = decodeURIComponent(raw); + } catch { + // A malformed percent sequence (`/%zz`) throws rather than decoding. The raw pathname + // is still a usable slug source, so fall back to it instead of failing the whole run. + decoded = raw; + } + // \p{L}/\p{N} rather than a-z0-9: `pathname` percent-encodes non-ASCII, so an ASCII-only + // rule turns `/es/zapatos-de-niño` into `es-zapatos-de-ni-c3-b1o` and any non-Latin script + // into unreadable hex that carries no page identity at all. + const full = decoded.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, ''); + if (full === '' || Buffer.byteLength(full) <= SLUG_MAX_BYTES) { + return full; + } + const hash = crypto.createHash('sha256').update(decoded).digest('hex').slice(0, SLUG_HASH_LENGTH); + const budget = SLUG_MAX_BYTES - (SLUG_HASH_LENGTH + 1); + // Walk backwards a character at a time so a multi-byte character is never cut in half. + const chars = [...full]; + let tail = ''; + let bytes = 0; + for (let i = chars.length - 1; i >= 0; i--) { + const size = Buffer.byteLength(chars[i]); + if (bytes + size > budget) { + break; + } + tail = chars[i] + tail; + bytes += size; + } + return `${tail.replace(/^-+/, '')}-${hash}`; +} + function buildFilename(url, command, suffix, ext = 'json') { - const hostname = new URL(url).hostname; + const { hostname } = new URL(url); + const slug = urlSlug(url); + const slugPart = slug ? `-${slug}` : ''; const suffixPart = suffix ? `-${suffix}` : ''; - const base = path.join(RESULTS_DIR, command, `${command}-${hostname}-${formatDate()}${suffixPart}`); + const base = path.join(RESULTS_DIR, command, `${command}-${hostname}${slugPart}-${formatDate()}${suffixPart}`); if (!fs.existsSync(`${base}.${ext}`)) { return `${base}.${ext}`; } @@ -265,6 +326,7 @@ async function runBatch(items, auditFn, { maxRequestsPerSecond, concurrency = 5, module.exports = { ensureCommandDir, buildFilename, + urlSlug, formatDate, formatElapsed, normalizeOrigin, diff --git a/lib/utils.test.js b/lib/utils.test.js index ee75cd4..411d580 100644 --- a/lib/utils.test.js +++ b/lib/utils.test.js @@ -6,6 +6,7 @@ const path = require('path'); const { formatDate, buildFilename, + urlSlug, formatElapsed, normalizeUrlForAi, normalizeUrlsToOrigins, @@ -116,6 +117,28 @@ describe('utils', () => { expect(result).toContain('rum-my-site.org-'); }); + it('should place the path slug between hostname and timestamp', () => { + existsSyncSpy.mockReturnValue(false); + const result = buildFilename('https://a.com/es/page-one', 'psi', 'mobile'); + expect(path.basename(result)).toBe('psi-a.com-es-page-one-2026-04-01-123456-mobile.json'); + }); + + it('should leave a root URL byte-identical to the pre-slug format', () => { + existsSyncSpy.mockReturnValue(false); + const withSlash = buildFilename('https://a.com/', 'psi', 'mobile'); + const without = buildFilename('https://a.com', 'psi', 'mobile'); + expect(path.basename(withSlash)).toBe('psi-a.com-2026-04-01-123456-mobile.json'); + expect(without).toBe(withSlash); + }); + + it('should still append _NN when two paths slug identically', () => { + const base = path.join(RESULTS_DIR, 'psi', 'psi-a.com-es-page-2026-04-01-123456'); + existsSyncSpy.mockImplementation((f) => f === `${base}.json`); + // Same page, different tracking parameters: the query is dropped, so both slug alike. + const result = buildFilename('https://a.com/es/page?utm_source=x', 'psi'); + expect(result).toBe(`${base}_01.json`); + }); + it('should include the command prefix in the filename', () => { existsSyncSpy.mockReturnValue(false); const result = buildFilename('https://example.com', 'collect'); @@ -148,6 +171,74 @@ describe('utils', () => { }); }); + describe('urlSlug', () => { + it('should return an empty slug for a root path', () => { + expect(urlSlug('https://a.com/')).toBe(''); + expect(urlSlug('https://a.com')).toBe(''); + }); + + it('should join path segments with single dashes', () => { + expect(urlSlug('https://a.com/es/productos/zapatos')).toBe('es-productos-zapatos'); + }); + + it('should lowercase and collapse runs of separators', () => { + expect(urlSlug('https://a.com/ES//Page__One/')).toBe('es-page-one'); + }); + + it('should drop the query string and the fragment', () => { + expect(urlSlug('https://a.com/es/page?utm_source=x&gclid=y')).toBe('es-page'); + expect(urlSlug('https://a.com/es/page#top')).toBe('es-page'); + }); + + // pathname percent-encodes non-ASCII. Slugging the encoded form would emit the UTF-8 + // bytes as text: /es/zapatos-de-niño -> es-zapatos-de-ni-c3-b1o. + it('should decode percent-encoding rather than slugging the hex', () => { + expect(urlSlug('https://a.com/es/zapatos-de-niño')).toBe('es-zapatos-de-niño'); + expect(urlSlug('https://a.com/es/año-nuevo')).toBe('es-año-nuevo'); + }); + + it('should keep non-Latin scripts readable and distinct', () => { + expect(urlSlug('https://a.com/日本語/ページ')).toBe('日本語-ページ'); + expect(urlSlug('https://a.com/日本語/製品')).toBe('日本語-製品'); + expect(urlSlug('https://a.com/ru/страница')).toBe('ru-страница'); + }); + + it('should fall back to the raw pathname on a malformed percent sequence', () => { + expect(urlSlug('https://a.com/es/%zz-page')).toBe('es-zz-page'); + }); + + // The discriminating part of a hierarchical URL is its tail, so truncation keeps the end. + it('should keep the tail when truncating, so sibling pages stay distinct', () => { + const a = urlSlug(`https://a.com/es/${'x'.repeat(90)}/hombre`); + const b = urlSlug(`https://a.com/es/${'x'.repeat(90)}/mujer`); + expect(a).toContain('hombre'); + expect(b).toContain('mujer'); + expect(a).not.toBe(b); + }); + + it('should append a hash only once the slug is truncated', () => { + expect(urlSlug('https://a.com/es/page-one')).toBe('es-page-one'); + expect(urlSlug(`https://a.com/${'x'.repeat(200)}`)).toMatch(/-[0-9a-f]{6}$/); + }); + + it('should produce a stable hash for the same path', () => { + const url = `https://a.com/${'x'.repeat(200)}`; + expect(urlSlug(url)).toBe(urlSlug(url)); + }); + + // A character cap would not hold: 80 CJK characters is 240 bytes on its own. + it('should cap the slug in bytes without splitting a multi-byte character', () => { + const slug = urlSlug(`https://a.com/${'製品'.repeat(60)}`); + expect(Buffer.byteLength(slug)).toBeLessThanOrEqual(80); + expect(slug).not.toContain('\uFFFD'); + expect(slug).toBe([...slug].join('')); + }); + + it('should throw on an invalid URL', () => { + expect(() => urlSlug('not-a-url')).toThrow(); + }); + }); + describe('buildFilename with custom ext', () => { let existsSyncSpy; diff --git a/types/lib/utils.d.ts b/types/lib/utils.d.ts index 0bbec63..032679f 100644 --- a/types/lib/utils.d.ts +++ b/types/lib/utils.d.ts @@ -1,5 +1,23 @@ export function ensureCommandDir(command: any): void; export function buildFilename(url: any, command: any, suffix: any, ext?: string): string; +/** + * Builds the filename-safe slug for a URL's path, so two pages of one host produce two + * distinguishable filenames. + * + * Query strings and fragments are dropped, which keeps tracking parameters (`utm_source`, + * `gclid`) from producing a different filename for the same page. A root path yields an empty + * slug, so single-page and origin-scoped runs keep the filenames they have always had. + * + * Truncation keeps the END of the path and appends a short hash. URLs are hierarchical, so the + * shared part is the prefix and the discriminating part is the suffix — keeping the head would + * collapse exactly the deep-category pages most likely to be audited together. The hash makes a + * truncated slug stable across runs, which the `_NN` fallback is not: `runBatch` records results + * in completion order, so the number a given URL receives varies run to run. + * + * @param {string} url - absolute URL + * @returns {string} the slug, or '' for a root path + */ +export function urlSlug(url: string): string; export function formatDate(): string; export function formatElapsed(ms: any): string; export function normalizeOrigin(url: any): string; From 02a9e2c5b4073105dae1afc4e426682c73ed93d1 Mon Sep 17 00:00:00 2001 From: hugoer Date: Fri, 4 Sep 2026 12:16:51 +0200 Subject: [PATCH 2/3] test(types): cover the utils subpath in the consumer guard consumer.ts imported nothing from web-perf-cli/utils, so the published declarations for that subpath were unchecked and urlSlug shipped uncovered. Asserts the return types and that urlSlug is not nullable, so a consumer never has to narrow it. --- type-tests/consumer.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/type-tests/consumer.ts b/type-tests/consumer.ts index 8c11d96..009bb27 100644 --- a/type-tests/consumer.ts +++ b/type-tests/consumer.ts @@ -12,6 +12,7 @@ import { runLabPlan, runLabAudit, runLabToDisk } from '@hugoer/web-perf-cli/lab' import { runCrux, runCruxAudit, runCruxBatch, DEFAULT_CRUX_FORM_FACTORS } from '@hugoer/web-perf-cli/crux'; import { runCruxHistoryAudit } from '@hugoer/web-perf-cli/crux-history'; import { buildRunSummary } from '@hugoer/web-perf-cli/variance'; +import { urlSlug, buildFilename } from '@hugoer/web-perf-cli/utils'; import type { LabReport, LabPlanResult } from '@hugoer/web-perf-cli/lab'; import type { CruxReport } from '@hugoer/web-perf-cli/crux'; import type { CruxHistoryReport } from '@hugoer/web-perf-cli/crux-history'; @@ -120,3 +121,11 @@ export const rejectsPort = runLabPlan(['u'], [{ profile: 'low' }], { }); export const audit = runLabAudit('https://example.com', { profile: 'low', silent: true }); + +// --- urlSlug and buildFilename are typed, so a consumer can predict an output path ---------- +export const slug: string = urlSlug('https://a.com/es/page-one'); +export const filename: string = buildFilename('https://a.com/es/page-one', 'psi', 'mobile'); +export const noSuffix: string = buildFilename('https://a.com', 'sitemap', undefined, 'txt'); + +// @ts-expect-error urlSlug returns a string, never null — callers should not have to narrow it +export const notNullable: null = urlSlug('https://a.com/'); From e7fd37b3b9d5cfcd5c26f0f71f518e08d70a3456 Mon Sep 17 00:00:00 2001 From: hugoer Date: Fri, 4 Sep 2026 12:16:51 +0200 Subject: [PATCH 3/3] docs: document the path slug in output filenames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The filename formats are written out in both README.md and CLAUDE.md and have drifted before, so both carry the [-] segment and the rules behind it: query and fragment dropped, percent-encoding decoded, Unicode letters kept, an 80-byte cap that keeps the path's tail with a hash. States plainly that a root path adds no segment, since the practical question for anyone with an existing results/ directory is whether their filenames changed — for single-page and origin-scoped runs they did not. Adds urlSlug to the library API table now that it is exported. --- CLAUDE.md | 32 ++++++++++++++++++++++++-------- README.md | 42 +++++++++++++++++++++++++++++++++++------- 2 files changed, 59 insertions(+), 15 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f99e561..d9ad838 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,18 +137,34 @@ module restates its signature to keep the published `.d.ts` describing a real Cr Each command writes to its own subdirectory under `results/`: -- `results/lab/` — lab (format: `lab--YYYY-MM-DD-HHMMSS-.json`) +- `results/lab/` — lab (format: `lab-[-]-YYYY-MM-DD-HHMMSS-.json`) - `results/lab/` — with `--runs=N`, each run gets a `-runNN` suffix plus one - `lab--YYYY-MM-DD-HHMMSS-.summary.json` per (URL x profile) pair. + `lab-[-]-YYYY-MM-DD-HHMMSS-.summary.json` per (URL x profile) pair. The summary is named after the group's FIRST run, so it sorts alongside `-run01`. -- `results/lab/clean/` — AI-friendly lab output when `--clean` is used (format: `lab--YYYY-MM-DD-HHMMSS-.clean.json`) -- `results/psi/` — psi (format: `psi--YYYY-MM-DD-HHMMSS-.json`, one file per strategy) -- `results/psi/clean/` — AI-friendly psi output when `--clean` is used (format: `psi--YYYY-MM-DD-HHMMSS-.clean.json`) -- `results/crux/` — crux (format: `crux--YYYY-MM-DD-HHMMSS-.json`, one file per form factor) -- `results/crux-history/` — crux-history (format: `crux-history--YYYY-MM-DD-HHMMSS-.json`, one file per form factor) -- `results/links/` — links (format: `links--YYYY-MM-DD-HHMMSS.json`) +- `results/lab/clean/` — AI-friendly lab output when `--clean` is used (format: `lab-[-]-YYYY-MM-DD-HHMMSS-.clean.json`) +- `results/psi/` — psi (format: `psi-[-]-YYYY-MM-DD-HHMMSS-.json`, one file per strategy) +- `results/psi/clean/` — AI-friendly psi output when `--clean` is used (format: `psi-[-]-YYYY-MM-DD-HHMMSS-.clean.json`) +- `results/crux/` — crux (format: `crux-[-]-YYYY-MM-DD-HHMMSS-.json`, one file per form factor) +- `results/crux-history/` — crux-history (format: `crux-history-[-]-YYYY-MM-DD-HHMMSS-.json`, one file per form factor) +- `results/links/` — links (format: `links-[-]-YYYY-MM-DD-HHMMSS.json`) - `results/sitemap/` — sitemap (format: `sitemap--YYYY-MM-DD-HHMMSS.json`) +`` is the URL's path, slugged by `urlSlug` (`lib/utils.js`) so two pages of one host do +not produce two filenames distinguishable only by a `_NN` counter. Rules: query string and +fragment dropped, percent-encoding decoded, lowercased, every run of non-letter/non-digit +collapsed to one `-`. Unicode letters survive, so `/es/zapatos-de-niño` slugs to +`es-zapatos-de-niño` rather than to the UTF-8 hex of its bytes. + +**A root path yields no slug segment at all**, so single-page runs and origin-scoped +`crux`/`crux-history` produce byte-identical filenames to before the slug existed. `sitemap` +is passed an origin, so it is unaffected. + +The slug is capped at 80 **bytes** — not characters, since 80 CJK characters is 240 bytes and +would exceed the 255-byte filename limit on its own. Past the cap it keeps the path's tail (the +discriminating end of a hierarchical URL) and appends a 6-char hash of the full path. The hash +is what makes a truncated name stable run to run; the `_NN` fallback is not, because `runBatch` +records results in completion order. + ## Environment Variables | Variable | Command | Description | diff --git a/README.md b/README.md index c7be215..6a5553f 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ that fail are excluded from the statistics and listed under `errors`. Scores are never rescaled after the fact: the variance is reported, not corrected. -**Output:** `results/lab/lab--YYYY-MM-DD-HHMMSS-.json` +**Output:** `results/lab/lab-[-]-YYYY-MM-DD-HHMMSS-.json` With `--runs=N`, each run is written as `...--runNN.json` plus one `...-.summary.json` per (URL x profile) pair, named after the group's first run. @@ -283,7 +283,7 @@ web-perf psi --category=performance,seo --api-key-path= 4. `WEB_PERF_PSI_API_KEY_PATH` env var (file path) 5. Interactive prompt -**Output:** `results/psi/psi--YYYY-MM-DD-HHMMSS-.json` (one file per URL per strategy — default produces both `-mobile.json` and `-desktop.json`) +**Output:** `results/psi/psi-[-]-YYYY-MM-DD-HHMMSS-.json` (one file per URL per strategy — default produces both `-mobile.json` and `-desktop.json`) --- @@ -320,7 +320,7 @@ Built-in quota protection: CrUX request starts are capped at 2.5 requests/second \* Not required when `--urls` or `--urls-file` is provided. \*\* A CrUX API key is required. Provide via `--api-key`, `--api-key-path`, or the `WEB_PERF_PSI_API_KEY` / `WEB_PERF_PSI_API_KEY_PATH` environment variables. -**Output:** `results/crux/crux--YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) +**Output:** `results/crux/crux-[-]-YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) --- @@ -357,7 +357,7 @@ Built-in quota protection: CrUX History request starts are capped at 2.5 request \* Not required when `--urls` or `--urls-file` is provided. \*\* A CrUX API key is required. Credential resolution is identical to `crux` (see above). -**Output:** `results/crux-history/crux-history--YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) +**Output:** `results/crux-history/crux-history-[-]-YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) --- @@ -409,7 +409,7 @@ web-perf links --output-ai | `` | Yes | URL to extract links from | | `--output-ai` | No | Generate AI-friendly `.txt` output (one URL per line, normalized) | -**Output:** `results/links/links--YYYY-MM-DD-HHMMSS.json` +**Output:** `results/links/links-[-]-YYYY-MM-DD-HHMMSS.json` ### `clean` — AI-friendly output @@ -427,11 +427,38 @@ web-perf clean 'results/**/*.json' ``` Clean files are written to a `clean/` subfolder next to the raw output: -- `results/lab/clean/lab--YYYY-MM-DD-HHMMSS-.clean.json` -- `results/psi/clean/psi--YYYY-MM-DD-HHMMSS-.clean.json` +- `results/lab/clean/lab-[-]-YYYY-MM-DD-HHMMSS-.clean.json` +- `results/psi/clean/psi-[-]-YYYY-MM-DD-HHMMSS-.clean.json` The clean file is self-describing: `JSON.parse(cleanFile)._clean === true`. +## Output filenames and the path slug + +Output filenames carry a slug of the URL's path, so auditing several pages of one host produces +files you can tell apart without opening them: + +``` +https://a.com/ -> psi-a.com-2026-09-02-171221-mobile.json +https://a.com/es/page-one -> psi-a.com-es-page-one-2026-09-02-171221-mobile.json +https://a.com/es/productos/zapatos -> lab-a.com-es-productos-zapatos-2026-09-02-171221-low.json +https://a.com/es/page?utm_source=x -> psi-a.com-es-page-2026-09-02-171221-mobile.json +https://a.com/es/zapatos-de-niño -> psi-a.com-es-zapatos-de-niño-2026-09-02-171221-mobile.json +``` + +The query string and fragment are dropped, so tracking parameters do not produce a different +filename for the same page. Percent-encoding is decoded and Unicode letters are kept, so +non-Latin paths stay readable (`/日本語/ページ` → `日本語-ページ`) instead of becoming the hex of +their UTF-8 bytes. + +**A root path adds no slug segment**, so single-page runs and origin-scoped `crux` / +`crux-history` keep exactly the filenames they produced before. `sitemap` is passed an origin and +is likewise unchanged. + +Slugs are capped at 80 bytes. Past that the path's tail is kept — the discriminating end of a +hierarchical URL — and a 6-character hash of the full path is appended. Two URLs that still +collide fall back to a `_NN` counter, and each file's `url` field remains authoritative either +way. `urlSlug` is exported from `web-perf-cli/utils` if you need to predict a filename. + ## Environment variables | Variable | Command | Description | @@ -498,6 +525,7 @@ const { buildRunSummary } = require('@hugoer/web-perf-cli/varian | `selectMedianRun(scores)` | `web-perf-cli/variance` | `number` (index; lower median, `-1` if empty) | | `assessStability(benchmarkIndexes)` | `web-perf-cli/variance` | `{ stable: boolean, warnings: string[] }` | | `buildRunSummary(runs, context?)` | `web-perf-cli/variance` | `RunSummary` | +| `urlSlug(url)` | `web-perf-cli/utils` | `string` (path slug used in output filenames; `''` for a root path) | The `variance` helpers are pure too — they take report-shaped plain objects and do no I/O, so they can summarise runs collected by any means, not just this CLI's.