diff --git a/CLAUDE.md b/CLAUDE.md index f99e561..d9ad838 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -137,18 +137,34 @@ module restates its signature to keep the published `.d.ts` describing a real Cr Each command writes to its own subdirectory under `results/`: -- `results/lab/` — lab (format: `lab--YYYY-MM-DD-HHMMSS-.json`) +- `results/lab/` — lab (format: `lab-[-]-YYYY-MM-DD-HHMMSS-.json`) - `results/lab/` — with `--runs=N`, each run gets a `-runNN` suffix plus one - `lab--YYYY-MM-DD-HHMMSS-.summary.json` per (URL x profile) pair. + `lab-[-]-YYYY-MM-DD-HHMMSS-.summary.json` per (URL x profile) pair. The summary is named after the group's FIRST run, so it sorts alongside `-run01`. -- `results/lab/clean/` — AI-friendly lab output when `--clean` is used (format: `lab--YYYY-MM-DD-HHMMSS-.clean.json`) -- `results/psi/` — psi (format: `psi--YYYY-MM-DD-HHMMSS-.json`, one file per strategy) -- `results/psi/clean/` — AI-friendly psi output when `--clean` is used (format: `psi--YYYY-MM-DD-HHMMSS-.clean.json`) -- `results/crux/` — crux (format: `crux--YYYY-MM-DD-HHMMSS-.json`, one file per form factor) -- `results/crux-history/` — crux-history (format: `crux-history--YYYY-MM-DD-HHMMSS-.json`, one file per form factor) -- `results/links/` — links (format: `links--YYYY-MM-DD-HHMMSS.json`) +- `results/lab/clean/` — AI-friendly lab output when `--clean` is used (format: `lab-[-]-YYYY-MM-DD-HHMMSS-.clean.json`) +- `results/psi/` — psi (format: `psi-[-]-YYYY-MM-DD-HHMMSS-.json`, one file per strategy) +- `results/psi/clean/` — AI-friendly psi output when `--clean` is used (format: `psi-[-]-YYYY-MM-DD-HHMMSS-.clean.json`) +- `results/crux/` — crux (format: `crux-[-]-YYYY-MM-DD-HHMMSS-.json`, one file per form factor) +- `results/crux-history/` — crux-history (format: `crux-history-[-]-YYYY-MM-DD-HHMMSS-.json`, one file per form factor) +- `results/links/` — links (format: `links-[-]-YYYY-MM-DD-HHMMSS.json`) - `results/sitemap/` — sitemap (format: `sitemap--YYYY-MM-DD-HHMMSS.json`) +`` is the URL's path, slugged by `urlSlug` (`lib/utils.js`) so two pages of one host do +not produce two filenames distinguishable only by a `_NN` counter. Rules: query string and +fragment dropped, percent-encoding decoded, lowercased, every run of non-letter/non-digit +collapsed to one `-`. Unicode letters survive, so `/es/zapatos-de-niño` slugs to +`es-zapatos-de-niño` rather than to the UTF-8 hex of its bytes. + +**A root path yields no slug segment at all**, so single-page runs and origin-scoped +`crux`/`crux-history` produce byte-identical filenames to before the slug existed. `sitemap` +is passed an origin, so it is unaffected. + +The slug is capped at 80 **bytes** — not characters, since 80 CJK characters is 240 bytes and +would exceed the 255-byte filename limit on its own. Past the cap it keeps the path's tail (the +discriminating end of a hierarchical URL) and appends a 6-char hash of the full path. The hash +is what makes a truncated name stable run to run; the `_NN` fallback is not, because `runBatch` +records results in completion order. + ## Environment Variables | Variable | Command | Description | diff --git a/README.md b/README.md index c7be215..6a5553f 100644 --- a/README.md +++ b/README.md @@ -216,7 +216,7 @@ that fail are excluded from the statistics and listed under `errors`. Scores are never rescaled after the fact: the variance is reported, not corrected. -**Output:** `results/lab/lab--YYYY-MM-DD-HHMMSS-.json` +**Output:** `results/lab/lab-[-]-YYYY-MM-DD-HHMMSS-.json` With `--runs=N`, each run is written as `...--runNN.json` plus one `...-.summary.json` per (URL x profile) pair, named after the group's first run. @@ -283,7 +283,7 @@ web-perf psi --category=performance,seo --api-key-path= 4. `WEB_PERF_PSI_API_KEY_PATH` env var (file path) 5. Interactive prompt -**Output:** `results/psi/psi--YYYY-MM-DD-HHMMSS-.json` (one file per URL per strategy — default produces both `-mobile.json` and `-desktop.json`) +**Output:** `results/psi/psi-[-]-YYYY-MM-DD-HHMMSS-.json` (one file per URL per strategy — default produces both `-mobile.json` and `-desktop.json`) --- @@ -320,7 +320,7 @@ Built-in quota protection: CrUX request starts are capped at 2.5 requests/second \* Not required when `--urls` or `--urls-file` is provided. \*\* A CrUX API key is required. Provide via `--api-key`, `--api-key-path`, or the `WEB_PERF_PSI_API_KEY` / `WEB_PERF_PSI_API_KEY_PATH` environment variables. -**Output:** `results/crux/crux--YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) +**Output:** `results/crux/crux-[-]-YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) --- @@ -357,7 +357,7 @@ Built-in quota protection: CrUX History request starts are capped at 2.5 request \* Not required when `--urls` or `--urls-file` is provided. \*\* A CrUX API key is required. Credential resolution is identical to `crux` (see above). -**Output:** `results/crux-history/crux-history--YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) +**Output:** `results/crux-history/crux-history-[-]-YYYY-MM-DD-HHMMSS-.json` (one file per form factor — default produces both `-phone.json` and `-desktop.json`) --- @@ -409,7 +409,7 @@ web-perf links --output-ai | `` | Yes | URL to extract links from | | `--output-ai` | No | Generate AI-friendly `.txt` output (one URL per line, normalized) | -**Output:** `results/links/links--YYYY-MM-DD-HHMMSS.json` +**Output:** `results/links/links-[-]-YYYY-MM-DD-HHMMSS.json` ### `clean` — AI-friendly output @@ -427,11 +427,38 @@ web-perf clean 'results/**/*.json' ``` Clean files are written to a `clean/` subfolder next to the raw output: -- `results/lab/clean/lab--YYYY-MM-DD-HHMMSS-.clean.json` -- `results/psi/clean/psi--YYYY-MM-DD-HHMMSS-.clean.json` +- `results/lab/clean/lab-[-]-YYYY-MM-DD-HHMMSS-.clean.json` +- `results/psi/clean/psi-[-]-YYYY-MM-DD-HHMMSS-.clean.json` The clean file is self-describing: `JSON.parse(cleanFile)._clean === true`. +## Output filenames and the path slug + +Output filenames carry a slug of the URL's path, so auditing several pages of one host produces +files you can tell apart without opening them: + +``` +https://a.com/ -> psi-a.com-2026-09-02-171221-mobile.json +https://a.com/es/page-one -> psi-a.com-es-page-one-2026-09-02-171221-mobile.json +https://a.com/es/productos/zapatos -> lab-a.com-es-productos-zapatos-2026-09-02-171221-low.json +https://a.com/es/page?utm_source=x -> psi-a.com-es-page-2026-09-02-171221-mobile.json +https://a.com/es/zapatos-de-niño -> psi-a.com-es-zapatos-de-niño-2026-09-02-171221-mobile.json +``` + +The query string and fragment are dropped, so tracking parameters do not produce a different +filename for the same page. Percent-encoding is decoded and Unicode letters are kept, so +non-Latin paths stay readable (`/日本語/ページ` → `日本語-ページ`) instead of becoming the hex of +their UTF-8 bytes. + +**A root path adds no slug segment**, so single-page runs and origin-scoped `crux` / +`crux-history` keep exactly the filenames they produced before. `sitemap` is passed an origin and +is likewise unchanged. + +Slugs are capped at 80 bytes. Past that the path's tail is kept — the discriminating end of a +hierarchical URL — and a 6-character hash of the full path is appended. Two URLs that still +collide fall back to a `_NN` counter, and each file's `url` field remains authoritative either +way. `urlSlug` is exported from `web-perf-cli/utils` if you need to predict a filename. + ## Environment variables | Variable | Command | Description | @@ -498,6 +525,7 @@ const { buildRunSummary } = require('@hugoer/web-perf-cli/varian | `selectMedianRun(scores)` | `web-perf-cli/variance` | `number` (index; lower median, `-1` if empty) | | `assessStability(benchmarkIndexes)` | `web-perf-cli/variance` | `{ stable: boolean, warnings: string[] }` | | `buildRunSummary(runs, context?)` | `web-perf-cli/variance` | `RunSummary` | +| `urlSlug(url)` | `web-perf-cli/utils` | `string` (path slug used in output filenames; `''` for a root path) | The `variance` helpers are pure too — they take report-shaped plain objects and do no I/O, so they can summarise runs collected by any means, not just this CLI's. diff --git a/lib/utils.js b/lib/utils.js index d2b396a..bdd545c 100644 --- a/lib/utils.js +++ b/lib/utils.js @@ -1,4 +1,5 @@ /* eslint-disable no-await-in-loop */ +const crypto = require('crypto'); const fs = require('fs'); const path = require('path'); @@ -23,10 +24,70 @@ function formatDate() { return `${date}-${hours}${mins}${secs}`; } +// Filenames are keyed on hostname, so every page of one site used to collide and the `_NN` +// tiebreaker carried no page identity. A path slug restores it. Capped in BYTES rather than +// characters: the slug keeps Unicode letters, and 80 CJK characters is 240 bytes, which alone +// would blow the 255-byte filename limit every mainstream filesystem enforces. +const SLUG_MAX_BYTES = 80; +const SLUG_HASH_LENGTH = 6; + +/** + * Builds the filename-safe slug for a URL's path, so two pages of one host produce two + * distinguishable filenames. + * + * Query strings and fragments are dropped, which keeps tracking parameters (`utm_source`, + * `gclid`) from producing a different filename for the same page. A root path yields an empty + * slug, so single-page and origin-scoped runs keep the filenames they have always had. + * + * Truncation keeps the END of the path and appends a short hash. URLs are hierarchical, so the + * shared part is the prefix and the discriminating part is the suffix — keeping the head would + * collapse exactly the deep-category pages most likely to be audited together. The hash makes a + * truncated slug stable across runs, which the `_NN` fallback is not: `runBatch` records results + * in completion order, so the number a given URL receives varies run to run. + * + * @param {string} url - absolute URL + * @returns {string} the slug, or '' for a root path + */ +function urlSlug(url) { + const raw = new URL(url).pathname; + let decoded; + try { + decoded = decodeURIComponent(raw); + } catch { + // A malformed percent sequence (`/%zz`) throws rather than decoding. The raw pathname + // is still a usable slug source, so fall back to it instead of failing the whole run. + decoded = raw; + } + // \p{L}/\p{N} rather than a-z0-9: `pathname` percent-encodes non-ASCII, so an ASCII-only + // rule turns `/es/zapatos-de-niño` into `es-zapatos-de-ni-c3-b1o` and any non-Latin script + // into unreadable hex that carries no page identity at all. + const full = decoded.toLowerCase().replace(/[^\p{L}\p{N}]+/gu, '-').replace(/^-+|-+$/g, ''); + if (full === '' || Buffer.byteLength(full) <= SLUG_MAX_BYTES) { + return full; + } + const hash = crypto.createHash('sha256').update(decoded).digest('hex').slice(0, SLUG_HASH_LENGTH); + const budget = SLUG_MAX_BYTES - (SLUG_HASH_LENGTH + 1); + // Walk backwards a character at a time so a multi-byte character is never cut in half. + const chars = [...full]; + let tail = ''; + let bytes = 0; + for (let i = chars.length - 1; i >= 0; i--) { + const size = Buffer.byteLength(chars[i]); + if (bytes + size > budget) { + break; + } + tail = chars[i] + tail; + bytes += size; + } + return `${tail.replace(/^-+/, '')}-${hash}`; +} + function buildFilename(url, command, suffix, ext = 'json') { - const hostname = new URL(url).hostname; + const { hostname } = new URL(url); + const slug = urlSlug(url); + const slugPart = slug ? `-${slug}` : ''; const suffixPart = suffix ? `-${suffix}` : ''; - const base = path.join(RESULTS_DIR, command, `${command}-${hostname}-${formatDate()}${suffixPart}`); + const base = path.join(RESULTS_DIR, command, `${command}-${hostname}${slugPart}-${formatDate()}${suffixPart}`); if (!fs.existsSync(`${base}.${ext}`)) { return `${base}.${ext}`; } @@ -265,6 +326,7 @@ async function runBatch(items, auditFn, { maxRequestsPerSecond, concurrency = 5, module.exports = { ensureCommandDir, buildFilename, + urlSlug, formatDate, formatElapsed, normalizeOrigin, diff --git a/lib/utils.test.js b/lib/utils.test.js index ee75cd4..411d580 100644 --- a/lib/utils.test.js +++ b/lib/utils.test.js @@ -6,6 +6,7 @@ const path = require('path'); const { formatDate, buildFilename, + urlSlug, formatElapsed, normalizeUrlForAi, normalizeUrlsToOrigins, @@ -116,6 +117,28 @@ describe('utils', () => { expect(result).toContain('rum-my-site.org-'); }); + it('should place the path slug between hostname and timestamp', () => { + existsSyncSpy.mockReturnValue(false); + const result = buildFilename('https://a.com/es/page-one', 'psi', 'mobile'); + expect(path.basename(result)).toBe('psi-a.com-es-page-one-2026-04-01-123456-mobile.json'); + }); + + it('should leave a root URL byte-identical to the pre-slug format', () => { + existsSyncSpy.mockReturnValue(false); + const withSlash = buildFilename('https://a.com/', 'psi', 'mobile'); + const without = buildFilename('https://a.com', 'psi', 'mobile'); + expect(path.basename(withSlash)).toBe('psi-a.com-2026-04-01-123456-mobile.json'); + expect(without).toBe(withSlash); + }); + + it('should still append _NN when two paths slug identically', () => { + const base = path.join(RESULTS_DIR, 'psi', 'psi-a.com-es-page-2026-04-01-123456'); + existsSyncSpy.mockImplementation((f) => f === `${base}.json`); + // Same page, different tracking parameters: the query is dropped, so both slug alike. + const result = buildFilename('https://a.com/es/page?utm_source=x', 'psi'); + expect(result).toBe(`${base}_01.json`); + }); + it('should include the command prefix in the filename', () => { existsSyncSpy.mockReturnValue(false); const result = buildFilename('https://example.com', 'collect'); @@ -148,6 +171,74 @@ describe('utils', () => { }); }); + describe('urlSlug', () => { + it('should return an empty slug for a root path', () => { + expect(urlSlug('https://a.com/')).toBe(''); + expect(urlSlug('https://a.com')).toBe(''); + }); + + it('should join path segments with single dashes', () => { + expect(urlSlug('https://a.com/es/productos/zapatos')).toBe('es-productos-zapatos'); + }); + + it('should lowercase and collapse runs of separators', () => { + expect(urlSlug('https://a.com/ES//Page__One/')).toBe('es-page-one'); + }); + + it('should drop the query string and the fragment', () => { + expect(urlSlug('https://a.com/es/page?utm_source=x&gclid=y')).toBe('es-page'); + expect(urlSlug('https://a.com/es/page#top')).toBe('es-page'); + }); + + // pathname percent-encodes non-ASCII. Slugging the encoded form would emit the UTF-8 + // bytes as text: /es/zapatos-de-niño -> es-zapatos-de-ni-c3-b1o. + it('should decode percent-encoding rather than slugging the hex', () => { + expect(urlSlug('https://a.com/es/zapatos-de-niño')).toBe('es-zapatos-de-niño'); + expect(urlSlug('https://a.com/es/año-nuevo')).toBe('es-año-nuevo'); + }); + + it('should keep non-Latin scripts readable and distinct', () => { + expect(urlSlug('https://a.com/日本語/ページ')).toBe('日本語-ページ'); + expect(urlSlug('https://a.com/日本語/製品')).toBe('日本語-製品'); + expect(urlSlug('https://a.com/ru/страница')).toBe('ru-страница'); + }); + + it('should fall back to the raw pathname on a malformed percent sequence', () => { + expect(urlSlug('https://a.com/es/%zz-page')).toBe('es-zz-page'); + }); + + // The discriminating part of a hierarchical URL is its tail, so truncation keeps the end. + it('should keep the tail when truncating, so sibling pages stay distinct', () => { + const a = urlSlug(`https://a.com/es/${'x'.repeat(90)}/hombre`); + const b = urlSlug(`https://a.com/es/${'x'.repeat(90)}/mujer`); + expect(a).toContain('hombre'); + expect(b).toContain('mujer'); + expect(a).not.toBe(b); + }); + + it('should append a hash only once the slug is truncated', () => { + expect(urlSlug('https://a.com/es/page-one')).toBe('es-page-one'); + expect(urlSlug(`https://a.com/${'x'.repeat(200)}`)).toMatch(/-[0-9a-f]{6}$/); + }); + + it('should produce a stable hash for the same path', () => { + const url = `https://a.com/${'x'.repeat(200)}`; + expect(urlSlug(url)).toBe(urlSlug(url)); + }); + + // A character cap would not hold: 80 CJK characters is 240 bytes on its own. + it('should cap the slug in bytes without splitting a multi-byte character', () => { + const slug = urlSlug(`https://a.com/${'製品'.repeat(60)}`); + expect(Buffer.byteLength(slug)).toBeLessThanOrEqual(80); + expect(slug).not.toContain('\uFFFD'); + expect(slug).toBe([...slug].join('')); + }); + + it('should throw on an invalid URL', () => { + expect(() => urlSlug('not-a-url')).toThrow(); + }); + }); + describe('buildFilename with custom ext', () => { let existsSyncSpy; diff --git a/type-tests/consumer.ts b/type-tests/consumer.ts index 8c11d96..009bb27 100644 --- a/type-tests/consumer.ts +++ b/type-tests/consumer.ts @@ -12,6 +12,7 @@ import { runLabPlan, runLabAudit, runLabToDisk } from '@hugoer/web-perf-cli/lab' import { runCrux, runCruxAudit, runCruxBatch, DEFAULT_CRUX_FORM_FACTORS } from '@hugoer/web-perf-cli/crux'; import { runCruxHistoryAudit } from '@hugoer/web-perf-cli/crux-history'; import { buildRunSummary } from '@hugoer/web-perf-cli/variance'; +import { urlSlug, buildFilename } from '@hugoer/web-perf-cli/utils'; import type { LabReport, LabPlanResult } from '@hugoer/web-perf-cli/lab'; import type { CruxReport } from '@hugoer/web-perf-cli/crux'; import type { CruxHistoryReport } from '@hugoer/web-perf-cli/crux-history'; @@ -120,3 +121,11 @@ export const rejectsPort = runLabPlan(['u'], [{ profile: 'low' }], { }); export const audit = runLabAudit('https://example.com', { profile: 'low', silent: true }); + +// --- urlSlug and buildFilename are typed, so a consumer can predict an output path ---------- +export const slug: string = urlSlug('https://a.com/es/page-one'); +export const filename: string = buildFilename('https://a.com/es/page-one', 'psi', 'mobile'); +export const noSuffix: string = buildFilename('https://a.com', 'sitemap', undefined, 'txt'); + +// @ts-expect-error urlSlug returns a string, never null — callers should not have to narrow it +export const notNullable: null = urlSlug('https://a.com/'); diff --git a/types/lib/utils.d.ts b/types/lib/utils.d.ts index 0bbec63..032679f 100644 --- a/types/lib/utils.d.ts +++ b/types/lib/utils.d.ts @@ -1,5 +1,23 @@ export function ensureCommandDir(command: any): void; export function buildFilename(url: any, command: any, suffix: any, ext?: string): string; +/** + * Builds the filename-safe slug for a URL's path, so two pages of one host produce two + * distinguishable filenames. + * + * Query strings and fragments are dropped, which keeps tracking parameters (`utm_source`, + * `gclid`) from producing a different filename for the same page. A root path yields an empty + * slug, so single-page and origin-scoped runs keep the filenames they have always had. + * + * Truncation keeps the END of the path and appends a short hash. URLs are hierarchical, so the + * shared part is the prefix and the discriminating part is the suffix — keeping the head would + * collapse exactly the deep-category pages most likely to be audited together. The hash makes a + * truncated slug stable across runs, which the `_NN` fallback is not: `runBatch` records results + * in completion order, so the number a given URL receives varies run to run. + * + * @param {string} url - absolute URL + * @returns {string} the slug, or '' for a root path + */ +export function urlSlug(url: string): string; export function formatDate(): string; export function formatElapsed(ms: any): string; export function normalizeOrigin(url: any): string;