diff --git a/CHANGELOG.md b/CHANGELOG.md index 08dcbe8..4138872 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,13 +48,14 @@ releases may contain breaking changes. against the most recent `save()`. - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents, same fidelity guarantees), automatic companion discovery/tracking on load - (`Lexicon.ranges_files`), `save()` writes companions together, + (`Lexicon.ranges_files`, matching companion filenames across case and + Unicode normalization differences), `save()` writes companions together, `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers, build-from-scratch helpers `Lexicon.add_ranges_file()` / `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and header-references a new companion beside the `.lift`); vendored - `schemas/lift-ranges-0.13.rng` — the first schema for standalone - ranges documents. + `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges + documents. - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and folder-wrapped layouts, junk entries like `__MACOSX` ignored), `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other diff --git a/src/sil_lift/_model.py b/src/sil_lift/_model.py index e130f9f..0a7d42c 100644 --- a/src/sil_lift/_model.py +++ b/src/sil_lift/_model.py @@ -11,6 +11,7 @@ from __future__ import annotations +import unicodedata from dataclasses import dataclass, field from datetime import date, datetime from pathlib import Path, PurePosixPath, PureWindowsPath @@ -452,6 +453,66 @@ def _normalize_href(href: str) -> Path | None: return Path(normalized) +def _fold(text: str) -> str: + """A filename reduced to what a case-folding filesystem treats as one name. + + ``casefold`` for what ``lower`` gets wrong (the Turkish dotless i), NFC + because FLEx mixes normalization forms within one export. An approximation + of NTFS's and APFS's tables, not a general equivalence. + """ + return unicodedata.normalize("NFC", text).casefold() + + +def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None: + """``candidate`` if it is a file, else one whose name folds onto it. + + LIFT folders are written on Windows, where the filesystem folds case, and + read everywhere: ``Dict.LIFT`` beside ``Dict.lift-ranges`` resolves there + and, before this fallback, silently did not on a case-sensitive filesystem. + + Only the final component folds — the hrefs this serves are basenames or + same-folder relatives — and only after the exact name misses, so a + case-folding filesystem never reaches this. Among names that fold together + the first in code point order wins: arbitrary, but stable across runs and + platforms, which directory order is not. ``listings`` caches one directory + read per folder. + """ + try: + if candidate.is_file(): + return candidate + except OSError: + pass # unstattable exact spelling: a case variant of it may still stat + folder = candidate.parent + if folder not in listings: + files: dict[str, Path] = {} + try: + # By name, not by Path: PurePath ordering is case-folded on Windows, + # which would leave the tie-break to directory order there. + for path in sorted(folder.iterdir(), key=lambda entry: entry.name): + if path.is_file(): + files.setdefault(_fold(path.name), path) + except OSError: + pass # unreadable folder: no candidate resolves out of it + listings[folder] = files + return listings[folder].get(_fold(candidate.name)) + + +def _same_file(left: Path, right: Path) -> bool: + """Whether two paths that fold together denote one file. + + ``Path.resolve()`` canonicalizes case on Windows but not on macOS, where + one file reached under two spellings yields two keys — tracked twice, and + written twice by :meth:`Lexicon.save`. The fold pre-check keeps the inode + comparison from conflating distinct files where ``st_ino`` is 0. + """ + if _fold(str(left)) != _fold(str(right)): + return False + try: + return left.samefile(right) + except OSError: + return False + + def _same_dir(left: Path, right: Path | None) -> bool: """Whether two paths denote the same directory, spelling aside. @@ -510,7 +571,11 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L ``range/@href`` both the href resolved as a path relative to the ``.lift`` file and its bare basename in the same directory (FLEx hrefs are usually dangling absolute ``file://C:/...`` paths from the - exporting machine, so the basename is what resolves locally). + exporting machine, so the basename is what resolves locally). A + candidate that no file matches exactly still resolves to a file whose + name differs only in case or Unicode normalization, so a folder + authored on Windows loads the same way on a case-sensitive filesystem. + The ``.lift`` itself is never taken as its own companion. A ``.zip`` path is treated as a packaged LIFT folder: it is extracted to a temporary directory (kept alive for the returned lexicon's @@ -532,9 +597,16 @@ def _resolve_ranges(self) -> None: if self.path is None: return base = self.path.parent + try: + own = self.path.resolve() + except OSError: + own = self.path candidates: list[Path] = [] - sibling = self.path.with_suffix(self.path.suffix + "-ranges") - candidates.append(sibling) + # with_name, not with_suffix: they agree on every name that has an + # extension, but with_suffix rejects "-ranges" outright on a name + # without one, and nothing upstream requires the document to be named + # ``.lift`` — parse_document never looks at the extension. + candidates.append(self.path.with_name(self.path.name + "-ranges")) for range_ in self.header.ranges: if range_.href is None: continue @@ -544,14 +616,24 @@ def _resolve_ranges(self) -> None: basename = range_.href.replace("\\", "/").rpartition("/")[2] if basename: candidates.append(base / basename) + listings: dict[Path, dict[str, Path]] = {} for candidate in candidates: + found = _existing_file(candidate, listings) + if found is None: + continue try: - resolved = candidate.resolve() - exists = candidate.is_file() + resolved = found.resolve() except OSError: continue - if exists and resolved not in self.ranges_files: - self.ranges_files[resolved] = RangesFile.load(candidate) + # Skip a spelling of something already tracked (macOS keeps two + # keys for one file) and the .lift itself, which a header href + # naming it in another case now folds onto — RangesFile.load would + # reject its root and take the whole load down with it. + if resolved in self.ranges_files or any( + _same_file(resolved, other) for other in (own, *self.ranges_files) + ): + continue + self.ranges_files[resolved] = RangesFile.load(found) def save(self, path: str | os.PathLike[str] | None = None) -> None: """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion. diff --git a/src/sil_lift/_validate.py b/src/sil_lift/_validate.py index 39e0b25..9680eb9 100644 --- a/src/sil_lift/_validate.py +++ b/src/sil_lift/_validate.py @@ -40,7 +40,7 @@ from lxml import etree from ._errors import LiftValidationError -from ._model import GrammaticalInfo, Lexicon, _normalize_href +from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href from ._text import Multitext, Trait if TYPE_CHECKING: @@ -435,9 +435,12 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve # (they are resolved by basename when the companion is in the same folder) # and are not checked here; this catches an exporter that writes a relative - # href but not the file. + # href but not the file. Existence is the same notion load resolves + # companions by (_existing_file), so a companion spelled in another case is + # not reported missing on a case-sensitive filesystem. if lexicon.path is not None: base = lexicon.path.parent + listings: dict[Path, dict[str, Path]] = {} for range_ in lexicon.header.ranges: if not range_.href or range_.elements: continue @@ -447,7 +450,7 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]: resolved = all_ranges.get(range_.id) if resolved is not None and resolved.elements: continue # supplied by a sibling companion instead - if not (base / relative).is_file(): + if _existing_file(base / relative, listings) is None: yield Problem( "warning", "dangling-ranges-href", diff --git a/tests/test_ranges_folder.py b/tests/test_ranges_folder.py index 9544db9..fc0d2b2 100644 --- a/tests/test_ranges_folder.py +++ b/tests/test_ranges_folder.py @@ -1,4 +1,5 @@ import shutil +import unicodedata from pathlib import Path import pytest @@ -225,6 +226,114 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None: assert [r.href for r in missing] == ["pictures\\sdd.png"] +def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path: + """A loadable .lift plus companion under arbitrary filename spellings. + + Deliberately not named after the fixture, so the header's ``range/@href`` + basename candidate finds nothing — only the sibling candidate resolves these. + """ + folder.mkdir(parents=True, exist_ok=True) + (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes()) + return folder / lift_name + + +def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path: + """The fixture .lift under another name, its companion href rewritten.""" + folder.mkdir(parents=True, exist_ok=True) + source = (PAIR_DIR / "test20080407.lift").read_bytes() + patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode()) + assert patched != source, "fixture href changed; the replacement no longer matches" + (folder / lift_name).write_bytes(patched) + return folder / lift_name + + +def _case_sensitive_filesystem(folder: Path) -> bool: + probe = folder / "CaseProbe" + probe.mkdir(exist_ok=True) + sensitive = not (folder / "caseprobe").exists() + probe.rmdir() + return sensitive + + +def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None: + lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES") + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None: + if not _case_sensitive_filesystem(tmp_path): + pytest.skip("needs a case-sensitive filesystem to hold both spellings at once") + # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the + # tie-break picks one: lexicographically first, the same one every run. + folder = tmp_path / "pkg" + lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges") + (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes()) + lexicon = sil_lift.load(lift) + assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"] + + +def test_absent_companion_stays_absent(tmp_path: Path) -> None: + # The fallback must not look outside the folder for a name not in it. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + assert sil_lift.load(folder / "Dict.lift").ranges_files == {} + + +def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None: + # FLEx mixes NFC and NFD within one export, and the mismatch reaches the + # filenames; only macOS folds the two forms together on its own. + composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift" + decomposed = unicodedata.normalize("NFD", f"{composed}-ranges") + lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed) + lexicon = sil_lift.load(lift) + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_lift_without_an_extension_loads(tmp_path: Path) -> None: + # Loading never inspects the extension, so the sibling candidate is built + # from a name that may have none; this companion is the href's basename. + folder = tmp_path / "pkg" + folder.mkdir() + (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes()) + shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder) + lexicon = sil_lift.load(folder / "Dict") + assert lexicon.all_ranges()["grammatical-info"].elements + + +def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None: + # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it + # as one would raise on the root and take the whole load down. + lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift") + assert sil_lift.load(lift).ranges_files == {} + + +# Defines the range the header points at, but no elements — so the merged view +# cannot vouch for the href and the check falls through to the filesystem. +ELEMENTLESS_RANGES = b""" + + + +""" + + +def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None: + folder = tmp_path / "pkg" + lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges") + (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES) + lexicon = sil_lift.load(lift) + assert lexicon.ranges_files # the companion resolved + assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == [] + + @pytest.mark.parametrize( ("href", "expected"), [