Resolve companion ranges across filename case differences

CHANGELOG.md

@@ -48,13 +48,15 @@ releases may contain breaking changes.
   against the most recent `save()`.
 - LIFT-folder handling: `RangesFile` (standalone `.lift-ranges` documents,
   same fidelity guarantees), automatic companion discovery/tracking on load
-  (`Lexicon.ranges_files`), `save()` writes companions together,
+  (`Lexicon.ranges_files`, resolving a companion whose filename differs from
+  the `.lift` only in case or Unicode normalization form, as Windows- and
+  FLEx-authored folders do), `save()` writes companions together,
   `all_ranges()` merged view, `media_refs()` / `missing_media()` helpers,
   build-from-scratch helpers `Lexicon.add_ranges_file()` /
   `RangesFile.add_range()` / `Range.add_element()` (`save()` writes and
   header-references a new companion beside the `.lift`); vendored
-  `schemas/lift-ranges-0.13.rng` — the first schema for standalone
-  ranges documents.
+  `schemas/lift-ranges-0.13.rng` — the first schema for standalone ranges
+  documents.
 - Zipped LIFT packages: `sil_lift.load()` reads a `.zip` (both the flat and
   folder-wrapped layouts, junk entries like `__MACOSX` ignored),
   `Lexicon.save_zip()` writes one (carrying media, `WritingSystems/`, and other

docs/en/guides/folder-media.md

@@ -12,7 +12,7 @@ lex.all_ranges()                            # merged {id: Range} view
 lex.all_ranges()["grammatical-info"].elements
 ```
 
-Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `<name>.lift-ranges` sibling is picked up even when nothing references it.
+Companion discovery handles the real world: a `range/@href` that points at an existing file is used; FieldWorks' dangling absolute `file://C:/...` hrefs fall back to the href's basename next to the `.lift`; and the conventional `<name>.lift-ranges` sibling is picked up even when nothing references it. Each of those resolves a name that differs only in case or Unicode normalization form, so a folder written on Windows — `Dict.LIFT` beside `Dict.lift-ranges` — loads the same way on a case-sensitive filesystem.
 
 `lex.save()` writes the `.lift` and every tracked companion together. Edits to a `RangesFile` save back to _its_ file; untouched ranges keep their exact bytes. Standalone use:
 

src/sil_lift/_model.py

@@ -11,6 +11,7 @@
 
 from __future__ import annotations
 
+import unicodedata
 from dataclasses import dataclass, field
 from datetime import date, datetime
 from pathlib import Path, PurePosixPath, PureWindowsPath
@@ -452,6 +453,75 @@ def _normalize_href(href: str) -> Path | None:
     return Path(normalized)
 
 
+def _fold(text: str) -> str:
+    """A filename reduced to what a forgiving filesystem treats as one name.
+
+    ``casefold`` rather than ``lower`` for the non-ASCII cases (Turkish dotted
+    and dotless I), and NFC because normalization forms get mixed within a
+    single export — FLEx writes the ``.lift`` in NFC and its companion in NFD —
+    and macOS folds them together where Linux does not. Neither NTFS's nor
+    APFS's own folding table is reproduced exactly; this is an approximation
+    over LIFT filenames, not a general equivalence.
+    """
+    return unicodedata.normalize("NFC", text).casefold()
+
+
+def _existing_file(candidate: Path, listings: dict[Path, dict[str, Path]]) -> Path | None:
+    """``candidate`` if it is a file, else one whose name differs only in spelling.
+
+    LIFT folders are written on Windows, where the filesystem folds case, and
+    read everywhere. A folder whose ``.lift`` and ``.lift-ranges`` disagree in
+    case (``Dict.LIFT`` beside ``Dict.lift-ranges``) resolves there and, before
+    this fallback, silently did not on a case-sensitive filesystem. See
+    :func:`_fold` for what counts as the same name.
+
+    The fallback runs only where the exact name matched no file, so a
+    case-folding filesystem never reaches it and nothing changes there. Only
+    the final path component is folded: a candidate under a *directory* spelled
+    in another case still does not resolve, which the hrefs this serves — bare
+    basenames, or relatives within the folder — do not need. Where several
+    names fold together, the first in code point order wins (so ``Dict.LIFT``
+    ahead of ``Dict.lift``) — arbitrary, but stable across runs and platforms,
+    which "whatever the directory yields first" would not be. ``listings``
+    caches one directory read per folder.
+    """
+    try:
+        if candidate.is_file():
+            return candidate
+    except OSError:
+        pass  # unstattable exact spelling: a case variant of it may still stat
+    folder = candidate.parent
+    if folder not in listings:
+        files: dict[str, Path] = {}
+        try:
+            # By name, not by Path: PurePath ordering is case-folded on Windows,
+            # which would leave the tie-break to directory order there.
+            for path in sorted(folder.iterdir(), key=lambda entry: entry.name):
+                if path.is_file():
+                    files.setdefault(_fold(path.name), path)
+        except OSError:
+            pass  # unreadable folder: no candidate resolves out of it
+        listings[folder] = files
+    return listings[folder].get(_fold(candidate.name))
+
+
+def _same_file(left: Path, right: Path) -> bool:
+    """Whether two paths differing only in spelling denote one file.
+
+    ``Path.resolve()`` canonicalizes case on Windows but not on macOS, so on
+    the latter one file reached under two spellings yields two distinct keys —
+    tracked twice, and written twice by :meth:`Lexicon.save`. Only paths that
+    :func:`_fold` together are compared, since the inode check alone would
+    conflate distinct files on the filesystems that report ``st_ino`` as 0.
+    """
+    if _fold(str(left)) != _fold(str(right)):
+        return False
+    try:
+        return left.samefile(right)
+    except OSError:
+        return False
+
+
 def _same_dir(left: Path, right: Path | None) -> bool:
     """Whether two paths denote the same directory, spelling aside.
 
@@ -510,7 +580,11 @@ def load(cls, path: str | os.PathLike[str], *, resolve_ranges: bool = True) -> L
         ``range/@href`` both the href resolved as a path relative to the
         ``.lift`` file and its bare basename in the same directory (FLEx
         hrefs are usually dangling absolute ``file://C:/...`` paths from the
-        exporting machine, so the basename is what resolves locally).
+        exporting machine, so the basename is what resolves locally). A
+        candidate that no file matches exactly still resolves to a file whose
+        name differs only in case or Unicode normalization, so a folder
+        authored on Windows loads the same way on a case-sensitive filesystem.
+        The ``.lift`` itself is never taken as its own companion.
 
         A ``.zip`` path is treated as a packaged LIFT folder: it is extracted
         to a temporary directory (kept alive for the returned lexicon's
@@ -532,9 +606,16 @@ def _resolve_ranges(self) -> None:
         if self.path is None:
             return
         base = self.path.parent
+        try:
+            own = self.path.resolve()
+        except OSError:
+            own = self.path
         candidates: list[Path] = []
-        sibling = self.path.with_suffix(self.path.suffix + "-ranges")
-        candidates.append(sibling)
+        # with_name, not with_suffix: they agree on every name that has an
+        # extension, but with_suffix rejects "-ranges" outright on a name
+        # without one, and nothing upstream requires the document to be named
+        # ``.lift`` — parse_document never looks at the extension.
+        candidates.append(self.path.with_name(self.path.name + "-ranges"))
         for range_ in self.header.ranges:
             if range_.href is None:
                 continue
@@ -544,14 +625,24 @@ def _resolve_ranges(self) -> None:
             basename = range_.href.replace("\\", "/").rpartition("/")[2]
             if basename:
                 candidates.append(base / basename)
+        listings: dict[Path, dict[str, Path]] = {}
         for candidate in candidates:
+            found = _existing_file(candidate, listings)
+            if found is None:
+                continue
             try:
-                resolved = candidate.resolve()
-                exists = candidate.is_file()
+                resolved = found.resolve()
             except OSError:
                 continue
-            if exists and resolved not in self.ranges_files:
-                self.ranges_files[resolved] = RangesFile.load(candidate)
+            # Skip a spelling of something already tracked (macOS keeps two
+            # keys for one file) and the .lift itself, which a header href
+            # naming it in another case now folds onto — RangesFile.load would
+            # reject its root and take the whole load down with it.
+            if resolved in self.ranges_files or any(
+                _same_file(resolved, other) for other in (own, *self.ranges_files)
+            ):
+                continue
+            self.ranges_files[resolved] = RangesFile.load(found)
 
     def save(self, path: str | os.PathLike[str] | None = None) -> None:
         """Write the ``.lift`` file and every tracked ``.lift-ranges`` companion.

src/sil_lift/_validate.py

@@ -40,7 +40,7 @@
 from lxml import etree
 
 from ._errors import LiftValidationError
-from ._model import GrammaticalInfo, Lexicon, _normalize_href
+from ._model import GrammaticalInfo, Lexicon, _existing_file, _normalize_href
 from ._text import Multitext, Trait
 
 if TYPE_CHECKING:
@@ -435,9 +435,12 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]:
     # Absolute/file:// hrefs are ones FLEx writes knowing they will not resolve
     # (they are resolved by basename when the companion is in the same folder)
     # and are not checked here; this catches an exporter that writes a relative
-    # href but not the file.
+    # href but not the file. Existence is the same notion load resolves
+    # companions by (_existing_file), so a companion spelled in another case is
+    # not reported missing on a case-sensitive filesystem.
     if lexicon.path is not None:
         base = lexicon.path.parent
+        listings: dict[Path, dict[str, Path]] = {}
         for range_ in lexicon.header.ranges:
             if not range_.href or range_.elements:
                 continue
@@ -447,7 +450,7 @@ def _main_doc_guids() -> Iterator[tuple[str, str, str | None, int | None]]:
             resolved = all_ranges.get(range_.id)
             if resolved is not None and resolved.elements:
                 continue  # supplied by a sibling companion instead
-            if not (base / relative).is_file():
+            if _existing_file(base / relative, listings) is None:
                 yield Problem(
                     "warning",
                     "dangling-ranges-href",

tests/test_ranges_folder.py

@@ -1,4 +1,5 @@
 import shutil
+import unicodedata
 from pathlib import Path
 
 import pytest
@@ -225,6 +226,114 @@ def test_missing_media_flags_broken_ref(tmp_path: Path) -> None:
     assert [r.href for r in missing] == ["pictures\\sdd.png"]
 
 
+def _write_case_variant_pair(folder: Path, lift_name: str, ranges_name: str) -> Path:
+    """A loadable .lift plus companion under arbitrary filename spellings.
+
+    Deliberately not named after the fixture, so the header's ``range/@href``
+    basename candidate finds nothing — only the sibling candidate resolves these.
+    """
+    folder.mkdir(parents=True, exist_ok=True)
+    (folder / lift_name).write_bytes((PAIR_DIR / "test20080407.lift").read_bytes())
+    (folder / ranges_name).write_bytes((PAIR_DIR / "test20080407.lift-ranges").read_bytes())
+    return folder / lift_name
+
+
+def _write_lift_with_href(folder: Path, lift_name: str, href: str) -> Path:
+    """The fixture .lift under another name, its companion href rewritten."""
+    folder.mkdir(parents=True, exist_ok=True)
+    source = (PAIR_DIR / "test20080407.lift").read_bytes()
+    patched = source.replace(b'"file://test20080407.lift-ranges"', f'"{href}"'.encode())
+    assert patched != source, "fixture href changed; the replacement no longer matches"
+    (folder / lift_name).write_bytes(patched)
+    return folder / lift_name
+
+
+def _case_sensitive_filesystem(folder: Path) -> bool:
+    probe = folder / "CaseProbe"
+    probe.mkdir(exist_ok=True)
+    sensitive = not (folder / "caseprobe").exists()
+    probe.rmdir()
+    return sensitive
+
+
+def test_companion_resolves_when_lift_suffix_is_uppercase(tmp_path: Path) -> None:
+    lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.LIFT", "Dict.lift-ranges")
+    lexicon = sil_lift.load(lift)
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_companion_resolves_when_companion_suffix_is_uppercase(tmp_path: Path) -> None:
+    lift = _write_case_variant_pair(tmp_path / "pkg", "Dict.lift", "Dict.LIFT-RANGES")
+    lexicon = sil_lift.load(lift)
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_case_folded_companions_resolve_deterministically(tmp_path: Path) -> None:
+    if not _case_sensitive_filesystem(tmp_path):
+        pytest.skip("needs a case-sensitive filesystem to hold both spellings at once")
+    # Neither spelling matches the Dict.LIFT-ranges candidate exactly, so the
+    # tie-break picks one: lexicographically first, the same one every run.
+    folder = tmp_path / "pkg"
+    lift = _write_case_variant_pair(folder, "Dict.LIFT", "Dict.lift-ranges")
+    (folder / "Dict.Lift-ranges").write_bytes((folder / "Dict.lift-ranges").read_bytes())
+    lexicon = sil_lift.load(lift)
+    assert [path.name for path in lexicon.ranges_files] == ["Dict.Lift-ranges"]
+
+
+def test_absent_companion_stays_absent(tmp_path: Path) -> None:
+    # The fallback must not look outside the folder for a name not in it.
+    folder = tmp_path / "pkg"
+    folder.mkdir()
+    (folder / "Dict.lift").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes())
+    assert sil_lift.load(folder / "Dict.lift").ranges_files == {}
+
+
+def test_companion_resolves_across_unicode_normalization(tmp_path: Path) -> None:
+    # FLEx mixes NFC and NFD within one export, and the mismatch reaches the
+    # filenames; only macOS folds the two forms together on its own.
+    composed = "Caf\N{LATIN SMALL LETTER E WITH ACUTE}.lift"
+    decomposed = unicodedata.normalize("NFD", f"{composed}-ranges")
+    lift = _write_case_variant_pair(tmp_path / "pkg", composed, decomposed)
+    lexicon = sil_lift.load(lift)
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_lift_without_an_extension_loads(tmp_path: Path) -> None:
+    # Loading never inspects the extension, so the sibling candidate is built
+    # from a name that may have none; this companion is the href's basename.
+    folder = tmp_path / "pkg"
+    folder.mkdir()
+    (folder / "Dict").write_bytes((PAIR_DIR / "test20080407.lift").read_bytes())
+    shutil.copy(PAIR_DIR / "test20080407.lift-ranges", folder)
+    lexicon = sil_lift.load(folder / "Dict")
+    assert lexicon.all_ranges()["grammatical-info"].elements
+
+
+def test_href_folding_onto_the_lift_itself_is_not_a_companion(tmp_path: Path) -> None:
+    # Dict.lift beside a Dict.LIFT is the lexicon, not its ranges: loading it
+    # as one would raise on the root and take the whole load down.
+    lift = _write_lift_with_href(tmp_path / "pkg", "Dict.LIFT", "Dict.lift")
+    assert sil_lift.load(lift).ranges_files == {}
+
+
+# Defines the range the header points at, but no elements — so the merged view
+# cannot vouch for the href and the check falls through to the filesystem.
+ELEMENTLESS_RANGES = b"""<?xml version="1.0" encoding="UTF-8"?>
+<lift-ranges>
+  <range id="grammatical-info"/>
+</lift-ranges>
+"""
+
+
+def test_case_variant_companion_is_not_reported_dangling(tmp_path: Path) -> None:
+    folder = tmp_path / "pkg"
+    lift = _write_lift_with_href(folder, "Dict.LIFT", "Dict.LIFT-ranges")
+    (folder / "Dict.lift-ranges").write_bytes(ELEMENTLESS_RANGES)
+    lexicon = sil_lift.load(lift)
+    assert lexicon.ranges_files  # the companion resolved
+    assert [p for p in lexicon.iter_problems() if p.code == "dangling-ranges-href"] == []
+
+
 @pytest.mark.parametrize(
     ("href", "expected"),
     [