diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2dad952..a75f42e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -121,3 +121,23 @@ jobs: files: dist/* generate_release_notes: true prerelease: ${{ github.ref_name == 'staging' }} + + # A floating alias tag literally named after the branch (main/staging) + # that always points at whatever was just released — unlike the + # v{version}[-staging] tag above (deleted/recreated, so it names one + # specific release), this one just moves forward every time. Runs only + # once the real release above has succeeded. + # + # Sharing its name with the branch is deliberate: `git` resolves the + # ambiguity deterministically (a tag always wins over a same-named + # branch — see gitrevisions(7) on ref disambiguation order), so + # `pip install ...@main` / `...@staging` always resolves to this tag, + # i.e. the latest release — not necessarily the exact branch tip. + # Expect (and ignore) a "refname 'main' is ambiguous" warning from + # git/pip when that happens. The push uses fully-qualified refs on + # both sides specifically to sidestep that same ambiguity for the + # push command itself. + - name: Move the branch-name alias tag to this release + run: | + git tag -f "${{ github.ref_name }}" + git push origin "refs/tags/${{ github.ref_name }}:refs/tags/${{ github.ref_name }}" --force diff --git a/README.md b/README.md index 6f36a1c..2155ad3 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,16 @@ Two main class domains for the EEA data lakehouse: These badges are live — each one queries the GitHub API directly and always shows whatever tag is *currently* released for that branch, updating on its own every time `main`/`staging` cuts a -new release (see [Releasing a new version](#releasing-a-new-version) — only one tag exists per -branch at a time, so pin to whatever the badge shows *now*, not a number copied from here). +new release (see [Releasing a new version](#releasing-a-new-version)). + +`@main`/`@staging` always installs whatever was most recently released for that branch — a +floating tag sharing the branch's own name, moved forward to the latest release automatically +each time one is cut, so there's nothing to look up or keep in sync yourself: ```bash -# main's latest release (stable) — pin to the tag the "main" badge above shows -pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.6" +pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@main" # latest stable +pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@staging" # latest early access +``` # staging's latest release (early access) — pin to the tag the "staging" badge above shows pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.7-staging" @@ -123,10 +127,32 @@ exact same arguments. count, without fetching any actual rows. **Wiki & tags** (Dremio's catalog collaboration API — REST-only, no SQL/Flight equivalent): -- `getwikifrom(path)` / `assignwikito(path, text)` — read, or create/overwrite, the wiki text on - a catalog entity. -- `gettagsfrom(path)` / `assigntagsto(path, tags)` / `deletetags(path, tags)` — read the full tag - list; replace it wholesale; or remove just the given tags, leaving the rest untouched. +- `getwikifrom(path)` / `setwikito(path, text, tags=None)` / `deletewiki(path)` — read, + create/overwrite, or clear the wiki text on a catalog entity. `deletewiki` is idempotent — a + missing `path`, or one with no wiki at all, is a no-op, not an error (Dremio's collaboration + API has no separate delete-wiki endpoint, so this clears the text to empty). `tags`, if given + to `setwikito`, is a list of `{"tag_name", "tag_value", "tag_title"}` dicts rendered into a + `# Meta Data` section appended to `text` (both a human-readable `title : value` line per tag + and the same data as `...`) — Dremio's wiki is + plain markdown with no structured-metadata concept of its own, so this is embedded directly in + the text. +- `setmeta2wiki(path, tags=None, overwrite=True)` / `getmetafromwiki(path, tag_name=None, + field=None)` — **folders only** (raises `CatalogOperationError` on a table/view — those have + Dremio's own tags/labels for this instead). `setmeta2wiki` updates just the `# Meta Data` + section of the wiki already at `path`, keeping whatever text comes before it untouched: + `overwrite=True` (the default) replaces the whole section with one built fresh from `tags`; + `overwrite=False` merges `tags` into whatever tags are already there (parsed back out of the + existing `` block), appended after them, with no deduplication. If `path` has no wiki + yet, starts from empty base text rather than raising. `getmetafromwiki` reads it back: + without `tag_name` (or if it doesn't match one there), returns every tag as a list; with a + matching `tag_name`, returns a single `{"tag_name", ...}` dict instead — both `tag_value` and + `tag_title` if `field` isn't given, or just the one `field` (`"tag_value"`/`"tag_title"`) asks + for. +- `gettagsfrom(path)` / `settagsto(path, tags)` / `deletetags(path, tags)` — **tables/views + only** (the mirror image of `setmeta2wiki`/`getmetafromwiki` — raises + `CatalogOperationError` on a folder, which has no Dremio tags/labels concept of its own). Read + the full tag list; replace it wholesale; or remove just the given tags, leaving the rest + untouched. **Folders** (REST-only, idempotent — an already-there/already-gone folder is not an error): - `createfolder(path, create_parents=False)` — `create_parents=False` (the default) raises if @@ -185,6 +211,13 @@ previous release *and* tag first, so **tags aren't permanent** — pin to whatev [Install](#install) badge shows *now*, not to an old tag number, since it won't exist once a newer release replaces it. +**A separate floating tag literally named `main`/`staging`** always points at that branch's +latest release — the workflow force-moves it (`git tag -f`, force-push) once the real release +above succeeds. It deliberately shares its name with the branch: git resolves the ambiguity +deterministically (a tag always wins over a same-named branch), so `@main`/`@staging` in an +install command means "latest release," not "current branch tip" — expect (and ignore) a +"refname is ambiguous" warning from git/pip when that happens. + The workflow also rewrites this README's `pip`/`%pip install ...@vX.Y.Z[-staging]` example lines to the version it just released, committing that change back to the branch (`[skip ci]`, so it doesn't re-trigger itself) — so the examples above never go stale, without anyone having to @@ -192,12 +225,19 @@ remember to update them by hand. ## Install in JupyterLab -Run this in a notebook cell (see the live badges under [Install](#install) for the current -`main`/`staging` release tags — the lines below are kept in sync with them automatically, see +Run this in a notebook cell — `@main`/`@staging` always resolves to whatever was most recently +released for that branch (see [Install](#install) above): + +```python +%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@main" # latest stable +%pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@staging" # latest early access +``` + +To pin to one specific release instead, use the exact tag the live badges under +[Install](#install) show (kept in sync automatically, see [Releasing a new version](#releasing-a-new-version)): ```python -# main's latest release (stable) — recommended %pip install "git+https://github.com/eeadata/EEALakeHouse.python.git@v0.1.6" # staging's latest release (early access) diff --git a/debugger/debug_run.py b/debugger/debug_run.py index 5f08b1f..873a827 100644 --- a/debugger/debug_run.py +++ b/debugger/debug_run.py @@ -255,27 +255,86 @@ def run_gettagsfrom() -> None: print(f" tags {tags}") -def run_assignwikito() -> None: +def run_setwikito() -> None: catalog = Catalog(DREMIO_BASE_URL, DREMIO_TOKEN, username=DREMIO_USERNAME) if DRY_RUN: print("DRY RUN — not setting the wiki. Set DRY_RUN = False to run this for real.") print(f" view {VIEW_PATH}") return - wiki_text = "# Bathing water assessments\n\nDebug-set wiki text for testing assignwikito." + wiki_text = "# Bathing water assessments\n\nDebug-set wiki text for testing setwikito." #wiki_text ="blabla" - catalog.assignwikito(VIEW_PATH, wiki_text, idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-set-wiki") - print(f"assignwikito wiki set on {VIEW_PATH}") + meta_tags = [ + {"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}, + {"tag_name": "status", "tag_value": "debug", "tag_title": "Status"}, + ] + catalog.setwikito( + VIEW_PATH, + wiki_text, + tags=meta_tags, + idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-set-wiki", + ) + print(f"setwikito wiki set on {VIEW_PATH} (with {len(meta_tags)} meta tags)") + + +def run_deletewiki() -> None: + catalog = Catalog(DREMIO_BASE_URL, DREMIO_TOKEN, username=DREMIO_USERNAME) + if DRY_RUN: + print("DRY RUN — not deleting the wiki. Set DRY_RUN = False to run this for real.") + print(f" view {VIEW_PATH}") + return + catalog.deletewiki(VIEW_PATH, idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-delete-wiki") + print(f"deletewiki wiki cleared on {VIEW_PATH} (if it had one)") -def run_assigntagsto() -> None: +def run_setmeta2wiki() -> None: + catalog = Catalog(DREMIO_BASE_URL, DREMIO_TOKEN, username=DREMIO_USERNAME) + # Folders only — tables/views (like VIEW_PATH used elsewhere) have + # Dremio's own tags/labels for this instead and would raise here. + folder_path = "catalog.water_management_resources.bathing_water.bwd.draft.altia_test" + if DRY_RUN: + print("DRY RUN — not updating wiki metadata. Set DRY_RUN = False to run this for real.") + print(f" folder {folder_path}") + return + new_tags = [ + {"tag_name": "test111", "tag_value": "oskar_value11", "tag_title": "oskar_title11"} + ] + catalog.setmeta2wiki( + folder_path, + tags=new_tags, + overwrite=True, # merge with whatever tags are already there + idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-write-meta", + ) + print(f"setmeta2wiki meta updated on {folder_path}") + + +def run_getmetafromwiki() -> None: + catalog = Catalog(DREMIO_BASE_URL, DREMIO_TOKEN, username=DREMIO_USERNAME) + folder_path = "catalog.water_management_resources.bathing_water.bwd.draft.altia_test" + # run_setmeta2wiki() must have run at least once first, so there's a + # "# Meta Data" section here to read back. + all_tags = catalog.getmetafromwiki( + folder_path, idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-read-meta-all" + ) + print(f"getmetafromwiki all tags on {folder_path}: {all_tags}") + + one_tag = catalog.getmetafromwiki( + folder_path, + "reviewed_by", + "tag_value", + idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-read-meta-one", + ) + print(f"getmetafromwiki reviewed_by's tag_value: {one_tag}") + + +def run_settagsto() -> None: catalog = Catalog(DREMIO_BASE_URL, DREMIO_TOKEN, username=DREMIO_USERNAME) if DRY_RUN: print("DRY RUN — not setting tags. Set DRY_RUN = False to run this for real.") print(f" view {VIEW_PATH}") return tags = ["bathing-water", "debug"] - catalog.assigntagsto(VIEW_PATH, tags, idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-set-tags") - print(f"assigntagsto tags set on {VIEW_PATH}: {tags}") + catalog.settagsto(VIEW_PATH, tags, idempotency_key=f"{TABLE2VIEW_IDEMPOTENCY_KEY}-set-tags") + print(f"settagsto tags set on {VIEW_PATH}: {tags}") def run_deletetags() -> None: @@ -292,7 +351,7 @@ def run_deletetags() -> None: def run_createfolder() -> None: catalog = Catalog(DREMIO_BASE_URL, DREMIO_TOKEN, username=DREMIO_USERNAME) - folder_path = "catalog.water_management_resources.bathing_water.bwd.draft.altia_test.new_folder.level1" + folder_path = "catalog.water_management_resources.bathing_water.bwd.draft.altia_test" if DRY_RUN: print("DRY RUN — not creating a folder. Set DRY_RUN = False to run this for real.") print(f" folder {folder_path}") @@ -584,8 +643,11 @@ def create_pat(host: str, access_token: str, username: str, label: str, #run_gettablesfrom() #run_gettableitemsfrom() - #run_assignwikito() - #run_assigntagsto() + #run_setwikito() + #run_deletewiki() + run_setmeta2wiki() + #run_getmetafromwiki() + #run_settagsto() #run_deletetags() #run_getwikifrom() @@ -599,9 +661,7 @@ def create_pat(host: str, access_token: str, username: str, label: str, - - - + """ ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument("--mode", choices=["azcli", "device", "sp"], default="device") @@ -648,4 +708,4 @@ def create_pat(host: str, access_token: str, username: str, label: str, print(pat.get("token") or json.dumps(pat)) print ("OK") - + """ diff --git a/src/eea_datalakehouse/catalog/__init__.py b/src/eea_datalakehouse/catalog/__init__.py index 2c1218b..ee28263 100644 --- a/src/eea_datalakehouse/catalog/__init__.py +++ b/src/eea_datalakehouse/catalog/__init__.py @@ -5,10 +5,11 @@ catalog.draft2version("bwd.draft.bw", "bwd.versions.v1", idempotency_key="bwd-v2025_1") table2view, draft2version, publishversion, datacopy, datamove, deleteview, -gettablesfrom, gettableitemsfrom, getwikifrom, gettagsfrom, assignwikito, -assigntagsto, deletetags, createfolder, deletefolder — over REST (the -default) or Arrow Flight (opt in via EEA_CATALOG_TRANSPORT=flight in .env), -with retry-later error handling for a Dremio engine that's still starting up. +gettablesfrom, gettableitemsfrom, getwikifrom, gettagsfrom, setwikito, +deletewiki, setmeta2wiki, getmetafromwiki, settagsto, deletetags, +createfolder, deletefolder — over REST (the default) or Arrow Flight (opt +in via EEA_CATALOG_TRANSPORT=flight in .env), with retry-later error +handling for a Dremio engine that's still starting up. The module-level functions in `operations` (table2view(executor, ...) etc.) are what `Catalog`'s methods delegate to — call them directly if you'd @@ -21,21 +22,24 @@ from .errors import CatalogOperationError, EngineStartingError from .operations import ( TableInfo, - assigntagsto, - assignwikito, createfolder, datacopy, datamove, deletefolder, deletetags, deleteview, + deletewiki, draft2version, gettableitemsfrom, gettablesfrom, gettagsfrom, getwikifrom, publishversion, + getmetafromwiki, retry_pending, + setmeta2wiki, + settagsto, + setwikito, table2view, ) from .rest import CatalogRestClient @@ -61,21 +65,24 @@ "SqlExecutor", "SqlResult", "TableInfo", - "assigntagsto", - "assignwikito", "createfolder", "datacopy", "datamove", "deletefolder", "deletetags", "deleteview", + "deletewiki", "draft2version", "gettableitemsfrom", "gettablesfrom", "gettagsfrom", "getwikifrom", "publishversion", + "getmetafromwiki", "resolve_executor", "retry_pending", + "setmeta2wiki", + "settagsto", + "setwikito", "table2view", ] diff --git a/src/eea_datalakehouse/catalog/client.py b/src/eea_datalakehouse/catalog/client.py index 3d37aaf..92d8337 100644 --- a/src/eea_datalakehouse/catalog/client.py +++ b/src/eea_datalakehouse/catalog/client.py @@ -263,13 +263,47 @@ def getwikifrom(self, path: str, *, idempotency_key: str) -> str: def gettagsfrom(self, path: str, *, idempotency_key: str) -> list[str]: return operations.gettagsfrom(self._catalog_rest, path, idempotency_key=idempotency_key) - def assignwikito(self, path: str, text: str, *, idempotency_key: str) -> None: - return operations.assignwikito( - self._catalog_rest, path, text, idempotency_key=idempotency_key + def setwikito( + self, + path: str, + text: str, + *, + tags: list[dict[str, str]] | None = None, + idempotency_key: str, + ) -> None: + return operations.setwikito( + self._catalog_rest, path, text, tags=tags, idempotency_key=idempotency_key + ) + + def deletewiki(self, path: str, *, idempotency_key: str) -> None: + return operations.deletewiki(self._catalog_rest, path, idempotency_key=idempotency_key) + + def setmeta2wiki( + self, + path: str, + *, + tags: list[dict[str, str]] | None = None, + overwrite: bool = True, + idempotency_key: str, + ) -> None: + return operations.setmeta2wiki( + self._catalog_rest, path, tags=tags, overwrite=overwrite, idempotency_key=idempotency_key + ) + + def getmetafromwiki( + self, + path: str, + tag_name: str | None = None, + field: Literal["tag_value", "tag_title"] | None = None, + *, + idempotency_key: str, + ) -> list[dict[str, str]] | dict[str, str]: + return operations.getmetafromwiki( + self._catalog_rest, path, tag_name, field, idempotency_key=idempotency_key ) - def assigntagsto(self, path: str, tags: list[str], *, idempotency_key: str) -> None: - return operations.assigntagsto( + def settagsto(self, path: str, tags: list[str], *, idempotency_key: str) -> None: + return operations.settagsto( self._catalog_rest, path, tags, idempotency_key=idempotency_key ) diff --git a/src/eea_datalakehouse/catalog/operations.py b/src/eea_datalakehouse/catalog/operations.py index 411feac..5442c83 100644 --- a/src/eea_datalakehouse/catalog/operations.py +++ b/src/eea_datalakehouse/catalog/operations.py @@ -1,7 +1,7 @@ """The catalog operations: table2view, draft2version, publishversion, datacopy, datamove, deleteview, gettablesfrom, gettableitemsfrom, -getwikifrom, gettagsfrom, assignwikito, assigntagsto, deletetags, -createfolder, deletefolder. +getwikifrom, gettagsfrom, setwikito, deletewiki, setmeta2wiki, +getmetafromwiki, settagsto, deletetags, createfolder, deletefolder. Each takes a :class:`~eea_datalakehouse.catalog.sql.SqlExecutor` (REST or Flight — the operation doesn't care which) and an `idempotency_key`. On @@ -50,16 +50,16 @@ ``datacopy``/``datamove`` run several statements (existence check, folder creation, the CTAS/DROP itself) through the same executor, so this session reuse is what keeps them from paying a fresh handshake per step. -* ``getwikifrom`` / ``gettagsfrom`` / ``assignwikito`` / ``assigntagsto`` are +* ``getwikifrom`` / ``gettagsfrom`` / ``setwikito`` / ``settagsto`` are REST-only for the same reason: Dremio wikis/tags have no SQL or Flight equivalent, so they take a ``catalog_rest`` directly rather than a ``SqlExecutor``. ``getwikifrom`` raises if the entity has no wiki at all; ``gettagsfrom`` doesn't — zero tags is normal, only a missing path raises. -* ``assignwikito``/``assigntagsto`` create the wiki/tags if the entity has +* ``setwikito``/``settagsto`` create the wiki/tags if the entity has none yet, or overwrite them if it already has some — Dremio's collaboration API is versioned for optimistic concurrency, and the exact semantics of that version field are unverified against a real deployment - (see rest.py's docstring). ``assigntagsto`` *replaces* the tag set, it + (see rest.py's docstring). ``settagsto`` *replaces* the tag set, it doesn't merge with the existing tags. * ``createfolder``/``deletefolder`` are REST-only for the same reason — folder creation/deletion has no SQL or Flight equivalent. Both are @@ -75,6 +75,7 @@ from __future__ import annotations import inspect +import re from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Literal @@ -736,12 +737,14 @@ def gettagsfrom( ) -> list[str]: """The Dremio tags attached to the catalog entity at `path`. - REST-only (see module docstring) — takes `catalog_rest` directly rather - than a `SqlExecutor`. Raises `CatalogOperationError` only if `path` - itself doesn't exist; an entity with zero tags is a normal state and - returns an empty list, not an error (unlike `getwikifrom`). + Tables/views only (see `_require_table_or_view`) — raises + `CatalogOperationError` if `path` doesn't exist or isn't one. REST-only + (see module docstring) — takes `catalog_rest` directly rather than a + `SqlExecutor`. An entity with zero tags is a normal state and returns + an empty list, not an error (unlike `getwikifrom`). """ try: + _require_table_or_view(catalog_rest, path, "gettagsfrom") tags = catalog_rest.get_tags(path) except EngineStartingError as exc: retry_state.record(idempotency_key, "gettagsfrom", path, str(exc), params={"path": path}) @@ -750,11 +753,65 @@ def gettagsfrom( return tags -def assignwikito( +_REQUIRED_TAG_KEYS = frozenset({"tag_name", "tag_value", "tag_title"}) + +# What separates the caller's own wiki text from the "# Meta Data" section +# setwikito/setmeta2wiki append — the exact prefix _render_wiki_metadata's +# output gets joined onto, so splitting on it is exact, not a guess. +_META_MARKER = "\n\n# Meta Data\n" + +_TAG_ELEMENT_RE = re.compile(r'') + + +def _validate_tags(tags: list[dict[str, str]]) -> None: + missing = [t for t in tags if not _REQUIRED_TAG_KEYS <= t.keys()] + if missing: + raise CatalogOperationError( + f"each tag needs {sorted(_REQUIRED_TAG_KEYS)} — missing from: {missing!r}" + ) + + +def _render_wiki_metadata(tags: list[dict[str, str]]) -> str: + """The "# Meta Data" section `setwikito`/`setmeta2wiki` append when + `tags` is given. + + Dremio's wiki has no structured-metadata concept of its own — it's + plain markdown text — so this is the only way to carry name/value/title + tags through it: one human-readable "title : value" line per tag, + followed by the same data machine-readable as + ``...``. + """ + display_lines = [f"\t{tag['tag_title']} : {tag['tag_value']}" for tag in tags] + meta_lines = [ + f'' + for tag in tags + ] + return "\n".join(["# Meta Data", *display_lines, "", *meta_lines, ""]) + + +def _split_wiki_metadata(text: str) -> tuple[str, list[dict[str, str]]]: + """`(base_text, existing_tags)`: `base_text` is whatever came before the + "# Meta Data" section (or all of `text`, if there isn't one yet); + `existing_tags` is parsed back out of that section's `` block, + best-effort — a wiki hand-edited outside this format (or with no + section at all) just yields no tags, not an error. + """ + idx = text.find(_META_MARKER) + if idx == -1: + return text, [] + existing_tags = [ + {"tag_name": name, "tag_value": value, "tag_title": title} + for name, value, title in _TAG_ELEMENT_RE.findall(text[idx:]) + ] + return text[:idx], existing_tags + + +def setwikito( catalog_rest: CatalogRestClient, path: str, text: str, *, + tags: list[dict[str, str]] | None = None, idempotency_key: str, ) -> None: """Create or overwrite the Dremio wiki text on the catalog entity at `path`. @@ -762,18 +819,203 @@ def assignwikito( REST-only (see module docstring) — takes `catalog_rest` directly rather than a `SqlExecutor`. Raises `CatalogOperationError` if `path` doesn't exist. + + `tags`, if given, is a list of ``{"tag_name": ..., "tag_value": ..., + "tag_title": ...}`` dicts (raises `CatalogOperationError` immediately if + any is missing a key) — each rendered into a "# Meta Data" section + appended to `text` before it's sent (see `_render_wiki_metadata`). The + rendered text, metadata included, is what a retry remembers and + re-sends — `tags` itself is never persisted to retry_state. """ + if tags: + _validate_tags(tags) + text = f"{text}\n\n{_render_wiki_metadata(tags)}" + try: catalog_rest.set_wiki(path, text) except EngineStartingError as exc: retry_state.record( - idempotency_key, "assignwikito", path, str(exc), params={"path": path, "text": text} + idempotency_key, "setwikito", path, str(exc), params={"path": path, "text": text} ) raise retry_state.clear(idempotency_key) -def assigntagsto( +def deletewiki( + catalog_rest: CatalogRestClient, + path: str, + *, + idempotency_key: str, +) -> None: + """Delete the wiki text on the catalog entity at `path`, if it has one. + + REST-only (see module docstring) — takes `catalog_rest` directly rather + than a `SqlExecutor`. Idempotent: a missing `path`, or one with no wiki + at all, is a no-op, not an error — same as `deleteview`'s + ``DROP VIEW IF EXISTS``. Dremio's collaboration API has no separate + delete-wiki endpoint to speak of, so this clears it by setting the text + to empty (the same workaround `deletetags` uses for tags). + """ + try: + has_wiki = True + try: + catalog_rest.get_wiki(path) + except CatalogOperationError: + has_wiki = False # path missing entirely, or just has no wiki — nothing to delete + if has_wiki: + catalog_rest.set_wiki(path, "") + except EngineStartingError as exc: + retry_state.record(idempotency_key, "deletewiki", path, str(exc), params={"path": path}) + raise + retry_state.clear(idempotency_key) + + +def _require_folder(catalog_rest: CatalogRestClient, path: str, operation_name: str) -> None: + """Gate for `setmeta2wiki`/`getmetafromwiki`: both are a folder-only + stand-in for tags, since Dremio's own tags/labels only exist on + tables/views, not folders — so this scheme and Dremio's real tags are + meant to cover different levels, not overlap. + """ + if not catalog_rest.exists(path): + raise CatalogOperationError(f"{path!r} does not exist") + if not catalog_rest.is_folder(path): + raise CatalogOperationError( + f"{operation_name} only works on folders — {path!r} is not one " + "(tables/views have Dremio's own tags/labels for this instead)" + ) + + +def _require_table_or_view(catalog_rest: CatalogRestClient, path: str, operation_name: str) -> None: + """Gate for `gettagsfrom`/`settagsto`: the mirror image of + `_require_folder` — Dremio's own tags/labels only exist on + tables/views, not folders (see `setmeta2wiki`/`getmetafromwiki` for + the folder-level stand-in) — so this and that scheme are meant to + cover different levels, not overlap. + """ + if not catalog_rest.exists(path): + raise CatalogOperationError(f"{path!r} does not exist") + if not catalog_rest.is_table_or_view(path): + raise CatalogOperationError( + f"{operation_name} only works on tables/views — {path!r} is not one " + "(folders have setmeta2wiki/getmetafromwiki for this instead)" + ) + + +def setmeta2wiki( + catalog_rest: CatalogRestClient, + path: str, + *, + tags: list[dict[str, str]] | None = None, + overwrite: bool = True, + idempotency_key: str, +) -> None: + """Append or update the "# Meta Data" section (see `setwikito`) on + the wiki already at `path`, keeping whatever text comes before it. + + Folders only (see `_require_folder`) — raises `CatalogOperationError` + if `path` doesn't exist or isn't one. + + `overwrite=True` (the default) replaces the whole section with one + rendered fresh from `tags`. `overwrite=False` instead merges: the tags + already embedded in the current section (parsed back out of its + ```` block) plus `tags`, appended after them — nothing is + deduplicated, so a repeated `tag_name` shows up twice. + + Raises `CatalogOperationError` if any tag is missing a required key. + If `path` has no wiki yet, starts from empty base text rather than + raising — same as `setwikito` creating one from scratch. + """ + try: + _require_folder(catalog_rest, path, "setmeta2wiki") + + try: + current_text = catalog_rest.get_wiki(path) + except CatalogOperationError: + current_text = "" # exists, but no wiki yet + + base_text, existing_tags = _split_wiki_metadata(current_text) + + new_tags = list(tags or []) if overwrite else existing_tags + list(tags or []) + if new_tags: + _validate_tags(new_tags) + full_text = f"{base_text}\n\n{_render_wiki_metadata(new_tags)}" + else: + full_text = base_text + + catalog_rest.set_wiki(path, full_text) + except EngineStartingError as exc: + # Params mirror this function's own arguments (not the computed + # full_text) — a retry re-runs the whole thing from scratch, + # re-fetching and re-merging against whatever the wiki looks like + # by then, rather than blindly resending a stale precomputed string. + retry_state.record( + idempotency_key, + "setmeta2wiki", + path, + str(exc), + params={"path": path, "tags": tags, "overwrite": overwrite}, + ) + raise + retry_state.clear(idempotency_key) + + +def getmetafromwiki( + catalog_rest: CatalogRestClient, + path: str, + tag_name: str | None = None, + field: Literal["tag_value", "tag_title"] | None = None, + *, + idempotency_key: str, +) -> list[dict[str, str]] | dict[str, str]: + """The metadata tags `setmeta2wiki` embedded in the wiki at `path`. + + Folders only (see `_require_folder`) — raises `CatalogOperationError` + if `path` doesn't exist or isn't one. + + Without `tag_name` — or if it doesn't actually match any tag there — + returns every tag as a list of ``{"tag_name", "tag_value", + "tag_title"}`` dicts. With a matching `tag_name`, returns a single dict + for just that tag instead: both ``tag_value`` and ``tag_title`` if + `field` isn't given, or ``tag_name`` plus whichever one `field` asks + for (``"tag_value"`` or ``"tag_title"``). + """ + if field is not None and field not in ("tag_value", "tag_title"): + raise CatalogOperationError( + f'field must be "tag_value", "tag_title", or None — got {field!r}' + ) + + try: + _require_folder(catalog_rest, path, "getmetafromwiki") + try: + text = catalog_rest.get_wiki(path) + except CatalogOperationError: + text = "" # exists (as a folder), but no wiki yet + except EngineStartingError as exc: + retry_state.record( + idempotency_key, + "getmetafromwiki", + path, + str(exc), + params={"path": path, "tag_name": tag_name, "field": field}, + ) + raise + retry_state.clear(idempotency_key) + + _, tags = _split_wiki_metadata(text) + + if tag_name is None: + return tags + + match = next((t for t in tags if t["tag_name"] == tag_name), None) + if match is None: + return tags + + if field is None: + return dict(match) + return {"tag_name": match["tag_name"], field: match[field]} + + +def settagsto( catalog_rest: CatalogRestClient, path: str, tags: list[str], @@ -782,16 +1024,19 @@ def assigntagsto( ) -> None: """Replace the Dremio tags on the catalog entity at `path` with `tags`. - REST-only (see module docstring) — takes `catalog_rest` directly rather - than a `SqlExecutor`. Raises `CatalogOperationError` if `path` doesn't - exist. This *replaces* the tag set — pass the union of old and new tags - if you want to keep the existing ones (e.g. via `gettagsfrom` first). + Tables/views only (see `_require_table_or_view`) — raises + `CatalogOperationError` if `path` doesn't exist or isn't one. REST-only + (see module docstring) — takes `catalog_rest` directly rather than a + `SqlExecutor`. This *replaces* the tag set — pass the union of old and + new tags if you want to keep the existing ones (e.g. via `gettagsfrom` + first). """ try: + _require_table_or_view(catalog_rest, path, "settagsto") catalog_rest.set_tags(path, tags) except EngineStartingError as exc: retry_state.record( - idempotency_key, "assigntagsto", path, str(exc), params={"path": path, "tags": tags} + idempotency_key, "settagsto", path, str(exc), params={"path": path, "tags": tags} ) raise retry_state.clear(idempotency_key) @@ -908,8 +1153,11 @@ def deletefolder( "gettableitemsfrom": gettableitemsfrom, "getwikifrom": getwikifrom, "gettagsfrom": gettagsfrom, - "assignwikito": assignwikito, - "assigntagsto": assigntagsto, + "setwikito": setwikito, + "deletewiki": deletewiki, + "setmeta2wiki": setmeta2wiki, + "getmetafromwiki": getmetafromwiki, + "settagsto": settagsto, "deletetags": deletetags, "createfolder": createfolder, "deletefolder": deletefolder, diff --git a/src/eea_datalakehouse/catalog/rest.py b/src/eea_datalakehouse/catalog/rest.py index f9a4a74..16a6fd5 100644 --- a/src/eea_datalakehouse/catalog/rest.py +++ b/src/eea_datalakehouse/catalog/rest.py @@ -96,6 +96,26 @@ def is_folder(self, path: str) -> bool: return False return entity is not None and entity.get("entityType") == "folder" + def is_table_or_view(self, path: str) -> bool: + """Whether `path` names an existing table or view — a Dremio + "dataset" entity, not a folder/space/source. + + Used to gate `gettagsfrom`/`settagsto` to the entities that + actually have Dremio's own tags/labels concept — folders don't + (see `setmeta2wiki`/`getmetafromwiki` for the folder-level + stand-in). + + Best-effort, same reasoning as `is_folder`: some Dremio source + types reject by-path lookups into nested items outright (400, not + 404), so any lookup failure here is treated the same as "not a + match", never raised. + """ + try: + entity = self._lookup_by_path(path) + except CatalogOperationError: + return False + return entity is not None and entity.get("entityType") == "dataset" + def get_wiki(self, path: str) -> str: """The Dremio wiki text attached to the catalog entity at `path`. diff --git a/tests/catalog/conftest.py b/tests/catalog/conftest.py index 91b5879..93f5db0 100644 --- a/tests/catalog/conftest.py +++ b/tests/catalog/conftest.py @@ -107,6 +107,12 @@ def exists(self, path: str) -> bool: def is_folder(self, path: str) -> bool: return path in self.folders + def is_table_or_view(self, path: str) -> bool: + # Every seeded path is either a folder (if also in self.folders) or, + # by default, a table/view — mirrors a real entity always being + # exactly one specific type. + return path in self.existing and path not in self.folders + def get_wiki(self, path: str) -> str: from eea_datalakehouse.catalog.errors import CatalogOperationError diff --git a/tests/catalog/test_client.py b/tests/catalog/test_client.py index 0cb5b8a..f2f6401 100644 --- a/tests/catalog/test_client.py +++ b/tests/catalog/test_client.py @@ -248,20 +248,73 @@ def test_gettagsfrom_delegates_to_operations() -> None: assert tags == ["pii"] -def test_assignwikito_delegates_to_operations() -> None: +def test_setwikito_delegates_to_operations() -> None: fake_rest = FakeCatalogRest(existing={"a.b"}) catalog = Catalog(BASE_URL, "pat", executor=FakeExecutor(), catalog_rest=fake_rest) - catalog.assignwikito("a.b", "# New docs", idempotency_key="k") + catalog.setwikito("a.b", "# New docs", idempotency_key="k") assert fake_rest._wikis["a.b"] == "# New docs" -def test_assigntagsto_delegates_to_operations() -> None: +def test_deletewiki_delegates_to_operations() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, wikis={"a.b": "# Docs"}) + catalog = Catalog(BASE_URL, "pat", executor=FakeExecutor(), catalog_rest=fake_rest) + + catalog.deletewiki("a.b", idempotency_key="k") + + assert fake_rest._wikis["a.b"] == "" + + +def test_setwikito_with_tags_delegates_to_operations() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}) + catalog = Catalog(BASE_URL, "pat", executor=FakeExecutor(), catalog_rest=fake_rest) + tags = [{"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}] + + catalog.setwikito("a.b", "# New docs", tags=tags, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == ( + '# New docs\n\n# Meta Data\n\tOwner : bwd-team\n\n' + '\n' + ) + + +def test_setmeta2wiki_delegates_to_operations() -> None: + existing_wiki = ( + '# Docs\n\n# Meta Data\n\tOwner : bwd-team\n\n' + '\n' + ) + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": existing_wiki}) + catalog = Catalog(BASE_URL, "pat", executor=FakeExecutor(), catalog_rest=fake_rest) + new_tags = [{"tag_name": "status", "tag_value": "published", "tag_title": "Status"}] + + catalog.setmeta2wiki("a.b", tags=new_tags, overwrite=False, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == ( + '# Docs\n\n# Meta Data\n\tOwner : bwd-team\n\tStatus : published\n\n' + '\n' + '\n' + ) + + +def test_getmetafromwiki_delegates_to_operations() -> None: + existing_wiki = ( + '# Docs\n\n# Meta Data\n\tOwner : bwd-team\n\n' + '\n' + ) + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": existing_wiki}) + catalog = Catalog(BASE_URL, "pat", executor=FakeExecutor(), catalog_rest=fake_rest) + + result = catalog.getmetafromwiki("a.b", "owner", "tag_value", idempotency_key="k") + + assert result == {"tag_name": "owner", "tag_value": "bwd-team"} + + +def test_settagsto_delegates_to_operations() -> None: fake_rest = FakeCatalogRest(existing={"a.b"}) catalog = Catalog(BASE_URL, "pat", executor=FakeExecutor(), catalog_rest=fake_rest) - catalog.assigntagsto("a.b", ["pii"], idempotency_key="k") + catalog.settagsto("a.b", ["pii"], idempotency_key="k") assert fake_rest._tags["a.b"] == ["pii"] diff --git a/tests/catalog/test_operations.py b/tests/catalog/test_operations.py index 91ddfad..2274121 100644 --- a/tests/catalog/test_operations.py +++ b/tests/catalog/test_operations.py @@ -677,9 +677,17 @@ def test_gettagsfrom_raises_when_path_does_not_exist() -> None: operations.gettagsfrom(fake_rest, "a.missing", idempotency_key="k") +def test_gettagsfrom_raises_when_path_is_a_folder() -> None: + # Folders have setmeta2wiki/getmetafromwiki for this instead. + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}) + + with pytest.raises(CatalogOperationError, match="only works on tables/views"): + operations.gettagsfrom(fake_rest, "a.b", idempotency_key="k") + + def test_gettagsfrom_engine_starting_is_remembered_and_retryable(executor) -> None: fake_rest = FakeCatalogRest( - raise_on_get_tags=EngineStartingError("stalled", idempotency_key="k") + existing={"a.b"}, raise_on_get_tags=EngineStartingError("stalled", idempotency_key="k") ) with pytest.raises(EngineStartingError): @@ -691,37 +699,36 @@ def test_gettagsfrom_engine_starting_is_remembered_and_retryable(executor) -> No assert pending.params == {"path": "a.b"} fake_rest._raise_on_get_tags = None - fake_rest.existing.add("a.b") fake_rest._tags["a.b"] = ["pii"] assert operations.retry_pending(executor, "k", catalog_rest=fake_rest) == ["pii"] -def test_assignwikito_sets_the_wiki() -> None: +def test_setwikito_sets_the_wiki() -> None: fake_rest = FakeCatalogRest(existing={"a.b"}) - operations.assignwikito(fake_rest, "a.b", "# New docs", idempotency_key="k") + operations.setwikito(fake_rest, "a.b", "# New docs", idempotency_key="k") assert fake_rest._wikis["a.b"] == "# New docs" -def test_assignwikito_raises_when_path_does_not_exist() -> None: +def test_setwikito_raises_when_path_does_not_exist() -> None: fake_rest = FakeCatalogRest() with pytest.raises(CatalogOperationError, match="does not exist"): - operations.assignwikito(fake_rest, "a.missing", "text", idempotency_key="k") + operations.setwikito(fake_rest, "a.missing", "text", idempotency_key="k") -def test_assignwikito_engine_starting_is_remembered_and_retryable(executor) -> None: +def test_setwikito_engine_starting_is_remembered_and_retryable(executor) -> None: fake_rest = FakeCatalogRest( existing={"a.b"}, raise_on_set_wiki=EngineStartingError("stalled", idempotency_key="k") ) with pytest.raises(EngineStartingError): - operations.assignwikito(fake_rest, "a.b", "# New docs", idempotency_key="k") + operations.setwikito(fake_rest, "a.b", "# New docs", idempotency_key="k") pending = retry_state.get("k") assert pending is not None - assert pending.operation == "assignwikito" + assert pending.operation == "setwikito" assert pending.params == {"path": "a.b", "text": "# New docs"} fake_rest._raise_on_set_wiki = None @@ -729,32 +736,382 @@ def test_assignwikito_engine_starting_is_remembered_and_retryable(executor) -> N assert fake_rest._wikis["a.b"] == "# New docs" -def test_assigntagsto_sets_the_tags() -> None: +def test_setwikito_appends_metadata_section_when_tags_given() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}) + tags = [ + {"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}, + {"tag_name": "status", "tag_value": "published", "tag_title": "Status"}, + ] + + operations.setwikito(fake_rest, "a.b", "# Docs", tags=tags, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == ( + "# Docs\n\n" + "# Meta Data\n" + "\tOwner : bwd-team\n" + "\tStatus : published\n" + "\n" + '\n' + '\n' + "" + ) + + +def test_setwikito_without_tags_leaves_text_untouched() -> None: fake_rest = FakeCatalogRest(existing={"a.b"}) - operations.assigntagsto(fake_rest, "a.b", ["pii", "reviewed"], idempotency_key="k") + operations.setwikito(fake_rest, "a.b", "# Docs", idempotency_key="k") + + assert fake_rest._wikis["a.b"] == "# Docs" + + +def test_setwikito_raises_when_a_tag_is_missing_a_key() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}) + tags = [{"tag_name": "owner", "tag_value": "bwd-team"}] # no tag_title + + with pytest.raises(CatalogOperationError, match="tag_title"): + operations.setwikito(fake_rest, "a.b", "# Docs", tags=tags, idempotency_key="k") + + # Fails before ever calling set_wiki. + assert "a.b" not in fake_rest._wikis + + +def test_setwikito_retry_resends_the_already_rendered_metadata(executor) -> None: + fake_rest = FakeCatalogRest( + existing={"a.b"}, raise_on_set_wiki=EngineStartingError("stalled", idempotency_key="k") + ) + tags = [{"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}] + + with pytest.raises(EngineStartingError): + operations.setwikito(fake_rest, "a.b", "# Docs", tags=tags, idempotency_key="k") + + pending = retry_state.get("k") + assert pending is not None + # tags itself is never persisted — only the fully-rendered text. + assert "tags" not in pending.params + assert pending.params["text"] == ( + '# Docs\n\n# Meta Data\n\tOwner : bwd-team\n\n' + '\n' + ) + + fake_rest._raise_on_set_wiki = None + operations.retry_pending(executor, "k", catalog_rest=fake_rest) + assert fake_rest._wikis["a.b"] == pending.params["text"] + + +def test_deletewiki_clears_an_existing_wiki() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, wikis={"a.b": "# Docs"}) + + operations.deletewiki(fake_rest, "a.b", idempotency_key="k") + + assert fake_rest._wikis["a.b"] == "" + + +def test_deletewiki_is_a_no_op_when_entity_has_no_wiki() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}) # exists, but no wiki set + + operations.deletewiki(fake_rest, "a.b", idempotency_key="k") + + assert "a.b" not in fake_rest._wikis + + +def test_deletewiki_is_a_no_op_when_path_does_not_exist() -> None: + fake_rest = FakeCatalogRest() + + operations.deletewiki(fake_rest, "a.missing", idempotency_key="k") # must not raise + + assert "a.missing" not in fake_rest._wikis + + +def test_deletewiki_engine_starting_is_remembered_and_retryable(executor) -> None: + fake_rest = FakeCatalogRest( + existing={"a.b"}, + wikis={"a.b": "# Docs"}, + raise_on_get_wiki=EngineStartingError("stalled", idempotency_key="k"), + ) + + with pytest.raises(EngineStartingError): + operations.deletewiki(fake_rest, "a.b", idempotency_key="k") + + pending = retry_state.get("k") + assert pending is not None + assert pending.operation == "deletewiki" + assert pending.params == {"path": "a.b"} + + fake_rest._raise_on_get_wiki = None + operations.retry_pending(executor, "k", catalog_rest=fake_rest) + assert fake_rest._wikis["a.b"] == "" + + +_EXISTING_WIKI_WITH_META = ( + "# Docs\n\nOriginal wiki text.\n\n" + "# Meta Data\n" + "\tOwner : bwd-team\n" + "\n" + '\n' + "" +) + + +def test_setmeta2wiki_overwrite_replaces_the_existing_meta_section() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _EXISTING_WIKI_WITH_META}) + new_tags = [{"tag_name": "status", "tag_value": "published", "tag_title": "Status"}] + + operations.setmeta2wiki(fake_rest, "a.b", tags=new_tags, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == ( + "# Docs\n\nOriginal wiki text.\n\n" + "# Meta Data\n" + "\tStatus : published\n" + "\n" + '\n' + "" + ) + + +def test_setmeta2wiki_no_overwrite_merges_with_existing_tags() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _EXISTING_WIKI_WITH_META}) + new_tags = [{"tag_name": "status", "tag_value": "published", "tag_title": "Status"}] + + operations.setmeta2wiki(fake_rest, "a.b", tags=new_tags, overwrite=False, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == ( + "# Docs\n\nOriginal wiki text.\n\n" + "# Meta Data\n" + "\tOwner : bwd-team\n" + "\tStatus : published\n" + "\n" + '\n' + '\n' + "" + ) + + +def test_setmeta2wiki_creates_meta_section_when_none_exists_yet() -> None: + fake_rest = FakeCatalogRest( + existing={"a.b"}, folders={"a.b"}, wikis={"a.b": "# Docs\n\nPlain wiki, no meta yet."} + ) + new_tags = [{"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}] + + operations.setmeta2wiki(fake_rest, "a.b", tags=new_tags, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == ( + "# Docs\n\nPlain wiki, no meta yet.\n\n" + "# Meta Data\n" + "\tOwner : bwd-team\n" + "\n" + '\n' + "" + ) + + +def test_setmeta2wiki_starts_from_empty_base_text_when_no_wiki_yet() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}) # exists, but no wiki set at all + new_tags = [{"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}] + + operations.setmeta2wiki(fake_rest, "a.b", tags=new_tags, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == ( + "\n\n# Meta Data\n" + "\tOwner : bwd-team\n" + "\n" + '\n' + "" + ) + + +def test_setmeta2wiki_raises_when_path_does_not_exist() -> None: + fake_rest = FakeCatalogRest() + + with pytest.raises(CatalogOperationError, match="does not exist"): + operations.setmeta2wiki(fake_rest, "a.missing", idempotency_key="k") + + +def test_setmeta2wiki_raises_when_path_is_not_a_folder() -> None: + # Exists, but as a table/view, not a folder — those have Dremio's own + # tags/labels for this instead. + fake_rest = FakeCatalogRest(existing={"a.b"}) + + with pytest.raises(CatalogOperationError, match="only works on folders"): + operations.setmeta2wiki(fake_rest, "a.b", idempotency_key="k") + + +def test_setmeta2wiki_raises_when_a_tag_is_missing_a_key() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": "# Docs"}) + bad_tags = [{"tag_name": "owner", "tag_value": "bwd-team"}] # no tag_title + + with pytest.raises(CatalogOperationError, match="tag_title"): + operations.setmeta2wiki(fake_rest, "a.b", tags=bad_tags, idempotency_key="k") + + assert fake_rest._wikis["a.b"] == "# Docs" + + +def test_setmeta2wiki_engine_starting_is_remembered_and_retryable(executor) -> None: + fake_rest = FakeCatalogRest( + existing={"a.b"}, + folders={"a.b"}, + wikis={"a.b": "# Docs"}, + raise_on_set_wiki=EngineStartingError("stalled", idempotency_key="k"), + ) + new_tags = [{"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}] + + with pytest.raises(EngineStartingError): + operations.setmeta2wiki(fake_rest, "a.b", tags=new_tags, idempotency_key="k") + + pending = retry_state.get("k") + assert pending is not None + assert pending.operation == "setmeta2wiki" + assert pending.params == {"path": "a.b", "tags": new_tags, "overwrite": True} + + fake_rest._raise_on_set_wiki = None + operations.retry_pending(executor, "k", catalog_rest=fake_rest) + assert fake_rest._wikis["a.b"] == ( + "# Docs\n\n" + "# Meta Data\n" + "\tOwner : bwd-team\n" + "\n" + '\n' + "" + ) + + +_WIKI_WITH_TWO_TAGS = ( + "# Docs\n\n" + "# Meta Data\n" + "\tOwner : bwd-team\n" + "\tStatus : published\n" + "\n" + '\n' + '\n' + "" +) +_ALL_TAGS = [ + {"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"}, + {"tag_name": "status", "tag_value": "published", "tag_title": "Status"}, +] + + +def test_getmetafromwiki_without_tag_name_returns_every_tag() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _WIKI_WITH_TWO_TAGS}) + + result = operations.getmetafromwiki(fake_rest, "a.b", idempotency_key="k") + + assert result == _ALL_TAGS + + +def test_getmetafromwiki_with_unknown_tag_name_returns_every_tag() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _WIKI_WITH_TWO_TAGS}) + + result = operations.getmetafromwiki(fake_rest, "a.b", "no-such-tag", idempotency_key="k") + + assert result == _ALL_TAGS + + +def test_getmetafromwiki_with_tag_name_and_no_field_returns_both() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _WIKI_WITH_TWO_TAGS}) + + result = operations.getmetafromwiki(fake_rest, "a.b", "owner", idempotency_key="k") + + assert result == {"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"} + + +def test_getmetafromwiki_with_tag_name_and_tag_value_field() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _WIKI_WITH_TWO_TAGS}) + + result = operations.getmetafromwiki(fake_rest, "a.b", "owner", "tag_value", idempotency_key="k") + + assert result == {"tag_name": "owner", "tag_value": "bwd-team"} + + +def test_getmetafromwiki_with_tag_name_and_tag_title_field() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _WIKI_WITH_TWO_TAGS}) + + result = operations.getmetafromwiki(fake_rest, "a.b", "owner", "tag_title", idempotency_key="k") + + assert result == {"tag_name": "owner", "tag_title": "Owner"} + + +def test_getmetafromwiki_raises_on_invalid_field() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}, wikis={"a.b": _WIKI_WITH_TWO_TAGS}) + + with pytest.raises(CatalogOperationError, match="field must be"): + operations.getmetafromwiki(fake_rest, "a.b", "owner", "bogus", idempotency_key="k") + + +def test_getmetafromwiki_returns_empty_list_when_no_wiki_yet() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}) + + assert operations.getmetafromwiki(fake_rest, "a.b", idempotency_key="k") == [] + + +def test_getmetafromwiki_raises_when_path_does_not_exist() -> None: + fake_rest = FakeCatalogRest() + + with pytest.raises(CatalogOperationError, match="does not exist"): + operations.getmetafromwiki(fake_rest, "a.missing", idempotency_key="k") + + +def test_getmetafromwiki_raises_when_path_is_not_a_folder() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}) # not in folders + + with pytest.raises(CatalogOperationError, match="only works on folders"): + operations.getmetafromwiki(fake_rest, "a.b", idempotency_key="k") + + +def test_getmetafromwiki_engine_starting_is_remembered_and_retryable(executor) -> None: + fake_rest = FakeCatalogRest( + existing={"a.b"}, + folders={"a.b"}, + wikis={"a.b": _WIKI_WITH_TWO_TAGS}, + raise_on_get_wiki=EngineStartingError("stalled", idempotency_key="k"), + ) + + with pytest.raises(EngineStartingError): + operations.getmetafromwiki(fake_rest, "a.b", "owner", idempotency_key="k") + + pending = retry_state.get("k") + assert pending is not None + assert pending.operation == "getmetafromwiki" + assert pending.params == {"path": "a.b", "tag_name": "owner", "field": None} + + fake_rest._raise_on_get_wiki = None + result = operations.retry_pending(executor, "k", catalog_rest=fake_rest) + assert result == {"tag_name": "owner", "tag_value": "bwd-team", "tag_title": "Owner"} + + +def test_settagsto_sets_the_tags() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}) + + operations.settagsto(fake_rest, "a.b", ["pii", "reviewed"], idempotency_key="k") assert fake_rest._tags["a.b"] == ["pii", "reviewed"] -def test_assigntagsto_raises_when_path_does_not_exist() -> None: +def test_settagsto_raises_when_path_does_not_exist() -> None: fake_rest = FakeCatalogRest() with pytest.raises(CatalogOperationError, match="does not exist"): - operations.assigntagsto(fake_rest, "a.missing", ["pii"], idempotency_key="k") + operations.settagsto(fake_rest, "a.missing", ["pii"], idempotency_key="k") + + +def test_settagsto_raises_when_path_is_a_folder() -> None: + fake_rest = FakeCatalogRest(existing={"a.b"}, folders={"a.b"}) + + with pytest.raises(CatalogOperationError, match="only works on tables/views"): + operations.settagsto(fake_rest, "a.b", ["pii"], idempotency_key="k") -def test_assigntagsto_engine_starting_is_remembered_and_retryable(executor) -> None: +def test_settagsto_engine_starting_is_remembered_and_retryable(executor) -> None: fake_rest = FakeCatalogRest( existing={"a.b"}, raise_on_set_tags=EngineStartingError("stalled", idempotency_key="k") ) with pytest.raises(EngineStartingError): - operations.assigntagsto(fake_rest, "a.b", ["pii"], idempotency_key="k") + operations.settagsto(fake_rest, "a.b", ["pii"], idempotency_key="k") pending = retry_state.get("k") assert pending is not None - assert pending.operation == "assigntagsto" + assert pending.operation == "settagsto" assert pending.params == {"path": "a.b", "tags": ["pii"]} fake_rest._raise_on_set_tags = None diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 4f6c710..522e166 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -96,6 +96,52 @@ def test_is_folder_false_on_a_broken_by_path_lookup_rather_than_raising() -> Non assert client.is_folder("catalog.deep.nested") is False +@respx.mock +def test_is_table_or_view_true_for_a_dataset_entity() -> None: + respx.get(f"{BASE_URL}/api/v3/catalog/by-path/a/b").mock( + return_value=httpx.Response(200, json={"entityType": "dataset"}) + ) + client = CatalogRestClient(BASE_URL, "pat") + + assert client.is_table_or_view("a.b") is True + + +@respx.mock +def test_is_table_or_view_false_for_a_folder_entity() -> None: + respx.get(f"{BASE_URL}/api/v3/catalog/by-path/a/b").mock( + return_value=httpx.Response(200, json={"entityType": "folder"}) + ) + client = CatalogRestClient(BASE_URL, "pat") + + assert client.is_table_or_view("a.b") is False + + +@respx.mock +def test_is_table_or_view_false_on_404() -> None: + respx.get(f"{BASE_URL}/api/v3/catalog/by-path/a/missing").mock(return_value=httpx.Response(404)) + client = CatalogRestClient(BASE_URL, "pat") + + assert client.is_table_or_view("a.missing") is False + + +@respx.mock +def test_is_table_or_view_false_on_a_broken_by_path_lookup_rather_than_raising() -> None: + respx.get(f"{BASE_URL}/api/v3/catalog/by-path/catalog/deep/nested").mock( + return_value=httpx.Response( + 400, + json={ + "errorMessage": ( + "Can not get internal item from non-filesystem source [catalog] " + "of type [com.dremio.plugins.dremiocatalog.store.DremioCatalogLocalPlugin]" + ) + }, + ) + ) + client = CatalogRestClient(BASE_URL, "pat") + + assert client.is_table_or_view("catalog.deep.nested") is False + + @respx.mock def test_ensure_folder_path_raises_when_space_missing() -> None: respx.get(f"{BASE_URL}/api/v3/catalog/by-path/a").mock(return_value=httpx.Response(404))