From 9a21cfefa150c5c83800490ed75392f7e0677ca2 Mon Sep 17 00:00:00 2001 From: Mike McCann Date: Tue, 18 Aug 2026 09:11:15 -0700 Subject: [PATCH] Withhold notification until all delayed mode data have been transferred from the vehicle. --- src/data/lrauv_deployment_plots.py | 49 ++++++++++++++--- src/data/test_lrauv_deployment_plots.py | 70 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 6 deletions(-) diff --git a/src/data/lrauv_deployment_plots.py b/src/data/lrauv_deployment_plots.py index d4648e5..8c6860b 100755 --- a/src/data/lrauv_deployment_plots.py +++ b/src/data/lrauv_deployment_plots.py @@ -139,20 +139,26 @@ def _nc_files_for_dir(self, deployment_dir: Path, rel_dir: str) -> list[str]: opendap_base = LRAUV_OPENDAP_BASE.rstrip("/") + "/" + rel_path return [opendap_base + name for name in nc_names] - def _collect_nc_files(self, deployment_dir: Path, dlist_content: str) -> list[Path | str]: - """Return *_{FREQ}.nc files (local Paths or OPeNDAP URL strings) for - each log directory listed in the .dlist. + def _dlist_log_dirs(self, dlist_content: str) -> list[str]: + """Return the log-directory names listed in .dlist content. Non-comment, non-empty lines in the .dlist are timestamp subdirectory names (e.g. ``20230213T183535``). Lines starting with ``#`` are skipped — including commented-out directories for short/excluded runs. - Missing or empty subdirectories generate a warning and are skipped. """ - log_dirs = [ + return [ line.strip() for line in dlist_content.splitlines() if line.strip() and not line.strip().startswith("#") ] + + def _collect_nc_files(self, deployment_dir: Path, dlist_content: str) -> list[Path | str]: + """Return *_{FREQ}.nc files (local Paths or OPeNDAP URL strings) for + each log directory listed in the .dlist. + + Missing or empty subdirectories generate a warning and are skipped. + """ + log_dirs = self._dlist_log_dirs(dlist_content) if not log_dirs: self.logger.warning("No log directories found in dlist content") return [] @@ -163,6 +169,16 @@ def _collect_nc_files(self, deployment_dir: Path, dlist_content: str) -> list[Pa return nc_files + def _pending_log_dirs(self, dlist_content: str, nc_files: list[Path | str]) -> list[str]: + """Return .dlist log directories with no corresponding nc_file yet. + + A non-empty result means the vehicle-to-archive data transfer for this + deployment is still in progress — used to withhold notifications until + every log directory in the .dlist has produced data. + """ + found_dirs = {str(f).rsplit("/", 2)[1] for f in nc_files} + return [d for d in self._dlist_log_dirs(dlist_content) if d not in found_dirs] + def _concat_datasets(self, nc_files: list[Path | str]) -> xr.Dataset | None: """Concatenate per-log datasets into a single deployment-wide Dataset. @@ -382,6 +398,14 @@ def plot_deployment( # noqa: C901, PLR0912, PLR0913, PLR0915 for f in nc_files: self.logger.info(" %s", f) + pending_log_dirs = self._pending_log_dirs(dlist_content, nc_files) + if pending_log_dirs: + self.logger.info( + "%d log dir(s) not yet transferred: %s", + len(pending_log_dirs), + ", ".join(sorted(pending_log_dirs)), + ) + if not force and self._deployment_has_outputs(deployment_dir, plot_name_stem, nc_files): self.logger.info( "Outputs already exist and are up to date for %s, skipping" @@ -445,6 +469,7 @@ def plot_deployment( # noqa: C901, PLR0912, PLR0913, PLR0915 update_ssds_provenance=update_ssds_provenance, force=force, notify=notify, + pending_log_dirs=pending_log_dirs, ) def _build_and_write_html( # noqa: PLR0913 @@ -460,6 +485,7 @@ def _build_and_write_html( # noqa: PLR0913 update_ssds_provenance: bool = False, # noqa: FBT001, FBT002 force: bool = False, # noqa: FBT001, FBT002 notify: list[str] | None = None, + pending_log_dirs: list[str] | None = None, ) -> None: """Fetch STOQS permalink and write per-PNG HTML pages.""" dlist_no_ext = str(Path(dlist).with_suffix("")) @@ -493,7 +519,18 @@ def _build_and_write_html( # noqa: PLR0913 archiver = Archiver(add_handlers=True, clobber=True) archiver.logger.setLevel(self._log_levels[min(verbose, 2)]) archiver.copy_lrauv_deployment(deployment_dir, plot_name_stem) - self._notify(notify, raw_name or plot_name_stem, html_paths, force=force) + if pending_log_dirs: + if notify is not None: + self.logger.warning( + "Processing finished but withholding notification for %s:" + " still waiting for %d log dir(s) to finish transferring" + " from the vehicle: %s", + raw_name or plot_name_stem, + len(pending_log_dirs), + ", ".join(sorted(pending_log_dirs)), + ) + else: + self._notify(notify, raw_name or plot_name_stem, html_paths, force=force) if update_ssds_provenance: self._submit_provenance( deployment_dir=deployment_dir, diff --git a/src/data/test_lrauv_deployment_plots.py b/src/data/test_lrauv_deployment_plots.py index 218baf1..eeb25ee 100644 --- a/src/data/test_lrauv_deployment_plots.py +++ b/src/data/test_lrauv_deployment_plots.py @@ -415,6 +415,76 @@ def test_missing_png_produces_no_html(self, dp, tmp_path): assert not (tmp_path / "ghost.html").exists() # noqa: S101 +class TestPendingLogDirs: + """_pending_log_dirs() identifies .dlist log directories still missing data, + i.e. still being transferred from the vehicle to the archive filesystem.""" + + _DLIST_CONTENT = "# Deployment name: CANON April 2025\n20250414T120000\n" + + def test_empty_when_all_dirs_present(self, dp): + assert dp._pending_log_dirs(self._DLIST_CONTENT, [_NC_URL]) == [] # noqa: S101 + + def test_lists_dir_with_no_nc_file(self, dp): + dlist_content = self._DLIST_CONTENT + "20250415T080000\n" + assert dp._pending_log_dirs(dlist_content, [_NC_URL]) == [ # noqa: S101 + "20250415T080000" + ] + + def test_ignores_commented_out_dirs(self, dp): + dlist_content = self._DLIST_CONTENT + "# 20250415T080000\n" + assert dp._pending_log_dirs(dlist_content, [_NC_URL]) == [] # noqa: S101 + + +class TestNotificationGating: + """_build_and_write_html() must withhold --notify until every .dlist log + directory has produced data, logging a warning instead in the meantime.""" + + _DLIST = "ahi/missionlogs/2025/20250414_20250418.dlist" + + def _call(self, dp, tmp_path, *, pending_log_dirs, notify=None): + png = tmp_path / "CANON_April_2025_2column_cmocean.png" + png.touch() + with ( + patch("make_permalink.requests.Session") as mock_session_cls, + patch.object(dp, "_url_exists", return_value=False), + patch.object(dp, "_stoqs_url_for_nc_url", return_value=None), + patch.object(dp, "_notify") as mock_notify, + ): + mock_session_cls.return_value.__enter__.return_value.get = _mock_session_get("7") + dp._build_and_write_html( + tmp_path, + self._DLIST, + "CANON_April_2025", + "CANON April 2025", + _make_ds("2025-04-14"), + [str(png)], + [_NC_URL], + notify=notify, + pending_log_dirs=pending_log_dirs, + ) + return mock_notify + + def test_notifies_when_fully_transferred(self, dp, tmp_path): + mock_notify = self._call(dp, tmp_path, pending_log_dirs=[], notify=["a@b.com"]) + mock_notify.assert_called_once() + + def test_withholds_notification_when_pending(self, dp, tmp_path, caplog): + caplog.set_level("WARNING") + mock_notify = self._call( + dp, tmp_path, pending_log_dirs=["20250415T080000"], notify=["a@b.com"] + ) + mock_notify.assert_not_called() + assert "still waiting" in caplog.text # noqa: S101 + assert "20250415T080000" in caplog.text # noqa: S101 + + def test_no_warning_when_notify_not_requested(self, dp, tmp_path, caplog): + """Nothing was ever going to be sent, so withholding it isn't warning-worthy.""" + caplog.set_level("WARNING") + mock_notify = self._call(dp, tmp_path, pending_log_dirs=["20250415T080000"], notify=None) + mock_notify.assert_not_called() + assert "still waiting" not in caplog.text # noqa: S101 + + class TestUpdateIndexHtml: """Regression tests for quick_look_plots.html accumulating multiple deployments.