Withhold notification until all delayed mode data have been transferred from the vehicle.

src/data/lrauv_deployment_plots.py

@@ -139,20 +139,26 @@ def _nc_files_for_dir(self, deployment_dir: Path, rel_dir: str) -> list[str]:
         opendap_base = LRAUV_OPENDAP_BASE.rstrip("/") + "/" + rel_path
         return [opendap_base + name for name in nc_names]
 
-    def _collect_nc_files(self, deployment_dir: Path, dlist_content: str) -> list[Path | str]:
-        """Return *_{FREQ}.nc files (local Paths or OPeNDAP URL strings) for
-        each log directory listed in the .dlist.
+    def _dlist_log_dirs(self, dlist_content: str) -> list[str]:
+        """Return the log-directory names listed in .dlist content.
 
         Non-comment, non-empty lines in the .dlist are timestamp subdirectory
         names (e.g. ``20230213T183535``).  Lines starting with ``#`` are
         skipped — including commented-out directories for short/excluded runs.
-        Missing or empty subdirectories generate a warning and are skipped.
         """
-        log_dirs = [
+        return [
             line.strip()
             for line in dlist_content.splitlines()
             if line.strip() and not line.strip().startswith("#")
         ]
+
+    def _collect_nc_files(self, deployment_dir: Path, dlist_content: str) -> list[Path | str]:
+        """Return *_{FREQ}.nc files (local Paths or OPeNDAP URL strings) for
+        each log directory listed in the .dlist.
+
+        Missing or empty subdirectories generate a warning and are skipped.
+        """
+        log_dirs = self._dlist_log_dirs(dlist_content)
         if not log_dirs:
             self.logger.warning("No log directories found in dlist content")
             return []
@@ -163,6 +169,16 @@ def _collect_nc_files(self, deployment_dir: Path, dlist_content: str) -> list[Pa
 
         return nc_files
 
+    def _pending_log_dirs(self, dlist_content: str, nc_files: list[Path | str]) -> list[str]:
+        """Return .dlist log directories with no corresponding nc_file yet.
+
+        A non-empty result means the vehicle-to-archive data transfer for this
+        deployment is still in progress — used to withhold notifications until
+        every log directory in the .dlist has produced data.
+        """
+        found_dirs = {str(f).rsplit("/", 2)[1] for f in nc_files}
+        return [d for d in self._dlist_log_dirs(dlist_content) if d not in found_dirs]
+
     def _concat_datasets(self, nc_files: list[Path | str]) -> xr.Dataset | None:
         """Concatenate per-log datasets into a single deployment-wide Dataset.
 
@@ -382,6 +398,14 @@ def plot_deployment(  # noqa: C901, PLR0912, PLR0913, PLR0915
         for f in nc_files:
             self.logger.info("  %s", f)
 
+        pending_log_dirs = self._pending_log_dirs(dlist_content, nc_files)
+        if pending_log_dirs:
+            self.logger.info(
+                "%d log dir(s) not yet transferred: %s",
+                len(pending_log_dirs),
+                ", ".join(sorted(pending_log_dirs)),
+            )
+
         if not force and self._deployment_has_outputs(deployment_dir, plot_name_stem, nc_files):
             self.logger.info(
                 "Outputs already exist and are up to date for %s, skipping"
@@ -445,6 +469,7 @@ def plot_deployment(  # noqa: C901, PLR0912, PLR0913, PLR0915
                 update_ssds_provenance=update_ssds_provenance,
                 force=force,
                 notify=notify,
+                pending_log_dirs=pending_log_dirs,
             )
 
     def _build_and_write_html(  # noqa: PLR0913
@@ -460,6 +485,7 @@ def _build_and_write_html(  # noqa: PLR0913
         update_ssds_provenance: bool = False,  # noqa: FBT001, FBT002
         force: bool = False,  # noqa: FBT001, FBT002
         notify: list[str] | None = None,
+        pending_log_dirs: list[str] | None = None,
     ) -> None:
         """Fetch STOQS permalink and write per-PNG HTML pages."""
         dlist_no_ext = str(Path(dlist).with_suffix(""))
@@ -493,7 +519,18 @@ def _build_and_write_html(  # noqa: PLR0913
         archiver = Archiver(add_handlers=True, clobber=True)
         archiver.logger.setLevel(self._log_levels[min(verbose, 2)])
         archiver.copy_lrauv_deployment(deployment_dir, plot_name_stem)
-        self._notify(notify, raw_name or plot_name_stem, html_paths, force=force)
+        if pending_log_dirs:
+            if notify is not None:
+                self.logger.warning(
+                    "Processing finished but withholding notification for %s:"
+                    " still waiting for %d log dir(s) to finish transferring"
+                    " from the vehicle: %s",
+                    raw_name or plot_name_stem,
+                    len(pending_log_dirs),
+                    ", ".join(sorted(pending_log_dirs)),
+                )
+        else:
+            self._notify(notify, raw_name or plot_name_stem, html_paths, force=force)
         if update_ssds_provenance:
             self._submit_provenance(
                 deployment_dir=deployment_dir,

src/data/test_lrauv_deployment_plots.py

@@ -415,6 +415,76 @@ def test_missing_png_produces_no_html(self, dp, tmp_path):
         assert not (tmp_path / "ghost.html").exists()  # noqa: S101
 
 
+class TestPendingLogDirs:
+    """_pending_log_dirs() identifies .dlist log directories still missing data,
+    i.e. still being transferred from the vehicle to the archive filesystem."""
+
+    _DLIST_CONTENT = "# Deployment name: CANON April 2025\n20250414T120000\n"
+
+    def test_empty_when_all_dirs_present(self, dp):
+        assert dp._pending_log_dirs(self._DLIST_CONTENT, [_NC_URL]) == []  # noqa: S101
+
+    def test_lists_dir_with_no_nc_file(self, dp):
+        dlist_content = self._DLIST_CONTENT + "20250415T080000\n"
+        assert dp._pending_log_dirs(dlist_content, [_NC_URL]) == [  # noqa: S101
+            "20250415T080000"
+        ]
+
+    def test_ignores_commented_out_dirs(self, dp):
+        dlist_content = self._DLIST_CONTENT + "# 20250415T080000\n"
+        assert dp._pending_log_dirs(dlist_content, [_NC_URL]) == []  # noqa: S101
+
+
+class TestNotificationGating:
+    """_build_and_write_html() must withhold --notify until every .dlist log
+    directory has produced data, logging a warning instead in the meantime."""
+
+    _DLIST = "ahi/missionlogs/2025/20250414_20250418.dlist"
+
+    def _call(self, dp, tmp_path, *, pending_log_dirs, notify=None):
+        png = tmp_path / "CANON_April_2025_2column_cmocean.png"
+        png.touch()
+        with (
+            patch("make_permalink.requests.Session") as mock_session_cls,
+            patch.object(dp, "_url_exists", return_value=False),
+            patch.object(dp, "_stoqs_url_for_nc_url", return_value=None),
+            patch.object(dp, "_notify") as mock_notify,
+        ):
+            mock_session_cls.return_value.__enter__.return_value.get = _mock_session_get("7")
+            dp._build_and_write_html(
+                tmp_path,
+                self._DLIST,
+                "CANON_April_2025",
+                "CANON April 2025",
+                _make_ds("2025-04-14"),
+                [str(png)],
+                [_NC_URL],
+                notify=notify,
+                pending_log_dirs=pending_log_dirs,
+            )
+        return mock_notify
+
+    def test_notifies_when_fully_transferred(self, dp, tmp_path):
+        mock_notify = self._call(dp, tmp_path, pending_log_dirs=[], notify=["a@b.com"])
+        mock_notify.assert_called_once()
+
+    def test_withholds_notification_when_pending(self, dp, tmp_path, caplog):
+        caplog.set_level("WARNING")
+        mock_notify = self._call(
+            dp, tmp_path, pending_log_dirs=["20250415T080000"], notify=["a@b.com"]
+        )
+        mock_notify.assert_not_called()
+        assert "still waiting" in caplog.text  # noqa: S101
+        assert "20250415T080000" in caplog.text  # noqa: S101
+
+    def test_no_warning_when_notify_not_requested(self, dp, tmp_path, caplog):
+        """Nothing was ever going to be sent, so withholding it isn't warning-worthy."""
+        caplog.set_level("WARNING")
+        mock_notify = self._call(dp, tmp_path, pending_log_dirs=["20250415T080000"], notify=None)
+        mock_notify.assert_not_called()
+        assert "still waiting" not in caplog.text  # noqa: S101
+
+
 class TestUpdateIndexHtml:
     """Regression tests for quick_look_plots.html accumulating multiple deployments.