From d272d4273de7330825af666116771f6afef1cfe2 Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Tue, 8 Sep 2026 13:38:29 -0600 Subject: [PATCH 1/3] fix(cli): recover dangling Codex marketplace registrations with --force Signed-off-by: Sara Tadayon --- crates/cli/src/installation/generation.rs | 327 +++++- .../cli/src/installation/marketplace/mod.rs | 768 +++++++++++++- .../cli/src/installation/marketplace/state.rs | 33 +- .../coverage/agents/plugin_install_tests.rs | 973 +++++++++++++++--- .../shared/install_generation_tests.rs | 193 +++- docs/nemo-relay-cli/plugin-installation.mdx | 52 +- 6 files changed, 2113 insertions(+), 233 deletions(-) diff --git a/crates/cli/src/installation/generation.rs b/crates/cli/src/installation/generation.rs index 367fb6830..923031d31 100644 --- a/crates/cli/src/installation/generation.rs +++ b/crates/cli/src/installation/generation.rs @@ -202,7 +202,7 @@ pub(crate) struct GenerationRetirement { lock: Option, lock_id: String, path: PathBuf, - original: GenerationMarker, + original: GenerationRetirementOriginal, changed: bool, committed: bool, lock_released_for_tree_mutation: bool, @@ -224,6 +224,36 @@ enum GenerationMarker { Retired { token: String, lock_path: PathBuf }, } +/// The filesystem state protected by a generation retirement transaction. +/// +/// Marker-absent recovery needs to remember that absence so rollback does not recreate a marker +/// for a generation that no longer exists. +#[derive(Clone, Debug, PartialEq, Eq)] +enum GenerationRetirementOriginal { + MarkerPresent(GenerationMarker), + MarkerAbsent { lock_path: PathBuf }, +} + +impl GenerationRetirementOriginal { + fn lock_path(&self) -> &Path { + match self { + Self::MarkerPresent(marker) => marker.lock_path(), + Self::MarkerAbsent { lock_path } => lock_path, + } + } + + fn marker(&self) -> Option<&GenerationMarker> { + match self { + Self::MarkerPresent(marker) => Some(marker), + Self::MarkerAbsent { .. } => None, + } + } + + fn marker_was_absent(&self) -> bool { + matches!(self, Self::MarkerAbsent { .. }) + } +} + impl GenerationMarker { fn active(token: impl Into, lock_path: impl Into) -> Self { Self::Active { @@ -308,7 +338,10 @@ impl GenerationRetirement { )); } let visible = read_generation_marker_path(path)?; - if visible != self.original { + if self.original.marker() != Some(&visible) + || visible.is_retired() + || !self.uses_lock_path(visible.lock_path())? + { return Err(format!( "failed to adopt promoted MCP install generation {} because its marker changed", path.display() @@ -340,13 +373,22 @@ impl GenerationRetirement { if self.lock_released_for_tree_mutation { return Ok(()); } + if self.original.marker_was_absent() { + // The external layout lock fences marker-absent recovery through cleanup and + // replacement. + return Ok(()); + } if !self.uses_lock_path(&generation_lock_path(&self.path))? { return Ok(()); } - if !self.original.is_retired() && !self.changed { + let original = self + .original + .marker() + .expect("marker-present generation retirement has an original marker"); + if !original.is_retired() && !self.changed { return Err(format!( "cannot release active MCP install generation lock {}", - self.original.lock_path().display() + original.lock_path().display() )); } let Some(file) = self.lock.take() else { @@ -359,7 +401,7 @@ impl GenerationRetirement { self.lock = Some(file); return Err(format!( "failed to release MCP install generation lock {} before moving its plugin tree: {error}", - self.original.lock_path().display() + original.lock_path().display() )); } self.lock_released_for_tree_mutation = true; @@ -378,6 +420,22 @@ impl GenerationRetirement { Self::acquire_impl(path, DEFAULT_GENERATION_LOCK_TIMEOUT, Some(external_lock)) } + /// Acquire the surviving external generation lock for a plugin whose marker is intentionally + /// absent. + /// + /// The surviving lock is the only fence for this recovery path, so its identity and path + /// shape are validated before and after exclusive acquisition. + pub(crate) fn acquire_missing_for_plugin( + marker_path: &Path, + external_lock: &Path, + ) -> Result { + Self::acquire_missing_with_timeout_impl( + marker_path, + external_lock, + DEFAULT_GENERATION_LOCK_TIMEOUT, + ) + } + #[cfg(test)] pub(crate) fn acquire_with_timeout( path: &Path, @@ -386,6 +444,54 @@ impl GenerationRetirement { Self::acquire_impl(path, timeout, None) } + #[cfg(test)] + pub(crate) fn acquire_missing_with_timeout( + marker_path: &Path, + external_lock: &Path, + timeout: Duration, + ) -> Result { + Self::acquire_missing_with_timeout_impl(marker_path, external_lock, timeout) + } + + fn acquire_missing_with_timeout_impl( + marker_path: &Path, + external_lock: &Path, + timeout: Duration, + ) -> Result { + ensure_path_absent_nofollow(marker_path, "MCP install generation")?; + let lock_path = absolute_lock_path(external_lock)?; + let file = open_existing_generation_lock_path_nofollow(&lock_path)?; + lock_exclusive_with_timeout(&file, marker_path, timeout)?; + + let locked_result = (|| -> Result { + let lock_id = read_generation_lock_identity(&file, &lock_path)? + .ok_or_else(|| empty_generation_lock_error(&lock_path))?; + if !visible_generation_lock_matches_nofollow(&file, &lock_path, &lock_id)? { + return Err(changed_generation_lock_error(&lock_path)); + } + ensure_path_absent_nofollow(marker_path, "MCP install generation")?; + // Close the revalidation window in the other direction too: a lock-path replacement + // racing with the marker check must not leave us holding an unlinked inode. + if !visible_generation_lock_matches_nofollow(&file, &lock_path, &lock_id)? { + return Err(changed_generation_lock_error(&lock_path)); + } + Ok(lock_id) + })(); + let lock_id = locked_result.inspect_err(|_| { + let _ = unlock_file(&file); + })?; + + Ok(Self { + lock: Some(file), + lock_id, + path: marker_path.to_owned(), + original: GenerationRetirementOriginal::MarkerAbsent { lock_path }, + changed: false, + committed: false, + lock_released_for_tree_mutation: false, + }) + } + fn acquire_impl( path: &Path, timeout: Duration, @@ -427,13 +533,34 @@ impl GenerationRetirement { lock: Some(file), lock_id, path: path.to_owned(), - original, + original: GenerationRetirementOriginal::MarkerPresent(original), changed: false, committed: false, lock_released_for_tree_mutation: false, })) } + /// Revalidate the marker-absent recovery precondition while retaining the exclusive lock. + /// + /// Callers use this immediately before their first host/filesystem mutation, after checking + /// the marketplace root under the same no-follow policy. + pub(crate) fn revalidate_missing_marker(&self) -> Result<(), String> { + if !self.original.marker_was_absent() { + return Err(format!( + "MCP install generation {} was not acquired as marker-absent recovery", + self.path.display() + )); + } + if !self.visible_lock_identity_matches_nofollow()? { + return Err(changed_generation_lock_error(self.original.lock_path())); + } + ensure_path_absent_nofollow(&self.path, "MCP install generation")?; + if !self.visible_lock_identity_matches_nofollow()? { + return Err(changed_generation_lock_error(self.original.lock_path())); + } + Ok(()) + } + /// Persistently invalidate this generation while retaining its exclusive transaction lock. /// /// Existing MCPs retain the stable generation lock while the marker can be atomically @@ -451,11 +578,17 @@ impl GenerationRetirement { if self.lock_released_for_tree_mutation { return Ok(()); } - if !self.changed && !self.original.is_retired() { + let Some(original) = self.original.marker() else { return Err(format!( - "cannot release active MCP install generation lock {}", + "cannot release marker-absent MCP install generation lock {} before recovery", self.original.lock_path().display() )); + }; + if !self.changed && !original.is_retired() { + return Err(format!( + "cannot release active MCP install generation lock {}", + original.lock_path().display() + )); } let Some(file) = self.lock.take() else { return Err(format!( @@ -467,7 +600,7 @@ impl GenerationRetirement { self.lock = Some(file); return Err(format!( "failed to release MCP install generation lock {} before refreshing: {error}", - self.original.lock_path().display() + original.lock_path().display() )); } self.lock_released_for_tree_mutation = true; @@ -478,13 +611,18 @@ impl GenerationRetirement { &mut self, write_retired: impl FnOnce(&Path, &GenerationMarker) -> Result<(), String>, ) -> Result<(), String> { - if self.original.is_retired() { + let Some(original) = self.original.marker().cloned() else { + // The external lock already fences marker-absent recovery, so no marker transition is + // needed. + return Ok(()); + }; + if original.is_retired() { return Ok(()); } if self.changed { return Ok(()); } - let retired = self.original.retired(); + let retired = original.retired(); self.lock.as_ref().ok_or_else(|| { format!( "MCP install generation {} is not locked", @@ -495,7 +633,7 @@ impl GenerationRetirement { if let Err(error) = write_retired(&self.path, &retired) { let restore_error = replace_generation_marker( &self.path, - &self.original, + &original, "restore after failed invalidation", ) .err(); @@ -525,7 +663,7 @@ impl GenerationRetirement { let visible = read_generation_marker_path(&self.path)?; if visible.is_retired() { return Err(format!( - "replacement MCP install generation {} is already retired", + "replacement MCP install generation {} changed or is already retired", self.path.display() )); } @@ -552,20 +690,35 @@ impl GenerationRetirement { /// Restore an invalidated marker before a rolled-back plugin is registered again. pub(crate) fn restore_after_rollback(&mut self) -> Result<(), String> { + if self.original.marker_was_absent() { + // Failed dangling recovery leaves the marker absent. Revalidate that state before + // releasing its only fence. + self.revalidate_missing_marker()?; + self.lock = None; + self.changed = false; + self.committed = false; + self.lock_released_for_tree_mutation = false; + return Ok(()); + } if !self.changed { self.lock = None; self.lock_released_for_tree_mutation = false; return Ok(()); } self.reacquire_transaction_lock()?; + let original = self + .original + .marker() + .cloned() + .expect("marker-present generation retirement has an original marker"); // Never publish the retired generation's token through a replacement tree. A failed // filesystem rollback can leave the promoted tree visible at the same path while this // transaction still owns the shared external lock. - self.verify_visible_state_for_rollback(&self.original.retired())?; - replace_generation_marker(&self.path, &self.original, "restore")?; + self.verify_visible_state_for_rollback(&original.retired())?; + replace_generation_marker(&self.path, &original, "restore")?; // Retain the post-write check so an unexpected path swap during restoration is still // reported before the old generation is considered active again. - self.verify_visible_state_for_rollback(&self.original)?; + self.verify_visible_state_for_rollback(&original)?; self.changed = false; self.committed = false; self.lock = None; @@ -603,6 +756,16 @@ impl GenerationRetirement { visible_generation_lock_matches(lock, self.original.lock_path(), &self.lock_id) } + fn visible_lock_identity_matches_nofollow(&self) -> Result { + let lock = self.lock.as_ref().ok_or_else(|| { + format!( + "MCP install generation {} has no transaction lock for rollback", + self.path.display() + ) + })?; + visible_generation_lock_matches_nofollow(lock, self.original.lock_path(), &self.lock_id) + } + fn reacquire_transaction_lock(&mut self) -> Result<(), String> { if self.lock.is_some() { return Ok(()); @@ -669,6 +832,72 @@ fn inspected_path_exists(path: &Path, description: &str) -> Result } } +/// Inspect the final path component without following it. +/// +/// Parent components retain the platform's ordinary path-resolution behavior; this specifically +/// prevents a marker or lock file itself from being replaced by a symlink and treated as the +/// expected Relay-owned object. +fn inspected_path_exists_nofollow(path: &Path, description: &str) -> Result { + match fs::symlink_metadata(path) { + Ok(_) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => { + inspected_path_exists(path, description) + } + Err(error) => Err(format!( + "failed to inspect {description} {} without following links: {error}", + path.display() + )), + } +} + +fn ensure_path_absent_nofollow(path: &Path, description: &str) -> Result<(), String> { + if !inspected_path_exists_nofollow(path, description)? { + return Ok(()); + } + let kind = fs::symlink_metadata(path) + .map(|metadata| { + if metadata.file_type().is_symlink() { + "a symlink" + } else { + "present" + } + }) + .unwrap_or("present"); + Err(format!( + "expected {description} {} to remain absent without following links, but it is {kind}", + path.display() + )) +} + +fn ensure_regular_file_nofollow(path: &Path, description: &str) -> Result<(), String> { + let metadata = fs::symlink_metadata(path).map_err(|error| { + format!( + "failed to open {description} {} safely without following links: {error}", + path.display() + ) + })?; + if metadata.file_type().is_symlink() { + return Err(format!( + "refusing to follow symlinked {description} {}", + path.display() + )); + } + if !metadata.is_file() { + return Err(format!( + "{description} {} is not a regular file", + path.display() + )); + } + Ok(()) +} + +fn changed_generation_lock_error(lock_path: &Path) -> String { + format!( + "MCP install generation lock {} changed identity while acquiring recovery fencing", + lock_path.display() + ) +} + fn replace_generation_marker( path: &Path, marker: &GenerationMarker, @@ -922,6 +1151,41 @@ fn open_existing_generation_lock_path(lock_path: &Path) -> Result }) } +fn open_existing_generation_lock_path_nofollow(lock_path: &Path) -> Result { + ensure_regular_file_nofollow(lock_path, "MCP install generation lock")?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // `O_NONBLOCK` prevents a path-shape race from hanging on a FIFO. It has no effect on + // regular files. `O_NOFOLLOW` closes the symlink race between inspection and open. + options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); + } + let file = options.open(lock_path).map_err(|error| { + format!( + "failed to open MCP install generation lock {}: {error}", + lock_path.display() + ) + })?; + let opened_metadata = file.metadata().map_err(|error| { + format!( + "failed to inspect opened MCP install generation lock {}: {error}", + lock_path.display() + ) + })?; + if !opened_metadata.is_file() { + return Err(format!( + "MCP install generation lock {} is not a regular file", + lock_path.display() + )); + } + // A replacement between the pre-open inspection and open is rejected even on platforms that + // do not expose `O_NOFOLLOW` through `OpenOptions`. + ensure_regular_file_nofollow(lock_path, "MCP install generation lock")?; + Ok(file) +} + fn ensure_generation_lock_identity_locked(file: &File, lock_path: &Path) -> Result { if let Some(identity) = read_generation_lock_identity(file, lock_path)? { return Ok(identity); @@ -987,10 +1251,37 @@ fn visible_generation_lock_matches( locked: &File, lock_path: &Path, expected_identity: &str, +) -> Result { + visible_generation_lock_matches_with( + locked, + lock_path, + expected_identity, + open_existing_generation_lock_path, + ) +} + +fn visible_generation_lock_matches_nofollow( + locked: &File, + lock_path: &Path, + expected_identity: &str, +) -> Result { + visible_generation_lock_matches_with( + locked, + lock_path, + expected_identity, + open_existing_generation_lock_path_nofollow, + ) +} + +fn visible_generation_lock_matches_with( + locked: &File, + lock_path: &Path, + expected_identity: &str, + open_visible: fn(&Path) -> Result, ) -> Result { #[cfg(windows)] { - let visible = open_existing_generation_lock_path(lock_path)?; + let visible = open_visible(lock_path)?; if windows_file_identity(locked, lock_path)? != windows_file_identity(&visible, lock_path)? { return Ok(false); @@ -1000,7 +1291,7 @@ fn visible_generation_lock_matches( } #[cfg(not(windows))] { - let visible = open_existing_generation_lock_path(lock_path)?; + let visible = open_visible(lock_path)?; #[cfg(unix)] if unix_file_identity(locked, lock_path)? != unix_file_identity(&visible, lock_path)? { return Ok(false); diff --git a/crates/cli/src/installation/marketplace/mod.rs b/crates/cli/src/installation/marketplace/mod.rs index e7daaeaee..d05870a87 100644 --- a/crates/cli/src/installation/marketplace/mod.rs +++ b/crates/cli/src/installation/marketplace/mod.rs @@ -32,9 +32,10 @@ use assets::{ write_plugin_marketplace, write_plugin_marketplace_for_generation, }; use host::{ - CommandRunner, RealCommandRunner, host_registration_report, require_host_cli, require_relay, - run_host_marketplace_registration, run_host_marketplace_removal, run_host_plugin_registration, - run_host_plugin_removal, validate_relay_hook_forward, validate_relay_mcp, + CommandRunner, HostRegistrationReport, RealCommandRunner, host_registration_report, + require_host_cli, require_relay, run_host_marketplace_registration, + run_host_marketplace_removal, run_host_plugin_registration, run_host_plugin_removal, + validate_relay_hook_forward, validate_relay_mcp, }; use setup::{ HostPluginSetupRunner, PluginSetupRunner, run_plugin_doctor_json, @@ -842,6 +843,9 @@ fn write_install_state( let runner = context.runner; let setup_runner = context.setup_runner; if let Err(error) = write_state(layout, options) { + let cleanup_committed = transaction.force_snapshot.as_ref().is_some_and(|snapshot| { + snapshot.recoverable_dangling_marketplace && snapshot.cleanup_committed + }); let _replacement_retirement = if transaction.force_snapshot.is_some() { let existing_retirement = transaction .replacement_generation_lock @@ -859,9 +863,35 @@ fn write_install_state( options, setup_runner, existing_retirement, + cleanup_committed, ) { Ok(retirement) => retirement, Err(retirement_error) => { + if cleanup_committed { + let retry_state = PluginState { + marketplace_root: layout.marketplace_root.clone(), + plugin_root: layout.plugin_root.clone(), + host_plugin_removed: true, + host_marketplace_removed: true, + plugin_setup_installed: false, + marker_absent_recovery: false, + }; + let retry_error = + write_state_for_host(host, &retry_state, &options.install_dir, options) + .err(); + return Err(retry_error.map_or_else( + || { + format!( + "{error}; replacement remains fenced after rollback refresh failed: {retirement_error}" + ) + }, + |retry_error| { + format!( + "{error}; replacement remains fenced after rollback refresh failed: {retirement_error}; additionally failed to persist retry state: {retry_error}" + ) + }, + )); + } return Err(format!( "{error}; refusing destructive rollback because the replacement MCP generation could not be retired: {retirement_error}" )); @@ -870,7 +900,9 @@ fn write_install_state( } else { None }; - let cleanup_error = remove_path(&layout.marketplace_root, options).err(); + let cleanup_error = (!cleanup_committed) + .then(|| remove_path(&layout.marketplace_root, options).err()) + .flatten(); let restore_error = transaction.force_snapshot.as_mut().and_then(|snapshot| { restore_force_replacement(host, layout, snapshot, options, runner, setup_runner).err() }); @@ -932,7 +964,7 @@ fn run_install_registration( options: &PluginInstallOptions, runner: &dyn CommandRunner, setup_runner: &dyn PluginSetupRunner, - transaction: &InstallTransactionState, + transaction: &mut InstallTransactionState, registration: &mut HostRegistrationProgress, registration_state_uncertain: &mut bool, setup_installed: &mut bool, @@ -949,16 +981,25 @@ fn run_install_registration( }) .map(GenerationRetirement::active_visible_token) .transpose()?; + if let Some(snapshot) = transaction.force_snapshot.as_mut() { + snapshot.replacement_registration.host_marketplace_added = true; + } run_host_marketplace_registration(host, &layout.marketplace_root, options, runner) .inspect_err(|_| { *registration_state_uncertain = true; })?; registration.host_marketplace_added = true; + if let Some(snapshot) = transaction.force_snapshot.as_mut() { + snapshot.replacement_registration.host_plugin_added = true; + } run_host_plugin_registration(host, options, runner).inspect_err(|_| { *registration_state_uncertain = true; })?; registration.host_plugin_added = true; *setup_installed = host.setup_may_mutate_before_success(); + if let Some(snapshot) = transaction.force_snapshot.as_mut() { + snapshot.replacement_setup_installed = *setup_installed; + } run_plugin_setup_with_generation( host, layout, @@ -967,6 +1008,9 @@ fn run_install_registration( generation_token.as_deref(), )?; *setup_installed = true; + if let Some(snapshot) = transaction.force_snapshot.as_mut() { + snapshot.replacement_setup_installed = true; + } mark_plugin_setup_installed(host, layout, options)?; if !options.skip_doctor { run_plugin_doctor_with_generation( @@ -993,19 +1037,36 @@ fn recover_failed_install_registration( setup_installed: bool, error: String, ) -> Result<(), String> { + let cleanup_committed = transaction.force_snapshot.as_ref().is_some_and(|snapshot| { + snapshot.recoverable_dangling_marketplace && snapshot.cleanup_committed + }); + let mut recovery_errors = Vec::new(); if registration_state_uncertain { - let observed = host_registration_report(host, options, runner).map_err(|report_error| { - format!( - "{error}; refusing destructive rollback because the host registration state could not be verified after a registration command failed: {report_error}" - ) - })?; - let observed_plugin_registered = observed.host_plugin_registered.ok_or_else(|| { - format!( - "{error}; refusing destructive rollback because the host plugin registration state could not be determined after a registration command failed" - ) - })?; - registration.host_plugin_added |= observed_plugin_registered; - registration.host_marketplace_added |= observed.host_marketplace_registered; + match host_registration_report(host, options, runner) { + Ok(observed) => { + if let Some(observed_plugin_registered) = observed.host_plugin_registered { + registration.host_plugin_added |= observed_plugin_registered; + } else if cleanup_committed { + recovery_errors.push( + "host plugin registration remained unknown after the replacement registration failed; cleanup was attempted conservatively" + .into(), + ); + } else { + return Err(format!( + "{error}; refusing destructive rollback because the host plugin registration state could not be determined after a registration command failed" + )); + } + registration.host_marketplace_added |= observed.host_marketplace_registered; + } + Err(report_error) if cleanup_committed => recovery_errors.push(format!( + "host registration state could not be verified after the replacement registration failed, so cleanup was attempted conservatively: {report_error}" + )), + Err(report_error) => { + return Err(format!( + "{error}; refusing destructive rollback because the host registration state could not be verified after a registration command failed: {report_error}" + )); + } + } } retire_live_replacement_before_rollback(host, layout, options, setup_runner, transaction, ®istration) .map_err(|retirement_error| { @@ -1013,6 +1074,24 @@ fn recover_failed_install_registration( "{error}; refusing destructive rollback because the replacement MCP generation could not be retired: {retirement_error}" ) })?; + if cleanup_committed { + if let Some(snapshot) = transaction.force_snapshot.as_mut() + && let Err(cleanup_error) = + restore_force_replacement(host, layout, snapshot, options, runner, setup_runner) + { + recovery_errors.push(format!( + "failed to preserve a clean forced-recovery retry state: {cleanup_error}" + )); + } + return if recovery_errors.is_empty() { + Err(error) + } else { + Err(format!( + "{error}; additionally {}", + recovery_errors.join("; ") + )) + }; + } let rollback_error = rollback_install( host, layout, @@ -1054,6 +1133,9 @@ fn retire_live_replacement_before_rollback( if transaction.force_snapshot.is_none() && !registration.host_plugin_added { return Ok(None); } + let cleanup_committed = transaction.force_snapshot.as_ref().is_some_and(|snapshot| { + snapshot.recoverable_dangling_marketplace && snapshot.cleanup_committed + }); let existing_retirement = transaction .replacement_generation_lock .as_mut() @@ -1064,7 +1146,14 @@ fn retire_live_replacement_before_rollback( .as_mut() .and_then(|snapshot| snapshot.generation_retirement.as_mut()) }); - retire_replacement_before_rollback(host, layout, options, setup_runner, existing_retirement) + retire_replacement_before_rollback( + host, + layout, + options, + setup_runner, + existing_retirement, + cleanup_committed, + ) } fn uninstall_host( @@ -1115,6 +1204,9 @@ fn uninstall_host_locked( let layout = PluginLayout::new(host, &options.install_dir); if let Some(state) = state.as_ref() { layout.validate_persisted_state(state)?; + if host.install_arg() == "codex" && state.marker_absent_recovery { + return Err(dangling_marketplace_requires_force_error(host, "uninstall")); + } } let plugin_root = state .as_ref() @@ -1132,8 +1224,8 @@ fn uninstall_host_locked( ); let mut generation_retirement = retire_installed_generation( host, + &layout, plugin_root, - &layout.generation_lock, local_install_exists, options, runner, @@ -1183,6 +1275,47 @@ fn force_uninstall_host_locked( setup_runner: &dyn PluginSetupRunner, ) -> Result<(), String> { let layout = PluginLayout::new(host, &options.install_dir); + if !options.dry_run && host.install_arg() == "codex" { + if let Some(state) = read_state(host, &options.install_dir) + && state.marker_absent_recovery + { + layout.validate_persisted_state(&state)?; + if !path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? + || !path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")? + { + return Err(unsafe_generation_fence_error( + host, + "or its marketplace root reappeared during a marker-absent cleanup retry", + )); + } + let retirement = acquire_dangling_generation_retirement(host, &layout)?; + return force_uninstall_dangling_marketplace_locked( + host, + &layout, + retirement, + options, + runner, + setup_runner, + host_registration_report(host, options, runner).ok(), + ); + } + if path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? + && path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")? + && let Ok(registration) = host_registration_report(host, options, runner) + && is_recoverable_dangling_marketplace(host, &layout, ®istration)? + { + let retirement = acquire_dangling_generation_retirement(host, &layout)?; + return force_uninstall_dangling_marketplace_locked( + host, + &layout, + retirement, + options, + runner, + setup_runner, + Some(registration), + ); + } + } let mut errors = Vec::new(); if !options.dry_run @@ -1243,10 +1376,161 @@ fn force_uninstall_host_locked( } } +#[allow(clippy::cognitive_complexity)] +fn force_uninstall_dangling_marketplace_locked( + host: impl MarketplaceHost, + layout: &PluginLayout, + mut retirement: GenerationRetirement, + options: &PluginInstallOptions, + runner: &dyn CommandRunner, + setup_runner: &dyn PluginSetupRunner, + registration: Option, +) -> Result<(), String> { + let persisted = read_state(host, &options.install_dir); + if let Some(state) = persisted.as_ref() { + layout.validate_persisted_state(state)?; + } + revalidate_dangling_paths(host, layout, &retirement)?; + let retry_progress = persisted + .as_ref() + .filter(|state| state.marker_absent_recovery); + + let mut errors = Vec::new(); + if let Err(error) = setup_runner.refresh_gateway() { + errors.push(format!("failed to stop the Relay-owned gateway: {error}")); + } + let plugin_setup_installed = + if retry_progress.is_some_and(|state| !state.plugin_setup_installed) { + false + } else { + match run_plugin_uninstall(host, &layout.plugin_root, options, setup_runner) { + Ok(()) => false, + Err(error) => { + errors.push(format!("failed to remove Relay host setup: {error}")); + true + } + } + }; + + let plugin_removal_error = if retry_progress.is_some_and(|state| state.host_plugin_removed) + && registration + .as_ref() + .is_some_and(|report| report.host_plugin_registered == Some(false)) + { + None + } else { + run_host_plugin_removal(host, options, runner).err() + }; + let marketplace_removal_error = if retry_progress + .is_some_and(|state| state.host_marketplace_removed) + && registration + .as_ref() + .is_some_and(|report| !report.host_marketplace_registered) + { + None + } else { + run_host_marketplace_removal(host, options, runner).err() + }; + let mut host_plugin_removed = plugin_removal_error.is_none(); + let mut host_marketplace_removed = marketplace_removal_error.is_none(); + if plugin_removal_error.is_some() || marketplace_removal_error.is_some() { + match host_registration_report(host, options, runner) { + Ok(report) => { + if plugin_removal_error.is_some() { + host_plugin_removed = report.host_plugin_registered == Some(false); + } + if marketplace_removal_error.is_some() { + host_marketplace_removed = !report.host_marketplace_registered; + } + } + Err(error) => { + errors.push(format!( + "could not verify host registration cleanup after removal failed: {error}" + )); + } + } + } + if !host_plugin_removed && let Some(error) = plugin_removal_error { + errors.push(format!("failed to unregister the host plugin: {error}")); + } + if !host_marketplace_removed && let Some(error) = marketplace_removal_error { + errors.push(format!( + "failed to unregister the host marketplace: {error}" + )); + } + if let Err(error) = remove_path(&layout.marketplace_root, options) { + errors.push(error); + } + + if errors.is_empty() + && let Err(error) = remove_path(&layout.state_path, options) + { + errors.push(error); + } + if !errors.is_empty() { + let retry_state = PluginState { + marketplace_root: layout.marketplace_root.clone(), + plugin_root: layout.plugin_root.clone(), + host_plugin_removed, + host_marketplace_removed, + plugin_setup_installed, + marker_absent_recovery: true, + }; + if let Err(error) = write_state_for_host(host, &retry_state, &options.install_dir, options) + { + errors.push(format!( + "failed to preserve Relay cleanup retry state: {error}" + )); + } + } + + if errors.is_empty() { + let lock_path = retirement.lock_path().to_owned(); + retirement.commit_replacement(); + drop(retirement); + match fs::remove_file(&lock_path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + errors.push(format!( + "failed to remove MCP generation lock {}: {error}", + lock_path.display() + )); + let retry_state = PluginState { + marketplace_root: layout.marketplace_root.clone(), + plugin_root: layout.plugin_root.clone(), + host_plugin_removed: true, + host_marketplace_removed: true, + plugin_setup_installed: false, + marker_absent_recovery: true, + }; + if let Err(state_error) = + write_state_for_host(host, &retry_state, &options.install_dir, options) + { + errors.push(format!( + "failed to preserve Relay cleanup retry state: {state_error}" + )); + } + } + } + } + + if errors.is_empty() { + println!("force-uninstalled {} plugin", host.label()); + Ok(()) + } else { + Err(format!( + "forced {} dangling-marketplace cleanup completed with errors: {}", + host.label(), + errors.join("; ") + )) + } +} + fn retire_installed_generation( host: impl MarketplaceHost, + layout: &PluginLayout, plugin_root: &Path, - expected_generation_lock: &Path, local_install_exists: bool, options: &PluginInstallOptions, runner: &dyn CommandRunner, @@ -1256,8 +1540,11 @@ fn retire_installed_generation( } let generation_fence = plugin_root.join(GENERATION_FILE_NAME); let mut existing_install = local_install_exists; - if !generation_fence.exists() { + if path_is_absent_no_follow(&generation_fence, "MCP generation marker")? { let registration = host_registration_report(host, options, runner)?; + if is_recoverable_dangling_marketplace(host, layout, ®istration)? { + return Err(dangling_marketplace_requires_force_error(host, "uninstall")); + } existing_install |= registration.host_plugin_may_be_registered() || registration.host_marketplace_registered; if existing_install && !legacy_plugin_without_mcp(host, plugin_root)? { @@ -1265,7 +1552,7 @@ fn retire_installed_generation( } } let retirement = - GenerationRetirement::acquire_for_plugin(&generation_fence, expected_generation_lock) + GenerationRetirement::acquire_for_plugin(&generation_fence, &layout.generation_lock) .map_err(|cause| invalid_generation_fence_error(host, &generation_fence, &cause))?; if retirement.is_none() && !existing_install { let registration = host_registration_report(host, options, runner)?; @@ -1284,6 +1571,7 @@ fn retire_replacement_before_rollback( options: &PluginInstallOptions, setup_runner: &dyn PluginSetupRunner, existing_retirement: Option<&mut GenerationRetirement>, + keep_retired_on_refresh_failure: bool, ) -> Result, String> { if options.dry_run { return Ok(None); @@ -1298,6 +1586,12 @@ fn retire_replacement_before_rollback( ) })?; if let Err(error) = setup_runner.refresh_gateway() { + if keep_retired_on_refresh_failure { + retirement.commit_replacement(); + return Err(format!( + "{error}; the incomplete replacement MCP generation remains retired for a forced retry" + )); + } return match retirement.restore_visible_replacement(visible) { Ok(()) => Err(error), Err(restore_error) => Err(format!( @@ -1320,6 +1614,12 @@ fn retire_replacement_before_rollback( ) })?; if let Err(error) = setup_runner.refresh_gateway() { + if keep_retired_on_refresh_failure { + retirement.commit_replacement(); + return Err(format!( + "{error}; the incomplete replacement MCP generation remains retired for a forced retry" + )); + } return match retirement.restore_after_rollback() { Ok(()) => Err(error), Err(restore_error) => Err(format!( @@ -1339,6 +1639,105 @@ fn existing_plugin_install_requires_force_error(host: impl MarketplaceHost) -> S ) } +/// Limits automatic recovery to Codex's known dangling-marketplace failure, avoiding recovery +/// from unrelated host failures. +fn is_recoverable_dangling_marketplace( + host: impl MarketplaceHost, + layout: &PluginLayout, + registration: &HostRegistrationReport, +) -> Result { + if host.install_arg() != "codex" + || registration.host_plugin_registered.is_some() + || !registration.host_marketplace_registered + || !registration.host_marketplace_unloadable + { + return Ok(false); + } + Ok( + path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? + && path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")?, + ) +} + +fn path_is_absent_no_follow(path: &Path, description: &str) -> Result { + match fs::symlink_metadata(path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(true), + Ok(_) => Ok(false), + Err(error) => Err(format!( + "failed to inspect {description} {} without following links: {error}", + path.display() + )), + } +} + +fn dangling_marketplace_requires_force_error( + host: impl MarketplaceHost, + operation: &str, +) -> String { + format!( + "the {} marketplace registration is dangling because its generated marketplace tree is missing; rerun `nemo-relay {operation} {} --force` to recover it safely", + host.label(), + host.install_arg() + ) +} + +fn acquire_dangling_generation_retirement( + host: impl MarketplaceHost, + layout: &PluginLayout, +) -> Result { + let retirement = GenerationRetirement::acquire_missing_for_plugin( + &layout.generation_fence, + &layout.generation_lock, + ) + .map_err(|cause| { + unsafe_generation_fence_error( + host, + &format!( + "is intentionally absent at {}, but its surviving lock at {} is unsafe: {cause}", + layout.generation_fence.display(), + layout.generation_lock.display() + ), + ) + })?; + revalidate_dangling_paths(host, layout, &retirement)?; + Ok(retirement) +} + +fn revalidate_dangling_paths( + host: impl MarketplaceHost, + layout: &PluginLayout, + retirement: &GenerationRetirement, +) -> Result<(), String> { + retirement.revalidate_missing_marker().map_err(|cause| { + unsafe_generation_fence_error( + host, + &format!( + "changed at {} during forced recovery: {cause}", + layout.generation_fence.display() + ), + ) + })?; + if !path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? { + return Err(unsafe_generation_fence_error( + host, + &format!( + "is absent, but marketplace root {} reappeared or became a symbolic link during forced recovery", + layout.marketplace_root.display() + ), + )); + } + retirement.revalidate_missing_marker().map_err(|cause| { + unsafe_generation_fence_error( + host, + &format!( + "or its surviving lock changed while marketplace root {} was revalidated: {cause}", + layout.marketplace_root.display() + ), + ) + })?; + Ok(()) +} + fn missing_generation_fence_error(host: impl MarketplaceHost, generation_fence: &Path) -> String { unsafe_generation_fence_error( host, @@ -1405,6 +1804,7 @@ fn uninstall_host_with_setup_override( host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: true, + marker_absent_recovery: false, }); layout.validate_persisted_state(&state)?; if let Err(_error) = require_relay(options, runner) @@ -1978,7 +2378,7 @@ struct PluginInstallPreflight { previous_marketplace_root: PathBuf, previous_plugin_root: PathBuf, previous_generation_fence: PathBuf, - plugin_registered: bool, + plugin_registered: Option, marketplace_registered: bool, previous_setup_installed: bool, previous_install_exists: bool, @@ -1987,6 +2387,7 @@ struct PluginInstallPreflight { // `force_uninstall_host_locked` writes one back as the cleanup-retry marker that // `installed_integrations` relies on to keep offering cleanup. previous_state_exists: bool, + recoverable_dangling_marketplace: bool, generation_retirement: Option, } @@ -2041,8 +2442,21 @@ fn prepare_plugin_install( // roots, so `local_install_exists` covers everything the state file could point at. let previous_install_exists = local_install_exists || plugin_may_be_registered || marketplace_registered; + let exact_dangling_marketplace = + is_recoverable_dangling_marketplace(host, layout, ®istration)?; + let marker_absent_retry = host.install_arg() == "codex" + && persisted + .as_ref() + .is_some_and(|state| state.marker_absent_recovery); + let recoverable_dangling_marketplace = exact_dangling_marketplace || marker_absent_retry; + let generation_fence_absent = + path_is_absent_no_follow(&previous_generation_fence, "MCP generation marker")?; + if (exact_dangling_marketplace || marker_absent_retry) && !options.force { + return Err(dangling_marketplace_requires_force_error(host, "install")); + } if previous_install_exists - && !previous_generation_fence.exists() + && generation_fence_absent + && !recoverable_dangling_marketplace && !legacy_plugin_without_mcp(host, &previous_plugin_root)? { return Err(missing_generation_fence_error( @@ -2050,14 +2464,18 @@ fn prepare_plugin_install( &previous_generation_fence, )); } - let plugin_registered = plugin_registration.ok_or_else(|| { - format!( + if plugin_registration.is_none() && !recoverable_dangling_marketplace { + return Err(format!( "refusing to modify the {} plugin because its host plugin registration state could not be determined", host.label() - ) - })?; - let previous_setup_installed = persisted_setup_installed || plugin_registered; - let generation_retirement = if previous_install_exists && previous_generation_fence.exists() { + )); + } + let previous_setup_installed = persisted_setup_installed + || plugin_registration == Some(true) + || (exact_dangling_marketplace && !marker_absent_retry); + let generation_retirement = if recoverable_dangling_marketplace && options.force { + Some(acquire_dangling_generation_retirement(host, layout)?) + } else if previous_install_exists && !generation_fence_absent { Some( GenerationRetirement::acquire_for_plugin( &previous_generation_fence, @@ -2078,11 +2496,12 @@ fn prepare_plugin_install( previous_marketplace_root, previous_plugin_root, previous_generation_fence, - plugin_registered, + plugin_registered: plugin_registration, marketplace_registered, previous_setup_installed, previous_install_exists, previous_state_exists: state_bytes_present, + recoverable_dangling_marketplace, generation_retirement, }) } @@ -2093,13 +2512,18 @@ struct ForceInstallSnapshot { original_marketplace_root: PathBuf, original_plugin_root: PathBuf, original_generation_fence: PathBuf, - plugin_registered: bool, + plugin_registered: Option, marketplace_registered: bool, backup_marketplace_root: PathBuf, backup_plugin_root: Option, marketplace_moved: bool, plugin_moved: bool, replacement_promoted: bool, + recoverable_dangling_marketplace: bool, + cleanup_committed: bool, + original_marketplace_removed: bool, + replacement_registration: HostRegistrationProgress, + replacement_setup_installed: bool, generation_retirement: Option, } @@ -2232,6 +2656,84 @@ fn stage_plugin_marketplace_at( }) } +fn cleanup_previous_install_for_replacement( + host: impl MarketplaceHost, + state: &mut PluginState, + snapshot: &mut ForceInstallSnapshot, + options: &PluginInstallOptions, + runner: &dyn CommandRunner, + setup_runner: &dyn PluginSetupRunner, +) -> Result<(), String> { + if state.plugin_setup_installed { + run_plugin_uninstall(host, &state.plugin_root, options, setup_runner)?; + state.plugin_setup_installed = false; + write_state_for_host(host, state, &options.install_dir, options)?; + } + let mut unknown_plugin_removal_error = None; + if !state.host_plugin_removed { + require_host_cli(host, options, runner)?; + // A dangling Codex marketplace makes plugin registration unknowable. Removal remains the + // conservative operation: it is safe whether the plugin was registered or not, while a + // rollback must never invent a registration that was only suspected. + match run_host_plugin_removal(host, options, runner) { + Ok(()) => { + state.host_plugin_removed = true; + write_state_for_host(host, state, &options.install_dir, options)?; + } + Err(error) + if snapshot.recoverable_dangling_marketplace + && snapshot.plugin_registered.is_none() => + { + // Codex may report "not installed" as a failed removal. The dangling marketplace + // must be removed before plugin-list can tell those two cases apart, so defer the + // decision until after that known registration is gone. + unknown_plugin_removal_error = Some(error); + } + Err(error) => return Err(error), + } + } + if !state.host_marketplace_removed { + require_host_cli(host, options, runner)?; + if let Err(marketplace_error) = run_host_marketplace_removal(host, options, runner) { + return Err(match unknown_plugin_removal_error { + Some(plugin_error) => format!( + "{plugin_error}; additionally failed to remove the dangling marketplace registration: {marketplace_error}" + ), + None => marketplace_error, + }); + } + state.host_marketplace_removed = true; + snapshot.original_marketplace_removed = true; + if let Some(plugin_error) = unknown_plugin_removal_error.take() { + match host_registration_report(host, options, runner) { + Ok(report) if report.host_plugin_registered == Some(false) => { + state.host_plugin_removed = true; + } + Ok(_) => return Err(plugin_error), + Err(report_error) => { + return Err(format!( + "{plugin_error}; additionally could not verify whether the unknown plugin registration was absent after marketplace cleanup: {report_error}" + )); + } + } + } + if snapshot.recoverable_dangling_marketplace { + // All old Relay-owned setup and registrations are now gone. From this point onward, + // rollback must converge to a clean retry state instead of recreating the dangling + // marketplace registration. + snapshot.cleanup_committed = true; + } + write_state_for_host(host, state, &options.install_dir, options)?; + } + if let Some(plugin_error) = unknown_plugin_removal_error { + return Err(plugin_error); + } + if snapshot.recoverable_dangling_marketplace { + snapshot.cleanup_committed = true; + } + Ok(()) +} + fn begin_force_replacement( host: impl MarketplaceHost, layout: &PluginLayout, @@ -2251,6 +2753,7 @@ fn begin_force_replacement( previous_setup_installed, previous_install_exists: _, previous_state_exists: _, + recoverable_dangling_marketplace, generation_retirement, } = preflight; let setup_snapshot = setup_runner.snapshot(host.install_arg())?; @@ -2286,31 +2789,71 @@ fn begin_force_replacement( marketplace_moved: false, plugin_moved: false, replacement_promoted: false, + recoverable_dangling_marketplace, + cleanup_committed: false, + original_marketplace_removed: false, + replacement_registration: HostRegistrationProgress::default(), + replacement_setup_installed: false, generation_retirement, }; let mut cleanup_state = persisted.unwrap_or_else(|| PluginState { marketplace_root: layout.marketplace_root.clone(), plugin_root: layout.plugin_root.clone(), - host_plugin_removed: !plugin_registered, + host_plugin_removed: plugin_registered == Some(false), host_marketplace_removed: !marketplace_registered, plugin_setup_installed: previous_setup_installed, + marker_absent_recovery: false, }); - cleanup_state.host_plugin_removed = !plugin_registered; - cleanup_state.host_marketplace_removed = !marketplace_registered; - let result = (|| { - if cleanup_state.plugin_setup_installed { - run_plugin_uninstall(host, &cleanup_state.plugin_root, options, setup_runner)?; - cleanup_state.plugin_setup_installed = false; + let cleanup_result = if recoverable_dangling_marketplace { + let persisted_recovery_progress = cleanup_state.marker_absent_recovery; + if persisted_recovery_progress { + // Persisted progress is useful only when fresh host evidence agrees. A registration + // may have reappeared between attempts; unknown evidence requires conservative + // removal. + cleanup_state.host_plugin_removed &= plugin_registered == Some(false); + cleanup_state.host_marketplace_removed &= !marketplace_registered; + } else { + cleanup_state.host_plugin_removed = plugin_registered == Some(false); + cleanup_state.host_marketplace_removed = !marketplace_registered; + cleanup_state.plugin_setup_installed = previous_setup_installed; } - run_host_unregistration( + cleanup_state.marker_absent_recovery = true; + revalidate_dangling_paths( + host, + layout, + snapshot + .generation_retirement + .as_ref() + .expect("dangling recovery holds the surviving generation lock"), + )?; + cleanup_previous_install_for_replacement( host, &mut cleanup_state, - &options.install_dir, + &mut snapshot, options, runner, + setup_runner, ) - })() - .and_then(|()| { + } else { + cleanup_state.host_plugin_removed = plugin_registered == Some(false); + cleanup_state.host_marketplace_removed = !marketplace_registered; + cleanup_state.plugin_setup_installed = previous_setup_installed; + cleanup_state.marker_absent_recovery = false; + (|| { + if cleanup_state.plugin_setup_installed { + run_plugin_uninstall(host, &cleanup_state.plugin_root, options, setup_runner)?; + cleanup_state.plugin_setup_installed = false; + } + run_host_unregistration( + host, + &mut cleanup_state, + &options.install_dir, + options, + runner, + ) + })() + }; + let result = cleanup_result.and_then(|()| { if let Some(retirement) = snapshot.generation_retirement.as_mut() { retirement.invalidate_for_replacement().map_err(|error| { format!( @@ -2379,8 +2922,12 @@ fn restore_force_replacement_after_error( setup_runner: &dyn PluginSetupRunner, original_error: String, ) -> Result { + let cleanup_committed = snapshot.recoverable_dangling_marketplace && snapshot.cleanup_committed; match restore_force_replacement(host, layout, snapshot, options, runner, setup_runner) { Ok(()) => Err(original_error), + Err(rollback_error) if cleanup_committed => Err(format!( + "{original_error}; additionally failed to preserve a clean forced-recovery retry state: {rollback_error}" + )), Err(rollback_error) => Err(format!( "{original_error}; additionally failed to restore previous install: {rollback_error}" )), @@ -2396,6 +2943,22 @@ fn restore_force_replacement( setup_runner: &dyn PluginSetupRunner, ) -> Result<(), String> { let mut errors = Vec::new(); + if snapshot.recoverable_dangling_marketplace && snapshot.cleanup_committed { + converge_committed_dangling_recovery( + host, + layout, + snapshot, + options, + runner, + setup_runner, + &mut errors, + ); + return if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + }; + } remove_promoted_replacement(host, layout, snapshot, options, runner, &mut errors); restore_replaced_paths(snapshot, &mut errors); if let Some(retirement) = snapshot.generation_retirement.as_mut() @@ -2413,6 +2976,101 @@ fn restore_force_replacement( } } +#[allow(clippy::too_many_arguments)] +fn converge_committed_dangling_recovery( + host: impl MarketplaceHost, + layout: &PluginLayout, + snapshot: &mut ForceInstallSnapshot, + options: &PluginInstallOptions, + runner: &dyn CommandRunner, + setup_runner: &dyn PluginSetupRunner, + errors: &mut Vec, +) { + let plugin_setup_installed = if snapshot.replacement_setup_installed { + match run_plugin_uninstall(host, &layout.plugin_root, options, setup_runner) { + Ok(()) => false, + Err(error) => { + errors.push(format!( + "failed to remove incomplete replacement host setup: {error}" + )); + true + } + } + } else { + false + }; + // Once cleanup commits, all registrations belong to the incomplete replacement. Remove them + // conservatively even when Codex still cannot report plugin state. + let plugin_removal_error = snapshot + .replacement_registration + .host_plugin_added + .then(|| run_host_plugin_removal(host, options, runner).err()) + .flatten(); + let marketplace_removal_error = snapshot + .replacement_registration + .host_marketplace_added + .then(|| run_host_marketplace_removal(host, options, runner).err()) + .flatten(); + let mut host_plugin_removed = plugin_removal_error.is_none(); + let mut host_marketplace_removed = marketplace_removal_error.is_none(); + if plugin_removal_error.is_some() || marketplace_removal_error.is_some() { + match host_registration_report(host, options, runner) { + Ok(report) => { + if plugin_removal_error.is_some() { + host_plugin_removed = report.host_plugin_registered == Some(false); + } + if marketplace_removal_error.is_some() { + host_marketplace_removed = !report.host_marketplace_registered; + } + } + Err(error) => errors.push(format!( + "failed to verify incomplete replacement registration cleanup: {error}" + )), + } + } + if !host_plugin_removed && let Some(error) = plugin_removal_error { + errors.push(format!( + "failed to remove incomplete replacement plugin registration: {error}" + )); + } + if !host_marketplace_removed && let Some(error) = marketplace_removal_error { + errors.push(format!( + "failed to remove incomplete replacement marketplace registration: {error}" + )); + } + let marketplace_root_removed = + if host_plugin_removed && host_marketplace_removed && !plugin_setup_installed { + match remove_path(&layout.marketplace_root, options) { + Ok(()) => { + snapshot.replacement_promoted = false; + true + } + Err(error) => { + errors.push(error); + false + } + } + } else { + false + }; + + let clean_state = PluginState { + marketplace_root: layout.marketplace_root.clone(), + plugin_root: layout.plugin_root.clone(), + host_plugin_removed, + host_marketplace_removed, + plugin_setup_installed, + // Keep a possibly registered replacement tree fenced by its retired marker. Removing the + // tree before marketplace removal is proven would recreate the dangling registration. + marker_absent_recovery: marketplace_root_removed, + }; + if let Err(error) = write_state_for_host(host, &clean_state, &options.install_dir, options) { + errors.push(format!( + "failed to preserve clean Relay recovery state: {error}" + )); + } +} + fn remove_promoted_replacement( host: impl MarketplaceHost, layout: &PluginLayout, @@ -2482,12 +3140,28 @@ fn reconcile_restored_registration( let report = match host_registration_report(host, options, runner) { Ok(report) => report, Err(error) => { + // Cleanup positively recorded removing the known-old marketplace even though the + // later host probe failed. Restore that known registration directly. The plugin's + // original tri-state remains unknown, so rollback must not invent it. + if snapshot.recoverable_dangling_marketplace + && snapshot.marketplace_registered + && snapshot.original_marketplace_removed + && let Err(restore_error) = run_host_marketplace_registration( + host, + &snapshot.original_marketplace_root, + options, + runner, + ) + { + errors.push(restore_error); + } errors.push(error); return; } }; let Some(host_plugin_registered) = report.host_plugin_registered else { if snapshot.marketplace_registered + && !report.host_marketplace_registered && let Err(error) = run_host_marketplace_registration( host, &snapshot.original_marketplace_root, @@ -2497,7 +3171,7 @@ fn reconcile_restored_registration( { errors.push(error); } - if snapshot.plugin_registered + if snapshot.plugin_registered == Some(true) && let Err(error) = run_host_plugin_registration(host, options, runner) { errors.push(error); @@ -2509,7 +3183,7 @@ fn reconcile_restored_registration( return; }; if host_plugin_registered - && !snapshot.plugin_registered + && snapshot.plugin_registered == Some(false) && let Err(error) = run_host_plugin_removal(host, options, runner) { errors.push(error); @@ -2531,7 +3205,7 @@ fn reconcile_restored_registration( { errors.push(error); } - if snapshot.plugin_registered + if snapshot.plugin_registered == Some(true) && !host_plugin_registered && let Err(error) = run_host_plugin_registration(host, options, runner) { @@ -2590,6 +3264,7 @@ fn force_cleanup_existing_install( host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: false, + marker_absent_recovery: false, }; run_host_unregistration(host, &mut state, &options.install_dir, options, runner)?; remove_path(&layout.marketplace_root, options)?; @@ -2616,6 +3291,7 @@ fn rollback_install( host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: false, + marker_absent_recovery: false, }); if registration.any_added() { state.host_plugin_removed |= !registration.host_plugin_added; diff --git a/crates/cli/src/installation/marketplace/state.rs b/crates/cli/src/installation/marketplace/state.rs index d3d6b905d..d44588174 100644 --- a/crates/cli/src/installation/marketplace/state.rs +++ b/crates/cli/src/installation/marketplace/state.rs @@ -148,6 +148,8 @@ pub(super) struct PluginState { pub(super) host_plugin_removed: bool, pub(super) host_marketplace_removed: bool, pub(super) plugin_setup_installed: bool, + /// True only for a forced recovery that owns an intentionally marker-absent lock. + pub(super) marker_absent_recovery: bool, } #[derive(Debug, Default, Deserialize, Serialize)] @@ -329,6 +331,7 @@ pub(super) fn write_state( host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: false, + marker_absent_recovery: false, }, layout .state_path @@ -349,6 +352,7 @@ pub(super) fn mark_plugin_setup_installed( host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: false, + marker_absent_recovery: false, }); state.plugin_setup_installed = true; write_state_for_host(host, &state, &options.install_dir, options) @@ -374,18 +378,19 @@ fn write_state_for_host_arg( println!("write {}", path.display()); return Ok(()); } - write_json( - &path, - &json!({ - "host": host_arg, - "marketplaceRoot": state.marketplace_root, - "pluginRoot": state.plugin_root, - "hostUnregistered": state.host_plugin_removed && state.host_marketplace_removed, - "hostPluginRemoved": state.host_plugin_removed, - "hostMarketplaceRemoved": state.host_marketplace_removed, - "pluginSetupInstalled": state.plugin_setup_installed - }), - ) + let mut value = json!({ + "host": host_arg, + "marketplaceRoot": state.marketplace_root, + "pluginRoot": state.plugin_root, + "hostUnregistered": state.host_plugin_removed && state.host_marketplace_removed, + "hostPluginRemoved": state.host_plugin_removed, + "hostMarketplaceRemoved": state.host_marketplace_removed, + "pluginSetupInstalled": state.plugin_setup_installed + }); + if state.marker_absent_recovery { + value["markerAbsentRecovery"] = Value::Bool(true); + } + write_json(&path, &value) } pub(super) fn read_state(host: impl MarketplaceHost, install_dir: &Path) -> Option { @@ -410,6 +415,10 @@ pub(super) fn read_state(host: impl MarketplaceHost, install_dir: &Path) -> Opti .get("pluginSetupInstalled") .and_then(Value::as_bool) .unwrap_or(true), + marker_absent_recovery: value + .get("markerAbsentRecovery") + .and_then(Value::as_bool) + .unwrap_or(false), }) } diff --git a/crates/cli/tests/coverage/agents/plugin_install_tests.rs b/crates/cli/tests/coverage/agents/plugin_install_tests.rs index 4ed0592c5..e482a66fa 100644 --- a/crates/cli/tests/coverage/agents/plugin_install_tests.rs +++ b/crates/cli/tests/coverage/agents/plugin_install_tests.rs @@ -63,13 +63,18 @@ fn force_snapshot_with_backups( original_marketplace_root: PathBuf::from("original-marketplace"), original_plugin_root: PathBuf::from("separate-original-plugin"), original_generation_fence: PathBuf::from("original-generation"), - plugin_registered: false, + plugin_registered: Some(false), marketplace_registered: false, backup_marketplace_root, backup_plugin_root, marketplace_moved: true, plugin_moved: true, replacement_promoted: false, + recoverable_dangling_marketplace: false, + cleanup_committed: false, + original_marketplace_removed: false, + replacement_registration: HostRegistrationProgress::default(), + replacement_setup_installed: false, generation_retirement: None, } } @@ -765,6 +770,7 @@ struct MockSetupRunner { calls: RefCell>, doctor_roots: RefCell>, failing_call: Option, + snapshot_reappearing_root: Option, } struct BlockingRefreshFailure { @@ -895,6 +901,10 @@ impl MockSetupRunner { impl PluginSetupRunner for MockSetupRunner { fn snapshot(&self, host_arg: &str) -> Result, String> { self.record(format!("snapshot {host_arg}"))?; + if let Some(path) = self.snapshot_reappearing_root.as_ref() { + std::fs::create_dir(path) + .map_err(|error| format!("failed to inject reappearing root: {error}"))?; + } Ok(Some(PluginSetupSnapshot::Mock)) } @@ -1031,6 +1041,36 @@ fn refresh_preflight_retires_multiple_directories_for_one_host() { } } +#[test] +fn refresh_preflight_skips_an_unsafe_dangling_codex_target_and_retires_later_targets() { + let home = tempdir().unwrap(); + let _home = HomeScope::enter(home.path()); + let dangling = tempdir().unwrap(); + let healthy = tempdir().unwrap(); + write_installed_state(CodingAgent::Codex, dangling.path()); + write_installed_state(CodingAgent::ClaudeCode, healthy.path()); + let dangling_layout = PluginLayout::new(CodingAgent::Codex, dangling.path()); + let healthy_layout = PluginLayout::new(CodingAgent::ClaudeCode, healthy.path()); + std::fs::remove_dir_all(&dangling_layout.marketplace_root).unwrap(); + std::fs::write(&dangling_layout.generation_lock, "not-a-generation-uuid\n").unwrap(); + + let _preflight = retire_integrations_for_refresh(&[ + (CodingAgent::Codex, dangling.path().to_path_buf()), + (CodingAgent::ClaudeCode, healthy.path().to_path_buf()), + ]) + .unwrap(); + + assert_eq!( + std::fs::read_to_string(&dangling_layout.generation_lock).unwrap(), + "not-a-generation-uuid\n" + ); + assert!( + std::fs::read_to_string(&healthy_layout.generation_fence) + .unwrap() + .starts_with("retired:") + ); +} + #[test] fn refresh_preflight_restores_earlier_generations_when_a_target_is_invalid() { let home = tempdir().unwrap(); @@ -1122,6 +1162,7 @@ fn write_relocated_codex_install(selected_dir: &Path, relocated_dir: &Path) -> P host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: true, + marker_absent_recovery: false, }, selected_dir, &options(selected_dir), @@ -3032,6 +3073,7 @@ fn persisted_roots_accept_an_equivalent_symlinked_install_path() { host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: true, + marker_absent_recovery: false, }; selected_layout.validate_persisted_state(&state).unwrap(); @@ -3117,7 +3159,7 @@ fn force_install_rejects_registered_legacy_plugin_without_generation_fence() { } #[test] -fn force_install_reports_a_dangling_codex_marketplace_as_an_unsafe_generation() { +fn force_install_rejects_a_dangling_codex_marketplace_without_a_surviving_lock() { let dir = tempdir().unwrap(); let runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") @@ -3137,7 +3179,7 @@ fn force_install_reports_a_dangling_codex_marketplace_as_an_unsafe_generation() let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); - assert_actionable_generation_error(&error, "MCP generation marker is missing"); + assert_actionable_generation_error(&error, "surviving lock"); assert!(!layout.marketplace_root.exists()); assert!(!layout.generation_lock.exists()); assert!(runner.commands().is_empty()); @@ -3146,7 +3188,7 @@ fn force_install_reports_a_dangling_codex_marketplace_as_an_unsafe_generation() } #[test] -fn force_install_preserves_orphaned_state_for_a_dangling_codex_marketplace() { +fn force_install_recovers_a_dangling_codex_marketplace_with_its_surviving_lock() { let dir = tempdir().unwrap(); let runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") @@ -3164,23 +3206,46 @@ fn force_install_preserves_orphaned_state_for_a_dangling_codex_marketplace() { }; write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); - let original_state = std::fs::read(&layout.state_path).unwrap(); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); assert!(layout.generation_lock.exists()); + let original_lock = std::fs::read_to_string(&layout.generation_lock).unwrap(); - let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); + install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); - assert_actionable_generation_error(&error, "MCP generation marker is missing"); - assert!(!layout.marketplace_root.exists()); - assert_eq!(std::fs::read(&layout.state_path).unwrap(), original_state); + assert!(layout.marketplace_root.exists()); + assert!(layout.generation_fence.exists()); + assert!(layout.state_path.exists()); assert!(layout.generation_lock.exists()); - assert!(runner.commands().is_empty()); - assert!(setup_runner.calls().is_empty()); + assert_eq!( + std::fs::read_to_string(&layout.generation_lock).unwrap(), + original_lock + ); + assert_eq!( + runner.commands(), + vec![ + "/bin/codex plugin remove nemo-relay-plugin@nemo-relay-local".to_string(), + "/bin/codex plugin marketplace remove nemo-relay-local".to_string(), + format!( + "/bin/codex plugin marketplace add {}", + layout.marketplace_root.display() + ), + "/bin/codex plugin add nemo-relay-plugin@nemo-relay-local".to_string(), + ] + ); + assert_eq!( + setup_runner.calls(), + vec![ + "snapshot codex".to_string(), + format!("uninstall codex {DEFAULT_GATEWAY_URL}"), + "refresh gateway".to_string(), + format!("setup codex {DEFAULT_GATEWAY_URL}"), + ] + ); assert_no_install_stage(dir.path()); } #[test] -fn force_install_preserves_a_generation_when_plugin_registration_is_unknown() { +fn plain_install_recommends_force_for_a_dangling_codex_marketplace() { let dir = tempdir().unwrap(); let runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") @@ -3192,44 +3257,46 @@ fn force_install_preserves_a_generation_when_plugin_registration_is_unknown() { DANGLING_CODEX_MARKETPLACE_ERROR, ); let setup_runner = MockSetupRunner::default(); - let options = PluginInstallOptions { - force: true, - ..options(dir.path()) - }; write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); - std::fs::remove_file(&layout.marketplace_manifest).unwrap(); - let generation_token = InstallGeneration::capture(layout.generation_fence.clone()) - .unwrap() - .token() - .to_owned(); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); - let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); + let error = install_host( + CodingAgent::Codex, + &options(dir.path()), + &runner, + &setup_runner, + ) + .unwrap_err(); assert!( - error.contains("registration state could not be determined"), + error.contains("nemo-relay install codex --force"), "{error}" ); - assert!(layout.marketplace_root.exists()); - assert!(layout.state_path.exists()); - assert_eq!( - InstallGeneration::capture(layout.generation_fence) - .unwrap() - .token(), - generation_token - ); + assert!(layout.generation_lock.exists()); assert!(runner.commands().is_empty()); assert!(setup_runner.calls().is_empty()); - assert_no_install_stage(dir.path()); } #[test] -fn force_install_rejects_unregistered_legacy_plugin_without_generation_fence() { +fn force_install_recovers_when_the_unknown_plugin_was_already_unregistered() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") - .with_codex_registration(false, false); + .with_capture_output("/bin/codex plugin marketplace list", ""); + runner.capture_output_sequences.get_mut().insert( + "/bin/codex plugin list".into(), + VecDeque::from([ + CommandOutput { + status: 1, + stdout: String::new(), + stderr: DANGLING_CODEX_MARKETPLACE_ERROR.into(), + }, + CommandOutput::success(String::new()), + ]), + ); + runner.failing_suffix = Some("plugin remove nemo-relay-plugin@nemo-relay-local".into()); let setup_runner = MockSetupRunner::default(); let options = PluginInstallOptions { force: true, @@ -3237,156 +3304,576 @@ fn force_install_rejects_unregistered_legacy_plugin_without_generation_fence() { }; write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); - std::fs::remove_file(&layout.generation_fence).unwrap(); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); - let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); + install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); + assert!(layout.generation_fence.exists()); assert!( - error.contains("MCP generation marker is missing"), - "{error}" + runner.commands().iter().any(|command| { + command.ends_with("plugin remove nemo-relay-plugin@nemo-relay-local") + }) + ); + assert!( + runner + .capture_commands() + .iter() + .filter(|command| { command.ends_with("codex plugin list") }) + .count() + >= 2 ); - assert!(layout.marketplace_root.exists()); - assert!(layout.state_path.exists()); - assert!(runner.commands().is_empty()); - assert!(setup_runner.calls().is_empty()); - assert_no_install_stage(dir.path()); -} - -#[test] -fn force_install_rejects_corrupt_generation_marker_without_mutating() { - for (corruption, cause) in [ - ("empty", "is empty"), - ("oversized", "exceeds the 128-byte limit"), - ("unreadable", "failed to"), - ] { - let dir = tempdir().unwrap(); - let runner = MockRunner::default() - .with_executable("nemo-relay", "/bin/nemo-relay") - .with_executable("codex", "/bin/codex") - .with_codex_registration(false, false); - let setup_runner = MockSetupRunner::default(); - let options = PluginInstallOptions { - force: true, - ..options(dir.path()) - }; - write_installed_state(CodingAgent::Codex, dir.path()); - let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); - corrupt_generation_fence(&layout.generation_fence, corruption); - - let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); - - assert_actionable_generation_error(&error, "is invalid or unreadable"); - assert!(error.contains(cause), "{corruption}: {error}"); - assert!(layout.marketplace_root.exists()); - assert!(layout.state_path.exists()); - assert!(runner.commands().is_empty()); - assert!(setup_runner.calls().is_empty()); - assert_no_install_stage(dir.path()); - } } #[test] -fn force_install_allows_a_clean_first_install_without_generation_fence() { +fn failed_dangling_force_install_leaves_a_fenced_retry_that_can_succeed() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let first_runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") - .with_codex_registration(false, false); - let setup_runner = MockSetupRunner::default(); - let options = PluginInstallOptions { + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + let first_setup = MockSetupRunner { + failing_call: Some("refresh gateway".into()), + ..MockSetupRunner::default() + }; + let force = PluginInstallOptions { force: true, ..options(dir.path()) }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + let lock_id = std::fs::read_to_string(&layout.generation_lock).unwrap(); - install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); + let error = install_host(CodingAgent::Codex, &force, &first_runner, &first_setup).unwrap_err(); - let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); - assert!(layout.generation_fence.exists()); - assert!(layout.state_path.exists()); -} + assert!(error.contains("refresh gateway failed"), "{error}"); + assert!(!layout.marketplace_root.exists()); + assert_eq!( + std::fs::read_to_string(&layout.generation_lock).unwrap(), + lock_id + ); + assert!( + read_state(CodingAgent::Codex, dir.path()) + .unwrap() + .marker_absent_recovery + ); -#[test] -fn force_install_uses_live_absent_registration_instead_of_stale_installed_state() { - let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let retry_runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_codex_registration(false, false); - let setup_runner = MockSetupRunner::default(); - let options = PluginInstallOptions { - force: true, - ..options(dir.path()) - }; - write_installed_state(CodingAgent::Codex, dir.path()); - - install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); + install_host( + CodingAgent::Codex, + &force, + &retry_runner, + &MockSetupRunner::default(), + ) + .unwrap(); - let commands = runner.commands(); - assert!( - commands - .iter() - .all(|command| !command.contains("plugin remove") - && !command.contains("marketplace remove")), - "unexpected removal commands: {commands:?}" + assert!(layout.generation_fence.exists()); + assert_eq!( + std::fs::read_to_string(&layout.generation_lock).unwrap(), + lock_id ); assert!( - commands - .iter() - .any(|command| command.ends_with("plugin add nemo-relay-plugin@nemo-relay-local")) + !read_state(CodingAgent::Codex, dir.path()) + .unwrap() + .marker_absent_recovery ); } #[test] -fn force_install_uses_live_present_registration_instead_of_stale_removed_state() { +fn precommit_dangling_cleanup_failure_never_invents_plugin_registration_and_can_retry() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut first_runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") - .with_codex_registration(true, true); - let setup_runner = MockSetupRunner::default(); - let options = PluginInstallOptions { + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + first_runner.failing_suffix = Some("plugin marketplace remove nemo-relay-local".into()); + let force = PluginInstallOptions { force: true, ..options(dir.path()) }; write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); - write_state_for_host( + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + + let error = install_host( CodingAgent::Codex, - &PluginState { - marketplace_root: layout.marketplace_root.clone(), - plugin_root: layout.plugin_root.clone(), - host_plugin_removed: true, - host_marketplace_removed: true, - plugin_setup_installed: true, - }, - dir.path(), - &options, + &force, + &first_runner, + &MockSetupRunner::default(), ) - .unwrap(); - - install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); + .unwrap_err(); - let commands = runner.commands(); - assert!(commands.iter().any(|command| { - command == "/bin/codex plugin remove nemo-relay-plugin@nemo-relay-local" - })); + assert!(error.contains("plugin marketplace remove"), "{error}"); assert!( - commands + first_runner + .commands() .iter() - .any(|command| { command == "/bin/codex plugin marketplace remove nemo-relay-local" }) + .all(|command| !command.ends_with("plugin add nemo-relay-plugin@nemo-relay-local")) ); -} - -#[test] -fn force_install_commit_does_not_fail_when_backup_cleanup_errors() { - let dir = tempdir().unwrap(); - let backup = dir.path().join("codex-marketplace-backup"); - std::fs::write(&backup, "not a directory").unwrap(); + assert!(!layout.marketplace_root.exists()); + assert!(layout.generation_lock.exists()); + + let retry_runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + install_host( + CodingAgent::Codex, + &force, + &retry_runner, + &MockSetupRunner::default(), + ) + .unwrap(); + assert!(layout.generation_fence.exists()); +} + +#[test] +fn precommit_dangling_probe_failure_restores_known_removed_marketplace_and_can_retry() { + let dir = tempdir().unwrap(); + let mut first_runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex"); + first_runner.failing_suffix = Some("plugin remove nemo-relay-plugin@nemo-relay-local".into()); + first_runner.capture_output_sequences.get_mut().insert( + "/bin/codex plugin list".into(), + vec![ + CommandOutput { + status: 1, + stdout: String::new(), + stderr: DANGLING_CODEX_MARKETPLACE_ERROR.into(), + }, + CommandOutput { + status: 2, + stdout: String::new(), + stderr: "post-removal probe failed".into(), + }, + CommandOutput { + status: 2, + stdout: String::new(), + stderr: "rollback probe failed".into(), + }, + ] + .into(), + ); + let force = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + + let error = install_host( + CodingAgent::Codex, + &force, + &first_runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + + assert!(error.contains("post-removal probe failed"), "{error}"); + assert!(error.contains("rollback probe failed"), "{error}"); + assert!(first_runner.commands().iter().any(|command| { + command.ends_with(&format!( + "plugin marketplace add {}", + layout.marketplace_root.display() + )) + })); + assert!( + first_runner + .commands() + .iter() + .all(|command| !command.ends_with("plugin add nemo-relay-plugin@nemo-relay-local")) + ); + assert!(!layout.marketplace_root.exists()); + assert!(layout.generation_lock.exists()); + + let retry_runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + install_host( + CodingAgent::Codex, + &force, + &retry_runner, + &MockSetupRunner::default(), + ) + .unwrap(); + assert!(layout.generation_fence.exists()); +} + +#[test] +fn postcommit_marketplace_removal_failure_keeps_a_retired_tree_for_retry() { + let dir = tempdir().unwrap(); + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + write_state_for_host( + CodingAgent::Codex, + &PluginState { + marketplace_root: layout.marketplace_root.clone(), + plugin_root: layout.plugin_root.clone(), + host_plugin_removed: true, + host_marketplace_removed: true, + plugin_setup_installed: false, + marker_absent_recovery: true, + }, + dir.path(), + &options(dir.path()), + ) + .unwrap(); + let mut first_runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration_sequence(&[(false, false), (true, true)]); + first_runner.failing_suffix = Some("plugin marketplace remove nemo-relay-local".into()); + let first_setup = MockSetupRunner { + failing_call: Some(format!("setup codex {DEFAULT_GATEWAY_URL}")), + ..MockSetupRunner::default() + }; + let force = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + + let error = install_host(CodingAgent::Codex, &force, &first_runner, &first_setup).unwrap_err(); + + assert!(error.contains("setup codex"), "{error}"); + assert!(error.contains("plugin marketplace remove"), "{error}"); + assert!(layout.marketplace_root.exists()); + assert!( + std::fs::read_to_string(&layout.generation_fence) + .unwrap() + .starts_with("retired:") + ); + assert!( + !read_state(CodingAgent::Codex, dir.path()) + .unwrap() + .marker_absent_recovery + ); + assert!(layout.generation_lock.exists()); + + let retry_runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(true, true); + install_host( + CodingAgent::Codex, + &force, + &retry_runner, + &MockSetupRunner::default(), + ) + .unwrap(); + InstallGeneration::capture(layout.generation_fence) + .unwrap() + .verify_current() + .unwrap(); +} + +#[cfg(unix)] +#[test] +fn force_install_rejects_a_symlinked_dangling_marketplace_root_without_mutation() { + use std::os::unix::fs::symlink; + + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + let force = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + let target = dir.path().join("unexpected-marketplace"); + std::fs::create_dir(&target).unwrap(); + symlink(&target, &layout.marketplace_root).unwrap(); + + let error = install_host( + CodingAgent::Codex, + &force, + &runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + + assert_actionable_generation_error(&error, "MCP generation marker is missing"); + assert!(layout.marketplace_root.is_symlink()); + assert!(runner.commands().is_empty()); +} + +#[test] +fn force_install_rechecks_a_dangling_root_after_setup_snapshot() { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + let force = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + let setup_runner = MockSetupRunner { + snapshot_reappearing_root: Some(layout.marketplace_root.clone()), + ..MockSetupRunner::default() + }; + + let error = install_host(CodingAgent::Codex, &force, &runner, &setup_runner).unwrap_err(); + + assert!(error.contains("reappeared"), "{error}"); + assert!(layout.marketplace_root.is_dir()); + assert!(runner.commands().is_empty()); + assert_eq!(setup_runner.calls(), vec!["snapshot codex"]); + assert_no_install_stage(dir.path()); +} + +#[test] +fn force_install_preserves_a_generation_when_plugin_registration_is_unknown() { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + let setup_runner = MockSetupRunner::default(); + let options = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_file(&layout.marketplace_manifest).unwrap(); + let generation_token = InstallGeneration::capture(layout.generation_fence.clone()) + .unwrap() + .token() + .to_owned(); + + let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); + + assert!( + error.contains("registration state could not be determined"), + "{error}" + ); + assert!(layout.marketplace_root.exists()); + assert!(layout.state_path.exists()); + assert_eq!( + InstallGeneration::capture(layout.generation_fence) + .unwrap() + .token(), + generation_token + ); + assert!(runner.commands().is_empty()); + assert!(setup_runner.calls().is_empty()); + assert_no_install_stage(dir.path()); +} + +#[test] +fn force_install_rejects_unregistered_legacy_plugin_without_generation_fence() { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + let setup_runner = MockSetupRunner::default(); + let options = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_file(&layout.generation_fence).unwrap(); + + let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); + + assert!( + error.contains("MCP generation marker is missing"), + "{error}" + ); + assert!(layout.marketplace_root.exists()); + assert!(layout.state_path.exists()); + assert!(runner.commands().is_empty()); + assert!(setup_runner.calls().is_empty()); + assert_no_install_stage(dir.path()); +} + +#[test] +fn force_install_rejects_corrupt_generation_marker_without_mutating() { + for (corruption, cause) in [ + ("empty", "is empty"), + ("oversized", "exceeds the 128-byte limit"), + ("unreadable", "failed to"), + ] { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + let setup_runner = MockSetupRunner::default(); + let options = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + corrupt_generation_fence(&layout.generation_fence, corruption); + + let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); + + assert_actionable_generation_error(&error, "is invalid or unreadable"); + assert!(error.contains(cause), "{corruption}: {error}"); + assert!(layout.marketplace_root.exists()); + assert!(layout.state_path.exists()); + assert!(runner.commands().is_empty()); + assert!(setup_runner.calls().is_empty()); + assert_no_install_stage(dir.path()); + } +} + +#[test] +fn force_install_allows_a_clean_first_install_without_generation_fence() { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + let setup_runner = MockSetupRunner::default(); + let options = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + + install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); + + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + assert!(layout.generation_fence.exists()); + assert!(layout.state_path.exists()); +} + +#[test] +fn force_install_uses_live_absent_registration_instead_of_stale_installed_state() { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + let setup_runner = MockSetupRunner::default(); + let options = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + + install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); + + let commands = runner.commands(); + assert!( + commands + .iter() + .all(|command| !command.contains("plugin remove") + && !command.contains("marketplace remove")), + "unexpected removal commands: {commands:?}" + ); + assert!( + commands + .iter() + .any(|command| command.ends_with("plugin add nemo-relay-plugin@nemo-relay-local")) + ); +} + +#[test] +fn force_install_uses_live_present_registration_instead_of_stale_removed_state() { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(true, true); + let setup_runner = MockSetupRunner::default(); + let options = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + write_state_for_host( + CodingAgent::Codex, + &PluginState { + marketplace_root: layout.marketplace_root.clone(), + plugin_root: layout.plugin_root.clone(), + host_plugin_removed: true, + host_marketplace_removed: true, + plugin_setup_installed: true, + marker_absent_recovery: false, + }, + dir.path(), + &options, + ) + .unwrap(); + + install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); + + let commands = runner.commands(); + assert!(commands.iter().any(|command| { + command == "/bin/codex plugin remove nemo-relay-plugin@nemo-relay-local" + })); + assert!( + commands + .iter() + .any(|command| { command == "/bin/codex plugin marketplace remove nemo-relay-local" }) + ); +} + +#[test] +fn force_install_commit_does_not_fail_when_backup_cleanup_errors() { + let dir = tempdir().unwrap(); + let backup = dir.path().join("codex-marketplace-backup"); + std::fs::write(&backup, "not a directory").unwrap(); ForceInstallSnapshot { state_bytes: None, setup_snapshot: None, - plugin_registered: false, + plugin_registered: Some(false), marketplace_registered: false, original_marketplace_root: dir.path().join("original-marketplace"), original_plugin_root: dir @@ -3400,6 +3887,11 @@ fn force_install_commit_does_not_fail_when_backup_cleanup_errors() { marketplace_moved: true, plugin_moved: false, replacement_promoted: true, + recoverable_dangling_marketplace: false, + cleanup_committed: false, + original_marketplace_removed: false, + replacement_registration: HostRegistrationProgress::default(), + replacement_setup_installed: false, generation_retirement: None, } .commit(&dir.path().join("replacement.lock")); @@ -3557,6 +4049,7 @@ fn replacement_retirement_aggregates_refresh_and_restore_failures_without_rewrit &options(&install_dir), &setup_runner, None, + false, ) }); @@ -3915,13 +4408,18 @@ fn force_replacement_restoration_aggregates_independent_cleanup_failures() { original_marketplace_root: original_marketplace_root.clone(), original_plugin_root: original_plugin_root.clone(), original_generation_fence: original_plugin_root.join(GENERATION_FILE_NAME), - plugin_registered: false, + plugin_registered: Some(false), marketplace_registered: false, backup_marketplace_root: dir.path().join("missing-marketplace-backup"), backup_plugin_root: Some(dir.path().join("missing-plugin-backup")), marketplace_moved: true, plugin_moved: true, replacement_promoted: true, + recoverable_dangling_marketplace: false, + cleanup_committed: false, + original_marketplace_removed: false, + replacement_registration: HostRegistrationProgress::default(), + replacement_setup_installed: false, generation_retirement: None, }; let mut runner = MockRunner::default() @@ -3971,13 +4469,18 @@ fn force_replacement_restoration_reports_failed_host_reregistration() { original_marketplace_root: original_marketplace_root.clone(), original_plugin_root: original_marketplace_root.join("plugins/nemo-relay-plugin"), original_generation_fence: original_marketplace_root.join(GENERATION_FILE_NAME), - plugin_registered: true, + plugin_registered: Some(true), marketplace_registered: true, backup_marketplace_root: dir.path().join("unused-marketplace-backup"), backup_plugin_root: None, marketplace_moved: false, plugin_moved: false, replacement_promoted: false, + recoverable_dangling_marketplace: false, + cleanup_committed: false, + original_marketplace_removed: false, + replacement_registration: HostRegistrationProgress::default(), + replacement_setup_installed: false, generation_retirement: None, }; let mut runner = MockRunner::default() @@ -4006,7 +4509,7 @@ fn force_replacement_restoration_reports_failed_host_reregistration() { } #[test] -fn force_replacement_restoration_reregisters_snapshot_when_plugin_state_is_unknown() { +fn force_replacement_restoration_does_not_reregister_a_known_present_marketplace() { let dir = tempdir().unwrap(); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); let original_marketplace_root = dir.path().join("original-marketplace"); @@ -4016,13 +4519,18 @@ fn force_replacement_restoration_reregisters_snapshot_when_plugin_state_is_unkno original_marketplace_root: original_marketplace_root.clone(), original_plugin_root: original_marketplace_root.join("plugins/nemo-relay-plugin"), original_generation_fence: original_marketplace_root.join(GENERATION_FILE_NAME), - plugin_registered: true, + plugin_registered: Some(true), marketplace_registered: true, backup_marketplace_root: dir.path().join("unused-marketplace-backup"), backup_plugin_root: None, marketplace_moved: false, plugin_moved: false, replacement_promoted: false, + recoverable_dangling_marketplace: false, + cleanup_committed: false, + original_marketplace_removed: false, + replacement_registration: HostRegistrationProgress::default(), + replacement_setup_installed: false, generation_retirement: None, }; let runner = MockRunner::default() @@ -4045,8 +4553,8 @@ fn force_replacement_restoration_reregisters_snapshot_when_plugin_state_is_unkno .unwrap_err(); assert!(error.contains("could not be determined"), "{error}"); - assert!(runner.commands().iter().any(|command| { - command.ends_with(&format!( + assert!(runner.commands().iter().all(|command| { + !command.ends_with(&format!( "plugin marketplace add {}", original_marketplace_root.display() )) @@ -4079,11 +4587,12 @@ fn force_replacement_moves_and_restores_a_separate_plugin_tree() { previous_marketplace_root: previous_marketplace_root.clone(), previous_plugin_root: previous_plugin_root.clone(), previous_generation_fence: previous_plugin_root.join(GENERATION_FILE_NAME), - plugin_registered: false, + plugin_registered: Some(false), marketplace_registered: false, previous_setup_installed: false, previous_install_exists: true, previous_state_exists: false, + recoverable_dangling_marketplace: false, generation_retirement: None, }; let setup_runner = MockSetupRunner::default(); @@ -4898,7 +5407,7 @@ fn uninstall_rejects_registered_legacy_plugin_without_generation_fence() { } #[test] -fn uninstall_preserves_orphaned_state_for_a_dangling_codex_marketplace() { +fn plain_uninstall_recommends_force_for_a_dangling_codex_marketplace() { let dir = tempdir().unwrap(); let runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") @@ -4924,7 +5433,10 @@ fn uninstall_preserves_orphaned_state_for_a_dangling_codex_marketplace() { ) .unwrap_err(); - assert_actionable_generation_error(&error, "MCP generation marker is missing"); + assert!( + error.contains("nemo-relay uninstall codex --force"), + "{error}" + ); assert!(!layout.marketplace_root.exists()); assert_eq!(std::fs::read(&layout.state_path).unwrap(), original_state); assert!(layout.generation_lock.exists()); @@ -4932,6 +5444,165 @@ fn uninstall_preserves_orphaned_state_for_a_dangling_codex_marketplace() { assert!(setup_runner.calls().is_empty()); } +#[test] +fn force_uninstall_recovers_a_dangling_codex_marketplace() { + let dir = tempdir().unwrap(); + let runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + let setup_runner = MockSetupRunner::default(); + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + let mut force = options(dir.path()); + force.force = true; + + uninstall_host(CodingAgent::Codex, &force, &runner, &setup_runner).unwrap(); + + assert!(!layout.marketplace_root.exists()); + assert!(!layout.state_path.exists()); + assert!(!layout.generation_lock.exists()); + assert_eq!( + runner.commands(), + vec![ + "/bin/codex plugin remove nemo-relay-plugin@nemo-relay-local".to_string(), + "/bin/codex plugin marketplace remove nemo-relay-local".to_string(), + ] + ); +} + +#[test] +fn force_uninstall_reconciles_a_marketplace_removal_error_after_success() { + let dir = tempdir().unwrap(); + let mut runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_capture_output("/bin/codex plugin marketplace list", ""); + runner.capture_output_sequences.get_mut().insert( + "/bin/codex plugin list".into(), + VecDeque::from([ + CommandOutput { + status: 1, + stdout: String::new(), + stderr: DANGLING_CODEX_MARKETPLACE_ERROR.into(), + }, + CommandOutput::success(String::new()), + ]), + ); + runner.failing_suffix = Some("plugin marketplace remove nemo-relay-local".into()); + let first_setup = MockSetupRunner { + failing_call: Some(format!("uninstall codex {DEFAULT_GATEWAY_URL}")), + ..MockSetupRunner::default() + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + let mut force = options(dir.path()); + force.force = true; + + let error = uninstall_host(CodingAgent::Codex, &force, &runner, &first_setup).unwrap_err(); + assert!( + error.contains("failed to remove Relay host setup"), + "{error}" + ); + + let retry_state = read_state(CodingAgent::Codex, dir.path()).unwrap(); + assert!(retry_state.host_plugin_removed); + assert!(retry_state.host_marketplace_removed); + assert_eq!( + runner.commands(), + vec![ + "/bin/codex plugin remove nemo-relay-plugin@nemo-relay-local".to_string(), + "/bin/codex plugin marketplace remove nemo-relay-local".to_string(), + ] + ); + + let retry_runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + uninstall_host( + CodingAgent::Codex, + &force, + &retry_runner, + &MockSetupRunner::default(), + ) + .unwrap(); + + assert!(retry_runner.commands().is_empty()); + assert!(!layout.state_path.exists()); + assert!(!layout.generation_lock.exists()); +} + +#[test] +fn partial_dangling_force_uninstall_retains_a_guarded_retry() { + let dir = tempdir().unwrap(); + let first_runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + let first_setup = MockSetupRunner { + failing_call: Some(format!("uninstall codex {DEFAULT_GATEWAY_URL}")), + ..MockSetupRunner::default() + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + let lock_id = std::fs::read_to_string(&layout.generation_lock).unwrap(); + let mut force = options(dir.path()); + force.force = true; + + let error = + uninstall_host(CodingAgent::Codex, &force, &first_runner, &first_setup).unwrap_err(); + + assert!( + error.contains("failed to remove Relay host setup"), + "{error}" + ); + let retry_state = read_state(CodingAgent::Codex, dir.path()).unwrap(); + assert!(retry_state.marker_absent_recovery); + assert!(retry_state.host_plugin_removed); + assert!(retry_state.host_marketplace_removed); + assert!(retry_state.plugin_setup_installed); + assert_eq!( + std::fs::read_to_string(&layout.generation_lock).unwrap(), + lock_id + ); + let plain_error = uninstall_host( + CodingAgent::Codex, + &options(dir.path()), + &MockRunner::default(), + &MockSetupRunner::default(), + ) + .unwrap_err(); + assert!( + plain_error.contains("nemo-relay uninstall codex --force"), + "{plain_error}" + ); + + let retry_runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + uninstall_host( + CodingAgent::Codex, + &force, + &retry_runner, + &MockSetupRunner::default(), + ) + .unwrap(); + + assert!(retry_runner.commands().is_empty()); + assert!(!layout.state_path.exists()); + assert!(!layout.generation_lock.exists()); +} + #[test] fn uninstall_rejects_unregistered_legacy_plugin_without_generation_fence() { let dir = tempdir().unwrap(); @@ -5201,6 +5872,7 @@ fn doctor_uses_plugin_root_persisted_in_install_state() { host_plugin_removed: false, host_marketplace_removed: false, plugin_setup_installed: true, + marker_absent_recovery: false, }, dir.path(), &install_options, @@ -6045,6 +6717,7 @@ fn uninstall_retry_skips_host_removal_after_prior_success() { host_plugin_removed: true, host_marketplace_removed: true, plugin_setup_installed: true, + marker_absent_recovery: false, }, dir.path(), &options(dir.path()), diff --git a/crates/cli/tests/coverage/shared/install_generation_tests.rs b/crates/cli/tests/coverage/shared/install_generation_tests.rs index ff7d2d063..08da25e1f 100644 --- a/crates/cli/tests/coverage/shared/install_generation_tests.rs +++ b/crates/cli/tests/coverage/shared/install_generation_tests.rs @@ -64,6 +64,158 @@ fn plugin_retirement_accepts_an_equivalent_symlinked_external_lock_path() { retirement.restore_after_rollback().unwrap(); } +fn surviving_external_lock(marker: &Path, lock: &Path) { + write_new_generation_with_token_at(marker, lock).unwrap(); + std::fs::remove_file(marker).unwrap(); +} + +fn expect_missing_retirement_error(result: Result) -> String { + match result { + Err(error) => error, + Ok(_) => panic!("unsafe marker-absent generation retirement was accepted"), + } +} + +#[test] +fn marker_absent_retirement_holds_and_reuses_the_surviving_lock_without_restoring_a_marker() { + let dir = tempdir().unwrap(); + let marker = dir.path().join("plugin").join(GENERATION_FILE_NAME); + let lock = dir.path().join("generation.lock"); + surviving_external_lock(&marker, &lock); + + let mut retirement = GenerationRetirement::acquire_missing_for_plugin(&marker, &lock).unwrap(); + assert!(retirement.original.marker_was_absent()); + retirement.invalidate_for_replacement().unwrap(); + assert!(!marker.exists()); + + let replacement = write_staged_generation_with_token(&marker, &lock).unwrap(); + assert_eq!(retirement.active_visible_token().unwrap(), replacement); + retirement.retire_visible_replacement().unwrap(); + std::fs::remove_file(&marker).unwrap(); + retirement.restore_after_rollback().unwrap(); + + assert!(!marker.exists()); + let retry = GenerationRetirement::acquire_missing_for_plugin(&marker, &lock).unwrap(); + drop(retry); + assert!(!marker.exists()); +} + +#[test] +fn marker_absent_retirement_rejects_missing_empty_malformed_and_non_regular_locks() { + let dir = tempdir().unwrap(); + let marker = dir.path().join("plugin").join(GENERATION_FILE_NAME); + let missing = dir.path().join("missing.lock"); + let error = expect_missing_retirement_error(GenerationRetirement::acquire_missing_for_plugin( + &marker, &missing, + )); + assert!(error.contains("failed to open"), "{error}"); + assert!(!missing.exists()); + + let empty = dir.path().join("empty.lock"); + std::fs::write(&empty, []).unwrap(); + let error = expect_missing_retirement_error(GenerationRetirement::acquire_missing_for_plugin( + &marker, &empty, + )); + assert!(error.contains("is empty"), "{error}"); + assert_eq!(std::fs::read(&empty).unwrap(), b""); + + let malformed = dir.path().join("malformed.lock"); + std::fs::write(&malformed, b"not-a-uuid\n").unwrap(); + let error = expect_missing_retirement_error(GenerationRetirement::acquire_missing_for_plugin( + &marker, &malformed, + )); + assert!(error.contains("invalid identity"), "{error}"); + assert_eq!(std::fs::read(&malformed).unwrap(), b"not-a-uuid\n"); + + let directory = dir.path().join("directory.lock"); + std::fs::create_dir(&directory).unwrap(); + let error = expect_missing_retirement_error(GenerationRetirement::acquire_missing_for_plugin( + &marker, &directory, + )); + assert!(error.contains("not a regular file"), "{error}"); +} + +#[test] +fn marker_absent_retirement_rejects_contention_without_changing_the_lock() { + let dir = tempdir().unwrap(); + let marker = dir.path().join("plugin").join(GENERATION_FILE_NAME); + let lock = dir.path().join("generation.lock"); + surviving_external_lock(&marker, &lock); + let contents = std::fs::read(&lock).unwrap(); + let first = GenerationRetirement::acquire_missing_for_plugin(&marker, &lock).unwrap(); + + let error = + expect_missing_retirement_error(GenerationRetirement::acquire_missing_with_timeout( + &marker, + &lock, + Duration::from_millis(20), + )); + + assert!(error.contains("timed out waiting"), "{error}"); + assert_eq!(std::fs::read(&lock).unwrap(), contents); + drop(first); +} + +#[test] +fn marker_absent_retirement_fails_closed_if_the_marker_reappears() { + let dir = tempdir().unwrap(); + let marker = dir.path().join("plugin").join(GENERATION_FILE_NAME); + let lock = dir.path().join("generation.lock"); + surviving_external_lock(&marker, &lock); + let retirement = GenerationRetirement::acquire_missing_for_plugin(&marker, &lock).unwrap(); + + std::fs::write(&marker, b"unexpected\n").unwrap(); + let error = retirement.revalidate_missing_marker().unwrap_err(); + + assert!(error.contains("remain absent"), "{error}"); + drop(retirement); + assert_eq!(std::fs::read(&marker).unwrap(), b"unexpected\n"); +} + +#[cfg(unix)] +#[test] +fn marker_absent_retirement_rejects_symlinked_locks_and_reappearing_marker_symlinks() { + use std::os::unix::fs::symlink; + + let dir = tempdir().unwrap(); + let marker = dir.path().join("plugin").join(GENERATION_FILE_NAME); + let lock_target = dir.path().join("generation-target.lock"); + let lock_link = dir.path().join("generation.lock"); + surviving_external_lock(&marker, &lock_target); + symlink(&lock_target, &lock_link).unwrap(); + + let error = expect_missing_retirement_error(GenerationRetirement::acquire_missing_for_plugin( + &marker, &lock_link, + )); + assert!(error.contains("symlinked"), "{error}"); + + let mut retirement = + GenerationRetirement::acquire_missing_for_plugin(&marker, &lock_target).unwrap(); + let marker_target = dir.path().join("unexpected-marker"); + std::fs::write(&marker_target, b"unexpected\n").unwrap(); + symlink(&marker_target, &marker).unwrap(); + let error = retirement.revalidate_missing_marker().unwrap_err(); + assert!(error.contains("a symlink"), "{error}"); + retirement.commit_replacement(); +} + +#[cfg(unix)] +#[test] +fn marker_absent_retirement_rejects_a_replaced_lock_inode_even_with_the_same_uuid() { + let dir = tempdir().unwrap(); + let marker = dir.path().join("plugin").join(GENERATION_FILE_NAME); + let lock = dir.path().join("generation.lock"); + surviving_external_lock(&marker, &lock); + let contents = std::fs::read(&lock).unwrap(); + let retirement = GenerationRetirement::acquire_missing_for_plugin(&marker, &lock).unwrap(); + + std::fs::remove_file(&lock).unwrap(); + std::fs::write(&lock, contents).unwrap(); + let error = retirement.revalidate_missing_marker().unwrap_err(); + + assert!(error.contains("changed identity"), "{error}"); +} + #[test] fn generation_markers_have_one_canonical_encoding() { let lock_path = PathBuf::from("generation.lock"); @@ -385,6 +537,29 @@ fn staged_generation_lock_remains_held_across_marketplace_promotion() { ); } +#[test] +fn promoted_generation_rejects_a_different_token_on_the_same_lock() { + let dir = tempdir().unwrap(); + let staged_plugin = dir.path().join("staged").join("plugin"); + let live_plugin = dir.path().join("live").join("plugin"); + let staged_marker = staged_plugin.join(GENERATION_FILE_NAME); + let live_marker = live_plugin.join(GENERATION_FILE_NAME); + let lock_path = dir.path().join("replacement-generation.lock"); + write_new_generation_with_token_at(&staged_marker, &lock_path).unwrap(); + let mut retirement = GenerationRetirement::acquire(&staged_marker) + .unwrap() + .unwrap(); + + std::fs::create_dir_all(live_plugin.parent().unwrap()).unwrap(); + std::fs::rename(&staged_plugin, &live_plugin).unwrap(); + write_staged_generation_with_token(&live_marker, &lock_path).unwrap(); + + let error = retirement + .retarget_promoted_marker(&live_marker) + .unwrap_err(); + assert!(error.contains("marker changed"), "{error}"); +} + #[test] fn legacy_sibling_lock_can_be_released_for_tree_move_and_reacquired_for_rollback() { let dir = tempdir().unwrap(); @@ -559,7 +734,10 @@ fn rollback_can_restore_with_the_original_lock_still_held() { lock: Some(lock), lock_id, path: path.clone(), - original: GenerationMarker::active("generation-a", generation_lock_path(&path)), + original: GenerationRetirementOriginal::MarkerPresent(GenerationMarker::active( + "generation-a", + generation_lock_path(&path), + )), changed: true, committed: false, lock_released_for_tree_mutation: false, @@ -584,7 +762,8 @@ fn rollback_restores_the_visible_path_after_atomic_marker_replacement() { let mut retirement = GenerationRetirement::acquire(&path).unwrap().unwrap(); retirement.invalidate_for_replacement().unwrap(); - atomic_write(&path, retirement.original.retired().encoded().as_bytes()).unwrap(); + let original_marker = retirement.original.marker().unwrap(); + atomic_write(&path, original_marker.retired().encoded().as_bytes()).unwrap(); retirement.restore_after_rollback().unwrap(); assert_eq!(std::fs::read(&path).unwrap(), original_bytes); @@ -614,7 +793,10 @@ fn invalidation_requires_a_live_exclusive_lock() { lock: None, lock_id: uuid::Uuid::nil().to_string(), path: path.clone(), - original: GenerationMarker::active("generation-a", generation_lock_path(&path)), + original: GenerationRetirementOriginal::MarkerPresent(GenerationMarker::active( + "generation-a", + generation_lock_path(&path), + )), changed: false, committed: false, lock_released_for_tree_mutation: false, @@ -698,7 +880,10 @@ fn rollback_requires_the_original_transaction_lock() { lock: None, lock_id: uuid::Uuid::nil().to_string(), path: path.clone(), - original: GenerationMarker::active("generation-a", generation_lock_path(&path)), + original: GenerationRetirementOriginal::MarkerPresent(GenerationMarker::active( + "generation-a", + generation_lock_path(&path), + )), changed: true, committed: false, lock_released_for_tree_mutation: false, diff --git a/docs/nemo-relay-cli/plugin-installation.mdx b/docs/nemo-relay-cli/plugin-installation.mdx index 38b97e8b0..82bc1046b 100644 --- a/docs/nemo-relay-cli/plugin-installation.mdx +++ b/docs/nemo-relay-cli/plugin-installation.mdx @@ -107,6 +107,41 @@ each user and host, even when two operations name different install directories. If another operation is still active after a short wait, Relay stops with a timeout instead of changing host-wide plugin state concurrently. +### Recover a Deleted Codex Marketplace + +If the generated Codex marketplace directory is deleted while Codex still has +`nemo-relay-local` registered, Codex reports the marketplace as present but +unloadable and cannot say whether `nemo-relay-plugin` is registered. Relay +recognizes this exact dangling state when both the expected marketplace root and +its generation marker are absent. Recovery is explicit: + +```bash +nemo-relay install codex --force +nemo-relay uninstall codex --force +``` + +Use the first command to rebuild the integration or the second to finish +removing it. The corresponding command without `--force` does not change the +dangling state and tells you which forced command to run. + +Recovery reuses the surviving generation lock and fails before changing +anything if that lock cannot be proved safe—for example, if it is missing, +malformed, replaced, symlinked, not a regular file, or held by another process, +or if the marketplace root or marker reappears while Relay acquires it. In that +case, follow the manual-remediation instructions in the error instead of +replacing the lock. + +During a forced reinstall, removal of the old provider and hooks, plugin +registration, and marketplace registration is one cleanup phase. Until all +three areas have been cleaned, a failure restores only prior state Relay can +prove existed. After all three are clean, cleanup is committed: a later install +failure does not recreate the dangling Codex registration. Relay keeps the +validated lock and records the clean progress so another +`nemo-relay install codex --force` or `nemo-relay integrations refresh` can +retry safely. A partially failed forced uninstall is likewise retryable with +`nemo-relay uninstall codex --force`; Relay removes the retained lock and state +only after cleanup succeeds completely. + ## What Install Changes For Claude Code and Codex, `nemo-relay install` writes a local marketplace named @@ -207,6 +242,11 @@ nemo-relay install codex --install-dir --force nemo-relay integrations refresh ``` +Refresh attempts each managed target through the same forced-install recovery +path. If one Codex target has an unsafe surviving lock, Relay reports that +target, continues refreshing the others, and returns an error after every +target has been attempted. + Manual MCP configurations are not changed; reinstall them through Relay if you want Relay to manage future refreshes. @@ -239,6 +279,9 @@ that file contains the generation-file path and immutable generation identity. Relay uses the configured hook failure policy. This prevents a legacy hook retained by a host process from reviving a retired installation. If install or uninstall reports a missing or invalid generation marker, follow its cleanup +instructions. If the generated Codex marketplace was deleted but its +registration remains, use the forced recovery described above. When forced +recovery cannot validate the surviving lock, follow the error's manual cleanup instructions: close the host and standalone `nemo-relay mcp` processes, remove the stale registration and state it identifies, and then run the requested `--force` command. @@ -392,13 +435,16 @@ selections no longer parse. The `--agent` option remains supported for Claude Code and Codex `run` flows. Existing fenced installations can be refreshed with `nemo-relay install --force`. Relay refuses to replace an older MCP installation without a valid generation marker because a cached host process -might still be running. In that case, follow the manual cleanup steps in the +might still be running. The exact deleted-Codex-marketplace state can be +recovered with the forced commands described above when its surviving lock is +safe. For other missing-marker states, follow the manual cleanup steps in the error before you retry the forced install. This release removes the internal `nemo-relay plugin-shim` command. Refresh a fenced generated installation with `nemo-relay install --force`; use the -manual cleanup described above when its generation marker is missing. For -custom automation, use these supported replacements: +deleted-marketplace recovery described above for that exact Codex state, and +use manual cleanup for other missing-marker states. For custom automation, use +these supported replacements: | Removed Internal Command | Supported Replacement | | --- | --- | From c38a81dfd8cc511de0f1639b44d5a3f66fccb1cb Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Tue, 8 Sep 2026 19:30:09 -0600 Subject: [PATCH 2/3] fix(cli): safely recover dangling Codex marketplace registrations Signed-off-by: Sara Tadayon --- .../cli/src/installation/marketplace/host.rs | 27 +- .../cli/src/installation/marketplace/mod.rs | 331 ++++++++++---- .../coverage/agents/plugin_install_tests.rs | 420 ++++++++++++++++-- .../shared/install_generation_tests.rs | 6 +- docs/nemo-relay-cli/plugin-installation.mdx | 35 +- 5 files changed, 658 insertions(+), 161 deletions(-) diff --git a/crates/cli/src/installation/marketplace/host.rs b/crates/cli/src/installation/marketplace/host.rs index 87c57d5e7..5de4ad10a 100644 --- a/crates/cli/src/installation/marketplace/host.rs +++ b/crates/cli/src/installation/marketplace/host.rs @@ -99,6 +99,8 @@ pub(crate) struct HostRegistrationReport { /// `None` means the host CLI could not determine whether the plugin is registered. pub(crate) host_plugin_registered: Option, pub(crate) host_marketplace_registered: bool, + /// Source Codex reported for its Relay marketplace when the marketplace cannot be loaded. + pub(crate) host_marketplace_source: Option, /// The registered Relay marketplace was identified but its snapshot could not be loaded. pub(crate) host_marketplace_unloadable: bool, } @@ -166,6 +168,7 @@ pub(super) fn host_registration_report( return Ok(HostRegistrationReport { host_plugin_registered: Some(true), host_marketplace_registered: true, + host_marketplace_source: None, host_marketplace_unloadable: false, }); } @@ -180,6 +183,7 @@ pub(crate) fn claude_registration_report( Ok(HostRegistrationReport { host_plugin_registered: Some(claude_plugin_registered(options, runner)?), host_marketplace_registered: claude_marketplace_registered(options, runner)?, + host_marketplace_source: None, host_marketplace_unloadable: false, }) } @@ -190,10 +194,11 @@ pub(crate) fn codex_registration_report( ) -> Result { let host_plugin_registered = match codex_plugin_registered(options, runner) { Ok(registered) => registered, - Err(error) if is_dangling_codex_marketplace_error(&error) => { + Err(error) if let Some(source) = dangling_codex_marketplace_source(&error) => { return Ok(HostRegistrationReport { host_plugin_registered: None, host_marketplace_registered: true, + host_marketplace_source: Some(source), host_marketplace_unloadable: true, }); } @@ -203,6 +208,7 @@ pub(crate) fn codex_registration_report( Ok(HostRegistrationReport { host_plugin_registered: Some(host_plugin_registered), host_marketplace_registered: codex_marketplace_registered(options, runner)?, + host_marketplace_source: None, host_marketplace_unloadable: false, }) } @@ -284,14 +290,19 @@ fn codex_marketplace_registered( .any(|name| name == MARKETPLACE_NAME)) } -fn is_dangling_codex_marketplace_error(error: &str) -> bool { +fn dangling_codex_marketplace_source(error: &str) -> Option { let plugin_list_snapshot_error = "failed to load configured marketplace snapshot(s):"; - let marketplace = format!("`{MARKETPLACE_NAME}`"); - let invalid_manifest = "marketplace root does not contain a supported manifest"; - error.contains(plugin_list_snapshot_error) - && error - .lines() - .any(|line| line.contains(&marketplace) && line.contains(invalid_manifest)) + if !error.contains(plugin_list_snapshot_error) { + return None; + } + let prefix = format!("- `{MARKETPLACE_NAME}` at "); + let suffix = ": marketplace root does not contain a supported manifest"; + error.lines().find_map(|line| { + line.strip_prefix(&prefix) + .and_then(|source| source.strip_suffix(suffix)) + .filter(|source| !source.is_empty()) + .map(PathBuf::from) + }) } fn plugin_entry_matches(entry: &Value) -> bool { diff --git a/crates/cli/src/installation/marketplace/mod.rs b/crates/cli/src/installation/marketplace/mod.rs index d05870a87..d84f87ec9 100644 --- a/crates/cli/src/installation/marketplace/mod.rs +++ b/crates/cli/src/installation/marketplace/mod.rs @@ -1280,30 +1280,9 @@ fn force_uninstall_host_locked( && state.marker_absent_recovery { layout.validate_persisted_state(&state)?; - if !path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? - || !path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")? - { - return Err(unsafe_generation_fence_error( - host, - "or its marketplace root reappeared during a marker-absent cleanup retry", - )); - } - let retirement = acquire_dangling_generation_retirement(host, &layout)?; - return force_uninstall_dangling_marketplace_locked( - host, - &layout, - retirement, - options, - runner, - setup_runner, - host_registration_report(host, options, runner).ok(), - ); - } - if path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? - && path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")? - && let Ok(registration) = host_registration_report(host, options, runner) - && is_recoverable_dangling_marketplace(host, &layout, ®istration)? - { + let registration = host_registration_report(host, options, runner)?; + let classification = classify_dangling_marketplace(host, &layout, ®istration)?; + validate_marker_absent_retry(host, &layout, &state, ®istration, classification)?; let retirement = acquire_dangling_generation_retirement(host, &layout)?; return force_uninstall_dangling_marketplace_locked( host, @@ -1315,6 +1294,26 @@ fn force_uninstall_host_locked( Some(registration), ); } + if let Ok(registration) = host_registration_report(host, options, runner) { + match classify_dangling_marketplace(host, &layout, ®istration)? { + DanglingMarketplaceClassification::Recoverable => { + let retirement = acquire_dangling_generation_retirement(host, &layout)?; + return force_uninstall_dangling_marketplace_locked( + host, + &layout, + retirement, + options, + runner, + setup_runner, + Some(registration), + ); + } + DanglingMarketplaceClassification::Unsafe => { + return Err(unsafe_dangling_marketplace_error(host)); + } + DanglingMarketplaceClassification::NotDangling => {} + } + } } let mut errors = Vec::new(); @@ -1412,7 +1411,7 @@ fn force_uninstall_dangling_marketplace_locked( } }; - let plugin_removal_error = if retry_progress.is_some_and(|state| state.host_plugin_removed) + let plugin_removal_error = if retry_progress.is_some() && registration .as_ref() .is_some_and(|report| report.host_plugin_registered == Some(false)) @@ -1421,8 +1420,7 @@ fn force_uninstall_dangling_marketplace_locked( } else { run_host_plugin_removal(host, options, runner).err() }; - let marketplace_removal_error = if retry_progress - .is_some_and(|state| state.host_marketplace_removed) + let marketplace_removal_error = if retry_progress.is_some() && registration .as_ref() .is_some_and(|report| !report.host_marketplace_registered) @@ -1431,24 +1429,18 @@ fn force_uninstall_dangling_marketplace_locked( } else { run_host_marketplace_removal(host, options, runner).err() }; - let mut host_plugin_removed = plugin_removal_error.is_none(); - let mut host_marketplace_removed = marketplace_removal_error.is_none(); - if plugin_removal_error.is_some() || marketplace_removal_error.is_some() { - match host_registration_report(host, options, runner) { - Ok(report) => { - if plugin_removal_error.is_some() { - host_plugin_removed = report.host_plugin_registered == Some(false); - } - if marketplace_removal_error.is_some() { - host_marketplace_removed = !report.host_marketplace_registered; - } - } - Err(error) => { - errors.push(format!( - "could not verify host registration cleanup after removal failed: {error}" - )); - } - } + let (host_plugin_removed, host_marketplace_removed, verification_error) = + reconcile_host_removal_results( + host, + options, + runner, + plugin_removal_error.as_deref(), + marketplace_removal_error.as_deref(), + ); + if let Some(error) = verification_error { + errors.push(format!( + "could not verify host registration cleanup after removal failed: {error}" + )); } if !host_plugin_removed && let Some(error) = plugin_removal_error { errors.push(format!("failed to unregister the host plugin: {error}")); @@ -1527,6 +1519,33 @@ fn force_uninstall_dangling_marketplace_locked( } } +fn reconcile_host_removal_results( + host: impl MarketplaceHost, + options: &PluginInstallOptions, + runner: &dyn CommandRunner, + plugin_removal_error: Option<&str>, + marketplace_removal_error: Option<&str>, +) -> (bool, bool, Option) { + let mut host_plugin_removed = plugin_removal_error.is_none(); + let mut host_marketplace_removed = marketplace_removal_error.is_none(); + if plugin_removal_error.is_none() && marketplace_removal_error.is_none() { + return (host_plugin_removed, host_marketplace_removed, None); + } + + match host_registration_report(host, options, runner) { + Ok(report) => { + if plugin_removal_error.is_some() { + host_plugin_removed = report.host_plugin_registered == Some(false); + } + if marketplace_removal_error.is_some() { + host_marketplace_removed = !report.host_marketplace_registered; + } + (host_plugin_removed, host_marketplace_removed, None) + } + Err(error) => (host_plugin_removed, host_marketplace_removed, Some(error)), + } +} + fn retire_installed_generation( host: impl MarketplaceHost, layout: &PluginLayout, @@ -1542,7 +1561,10 @@ fn retire_installed_generation( let mut existing_install = local_install_exists; if path_is_absent_no_follow(&generation_fence, "MCP generation marker")? { let registration = host_registration_report(host, options, runner)?; - if is_recoverable_dangling_marketplace(host, layout, ®istration)? { + if matches!( + classify_dangling_marketplace(host, layout, ®istration)?, + DanglingMarketplaceClassification::Recoverable + ) { return Err(dangling_marketplace_requires_force_error(host, "uninstall")); } existing_install |= registration.host_plugin_may_be_registered() @@ -1639,23 +1661,116 @@ fn existing_plugin_install_requires_force_error(host: impl MarketplaceHost) -> S ) } -/// Limits automatic recovery to Codex's known dangling-marketplace failure, avoiding recovery -/// from unrelated host failures. -fn is_recoverable_dangling_marketplace( +#[derive(Clone, Copy)] +enum DanglingMarketplaceClassification { + NotDangling, + Recoverable, + Unsafe, +} + +fn classify_dangling_marketplace( host: impl MarketplaceHost, layout: &PluginLayout, registration: &HostRegistrationReport, -) -> Result { +) -> Result { if host.install_arg() != "codex" || registration.host_plugin_registered.is_some() || !registration.host_marketplace_registered || !registration.host_marketplace_unloadable { - return Ok(false); + return Ok(DanglingMarketplaceClassification::NotDangling); } - Ok( - path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? - && path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")?, + if !marketplace_source_matches_layout( + registration.host_marketplace_source.as_deref(), + &layout.marketplace_root, + ) { + return Err(unsafe_generation_fence_error( + host, + "is an unloadable Codex marketplace registration, but Codex reports a different marketplace source than the selected install directory", + )); + } + if path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? + && path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")? + { + Ok(DanglingMarketplaceClassification::Recoverable) + } else { + Ok(DanglingMarketplaceClassification::Unsafe) + } +} + +fn marketplace_source_matches_layout(source: Option<&Path>, selected: &Path) -> bool { + let Some(source) = source else { + return false; + }; + source == selected + || source.file_name() == selected.file_name() + && source + .parent() + .and_then(|parent| parent.canonicalize().ok()) + .zip( + selected + .parent() + .and_then(|parent| parent.canonicalize().ok()), + ) + .is_some_and(|(source, selected)| source == selected) +} + +fn validate_marker_absent_retry( + host: impl MarketplaceHost, + layout: &PluginLayout, + state: &PluginState, + registration: &HostRegistrationReport, + classification: DanglingMarketplaceClassification, +) -> Result<(), String> { + if !path_is_absent_no_follow(&layout.marketplace_root, "marketplace root")? + || !path_is_absent_no_follow(&layout.generation_fence, "MCP generation marker")? + { + return Err(unsafe_dangling_marketplace_error(host)); + } + match classification { + DanglingMarketplaceClassification::Unsafe => Err(unsafe_dangling_marketplace_error(host)), + DanglingMarketplaceClassification::Recoverable if state.host_marketplace_removed => { + Err(marker_absent_retry_conflict_error(host)) + } + DanglingMarketplaceClassification::Recoverable => Ok(()), + DanglingMarketplaceClassification::NotDangling + if registration.host_plugin_registered == Some(false) + && !registration.host_marketplace_registered => + { + Ok(()) + } + DanglingMarketplaceClassification::NotDangling + if state.host_plugin_removed && registration.host_plugin_registered != Some(false) => + { + Err(marker_absent_retry_conflict_error(host)) + } + DanglingMarketplaceClassification::NotDangling + if state.host_marketplace_removed && registration.host_marketplace_registered => + { + Err(marker_absent_retry_conflict_error(host)) + } + DanglingMarketplaceClassification::NotDangling + if state.host_plugin_removed || state.host_marketplace_removed => + { + Ok(()) + } + DanglingMarketplaceClassification::NotDangling => { + Err(marker_absent_retry_conflict_error(host)) + } + } +} + +fn marker_absent_retry_conflict_error(host: impl MarketplaceHost) -> String { + unsafe_generation_fence_error( + host, + "has marker-absent recovery state that conflicts with the current Codex registration; do not remove that registration automatically", + ) +} + +fn unsafe_dangling_marketplace_error(host: impl MarketplaceHost) -> String { + unsafe_generation_fence_error( + host, + "is an unloadable Codex marketplace registration, but its marketplace root and generation marker are not both safely absent", ) } @@ -2442,12 +2557,24 @@ fn prepare_plugin_install( // roots, so `local_install_exists` covers everything the state file could point at. let previous_install_exists = local_install_exists || plugin_may_be_registered || marketplace_registered; - let exact_dangling_marketplace = - is_recoverable_dangling_marketplace(host, layout, ®istration)?; - let marker_absent_retry = host.install_arg() == "codex" - && persisted - .as_ref() - .is_some_and(|state| state.marker_absent_recovery); + let dangling_classification = classify_dangling_marketplace(host, layout, ®istration)?; + if matches!( + dangling_classification, + DanglingMarketplaceClassification::Unsafe + ) { + return Err(unsafe_dangling_marketplace_error(host)); + } + let exact_dangling_marketplace = matches!( + dangling_classification, + DanglingMarketplaceClassification::Recoverable + ); + let marker_absent_retry_state = persisted + .as_ref() + .filter(|state| host.install_arg() == "codex" && state.marker_absent_recovery); + if let Some(state) = marker_absent_retry_state { + validate_marker_absent_retry(host, layout, state, ®istration, dangling_classification)?; + } + let marker_absent_retry = marker_absent_retry_state.is_some(); let recoverable_dangling_marketplace = exact_dangling_marketplace || marker_absent_retry; let generation_fence_absent = path_is_absent_no_follow(&previous_generation_fence, "MCP generation marker")?; @@ -2704,6 +2831,7 @@ fn cleanup_previous_install_for_replacement( } state.host_marketplace_removed = true; snapshot.original_marketplace_removed = true; + write_state_for_host(host, state, &options.install_dir, options)?; if let Some(plugin_error) = unknown_plugin_removal_error.take() { match host_registration_report(host, options, runner) { Ok(report) if report.host_plugin_registered == Some(false) => { @@ -2806,16 +2934,13 @@ fn begin_force_replacement( }); let cleanup_result = if recoverable_dangling_marketplace { let persisted_recovery_progress = cleanup_state.marker_absent_recovery; - if persisted_recovery_progress { - // Persisted progress is useful only when fresh host evidence agrees. A registration - // may have reappeared between attempts; unknown evidence requires conservative - // removal. - cleanup_state.host_plugin_removed &= plugin_registered == Some(false); - cleanup_state.host_marketplace_removed &= !marketplace_registered; - } else { + if !persisted_recovery_progress { cleanup_state.host_plugin_removed = plugin_registered == Some(false); cleanup_state.host_marketplace_removed = !marketplace_registered; cleanup_state.plugin_setup_installed = previous_setup_installed; + } else { + cleanup_state.host_plugin_removed |= plugin_registered == Some(false); + cleanup_state.host_marketplace_removed |= !marketplace_registered; } cleanup_state.marker_absent_recovery = true; revalidate_dangling_paths( @@ -2959,6 +3084,14 @@ fn restore_force_replacement( Err(errors.join("; ")) }; } + if snapshot.recoverable_dangling_marketplace && snapshot.original_marketplace_removed { + preserve_partial_dangling_recovery(host, layout, options, &mut errors); + return if errors.is_empty() { + Ok(()) + } else { + Err(errors.join("; ")) + }; + } remove_promoted_replacement(host, layout, snapshot, options, runner, &mut errors); restore_replaced_paths(snapshot, &mut errors); if let Some(retirement) = snapshot.generation_retirement.as_mut() @@ -2976,6 +3109,27 @@ fn restore_force_replacement( } } +fn preserve_partial_dangling_recovery( + host: impl MarketplaceHost, + layout: &PluginLayout, + options: &PluginInstallOptions, + errors: &mut Vec, +) { + let retry_state = PluginState { + marketplace_root: layout.marketplace_root.clone(), + plugin_root: layout.plugin_root.clone(), + host_plugin_removed: false, + host_marketplace_removed: true, + plugin_setup_installed: false, + marker_absent_recovery: true, + }; + if let Err(error) = write_state_for_host(host, &retry_state, &options.install_dir, options) { + errors.push(format!( + "failed to preserve Relay cleanup retry state: {error}" + )); + } +} + #[allow(clippy::too_many_arguments)] fn converge_committed_dangling_recovery( host: impl MarketplaceHost, @@ -3011,22 +3165,18 @@ fn converge_committed_dangling_recovery( .host_marketplace_added .then(|| run_host_marketplace_removal(host, options, runner).err()) .flatten(); - let mut host_plugin_removed = plugin_removal_error.is_none(); - let mut host_marketplace_removed = marketplace_removal_error.is_none(); - if plugin_removal_error.is_some() || marketplace_removal_error.is_some() { - match host_registration_report(host, options, runner) { - Ok(report) => { - if plugin_removal_error.is_some() { - host_plugin_removed = report.host_plugin_registered == Some(false); - } - if marketplace_removal_error.is_some() { - host_marketplace_removed = !report.host_marketplace_registered; - } - } - Err(error) => errors.push(format!( - "failed to verify incomplete replacement registration cleanup: {error}" - )), - } + let (host_plugin_removed, host_marketplace_removed, verification_error) = + reconcile_host_removal_results( + host, + options, + runner, + plugin_removal_error.as_deref(), + marketplace_removal_error.as_deref(), + ); + if let Some(error) = verification_error { + errors.push(format!( + "failed to verify incomplete replacement registration cleanup: {error}" + )); } if !host_plugin_removed && let Some(error) = plugin_removal_error { errors.push(format!( @@ -3140,21 +3290,6 @@ fn reconcile_restored_registration( let report = match host_registration_report(host, options, runner) { Ok(report) => report, Err(error) => { - // Cleanup positively recorded removing the known-old marketplace even though the - // later host probe failed. Restore that known registration directly. The plugin's - // original tri-state remains unknown, so rollback must not invent it. - if snapshot.recoverable_dangling_marketplace - && snapshot.marketplace_registered - && snapshot.original_marketplace_removed - && let Err(restore_error) = run_host_marketplace_registration( - host, - &snapshot.original_marketplace_root, - options, - runner, - ) - { - errors.push(restore_error); - } errors.push(error); return; } diff --git a/crates/cli/tests/coverage/agents/plugin_install_tests.rs b/crates/cli/tests/coverage/agents/plugin_install_tests.rs index e482a66fa..ef725b780 100644 --- a/crates/cli/tests/coverage/agents/plugin_install_tests.rs +++ b/crates/cli/tests/coverage/agents/plugin_install_tests.rs @@ -53,6 +53,13 @@ const LOCK_HELPER_RELEASE_ENV: &str = "NEMO_RELAY_TEST_LOCK_RELEASE"; const TEST_GENERATION_TOKEN: &str = "test-generation"; const DANGLING_CODEX_MARKETPLACE_ERROR: &str = "Error: failed to load configured marketplace snapshot(s):\n\n- `nemo-relay-local` at /tmp/plugins/codex-marketplace: marketplace root does not contain a supported manifest\n"; +fn dangling_codex_marketplace_error(marketplace_root: &Path) -> String { + format!( + "Error: failed to load configured marketplace snapshot(s):\n\n- `nemo-relay-local` at {}: marketplace root does not contain a supported manifest\n", + marketplace_root.display() + ) +} + fn force_snapshot_with_backups( backup_marketplace_root: PathBuf, backup_plugin_root: Option, @@ -541,9 +548,26 @@ struct MockRunner { failing_suffix: Option, failing_suffixes: Vec, failing_quiet_suffix: Option, + capture_reappearing_root: Option, } impl MockRunner { + fn set_dangling_marketplace_source(&mut self, marketplace_root: &Path) { + let error = dangling_codex_marketplace_error(marketplace_root); + for output in self.capture_outputs.values_mut() { + if output.stderr == DANGLING_CODEX_MARKETPLACE_ERROR { + output.stderr.clone_from(&error); + } + } + for outputs in self.capture_output_sequences.get_mut().values_mut() { + for output in outputs { + if output.stderr == DANGLING_CODEX_MARKETPLACE_ERROR { + output.stderr.clone_from(&error); + } + } + } + } + fn with_current_executable(mut self, path: &str) -> Self { self.current_executable = Some(PathBuf::from(path)); self @@ -733,6 +757,12 @@ impl CommandRunner for MockRunner { .join(" ") ); self.capture_commands.borrow_mut().push(rendered.clone()); + if rendered.ends_with("codex plugin list") + && let Some(path) = self.capture_reappearing_root.as_ref() + { + std::fs::create_dir(path) + .map_err(|error| format!("failed to inject reappearing root: {error}"))?; + } if let Some(output) = self .capture_output_sequences .borrow_mut() @@ -771,6 +801,7 @@ struct MockSetupRunner { doctor_roots: RefCell>, failing_call: Option, snapshot_reappearing_root: Option, + snapshot_replaced_lock: Option, } struct BlockingRefreshFailure { @@ -905,6 +936,12 @@ impl PluginSetupRunner for MockSetupRunner { std::fs::create_dir(path) .map_err(|error| format!("failed to inject reappearing root: {error}"))?; } + if let Some(path) = self.snapshot_replaced_lock.as_ref() { + std::fs::remove_file(path) + .map_err(|error| format!("failed to remove generation lock: {error}"))?; + std::fs::write(path, format!("{}\n", uuid::Uuid::now_v7())) + .map_err(|error| format!("failed to replace generation lock: {error}"))?; + } Ok(Some(PluginSetupSnapshot::Mock)) } @@ -1988,6 +2025,7 @@ fn host_command_helpers_cover_dry_run_missing_failure_and_reporting() { HostRegistrationReport { host_plugin_registered: Some(false), host_marketplace_registered: true, + host_marketplace_source: None, host_marketplace_unloadable: false, } .to_json()["host_plugin_registered"], @@ -2220,6 +2258,10 @@ fn codex_registration_report_identifies_a_dangling_relay_marketplace() { assert_eq!(report.host_plugin_registered, None); assert!(report.host_marketplace_registered); assert!(report.host_marketplace_unloadable); + assert_eq!( + report.host_marketplace_source, + Some(PathBuf::from("/tmp/plugins/codex-marketplace")) + ); assert!(!report.ok()); assert_eq!(runner.capture_commands(), vec!["/bin/codex plugin list"]); } @@ -3161,7 +3203,7 @@ fn force_install_rejects_registered_legacy_plugin_without_generation_fence() { #[test] fn force_install_rejects_a_dangling_codex_marketplace_without_a_surviving_lock() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3176,6 +3218,7 @@ fn force_install_rejects_a_dangling_codex_marketplace_without_a_surviving_lock() ..options(dir.path()) }; let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + runner.set_dangling_marketplace_source(&layout.marketplace_root); let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); @@ -3190,7 +3233,7 @@ fn force_install_rejects_a_dangling_codex_marketplace_without_a_surviving_lock() #[test] fn force_install_recovers_a_dangling_codex_marketplace_with_its_surviving_lock() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3207,6 +3250,7 @@ fn force_install_recovers_a_dangling_codex_marketplace_with_its_surviving_lock() write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); assert!(layout.generation_lock.exists()); let original_lock = std::fs::read_to_string(&layout.generation_lock).unwrap(); @@ -3247,7 +3291,7 @@ fn force_install_recovers_a_dangling_codex_marketplace_with_its_surviving_lock() #[test] fn plain_install_recommends_force_for_a_dangling_codex_marketplace() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3260,6 +3304,7 @@ fn plain_install_recommends_force_for_a_dangling_codex_marketplace() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); let error = install_host( CodingAgent::Codex, @@ -3305,6 +3350,7 @@ fn force_install_recovers_when_the_unknown_plugin_was_already_unregistered() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap(); @@ -3327,7 +3373,7 @@ fn force_install_recovers_when_the_unknown_plugin_was_already_unregistered() { #[test] fn failed_dangling_force_install_leaves_a_fenced_retry_that_can_succeed() { let dir = tempdir().unwrap(); - let first_runner = MockRunner::default() + let mut first_runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3347,6 +3393,7 @@ fn failed_dangling_force_install_leaves_a_fenced_retry_that_can_succeed() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + first_runner.set_dangling_marketplace_source(&layout.marketplace_root); let lock_id = std::fs::read_to_string(&layout.generation_lock).unwrap(); let error = install_host(CodingAgent::Codex, &force, &first_runner, &first_setup).unwrap_err(); @@ -3407,6 +3454,7 @@ fn precommit_dangling_cleanup_failure_never_invents_plugin_registration_and_can_ write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + first_runner.set_dangling_marketplace_source(&layout.marketplace_root); let error = install_host( CodingAgent::Codex, @@ -3426,7 +3474,7 @@ fn precommit_dangling_cleanup_failure_never_invents_plugin_registration_and_can_ assert!(!layout.marketplace_root.exists()); assert!(layout.generation_lock.exists()); - let retry_runner = MockRunner::default() + let mut retry_runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3435,6 +3483,7 @@ fn precommit_dangling_cleanup_failure_never_invents_plugin_registration_and_can_ "", DANGLING_CODEX_MARKETPLACE_ERROR, ); + retry_runner.set_dangling_marketplace_source(&layout.marketplace_root); install_host( CodingAgent::Codex, &force, @@ -3446,7 +3495,62 @@ fn precommit_dangling_cleanup_failure_never_invents_plugin_registration_and_can_ } #[test] -fn precommit_dangling_probe_failure_restores_known_removed_marketplace_and_can_retry() { +fn dangling_recovery_rejects_a_registration_from_another_install_directory() { + let dir = tempdir().unwrap(); + let selected_dir = dir.path().join("selected"); + let registered_dir = dir.path().join("registered"); + write_installed_state(CodingAgent::Codex, &selected_dir); + write_installed_state(CodingAgent::Codex, ®istered_dir); + let selected = PluginLayout::new(CodingAgent::Codex, &selected_dir); + let registered = PluginLayout::new(CodingAgent::Codex, ®istered_dir); + std::fs::remove_dir_all(&selected.marketplace_root).unwrap(); + std::fs::remove_dir_all(®istered.marketplace_root).unwrap(); + let lock_contents = std::fs::read(®istered.generation_lock).unwrap(); + let held_registered_lock = GenerationRetirement::acquire_missing_for_plugin( + ®istered.generation_fence, + ®istered.generation_lock, + ) + .unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + dangling_codex_marketplace_error(®istered.marketplace_root), + ); + let mut force = options(&selected_dir); + force.force = true; + + let install_error = install_host( + CodingAgent::Codex, + &force, + &runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + assert!(install_error.contains("different marketplace source")); + + let uninstall_error = uninstall_host( + CodingAgent::Codex, + &force, + &runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + assert!(uninstall_error.contains("different marketplace source")); + assert_eq!( + std::fs::read(®istered.generation_lock).unwrap(), + lock_contents + ); + assert!(selected.generation_lock.exists()); + assert!(runner.commands().is_empty()); + drop(held_registered_lock); +} + +#[test] +fn precommit_dangling_probe_failure_keeps_marketplace_removal_progress_for_retry() { let dir = tempdir().unwrap(); let mut first_runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") @@ -3465,11 +3569,6 @@ fn precommit_dangling_probe_failure_restores_known_removed_marketplace_and_can_r stdout: String::new(), stderr: "post-removal probe failed".into(), }, - CommandOutput { - status: 2, - stdout: String::new(), - stderr: "rollback probe failed".into(), - }, ] .into(), ); @@ -3480,6 +3579,7 @@ fn precommit_dangling_probe_failure_restores_known_removed_marketplace_and_can_r write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + first_runner.set_dangling_marketplace_source(&layout.marketplace_root); let error = install_host( CodingAgent::Codex, @@ -3490,31 +3590,24 @@ fn precommit_dangling_probe_failure_restores_known_removed_marketplace_and_can_r .unwrap_err(); assert!(error.contains("post-removal probe failed"), "{error}"); - assert!(error.contains("rollback probe failed"), "{error}"); - assert!(first_runner.commands().iter().any(|command| { - command.ends_with(&format!( - "plugin marketplace add {}", - layout.marketplace_root.display() - )) + assert!(first_runner.commands().iter().all(|command| { + !command.ends_with("plugin add nemo-relay-plugin@nemo-relay-local") + && !command.ends_with(&format!( + "plugin marketplace add {}", + layout.marketplace_root.display() + )) })); - assert!( - first_runner - .commands() - .iter() - .all(|command| !command.ends_with("plugin add nemo-relay-plugin@nemo-relay-local")) - ); assert!(!layout.marketplace_root.exists()); assert!(layout.generation_lock.exists()); + let retry_state = read_state(CodingAgent::Codex, dir.path()).unwrap(); + assert!(!retry_state.host_plugin_removed); + assert!(retry_state.host_marketplace_removed); + assert!(retry_state.marker_absent_recovery); let retry_runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") - .with_capture_status( - "/bin/codex plugin list", - 1, - "", - DANGLING_CODEX_MARKETPLACE_ERROR, - ); + .with_codex_registration(true, false); install_host( CodingAgent::Codex, &force, @@ -3523,6 +3616,11 @@ fn precommit_dangling_probe_failure_restores_known_removed_marketplace_and_can_r ) .unwrap(); assert!(layout.generation_fence.exists()); + assert!( + retry_runner.commands().iter().any(|command| { + command.ends_with("plugin remove nemo-relay-plugin@nemo-relay-local") + }) + ); } #[test] @@ -3599,7 +3697,7 @@ fn force_install_rejects_a_symlinked_dangling_marketplace_root_without_mutation( use std::os::unix::fs::symlink; let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3615,6 +3713,7 @@ fn force_install_rejects_a_symlinked_dangling_marketplace_root_without_mutation( write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); let target = dir.path().join("unexpected-marketplace"); std::fs::create_dir(&target).unwrap(); symlink(&target, &layout.marketplace_root).unwrap(); @@ -3627,7 +3726,7 @@ fn force_install_rejects_a_symlinked_dangling_marketplace_root_without_mutation( ) .unwrap_err(); - assert_actionable_generation_error(&error, "MCP generation marker is missing"); + assert_actionable_generation_error(&error, "unloadable Codex marketplace"); assert!(layout.marketplace_root.is_symlink()); assert!(runner.commands().is_empty()); } @@ -3635,7 +3734,7 @@ fn force_install_rejects_a_symlinked_dangling_marketplace_root_without_mutation( #[test] fn force_install_rechecks_a_dangling_root_after_setup_snapshot() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3651,6 +3750,7 @@ fn force_install_rechecks_a_dangling_root_after_setup_snapshot() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); let setup_runner = MockSetupRunner { snapshot_reappearing_root: Some(layout.marketplace_root.clone()), ..MockSetupRunner::default() @@ -3665,10 +3765,44 @@ fn force_install_rechecks_a_dangling_root_after_setup_snapshot() { assert_no_install_stage(dir.path()); } +#[cfg(unix)] +#[test] +fn force_install_rechecks_the_dangling_lock_after_setup_snapshot() { + let dir = tempdir().unwrap(); + let mut runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + let force = PluginInstallOptions { + force: true, + ..options(dir.path()) + }; + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); + let setup_runner = MockSetupRunner { + snapshot_replaced_lock: Some(layout.generation_lock.clone()), + ..MockSetupRunner::default() + }; + + let error = install_host(CodingAgent::Codex, &force, &runner, &setup_runner).unwrap_err(); + + assert!(error.contains("changed identity"), "{error}"); + assert!(runner.commands().is_empty()); + assert_eq!(setup_runner.calls(), vec!["snapshot codex"]); + assert_no_install_stage(dir.path()); +} + #[test] fn force_install_preserves_a_generation_when_plugin_registration_is_unknown() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -3685,6 +3819,7 @@ fn force_install_preserves_a_generation_when_plugin_registration_is_unknown() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_file(&layout.marketplace_manifest).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); let generation_token = InstallGeneration::capture(layout.generation_fence.clone()) .unwrap() .token() @@ -3692,10 +3827,7 @@ fn force_install_preserves_a_generation_when_plugin_registration_is_unknown() { let error = install_host(CodingAgent::Codex, &options, &runner, &setup_runner).unwrap_err(); - assert!( - error.contains("registration state could not be determined"), - "{error}" - ); + assert_actionable_generation_error(&error, "unloadable Codex marketplace"); assert!(layout.marketplace_root.exists()); assert!(layout.state_path.exists()); assert_eq!( @@ -5409,7 +5541,7 @@ fn uninstall_rejects_registered_legacy_plugin_without_generation_fence() { #[test] fn plain_uninstall_recommends_force_for_a_dangling_codex_marketplace() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("nemo-relay", "/bin/nemo-relay") .with_executable("codex", "/bin/codex") .with_capture_status( @@ -5423,6 +5555,7 @@ fn plain_uninstall_recommends_force_for_a_dangling_codex_marketplace() { let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); let original_state = std::fs::read(&layout.state_path).unwrap(); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); assert!(layout.generation_lock.exists()); let error = uninstall_host( @@ -5447,7 +5580,7 @@ fn plain_uninstall_recommends_force_for_a_dangling_codex_marketplace() { #[test] fn force_uninstall_recovers_a_dangling_codex_marketplace() { let dir = tempdir().unwrap(); - let runner = MockRunner::default() + let mut runner = MockRunner::default() .with_executable("codex", "/bin/codex") .with_capture_status( "/bin/codex plugin list", @@ -5459,6 +5592,7 @@ fn force_uninstall_recovers_a_dangling_codex_marketplace() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); let mut force = options(dir.path()); force.force = true; @@ -5476,6 +5610,210 @@ fn force_uninstall_recovers_a_dangling_codex_marketplace() { ); } +#[cfg(unix)] +#[test] +fn force_uninstall_rejects_a_symlinked_dangling_marketplace_root() { + use std::os::unix::fs::symlink; + + let dir = tempdir().unwrap(); + let mut runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); + let unexpected_root = dir.path().join("unexpected-marketplace"); + std::fs::create_dir(&unexpected_root).unwrap(); + symlink(&unexpected_root, &layout.marketplace_root).unwrap(); + let lock_contents = std::fs::read(&layout.generation_lock).unwrap(); + let mut force = options(dir.path()); + force.force = true; + + let error = uninstall_host( + CodingAgent::Codex, + &force, + &runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + + assert!(error.contains("unloadable Codex marketplace"), "{error}"); + assert!(layout.marketplace_root.is_symlink()); + assert_eq!( + std::fs::read(&layout.generation_lock).unwrap(), + lock_contents + ); + assert!(runner.commands().is_empty()); +} + +#[test] +fn force_uninstall_rejects_a_dangling_root_that_reappears_during_classification() { + let dir = tempdir().unwrap(); + let mut runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_capture_status( + "/bin/codex plugin list", + 1, + "", + DANGLING_CODEX_MARKETPLACE_ERROR, + ); + write_installed_state(CodingAgent::Codex, dir.path()); + let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); + std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); + runner.capture_reappearing_root = Some(layout.marketplace_root.clone()); + let mut force = options(dir.path()); + force.force = true; + + let error = uninstall_host( + CodingAgent::Codex, + &force, + &runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + + assert!(error.contains("unloadable Codex marketplace"), "{error}"); + assert!(layout.marketplace_root.is_dir()); + assert!(runner.commands().is_empty()); +} + +#[test] +fn marker_absent_retry_does_not_remove_a_newer_codex_registration() { + let dir = tempdir().unwrap(); + let previous_dir = dir.path().join("previous"); + let replacement_dir = dir.path().join("replacement"); + write_installed_state(CodingAgent::Codex, &previous_dir); + write_installed_state(CodingAgent::Codex, &replacement_dir); + let previous_layout = PluginLayout::new(CodingAgent::Codex, &previous_dir); + let replacement_layout = PluginLayout::new(CodingAgent::Codex, &replacement_dir); + std::fs::remove_dir_all(&previous_layout.marketplace_root).unwrap(); + let mut retry_state = read_state(CodingAgent::Codex, &previous_dir).unwrap(); + retry_state.marker_absent_recovery = true; + retry_state.host_plugin_removed = true; + retry_state.host_marketplace_removed = true; + retry_state.plugin_setup_installed = false; + write_state_for_host( + CodingAgent::Codex, + &retry_state, + &previous_dir, + &options(&previous_dir), + ) + .unwrap(); + let runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(true, true); + let mut force = options(&previous_dir); + force.force = true; + + let install_error = install_host( + CodingAgent::Codex, + &force, + &runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + assert!(install_error.contains("conflicts with the current Codex registration")); + + let uninstall_error = uninstall_host( + CodingAgent::Codex, + &force, + &runner, + &MockSetupRunner::default(), + ) + .unwrap_err(); + assert!(uninstall_error.contains("conflicts with the current Codex registration")); + assert!(replacement_layout.marketplace_root.exists()); + assert!(replacement_layout.generation_fence.exists()); + assert!(runner.commands().is_empty()); +} + +#[test] +fn clean_marker_absent_retry_resumes_for_forced_install_and_uninstall() { + let dir = tempdir().unwrap(); + let install_dir = dir.path().join("install"); + write_installed_state(CodingAgent::Codex, &install_dir); + let install_layout = PluginLayout::new(CodingAgent::Codex, &install_dir); + std::fs::remove_dir_all(&install_layout.marketplace_root).unwrap(); + let retry_state = PluginState { + marketplace_root: install_layout.marketplace_root.clone(), + plugin_root: install_layout.plugin_root.clone(), + host_plugin_removed: false, + host_marketplace_removed: false, + plugin_setup_installed: false, + marker_absent_recovery: true, + }; + write_state_for_host( + CodingAgent::Codex, + &retry_state, + &install_dir, + &options(&install_dir), + ) + .unwrap(); + let install_runner = MockRunner::default() + .with_executable("nemo-relay", "/bin/nemo-relay") + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + let mut force_install = options(&install_dir); + force_install.force = true; + + install_host( + CodingAgent::Codex, + &force_install, + &install_runner, + &MockSetupRunner::default(), + ) + .unwrap(); + assert!(install_layout.generation_fence.exists()); + assert!(install_runner.commands().iter().all(|command| { + !command.ends_with("plugin remove nemo-relay-plugin@nemo-relay-local") + && !command.ends_with("plugin marketplace remove nemo-relay-local") + })); + + let uninstall_dir = dir.path().join("uninstall"); + write_installed_state(CodingAgent::Codex, &uninstall_dir); + let uninstall_layout = PluginLayout::new(CodingAgent::Codex, &uninstall_dir); + std::fs::remove_dir_all(&uninstall_layout.marketplace_root).unwrap(); + let retry_state = PluginState { + marketplace_root: uninstall_layout.marketplace_root.clone(), + plugin_root: uninstall_layout.plugin_root.clone(), + host_plugin_removed: false, + host_marketplace_removed: false, + plugin_setup_installed: false, + marker_absent_recovery: true, + }; + write_state_for_host( + CodingAgent::Codex, + &retry_state, + &uninstall_dir, + &options(&uninstall_dir), + ) + .unwrap(); + let uninstall_runner = MockRunner::default() + .with_executable("codex", "/bin/codex") + .with_codex_registration(false, false); + let mut force_uninstall = options(&uninstall_dir); + force_uninstall.force = true; + + uninstall_host( + CodingAgent::Codex, + &force_uninstall, + &uninstall_runner, + &MockSetupRunner::default(), + ) + .unwrap(); + assert!(!uninstall_layout.state_path.exists()); + assert!(!uninstall_layout.generation_lock.exists()); + assert!(uninstall_runner.commands().is_empty()); +} + #[test] fn force_uninstall_reconciles_a_marketplace_removal_error_after_success() { let dir = tempdir().unwrap(); @@ -5501,6 +5839,7 @@ fn force_uninstall_reconciles_a_marketplace_removal_error_after_success() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + runner.set_dangling_marketplace_source(&layout.marketplace_root); let mut force = options(dir.path()); force.force = true; @@ -5540,7 +5879,7 @@ fn force_uninstall_reconciles_a_marketplace_removal_error_after_success() { #[test] fn partial_dangling_force_uninstall_retains_a_guarded_retry() { let dir = tempdir().unwrap(); - let first_runner = MockRunner::default() + let mut first_runner = MockRunner::default() .with_executable("codex", "/bin/codex") .with_capture_status( "/bin/codex plugin list", @@ -5555,6 +5894,7 @@ fn partial_dangling_force_uninstall_retains_a_guarded_retry() { write_installed_state(CodingAgent::Codex, dir.path()); let layout = PluginLayout::new(CodingAgent::Codex, dir.path()); std::fs::remove_dir_all(&layout.marketplace_root).unwrap(); + first_runner.set_dangling_marketplace_source(&layout.marketplace_root); let lock_id = std::fs::read_to_string(&layout.generation_lock).unwrap(); let mut force = options(dir.path()); force.force = true; diff --git a/crates/cli/tests/coverage/shared/install_generation_tests.rs b/crates/cli/tests/coverage/shared/install_generation_tests.rs index 08da25e1f..e1ef4a6ed 100644 --- a/crates/cli/tests/coverage/shared/install_generation_tests.rs +++ b/crates/cli/tests/coverage/shared/install_generation_tests.rs @@ -210,10 +210,14 @@ fn marker_absent_retirement_rejects_a_replaced_lock_inode_even_with_the_same_uui let retirement = GenerationRetirement::acquire_missing_for_plugin(&marker, &lock).unwrap(); std::fs::remove_file(&lock).unwrap(); - std::fs::write(&lock, contents).unwrap(); + std::fs::write(&lock, &contents).unwrap(); let error = retirement.revalidate_missing_marker().unwrap_err(); assert!(error.contains("changed identity"), "{error}"); + assert!(!marker.exists()); + drop(retirement); + assert!(!marker.exists()); + assert_eq!(std::fs::read(&lock).unwrap(), contents); } #[test] diff --git a/docs/nemo-relay-cli/plugin-installation.mdx b/docs/nemo-relay-cli/plugin-installation.mdx index 82bc1046b..310eaf6a4 100644 --- a/docs/nemo-relay-cli/plugin-installation.mdx +++ b/docs/nemo-relay-cli/plugin-installation.mdx @@ -134,7 +134,9 @@ replacing the lock. During a forced reinstall, removal of the old provider and hooks, plugin registration, and marketplace registration is one cleanup phase. Until all three areas have been cleaned, a failure restores only prior state Relay can -prove existed. After all three are clean, cleanup is committed: a later install +prove existed. Marketplace removal is durable progress because the generated +tree is already gone, so a later probe failure records that removal and retries +forward. After all three are clean, cleanup is committed: a later install failure does not recreate the dangling Codex registration. Relay keeps the validated lock and records the clean progress so another `nemo-relay install codex --force` or `nemo-relay integrations refresh` can @@ -242,10 +244,12 @@ nemo-relay install codex --install-dir --force nemo-relay integrations refresh ``` -Refresh attempts each managed target through the same forced-install recovery -path. If one Codex target has an unsafe surviving lock, Relay reports that -target, continues refreshing the others, and returns an error after every -target has been attempted. +Refresh preflight validates installed targets and retires their active MCP +generations. A preflight failure stops the refresh before any target is +attempted. A deleted Codex marketplace has no generation to retire, so its +recovery remains in the per-target forced-install loop. If its surviving lock is +unsafe, Relay reports that target, continues refreshing the others, and returns +an error after every target has been attempted. Manual MCP configurations are not changed; reinstall them through Relay if you want Relay to manage future refreshes. @@ -278,10 +282,12 @@ persistent hook commands reference a private Relay-owned configuration file; that file contains the generation-file path and immutable generation identity. Relay uses the configured hook failure policy. This prevents a legacy hook retained by a host process from reviving a retired installation. If install or -uninstall reports a missing or invalid generation marker, follow its cleanup -instructions. If the generated Codex marketplace was deleted but its -registration remains, use the forced recovery described above. When forced -recovery cannot validate the surviving lock, follow the error's manual cleanup +uninstall reports a missing or invalid generation marker, follow the error's +cleanup instructions. If the generated Codex marketplace was deleted but its +registration remains, see +[Recover a Deleted Codex Marketplace](#recover-a-deleted-codex-marketplace). +For other missing or invalid generation markers, or when forced recovery +cannot validate the surviving lock, follow the error's manual cleanup instructions: close the host and standalone `nemo-relay mcp` processes, remove the stale registration and state it identifies, and then run the requested `--force` command. @@ -436,15 +442,16 @@ Code and Codex `run` flows. Existing fenced installations can be refreshed with `nemo-relay install --force`. Relay refuses to replace an older MCP installation without a valid generation marker because a cached host process might still be running. The exact deleted-Codex-marketplace state can be -recovered with the forced commands described above when its surviving lock is +recovered through [Recover a Deleted Codex +Marketplace](#recover-a-deleted-codex-marketplace) when its surviving lock is safe. For other missing-marker states, follow the manual cleanup steps in the error before you retry the forced install. This release removes the internal `nemo-relay plugin-shim` command. Refresh a -fenced generated installation with `nemo-relay install --force`; use the -deleted-marketplace recovery described above for that exact Codex state, and -use manual cleanup for other missing-marker states. For custom automation, use -these supported replacements: +fenced generated installation with `nemo-relay install --force`; use +[Recover a Deleted Codex Marketplace](#recover-a-deleted-codex-marketplace) for +that exact Codex state, and use manual cleanup for other missing-marker states. +For custom automation, use these supported replacements: | Removed Internal Command | Supported Replacement | | --- | --- | From 62f2033451fd3e58b99b1fd11b40d24a86241bad Mon Sep 17 00:00:00 2001 From: Sara Tadayon Date: Wed, 9 Sep 2026 09:11:36 -0600 Subject: [PATCH 3/3] test(cli): make generation-lock contention checks portable Signed-off-by: Sara Tadayon --- crates/cli/tests/coverage/agents/plugin_install_tests.rs | 6 +++--- .../cli/tests/coverage/shared/install_generation_tests.rs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/cli/tests/coverage/agents/plugin_install_tests.rs b/crates/cli/tests/coverage/agents/plugin_install_tests.rs index ef725b780..4405c9f63 100644 --- a/crates/cli/tests/coverage/agents/plugin_install_tests.rs +++ b/crates/cli/tests/coverage/agents/plugin_install_tests.rs @@ -3540,13 +3540,13 @@ fn dangling_recovery_rejects_a_registration_from_another_install_directory() { ) .unwrap_err(); assert!(uninstall_error.contains("different marketplace source")); + assert!(selected.generation_lock.exists()); + assert!(runner.commands().is_empty()); + drop(held_registered_lock); assert_eq!( std::fs::read(®istered.generation_lock).unwrap(), lock_contents ); - assert!(selected.generation_lock.exists()); - assert!(runner.commands().is_empty()); - drop(held_registered_lock); } #[test] diff --git a/crates/cli/tests/coverage/shared/install_generation_tests.rs b/crates/cli/tests/coverage/shared/install_generation_tests.rs index e1ef4a6ed..ab53eb305 100644 --- a/crates/cli/tests/coverage/shared/install_generation_tests.rs +++ b/crates/cli/tests/coverage/shared/install_generation_tests.rs @@ -152,8 +152,8 @@ fn marker_absent_retirement_rejects_contention_without_changing_the_lock() { )); assert!(error.contains("timed out waiting"), "{error}"); - assert_eq!(std::fs::read(&lock).unwrap(), contents); drop(first); + assert_eq!(std::fs::read(&lock).unwrap(), contents); } #[test]