diff --git a/crates/edgezero-adapter-fastly/src/cli.rs b/crates/edgezero-adapter-fastly/src/cli.rs index 2c0d9bc1..210973fe 100644 --- a/crates/edgezero-adapter-fastly/src/cli.rs +++ b/crates/edgezero-adapter-fastly/src/cli.rs @@ -353,6 +353,11 @@ struct TempFileGuard { path: Option, } +struct RuntimeStoreNameReconciliation { + deletes: Vec, + upserts: Vec<(String, String)>, +} + // The three `validate_*` trait methods exist on `Adapter` because // spin requires them (variable-name regex, `[component.*]` // discovery, flat-namespace collision). The trait surface is typed @@ -468,6 +473,7 @@ impl Adapter for FastlyCliAdapter { ); }; let fastly_path = manifest_root.join(rel); + let manifest_dir = fastly_path.parent().unwrap_or(manifest_root); let mut out = Vec::new(); for (kind, ids) in [ @@ -498,7 +504,7 @@ impl Adapter for FastlyCliAdapter { )); continue; } - create_fastly_store(kind, name)?; + create_fastly_store_in(kind, name, manifest_dir)?; // If the platform store was created but the // writeback fails, remote state and the local // manifest are out of sync. Re-running `provision` @@ -560,7 +566,7 @@ impl Adapter for FastlyCliAdapter { fastly_path.display() )); } else if !setup_block_present(&fastly_path, runtime_env_kind, runtime_env_name)? { - create_fastly_store(runtime_env_kind, runtime_env_name)?; + create_fastly_store_in(runtime_env_kind, runtime_env_name, manifest_dir)?; append_fastly_setup(&fastly_path, runtime_env_kind, runtime_env_name).map_err( |err| { format!( @@ -583,7 +589,7 @@ impl Adapter for FastlyCliAdapter { // selector via `edgezero_runtime_env_staging`, wired automatically by // a staged deploy; nothing here should be edited to stage config. let mut line = format!( - "created fastly {runtime_env_kind}-store `{runtime_env_name}` (EdgeZero runtime override store, read by the ACTIVE version); appended setup tables to {}\n It already selects each store's default key, so no edit is needed for a normal setup.\n To point PRODUCTION at a different key (e.g. a renamed store), and only then:\n fastly config-store-entry update --store-id= --key=EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY --value= --upsert\n Do NOT set a `_staging` key here: staged config is isolated by a per-service `{RUNTIME_ENV_STAGING_STORE_PREFIX}_` store, which a staged deploy creates and links automatically.", + "created fastly {runtime_env_kind}-store `{runtime_env_name}` (EdgeZero runtime override store, read by the ACTIVE version); appended setup tables to {}\n Provision writes non-default store-name mappings below. Config stores still select their logical id as the default key.\n To point PRODUCTION at a different config key, and only then:\n fastly config-store-entry update --store-id= --key=EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY --value= --upsert\n Do NOT set a `_staging` key here: staged config is isolated by a per-service `{RUNTIME_ENV_STAGING_STORE_PREFIX}_` store, which a staged deploy creates and links automatically.", fastly_path.display() ); if let Some(note) = post_create_note { @@ -595,6 +601,12 @@ impl Adapter for FastlyCliAdapter { // Already declared; nothing to do. } + out.extend(persist_runtime_env_store_name_entries( + stores, + dry_run, + manifest_dir, + )?); + // The STAGING twin of the runtime-override store is created and // populated entirely by a staged deploy (see // `relink_runtime_env_for_staging` → `mirror_production_to_staging`), so @@ -1465,19 +1477,20 @@ fn classify_resolved_read( /// # Errors /// Returns an error if `fastly` isn't on `PATH`, the child fails to /// spawn, or the exit status is non-zero. -fn create_fastly_store(kind: &str, name: &str) -> Result<(), String> { +fn create_fastly_store_in(kind: &str, name: &str, cwd: &Path) -> Result<(), String> { let subcommand = format!("{kind}-store"); let name_arg = format!("--name={name}"); - let output = Command::new("fastly") + let mut command = Command::new("fastly"); + command .args([subcommand.as_str(), "create", name_arg.as_str()]) - .output() - .map_err(|err| { - if err.kind() == ErrorKind::NotFound { - format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") - } else { - format!("failed to spawn `fastly`: {err}") - } - })?; + .current_dir(cwd); + let output = command.output().map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; if output.status.success() { return Ok(()); } @@ -3243,17 +3256,39 @@ where /// bytes out of argv and lifts the size cap to whatever the OS /// pipe buffer + the CLI's read accept (megabytes in practice). fn create_config_store_entry(store_id: &str, key: &str, value: &str) -> Result<(), String> { + create_config_store_entry_with_cwd(store_id, key, value, None) +} + +fn create_config_store_entry_in( + store_id: &str, + key: &str, + value: &str, + cwd: &Path, +) -> Result<(), String> { + create_config_store_entry_with_cwd(store_id, key, value, Some(cwd)) +} + +fn create_config_store_entry_with_cwd( + store_id: &str, + key: &str, + value: &str, + cwd: Option<&Path>, +) -> Result<(), String> { let store_arg = format!("--store-id={store_id}"); let key_arg = format!("--key={key}"); - let mut child = Command::new("fastly") - .args([ - "config-store-entry", - "update", - store_arg.as_str(), - key_arg.as_str(), - "--upsert", - "--stdin", - ]) + let mut command = Command::new("fastly"); + command.args([ + "config-store-entry", + "update", + store_arg.as_str(), + key_arg.as_str(), + "--upsert", + "--stdin", + ]); + if let Some(command_cwd) = cwd { + command.current_dir(command_cwd); + } + let mut child = command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()) @@ -3412,9 +3447,9 @@ fn parse_config_store_entries(stdout: &str) -> Result, Str /// `fastly config-store-entry delete --store-id= --key=`, run in the /// app manifest directory. Distinct from the `config gc` `delete_config_store_entry` -/// (which runs in the process cwd with redacted diagnostics); staging reconciliation -/// must run `fastly` in `cwd` so it resolves the right service context. -fn delete_staging_config_store_entry(store_id: &str, key: &str, cwd: &Path) -> Result<(), String> { +/// (which runs in the process cwd with redacted diagnostics); runtime-env +/// reconciliation must run `fastly` in `cwd` so it resolves the right service context. +fn delete_config_store_entry_in(store_id: &str, key: &str, cwd: &Path) -> Result<(), String> { run_fastly_status( &[ "config-store-entry".to_owned(), @@ -3460,6 +3495,62 @@ fn staging_entries_from_production( out } +fn is_runtime_store_name_key(key: &str) -> bool { + let mut segments = key.split("__"); + matches!( + ( + segments.next(), + segments.next(), + segments.next(), + segments.next(), + segments.next(), + segments.next(), + ), + ( + Some("EDGEZERO"), + Some("STORES"), + Some("CONFIG" | "KV" | "SECRETS"), + Some(id), + Some("NAME"), + None, + ) if !id.is_empty() + ) +} + +fn runtime_store_name_entries_from_vars( + vars: impl IntoIterator, +) -> Result, String> { + let mut entries = Vec::new(); + for (key, value) in vars { + if !is_runtime_store_name_key(&key) { + continue; + } + if value.is_empty() || value.trim() != value || value.chars().any(char::is_control) { + return Err(format!( + "runtime store-name override `{key}` must be non-empty and contain no surrounding whitespace or control characters" + )); + } + entries.push((key, value)); + } + entries.sort_by(|left, right| left.0.cmp(&right.0)); + Ok(entries) +} + +fn overlay_runtime_store_name_entries( + base: &[(String, String)], + overrides: &[(String, String)], +) -> Vec<(String, String)> { + let mut entries = base.to_vec(); + for (key, value) in overrides { + if let Some((_, current)) = entries.iter_mut().find(|(candidate, _)| candidate == key) { + current.clone_from(value); + } else { + entries.push((key.clone(), value.clone())); + } + } + entries +} + /// Resolve the staging twin store, creating it on demand. A staged deploy owns /// this store end to end (it is never linked on the ACTIVE version), so it does /// not depend on `provision` having created it first. Fails closed on a lookup @@ -3471,15 +3562,15 @@ fn staging_selector_store_name(service_id: &str) -> String { format!("{RUNTIME_ENV_STAGING_STORE_PREFIX}_{service_id}") } -fn ensure_staging_selector_store(store_name: &str) -> Result { - match classify_remote_config_store(store_name)? { +fn ensure_staging_selector_store(store_name: &str, cwd: &Path) -> Result { + match classify_remote_config_store_in(store_name, cwd)? { ConfigStoreLookup::Found(id) => Ok(id), ConfigStoreLookup::NotFound => { - create_fastly_store("config", store_name)?; + create_fastly_store_in("config", store_name, cwd)?; // resolve_remote_config_store_id now yields a typed absence; we just // created the store, so a None here is fail-closed (the listing did // not reflect our own create), not a genuine absence. - resolve_remote_config_store_id(store_name) + resolve_remote_config_store_id_in(store_name, cwd) .map_err(|err| { format!( "created fastly config-store `{store_name}` but could not resolve its id: {err}" @@ -3525,20 +3616,154 @@ fn mirror_production_to_staging( config_logical_ids: &[String], cwd: &Path, ) -> Result<(), String> { - let desired = staging_entries_from_production(production, config_logical_ids); + let process_overrides = runtime_store_name_entries_from_vars(env::vars())?; + let effective_production = overlay_runtime_store_name_entries(production, &process_overrides); + let desired = staging_entries_from_production(&effective_production, config_logical_ids); for (key, value) in &desired { - create_config_store_entry(staging_id, key, value)?; + create_config_store_entry_in(staging_id, key, value, cwd)?; } let current = read_config_store_entries(staging_id, cwd)?; for (key, _) in ¤t { if !desired.iter().any(|(dk, _)| dk == key) { - delete_staging_config_store_entry(staging_id, key, cwd)?; + delete_config_store_entry_in(staging_id, key, cwd)?; } } Ok(()) } +fn runtime_store_name_key(kind: &str, logical: &str) -> String { + format!( + "EDGEZERO__STORES__{kind}__{}__NAME", + logical.to_ascii_uppercase() + ) +} + +/// Return the runtime entries required when logical store ids map to different +/// Fastly resource names. +fn runtime_env_store_name_entries(stores: &ProvisionStores<'_>) -> Vec<(String, String)> { + let mut entries = Vec::new(); + for (kind, ids) in [ + ("CONFIG", stores.config), + ("KV", stores.kv), + ("SECRETS", stores.secrets), + ] { + for store in ids { + if store.logical == store.platform { + continue; + } + entries.push(( + runtime_store_name_key(kind, &store.logical), + store.platform.clone(), + )); + } + } + entries +} + +fn runtime_env_store_name_keys(stores: &ProvisionStores<'_>) -> Vec { + let mut keys = Vec::new(); + for (kind, ids) in [ + ("CONFIG", stores.config), + ("KV", stores.kv), + ("SECRETS", stores.secrets), + ] { + keys.extend( + ids.iter() + .map(|store| runtime_store_name_key(kind, &store.logical)), + ); + } + keys +} + +/// Compute the minimal changes needed for store-name mappings owned by the +/// logical ids this app currently declares. Entries for undeclared ids and +/// unrelated runtime settings are preserved because the production runtime-env +/// store can be linked by more than one service in the same Fastly account. +fn runtime_store_name_reconciliation( + stores: &ProvisionStores<'_>, + current: &[(String, String)], +) -> RuntimeStoreNameReconciliation { + let desired = runtime_env_store_name_entries(stores); + let declared = runtime_env_store_name_keys(stores); + + let mut upserts = desired + .iter() + .filter(|(key, value)| { + current + .iter() + .find(|(current_key, _)| current_key == key) + .is_none_or(|(_, current_value)| current_value != value) + }) + .cloned() + .collect::>(); + let mut deletes = current + .iter() + .filter(|(key, _)| { + declared.iter().any(|declared_key| declared_key == key) + && !desired.iter().any(|(desired_key, _)| desired_key == key) + }) + .map(|(key, _)| key.clone()) + .collect::>(); + upserts.sort_by(|left, right| left.0.cmp(&right.0)); + deletes.sort(); + + RuntimeStoreNameReconciliation { deletes, upserts } +} + +fn persist_runtime_env_store_name_entries( + stores: &ProvisionStores<'_>, + dry_run: bool, + cwd: &Path, +) -> Result, String> { + let entries = runtime_env_store_name_entries(stores); + let declared = runtime_env_store_name_keys(stores); + if declared.is_empty() { + return Ok(Vec::new()); + } + if dry_run { + let mut out = entries + .iter() + .map(|(key, value)| { + format!( + "would upsert `{key}={value}` into fastly config-store `{RUNTIME_ENV_STORE}`" + ) + }) + .collect::>(); + out.extend( + declared + .iter() + .filter(|key| !entries.iter().any(|(entry_key, _)| entry_key == *key)) + .map(|key| { + format!( + "would remove `{key}` from fastly config-store `{RUNTIME_ENV_STORE}` if a stale mapping is present" + ) + }), + ); + return Ok(out); + } + + let runtime_env_store_id = resolve_remote_config_store_id_in(RUNTIME_ENV_STORE, cwd)? + .ok_or_else(|| no_matching_store_error(RUNTIME_ENV_STORE))?; + let current = read_config_store_entries(&runtime_env_store_id, cwd)?; + let reconciliation = runtime_store_name_reconciliation(stores, ¤t); + if reconciliation.upserts.is_empty() && reconciliation.deletes.is_empty() { + return Ok(Vec::new()); + } + + push_entries_with_committer(&reconciliation.upserts, |key, value| { + create_config_store_entry_in(&runtime_env_store_id, key, value, cwd) + })?; + for key in &reconciliation.deletes { + delete_config_store_entry_in(&runtime_env_store_id, key, cwd)?; + } + Ok(vec![format!( + "reconciled store-name mappings in fastly config-store `{RUNTIME_ENV_STORE}`: upserted {}, removed {} stale mapping(s)", + reconciliation.upserts.len(), + reconciliation.deletes.len() + )]) +} + /// The runtime-override entry naming the config-store KEY for logical store /// `id` — `EDGEZERO__STORES__CONFIG____KEY`. /// @@ -3706,7 +3931,23 @@ fn shape_summary(value: &serde_json::Value) -> &'static str { /// spawn/list failure, schema drift) stays `Err` -- callers that read for a diff /// must not treat an operational failure as "store absent" and overwrite. fn resolve_remote_config_store_id(name: &str) -> Result, String> { - match classify_remote_config_store(name)? { + resolve_remote_config_store_id_with_cwd(name, None) +} + +fn resolve_remote_config_store_id_in(name: &str, cwd: &Path) -> Result, String> { + resolve_remote_config_store_id_with_cwd(name, Some(cwd)) +} + +fn resolve_remote_config_store_id_with_cwd( + name: &str, + cwd: Option<&Path>, +) -> Result, String> { + let lookup = if let Some(command_cwd) = cwd { + classify_remote_config_store_in(name, command_cwd)? + } else { + classify_remote_config_store(name)? + }; + match lookup { ConfigStoreLookup::Found(id) => Ok(Some(id)), ConfigStoreLookup::NotFound => Ok(None), ConfigStoreLookup::SchemaDrift(detail) => Err(format!( @@ -3724,16 +3965,29 @@ fn resolve_remote_config_store_id(name: &str) -> Result, String> /// `Err` is only for a failure to OBTAIN an answer; a successful listing that /// simply doesn't contain `name` is `Ok(ConfigStoreLookup::NotFound)`. fn classify_remote_config_store(name: &str) -> Result { - let output = Command::new("fastly") - .args(["config-store", "list", "--json"]) - .output() - .map_err(|err| { - if err.kind() == ErrorKind::NotFound { - format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") - } else { - format!("failed to spawn `fastly`: {err}") - } - })?; + classify_remote_config_store_with_cwd(name, None) +} + +fn classify_remote_config_store_in(name: &str, cwd: &Path) -> Result { + classify_remote_config_store_with_cwd(name, Some(cwd)) +} + +fn classify_remote_config_store_with_cwd( + name: &str, + cwd: Option<&Path>, +) -> Result { + let mut command = Command::new("fastly"); + command.args(["config-store", "list", "--json"]); + if let Some(command_cwd) = cwd { + command.current_dir(command_cwd); + } + let output = command.output().map_err(|err| { + if err.kind() == ErrorKind::NotFound { + format!("`fastly` not found on PATH; {FASTLY_INSTALL_HINT}") + } else { + format!("failed to spawn `fastly`: {err}") + } + })?; if !output.status.success() { return Err(format!( "`fastly config-store list --json` exited with status {}\nstderr: {}", @@ -4852,7 +5106,7 @@ fn relink_runtime_env_for_staging( // isolated. There is simply nothing to mirror — the twin gets only the // derived `_staging` selectors, and the staged draft is relinked to // it so it reads staged config while production keeps its default key. - let production = match classify_remote_config_store(RUNTIME_ENV_STORE)? { + let production = match classify_remote_config_store_in(RUNTIME_ENV_STORE, manifest_dir)? { ConfigStoreLookup::Found(id) => read_config_store_entries(&id, manifest_dir)?, ConfigStoreLookup::NotFound => Vec::new(), ConfigStoreLookup::SchemaDrift(detail) => { @@ -4867,7 +5121,7 @@ fn relink_runtime_env_for_staging( // THIS draft at the twin. Create the twin on demand so a staged deploy never // depends on a prior provision having created it. let staging_store_name = staging_selector_store_name(service_id); - let staging_store_id = ensure_staging_selector_store(&staging_store_name)?; + let staging_store_id = ensure_staging_selector_store(&staging_store_name, manifest_dir)?; mirror_production_to_staging( &production, &staging_store_id, @@ -6614,10 +6868,10 @@ build = \"cargo build --release\" let out = FastlyCliAdapter .provision(dir.path(), Some("fastly.toml"), None, &stores, true) .expect("dry-run succeeds"); - // 1 KV + 1 config + 1 secret + runtime-env = 4 status lines. The staging - // twin is created and populated by a staged deploy, NOT by provision, so - // it does not appear here. - assert_eq!(out.len(), 4, "dry-run rows: {out:?}"); + // 1 KV + 1 config + 1 secret + runtime-env + 3 possible stale-mapping + // removals = 7 status lines. The staging twin is created and populated by + // a staged deploy, NOT by provision, so it does not appear here. + assert_eq!(out.len(), 7, "dry-run rows: {out:?}"); assert!(out[0].contains("would run `fastly kv-store create --name=sessions`")); assert!(out[1].contains("would run `fastly config-store create --name=app_config`")); assert!(out[2].contains("would run `fastly secret-store create --name=default`")); @@ -6625,6 +6879,11 @@ build = \"cargo build --release\" out[3].contains("would run `fastly config-store create --name=edgezero_runtime_env`"), "runtime-env store row: {out:?}", ); + assert!( + out.iter() + .any(|row| row.contains("EDGEZERO__STORES__KV__SESSIONS__NAME")), + "dry-run reports possible stale mapping cleanup: {out:?}", + ); assert!( !out.iter() .any(|row| row.contains("edgezero_runtime_env_staging")), @@ -6635,6 +6894,105 @@ build = \"cargo build --release\" assert_eq!(after, "name = \"demo\"\n", "dry-run mutated fastly.toml"); } + #[test] + fn provision_dry_run_reports_non_default_store_name_mapping() { + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write(&path, "name = \"demo\"\n").expect("write"); + let secret_ids = vec![ResolvedStoreId::new("default", "production_secrets")]; + let stores = ProvisionStores { + config: &[], + kv: &[], + secrets: &secret_ids, + }; + + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, true) + .expect("dry-run succeeds"); + + assert!(out.iter().any(|line| { + line.contains("EDGEZERO__STORES__SECRETS__DEFAULT__NAME=production_secrets") + })); + } + + #[cfg(unix)] + #[test] + fn provision_reconciles_runtime_store_name_mappings() { + let _lock = path_mutation_guard().lock().expect("guard"); + let dir = tempdir().expect("tempdir"); + let path = dir.path().join("fastly.toml"); + fs::write( + &path, + "[setup.kv_stores.production_sessions]\n\ + [setup.secret_stores.default]\n", + ) + .expect("write"); + let kv = vec![ResolvedStoreId::new("sessions", "production_sessions")]; + let secrets = vec![ResolvedStoreId::from_logical("default")]; + let stores = ProvisionStores { + config: &[], + kv: &kv, + secrets: &secrets, + }; + let current = vec![ + ( + "EDGEZERO__STORES__KV__SESSIONS__NAME".to_owned(), + "old_sessions".to_owned(), + ), + ( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + "old_secrets".to_owned(), + ), + ( + "EDGEZERO__STORES__KV__OTHER__NAME".to_owned(), + "other_service".to_owned(), + ), + ("EDGEZERO__LOGGING__LEVEL".to_owned(), "debug".to_owned()), + ]; + let oplog = dir.path().join("oplog.txt"); + let fake = fake_fastly_runtime_mapping(¤t, &oplog); + let _path = PathPrepend::new(fake.path()); + + let out = FastlyCliAdapter + .provision(dir.path(), Some("fastly.toml"), None, &stores, false) + .expect("mapping reconciliation succeeds"); + let log = fs::read_to_string(&oplog).expect("oplog"); + let manifest_dir = fs::canonicalize(dir.path()).expect("canonical manifest dir"); + + assert!( + log.contains(&format!("store-create cwd={}", manifest_dir.display())), + "runtime-env store creation runs in the manifest directory: {log}" + ); + assert!( + log.contains(&format!("store-list cwd={}", manifest_dir.display())), + "runtime-env store lookup runs in the manifest directory: {log}" + ); + assert!( + log.contains(&format!( + "update EDGEZERO__STORES__KV__SESSIONS__NAME=production_sessions cwd={}", + manifest_dir.display() + )), + "changed non-default mapping is upserted in the manifest directory: {log}" + ); + assert!( + log.contains(&format!( + "delete EDGEZERO__STORES__SECRETS__DEFAULT__NAME cwd={}", + manifest_dir.display() + )), + "stale mapping is removed in the manifest directory: {log}" + ); + assert!( + !log.contains("delete EDGEZERO__STORES__KV__OTHER__NAME") + && !log.contains("EDGEZERO__LOGGING__LEVEL="), + "unrelated runtime entries are preserved: {log}" + ); + assert!( + out.iter() + .any(|line| line.contains("upserted 1, removed 1")), + "status reports both mutations: {out:?}" + ); + } + #[test] fn provision_errors_when_adapter_manifest_path_missing() { let dir = tempdir().expect("tempdir"); @@ -6676,13 +7034,12 @@ build = \"cargo build --release\" assert_eq!(out, vec!["fastly has no declared stores to provision"]); } + #[cfg(unix)] #[test] - fn provision_skips_id_when_setup_block_already_present() { - // setup_block_present's role in the flow: re-running - // provision after the user already declared a store in - // fastly.toml must be a no-op (no shell-out to fastly). - // We can verify this in a real (non-dry-run) call because - // the skip path bypasses create_fastly_store entirely. + fn provision_skips_store_creation_when_setup_block_already_present() { + // Re-running provision skips resource creation but still reads the + // runtime-env store to reconcile a mapping that may have been removed. + let _lock = path_mutation_guard().lock().expect("guard"); let dir = tempdir().expect("tempdir"); let path = dir.path().join("fastly.toml"); fs::write( @@ -6697,11 +7054,21 @@ build = \"cargo build --release\" kv: &kv_ids, secrets: &[], }; + let oplog = dir.path().join("oplog.txt"); + let fake = fake_fastly_runtime_mapping(&[], &oplog); + let _path = PathPrepend::new(fake.path()); + let out = FastlyCliAdapter .provision(dir.path(), Some("fastly.toml"), None, &stores, false) - .expect("skip path succeeds without invoking fastly"); + .expect("skip path succeeds"); assert_eq!(out.len(), 1); assert!(out[0].contains("already declared"), "got: {out:?}"); + let manifest_dir = fs::canonicalize(dir.path()).expect("canonical manifest dir"); + assert_eq!( + fs::read_to_string(oplog).expect("oplog"), + format!("store-list cwd={0}\nlist cwd={0}\n", manifest_dir.display()), + "runtime mapping is inspected in the manifest directory without mutation" + ); } /// When `fastly.toml` declares `service_id`, the next @@ -7213,6 +7580,64 @@ build = \"cargo build --release\" // ---------- read_config_entry (fake fastly, remote shell-out) ---------- + /// Build a fake `fastly` for live runtime store-name reconciliation. + /// The current runtime-env entries are listed verbatim and every update or + /// delete is recorded in `oplog`. + #[cfg(unix)] + fn fake_fastly_runtime_mapping( + current: &[(String, String)], + oplog: &Path, + ) -> tempfile::TempDir { + use std::os::unix::fs::PermissionsExt as _; + + let dir = tempdir().expect("tempdir"); + let store_list = dir.path().join("stores.json"); + let entry_list = dir.path().join("entries.json"); + fs::write( + &store_list, + format!(r#"[{{"name":"{RUNTIME_ENV_STORE}","id":"runtime-env-123"}}]"#), + ) + .expect("store list"); + let entries = current + .iter() + .map(|(key, value)| { + serde_json::json!({ + "item_key": key, + "item_value": value, + }) + }) + .collect::>(); + fs::write( + &entry_list, + serde_json::to_string(&entries).expect("entry list json"), + ) + .expect("entry list"); + + let script = format!( + r#"#!/bin/sh +if [ "$1" = "config-store" ] && [ "$2" = "create" ]; then printf 'store-create cwd=%s\n' "$PWD" >> '{oplog}'; exit 0; fi +if [ "$1" = "config-store" ]; then printf 'store-list cwd=%s\n' "$PWD" >> '{oplog}'; cat '{stores}'; exit 0; fi +sub="$2" +key="" +for arg in "$@"; do case "$arg" in --key=*) key="${{arg#--key=}}";; esac; done +if [ "$sub" = "list" ]; then printf 'list cwd=%s\n' "$PWD" >> '{oplog}'; cat '{entries}'; exit 0; fi +if [ "$sub" = "update" ]; then value=$(cat); printf 'update %s=%s cwd=%s\n' "$key" "$value" "$PWD" >> '{oplog}'; exit 0; fi +if [ "$sub" = "delete" ]; then printf 'delete %s cwd=%s\n' "$key" "$PWD" >> '{oplog}'; exit 0; fi +echo 'unexpected fastly invocation' >&2 +exit 1 +"#, + stores = store_list.display(), + entries = entry_list.display(), + oplog = oplog.display(), + ); + let script_path = dir.path().join("fastly"); + fs::write(&script_path, script).expect("script"); + let mut perms = fs::metadata(&script_path).expect("meta").permissions(); + perms.set_mode(0o755); + fs::set_permissions(&script_path, perms).expect("chmod"); + dir + } + /// Build a tempdir containing a `fastly` shim script that: /// - Responds to `config-store list --json` with a store-list JSON containing /// `TEST_CONFIG_ID` mapped to `store-abc123`. @@ -10213,6 +10638,40 @@ echo 'unexpected' >&2; exit 1 } } + #[test] + fn runtime_env_store_name_entries_include_only_non_default_mappings() { + use edgezero_core::env_config::EnvConfig; + + let config = vec![ResolvedStoreId::from_logical("app_config")]; + let kv = vec![ResolvedStoreId::new("sessions", "production_sessions")]; + let secrets = vec![ResolvedStoreId::new("default", "production_secrets")]; + let stores = ProvisionStores { + config: &config, + kv: &kv, + secrets: &secrets, + }; + + let entries = runtime_env_store_name_entries(&stores); + assert_eq!( + entries, + vec![ + ( + "EDGEZERO__STORES__KV__SESSIONS__NAME".to_owned(), + "production_sessions".to_owned(), + ), + ( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + "production_secrets".to_owned(), + ), + ] + ); + + let env = EnvConfig::from_vars(entries); + assert_eq!(env.store_name("config", "app_config"), "app_config"); + assert_eq!(env.store_name("kv", "sessions"), "production_sessions"); + assert_eq!(env.store_name("secrets", "default"), "production_secrets"); + } + #[test] fn runtime_env_key_matches_what_the_runtime_reads() { use edgezero_core::env_config::EnvConfig; @@ -10294,6 +10753,84 @@ echo 'unexpected' >&2; exit 1 ); } + #[test] + fn runtime_store_name_entries_from_vars_filters_and_validates() { + let entries = runtime_store_name_entries_from_vars([ + ( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + "physical_secrets".to_owned(), + ), + ( + "EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY".to_owned(), + "ignored_selector".to_owned(), + ), + ( + "EDGEZERO__STORES__KV__A__B__NAME".to_owned(), + "ignored_nested_id".to_owned(), + ), + ( + "EDGEZERO__STORES__KV__A__NAME__EXTRA".to_owned(), + "ignored_extra_segment".to_owned(), + ), + ( + "EDGEZERO__STORES__kv__A__NAME".to_owned(), + "ignored_lowercase_kind".to_owned(), + ), + ("UNRELATED".to_owned(), "ignored".to_owned()), + ]) + .expect("valid store-name override"); + + assert_eq!( + entries, + vec![( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + "physical_secrets".to_owned(), + )] + ); + for invalid in [ + String::new(), + "prod\nsecrets".to_owned(), + "prod\0secrets".to_owned(), + ] { + assert!( + runtime_store_name_entries_from_vars([( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + invalid, + )]) + .is_err(), + "an invalid mapped resource name must fail closed" + ); + } + } + + #[test] + fn process_store_name_overrides_win_before_staging_mirror() { + let production = vec![ + ( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + "old_secrets".to_owned(), + ), + ("EDGEZERO__LOGGING__LEVEL".to_owned(), "info".to_owned()), + ]; + let overrides = vec![( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + "new_secrets".to_owned(), + )]; + + let effective = overlay_runtime_store_name_entries(&production, &overrides); + let staging = staging_entries_from_production(&effective, &["app_config".to_owned()]); + + assert!(staging.contains(&( + "EDGEZERO__STORES__SECRETS__DEFAULT__NAME".to_owned(), + "new_secrets".to_owned(), + ))); + assert!(!staging.iter().any(|(_, value)| value == "old_secrets")); + assert!(staging.contains(&( + "EDGEZERO__STORES__CONFIG__APP_CONFIG__KEY".to_owned(), + "app_config_staging".to_owned(), + ))); + } + #[test] fn find_resource_link_id_matches_on_link_name_not_resource_name() { // The link's `name` is an alias defaulting to the resource's name. The diff --git a/crates/edgezero-adapter-fastly/src/lib.rs b/crates/edgezero-adapter-fastly/src/lib.rs index 4df9ef7d..454b7655 100644 --- a/crates/edgezero-adapter-fastly/src/lib.rs +++ b/crates/edgezero-adapter-fastly/src/lib.rs @@ -174,7 +174,9 @@ where /// what the pre-fix code did, just without the env-driven override /// path the spec promises). #[cfg(feature = "fastly")] -fn env_config_from_runtime_dictionary(stores: StoresMetadata) -> EnvConfig { +#[inline] +#[must_use] +pub fn env_config_from_runtime_dictionary(stores: StoresMetadata) -> EnvConfig { use fastly::ConfigStore; use std::iter::empty; let Ok(dict) = ConfigStore::try_open("edgezero_runtime_env") else { diff --git a/crates/edgezero-cli/src/config.rs b/crates/edgezero-cli/src/config.rs index 079069d7..e1cc0a06 100644 --- a/crates/edgezero-cli/src/config.rs +++ b/crates/edgezero-cli/src/config.rs @@ -1633,6 +1633,44 @@ pub(crate) fn reject_merged_id_collisions( Ok(()) } +fn collect_secret_leaf<'raw>( + node: &'raw Value, + field: &SecretField, + name: &str, + rendered: &str, + optional_segment: bool, + out: &mut Vec>, +) -> Result<(), String> { + let parent = node + .as_table() + .ok_or_else(|| format!("expected a table containing `{name}` at `{rendered}`"))?; + let leaf_label = if rendered.is_empty() { + name.to_owned() + } else { + format!("{rendered}.{name}") + }; + match parent.get(name).and_then(Value::as_str) { + Some(value) => { + let store_ref_value = match field.kind { + SecretKind::KeyInNamedStore { store_ref_field } => { + parent.get(store_ref_field).and_then(Value::as_str) + } + SecretKind::KeyInDefault | SecretKind::StoreRef => None, + }; + out.push(ResolvedTomlLeaf { + label: leaf_label, + store_ref_value, + value, + }); + Ok(()) + } + None if (field.optional || optional_segment) && parent.get(name).is_none() => Ok(()), + None => Err(format!( + "`#[secret]` field `{leaf_label}` is missing or not a string" + )), + } +} + /// Collect every concrete secret leaf a `SecretField` resolves to in the /// raw app-config TOML, navigating `Field` (table descent) and `ArrayEach` /// (per-element) segments. `label` uses concrete `[n]` indices and, for a @@ -1652,36 +1690,26 @@ fn collect_secret_leaves<'raw>( ) -> Result<(), String> { match remaining.split_first() { Some((SecretPathSegment::Field(name), [])) => { - let parent = node.as_table().ok_or_else(|| { - format!("expected a table containing `{name}` at `{rendered}`") - })?; - let leaf_label = if rendered.is_empty() { + collect_secret_leaf(node, field, name, rendered, false, out) + } + Some((SecretPathSegment::OptionalField(name), [])) => { + collect_secret_leaf(node, field, name, rendered, true, out) + } + Some((SecretPathSegment::Field(name), rest)) => { + let table = node + .as_table() + .ok_or_else(|| format!("expected a table at `{rendered}`"))?; + let next_rendered = if rendered.is_empty() { name.to_string() } else { format!("{rendered}.{name}") }; - match parent.get(name.as_ref()).and_then(Value::as_str) { - Some(value) => { - let store_ref_value = match field.kind { - SecretKind::KeyInNamedStore { store_ref_field } => { - parent.get(store_ref_field).and_then(Value::as_str) - } - SecretKind::KeyInDefault | SecretKind::StoreRef => None, - }; - out.push(ResolvedTomlLeaf { - label: leaf_label, - store_ref_value, - value, - }); - Ok(()) - } - None if field.optional && parent.get(name.as_ref()).is_none() => Ok(()), - None => Err(format!( - "`#[secret]` field `{leaf_label}` is missing or not a string" - )), + match table.get(name.as_ref()) { + Some(child) => walk(child, field, rest, &next_rendered, out), + None => Err(format!("missing `{next_rendered}`")), } } - Some((SecretPathSegment::Field(name), rest)) => { + Some((SecretPathSegment::OptionalField(name), rest)) => { let table = node .as_table() .ok_or_else(|| format!("expected a table at `{rendered}`"))?; @@ -1690,13 +1718,9 @@ fn collect_secret_leaves<'raw>( } else { format!("{rendered}.{name}") }; - // Intermediates are always required — `field.optional` reflects - // only the leaf, and the derive never nests through `Option`. This - // matches the runtime walk (`resolve_secret_field`) so `config - // validate` catches exactly what the runtime would reject. match table.get(name.as_ref()) { Some(child) => walk(child, field, rest, &next_rendered, out), - None => Err(format!("missing `{next_rendered}`")), + None => Ok(()), } } Some((SecretPathSegment::ArrayEach, rest)) => { @@ -1709,6 +1733,10 @@ fn collect_secret_leaves<'raw>( } Ok(()) } + Some((_unsupported, _)) => Err(format!( + "unsupported secret path segment in `{}`", + field.dotted_path() + )), None => Ok(()), } } @@ -2828,12 +2856,28 @@ other = "x" assert!(leaves.is_empty(), "absent optional leaf yields nothing"); } + #[test] + fn collect_secret_leaves_skips_absent_optional_intermediate() { + let raw: Value = toml::from_str("[integrations]\n").expect("toml"); + let field = SecretField { + kind: SecretKind::KeyInDefault, + path: vec![ + SecretPathSegment::Field(Cow::Borrowed("integrations")), + SecretPathSegment::OptionalField(Cow::Borrowed("datadome")), + SecretPathSegment::Field(Cow::Borrowed("webhook_key")), + ], + optional: true, + }; + let leaves = collect_secret_leaves(&raw, &field) + .expect("absent optional intermediate should be skipped"); + assert!( + leaves.is_empty(), + "absent optional intermediate yields nothing" + ); + } + #[test] fn collect_secret_leaves_errors_on_missing_required_intermediate() { - // A missing INTERMEDIATE (the `integrations` table) is an error even - // when the leaf is optional — `optional` reflects only the leaf, and - // intermediates are structurally required. Locks alignment with the - // runtime walk (`resolve_secret_field`). let raw: Value = toml::from_str("other = \"x\"\n").expect("toml"); let field = SecretField { kind: SecretKind::KeyInDefault, diff --git a/crates/edgezero-core/src/app_config.rs b/crates/edgezero-core/src/app_config.rs index 509e5684..6cf96235 100644 --- a/crates/edgezero-core/src/app_config.rs +++ b/crates/edgezero-core/src/app_config.rs @@ -32,11 +32,17 @@ use validator::{Validate, ValidationErrors}; /// One segment of a [`SecretField`] path. #[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] pub enum SecretPathSegment { /// Every element of an array/`Vec` at this position. ArrayEach, /// An object key — a Rust field name, verbatim (no `serde(rename)`). Field(Cow<'static, str>), + /// An optional field that skips the rest of this secret path when absent or null. + /// + /// Available to hand-written [`AppConfigMeta`] implementations. The + /// `AppConfig` derive does not currently emit optional intermediate fields. + OptionalField(Cow<'static, str>), } /// One field's worth of secret-annotation metadata. @@ -64,7 +70,7 @@ impl SecretField { let mut out = String::new(); for segment in &self.path { match segment { - SecretPathSegment::Field(name) => { + SecretPathSegment::Field(name) | SecretPathSegment::OptionalField(name) => { if !out.is_empty() { out.push('.'); } @@ -324,10 +330,13 @@ fn prune_secret_leaf(errors: &mut ValidationErrors, path: &[SecretPathSegment]) let Some((head, rest)) = path.split_first() else { return; }; - let SecretPathSegment::Field(name) = head else { - // `ArrayEach` only appears immediately after a `Field` (the root is - // always a struct), so it is consumed by the peek below, never a head. - return; + let name = match head { + SecretPathSegment::Field(name) | SecretPathSegment::OptionalField(name) => name, + SecretPathSegment::ArrayEach => { + // `ArrayEach` only appears immediately after a field (the root is + // always a struct), so it is consumed by the peek below, never a head. + return; + } }; // Leaf reached: drop the validator error keyed by this field name. @@ -1554,7 +1563,7 @@ greeting = "hello" kind: SecretKind::KeyInDefault, path: vec![ Field(Cow::Borrowed("integrations")), - Field(Cow::Borrowed("datadome")), + OptionalField(Cow::Borrowed("datadome")), Field(Cow::Borrowed("server_side_key")), ], optional: false, diff --git a/crates/edgezero-core/src/extractor.rs b/crates/edgezero-core/src/extractor.rs index 7cbb4a85..22205682 100644 --- a/crates/edgezero-core/src/extractor.rs +++ b/crates/edgezero-core/src/extractor.rs @@ -999,15 +999,22 @@ fn resolve_secret_field<'walk>( ) -> Pin> + 'walk>> { Box::pin(async move { match remaining.split_first() { - // Leaf reached: `node` is the PARENT object; the last Field is the key. + // Leaf reached: `node` is the PARENT object; the last field is the key. Some((SecretPathSegment::Field(name), [])) => { resolve_leaf(ctx, node, field, name.as_ref(), &rendered).await } - // Descend into an object key. Intermediates are ALWAYS required — - // `field.optional` reflects only the LEAF (`Option`), and the - // derive never nests through `Option`/`Box`, so a missing/null parent - // is a stale blob. (Skipping it here would let the whole subtree pass - // silently and only fail later with a vaguer serde error.) + Some((SecretPathSegment::OptionalField(name), [])) => { + if matches!( + node.get(name.as_ref()), + None | Some(serde_json::Value::Null) + ) { + return Ok(()); + } + resolve_leaf(ctx, node, field, name.as_ref(), &rendered).await + } + // Required intermediates still reject stale blobs. Optional + // intermediates are represented explicitly below rather than by the + // leaf's `field.optional` flag. Some((SecretPathSegment::Field(name), rest)) => { let next_rendered = join_field(&rendered, name.as_ref()); match node.get_mut(name.as_ref()) { @@ -1020,8 +1027,17 @@ fn resolve_secret_field<'walk>( } } } + Some((SecretPathSegment::OptionalField(name), rest)) => { + let next_rendered = join_field(&rendered, name.as_ref()); + match node.get_mut(name.as_ref()) { + None | Some(serde_json::Value::Null) => Ok(()), + Some(child) => { + resolve_secret_field(ctx, child, field, rest, next_rendered).await + } + } + } // Iterate every array element. The array itself is a required - // intermediate (see above), so a non-array is always an error. + // intermediate unless its containing field was optional above. Some((SecretPathSegment::ArrayEach, rest)) => { let Some(items) = node.as_array_mut() else { return Err(EdgeError::config_out_of_date( @@ -1394,7 +1410,8 @@ mod tests { } } - // Optional leaf behind required intermediates: integrations.datadome.webhook_key + // Optional leaf behind one required and one optional intermediate: + // integrations.datadome.webhook_key struct OptionalNestedCfg; impl AppConfigMeta for OptionalNestedCfg { fn secret_fields() -> Vec { @@ -1402,7 +1419,7 @@ mod tests { kind: SecretKind::KeyInDefault, path: vec![ SecretPathSegment::Field(Cow::Borrowed("integrations")), - SecretPathSegment::Field(Cow::Borrowed("datadome")), + SecretPathSegment::OptionalField(Cow::Borrowed("datadome")), SecretPathSegment::Field(Cow::Borrowed("webhook_key")), ], optional: true, @@ -2861,9 +2878,24 @@ mod tests { ); } + #[test] + fn secret_walk_skips_absent_optional_intermediate() { + let ctx = ctx_with_default_secret_store("unused", "unused"); + let mut data = serde_json::json!({ "integrations": {} }); + block_on(secret_walk::(&ctx, &mut data)) + .expect("absent optional intermediate is fine"); + } + + #[test] + fn secret_walk_skips_null_optional_intermediate() { + let ctx = ctx_with_default_secret_store("unused", "unused"); + let mut data = serde_json::json!({ "integrations": { "datadome": null } }); + block_on(secret_walk::(&ctx, &mut data)) + .expect("null optional intermediate is fine"); + } + #[test] fn secret_walk_present_intermediate_absent_optional_leaf_is_ok() { - // The mirror case: intermediates present, optional leaf absent -> skip. let ctx = ctx_with_default_secret_store("unused", "unused"); let mut data = serde_json::json!({ "integrations": { "datadome": {} } }); block_on(secret_walk::(&ctx, &mut data)) diff --git a/docs/guide/cli-reference.md b/docs/guide/cli-reference.md index 37607371..74c57b82 100644 --- a/docs/guide/cli-reference.md +++ b/docs/guide/cli-reference.md @@ -510,12 +510,12 @@ edgezero provision --adapter [--manifest ] [--dry-run] **Per-adapter behaviour:** -| `--adapter` | Behaviour | -| ------------ | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `axum` | Local-only — prints one note per declared store id and exits 0 (KV in-memory; config in `.edgezero/local-config-.json`). | -| `cloudflare` | For each KV id + config id: shells out to `wrangler kv namespace create ` (where `` resolves from `EDGEZERO__STORES______NAME` or falls back to the logical ``), parses the namespace id from stdout, appends `[[kv_namespaces]] binding = "", id = ""` to `wrangler.toml` (idempotent on the binding name; preserves existing entries and comments). Secrets are runtime-managed via `wrangler secret put` — no-op. | -| `fastly` | For each KV / config / secret id: shells out to `fastly -store create --name=` (using the same `` resolution), then appends the `[setup._stores.]` table to `fastly.toml`. Provision writes ONLY `[setup.*]` (the remote/deploy half); the `[local_server.*]` seeding is written by `config push --local` (config stores only). Idempotent: if the setup table is already present the id is skipped (no shell-out, no edit). Store IDs are not persisted — `config push` resolves them on demand. | -| `spin` | Pure `spin.toml` editing — no shell-out (Spin KV stores are runtime-resolved). For each declared KV id AND each declared `[stores.config]` id (both KV-backed at runtime), appends the platform-resolved label to the resolved `[component.].key_value_stores = [...]` array (idempotent on the label). Secret variables are still manual: `[stores.secrets]` ids get a `nothing to do here` status line and the operator declares `[variables]. = { secret = true }` + the per-component binding by hand. | +| `--adapter` | Behaviour | +| ------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `axum` | Local-only — prints one note per declared store id and exits 0 (KV in-memory; config in `.edgezero/local-config-.json`). | +| `cloudflare` | For each KV id + config id: shells out to `wrangler kv namespace create ` (where `` resolves from `EDGEZERO__STORES______NAME` or falls back to the logical ``), parses the namespace id from stdout, appends `[[kv_namespaces]] binding = "", id = ""` to `wrangler.toml` (idempotent on the binding name; preserves existing entries and comments). Secrets are runtime-managed via `wrangler secret put` — no-op. | +| `fastly` | For each KV / config / secret id: shells out to `fastly -store create --name=` (using the same `` resolution), then appends the `[setup._stores.]` table to `fastly.toml`. Provision writes ONLY `[setup.*]` (the remote/deploy half); the `[local_server.*]` seeding is written by `config push --local` (config stores only). If the setup table is already present, resource creation and manifest editing are skipped. A live run still reconciles declared logical-to-physical name mappings in `edgezero_runtime_env`, removing stale mappings when an override returns to its logical default. Store IDs are not persisted — `config push` resolves them on demand. | +| `spin` | Pure `spin.toml` editing — no shell-out (Spin KV stores are runtime-resolved). For each declared KV id AND each declared `[stores.config]` id (both KV-backed at runtime), appends the platform-resolved label to the resolved `[component.].key_value_stores = [...]` array (idempotent on the label). Secret variables are still manual: `[stores.secrets]` ids get a `nothing to do here` status line and the operator declares `[variables]. = { secret = true }` + the per-component binding by hand. | **`--dry-run`** prints what each adapter _would_ do without performing it. For `axum` the output is identical to a real run @@ -530,8 +530,10 @@ existing `binding`s are detected and skipped. The `fastly` flow requires `fastly` on `PATH` and `[adapters.fastly.adapter].manifest` pointing at the project's -`fastly.toml`. Re-running is safe: provision skips any id whose -`[setup._stores.]` block already exists in the manifest. +`fastly.toml`. Re-running is safe: provision skips resource creation for any id +whose `[setup._stores.]` block already exists, then reads +`edgezero_runtime_env` and reconciles mappings for the app's declared logical +ids. It does not delete mappings for undeclared ids or unrelated runtime entries. The `spin` flow needs no native CLI but does require `[adapters.spin.adapter].manifest` pointing at the project's