From fcf5fa77c6a4bb3e410a9e0100ee6ec09b29e6d5 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Tue, 18 Aug 2026 21:28:36 +0000 Subject: [PATCH 1/8] Add `allow_ddm_traffic` parameter for link creation Plumb this through such front ports can run ddm for multirack. --- dpd-api/src/lib.rs | 88 +++++++++++++++++++ dpd-client/tests/chaos_tests/port_settings.rs | 7 ++ .../tests/integration_tests/port_api.rs | 1 + .../versions/src/allow_ddm_traffic/link.rs | 75 ++++++++++++++++ .../versions/src/allow_ddm_traffic/mod.rs | 13 +++ .../versions/src/allow_ddm_traffic/port.rs | 72 +++++++++++++++ dpd-types/versions/src/latest.rs | 8 +- dpd-types/versions/src/lib.rs | 2 + dpd/src/api_server.rs | 1 + dpd/src/link.rs | 18 +++- dpd/src/macaddrs.rs | 1 + dpd/src/main.rs | 1 + dpd/src/port_settings.rs | 4 + openapi/dpd/dpd-12.0.0-a135ff.json.gitstub | 1 + ...0.0-a135ff.json => dpd-13.0.0-040180.json} | 7 +- openapi/dpd/dpd-latest.json | 2 +- swadm/src/compliance.rs | 1 + swadm/src/link.rs | 21 ++++- tfportd/src/simport.rs | 1 + 19 files changed, 316 insertions(+), 8 deletions(-) create mode 100644 dpd-types/versions/src/allow_ddm_traffic/link.rs create mode 100644 dpd-types/versions/src/allow_ddm_traffic/mod.rs create mode 100644 dpd-types/versions/src/allow_ddm_traffic/port.rs create mode 100644 openapi/dpd/dpd-12.0.0-a135ff.json.gitstub rename openapi/dpd/{dpd-12.0.0-a135ff.json => dpd-13.0.0-040180.json} (99%) diff --git a/dpd-api/src/lib.rs b/dpd-api/src/lib.rs index 3aac525b..b71363f5 100644 --- a/dpd-api/src/lib.rs +++ b/dpd-api/src/lib.rs @@ -39,6 +39,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (13, ALLOW_DDM_TRAFFIC), (12, PRBS_ERROR_TRACKING), (11, WALLCLOCK_HISTORY), (10, ASIC_DETAILS), @@ -717,6 +718,7 @@ pub trait DpdApi { /// physical port to create an interface of the desired speed, if possible. #[endpoint { method = POST, + versions = VERSION_ALLOW_DDM_TRAFFIC.., path = "/ports/{port_id}/links" }] async fn link_create( @@ -725,6 +727,25 @@ pub trait DpdApi { params: TypedBody, ) -> Result, HttpError>; + /// Create a link on a switch port. + /// + /// Create an interface that can be used for sending Ethernet frames on the + /// provided switch port. This will use the first available lanes in the + /// physical port to create an interface of the desired speed, if possible. + #[endpoint { + method = POST, + versions = ..VERSION_ALLOW_DDM_TRAFFIC, + path = "/ports/{port_id}/links", + operation_id = "link_create", + }] + async fn link_create_v1( + rqctx: RequestContext, + path: Path, + params: TypedBody, + ) -> Result, HttpError> { + Self::link_create(rqctx, path, params.map(Into::into)).await + } + /// Get an existing link by ID. #[endpoint { method = GET, @@ -1631,6 +1652,7 @@ pub trait DpdApi { */ #[endpoint { method = POST, + versions = VERSION_ALLOW_DDM_TRAFFIC.., path = "/port/{port_id}/settings" }] async fn port_settings_apply( @@ -1640,11 +1662,38 @@ pub trait DpdApi { body: TypedBody, ) -> Result, HttpError>; + /** + * Apply port settings atomically. + * + * These settings will be applied holistically, and to the extent possible + * atomically to a given port. In the event of a failure a rollback is + * attempted. If the rollback fails there will be inconsistent state. This + * failure mode returns the error code "rollback failure". For more details see + * the docs on the [`PortSettings`] type. + */ + #[endpoint { + method = POST, + versions = ..VERSION_ALLOW_DDM_TRAFFIC, + path = "/port/{port_id}/settings", + operation_id = "port_settings_apply", + }] + async fn port_settings_apply_v1( + rqctx: RequestContext, + path: Path, + query: Query, + body: TypedBody, + ) -> Result, HttpError> { + Self::port_settings_apply(rqctx, path, query, body.map(Into::into)) + .await + .map(|resp| resp.map(Into::into)) + } + /** * Clear port settings atomically. */ #[endpoint { method = DELETE, + versions = VERSION_ALLOW_DDM_TRAFFIC.., path = "/port/{port_id}/settings" }] async fn port_settings_clear( @@ -1653,11 +1702,31 @@ pub trait DpdApi { query: Query, ) -> Result, HttpError>; + /** + * Clear port settings atomically. + */ + #[endpoint { + method = DELETE, + versions = ..VERSION_ALLOW_DDM_TRAFFIC, + path = "/port/{port_id}/settings", + operation_id = "port_settings_clear", + }] + async fn port_settings_clear_v1( + rqctx: RequestContext, + path: Path, + query: Query, + ) -> Result, HttpError> { + Self::port_settings_clear(rqctx, path, query) + .await + .map(|resp| resp.map(Into::into)) + } + /** * Get port settings atomically. */ #[endpoint { method = GET, + versions = VERSION_ALLOW_DDM_TRAFFIC.., path = "/port/{port_id}/settings" }] async fn port_settings_get( @@ -1666,6 +1735,25 @@ pub trait DpdApi { query: Query, ) -> Result, HttpError>; + /** + * Get port settings atomically. + */ + #[endpoint { + method = GET, + versions = ..VERSION_ALLOW_DDM_TRAFFIC, + path = "/port/{port_id}/settings", + operation_id = "port_settings_get", + }] + async fn port_settings_get_v1( + rqctx: RequestContext, + path: Path, + query: Query, + ) -> Result, HttpError> { + Self::port_settings_get(rqctx, path, query) + .await + .map(|resp| resp.map(Into::into)) + } + /// Get switch identifiers. /// /// This endpoint returns the switch identifiers, which can be used for diff --git a/dpd-client/tests/chaos_tests/port_settings.rs b/dpd-client/tests/chaos_tests/port_settings.rs index a3dfebcb..bd781779 100644 --- a/dpd-client/tests/chaos_tests/port_settings.rs +++ b/dpd-client/tests/chaos_tests/port_settings.rs @@ -92,6 +92,7 @@ async fn test_basic_autoneg_chaos() -> anyhow::Result<()> { speed: PortSpeed::Speed100G, fec: Some(PortFec::None), tx_eq: None, + allow_ddm_traffic: false, }, ) .await @@ -128,6 +129,7 @@ async fn test_port_settings_addr_fail_1() -> anyhow::Result<()> { fec: Some(PortFec::None), speed: PortSpeed::Speed100G, tx_eq: None, + allow_ddm_traffic: false, }, addrs: vec!["203.0.113.47".parse().unwrap()], }, @@ -169,6 +171,7 @@ async fn test_port_settings_addr_success_1() -> anyhow::Result<()> { fec: Some(PortFec::None), speed: PortSpeed::Speed100G, tx_eq: None, + allow_ddm_traffic: false, }, addrs: vec!["203.0.113.47".parse().unwrap()], }, @@ -208,6 +211,7 @@ async fn test_port_settings_addr_success_multi() -> anyhow::Result<()> { fec: Some(PortFec::None), speed: PortSpeed::Speed100G, tx_eq: None, + allow_ddm_traffic: false, }, addrs: vec!["203.0.113.47".parse().unwrap()], }, @@ -237,6 +241,7 @@ async fn test_port_settings_addr_success_multi() -> anyhow::Result<()> { fec: Some(PortFec::None), speed: PortSpeed::Speed100G, tx_eq: None, + allow_ddm_traffic: false, }, addrs: vec![ "203.0.113.46".parse().unwrap(), @@ -277,6 +282,7 @@ async fn test_port_settings_addr_success_multi() -> anyhow::Result<()> { fec: Some(PortFec::None), speed: PortSpeed::Speed100G, tx_eq: None, + allow_ddm_traffic: false, }, addrs: vec![ "203.0.113.47".parse().unwrap(), @@ -522,6 +528,7 @@ fn random_port_settings() -> PortSettings { speed: PortSpeed::Speed100G, tx_eq: None, fec: Some(PortFec::None), + allow_ddm_traffic: false, }; let link_id = 0; diff --git a/dpd-client/tests/integration_tests/port_api.rs b/dpd-client/tests/integration_tests/port_api.rs index 98fe8675..838b3796 100644 --- a/dpd-client/tests/integration_tests/port_api.rs +++ b/dpd-client/tests/integration_tests/port_api.rs @@ -741,6 +741,7 @@ async fn test_set_mac_on_new_link_succeeds() -> TestResult { fec: link.fec, speed: link.speed, tx_eq: None, + allow_ddm_traffic: false, }; let _new_link = switch .client diff --git a/dpd-types/versions/src/allow_ddm_traffic/link.rs b/dpd-types/versions/src/allow_ddm_traffic/link.rs new file mode 100644 index 00000000..9a414934 --- /dev/null +++ b/dpd-types/versions/src/allow_ddm_traffic/link.rs @@ -0,0 +1,75 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use common::ports::{PortFec, PortSpeed, TxEq}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v1; + +/// Parameters used to create a link on a switch port. +#[derive(Clone, Debug, Deserialize, Serialize, JsonSchema)] +pub struct LinkCreate { + /// The first lane of the port to use for the new link + pub lane: Option, + /// The requested speed of the link. + pub speed: PortSpeed, + /// The requested forward-error correction method. If this is None, the + /// standard FEC for the underlying media will be applied if it can be + /// determined. + pub fec: Option, + /// Whether the link is configured to autonegotiate with its peer during + /// link training. + /// + /// This is generally only true for backplane links, and defaults to + /// `false`. + #[serde(default)] + pub autoneg: bool, + /// Whether the link is configured in KR mode, an electrical specification + /// generally only true for backplane link. + /// + /// This defaults to `false`. + #[serde(default)] + pub kr: bool, + + /// Transceiver equalization adjustment parameters. + /// This defaults to `None`. + #[serde(default)] + pub tx_eq: Option, + + /// Whether DDM traffic is allowed on this link. + /// + /// This defaults to `false`. + #[serde(default)] + pub allow_ddm_traffic: bool, +} + +impl From for LinkCreate { + fn from(old: v1::link::LinkCreate) -> Self { + Self { + lane: old.lane, + speed: old.speed, + fec: old.fec, + autoneg: old.autoneg, + kr: old.kr, + tx_eq: old.tx_eq, + allow_ddm_traffic: false, + } + } +} + +impl From for v1::link::LinkCreate { + fn from(new: LinkCreate) -> Self { + Self { + lane: new.lane, + speed: new.speed, + fec: new.fec, + autoneg: new.autoneg, + kr: new.kr, + tx_eq: new.tx_eq, + } + } +} diff --git a/dpd-types/versions/src/allow_ddm_traffic/mod.rs b/dpd-types/versions/src/allow_ddm_traffic/mod.rs new file mode 100644 index 00000000..4563d845 --- /dev/null +++ b/dpd-types/versions/src/allow_ddm_traffic/mod.rs @@ -0,0 +1,13 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +//! Version `ALLOW_DDM_TRAFFIC` of the DPD API. +//! +//! Added an `allow_ddm_traffic` field to `LinkCreate`. `LinkSettings` and +//! `PortSettings` are updated to carry the new `LinkCreate`. + +pub mod link; +pub mod port; diff --git a/dpd-types/versions/src/allow_ddm_traffic/port.rs b/dpd-types/versions/src/allow_ddm_traffic/port.rs new file mode 100644 index 00000000..3cbfe60e --- /dev/null +++ b/dpd-types/versions/src/allow_ddm_traffic/port.rs @@ -0,0 +1,72 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +use std::collections::{HashMap, HashSet}; +use std::net::IpAddr; + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use crate::v1; + +use super::link::LinkCreate; + +/// A port settings transaction object. When posted to the +/// `/port-settings/{port_id}` API endpoint, these settings will be applied +/// holistically, and to the extent possible atomically to a given port. +#[derive(Default, Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct PortSettings { + /// The link settings to apply to the port on a per-link basis. Any links + /// not in this map that are resident on the switch port will be removed. + /// Any links that are in this map that are not resident on the switch port + /// will be added. Any links that are resident on the switch port and in + /// this map, and are different, will be modified. Links are indexed by + /// spatial index within the port. + pub links: HashMap, +} + +/// An object with link settings used in concert with [`PortSettings`]. +#[derive(Clone, Debug, Deserialize, JsonSchema, Serialize)] +pub struct LinkSettings { + pub params: LinkCreate, + pub addrs: HashSet, +} + +impl From for PortSettings { + fn from(old: v1::port::PortSettings) -> Self { + Self { + links: old + .links + .into_iter() + .map(|(index, settings)| (index, settings.into())) + .collect(), + } + } +} + +impl From for v1::port::PortSettings { + fn from(new: PortSettings) -> Self { + Self { + links: new + .links + .into_iter() + .map(|(index, settings)| (index, settings.into())) + .collect(), + } + } +} + +impl From for LinkSettings { + fn from(old: v1::port::LinkSettings) -> Self { + Self { params: old.params.into(), addrs: old.addrs } + } +} + +impl From for v1::port::LinkSettings { + fn from(new: LinkSettings) -> Self { + Self { params: new.params.into(), addrs: new.addrs } + } +} diff --git a/dpd-types/versions/src/latest.rs b/dpd-types/versions/src/latest.rs index 3350c606..7a04d1db 100644 --- a/dpd-types/versions/src/latest.rs +++ b/dpd-types/versions/src/latest.rs @@ -30,7 +30,6 @@ pub mod fault { } pub mod link { - pub use crate::v1::link::LinkCreate; pub use crate::v1::link::LinkEvent; pub use crate::v1::link::LinkFilter; pub use crate::v1::link::LinkFsmCounter; @@ -47,6 +46,8 @@ pub mod link { pub use crate::v12::link::LinkView; pub use crate::v12::link::MsDuration; + + pub use crate::v13::link::LinkCreate; } pub mod loopback { @@ -100,14 +101,15 @@ pub mod nat { pub mod port { pub use crate::v1::port::FreeChannels; - pub use crate::v1::port::LinkSettings; pub use crate::v1::port::PortCreateParams; pub use crate::v1::port::PortIdPathParams; pub use crate::v1::port::PortIpv4Path; pub use crate::v1::port::PortIpv6Path; - pub use crate::v1::port::PortSettings; pub use crate::v1::port::PortSettingsTag; pub use crate::v1::port::PortToken; + + pub use crate::v13::port::LinkSettings; + pub use crate::v13::port::PortSettings; } pub mod port_map { diff --git a/dpd-types/versions/src/lib.rs b/dpd-types/versions/src/lib.rs index 5f99f707..617a58c7 100644 --- a/dpd-types/versions/src/lib.rs +++ b/dpd-types/versions/src/lib.rs @@ -41,6 +41,8 @@ pub mod v10; pub mod v11; #[path = "prbs_error_tracking/mod.rs"] pub mod v12; +#[path = "allow_ddm_traffic/mod.rs"] +pub mod v13; #[path = "attached_subnets/mod.rs"] pub mod v3; #[path = "v4_over_v6_routes/mod.rs"] diff --git a/dpd/src/api_server.rs b/dpd/src/api_server.rs index ea111235..8dc859ad 100644 --- a/dpd/src/api_server.rs +++ b/dpd/src/api_server.rs @@ -2968,6 +2968,7 @@ impl From<&crate::link::Link> for LinkSettings { autoneg: l.config.autoneg, kr: l.config.kr, tx_eq: l.tx_eq, + allow_ddm_traffic: false, }, addrs, } diff --git a/dpd/src/link.rs b/dpd/src/link.rs index 3834a69d..ac4ee699 100644 --- a/dpd/src/link.rs +++ b/dpd/src/link.rs @@ -337,6 +337,9 @@ pub struct LinkConfig { /// This link is expected to be connected to the outside world, and /// should only accept inbound traffic that matches a NAT mapping. pub uplink: bool, + + /// This links should allow ddm traffic for multirack setups + pub allow_ddm_traffic: bool, } // This struct represents the state of the link as it actually exists in the @@ -401,6 +404,7 @@ pub struct LinkParams { pub autoneg: bool, pub kr: bool, pub tx_eq: Option, + pub allow_ddm_traffic: bool, } impl Link { @@ -417,10 +421,18 @@ impl Link { // By default, we enable ipv6 on backplane and internal links, but // disable it for external-facing qsfp links. This allows the site // admin to determine the kinds of traffic we send to their network. - let ipv6_enabled = !matches!(port_id, PortId::Qsfp(_)); + // + // For multirack ddm traffic on the front ports we must also enable + // ipv6. + let ipv6_enabled = + !matches!(port_id, PortId::Qsfp(_)) || params.allow_ddm_traffic; // By default we expect external-facing links to be used as uplinks and // internal-facing links for backplane traffic. - let uplink = matches!(port_id, PortId::Qsfp(_)); + // + // We need to disable uplinks for multirack ddm traffic on the front + // ports. + let uplink = + matches!(port_id, PortId::Qsfp(_)) && !params.allow_ddm_traffic; let config = LinkConfig { delete_me: false, @@ -432,6 +444,7 @@ impl Link { speed: params.speed, uplink, mac, + allow_ddm_traffic: params.allow_ddm_traffic, }; let plumbed = LinkPlumbed { link_created: false, @@ -557,6 +570,7 @@ impl Switch { kr: params.kr, tx_eq: params.tx_eq, fec: params.fec, + allow_ddm_traffic: params.allow_ddm_traffic, }; let mut links = self.links.lock().unwrap(); diff --git a/dpd/src/macaddrs.rs b/dpd/src/macaddrs.rs index f526098b..52f850d4 100644 --- a/dpd/src/macaddrs.rs +++ b/dpd/src/macaddrs.rs @@ -411,6 +411,7 @@ impl Switch { autoneg: true, kr: true, tx_eq: None, + allow_ddm_traffic: false, }; let params = match &autoconfig_links { Some(links) => links diff --git a/dpd/src/main.rs b/dpd/src/main.rs index 1e2d91c4..5fc08ec5 100644 --- a/dpd/src/main.rs +++ b/dpd/src/main.rs @@ -767,6 +767,7 @@ async fn sidecar_main(mut switch: Switch) -> anyhow::Result<()> { kr: true, lane: Some(dpd_types::link::LinkId(0)), tx_eq: None, + allow_ddm_traffic: false, }; Some((*port_id, create)) } else { diff --git a/dpd/src/port_settings.rs b/dpd/src/port_settings.rs index f9575174..1af0709e 100644 --- a/dpd/src/port_settings.rs +++ b/dpd/src/port_settings.rs @@ -102,6 +102,7 @@ struct LinkSpec { pub ipv4: BTreeSet, pub ipv6: BTreeSet, pub tx_eq: Option, + pub allow_ddm_traffic: bool, } impl From<&Link> for LinkSpec { @@ -115,6 +116,7 @@ impl From<&Link> for LinkSpec { delete_me: p.config.delete_me, ipv4: p.ipv4.iter().map(|x| x.addr).collect(), ipv6: p.ipv6.iter().map(|x| x.addr).collect(), + allow_ddm_traffic: p.config.allow_ddm_traffic, } } } @@ -144,6 +146,7 @@ impl From<&LinkSettings> for LinkSpec { ) .copied() .collect(), + allow_ddm_traffic: l.params.allow_ddm_traffic, } } } @@ -314,6 +317,7 @@ impl PortSettingsDiff { kr: spec.kr, tx_eq: spec.tx_eq, fec: spec.fec, + allow_ddm_traffic: spec.allow_ddm_traffic, }; let port_id = ctx.port_id; let asic_port_id = ctx.switch.port_link_to_asic_id(port_id, link_id)?; diff --git a/openapi/dpd/dpd-12.0.0-a135ff.json.gitstub b/openapi/dpd/dpd-12.0.0-a135ff.json.gitstub new file mode 100644 index 00000000..ac977a6a --- /dev/null +++ b/openapi/dpd/dpd-12.0.0-a135ff.json.gitstub @@ -0,0 +1 @@ +ef7978f916c17d5851935b8e7c2c12db48f72097:openapi/dpd/dpd-12.0.0-a135ff.json diff --git a/openapi/dpd/dpd-12.0.0-a135ff.json b/openapi/dpd/dpd-13.0.0-040180.json similarity index 99% rename from openapi/dpd/dpd-12.0.0-a135ff.json rename to openapi/dpd/dpd-13.0.0-040180.json index 8e02fbd4..f11306a3 100644 --- a/openapi/dpd/dpd-12.0.0-a135ff.json +++ b/openapi/dpd/dpd-13.0.0-040180.json @@ -7,7 +7,7 @@ "url": "https://oxide.computer", "email": "api@oxide.computer" }, - "version": "12.0.0" + "version": "13.0.0" }, "paths": { "/all-settings": { @@ -7640,6 +7640,11 @@ "description": "Parameters used to create a link on a switch port.", "type": "object", "properties": { + "allow_ddm_traffic": { + "description": "Whether DDM traffic is allowed on this link.\n\nThis defaults to `false`.", + "default": false, + "type": "boolean" + }, "autoneg": { "description": "Whether the link is configured to autonegotiate with its peer during link training.\n\nThis is generally only true for backplane links, and defaults to `false`.", "default": false, diff --git a/openapi/dpd/dpd-latest.json b/openapi/dpd/dpd-latest.json index bab102da..d1387757 120000 --- a/openapi/dpd/dpd-latest.json +++ b/openapi/dpd/dpd-latest.json @@ -1 +1 @@ -dpd-12.0.0-a135ff.json \ No newline at end of file +dpd-13.0.0-040180.json \ No newline at end of file diff --git a/swadm/src/compliance.rs b/swadm/src/compliance.rs index 86adb53c..e0f92b70 100644 --- a/swadm/src/compliance.rs +++ b/swadm/src/compliance.rs @@ -382,6 +382,7 @@ async fn compliance_ports_setup( autoneg, kr, tx_eq: None, + allow_ddm_traffic: false, }; match client.link_create(&port_id, ¶ms).await { diff --git a/swadm/src/link.rs b/swadm/src/link.rs index dd02b39f..41ce7c41 100644 --- a/swadm/src/link.rs +++ b/swadm/src/link.rs @@ -445,6 +445,10 @@ pub enum Link { #[clap(long)] kr: bool, + /// Whether DDM traffic is allowed on this link. + #[clap(long)] + allow_ddm_traffic: bool, + /// Uniform equalization parameter to apply to all cursors. #[clap(long)] tx_eq: Option, @@ -498,6 +502,10 @@ pub struct LinkCreate { /// This is generally only appropriate for backplane links. #[clap(short, long)] kr: bool, + + /// If provided, allow DDM traffic on the link. + #[clap(long)] + allow_ddm_traffic: bool, } #[derive(Clone, Copy, Debug, PartialEq, Subcommand)] @@ -1635,7 +1643,15 @@ fn apply_filter( pub async fn link_cmd(client: &Client, link: Link) -> anyhow::Result<()> { match link { - Link::Create(LinkCreate { port_id, speed, lane, fec, autoneg, kr }) => { + Link::Create(LinkCreate { + port_id, + speed, + lane, + fec, + autoneg, + kr, + allow_ddm_traffic, + }) => { let params = types::LinkCreate { lane, speed: speed.into(), @@ -1643,6 +1659,7 @@ pub async fn link_cmd(client: &Client, link: Link) -> anyhow::Result<()> { autoneg, kr, tx_eq: None, + allow_ddm_traffic, }; let link_id = client .link_create(&port_id, ¶ms) @@ -2115,6 +2132,7 @@ pub async fn link_cmd(client: &Client, link: Link) -> anyhow::Result<()> { fec, autoneg, kr, + allow_ddm_traffic, tx_eq, pre1, pre2, @@ -2146,6 +2164,7 @@ pub async fn link_cmd(client: &Client, link: Link) -> anyhow::Result<()> { types::LinkSettings { addrs: Vec::default(), params: types::LinkCreate { + allow_ddm_traffic, autoneg, fec: fec.map(|f| f.into()), kr, diff --git a/tfportd/src/simport.rs b/tfportd/src/simport.rs index 394894fa..81920ce7 100644 --- a/tfportd/src/simport.rs +++ b/tfportd/src/simport.rs @@ -121,6 +121,7 @@ async fn simnet_process(g: &Global) -> anyhow::Result<()> { autoneg: false, kr: false, tx_eq: None, + allow_ddm_traffic: false, }; if let Err(e) = g.client.link_create(&port_id, ¶ms).await { error!( From 5bac5604a450ae882e5548743c9b6431ef062f10 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Tue, 18 Aug 2026 22:33:15 +0000 Subject: [PATCH 2/8] fix types after merge --- dpd-types/versions/src/allow_ddm_traffic/link.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpd-types/versions/src/allow_ddm_traffic/link.rs b/dpd-types/versions/src/allow_ddm_traffic/link.rs index 9a414934..650e83ce 100644 --- a/dpd-types/versions/src/allow_ddm_traffic/link.rs +++ b/dpd-types/versions/src/allow_ddm_traffic/link.rs @@ -4,9 +4,9 @@ // // Copyright 2026 Oxide Computer Company -use common::ports::{PortFec, PortSpeed, TxEq}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use v1::port::{PortFec, PortSpeed, TxEq}; use crate::v1; From d47b431b4a64c1fe5003c1acb69c95f39845b6ab Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Tue, 18 Aug 2026 22:34:50 +0000 Subject: [PATCH 3/8] fix typo --- dpd/src/link.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpd/src/link.rs b/dpd/src/link.rs index ac4ee699..83600b48 100644 --- a/dpd/src/link.rs +++ b/dpd/src/link.rs @@ -338,7 +338,7 @@ pub struct LinkConfig { /// should only accept inbound traffic that matches a NAT mapping. pub uplink: bool, - /// This links should allow ddm traffic for multirack setups + /// This link should allow ddm traffic for multirack setups pub allow_ddm_traffic: bool, } From 66b6b2039b86790df1581dc31676007aca9f03ba Mon Sep 17 00:00:00 2001 From: Nicolas Kagami Date: Wed, 19 Aug 2026 17:01:01 -0300 Subject: [PATCH 4/8] pass allow_ddm_traffic through for LinkSettings --- dpd/src/api_server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpd/src/api_server.rs b/dpd/src/api_server.rs index 419f1309..5e108a04 100644 --- a/dpd/src/api_server.rs +++ b/dpd/src/api_server.rs @@ -2963,7 +2963,7 @@ impl From<&crate::link::Link> for LinkSettings { autoneg: l.config.autoneg, kr: l.config.kr, tx_eq: l.tx_eq, - allow_ddm_traffic: false, + allow_ddm_traffic: l.config.allow_ddm_traffic, }, addrs, } From 7da3486a00012c313f49ddf49bd62b79d48b2a98 Mon Sep 17 00:00:00 2001 From: Nicolas Kagami Date: Wed, 19 Aug 2026 18:25:24 -0300 Subject: [PATCH 5/8] impose allod_ddm_traffic logic on modify_link --- dpd/src/port_settings.rs | 76 +++++++++++++++++++++++++++++++--------- 1 file changed, 59 insertions(+), 17 deletions(-) diff --git a/dpd/src/port_settings.rs b/dpd/src/port_settings.rs index 1af0709e..cfcea772 100644 --- a/dpd/src/port_settings.rs +++ b/dpd/src/port_settings.rs @@ -403,21 +403,60 @@ impl PortSettingsDiff { let link_lock = ctx.link(link_id)?; let mut link = link_lock.lock().unwrap(); - let speed_before = spec.before.speed; - let fec_before = spec.before.fec; - let an_before = spec.before.autoneg; - let kr_before = spec.before.kr; - let txeq_before = spec.before.tx_eq; - let delete_before = spec.before.delete_me; - link.config.speed = spec.after.speed; - link.config.fec = spec.after.fec; - link.config.autoneg = spec.after.autoneg; - link.config.kr = spec.after.kr; - link.tx_eq = spec.after.tx_eq; + let Modify { before, after } = spec; + + let &LinkSpec { + speed: speed_before, + fec: fec_before, + autoneg: an_before, + kr: kr_before, + delete_me: delete_before, + ipv4: ref ipv4_before, + ipv6: ref ipv6_before, + tx_eq: tx_eq_before, + allow_ddm_traffic: allow_ddm_traffic_before, + } = before; + + let &LinkSpec { + speed: speed_after, + fec: fec_after, + autoneg: an_after, + kr: kr_after, + delete_me: _, + ipv4: ref ipv4_after, + ipv6: ref ipv6_after, + tx_eq: tx_eq_after, + allow_ddm_traffic: allow_ddm_traffic_after, + } = after; + + debug_assert_eq!( + link.config.allow_ddm_traffic, + allow_ddm_traffic_before, + ); + + let ipv6_enabled_before = link.ipv6_enabled; + let uplink_before = link.config.uplink; + + if allow_ddm_traffic_before != allow_ddm_traffic_after { + let is_front_port = matches!(ctx.port_id, PortId::Qsfp(_)); + + // Following the logic from Link::new(). Would be nice to factor it out. + link.ipv6_enabled = !is_front_port || allow_ddm_traffic_after; + link.config.uplink = is_front_port && !allow_ddm_traffic_after; + } + + link.config.speed = speed_after; + link.config.fec = fec_after; + link.config.autoneg = an_after; + link.config.kr = kr_after; + link.tx_eq = tx_eq_after; link.config.delete_me = false; - if spec.before.tx_eq != spec.after.tx_eq { + link.config.allow_ddm_traffic = allow_ddm_traffic_after; + + if tx_eq_before != tx_eq_after { link.plumbed.tx_eq_pushed = false; } + rb.wind(move |ctx: &mut Context<'_>| -> DpdResult<()> { let link_lock = ctx.link(link_id)?; let mut link = link_lock.lock().unwrap(); @@ -425,17 +464,20 @@ impl PortSettingsDiff { link.config.fec = fec_before; link.config.autoneg = an_before; link.config.kr = kr_before; - link.tx_eq = txeq_before; + link.tx_eq = tx_eq_before; link.config.delete_me = delete_before; + link.config.allow_ddm_traffic = allow_ddm_traffic_before; + link.ipv6_enabled = ipv6_enabled_before; + link.config.uplink = uplink_before; Ok(()) }); // ipv4 addrs let v4_add: BTreeSet = - spec.after.ipv4.difference(&spec.before.ipv4).copied().collect(); + ipv4_after.difference(ipv4_before).copied().collect(); let v4_del: BTreeSet = - spec.before.ipv4.difference(&spec.after.ipv4).copied().collect(); + ipv4_before.difference(ipv4_after).copied().collect(); for addr in v4_add { Self::addr_add_v4(ctx, &mut link, rb, addr)?; @@ -446,10 +488,10 @@ impl PortSettingsDiff { // ipv6 addrs let v6_add: BTreeSet = - spec.after.ipv6.difference(&spec.before.ipv6).copied().collect(); + ipv6_after.difference(ipv6_before).copied().collect(); let v6_del: BTreeSet = - spec.before.ipv6.difference(&spec.after.ipv6).copied().collect(); + ipv6_before.difference(ipv6_after).copied().collect(); for addr in v6_add { Self::addr_add_v6(ctx, &mut link, rb, addr)?; From 1cab7c758c9af07b54cf3bc27fb38d264482e864 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Wed, 19 Aug 2026 23:36:54 +0000 Subject: [PATCH 6/8] fix docs --- dpd-types/versions/src/allow_ddm_traffic/link.rs | 6 ++++++ .../dpd/{dpd-13.0.0-040180.json => dpd-13.0.0-5db8bd.json} | 2 +- openapi/dpd/dpd-latest.json | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) rename openapi/dpd/{dpd-13.0.0-040180.json => dpd-13.0.0-5db8bd.json} (99%) diff --git a/dpd-types/versions/src/allow_ddm_traffic/link.rs b/dpd-types/versions/src/allow_ddm_traffic/link.rs index 650e83ce..e4a453bf 100644 --- a/dpd-types/versions/src/allow_ddm_traffic/link.rs +++ b/dpd-types/versions/src/allow_ddm_traffic/link.rs @@ -42,6 +42,12 @@ pub struct LinkCreate { /// Whether DDM traffic is allowed on this link. /// + /// This only applies to the qsfp front ports. Rear ports always have DDM + /// enabled, regardless of this setting. + /// + /// The actual effect of this is to enable IPv6 routing on front ports and + /// ensure they are not an uplink. + /// /// This defaults to `false`. #[serde(default)] pub allow_ddm_traffic: bool, diff --git a/openapi/dpd/dpd-13.0.0-040180.json b/openapi/dpd/dpd-13.0.0-5db8bd.json similarity index 99% rename from openapi/dpd/dpd-13.0.0-040180.json rename to openapi/dpd/dpd-13.0.0-5db8bd.json index f11306a3..22694237 100644 --- a/openapi/dpd/dpd-13.0.0-040180.json +++ b/openapi/dpd/dpd-13.0.0-5db8bd.json @@ -7641,7 +7641,7 @@ "type": "object", "properties": { "allow_ddm_traffic": { - "description": "Whether DDM traffic is allowed on this link.\n\nThis defaults to `false`.", + "description": "Whether DDM traffic is allowed on this link.\n\nThis only applies to the qsfp front ports. Rear ports always have DDM enabled, regardless of this setting.\n\nThe actual effect of this is to enable IPv6 routing on front ports and ensure they are not an uplink.\n\nThis defaults to `false`.", "default": false, "type": "boolean" }, diff --git a/openapi/dpd/dpd-latest.json b/openapi/dpd/dpd-latest.json index d1387757..d1a5660e 120000 --- a/openapi/dpd/dpd-latest.json +++ b/openapi/dpd/dpd-latest.json @@ -1 +1 @@ -dpd-13.0.0-040180.json \ No newline at end of file +dpd-13.0.0-5db8bd.json \ No newline at end of file From 67635f6971b4ce67fe18eab71a9d9214b8585cd9 Mon Sep 17 00:00:00 2001 From: "Andrew J. Stone" Date: Wed, 19 Aug 2026 23:40:17 +0000 Subject: [PATCH 7/8] more doc fixes --- dpd/src/link.rs | 5 ++++- swadm/src/link.rs | 6 ++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/dpd/src/link.rs b/dpd/src/link.rs index 83600b48..ae60baf4 100644 --- a/dpd/src/link.rs +++ b/dpd/src/link.rs @@ -338,7 +338,10 @@ pub struct LinkConfig { /// should only accept inbound traffic that matches a NAT mapping. pub uplink: bool, - /// This link should allow ddm traffic for multirack setups + /// This link should allow DDM traffic for multirack setups. + /// + /// This setting only applies to qsfp front ports. Rear ports always have + /// DDM enabled. pub allow_ddm_traffic: bool, } diff --git a/swadm/src/link.rs b/swadm/src/link.rs index 41ce7c41..47e7e34a 100644 --- a/swadm/src/link.rs +++ b/swadm/src/link.rs @@ -446,6 +446,9 @@ pub enum Link { kr: bool, /// Whether DDM traffic is allowed on this link. + /// + /// This setting only applies to qsfp front ports. Rear ports always + /// have DDM enabled. #[clap(long)] allow_ddm_traffic: bool, @@ -504,6 +507,9 @@ pub struct LinkCreate { kr: bool, /// If provided, allow DDM traffic on the link. + /// + /// This setting only applies to qsfp front ports. Rear ports always have + /// DDM enabled. #[clap(long)] allow_ddm_traffic: bool, } From 3e0ef67860f7930f8da5fe1d8a3282820598aa2d Mon Sep 17 00:00:00 2001 From: Cory Zimmerman Date: Sun, 16 Aug 2026 08:16:55 +0000 Subject: [PATCH 8/8] Tx eq settings TODO: magnificent testing --- aal/src/lib.rs | 26 ++- asic/src/chaos/mod.rs | 17 ++ asic/src/softnpu/mod.rs | 30 +++- asic/src/tofino_asic/imported_bf_functions | 1 + asic/src/tofino_asic/mod.rs | 42 ++++- asic/src/tofino_asic/ports.rs | 15 +- asic/src/tofino_asic/serdes.rs | 152 ++++++++-------- asic/src/tofino_stub/mod.rs | 19 +- common/src/ports.rs | 4 +- dpd-api/src/lib.rs | 37 +++- dpd-types/versions/src/explicit_txeq/mod.rs | 7 + dpd-types/versions/src/explicit_txeq/txeq.rs | 142 +++++++++++++++ dpd-types/versions/src/latest.rs | 9 +- dpd-types/versions/src/lib.rs | 2 + dpd/src/api_server.rs | 86 ++------- dpd/src/link.rs | 14 +- dpd/src/main.rs | 3 +- dpd/src/port_map.rs | 5 + dpd/src/port_settings.rs | 6 +- dpd/src/switch_port.rs | 175 +++++++++++-------- dpd/src/transceivers/tofino_impl.rs | 63 ++++--- openapi/dpd/dpd-latest.json | 2 +- swadm/src/link.rs | 121 ++++++++----- 23 files changed, 664 insertions(+), 314 deletions(-) create mode 100644 dpd-types/versions/src/explicit_txeq/mod.rs create mode 100644 dpd-types/versions/src/explicit_txeq/txeq.rs diff --git a/aal/src/lib.rs b/aal/src/lib.rs index 3fa1ea56..1dca814e 100644 --- a/aal/src/lib.rs +++ b/aal/src/lib.rs @@ -8,7 +8,9 @@ use std::hash::Hash; use thiserror::Error; -use common::ports::{PortFec, PortMedia, PortPrbsMode, PortSpeed, TxEq}; +use common::ports::{ + PortFec, PortMedia, PortPrbsMode, PortSpeed, TxEq, TxEqSwHw, +}; use common::table::TableType; pub use dpd_types::table::{TableEntryAction, TableEntryKey}; @@ -245,6 +247,16 @@ pub trait AsicOps { settings: &TxEq, ) -> AsicResult<()>; + /// Read the current transceiver equalization settings on this port. + /// Returns an ordered iterator mapping each lane to its [`TxEqSwHw`] settings. + /// + /// - The outer error indicates failure accessing the port. + /// - The iterator error indicates failure accessing the specific lane. + fn port_tx_eq_get( + &self, + port_hdl: PortHdl, + ) -> AsicResult>>; + /// For the given connector, return a list of all of its channels which have /// not yet been assigned to a logical port. fn connector_avail_channels( @@ -252,6 +264,18 @@ pub trait AsicOps { connector: Connector, ) -> AsicResult>; + /// Returns the tx equalization settings configured for this + /// connector in the SDE board map. + /// + /// These aren't necessarily the _current_ tx eq settings if runtime + /// modifications have been made. + /// + /// Returns an iterator mapping each u8 channel to its settings. + fn connector_tx_eq_defaults( + &self, + connector: Connector, + ) -> impl ExactSizeIterator>; + /// Get sidecar identifiers of the device being managed. fn get_sidecar_identifiers(&self) -> AsicResult; diff --git a/asic/src/chaos/mod.rs b/asic/src/chaos/mod.rs index e5d58831..df5b4ebf 100644 --- a/asic/src/chaos/mod.rs +++ b/asic/src/chaos/mod.rs @@ -529,6 +529,23 @@ impl AsicOps for Handle { Ok(()) } + fn connector_tx_eq_defaults( + &self, + _: Connector, + ) -> impl ExactSizeIterator> + { + std::iter::empty() + } + + fn port_tx_eq_get( + &self, + _: PortHdl, + ) -> AsicResult< + impl ExactSizeIterator>, + > { + Ok(std::iter::empty()) + } + fn port_prbs_set( &self, _port_hdl: PortHdl, diff --git a/asic/src/softnpu/mod.rs b/asic/src/softnpu/mod.rs index cfb497ad..4b14f51e 100644 --- a/asic/src/softnpu/mod.rs +++ b/asic/src/softnpu/mod.rs @@ -133,11 +133,6 @@ impl Handle { }) } - pub fn port_tx_eq_get(&self, port_hdl: PortHdl) -> AsicResult { - let ports = self.ports.lock().unwrap(); - Ok(get_port(&ports, port_hdl)?.tx_eq) - } - pub fn is_model(&self) -> bool { true } @@ -292,10 +287,33 @@ impl AsicOps for Handle { tx_eq: &TxEq, ) -> AsicResult<()> { let mut ports = self.ports.lock().unwrap(); - get_port_mut(&mut ports, port_hdl)?.tx_eq = tx_eq.main.unwrap_or(0); + get_port_mut(&mut ports, port_hdl)?.tx_eq = tx_eq.main; Ok(()) } + fn port_tx_eq_get( + &self, + port_hdl: PortHdl, + ) -> AsicResult< + impl ExactSizeIterator>, + > { + let tx_eq_main = + self::get_port(&self.ports.lock().unwrap(), port_hdl)?.tx_eq; + let tx_eq = TxEq { main: tx_eq_main, ..Default::default() }; + + let lanes = self.port_get_lane_cnt(port_hdl)?; + + Ok((0..lanes) + .map(move |_| Ok(common::ports::TxEqSwHw { hw: tx_eq, sw: tx_eq }))) + } + + fn connector_tx_eq_defaults( + &self, + _: Connector, + ) -> impl ExactSizeIterator> { + std::iter::once(Ok((0, TxEq::default()))) + } + fn port_autoneg_set( &self, _port_hdl: PortHdl, diff --git a/asic/src/tofino_asic/imported_bf_functions b/asic/src/tofino_asic/imported_bf_functions index 547abe30..dddd74a5 100644 --- a/asic/src/tofino_asic/imported_bf_functions +++ b/asic/src/tofino_asic/imported_bf_functions @@ -15,6 +15,7 @@ bf_drv_device_type_get # Board-related bf_bd_is_this_port_internal +bf_bd_port_serdes_tx_params_get # multicast manager calls bf_mc_init diff --git a/asic/src/tofino_asic/mod.rs b/asic/src/tofino_asic/mod.rs index 18eb3eed..86b27ada 100644 --- a/asic/src/tofino_asic/mod.rs +++ b/asic/src/tofino_asic/mod.rs @@ -317,8 +317,35 @@ impl AsicOps for Handle { port_hdl: PortHdl, settings: &TxEq, ) -> AsicResult<()> { - let settings = serdes::TxEqSettings::from(*settings); - serdes::port_tx_eq_set(self, port_hdl, &settings) + serdes::port_tx_eq_set(self, port_hdl, settings) + } + + fn port_tx_eq_get( + &self, + port_hdl: PortHdl, + ) -> AsicResult>> { + let lanes = serdes::lane_count(self, port_hdl)?; + let lanes = u8::try_from(lanes).map_err(|e| { + AsicError::Internal(format!( + "Any reasonable lane count should fit in a u8. Found lanes = {lanes:?}: {e:?}" + )) + })?; + + Ok((0..lanes) + .map(move |lane| serdes::lane_tx_eq_get(self, port_hdl, lane))) + } + + fn connector_tx_eq_defaults( + &self, + connector: Connector, + ) -> impl ExactSizeIterator> { + (0..tofino_common::ports::CHANNELS_PER_SWITCH_PORT).map( + move |channel| { + let tx_eq = + serdes::connector_tx_eq_default(self, connector, channel)?; + Ok((channel, tx_eq)) + }, + ) } } @@ -472,6 +499,17 @@ impl Handle { pub fn clear_qsfp_state(&self) { qsfp::clear_transceiver_tx(); } + + /// Returns the board map ID of the given connector or an error + /// if ID couldn't be determined. + pub fn connector_id(&self, conn: Connector) -> AsicResult { + match (conn, self.eth_connector_id) { + (Connector::QSFP(id), _) | (Connector::CPU, Some(id)) => Ok(id), + (Connector::CPU, None) => { + Err(AsicError::InvalidArg("no CPU ports found".to_string())) + } + } + } } pub fn sde_error(ctx: impl ToString, err: bf_status_t) -> AsicError { diff --git a/asic/src/tofino_asic/ports.rs b/asic/src/tofino_asic/ports.rs index ce866d61..31fcde8c 100644 --- a/asic/src/tofino_asic/ports.rs +++ b/asic/src/tofino_asic/ports.rs @@ -108,16 +108,11 @@ impl FrontPortHandle { /// Given a PortHdl, return the front-panel port it maps to pub fn from_port_hdl(hdl: &Handle, port_hdl: PortHdl) -> AsicResult { - let connector = match port_hdl.connector { - Connector::CPU => match hdl.eth_connector_id { - Some(id) => Ok(id), - None => { - Err(AsicError::InvalidArg("no CPU ports found".to_string())) - } - }, - Connector::QSFP(c) => Ok(c), - }?; - Ok(FrontPortHandle::new(hdl.dev_id, connector, port_hdl.channel)) + Ok(FrontPortHandle::new( + hdl.dev_id, + hdl.connector_id(port_hdl.connector)?, + port_hdl.channel, + )) } /// Get the first front-panel port. Whether this is actually "first" in any diff --git a/asic/src/tofino_asic/serdes.rs b/asic/src/tofino_asic/serdes.rs index 32fbc42f..2978b2d7 100644 --- a/asic/src/tofino_asic/serdes.rs +++ b/asic/src/tofino_asic/serdes.rs @@ -8,9 +8,11 @@ #![allow(non_snake_case)] #![allow(clippy::manual_range_contains)] +use aal::Connector; +use common::ports::TxEq; +use common::ports::TxEqSwHw; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; -use std::convert::From; use crate::tofino_asic::genpd::*; use crate::tofino_asic::ports; @@ -179,69 +181,20 @@ pub fn port_rx_sig_info_get( Ok(rval) } -/// There are two groups of TxEqSettings: the one cached in the software and the -/// one currently set in the hardware. -#[derive(Deserialize, Serialize, JsonSchema)] -pub struct TxEqHwSw { - /// Value cached in software - pub sw: TxEqSettings, - /// The value actually in use by the hardware - pub hw: TxEqSettings, -} - -/// Tx equalization settings -#[derive(Clone, Default, Debug, Deserialize, Serialize, JsonSchema)] -pub struct TxEqSettings { - /// Precursor 2 - pub pre2: i32, - /// Precursor 1 - pub pre1: i32, - /// Main - pub main: i32, - /// Postcursor 1 - pub post1: i32, - /// Postcursor 2 - pub post2: i32, -} - -impl From for common::ports::TxEq { - fn from(txeq: TxEqSettings) -> Self { - common::ports::TxEq { - pre1: Some(txeq.pre1), - pre2: Some(txeq.pre2), - main: Some(txeq.main), - post2: Some(txeq.post2), - post1: Some(txeq.post1), - } - } -} - -impl From for TxEqSettings { - fn from(txeq: common::ports::TxEq) -> Self { - TxEqSettings { - pre1: txeq.pre1.unwrap_or(0), - pre2: txeq.pre2.unwrap_or(0), - main: txeq.main.unwrap_or(0), - post2: txeq.post2.unwrap_or(0), - post1: txeq.post1.unwrap_or(0), - } - } -} - // Fetch the currently applied tx eq settings for the specified port and -// logical lane -fn lane_tx_eq_get( +// logical lane. +pub fn lane_tx_eq_get( hdl: &Handle, port: PortHdl, - lane: u32, -) -> AsicResult { + lane: u8, +) -> AsicResult { let port_id = ports::to_asic_id(hdl, port)?; - let mut sw = TxEqSettings::default(); + let mut sw = TxEq::default(); unsafe { bf_tof2_serdes_tx_taps_get( hdl.dev_id, port_id as i32, - lane, + lane.into(), &mut sw.pre2, &mut sw.pre1, &mut sw.main, @@ -250,12 +203,12 @@ fn lane_tx_eq_get( ) .check_error("fetching sw tx eq settings")?; } - let mut hw = TxEqSettings::default(); + let mut hw = TxEq::default(); unsafe { bf_tof2_serdes_tx_taps_hw_get( hdl.dev_id, port_id as i32, - lane, + lane.into(), &mut hw.pre2, &mut hw.pre1, &mut hw.main, @@ -265,23 +218,7 @@ fn lane_tx_eq_get( .check_error("fetching hw tx eq settings")? }; - Ok(TxEqHwSw { sw, hw }) -} - -/// Collect all of the per-lane eq settings for the specified port. -/// -/// The returned value contains a vector of `TxEqHwSw` structures, indexed by -/// the logical lane ID within the link. -pub fn port_tx_eq_get( - hdl: &Handle, - port: PortHdl, -) -> AsicResult> { - let lanes = lane_count(hdl, port)?; - let mut rval = Vec::with_capacity(lanes as usize); - for lane in 0..lanes { - rval.push(lane_tx_eq_get(hdl, port, lane)?) - } - Ok(rval) + Ok(TxEqSwHw { sw, hw }) } /// Update the currently applied tx eq settings in both the hardware and the @@ -290,7 +227,7 @@ pub fn lane_tx_eq_set( hdl: &Handle, port: PortHdl, lane: u32, - settings: &TxEqSettings, + settings: &TxEq, ) -> AsicResult<()> { let port_id = ports::to_asic_id(hdl, port)?; let lanes = lane_count(hdl, port)?; @@ -320,7 +257,7 @@ pub fn lane_tx_eq_set( pub fn port_tx_eq_set( hdl: &Handle, port: PortHdl, - settings: &TxEqSettings, + settings: &TxEq, ) -> AsicResult<()> { let lanes = lane_count(hdl, port)?; for lane in 0..lanes { @@ -335,6 +272,67 @@ pub fn port_tx_eq_set( Ok(()) } +/// Returns the board default tx equalization settings for the +/// given connector and channel. +/// +/// Informed by the SDE board map: +/// +/// +pub fn connector_tx_eq_default( + handle: &Handle, + connector: Connector, + channel: u8, +) -> AsicResult { + let mut port_info = bf_pltfm_port_info_t { + conn_id: handle.connector_id(connector)?, + chnl_id: channel.into(), + }; + + // Either encoding mode should work because we set the same config + // for both. This is not true to the actual encoding mode used + // by the port, but querying that adds failure cases that will rely + // on this conclusion anyway. + // + // - Definition: https://github.com/oxidecomputer/tofino-sde/blob/oxide/pkgsrc/bf-platforms/drivers/include/bf_bd_cfg/bf_bd_cfg_bd_map.h#L101 + // - Where this is assigned: https://github.com/oxidecomputer/tofino-sde/blob/oxide/pkgsrc/bf-platforms/platforms/sidecar/src/platform_mgr/board.c#L200 + // - Where this is read: https://github.com/oxidecomputer/tofino-sde/blob/oxide/pkgsrc/bf-platforms/drivers/src/bf_bd_cfg/bf_bd_cfg_intf.c#L727-L731 + let encoding = bf_pltfm_encoding_type__BF_PLTFM_ENCODING_NRZ; + + // Any of the options other than `unknown` should work because we set the + // same config for all. + // + // - Assignment: https://github.com/oxidecomputer/tofino-sde/blob/oxide/pkgsrc/bf-platforms/platforms/sidecar/src/platform_mgr/board.c#L203-L216 + // - MAX: https://github.com/oxidecomputer/tofino-sde/blob/oxide/pkgsrc/bf-platforms/drivers/include/bf_bd_cfg/bf_bd_cfg_bd_map.h#L20 + // - No unknown: https://github.com/oxidecomputer/tofino-sde/blob/oxide/pkgsrc/bf-platforms/drivers/src/bf_bd_cfg/bf_bd_cfg_intf.c#L706-L711 + let qsfpdd = bf_pltfm_qsfpdd_type_t_BF_PLTFM_QSFPDD_OPT; + + let mut taps = bf_pltfm_serdes_lane_tx_eq_t { + tx_main: 0, + tx_pre1: 0, + tx_pre2: 0, + tx_post1: 0, + tx_post2: 0, + }; + + unsafe { + bf_bd_port_serdes_tx_params_get( + &mut port_info, + qsfpdd, + &mut taps, + encoding, + ) + .check_error("fetching board default tx eq settings")?; + } + + Ok(TxEq { + pre2: taps.tx_pre2, + pre1: taps.tx_pre1, + main: taps.tx_main, + post1: taps.tx_post1, + post2: taps.tx_post2, + }) +} + // Fetch the state of the Rx Decision Feedback Equalizer adaptation for the // specified port and logical lane fn lane_adapt_state_get( diff --git a/asic/src/tofino_stub/mod.rs b/asic/src/tofino_stub/mod.rs index 8f4224c0..3a2ec1ce 100644 --- a/asic/src/tofino_stub/mod.rs +++ b/asic/src/tofino_stub/mod.rs @@ -186,12 +186,29 @@ impl AsicOps for StubHandle { ports::set_autoneg_mode(self, port_hdl, an) } + fn connector_tx_eq_defaults( + &self, + _: Connector, + ) -> impl ExactSizeIterator> + { + std::iter::once(Ok((0, common::ports::TxEq::default()))) + } + fn port_tx_eq_set( &self, _port_hdl: PortHdl, _settings: &common::ports::TxEq, ) -> AsicResult<()> { - Ok(()) + Err(AsicError::OperationUnsupported) + } + + fn port_tx_eq_get( + &self, + _: PortHdl, + ) -> AsicResult< + impl ExactSizeIterator>, + > { + Result::, _>::Err(AsicError::OperationUnsupported) } fn port_prbs_set( diff --git a/common/src/ports.rs b/common/src/ports.rs index f5c2803f..e4a1d634 100644 --- a/common/src/ports.rs +++ b/common/src/ports.rs @@ -12,7 +12,7 @@ use crate::network::MacAddr; pub use dpd_types::port::{ InternalPort, Ipv4Entry, Ipv6Entry, PORT_COUNT_INTERNAL, PORT_COUNT_QSFP, PORT_COUNT_REAR, PortFec, PortId, PortMedia, PortPrbsMode, PortSpeed, - QsfpPort, RearPort, TxEq, TxEqSwHw, + QsfpPort, RearPort, TxEq, TxEqConfig, TxEqSwHw, }; #[derive(Clone, Copy, Debug, Deserialize, Serialize, JsonSchema)] @@ -46,5 +46,5 @@ pub struct XcvrSettings { /// FEC setting pub fec: Option, /// Equalization settings - pub tx_eq: Option, + pub tx_eq: TxEqConfig, } diff --git a/dpd-api/src/lib.rs b/dpd-api/src/lib.rs index 7441178e..6c716058 100644 --- a/dpd-api/src/lib.rs +++ b/dpd-api/src/lib.rs @@ -29,6 +29,7 @@ api_versions!([ // | example for the next person. // v // (next_int, IDENT), + (14, EXPLICIT_TXEQ), (13, ALLOW_DDM_TRAFFIC), (12, PRBS_ERROR_TRACKING), (11, WALLCLOCK_HISTORY), @@ -2654,6 +2655,23 @@ pub trait DpdApi { */ #[endpoint { method = GET, + versions = ..VERSION_EXPLICIT_TXEQ, + path = "/ports/{port_id}/links/{link_id}/serdes/tx_eq", + operation_id = "link_tx_eq_get" + }] + async fn link_tx_eq_get_v1( + rqctx: RequestContext, + path: Path, + ) -> Result>, HttpError> { + Ok(Self::link_tx_eq_get(rqctx, path) + .await? + .map(|inner| inner.into_iter().map(Into::into).collect())) + } + + /// Get the tx eq settings for each lane on this link. + #[endpoint { + method = GET, + versions = VERSION_EXPLICIT_TXEQ.., path = "/ports/{port_id}/links/{link_id}/serdes/tx_eq", }] async fn link_tx_eq_get( @@ -2666,12 +2684,29 @@ pub trait DpdApi { */ #[endpoint { method = PUT, + versions = ..VERSION_EXPLICIT_TXEQ, + path = "/ports/{port_id}/links/{link_id}/serdes/tx_eq", + operation_id = "link_tx_eq_set" + }] + async fn link_tx_eq_set_v1( + rqctx: RequestContext, + path: Path, + args: TypedBody, + ) -> Result { + Self::link_tx_eq_set(rqctx, path, args.map(|tx_eq| Some(tx_eq).into())) + .await + } + + /// Update the tx eq settings for all lanes on this link + #[endpoint { + method = PUT, + versions = VERSION_EXPLICIT_TXEQ.., path = "/ports/{port_id}/links/{link_id}/serdes/tx_eq", }] async fn link_tx_eq_set( rqctx: RequestContext, path: Path, - args: TypedBody, + args: TypedBody, ) -> Result; /** diff --git a/dpd-types/versions/src/explicit_txeq/mod.rs b/dpd-types/versions/src/explicit_txeq/mod.rs new file mode 100644 index 00000000..0841fa76 --- /dev/null +++ b/dpd-types/versions/src/explicit_txeq/mod.rs @@ -0,0 +1,7 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +pub mod txeq; diff --git a/dpd-types/versions/src/explicit_txeq/txeq.rs b/dpd-types/versions/src/explicit_txeq/txeq.rs new file mode 100644 index 00000000..371bf3b7 --- /dev/null +++ b/dpd-types/versions/src/explicit_txeq/txeq.rs @@ -0,0 +1,142 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at https://mozilla.org/MPL/2.0/ +// +// Copyright 2026 Oxide Computer Company + +//! This module defines a new API for tx equalization parameters +//! on a switch link. +//! +//! ### Motivation +//! +//! - The previous API used `Option` for tap values. However, since +//! Tofino only supports updating all taps simultaneously, `None` values +//! were unwrapped to zero inside the AAL. This isn't an obvious behavior. +//! One could also expect `None` values to remain unmodified. +//! - The previous API offered no easy way to reset taps. `dpd` knows the +//! default tx eq settings for a port, but there was no way to (re)apply +//! them without an explicit command. +//! +//! ### Changes +//! +//! - All taps must be defined in a `TxEq` command. This is arguably less +//! convenient than the previous design, but it matches the hardware and +//! is less surprising. +//! - There's now an explicit way to command default taps for a link without +//! actually knowing the values. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +/// Parameters to adjust the transceiver equalization settings for a +/// link on a switch. +#[derive( + Clone, + Copy, + Debug, + Default, + Eq, + PartialEq, + Deserialize, + Serialize, + JsonSchema, +)] +pub struct TxEq { + pub pre1: i32, + pub pre2: i32, + pub main: i32, + pub post2: i32, + pub post1: i32, +} + +/// The tx equalization settings in use by a transceiver. +/// +/// - `sw` is the configured value. +/// - `hw` is what the hardware is actually using. +/// +/// These differ on transceivers that tune their own settings during run time. +#[derive( + Clone, Copy, Debug, Eq, PartialEq, Deserialize, Serialize, JsonSchema, +)] +pub struct TxEqSwHw { + pub sw: TxEq, + pub hw: TxEq, +} + +/// The tx equalization settings assigned to a port. +#[derive( + Clone, + Copy, + Debug, + Default, + Eq, + PartialEq, + Deserialize, + Serialize, + JsonSchema, +)] +pub enum TxEqConfig { + /// Apply any pre-defined tx equalization settings for this port. + /// The first config found among these options is chosen: + /// + /// - Override level: + /// - Board level: + #[default] + Preset, + + /// Apply custom tx equalization settings. All taps must be defined + /// together, which is an SDE/board driven requirement. + Custom(TxEq), +} + +impl TxEqConfig { + /// Returns custom taps if any exist or `None` if using + /// the preset config. + pub fn taps(&self) -> Option<&TxEq> { + match self { + Self::Preset => None, + Self::Custom(tx_eq) => Some(tx_eq), + } + } +} + +impl From> for TxEqConfig { + fn from(value: Option) -> Self { + match value { + None + | Some(crate::v1::port::TxEq { + pre1: None, + pre2: None, + main: None, + post2: None, + post1: None, + }) => TxEqConfig::Preset, + // This is what the tofino_asic AAL did previously. + Some(config) => TxEqConfig::Custom(TxEq { + pre2: config.pre2.unwrap_or_default(), + pre1: config.pre1.unwrap_or_default(), + main: config.main.unwrap_or_default(), + post2: config.post2.unwrap_or_default(), + post1: config.post1.unwrap_or_default(), + }), + } + } +} + +impl From for crate::v1::port::TxEq { + fn from(value: TxEq) -> Self { + Self { + pre2: Some(value.pre2), + pre1: Some(value.pre1), + main: Some(value.main), + post1: Some(value.post1), + post2: Some(value.post2), + } + } +} + +impl From for crate::v1::port::TxEqSwHw { + fn from(value: TxEqSwHw) -> Self { + Self { sw: value.sw.into(), hw: value.hw.into() } + } +} diff --git a/dpd-types/versions/src/latest.rs b/dpd-types/versions/src/latest.rs index 3b56eb39..bbe8c645 100644 --- a/dpd-types/versions/src/latest.rs +++ b/dpd-types/versions/src/latest.rs @@ -147,12 +147,15 @@ pub mod port { pub use crate::v1::port::QsfpPort; pub use crate::v1::port::RearPort; - pub use crate::v1::port::TxEq; - pub use crate::v1::port::TxEqSwHw; + + pub use crate::v12::port::PortPrbsMode; + pub use crate::v13::port::LinkSettings; pub use crate::v13::port::PortSettings; - pub use crate::v12::port::PortPrbsMode; + pub use crate::v14::txeq::TxEq; + pub use crate::v14::txeq::TxEqConfig; + pub use crate::v14::txeq::TxEqSwHw; } pub mod port_map { diff --git a/dpd-types/versions/src/lib.rs b/dpd-types/versions/src/lib.rs index 617a58c7..1b72af46 100644 --- a/dpd-types/versions/src/lib.rs +++ b/dpd-types/versions/src/lib.rs @@ -43,6 +43,8 @@ pub mod v11; pub mod v12; #[path = "allow_ddm_traffic/mod.rs"] pub mod v13; +#[path = "explicit_txeq/mod.rs"] +pub mod v14; #[path = "attached_subnets/mod.rs"] pub mod v3; #[path = "v4_over_v6_routes/mod.rs"] diff --git a/dpd/src/api_server.rs b/dpd/src/api_server.rs index 5e108a04..a7e5b576 100644 --- a/dpd/src/api_server.rs +++ b/dpd/src/api_server.rs @@ -90,13 +90,8 @@ use dropshot::WhichPage; use slog::{debug, error, info, o}; use slog_error_chain::InlineErrorChain; -#[cfg(feature = "softnpu")] -use aal::AsicOps; -#[cfg(feature = "softnpu")] -use common::ports::TxEq; - #[cfg(any(feature = "softnpu", feature = "tofino_asic"))] -use common::ports::TxEqSwHw; +use aal::{AsicOps, AsicResult}; use crate::attached_subnet; use crate::counters; @@ -2590,49 +2585,23 @@ impl DpdApi for DpdApiImpl { )) } - #[cfg(feature = "tofino_asic")] - async fn link_tx_eq_get( - rqctx: RequestContext, - path: Path, - ) -> Result>, HttpError> { - let switch: &Switch = rqctx.context(); - let params = path.into_inner(); - let port_id = params.port_id; - let link_id = params.link_id; - let port_handle = switch.link_id_to_hdl(port_id, link_id)?; - Ok(HttpResponseOk( - serdes::port_tx_eq_get(&switch.asic_hdl, port_handle) - .map_err(|e| HttpError::from(DpdError::from(e)))? - .into_iter() - .map(|t| TxEqSwHw { sw: t.sw.into(), hw: t.hw.into() }) - .collect(), - )) - } - - #[cfg(feature = "softnpu")] + #[cfg(any(feature = "tofino_asic", feature = "softnpu"))] async fn link_tx_eq_get( rqctx: RequestContext, path: Path, ) -> Result>, HttpError> { let switch: &Switch = rqctx.context(); let params = path.into_inner(); - let port_id = params.port_id; - let link_id = params.link_id; - let port_handle = switch.link_id_to_hdl(port_id, link_id)?; - let lane_cnt = switch - .asic_hdl - .port_get_lane_cnt(port_handle) - .map_err(|e| HttpError::from(DpdError::from(e)))?; + let port_handle = + switch.link_id_to_hdl(params.port_id, params.link_id)?; - let tx_eq = switch - .asic_hdl - .port_tx_eq_get(port_handle) - .map_err(|e| HttpError::from(DpdError::from(e)))?; - let softnpu_tx_eq = TxEq { main: Some(tx_eq), ..Default::default() }; Ok(HttpResponseOk( - (0..lane_cnt) - .map(|_| TxEqSwHw { sw: softnpu_tx_eq, hw: softnpu_tx_eq }) - .collect(), + switch + .asic_hdl + .port_tx_eq_get(port_handle) + .map_err(|e| HttpError::from(DpdError::from(e)))? + .collect::>() + .map_err(|e| HttpError::from(DpdError::from(e)))?, )) } @@ -2647,47 +2616,26 @@ impl DpdApi for DpdApiImpl { )) } - #[cfg(feature = "tofino_asic")] + #[cfg(any(feature = "tofino_asic", feature = "softnpu"))] async fn link_tx_eq_set( rqctx: RequestContext, path: Path, - args: TypedBody, + args: TypedBody, ) -> Result { let switch: &Switch = rqctx.context(); let params = path.into_inner(); - let port_id = params.port_id; - let link_id = params.link_id; - let settings = args.into_inner(); switch - .set_link_tx_eq(port_id, link_id, settings) + .set_link_tx_eq(params.port_id, params.link_id, args.into_inner()) .map(|_| HttpResponseUpdatedNoContent()) - .map_err(|e| e.into()) - } - - #[cfg(feature = "softnpu")] - async fn link_tx_eq_set( - rqctx: RequestContext, - path: Path, - args: TypedBody, - ) -> Result { - let switch: &Switch = rqctx.context(); - let params = path.into_inner(); - let port_id = params.port_id; - let link_id = params.link_id; - - let settings = args.into_inner(); - switch - .set_link_tx_eq(port_id, link_id, settings) - .map(|_| HttpResponseUpdatedNoContent()) - .map_err(|e| e.into()) + .map_err(HttpError::from) } #[cfg(not(any(feature = "tofino_asic", feature = "softnpu")))] async fn link_tx_eq_set( _rqctx: RequestContext, _path: Path, - _args: TypedBody, + _args: TypedBody, ) -> Result { Err(HttpError::for_unavail( None, @@ -2962,8 +2910,8 @@ impl From<&crate::link::Link> for LinkSettings { fec: l.config.fec, autoneg: l.config.autoneg, kr: l.config.kr, - tx_eq: l.tx_eq, - allow_ddm_traffic: l.config.allow_ddm_traffic, + tx_eq: l.tx_eq.taps().copied().map(|t| t.into()), + allow_ddm_traffic: false, }, addrs, } diff --git a/dpd/src/link.rs b/dpd/src/link.rs index ae60baf4..8677832b 100644 --- a/dpd/src/link.rs +++ b/dpd/src/link.rs @@ -33,7 +33,6 @@ use common::ports::PortId; use common::ports::PortMedia; use common::ports::PortPrbsMode; use common::ports::PortSpeed; -use common::ports::TxEq; use dpd_types::link::LinkCreate; use dpd_types::link::LinkFsmCounters; use dpd_types::link::LinkHistory; @@ -42,6 +41,7 @@ use dpd_types::link::LinkState; use dpd_types::link::LinkUpCounter; use dpd_types::link::LinkView; use dpd_types::link::TfportData; +use dpd_types::port::TxEqConfig; use dpd_types::serdes::Ber; use slog::debug; use slog::error; @@ -229,7 +229,7 @@ pub struct Link { /// responsible for configuring the corresponding illumos port. pub ipv6_enabled: bool, /// Optional transceiver equalization settings - pub tx_eq: Option, + pub tx_eq: TxEqConfig, /// Latest top-level port FSM state pub fsm_state: asic::PortFsmState, /// The state of the link. @@ -406,7 +406,7 @@ pub struct LinkParams { pub fec: Option, pub autoneg: bool, pub kr: bool, - pub tx_eq: Option, + pub tx_eq: TxEqConfig, pub allow_ddm_traffic: bool, } @@ -571,7 +571,7 @@ impl Switch { speed: params.speed, autoneg: params.autoneg, kr: params.kr, - tx_eq: params.tx_eq, + tx_eq: params.tx_eq.into(), fec: params.fec, allow_ddm_traffic: params.allow_ddm_traffic, }; @@ -1275,10 +1275,10 @@ impl Switch { &self, port_id: PortId, link_id: LinkId, - tx_eq: TxEq, + tx_eq: TxEqConfig, ) -> DpdResult<()> { self.link_update(port_id, link_id, |link| { - link.tx_eq = Some(tx_eq); + link.tx_eq = tx_eq; link.plumbed.tx_eq_pushed = false; self.reconciler.trigger(port_id, link_id); Ok(()) @@ -1963,7 +1963,7 @@ async fn reconcile_link( } if link.config.enabled && !link.plumbed.tx_eq_pushed { - if let Err(e) = switch.push_tx_eq(&link, &mpn) { + if let Err(e) = switch.push_tx_eq(&link, mpn.as_deref()) { record_plumb_failure( switch, &mut link, diff --git a/dpd/src/main.rs b/dpd/src/main.rs index 5fc08ec5..ab8d1f9b 100644 --- a/dpd/src/main.rs +++ b/dpd/src/main.rs @@ -275,7 +275,8 @@ impl Switch { let switch_ports = SwitchPorts::new( get_sidecar_revision(&config)?, - &config.xcvr_defaults, + config.xcvr_defaults.as_deref(), + Some(&asic_hdl), )?; let counters = counters::init(&asic_hdl) diff --git a/dpd/src/port_map.rs b/dpd/src/port_map.rs index 3a9c136a..dbef2d32 100644 --- a/dpd/src/port_map.rs +++ b/dpd/src/port_map.rs @@ -160,6 +160,11 @@ impl PortMap { pub fn port_ids(&self) -> impl Iterator { self.id_to_connector.keys() } + + /// Returns an iterator over all unique `Connector`s. + pub fn connectors(&self) -> impl Iterator { + self.connector_to_id.keys() + } } fn rev_ab_port_map() -> BTreeMap { diff --git a/dpd/src/port_settings.rs b/dpd/src/port_settings.rs index cfcea772..1f03048a 100644 --- a/dpd/src/port_settings.rs +++ b/dpd/src/port_settings.rs @@ -15,10 +15,10 @@ use common::ports::Ipv6Entry; use common::ports::PortFec; use common::ports::PortId; use common::ports::PortSpeed; -use common::ports::TxEq; use dpd_types::link::LinkId; use dpd_types::port::LinkSettings; use dpd_types::port::PortSettings; +use dpd_types::port::TxEqConfig; use slog::Logger; use slog::debug; use slog::error; @@ -101,7 +101,7 @@ struct LinkSpec { pub delete_me: bool, pub ipv4: BTreeSet, pub ipv6: BTreeSet, - pub tx_eq: Option, + pub tx_eq: TxEqConfig, pub allow_ddm_traffic: bool, } @@ -128,7 +128,7 @@ impl From<&LinkSettings> for LinkSpec { fec: l.params.fec, autoneg: l.params.autoneg, kr: l.params.kr, - tx_eq: l.params.tx_eq, + tx_eq: l.params.tx_eq.into(), delete_me: false, ipv4: l .addrs diff --git a/dpd/src/switch_port.rs b/dpd/src/switch_port.rs index 9ba24091..982cb010 100644 --- a/dpd/src/switch_port.rs +++ b/dpd/src/switch_port.rs @@ -7,6 +7,7 @@ //! Types for describing and managing physical ports on the Sidecar switch. use anyhow::Context; +use dpd_types::port::TxEqConfig; use dpd_types::port_map::BackplaneLink; use dpd_types::switch_port::Led; use dpd_types::switch_port::LedPolicy; @@ -24,6 +25,7 @@ use crate::transceivers::FakeQsfpModule; use crate::types::DpdError; use crate::types::DpdResult; use aal::AsicOps; +use aal::PortHdl; use common::ports::InternalPort; use common::ports::PortFec; use common::ports::PortId; @@ -68,42 +70,6 @@ struct XcvrDefaultsEntry { pub post2: Option, } -/// Parse the provided CSV file, extracting optional settings for a -/// subset of our supported transceivers. -pub fn load_xcvr_defaults( - csv_file: &str, -) -> anyhow::Result> { - let mut rdr = csv::ReaderBuilder::new() - .has_headers(false) - .comment(Some(b'#')) - .from_path(csv_file) - .with_context(|| format!("parsing xcvr config file {csv_file}"))?; - - let mut settings = BTreeMap::new(); - for entry in rdr.deserialize() { - let e: XcvrDefaultsEntry = entry?; - - let tx_eq = if e.pre2.is_some() - || e.pre1.is_some() - || e.main.is_some() - || e.post2.is_some() - || e.post1.is_some() - { - Some(TxEq { - pre2: e.pre2, - pre1: e.pre1, - main: e.main, - post1: e.post1, - post2: e.post2, - }) - } else { - None - }; - settings.insert(e.mpn, XcvrSettings { tx_eq, fec: e.fec }); - } - Ok(settings) -} - /// The physical ports on the switch. /// /// This is really the container for almost all of Dendrite's state about the @@ -119,13 +85,28 @@ pub struct SwitchPorts { /// we find transceivers that do not work correctly with the default values /// assigned by the SDE. Both the SDE defaults and these settings may be /// explicitly overridden by per-link settings configured by the admin. - pub xcvr_defaults: BTreeMap, + pub xcvr_mpn_defaults: BTreeMap, + /// The SDE default settings referenced above. + /// These only apply to ports without per-MPN settings or + /// an active user config. + /// + /// This may be empty if no asic handle was passed + /// to the constructor, which is often the case in tests. + pub xcvr_board_defaults: BTreeMap, } impl SwitchPorts { + /// Builds a new instance tracking metadata about the ports on a switch. + /// + /// If the asic handle is provided, this loads and caches useful board map + /// settings from the SDE. + /// + /// If the transceiver defaults file is provided, this reads and caches + /// a map of device configs that supercede what is found in the SDE board map. pub fn new( revision: SidecarRevision, - xcvr_defaults_file: &Option, + xcvr_defaults_file: Option<&str>, + handle: Option<&asic::Handle>, ) -> anyhow::Result { let port_map = PortMap::new(revision); let ports = port_map @@ -133,12 +114,20 @@ impl SwitchPorts { .copied() .map(|port_id| (port_id, Mutex::new(SwitchPort::new(port_id)))) .collect(); - let xcvr_defaults = match xcvr_defaults_file { - Some(f) => load_xcvr_defaults(f)?, - None => BTreeMap::new(), - }; - Ok(Self { port_map, ports, xcvr_defaults }) + let xcvr_mpn_defaults = xcvr_defaults_file + .map(self::load_xcvr_defaults) + .transpose()? + .unwrap_or_default(); + + // It should be fatal if this fails. On tofino, we're basically reading + // the board defaults file from a hash map in the SDE. + let xcvr_board_defaults = handle + .map(|asic| self::load_board_defaults(&port_map, asic).collect()) + .transpose()? + .unwrap_or_default(); + + Ok(Self { port_map, ports, xcvr_mpn_defaults, xcvr_board_defaults }) } pub fn verify_exists(&self, port_id: PortId) -> DpdResult<()> { @@ -281,6 +270,50 @@ impl From<&SwitchPort> for SwitchPortView { } } +/// Parse the provided CSV file, extracting optional settings for a +/// subset of our supported transceivers. +fn load_xcvr_defaults( + csv_file: &str, +) -> anyhow::Result> { + csv::ReaderBuilder::new() + .has_headers(false) + .comment(Some(b'#')) + .from_path(csv_file) + .with_context(|| format!("parsing xcvr config file {csv_file}"))? + .deserialize() + .map(|entry| { + let e: XcvrDefaultsEntry = entry?; + let tx_eq = + TxEqConfig::from(Some(dpd_types_versions::v1::port::TxEq { + pre2: e.pre2, + pre1: e.pre1, + main: e.main, + post1: e.post1, + post2: e.post2, + })); + Ok((e.mpn, XcvrSettings { tx_eq, fec: e.fec })) + }) + .collect() +} + +fn load_board_defaults( + port_map: &PortMap, + handle: &asic::Handle, +) -> impl Iterator> { + port_map + .connectors() + .copied() + .flat_map(|connector| { + handle + .connector_tx_eq_defaults(connector) + .zip(std::iter::repeat(connector)) + }) + .map(|(info, connector)| { + let (channel, settings) = info?; + Ok((PortHdl { connector, channel }, settings)) + }) +} + /// Data specific to each kind of fixed-side switch port. #[derive(Clone, Debug)] pub enum FixedSideDevice { @@ -313,34 +346,34 @@ impl crate::Switch { /// If this port's transceiver has an alternate set of tx_eq default values, /// apply them now. We first check for an explicit override value from the /// admin, then for an alternate default for this xcvr type. - pub fn push_tx_eq( - &self, - link: &Link, - mpn: &Option, - ) -> DpdResult<()> { - let port_hdl = link.port_hdl; + pub fn push_tx_eq(&self, link: &Link, mpn: Option<&str>) -> DpdResult<()> { + let tx_eq = link + .tx_eq + .taps() + .or_else(|| { + self.switch_ports.xcvr_mpn_defaults.get(mpn?)?.tx_eq.taps() + }) + .or_else(|| { + self.switch_ports.xcvr_board_defaults.get(&link.port_hdl) + }) + .copied(); + + let Some(tx_eq) = tx_eq else { + return Ok(()); + }; + + slog::debug!( + self.log, + "Applying alternate tx settings for {link} ({}): {tx_eq:?}", + mpn.unwrap_or("unknown transceiver") + ); + + self.asic_hdl + .port_tx_eq_set(link.port_hdl, &tx_eq) + .map_err(DpdError::Switch)?; - if let Some(tx_eq) = match (link.tx_eq, &mpn) { - (Some(user_defined), _) => Some(user_defined), - (None, Some(mpn)) => { - self.switch_ports.xcvr_defaults.get(mpn).and_then(|x| x.tx_eq) - } - (_, _) => None, - } { - let mpn = mpn - .clone() - .unwrap_or_else(|| "unknown transceiver".to_string()); - slog::debug!( - self.log, - "Applying alternate tx settings for {link} ({mpn}): {tx_eq:?}" - ); - self.asic_hdl - .port_tx_eq_set(port_hdl, &tx_eq) - .map_err(DpdError::Switch)?; - } Ok(()) } - /// Return the standard FEC method (if any) for the transceiver plugged into /// this port. /// @@ -353,7 +386,11 @@ impl crate::Switch { /// also fail. pub fn qsfp_default_fec(&self, qsfp_mpn: &str) -> DpdResult { slog::debug!(self.log, "looking up default FEC for {qsfp_mpn}"); - match self.switch_ports.xcvr_defaults.get(qsfp_mpn).and_then(|x| x.fec) + match self + .switch_ports + .xcvr_mpn_defaults + .get(qsfp_mpn) + .and_then(|x| x.fec) { Some(fec) => Ok(fec), None => Err(DpdError::Missing( diff --git a/dpd/src/transceivers/tofino_impl.rs b/dpd/src/transceivers/tofino_impl.rs index af46fb23..91a54cb6 100644 --- a/dpd/src/transceivers/tofino_impl.rs +++ b/dpd/src/transceivers/tofino_impl.rs @@ -3790,7 +3790,8 @@ mod tests { // We're going to alternate in the ordering of the switch ports as // enumerated by the port map, which is a permutation of the ordering in // terms of Tofino connectors. - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let mut n_ports = 0; for (i, (_, port)) in switch_ports.ports.iter().enumerate() { if let Some(device) = port.lock().await.as_backplane_mut() { @@ -3833,7 +3834,8 @@ mod tests { #[tokio::test] async fn test_handle_detect_request_qsfp() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); // Should access the controller exactly once for each module. Let's // pretend we have alternating presence. @@ -3897,7 +3899,8 @@ mod tests { #[tokio::test] async fn test_handle_detect_request_invalid_port() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); // Should never access the controller at all. let controller = Controller::default(); @@ -3919,7 +3922,8 @@ mod tests { #[tokio::test] async fn test_handle_detect_request_qsfp_invalid_interface() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); // The controller will be used to detect one module, which we'll pretend // has an unsupported interface. We'll then check that we disable power @@ -3967,7 +3971,8 @@ mod tests { #[tokio::test] async fn test_handle_presence_mask_request() { // Simulate all but the first backplane present. - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let backplane_port = PortId::Rear(RearPort::new(0).unwrap()); { switch_ports @@ -4030,7 +4035,8 @@ mod tests { #[tokio::test] async fn test_handle_lp_mode_mask_request() { // Simulate all but the first backplane in LPMode. - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let backplane_port = PortId::Rear(RearPort::new(0).unwrap()); { switch_ports @@ -4141,7 +4147,8 @@ mod tests { #[tokio::test] async fn test_handle_set_lp_mode_request_invalid_port() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); // Should never access the controller at all. let controller = Controller::default(); @@ -4175,7 +4182,8 @@ mod tests { #[tokio::test] async fn test_handle_set_lp_mode_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); // We'll expect to set every other module into LPMode. let mut sequence = mockall::Sequence::new(); @@ -4276,7 +4284,8 @@ mod tests { #[tokio::test] async fn test_handle_assert_reset_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); // We'll expect to assert reset on every other module. let mut sequence = mockall::Sequence::new(); @@ -4381,7 +4390,8 @@ mod tests { #[tokio::test] async fn test_handle_backplane_write_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); // We should never touch the controller. let controller = Controller::default(); @@ -4425,7 +4435,8 @@ mod tests { #[tokio::test] async fn test_handle_sff_8636_write_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the write operation that the SDE should send. @@ -4491,7 +4502,8 @@ mod tests { #[tokio::test] async fn test_handle_sff_8636_write_request_with_nonzero_bank_fails() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the write operation that the SDE should send. @@ -4543,7 +4555,8 @@ mod tests { #[tokio::test] async fn test_handle_sff_8636_upper_page_write_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the write operation that the SDE should send. @@ -4611,7 +4624,8 @@ mod tests { #[tokio::test] async fn test_handle_cmis_small_write_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the write operation that the SDE should send. @@ -4679,7 +4693,8 @@ mod tests { #[tokio::test] async fn test_handle_cmis_large_write_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the write operation that the SDE should send. It will @@ -4755,7 +4770,8 @@ mod tests { #[tokio::test] async fn test_handle_backplane_read_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let port_id = PortId::Rear(RearPort::new(0).unwrap()); let expected_data = vec![0, 1, 2, 3]; @@ -4806,7 +4822,8 @@ mod tests { #[tokio::test] async fn test_handle_sff_8636_read_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the read operation that the SDE should send. @@ -4863,7 +4880,8 @@ mod tests { #[tokio::test] async fn test_handle_sff_8636_upper_page_read_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the read operation that the SDE should send. @@ -4925,7 +4943,8 @@ mod tests { #[tokio::test] async fn test_handle_sff_8636_read_request_with_nonzero_bank_fails() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the read operation that the SDE should send. @@ -4971,7 +4990,8 @@ mod tests { #[tokio::test] async fn test_handle_cmis_small_read_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the read operation that the SDE should send. @@ -5029,7 +5049,8 @@ mod tests { #[tokio::test] async fn test_handle_cmis_large_read_request() { - let switch_ports = SwitchPorts::new(SidecarRevision::B, &None).unwrap(); + let switch_ports = + SwitchPorts::new(SidecarRevision::B, None, None).unwrap(); let log = logger(); // Describe the read operation that the SDE should send. It will diff --git a/openapi/dpd/dpd-latest.json b/openapi/dpd/dpd-latest.json index d1a5660e..40fb666c 120000 --- a/openapi/dpd/dpd-latest.json +++ b/openapi/dpd/dpd-latest.json @@ -1 +1 @@ -dpd-13.0.0-5db8bd.json \ No newline at end of file +dpd-14.0.0-70b4ed.json \ No newline at end of file diff --git a/swadm/src/link.rs b/swadm/src/link.rs index 47e7e34a..1780dad5 100644 --- a/swadm/src/link.rs +++ b/swadm/src/link.rs @@ -221,28 +221,63 @@ pub enum GetSerdes { }, } +/// Diagnostic commands to update the settings of a link's SERDES #[derive(Debug, Subcommand)] -/// Diagnostic commands to updated the settings of a link's SERDES pub enum SetSerdes { - /// Update the tx equalization settings for this port. Only the main setting is - /// required. All others will default to 0. Note: to set a negative value, - /// you must use the "=" option syntax. e.g., "--pre1=-1" - #[clap(visible_alias = "txeq")] + /// Update the tx equalization settings for this port. + #[clap( + visible_alias = "txeq", + after_help = r#" +Examples: + Set every tx eq tap on this link to -1: + swadm link serdes set txeq "rear0/0" all -1 + + Reset the link tx eq settings to their initial state: + swadm link serdes set txeq "rear0/0" default + + Declare full tx eq settings on this link: + swadm link serdes set txeq "rear0/0" taps --pre2 0 --pre1 0 --main 25 --post1 -5 --post2 0"# + )] TxEq { /// The link path, specified as `switch_port/link`. /// /// For example `rear0/0` is the first link on the rear0 switch port. link_path: LinkPath, - #[clap(long)] - pre2: Option, - #[clap(long)] - pre1: Option, - #[clap(long)] - main: Option, - #[clap(long)] - post1: Option, - #[clap(long)] - post2: Option, + + #[command(subcommand)] + options: TxEqOptions, + }, +} + +#[derive(Subcommand, Debug)] +pub enum TxEqOptions { + /// Commands the switch to reset its tx equalization parameters on this + /// link to whatever was assigned at start time. + Default, + + /// Configures the same value for every tap on this link. + All { + /// Command every tap on the link to this signed integer. + #[clap(value_name = "TAP", allow_hyphen_values(true))] + all: i32, + }, + + /// Configures each tap on the link. All five must be declared. + Taps { + #[clap(long, allow_hyphen_values(true))] + pre2: i32, + + #[clap(long, allow_hyphen_values(true))] + pre1: i32, + + #[clap(long, allow_hyphen_values(true))] + main: i32, + + #[clap(long, allow_hyphen_values(true))] + post1: i32, + + #[clap(long, allow_hyphen_values(true))] + post2: i32, }, } @@ -1151,8 +1186,8 @@ macro_rules! print_txeq_fields { ($label:expr, $all:ident, $($field_path:ident).+) => { print!("{:6}", $label); for lane in & $all { - let sw = lane.sw.$($field_path).+.unwrap_or(0); - let hw = lane.hw.$($field_path).+.unwrap_or(0); + let sw = lane.sw.$($field_path).+; + let hw = lane.hw.$($field_path).+; print!(" {:>3} ({:>3})", sw.to_string(), hw.to_string()); } println!(); @@ -1183,21 +1218,35 @@ async fn link_serdes_tx_eq_get( Ok(()) } -#[allow(clippy::too_many_arguments)] async fn link_serdes_tx_eq_set( client: &Client, link: LinkPath, - pre2: Option, - pre1: Option, - main: Option, - post1: Option, - post2: Option, + args: TxEqOptions, ) -> anyhow::Result<()> { - let settings = types::TxEq { pre2, pre1, main, post1, post2 }; - let port = link.port_id; - let link = link.link_id; + let settings = match args { + TxEqOptions::Default => types::TxEqConfig::Preset, + TxEqOptions::All { all: tap } => { + types::TxEqConfig::Custom(types::TxEq2 { + pre2: tap, + pre1: tap, + main: tap, + post1: tap, + post2: tap, + }) + } + TxEqOptions::Taps { pre2, pre1, main, post1, post2 } => { + types::TxEqConfig::Custom(types::TxEq2 { + pre2, + pre1, + main, + post1, + post2, + }) + } + }; + client - .link_tx_eq_set(&port, &link, &settings) + .link_tx_eq_set(&link.port_id, &link.link_id, &settings) .await .map(|r| r.into_inner()) .context("failed to set tx eq settings")?; @@ -2113,19 +2162,10 @@ pub async fn link_cmd(client: &Client, link: Link) -> anyhow::Result<()> { } }, Serdes::Set { cmd: set } => match set { - SetSerdes::TxEq { - link_path, - pre2, - pre1, - main, - post1, - post2, - } => { - link_serdes_tx_eq_set( - client, link_path, pre2, pre1, main, post1, post2, - ) - .await - .context("failed to set tx eq values")?; + SetSerdes::TxEq { link_path, options } => { + link_serdes_tx_eq_set(client, link_path, options) + .await + .context("failed to set tx eq values")?; } }, }, @@ -2146,6 +2186,7 @@ pub async fn link_cmd(client: &Client, link: Link) -> anyhow::Result<()> { post1, post2, } => { + // TODO::cory: should this be migrated or not? let port_id = &link.port_id; let mut body = types::PortSettings { links: HashMap::default() };