From dae76dede9a838243f537150b5cc10fbce113f55 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 13 Apr 2026 12:12:32 +0200 Subject: [PATCH 01/25] ixgbe: remove legacy rx Similarly, like for virtual function in ixgbevf, remove ixgbe_construct_skb(), the legacy-rx private flag, and all ring_uses_build_skb() conditionals. build_skb is now the only RX code path. This is a preparation for the conversion to libeth and page_pool based Rx buffer management. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 24 +-- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 13 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 176 ++---------------- 3 files changed, 27 insertions(+), 186 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index dce4936708eb44..0abc460b55765d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -129,14 +129,7 @@ static inline int ixgbe_skb_pad(void) #define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) #endif -/* - * NOTE: netdev_alloc_skb reserves up to 64 bytes, NET_IP_ALIGN means we - * reserve 64 more, and skb_shared_info adds an additional 320 bytes more, - * this adds up to 448 bytes of extra data. - * - * Since netdev_alloc_skb now allocates a page fragment we can use a value - * of 256 and the resultant skb will have a truesize of 960 or less. - */ +/* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -312,7 +305,6 @@ struct ixgbe_rx_queue_stats { enum ixgbe_ring_state_t { __IXGBE_RX_3K_BUFFER, - __IXGBE_RX_BUILD_SKB_ENABLED, __IXGBE_RX_RSC_ENABLED, __IXGBE_RX_CSUM_UDP_ZERO_ERR, __IXGBE_RX_FCOE, @@ -324,9 +316,6 @@ enum ixgbe_ring_state_t { __IXGBE_TX_DISABLED, }; -#define ring_uses_build_skb(ring) \ - test_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &(ring)->state) - struct ixgbe_fwd_adapter { unsigned long active_vlans[BITS_TO_LONGS(VLAN_N_VID)]; struct net_device *netdev; @@ -397,7 +386,6 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; - u16 rx_offset; struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; @@ -456,8 +444,7 @@ static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) return IXGBE_RXBUFFER_3K; #if (PAGE_SIZE < 8192) - if (ring_uses_build_skb(ring)) - return IXGBE_MAX_2K_FRAME_BUILD_SKB; + return IXGBE_MAX_2K_FRAME_BUILD_SKB; #endif return IXGBE_RXBUFFER_2K; } @@ -472,6 +459,11 @@ static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) } #define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) +static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) +{ + return IXGBE_SKB_PAD; +} + #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 @@ -672,7 +664,7 @@ struct ixgbe_adapter { #define IXGBE_FLAG2_VLAN_PROMISC BIT(13) #define IXGBE_FLAG2_EEE_CAPABLE BIT(14) #define IXGBE_FLAG2_EEE_ENABLED BIT(15) -#define IXGBE_FLAG2_RX_LEGACY BIT(16) +/* BIT16 used to be reserved for legacy RX flag */ #define IXGBE_FLAG2_IPSEC_ENABLED BIT(17) #define IXGBE_FLAG2_VF_IPSEC_ENABLED BIT(18) #define IXGBE_FLAG2_AUTO_DISABLE_VF BIT(19) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 2ad81f687a844a..eaf50ce47fe5e1 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -132,11 +132,9 @@ static const char ixgbe_gstrings_test[][ETH_GSTRING_LEN] = { #define IXGBE_TEST_LEN sizeof(ixgbe_gstrings_test) / ETH_GSTRING_LEN static const char ixgbe_priv_flags_strings[][ETH_GSTRING_LEN] = { -#define IXGBE_PRIV_FLAGS_LEGACY_RX BIT(0) - "legacy-rx", -#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(1) +#define IXGBE_PRIV_FLAGS_VF_IPSEC_EN BIT(0) "vf-ipsec", -#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(2) +#define IXGBE_PRIV_FLAGS_AUTO_DISABLE_VF BIT(1) "mdd-disable-vf", }; @@ -3667,9 +3665,6 @@ static u32 ixgbe_get_priv_flags(struct net_device *netdev) struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); u32 priv_flags = 0; - if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - priv_flags |= IXGBE_PRIV_FLAGS_LEGACY_RX; - if (adapter->flags2 & IXGBE_FLAG2_VF_IPSEC_ENABLED) priv_flags |= IXGBE_PRIV_FLAGS_VF_IPSEC_EN; @@ -3685,10 +3680,6 @@ static int ixgbe_set_priv_flags(struct net_device *netdev, u32 priv_flags) unsigned int flags2 = adapter->flags2; unsigned int i; - flags2 &= ~IXGBE_FLAG2_RX_LEGACY; - if (priv_flags & IXGBE_PRIV_FLAGS_LEGACY_RX) - flags2 |= IXGBE_FLAG2_RX_LEGACY; - flags2 &= ~IXGBE_FLAG2_VF_IPSEC_ENABLED; if (priv_flags & IXGBE_PRIV_FLAGS_VF_IPSEC_EN) flags2 |= IXGBE_FLAG2_VF_IPSEC_ENABLED; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c58051e4350be2..0533fbb92339eb 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1730,11 +1730,6 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, } } -static unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) -{ - return ring_uses_build_skb(rx_ring) ? IXGBE_SKB_PAD : 0; -} - static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *bi) { @@ -1771,7 +1766,7 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, bi->dma = dma; bi->page = page; - bi->page_offset = rx_ring->rx_offset; + bi->page_offset = ixgbe_rx_offset(rx_ring); page_ref_add(page, USHRT_MAX - 1); bi->pagecnt_bias = USHRT_MAX; rx_ring->rx_stats.alloc_rx_page++; @@ -1979,48 +1974,6 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, return true; } -/** - * ixgbe_pull_tail - ixgbe specific version of skb_pull_tail - * @rx_ring: rx descriptor ring packet is being transacted on - * @skb: pointer to current skb being adjusted - * - * This function is an ixgbe specific version of __pskb_pull_tail. The - * main difference between this version and the original function is that - * this function can make several assumptions about the state of things - * that allow for significant optimizations versus the standard function. - * As a result we can do things like drop a frag and maintain an accurate - * truesize for the skb. - */ -static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) -{ - skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; - unsigned char *va; - unsigned int pull_len; - - /* - * it is valid to use page_address instead of kmap since we are - * working with pages allocated out of the lomem pool per - * alloc_page(GFP_ATOMIC) - */ - va = skb_frag_address(frag); - - /* - * we need the header to contain the greater of either ETH_HLEN or - * 60 bytes if the skb->len is less than 60 for skb_pad. - */ - pull_len = eth_get_headlen(skb->dev, va, IXGBE_RX_HDR_SIZE); - - /* align pull length to size of long to optimize memcpy performance */ - skb_copy_to_linear_data(skb, va, ALIGN(pull_len, sizeof(long))); - - /* update all of the pointers */ - skb_frag_size_sub(frag, pull_len); - skb_frag_off_add(frag, pull_len); - skb->data_len -= pull_len; - skb->tail += pull_len; -} - /** * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB * @rx_ring: rx descriptor ring packet is being transacted on @@ -2034,24 +1987,12 @@ static void ixgbe_pull_tail(struct ixgbe_ring *rx_ring, static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, struct sk_buff *skb) { - if (ring_uses_build_skb(rx_ring)) { - unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; - unsigned long offset = (unsigned long)(skb->data) & mask; - - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - offset, - skb_headlen(skb), - DMA_FROM_DEVICE); - } else { - skb_frag_t *frag = &skb_shinfo(skb)->frags[0]; + unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; + unsigned long offset = (unsigned long)(skb->data) & mask; - dma_sync_single_range_for_cpu(rx_ring->dev, - IXGBE_CB(skb)->dma, - skb_frag_off(frag), - skb_frag_size(frag), - DMA_FROM_DEVICE); - } + dma_sync_single_range_for_cpu(rx_ring->dev, IXGBE_CB(skb)->dma, + offset, skb_headlen(skb), + DMA_FROM_DEVICE); /* If the page was released, just unmap it. */ if (unlikely(IXGBE_CB(skb)->page_released)) { @@ -2101,10 +2042,6 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return true; } - /* place header in linear portion of buffer */ - if (!skb_headlen(skb)) - ixgbe_pull_tail(rx_ring, skb); - #ifdef IXGBE_FCOE /* do not attempt to pad FCoE Frames as this will disrupt DDP */ if (ixgbe_rx_is_fcoe(rx_ring, rx_desc)) @@ -2192,13 +2129,9 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, * @skb: sk_buff to place the data into * @size: size of data in rx_buffer * - * This function will add the data contained in rx_buffer->page to the skb. - * This is done either through a direct copy if the data in the buffer is - * less than the skb header size, otherwise it will just attach the page as - * a frag to the skb. - * - * The function will then update the page offset if necessary and return - * true if the buffer can be reused by the adapter. + * This function attaches the page contained in rx_buffer to the skb as + * a fragment. It then updates the page offset based on the configured + * PAGE_SIZE to prepare the buffer for the next packet. **/ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, @@ -2208,9 +2141,7 @@ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, #if (PAGE_SIZE < 8192) unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; #else - unsigned int truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) : - SKB_DATA_ALIGN(size); + unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); #endif skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); @@ -2291,65 +2222,6 @@ static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, rx_buffer->skb = NULL; } -static struct sk_buff *ixgbe_construct_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int size = xdp->data_end - xdp->data; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else - unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); -#endif - struct sk_buff *skb; - - /* prefetch first cache line of first page */ - net_prefetch(xdp->data); - - /* Note, we get here by enabling legacy-rx via: - * - * ethtool --set-priv-flags legacy-rx on - * - * In this mode, we currently get 0 extra XDP headroom as - * opposed to having legacy-rx off, where we process XDP - * packets going to stack via ixgbe_build_skb(). The latter - * provides us currently with 192 bytes of headroom. - * - * For ixgbe_construct_skb() mode it means that the - * xdp->data_meta will always point to xdp->data, since - * the helper cannot expand the head. Should this ever - * change in future for legacy-rx mode on, then lets also - * add xdp->data_meta handling here. - */ - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, IXGBE_RX_HDR_SIZE); - if (unlikely(!skb)) - return NULL; - - if (size > IXGBE_RX_HDR_SIZE) { - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - skb_add_rx_frag(skb, 0, rx_buffer->page, - xdp->data - page_address(rx_buffer->page), - size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - } else { - memcpy(__skb_put(skb, size), - xdp->data, ALIGN(size, sizeof(long))); - rx_buffer->pagecnt_bias++; - } - - return skb; -} - static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, struct xdp_buff *xdp, @@ -2460,10 +2332,8 @@ static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, #if (PAGE_SIZE < 8192) truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ #else - truesize = rx_ring->rx_offset ? - SKB_DATA_ALIGN(rx_ring->rx_offset + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) : - SKB_DATA_ALIGN(size); + truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) + + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); #endif return truesize; } @@ -2504,7 +2374,6 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); - unsigned int offset = rx_ring->rx_offset; unsigned int xdp_xmit = 0; struct xdp_buff xdp; int xdp_res = 0; @@ -2543,6 +2412,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* retrieve a buffer from the ring */ if (!skb) { + unsigned int offset = ixgbe_rx_offset(rx_ring); unsigned char *hard_start; hard_start = page_address(rx_buffer->page) + @@ -2567,12 +2437,9 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_bytes += size; } else if (skb) { ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size); - } else if (ring_uses_build_skb(rx_ring)) { + } else { skb = ixgbe_build_skb(rx_ring, rx_buffer, &xdp, rx_desc); - } else { - skb = ixgbe_construct_skb(rx_ring, rx_buffer, - &xdp, rx_desc); } /* exit if we failed to retrieve a buffer */ @@ -4557,15 +4424,12 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, * This can happen in SRIOV mode when the MTU of the VF is * higher than the MTU of the PF. */ - if (ring_uses_build_skb(ring) && - !test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) + if (!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | IXGBE_RXDCTL_RLPML_EN; #endif } - ring->rx_offset = ixgbe_rx_offset(ring); - if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); @@ -4734,7 +4598,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) clear_ring_rsc_enabled(rx_ring); clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - clear_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) set_ring_rsc_enabled(rx_ring); @@ -4742,11 +4605,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - if (adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) - continue; - - set_bit(__IXGBE_RX_BUILD_SKB_ENABLED, &rx_ring->state); - #if (PAGE_SIZE < 8192) if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); @@ -7199,7 +7057,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0) goto err; - WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog); + rx_ring->xdp_prog = adapter->xdp_prog; return 0; err: @@ -7335,7 +7193,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) */ static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter) { - if (PAGE_SIZE >= 8192 || adapter->flags2 & IXGBE_FLAG2_RX_LEGACY) + if (PAGE_SIZE >= 8192) return IXGBE_RXBUFFER_2K; else return IXGBE_RXBUFFER_3K; From 42f9bf5f2cbb4861bdf243647de05523087a6b8b Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 7 May 2026 23:33:54 +0200 Subject: [PATCH 02/25] ixgbe: do not share pages between packets Stop sharing pages between multiple Rx buffers. Convert ixgbe Rx path from page-sharing/recycling to a simple one-page-per-packet model. Remove logic that refers to page-sharing buffer management. This is a preparatory step before converting the Rx path to page_pool and libeth. Fixed XDP_DROP/ABORTED: reaches ixgbe_put_rx_buffer() but page is never freed. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 91 +--- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 8 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 413 ++++-------------- 3 files changed, 86 insertions(+), 426 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 0abc460b55765d..ef57e6029e492f 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -69,65 +69,13 @@ /* Supported Rx Buffer Sizes */ #define IXGBE_RXBUFFER_256 256 /* Used for skb receive header */ -#define IXGBE_RXBUFFER_1536 1536 -#define IXGBE_RXBUFFER_2K 2048 #define IXGBE_RXBUFFER_3K 3072 #define IXGBE_RXBUFFER_4K 4096 #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ #define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) -/* Attempt to maximize the headroom available for incoming frames. We - * use a 2K buffer for receives and need 1536/1534 to store the data for - * the frame. This leaves us with 512 bytes of room. From that we need - * to deduct the space needed for the shared info and the padding needed - * to IP align the frame. - * - * Note: For cache line sizes 256 or larger this value is going to end - * up negative. In these cases we should fall back to the 3K - * buffers. - */ -#if (PAGE_SIZE < 8192) -#define IXGBE_MAX_2K_FRAME_BUILD_SKB (IXGBE_RXBUFFER_1536 - NET_IP_ALIGN) -#define IXGBE_2K_TOO_SMALL_WITH_PADDING \ -((NET_SKB_PAD + IXGBE_RXBUFFER_1536) > SKB_WITH_OVERHEAD(IXGBE_RXBUFFER_2K)) - -static inline int ixgbe_compute_pad(int rx_buf_len) -{ - int page_size, pad_size; - - page_size = ALIGN(rx_buf_len, PAGE_SIZE / 2); - pad_size = SKB_WITH_OVERHEAD(page_size) - rx_buf_len; - - return pad_size; -} - -static inline int ixgbe_skb_pad(void) -{ - int rx_buf_len; - - /* If a 2K buffer cannot handle a standard Ethernet frame then - * optimize padding for a 3K buffer instead of a 1.5K buffer. - * - * For a 3K buffer we need to add enough padding to allow for - * tailroom due to NET_IP_ALIGN possibly shifting us out of - * cache-line alignment. - */ - if (IXGBE_2K_TOO_SMALL_WITH_PADDING) - rx_buf_len = IXGBE_RXBUFFER_3K + SKB_DATA_ALIGN(NET_IP_ALIGN); - else - rx_buf_len = IXGBE_RXBUFFER_1536; - - /* if needed make room for NET_IP_ALIGN */ - rx_buf_len -= NET_IP_ALIGN; - - return ixgbe_compute_pad(rx_buf_len); -} - -#define IXGBE_SKB_PAD ixgbe_skb_pad() -#else #define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) -#endif /* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 @@ -267,11 +215,9 @@ struct ixgbe_tx_buffer { struct ixgbe_rx_buffer { union { struct { - struct sk_buff *skb; dma_addr_t dma; struct page *page; __u32 page_offset; - __u16 pagecnt_bias; }; struct { bool discard; @@ -304,7 +250,6 @@ struct ixgbe_rx_queue_stats { #define IXGBE_TS_HDR_LEN 8 enum ixgbe_ring_state_t { - __IXGBE_RX_3K_BUFFER, __IXGBE_RX_RSC_ENABLED, __IXGBE_RX_CSUM_UDP_ZERO_ERR, __IXGBE_RX_FCOE, @@ -371,12 +316,9 @@ struct ixgbe_ring { unsigned long last_rx_timestamp; - union { - u16 next_to_alloc; - struct { - u8 atr_sample_rate; - u8 atr_count; - }; + struct { + u8 atr_sample_rate; + u8 atr_count; }; u8 dcb_tc; @@ -386,6 +328,7 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; + struct sk_buff *skb; /* partial Rx skb across non-EOP */ struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; @@ -434,31 +377,6 @@ struct ixgbe_ring_feature { #define IXGBE_82599_VMDQ_4Q_MASK 0x7C #define IXGBE_82599_VMDQ_2Q_MASK 0x7E -/* - * FCoE requires that all Rx buffers be over 2200 bytes in length. Since - * this is twice the size of a half page we need to double the page order - * for FCoE enabled Rx queues. - */ -static inline unsigned int ixgbe_rx_bufsz(struct ixgbe_ring *ring) -{ - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return IXGBE_RXBUFFER_3K; -#if (PAGE_SIZE < 8192) - return IXGBE_MAX_2K_FRAME_BUILD_SKB; -#endif - return IXGBE_RXBUFFER_2K; -} - -static inline unsigned int ixgbe_rx_pg_order(struct ixgbe_ring *ring) -{ -#if (PAGE_SIZE < 8192) - if (test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - return 1; -#endif - return 0; -} -#define ixgbe_rx_pg_size(_ring) (PAGE_SIZE << ixgbe_rx_pg_order(_ring)) - static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) { return IXGBE_SKB_PAD; @@ -911,7 +829,6 @@ struct ixgbe_cb { }; dma_addr_t dma; u16 append_cnt; - bool page_released; }; #define IXGBE_CB(skb) ((struct ixgbe_cb *)(skb)->cb) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index eaf50ce47fe5e1..b9db6ee9668e93 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1961,6 +1961,10 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) hw->mac.ops.disable_rx(hw); + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); rctl = IXGBE_READ_REG(&adapter->hw, IXGBE_RXCTRL); @@ -2119,7 +2123,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, /* sync Rx buffer for CPU read */ dma_sync_single_for_cpu(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* verify contents of skb */ @@ -2131,7 +2135,7 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, /* sync Rx buffer for device write */ dma_sync_single_for_device(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* increment Rx next to clean counter */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 0533fbb92339eb..73e46aea5c671d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -592,7 +592,6 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) struct my_u0 { u64 a; u64 b; } *u0; struct ixgbe_ring *rx_ring; union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer_info; int i = 0; if (!netif_msg_hw(adapter)) @@ -790,14 +789,12 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) pr_info("------------------------------------\n"); pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index); pr_info("------------------------------------\n"); - pr_info("%s%s%s\n", + pr_info("%s%s\n", "R [desc] [ PktBuf A0] ", - "[ HeadBuf DD] [bi->dma ] [bi->skb ] ", - "<-- Adv Rx Read format"); - pr_info("%s%s%s\n", + "[ HeadBuf DD] <-- Adv Rx Read format"); + pr_info("%s%s\n", "RWB[desc] [PcsmIpSHl PtRs] ", - "[vl er S cks ln] ---------------- [bi->skb ] ", - "<-- Adv Rx Write-Back format"); + "[vl er S cks ln] <-- Adv Rx Write-Back format"); for (i = 0; i < rx_ring->count; i++) { const char *ring_desc; @@ -809,34 +806,30 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) else ring_desc = ""; - rx_buffer_info = &rx_ring->rx_buffer_info[i]; rx_desc = IXGBE_RX_DESC(rx_ring, i); u0 = (struct my_u0 *)rx_desc; if (rx_desc->wb.upper.length) { - /* Descriptor Done */ - pr_info("RWB[0x%03X] %016llX %016llX ---------------- %p%s\n", + pr_info("RWB[0x%03X] %016llX %016llX%s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - rx_buffer_info->skb, ring_desc); } else { - pr_info("R [0x%03X] %016llX %016llX %016llX %p%s\n", + pr_info("R [0x%03X] %016llX %016llX %016llX %s\n", i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - (u64)rx_buffer_info->dma, - rx_buffer_info->skb, + (u64)rx_ring->rx_buffer_info[i].dma, ring_desc); + } - if (netif_msg_pktdata(adapter) && - rx_buffer_info->dma) { - print_hex_dump(KERN_INFO, "", - DUMP_PREFIX_ADDRESS, 16, 1, - page_address(rx_buffer_info->page) + - rx_buffer_info->page_offset, - ixgbe_rx_bufsz(rx_ring), true); - } + if (netif_msg_pktdata(adapter) && + rx_ring->rx_buffer_info[i].dma) { + print_hex_dump(KERN_INFO, "", + DUMP_PREFIX_ADDRESS, 16, 1, + page_address(rx_ring->rx_buffer_info[i].page) + + rx_ring->rx_buffer_info[i].page_offset, + IXGBE_RXBUFFER_3K, true); } } } @@ -1736,29 +1729,19 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, struct page *page = bi->page; dma_addr_t dma; - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) - return true; - /* alloc new page for storage */ - page = dev_alloc_pages(ixgbe_rx_pg_order(rx_ring)); + page = dev_alloc_page(); if (unlikely(!page)) { rx_ring->rx_stats.alloc_rx_page_failed++; return false; } /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); + dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - /* - * if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ if (dma_mapping_error(rx_ring->dev, dma)) { - __free_pages(page, ixgbe_rx_pg_order(rx_ring)); + __free_page(page); rx_ring->rx_stats.alloc_rx_page_failed++; return false; @@ -1766,9 +1749,8 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, bi->dma = dma; bi->page = page; - bi->page_offset = ixgbe_rx_offset(rx_ring); - page_ref_add(page, USHRT_MAX - 1); - bi->pagecnt_bias = USHRT_MAX; + bi->page_offset = rx_ring->xdp_prog ? XDP_PACKET_HEADROOM : + IXGBE_SKB_PAD; rx_ring->rx_stats.alloc_rx_page++; return true; @@ -1784,7 +1766,6 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *bi; u16 i = rx_ring->next_to_use; - u16 bufsz; /* nothing to do */ if (!cleaned_count) @@ -1794,15 +1775,14 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) bi = &rx_ring->rx_buffer_info[i]; i -= rx_ring->count; - bufsz = ixgbe_rx_bufsz(rx_ring); - do { if (!ixgbe_alloc_mapped_page(rx_ring, bi)) break; /* sync the buffer for use by the device */ dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, bufsz, + bi->page_offset, + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* @@ -1831,9 +1811,6 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) if (rx_ring->next_to_use != i) { rx_ring->next_to_use = i; - /* update next to alloc since we have filled the ring */ - rx_ring->next_to_alloc = i; - /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, @@ -1967,41 +1944,11 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) return false; - /* place skb in next buffer to be received */ - rx_ring->rx_buffer_info[ntc].skb = skb; rx_ring->rx_stats.non_eop_descs++; return true; } -/** - * ixgbe_dma_sync_frag - perform DMA sync for first frag of SKB - * @rx_ring: rx descriptor ring packet is being transacted on - * @skb: pointer to current skb being updated - * - * This function provides a basic DMA sync up for the first fragment of an - * skb. The reason for doing this is that the first fragment cannot be - * unmapped until we have reached the end of packet descriptor for a buffer - * chain. - */ -static void ixgbe_dma_sync_frag(struct ixgbe_ring *rx_ring, - struct sk_buff *skb) -{ - unsigned long mask = (unsigned long)ixgbe_rx_pg_size(rx_ring) - 1; - unsigned long offset = (unsigned long)(skb->data) & mask; - - dma_sync_single_range_for_cpu(rx_ring->dev, IXGBE_CB(skb)->dma, - offset, skb_headlen(skb), - DMA_FROM_DEVICE); - - /* If the page was released, just unmap it. */ - if (unlikely(IXGBE_CB(skb)->page_released)) { - dma_unmap_page_attrs(rx_ring->dev, IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } -} /** * ixgbe_cleanup_headers - Correct corrupted or empty headers @@ -2048,80 +1995,10 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; #endif - /* if eth_skb_pad returns an error the skb was freed */ - if (eth_skb_pad(skb)) - return true; return false; } -/** - * ixgbe_reuse_rx_page - page flip buffer and store it back on the ring - * @rx_ring: rx descriptor ring to store buffers on - * @old_buff: donor buffer to have page reused - * - * Synchronizes page for reuse by the adapter - **/ -static void ixgbe_reuse_rx_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *old_buff) -{ - struct ixgbe_rx_buffer *new_buff; - u16 nta = rx_ring->next_to_alloc; - - new_buff = &rx_ring->rx_buffer_info[nta]; - - /* update, and store next to alloc */ - nta++; - rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0; - - /* Transfer page from old buffer to new buffer. - * Move each member individually to avoid possible store - * forwarding stalls and unnecessary copy of skb. - */ - new_buff->dma = old_buff->dma; - new_buff->page = old_buff->page; - new_buff->page_offset = old_buff->page_offset; - new_buff->pagecnt_bias = old_buff->pagecnt_bias; -} - -static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, - int rx_buffer_pgcnt) -{ - unsigned int pagecnt_bias = rx_buffer->pagecnt_bias; - struct page *page = rx_buffer->page; - - /* avoid re-using remote and pfmemalloc pages */ - if (!dev_page_is_reusable(page)) - return false; - -#if (PAGE_SIZE < 8192) - /* if we are only owner of page we can reuse it */ - if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1)) - return false; -#else - /* The last offset is a bit aggressive in that we assume the - * worst case of FCoE being enabled and using a 3K buffer. - * However this should have minimal impact as the 1K extra is - * still less than one buffer in size. - */ -#define IXGBE_LAST_OFFSET \ - (SKB_WITH_OVERHEAD(PAGE_SIZE) - IXGBE_RXBUFFER_3K) - if (rx_buffer->page_offset > IXGBE_LAST_OFFSET) - return false; -#endif - - /* If we have drained the page fragment pool we need to update - * the pagecnt_bias and page count so that we fully restock the - * number of references the driver holds. - */ - if (unlikely(pagecnt_bias == 1)) { - page_ref_add(page, USHRT_MAX - 1); - rx_buffer->pagecnt_bias = USHRT_MAX; - } - - return true; -} - /** * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff * @rx_ring: rx descriptor ring to transact packets on @@ -2129,97 +2006,42 @@ static bool ixgbe_can_reuse_rx_page(struct ixgbe_rx_buffer *rx_buffer, * @skb: sk_buff to place the data into * @size: size of data in rx_buffer * - * This function attaches the page contained in rx_buffer to the skb as - * a fragment. It then updates the page offset based on the configured - * PAGE_SIZE to prepare the buffer for the next packet. + * This function will add the data contained in rx_buffer->page to the skb. **/ static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, struct ixgbe_rx_buffer *rx_buffer, struct sk_buff *skb, unsigned int size) { -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); -#endif + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, rx_buffer->page_offset, size, truesize); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif } static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff **skb, - const unsigned int size, - int *rx_buffer_pgcnt) + const unsigned int size) { struct ixgbe_rx_buffer *rx_buffer; rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - *rx_buffer_pgcnt = -#if (PAGE_SIZE < 8192) - page_count(rx_buffer->page); -#else - 0; -#endif prefetchw(rx_buffer->page); - *skb = rx_buffer->skb; - - /* Delay unmapping of the first packet. It carries the header - * information, HW may still access the header after the writeback. - * Only unmap it when EOP is reached - */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) { - if (!*skb) - goto skip_sync; - } else { - if (*skb) - ixgbe_dma_sync_frag(rx_ring, *skb); - } - /* we are reusing so sync this buffer for CPU use */ + /* sync this buffer for CPU use */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, size, DMA_FROM_DEVICE); -skip_sync: - rx_buffer->pagecnt_bias--; return rx_buffer; } static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - int rx_buffer_pgcnt) + struct ixgbe_rx_buffer *rx_buffer) { - if (ixgbe_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) { - /* hand second half of page back to the ring */ - ixgbe_reuse_rx_page(rx_ring, rx_buffer); - } else { - if (skb && IXGBE_CB(skb)->dma == rx_buffer->dma) { - /* the page has been released from the ring */ - IXGBE_CB(skb)->page_released = true; - } else { - /* we are not reusing the buffer so unmap it */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - } - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); - } - - /* clear contents of rx_buffer */ - rx_buffer->page = NULL; - rx_buffer->skb = NULL; + dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, + DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); } static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, @@ -2228,13 +2050,9 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc) { unsigned int metasize = xdp->data - xdp->data_meta; -#if (PAGE_SIZE < 8192) - unsigned int truesize = ixgbe_rx_pg_size(rx_ring) / 2; -#else unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + SKB_DATA_ALIGN(xdp->data_end - xdp->data_hard_start); -#endif struct sk_buff *skb; /* Prefetch first cache line of first page. If xdp->data_meta @@ -2259,13 +2077,6 @@ static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) IXGBE_CB(skb)->dma = rx_buffer->dma; - /* update buffer offset */ -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif - return skb; } @@ -2324,31 +2135,6 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, return result; } -static unsigned int ixgbe_rx_frame_truesize(struct ixgbe_ring *rx_ring, - unsigned int size) -{ - unsigned int truesize; - -#if (PAGE_SIZE < 8192) - truesize = ixgbe_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */ -#else - truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size) + - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); -#endif - return truesize; -} - -static void ixgbe_rx_buffer_flip(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - unsigned int size) -{ - unsigned int truesize = ixgbe_rx_frame_truesize(rx_ring, size); -#if (PAGE_SIZE < 8192) - rx_buffer->page_offset ^= truesize; -#else - rx_buffer->page_offset += truesize; -#endif -} /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf @@ -2367,7 +2153,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, const int budget) { - unsigned int total_rx_bytes = 0, total_rx_packets = 0, frame_sz = 0; + unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; #ifdef IXGBE_FCOE int ddp_bytes; @@ -2376,19 +2162,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, u16 cleaned_count = ixgbe_desc_unused(rx_ring); unsigned int xdp_xmit = 0; struct xdp_buff xdp; + struct sk_buff *skb = rx_ring->skb; int xdp_res = 0; - /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */ -#if (PAGE_SIZE < 8192) - frame_sz = ixgbe_rx_frame_truesize(rx_ring, 0); -#endif - xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + xdp_init_buff(&xdp, IXGBE_RXBUFFER_3K, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct ixgbe_rx_buffer *rx_buffer; - struct sk_buff *skb; - int rx_buffer_pgcnt; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2408,7 +2189,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - rx_buffer = ixgbe_get_rx_buffer(rx_ring, rx_desc, &skb, size, &rx_buffer_pgcnt); + rx_buffer = ixgbe_get_rx_buffer(rx_ring, size); /* retrieve a buffer from the ring */ if (!skb) { @@ -2419,20 +2200,13 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, rx_buffer->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); xdp_buff_clear_frags_flag(&xdp); -#if (PAGE_SIZE > 4096) - /* At larger PAGE_SIZE, frame_sz depend on len size */ - xdp.frame_sz = ixgbe_rx_frame_truesize(rx_ring, size); -#endif xdp_res = ixgbe_run_xdp(adapter, rx_ring, &xdp); } if (xdp_res) { - if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) { + if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; - ixgbe_rx_buffer_flip(rx_ring, rx_buffer, size); - } else { - rx_buffer->pagecnt_bias++; - } + total_rx_packets++; total_rx_bytes += size; } else if (skb) { @@ -2445,11 +2219,13 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (!xdp_res && !skb) { rx_ring->rx_stats.alloc_rx_buff_failed++; - rx_buffer->pagecnt_bias++; + cleaned_count++; break; } - ixgbe_put_rx_buffer(rx_ring, rx_buffer, skb, rx_buffer_pgcnt); + ixgbe_put_rx_buffer(rx_ring, rx_buffer); + if (xdp_res == IXGBE_XDP_CONSUMED) + __free_page(rx_buffer->page); cleaned_count++; /* place incomplete frames back on ring for completion */ @@ -2497,6 +2273,8 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; } + rx_ring->skb = skb; + if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -4040,10 +3818,8 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; else srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else if (test_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state)) { - srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } else { - srrctl |= IXGBE_RXBUFFER_2K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; + srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; } /* configure descriptor type */ @@ -4419,17 +4195,8 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, } else if (hw->mac.type != ixgbe_mac_82599EB) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); - - /* Limit the maximum frame size so we don't overrun the skb. - * This can happen in SRIOV mode when the MTU of the VF is - * higher than the MTU of the PF. - */ - if (!test_bit(__IXGBE_RX_3K_BUFFER, &ring->state)) - rxdctl |= IXGBE_MAX_2K_FRAME_BUILD_SKB | - IXGBE_RXDCTL_RLPML_EN; -#endif } - +#endif if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); @@ -4439,11 +4206,9 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, ring->rx_buf_len = xsk_buf_len; } - /* initialize rx_buffer_info */ memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbe_rx_buffer) * ring->count); - + sizeof(struct ixgbe_rx_buffer) * ring->count); /* initialize Rx descriptor 0 */ rx_desc = IXGBE_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -4560,8 +4325,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) struct ixgbe_hw *hw = &adapter->hw; struct net_device *netdev = adapter->netdev; int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - struct ixgbe_ring *rx_ring; - int i; u32 mhadd, hlreg0; #ifdef IXGBE_FCOE @@ -4569,7 +4332,6 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; - #endif /* IXGBE_FCOE */ /* adjust max frame to be at least the size of a standard frame */ @@ -4585,35 +4347,11 @@ static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) } hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); - /* set jumbo enable since MHADD.MFS is keeping size locked at max_frame */ + /* set jumbo enable since MHADD.MFS is keeping size locked at + * max_frame + */ hlreg0 |= IXGBE_HLREG0_JUMBOEN; IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); - - /* - * Setup the HW Rx Head and Tail Descriptor Pointers and - * the Base and Length of the Rx Descriptor Ring - */ - for (i = 0; i < adapter->num_rx_queues; i++) { - rx_ring = adapter->rx_ring[i]; - - clear_ring_rsc_enabled(rx_ring); - clear_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_ring_rsc_enabled(rx_ring); - - if (test_bit(__IXGBE_RX_FCOE, &rx_ring->state)) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - -#if (PAGE_SIZE < 8192) - if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); - - if (IXGBE_2K_TOO_SMALL_WITH_PADDING || - (max_frame > (ETH_FRAME_LEN + ETH_FCS_LEN))) - set_bit(__IXGBE_RX_3K_BUFFER, &rx_ring->state); -#endif - } } static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) @@ -4698,8 +4436,15 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ - for (i = 0; i < adapter->num_rx_queues; i++) - ixgbe_configure_rx_ring(adapter, adapter->rx_ring[i]); + for (i = 0; i < adapter->num_rx_queues; i++) { + struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + + ixgbe_configure_rx_ring(adapter, rx_ring); + } rxctrl = IXGBE_READ_REG(hw, IXGBE_RXCTRL); /* disable drop enable for 82598 parts */ @@ -5605,35 +5350,28 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) goto skip_free; } - /* Free all the Rx ring sk_buffs */ - while (i != rx_ring->next_to_alloc) { - if (rx_buffer->skb) { - struct sk_buff *skb = rx_buffer->skb; - if (IXGBE_CB(skb)->page_released) - dma_unmap_page_attrs(rx_ring->dev, - IXGBE_CB(skb)->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - dev_kfree_skb(skb); - } + /* Free Rx ring sk_buff */ + if (rx_ring->skb) { + dev_kfree_skb(rx_ring->skb); + rx_ring->skb = NULL; + } + /* Free all the Rx ring pages */ + while (i != rx_ring->next_to_use) { /* Invalidate cache lines that may have been written to by * device so that we avoid corrupting memory. */ dma_sync_single_range_for_cpu(rx_ring->dev, rx_buffer->dma, rx_buffer->page_offset, - ixgbe_rx_bufsz(rx_ring), + IXGBE_RXBUFFER_3K, DMA_FROM_DEVICE); /* free resources associated with mapping */ dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - ixgbe_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, + PAGE_SIZE, DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - __page_frag_cache_drain(rx_buffer->page, - rx_buffer->pagecnt_bias); + __free_page(rx_buffer->page); i++; rx_buffer++; @@ -5644,7 +5382,6 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) } skip_free: - rx_ring->next_to_alloc = 0; rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; } @@ -7193,10 +6930,7 @@ static void ixgbe_free_all_rx_resources(struct ixgbe_adapter *adapter) */ static int ixgbe_max_xdp_frame_size(struct ixgbe_adapter *adapter) { - if (PAGE_SIZE >= 8192) - return IXGBE_RXBUFFER_2K; - else - return IXGBE_RXBUFFER_3K; + return IXGBE_RXBUFFER_3K; } /** @@ -10761,11 +10495,11 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (ring_is_rsc_enabled(ring)) return -EINVAL; - - if (frame_size > ixgbe_rx_bufsz(ring)) - return -EINVAL; } + if (frame_size > IXGBE_RXBUFFER_3K) + return -EINVAL; + /* if the number of cpus is much larger than the maximum of queues, * we should stop it and then return with ENOMEM like before. */ @@ -11131,6 +10865,11 @@ void ixgbe_txrx_ring_enable(struct ixgbe_adapter *adapter, int ring) ixgbe_configure_tx_ring(adapter, tx_ring); if (xdp_ring) ixgbe_configure_tx_ring(adapter, xdp_ring); + + clear_ring_rsc_enabled(rx_ring); + if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) + set_ring_rsc_enabled(rx_ring); + ixgbe_configure_rx_ring(adapter, rx_ring); clear_bit(__IXGBE_TX_DISABLED, &tx_ring->state); From 31040ff5982bd4ac7b2812210b46e165e768c5a5 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 7 May 2026 23:34:48 +0200 Subject: [PATCH 03/25] ixgbe: use libeth in Rx processing Convert ixgbe Rx path from manual page-per-packet DMA management to libeth/page_pool infrastructure, and integrates libeth XDP helpers. Each packet gets its own buffer instead of sharing pages between packets. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/Kconfig | 1 + drivers/net/ethernet/intel/ixgbe/ixgbe.h | 39 +- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 23 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 1149 ++++++++--------- drivers/net/ethernet/intel/ixgbe/ixgbe_type.h | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 19 +- 6 files changed, 554 insertions(+), 679 deletions(-) diff --git a/drivers/net/ethernet/intel/Kconfig b/drivers/net/ethernet/intel/Kconfig index b513baf3cbb296..072133b4a4416d 100644 --- a/drivers/net/ethernet/intel/Kconfig +++ b/drivers/net/ethernet/intel/Kconfig @@ -146,6 +146,7 @@ config IXGBE tristate "Intel(R) 10GbE PCI Express adapters support" depends on PCI depends on PTP_1588_CLOCK_OPTIONAL + select LIBETH_XDP select LIBIE_FWLOG if DEBUG_FS select MDIO select NET_DEVLINK diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index ef57e6029e492f..f5842f35d498bd 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -33,6 +33,7 @@ #include "ixgbe_ipsec.h" #include +#include /* common prefix used by pr_<> macros */ #undef pr_fmt @@ -73,9 +74,8 @@ #define IXGBE_RXBUFFER_4K 4096 #define IXGBE_MAX_RXBUFFER 16384 /* largest size for a single descriptor */ -#define IXGBE_PKT_HDR_PAD (ETH_HLEN + ETH_FCS_LEN + (VLAN_HLEN * 2)) - -#define IXGBE_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) +#define IXGBE_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) /* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 @@ -212,18 +212,9 @@ struct ixgbe_tx_buffer { u32 tx_flags; }; -struct ixgbe_rx_buffer { - union { - struct { - dma_addr_t dma; - struct page *page; - __u32 page_offset; - }; - struct { - bool discard; - struct xdp_buff *xdp; - }; - }; +struct ixgbe_xsk_rx_buffer { + bool discard; + struct xdp_buff *xdp; }; struct ixgbe_queue_stats { @@ -291,17 +282,22 @@ struct ixgbe_ring { struct ixgbe_ring *next; /* pointer to next ring in q_vector */ struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ struct net_device *netdev; /* netdev ring belongs to */ - struct bpf_prog *xdp_prog; - struct device *dev; /* device for DMA mapping */ + struct bpf_prog __rcu *xdp_prog; + union { + struct page_pool *pp; /* Rx ring */ + struct device *dev; /* Tx ring */ + }; void *desc; /* descriptor ring memory */ union { + struct libeth_fqe *rx_fqes; + struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; struct ixgbe_tx_buffer *tx_buffer_info; - struct ixgbe_rx_buffer *rx_buffer_info; }; unsigned long state; u8 __iomem *tail; dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ + u32 truesize; u16 count; /* amount of descriptors */ @@ -333,7 +329,7 @@ struct ixgbe_ring { spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ - u16 rx_buf_len; + u32 rx_buf_len; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { @@ -377,11 +373,6 @@ struct ixgbe_ring_feature { #define IXGBE_82599_VMDQ_4Q_MASK 0x7C #define IXGBE_82599_VMDQ_2Q_MASK 0x7E -static inline unsigned int ixgbe_rx_offset(struct ixgbe_ring *rx_ring) -{ - return IXGBE_SKB_PAD; -} - #define IXGBE_ITR_ADAPTIVE_MIN_INC 2 #define IXGBE_ITR_ADAPTIVE_MIN_USECS 10 #define IXGBE_ITR_ADAPTIVE_MAX_USECS 126 diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index b9db6ee9668e93..a2cc9b39eb634c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -1949,7 +1949,6 @@ static int ixgbe_setup_desc_rings(struct ixgbe_adapter *adapter) /* Setup Rx Descriptor ring and Rx buffers */ rx_ring->count = IXGBE_DEFAULT_RXD; rx_ring->queue_index = 0; - rx_ring->dev = &adapter->pdev->dev; rx_ring->netdev = adapter->netdev; rx_ring->reg_idx = adapter->rx_ring[0]->reg_idx; @@ -2060,14 +2059,16 @@ static void ixgbe_create_lbtest_frame(struct sk_buff *skb, skb->data[frame_size + 12] = 0xAF; } -static bool ixgbe_check_lbtest_frame(struct ixgbe_rx_buffer *rx_buffer, +static bool ixgbe_check_lbtest_frame(const struct libeth_fqe *rx_buffer, unsigned int frame_size) { + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; unsigned char *data; frame_size >>= 1; - data = page_address(rx_buffer->page) + rx_buffer->page_offset; + data = page_address(__netmem_to_page(rx_buffer->netmem)) + + rx_buffer->offset + hr; return data[3] == 0xFF && data[frame_size + 10] == 0xBE && data[frame_size + 12] == 0xAF; @@ -2115,16 +2116,13 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, } while (rx_desc->wb.upper.length) { - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; /* check Rx buffer */ - rx_buffer = &rx_ring->rx_buffer_info[rx_ntc]; + rx_buffer = &rx_ring->rx_fqes[rx_ntc]; /* sync Rx buffer for CPU read */ - dma_sync_single_for_cpu(rx_ring->dev, - rx_buffer->dma, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + libeth_rx_sync_for_cpu(rx_buffer, rx_ring->rx_buf_len); /* verify contents of skb */ if (ixgbe_check_lbtest_frame(rx_buffer, size)) @@ -2132,11 +2130,8 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, else break; - /* sync Rx buffer for device write */ - dma_sync_single_for_device(rx_ring->dev, - rx_buffer->dma, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + /* recycle the page back to the pool */ + libeth_rx_recycle_slow(rx_buffer->netmem); /* increment Rx next to clean counter */ rx_ntc++; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 73e46aea5c671d..c50a5a1e73eb19 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -54,16 +55,16 @@ char ixgbe_driver_name[] = "ixgbe"; static const char ixgbe_driver_string[] = - "Intel(R) 10 Gigabit PCI Express Network Driver"; + "Intel(R) 10 Gigabit PCI Express Network Driver"; #ifdef IXGBE_FCOE char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #else static char ixgbe_default_device_descr[] = - "Intel(R) 10 Gigabit Network Connection"; + "Intel(R) 10 Gigabit Network Connection"; #endif static const char ixgbe_copyright[] = - "Copyright (c) 1999-2016 Intel Corporation."; + "Copyright (c) 1999-2016 Intel Corporation."; static const char ixgbe_overheat_msg[] = "Network adapter has been stopped because it has over heated. Restart the computer. If the problem persists, power off the system and replace the adapter"; @@ -147,7 +148,7 @@ MODULE_DEVICE_TABLE(pci, ixgbe_pci_tbl); #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *, unsigned long event, - void *p); + void *p); static struct notifier_block dca_notifier = { .notifier_call = ixgbe_notify_dca, .next = NULL, @@ -174,6 +175,8 @@ MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)"); MODULE_IMPORT_NS("LIBIE_FWLOG"); MODULE_DESCRIPTION("Intel(R) 10 Gigabit PCI Express Network Driver"); +MODULE_IMPORT_NS("LIBETH"); +MODULE_IMPORT_NS("LIBETH_XDP"); MODULE_LICENSE("GPL v2"); DEFINE_STATIC_KEY_FALSE(ixgbe_xdp_locking_key); @@ -212,7 +215,7 @@ static int ixgbe_read_pci_cfg_word_parent(struct ixgbe_adapter *adapter, pcie_capability_read_word(parent_dev, reg, value); if (*value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) + ixgbe_check_cfg_remove(&adapter->hw, parent_dev)) return -1; return 0; } @@ -264,7 +267,7 @@ static bool ixgbe_pcie_from_parent(struct ixgbe_hw *hw) } static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, - int expected_gts) + int expected_gts) { struct ixgbe_hw *hw = &adapter->hw; struct pci_dev *pdev; @@ -288,8 +291,8 @@ static void ixgbe_check_minimum_link(struct ixgbe_adapter *adapter, static void ixgbe_service_event_schedule(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_and_set_bit(__IXGBE_SERVICE_SCHED, &adapter->state)) queue_work(ixgbe_wq, &adapter->service_task); } @@ -354,7 +357,7 @@ u32 ixgbe_read_reg(struct ixgbe_hw *hw, u32 reg) if (ixgbe_removed(reg_addr)) return IXGBE_FAILED_READ_REG; if (unlikely(hw->phy.nw_mng_if_sel & - IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { + IXGBE_NW_MNG_IF_SEL_SGMII_ENABLE)) { struct ixgbe_adapter *adapter; int i; @@ -401,7 +404,7 @@ u16 ixgbe_read_pci_cfg_word(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_WORD; pci_read_config_word(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_WORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_WORD; return value; } @@ -416,7 +419,7 @@ static u32 ixgbe_read_pci_cfg_dword(struct ixgbe_hw *hw, u32 reg) return IXGBE_FAILED_READ_CFG_DWORD; pci_read_config_dword(adapter->pdev, reg, &value); if (value == IXGBE_FAILED_READ_CFG_DWORD && - ixgbe_check_cfg_remove(hw, adapter->pdev)) + ixgbe_check_cfg_remove(hw, adapter->pdev)) return IXGBE_FAILED_READ_CFG_DWORD; return value; } @@ -612,7 +615,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) dev_info(&adapter->pdev->dev, "Register Dump\n"); pr_info(" Register Name Value\n"); for (reginfo = (struct ixgbe_reg_info *)ixgbe_reg_info_tbl; - reginfo->name; reginfo++) { + reginfo->name; reginfo++) { ixgbe_regdump(hw, reginfo); } @@ -693,7 +696,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) const char *ring_desc; if (i == ring->next_to_use && - i == ring->next_to_clean) + i == ring->next_to_clean) ring_desc = " NTC/U"; else if (i == ring->next_to_use) ring_desc = " NTU"; @@ -713,7 +716,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) ring_desc); if (netif_msg_pktdata(adapter) && - tx_buffer->skb) + tx_buffer->skb) print_hex_dump(KERN_INFO, "", DUMP_PREFIX_ADDRESS, 16, 1, tx_buffer->skb->data, @@ -819,17 +822,19 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) i, le64_to_cpu((__force __le64)u0->a), le64_to_cpu((__force __le64)u0->b), - (u64)rx_ring->rx_buffer_info[i].dma, + (u64)page_pool_get_dma_addr_netmem(rx_ring->rx_fqes[i].netmem), ring_desc); } if (netif_msg_pktdata(adapter) && - rx_ring->rx_buffer_info[i].dma) { + rx_ring->rx_fqes[i].netmem) { + struct libeth_fqe *fqe = &rx_ring->rx_fqes[i]; + print_hex_dump(KERN_INFO, "", - DUMP_PREFIX_ADDRESS, 16, 1, - page_address(rx_ring->rx_buffer_info[i].page) + - rx_ring->rx_buffer_info[i].page_offset, - IXGBE_RXBUFFER_3K, true); + DUMP_PREFIX_ADDRESS, 16, 1, + page_address(__netmem_to_page(fqe->netmem)) + + fqe->offset, + IXGBE_RXBUFFER_3K, true); } } } @@ -910,7 +915,7 @@ static void ixgbe_set_ivar(struct ixgbe_adapter *adapter, s8 direction, } void ixgbe_irq_rearm_queues(struct ixgbe_adapter *adapter, - u64 qmask) + u64 qmask) { u32 mask; @@ -943,7 +948,7 @@ static void ixgbe_update_xoff_rx_lfc(struct ixgbe_adapter *adapter) u32 data; if ((hw->fc.current_mode != ixgbe_fc_full) && - (hw->fc.current_mode != ixgbe_fc_rx_pause)) + (hw->fc.current_mode != ixgbe_fc_rx_pause)) return; switch (hw->mac.type) { @@ -1127,7 +1132,7 @@ static void ixgbe_tx_timeout_reset(struct ixgbe_adapter *adapter) * @maxrate: desired maximum transmit bitrate **/ static int ixgbe_tx_maxrate(struct net_device *netdev, - int queue_index, u32 maxrate) + int queue_index, u32 maxrate) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1199,7 +1204,7 @@ void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring, * Prints a message containing details about the tx hang. */ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, - unsigned int next) + unsigned int next) { struct ixgbe_adapter *adapter = netdev_priv(tx_ring->netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -1212,14 +1217,14 @@ static void ixgbe_pf_handle_tx_hang(struct ixgbe_ring *tx_ring, "tx_buffer_info[next_to_clean]\n" " time_stamp <%lx>\n" " jiffies <%lx>\n", - tx_ring->queue_index, - IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), - IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), - tx_ring->next_to_use, next, - tx_ring->tx_buffer_info[next].time_stamp, jiffies); + tx_ring->queue_index, + IXGBE_READ_REG(hw, IXGBE_TDH(tx_ring->reg_idx)), + IXGBE_READ_REG(hw, IXGBE_TDT(tx_ring->reg_idx)), + tx_ring->next_to_use, next, + tx_ring->tx_buffer_info[next].time_stamp, jiffies); netif_stop_subqueue(tx_ring->netdev, - tx_ring->queue_index); + tx_ring->queue_index); } /** @@ -1238,8 +1243,8 @@ static void ixgbe_vf_handle_tx_hang(struct ixgbe_adapter *adapter, u16 vf) return; e_warn(drv, - "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", - hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection tx hang detected on PF %d VF %d MAC: %pM", + hw->bus.func, vf, adapter->vfinfo[vf].vf_mac_addresses); adapter->tx_hang_count[vf]++; if (adapter->tx_hang_count[vf] == IXGBE_MAX_TX_VF_HANGS) { @@ -1266,7 +1271,7 @@ static u32 ixgbe_poll_tx_icache(struct ixgbe_hw *hw, u16 queue, u16 idx) * Returns: true if queue contain illegal packet. */ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, - u16 queue) + u16 queue) { u32 hdr_len_reg, mss_len_reg, type_reg; struct ixgbe_hw *hw = &adapter->hw; @@ -1289,13 +1294,13 @@ static bool ixgbe_check_illegal_queue(struct ixgbe_adapter *adapter, /* check if Advanced Context Descriptor */ if (FIELD_GET(IXGBE_ADVTXD_DTYP_MASK, type_reg) != - IXGBE_ADVTXD_DTYP_CTXT) + IXGBE_ADVTXD_DTYP_CTXT) continue; /* check for illegal MSS and Header length */ mss_len = FIELD_GET(IXGBE_ADVTXD_MSS_MASK, mss_len_reg); header_len = FIELD_GET(IXGBE_ADVTXD_HEADER_LEN_MASK, - hdr_len_reg); + hdr_len_reg); if ((mss_len + header_len) > SZ_16K) { e_warn(probe, "mss len + header len too long\n"); return true; @@ -1326,7 +1331,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, /* malicious vf not detected */ for (q = 0; q < IXGBE_MAX_TX_QUEUES; q++) { if (ixgbe_check_illegal_queue(adapter, q) && - !ixgbe_get_vf_idx(adapter, q, &vf)) + !ixgbe_get_vf_idx(adapter, q, &vf)) /* illegal queue detected */ ixgbe_vf_handle_tx_hang(adapter, vf); } @@ -1340,7 +1345,7 @@ static void ixgbe_handle_mdd_event(struct ixgbe_adapter *adapter, * @napi_budget: Used to determine if we are in netpoll **/ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *tx_ring, int napi_budget) + struct ixgbe_ring *tx_ring, int napi_budget) { struct ixgbe_adapter *adapter = q_vector->adapter; struct ixgbe_tx_buffer *tx_buffer; @@ -1409,9 +1414,9 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) { dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); } } @@ -1449,7 +1454,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, ixgbe_pf_handle_tx_hang(tx_ring, i); e_info(probe, - "tx hang %d detected on queue %d, resetting adapter\n", + "tx hang %d detected on queue %d, resetting adapter\n", adapter->tx_timeout_count + 1, tx_ring->queue_index); /* schedule immediate reset if we believe we hung */ @@ -1518,7 +1523,7 @@ static void ixgbe_update_rx_dca(struct ixgbe_adapter *adapter, u8 reg_idx = rx_ring->reg_idx; if (adapter->flags & IXGBE_FLAG_DCA_ENABLED) - rxctrl = dca3_get_tag(rx_ring->dev, cpu); + rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu); switch (hw->mac.type) { case ixgbe_mac_82599EB: @@ -1636,8 +1641,8 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, return; skb_set_hash(skb, le32_to_cpu(rx_desc->wb.lower.hi_dword.rss), - (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? - PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); + (IXGBE_RSS_L4_TYPES_MASK & (1ul << rss_type)) ? + PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3); } #ifdef IXGBE_FCOE @@ -1649,14 +1654,14 @@ static inline void ixgbe_rx_hash(struct ixgbe_ring *ring, * Returns : true if it is FCoE pkt */ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc) + union ixgbe_adv_rx_desc *rx_desc) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; return test_bit(__IXGBE_RX_FCOE, &ring->state) && - ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == + ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_ETQF_MASK)) == (cpu_to_le16(IXGBE_ETQF_FILTER_FCOE << - IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); + IXGBE_RXDADV_PKTTYPE_ETQF_SHIFT))); } #endif /* IXGBE_FCOE */ @@ -1667,8 +1672,8 @@ static inline bool ixgbe_rx_is_fcoe(struct ixgbe_ring *ring, * @skb: skb currently being received and modified **/ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { __le16 pkt_info = rx_desc->wb.lower.lo_dword.hs_rss.pkt_info; bool encap_pkt = false; @@ -1687,7 +1692,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, /* if IP and error */ if (ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_IPCS) && - ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_IPE)) { ring->rx_stats.csum_err++; return; } @@ -1701,7 +1706,7 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, * checksum errors. */ if ((pkt_info & cpu_to_le16(IXGBE_RXDADV_PKTTYPE_UDP)) && - test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) + test_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state)) return; ring->rx_stats.csum_err++; @@ -1723,39 +1728,6 @@ static inline void ixgbe_rx_checksum(struct ixgbe_ring *ring, } } -static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *bi) -{ - struct page *page = bi->page; - dma_addr_t dma; - - /* alloc new page for storage */ - page = dev_alloc_page(); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); - - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_page(page); - - rx_ring->rx_stats.alloc_rx_page_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = rx_ring->xdp_prog ? XDP_PACKET_HEADROOM : - IXGBE_SKB_PAD; - rx_ring->rx_stats.alloc_rx_page++; - - return true; -} - /** * ixgbe_alloc_rx_buffers - Replace used receive buffers * @rx_ring: ring to place buffers on @@ -1764,40 +1736,34 @@ static bool ixgbe_alloc_mapped_page(struct ixgbe_ring *rx_ring, void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; - u16 i = rx_ring->next_to_use; + const struct libeth_fq_fp fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + .truesize = rx_ring->truesize, + .count = rx_ring->count, + }; + u16 ntu = rx_ring->next_to_use; /* nothing to do */ if (!cleaned_count) return; - - rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; - i -= rx_ring->count; + rx_desc = IXGBE_RX_DESC(rx_ring, ntu); do { - if (!ixgbe_alloc_mapped_page(rx_ring, bi)) - break; + dma_addr_t addr; - /* sync the buffer for use by the device */ - dma_sync_single_range_for_device(rx_ring->dev, bi->dma, - bi->page_offset, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); + addr = libeth_rx_alloc(&fq, ntu); + if (addr == DMA_MAPPING_ERROR) + break; - /* - * Refresh the desc even if buffer_addrs didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset); + rx_desc->read.pkt_addr = cpu_to_le64(addr); rx_desc++; - bi++; - i++; - if (unlikely(!i)) { + ntu++; + + if (unlikely(ntu == rx_ring->count)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; - i -= rx_ring->count; + ntu = 0; } /* clear the length for the next_to_use descriptor */ @@ -1806,18 +1772,16 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) cleaned_count--; } while (cleaned_count); - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { - rx_ring->next_to_use = i; - + if (likely(rx_ring->next_to_use != ntu)) { + /* record the next descriptor to use */ + rx_ring->next_to_use = ntu; /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only * applicable for weak-ordered memory model archs, * such as IA-64). */ wmb(); - writel(i, rx_ring->tail); + writel(ntu, rx_ring->tail); } } @@ -1859,8 +1823,8 @@ static void ixgbe_update_rsc_stats(struct ixgbe_ring *rx_ring, * other fields within the skb. **/ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { struct net_device *dev = rx_ring->netdev; u32 flags = rx_ring->q_vector->adapter->flags; @@ -1875,7 +1839,7 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, ixgbe_ptp_rx_hwtstamp(rx_ring, rx_desc, skb); if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) && - ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { + ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_VP)) { u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan); __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid); } @@ -1889,8 +1853,6 @@ void ixgbe_process_skb_fields(struct ixgbe_ring *rx_ring, else macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true, false); - - skb->protocol = eth_type_trans(skb, dev); } void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, @@ -1911,8 +1873,8 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * that this is in fact a non-EOP buffer. **/ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + struct sk_buff *skb) { u32 ntc = rx_ring->next_to_clean + 1; @@ -1925,7 +1887,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, /* update RSC append count if present */ if (ring_is_rsc_enabled(rx_ring)) { __le32 rsc_enabled = rx_desc->wb.lower.lo_dword.data & - cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); + cpu_to_le32(IXGBE_RXDADV_RSCCNT_MASK); if (unlikely(rsc_enabled)) { u32 rsc_cnt = le32_to_cpu(rsc_enabled); @@ -1982,9 +1944,9 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, * errors that would be unacceptable to the netdev. */ if (!netdev || - (unlikely(ixgbe_test_staterr(rx_desc, + (unlikely(ixgbe_test_staterr(rx_desc, IXGBE_RXDADV_ERR_FRAME_ERR_MASK) && - !(netdev->features & NETIF_F_RXALL)))) { + !(netdev->features & NETIF_F_RXALL)))) { dev_kfree_skb_any(skb); return true; } @@ -2007,82 +1969,28 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, * @size: size of data in rx_buffer * * This function will add the data contained in rx_buffer->page to the skb. + * This is done either through a direct copy if the data in the buffer is + * less than the skb header size, otherwise it will just attach the page as + * a frag to the skb. + * + * The function will then update the page offset if necessary and return + * true if the buffer can be reused by the adapter. **/ -static void ixgbe_add_rx_frag(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct sk_buff *skb, - unsigned int size) +static void ixgbe_add_rx_frag(const struct libeth_fqe *rx_buffer, + struct sk_buff *skb, + unsigned int size) { - unsigned int truesize = SKB_DATA_ALIGN(IXGBE_SKB_PAD + size); + u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page, - rx_buffer->page_offset, size, truesize); + skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, + rx_buffer->netmem, rx_buffer->offset + hr, + size, rx_buffer->truesize); } -static struct ixgbe_rx_buffer *ixgbe_get_rx_buffer(struct ixgbe_ring *rx_ring, - const unsigned int size) -{ - struct ixgbe_rx_buffer *rx_buffer; - - rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; - prefetchw(rx_buffer->page); - - /* sync this buffer for CPU use */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - size, - DMA_FROM_DEVICE); - - return rx_buffer; -} - -static void ixgbe_put_rx_buffer(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer) -{ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, PAGE_SIZE, - DMA_FROM_DEVICE, IXGBE_RX_DMA_ATTR); -} - -static struct sk_buff *ixgbe_build_skb(struct ixgbe_ring *rx_ring, - struct ixgbe_rx_buffer *rx_buffer, - struct xdp_buff *xdp, - union ixgbe_adv_rx_desc *rx_desc) -{ - unsigned int metasize = xdp->data - xdp->data_meta; - unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + - SKB_DATA_ALIGN(xdp->data_end - - xdp->data_hard_start); - struct sk_buff *skb; - - /* Prefetch first cache line of first page. If xdp->data_meta - * is unused, this points exactly as xdp->data, otherwise we - * likely have a consumer accessing first few bytes of meta - * data, and then actual data. - */ - net_prefetch(xdp->data_meta); - - /* build an skb to around the page buffer */ - skb = napi_build_skb(xdp->data_hard_start, truesize); - if (unlikely(!skb)) - return NULL; - - /* update pointers within the skb to store the data */ - skb_reserve(skb, xdp->data - xdp->data_hard_start); - __skb_put(skb, xdp->data_end - xdp->data); - if (metasize) - skb_metadata_set(skb, metasize); - - /* record DMA address if this is the start of a chain of buffers */ - if (!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP)) - IXGBE_CB(skb)->dma = rx_buffer->dma; - - return skb; -} static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, - struct xdp_buff *xdp) + struct libeth_xdp_buff *xdp) { int err, result = IXGBE_XDP_PASS; struct bpf_prog *xdp_prog; @@ -2095,14 +2003,14 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, if (!xdp_prog) goto xdp_out; - prefetchw(xdp->data_hard_start); /* xdp_frame write */ + prefetchw(xdp->base.data_hard_start); /* xdp_frame write */ - act = bpf_prog_run_xdp(xdp_prog, xdp); + act = bpf_prog_run_xdp(xdp_prog, &xdp->base); switch (act) { case XDP_PASS: break; case XDP_TX: - xdpf = xdp_convert_buff_to_frame(xdp); + xdpf = xdp_convert_buff_to_frame(&xdp->base); if (unlikely(!xdpf)) goto out_failure; ring = ixgbe_determine_xdp_ring(adapter); @@ -2115,7 +2023,7 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, goto out_failure; break; case XDP_REDIRECT: - err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog); + err = xdp_do_redirect(adapter->netdev, &xdp->base, xdp_prog); if (err) goto out_failure; result = IXGBE_XDP_REDIR; @@ -2129,13 +2037,13 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, fallthrough; /* handle aborts by dropping packet */ case XDP_DROP: result = IXGBE_XDP_CONSUMED; + libeth_xdp_return_buff(xdp); break; } xdp_out: return result; } - /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2150,8 +2058,8 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, * Returns amount of work completed **/ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring *rx_ring, - const int budget) + struct ixgbe_ring *rx_ring, + const int budget) { unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; @@ -2160,16 +2068,17 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); - unsigned int xdp_xmit = 0; - struct xdp_buff xdp; + LIBETH_XDP_ONSTACK_BUFF(xdp); struct sk_buff *skb = rx_ring->skb; + unsigned int xdp_xmit = 0; int xdp_res = 0; - xdp_init_buff(&xdp, IXGBE_RXBUFFER_3K, &rx_ring->xdp_rxq); + xdp->data = NULL; + xdp->base.rxq = &rx_ring->xdp_rxq; while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *rx_buffer; + struct libeth_fqe *rx_buffer; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2180,7 +2089,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -2189,18 +2098,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - rx_buffer = ixgbe_get_rx_buffer(rx_ring, size); + rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; /* retrieve a buffer from the ring */ if (!skb) { - unsigned int offset = ixgbe_rx_offset(rx_ring); - unsigned char *hard_start; - - hard_start = page_address(rx_buffer->page) + - rx_buffer->page_offset - offset; - xdp_prepare_buff(&xdp, hard_start, offset, size, true); - xdp_buff_clear_frags_flag(&xdp); - xdp_res = ixgbe_run_xdp(adapter, rx_ring, &xdp); + libeth_xdp_process_buff(xdp, rx_buffer, size); + xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); + } else { + libeth_rx_sync_for_cpu(rx_buffer, size); } if (xdp_res) { @@ -2210,22 +2115,19 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; total_rx_bytes += size; } else if (skb) { - ixgbe_add_rx_frag(rx_ring, rx_buffer, skb, size); + ixgbe_add_rx_frag(rx_buffer, skb, size); } else { - skb = ixgbe_build_skb(rx_ring, rx_buffer, - &xdp, rx_desc); + skb = xdp_build_skb_from_buff(&xdp->base); } /* exit if we failed to retrieve a buffer */ - if (!xdp_res && !skb) { + if (unlikely(!xdp_res && !skb)) { + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; cleaned_count++; break; } - ixgbe_put_rx_buffer(rx_ring, rx_buffer); - if (xdp_res == IXGBE_XDP_CONSUMED) - __free_page(rx_buffer->page); cleaned_count++; /* place incomplete frames back on ring for completion */ @@ -2233,8 +2135,10 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || ixgbe_cleanup_headers(rx_ring, rx_desc, skb)) + if (xdp_res || unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + skb = NULL; continue; + } /* probably a little skewed due to removing CRC */ total_rx_bytes += skb->len; @@ -2262,12 +2166,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } if (!ddp_bytes) { dev_kfree_skb_any(skb); + skb = NULL; continue; } } #endif /* IXGBE_FCOE */ ixgbe_rx_skb(q_vector, skb); + skb = NULL; /* update budget accounting */ total_rx_packets++; @@ -2329,7 +2235,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) switch (adapter->hw.mac.type) { case ixgbe_mac_82598EB: ixgbe_set_ivar(adapter, -1, IXGBE_IVAR_OTHER_CAUSES_INDEX, - v_idx); + v_idx); break; case ixgbe_mac_82599EB: case ixgbe_mac_X540: @@ -2370,7 +2276,7 @@ static void ixgbe_configure_msix(struct ixgbe_adapter *adapter) * while increasing bulk throughput. **/ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, - struct ixgbe_ring_container *ring_container) + struct ixgbe_ring_container *ring_container) { unsigned int itr = IXGBE_ITR_ADAPTIVE_MIN_USECS | IXGBE_ITR_ADAPTIVE_LATENCY; @@ -2519,8 +2425,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, case IXGBE_LINK_SPEED_100_FULL: default: itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 256) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; case IXGBE_LINK_SPEED_2_5GB_FULL: case IXGBE_LINK_SPEED_1GB_FULL: @@ -2528,8 +2434,8 @@ static void ixgbe_update_itr(struct ixgbe_q_vector *q_vector, if (avg_wire_size > 8064) avg_wire_size = 8064; itr += DIV_ROUND_UP(avg_wire_size, - IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * - IXGBE_ITR_ADAPTIVE_MIN_INC; + IXGBE_ITR_ADAPTIVE_MIN_INC * 64) * + IXGBE_ITR_ADAPTIVE_MIN_INC; break; } @@ -2631,7 +2537,7 @@ static void ixgbe_check_overtemp_subtask(struct ixgbe_adapter *adapter) * check if we got a LSC */ if (!(eicr & IXGBE_EICR_GPI_SDP0_8259X) && - !(eicr & IXGBE_EICR_LSC)) + !(eicr & IXGBE_EICR_LSC)) return; if (!(eicr & IXGBE_EICR_LSC) && hw->mac.ops.check_link) { @@ -2671,7 +2577,7 @@ static void ixgbe_check_fan_failure(struct ixgbe_adapter *adapter, u32 eicr) struct ixgbe_hw *hw = &adapter->hw; if ((adapter->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { e_crit(probe, "Fan has stopped, replace the adapter\n"); /* write to clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); @@ -2692,8 +2598,8 @@ static void ixgbe_check_overtemp_event(struct ixgbe_adapter *adapter, u32 eicr) * on service task */ if (((eicr & IXGBE_EICR_GPI_SDP0(hw)) || - (eicr & IXGBE_EICR_LSC)) && - (!test_bit(__IXGBE_DOWN, &adapter->state))) { + (eicr & IXGBE_EICR_LSC)) && + (!test_bit(__IXGBE_DOWN, &adapter->state))) { adapter->interrupt_event = eicr; adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_EVENT; ixgbe_service_event_schedule(adapter); @@ -2768,7 +2674,7 @@ static void ixgbe_check_sfp_event(struct ixgbe_adapter *adapter, u32 eicr) } if (adapter->hw.mac.type == ixgbe_mac_82599EB && - (eicr & IXGBE_EICR_GPI_SDP1(hw))) { + (eicr & IXGBE_EICR_GPI_SDP1(hw))) { /* Clear the interrupt */ IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1(hw)); if (!test_bit(__IXGBE_DOWN, &adapter->state)) { @@ -2800,7 +2706,7 @@ static void ixgbe_check_lsc(struct ixgbe_adapter *adapter) * Check if external PHY FW load failed and print an error message if it did. */ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { if (!(link_cfg_err & IXGBE_ACI_LINK_EXTERNAL_PHY_LOAD_FAILURE)) { adapter->flags2 &= ~IXGBE_FLAG2_PHY_FW_LOAD_FAILED; @@ -2825,11 +2731,11 @@ static void ixgbe_check_phy_fw_load(struct ixgbe_adapter *adapter, * and print error messages if module power level is not supported. */ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { /* If module power level is supported, clear the flag. */ if (!(link_cfg_err & (IXGBE_ACI_LINK_INVAL_MAX_POWER_LIMIT | - IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { + IXGBE_ACI_LINK_MODULE_POWER_UNSUPPORTED))) { adapter->flags2 &= ~IXGBE_FLAG2_MOD_POWER_UNSUPPORTED; return; } @@ -2858,7 +2764,7 @@ static void ixgbe_check_module_power(struct ixgbe_adapter *adapter, * link_cfg_err parameter in the link info structure. */ static void ixgbe_check_link_cfg_err(struct ixgbe_adapter *adapter, - u8 link_cfg_err) + u8 link_cfg_err) { ixgbe_check_module_power(adapter, link_cfg_err); ixgbe_check_phy_fw_load(adapter, link_cfg_err); @@ -2897,12 +2803,12 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, /* Turn off PHY if media was removed. */ if (!(adapter->flags2 & IXGBE_FLAG2_NO_MEDIA) && - !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) + !(hw->link.link_info.link_info & IXGBE_ACI_MEDIA_AVAILABLE)) adapter->flags2 |= IXGBE_FLAG2_NO_MEDIA; if (link_up == adapter->link_up && - link_up == netif_carrier_ok(adapter->netdev) && - link_speed == adapter->link_speed) + link_up == netif_carrier_ok(adapter->netdev) && + link_speed == adapter->link_speed) return 0; adapter->flags |= IXGBE_FLAG_NEED_LINK_UPDATE; @@ -2924,7 +2830,7 @@ ixgbe_process_link_status_event(struct ixgbe_adapter *adapter, bool link_up, */ static void ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, - struct ixgbe_aci_event *e) + struct ixgbe_aci_event *e) { struct ixgbe_aci_cmd_get_link_status_data *link_data; u16 link_speed; @@ -2949,8 +2855,8 @@ ixgbe_handle_link_status_event(struct ixgbe_adapter *adapter, static void ixgbe_schedule_fw_event(struct ixgbe_adapter *adapter) { if (!test_bit(__IXGBE_DOWN, &adapter->state) && - !test_bit(__IXGBE_REMOVING, &adapter->state) && - !test_bit(__IXGBE_RESETTING, &adapter->state)) { + !test_bit(__IXGBE_REMOVING, &adapter->state) && + !test_bit(__IXGBE_RESETTING, &adapter->state)) { adapter->flags2 |= IXGBE_FLAG2_FW_ASYNC_EVENT; ixgbe_service_event_schedule(adapter); } @@ -3003,7 +2909,7 @@ static void ixgbe_handle_fw_event(struct ixgbe_adapter *adapter) break; case libie_aqc_opc_fw_logs_event: libie_get_fwlog_data(&hw->fwlog, event.msg_buf, - le16_to_cpu(event.desc.datalen)); + le16_to_cpu(event.desc.datalen)); break; default: e_warn(hw, "unknown FW async event captured\n"); @@ -3049,7 +2955,7 @@ static inline void ixgbe_irq_enable_queues(struct ixgbe_adapter *adapter, * @flush: flush register write **/ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, - bool flush) + bool flush) { struct ixgbe_hw *hw = &adapter->hw; u32 mask = (IXGBE_EIMS_ENABLE_MASK & ~IXGBE_EIMS_RTX_QUEUE); @@ -3087,8 +2993,8 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, fallthrough; case ixgbe_mac_x550em_a: if (adapter->hw.device_id == IXGBE_DEV_ID_X550EM_X_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || - adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP || + adapter->hw.device_id == IXGBE_DEV_ID_X550EM_A_SFP_N) mask |= IXGBE_EIMS_GPI_SDP0(&adapter->hw); if (adapter->hw.phy.type == ixgbe_phy_x550em_ext_t) mask |= IXGBE_EICR_GPI_SDP0_X540; @@ -3100,7 +3006,7 @@ static inline void ixgbe_irq_enable(struct ixgbe_adapter *adapter, bool queues, } if ((adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE) && - !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) + !(adapter->flags2 & IXGBE_FLAG2_FDIR_REQUIRES_REINIT)) mask |= IXGBE_EIMS_FLOW_DIR; IXGBE_WRITE_REG(&adapter->hw, IXGBE_EIMS, mask); @@ -3152,7 +3058,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) case ixgbe_mac_x550em_a: case ixgbe_mac_e610: if (hw->phy.type == ixgbe_phy_x550em_ext_t && - (eicr & IXGBE_EICR_GPI_SDP0_X540)) { + (eicr & IXGBE_EICR_GPI_SDP0_X540)) { adapter->flags2 |= IXGBE_FLAG2_PHY_INTERRUPT; ixgbe_service_event_schedule(adapter); IXGBE_WRITE_REG(hw, IXGBE_EICR, @@ -3171,7 +3077,7 @@ static irqreturn_t ixgbe_msix_other(int irq, void *data) for (i = 0; i < adapter->num_tx_queues; i++) { struct ixgbe_ring *ring = adapter->tx_ring[i]; if (test_and_clear_bit(__IXGBE_TX_FDIR_INIT_DONE, - &ring->state)) + &ring->state)) reinit_count++; } if (reinit_count) { @@ -3255,9 +3161,9 @@ int ixgbe_poll(struct napi_struct *napi, int budget) ixgbe_for_each_ring(ring, q_vector->rx) { int cleaned = ring->xsk_pool ? - ixgbe_clean_rx_irq_zc(q_vector, ring, - per_ring_budget) : - ixgbe_clean_rx_irq(q_vector, ring, + ixgbe_clean_rx_irq_zc(q_vector, ring, + per_ring_budget) : + ixgbe_clean_rx_irq(q_vector, ring, per_ring_budget); work_done += cleaned; @@ -3316,7 +3222,7 @@ static int ixgbe_request_msix_irqs(struct ixgbe_adapter *adapter) q_vector->name, q_vector); if (err) { e_err(probe, "request_irq failed for MSIX interrupt " - "Error: %d\n", err); + "Error: %d\n", err); goto free_queue_irqs; } /* If Flow Director is enabled, set interrupt affinity */ @@ -3546,7 +3452,7 @@ static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) * Configure the Tx descriptor ring after a reset. **/ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u64 tdba = ring->dma; @@ -3608,22 +3514,22 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, if (q_vector) netif_set_xps_queue(ring->netdev, - &q_vector->affinity_mask, - ring->queue_index); + &q_vector->affinity_mask, + ring->queue_index); } clear_bit(__IXGBE_HANG_CHECK_ARMED, &ring->state); /* reinitialize tx_buffer_info */ memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbe_tx_buffer) * ring->count); + sizeof(struct ixgbe_tx_buffer) * ring->count); /* enable queue */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); /* TXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* poll to verify queue is enabled */ @@ -3768,7 +3674,7 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) * and performance reasons. */ if (adapter->num_vfs || (adapter->num_rx_queues > 1 && - !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { + !(adapter->hw.fc.current_mode & ixgbe_fc_tx_pause) && !pfc_en)) { for (i = 0; i < adapter->num_rx_queues; i++) ixgbe_enable_rx_drop(adapter, adapter->rx_ring[i]); } else { @@ -3783,11 +3689,11 @@ static void ixgbe_set_rx_drop_en(struct ixgbe_adapter *adapter) #define IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT 2 static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; u32 srrctl; - u8 reg_idx = rx_ring->reg_idx; + u8 reg_idx = ring->reg_idx; if (hw->mac.type == ixgbe_mac_82598EB) { u16 mask = adapter->ring_feature[RING_F_RSS].mask; @@ -3802,25 +3708,12 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, /* configure header buffer length, needed for RSC */ srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - /* configure the packet buffer length */ - if (rx_ring->xsk_pool) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(rx_ring->xsk_pool); - - /* If the MAC support setting RXDCTL.RLPML, the - * SRRCTL[n].BSIZEPKT is set to PAGE_SIZE and - * RXDCTL.RLPML is set to the actual UMEM buffer - * size. If not, then we are stuck with a 1k buffer - * size resolution. In this case frames larger than - * the UMEM buffer size viewed in a 1k resolution will - * be dropped. - */ - if (hw->mac.type != ixgbe_mac_82599EB) - srrctl |= PAGE_SIZE >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - else - srrctl |= xsk_buf_len >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } else { - srrctl |= IXGBE_RXBUFFER_3K >> IXGBE_SRRCTL_BSIZEPKT_SHIFT; - } + if (ring->xsk_pool) + srrctl |= DIV_ROUND_UP(xsk_pool_get_rx_frame_size(ring->xsk_pool), + IXGBE_SRRCTL_BSIZEPKT_STEP); + else + srrctl |= DIV_ROUND_UP(ring->rx_buf_len, + IXGBE_SRRCTL_BSIZEPKT_STEP); /* configure descriptor type */ srrctl |= IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; @@ -4051,9 +3944,9 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) /* Perform hash on these packet types */ rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4 | - IXGBE_MRQC_RSS_FIELD_IPV4_TCP | - IXGBE_MRQC_RSS_FIELD_IPV6 | - IXGBE_MRQC_RSS_FIELD_IPV6_TCP; + IXGBE_MRQC_RSS_FIELD_IPV4_TCP | + IXGBE_MRQC_RSS_FIELD_IPV6 | + IXGBE_MRQC_RSS_FIELD_IPV6_TCP; if (adapter->flags2 & IXGBE_FLAG2_RSS_FIELD_IPV4_UDP) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV4_UDP; @@ -4061,7 +3954,7 @@ static void ixgbe_setup_mrqc(struct ixgbe_adapter *adapter) rss_field |= IXGBE_MRQC_RSS_FIELD_IPV6_UDP; if ((hw->mac.type >= ixgbe_mac_X550) && - (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { + (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED)) { u16 pool = adapter->num_rx_pools; /* Enable VF RSS mode */ @@ -4112,7 +4005,7 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, #define IXGBE_MAX_RX_DESC_POLL 10 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; int wait_loop = IXGBE_MAX_RX_DESC_POLL; @@ -4123,7 +4016,7 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, return; /* RXDCTL.EN will return 0 on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; do { @@ -4133,12 +4026,12 @@ static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, if (!wait_loop) { e_err(drv, "RXDCTL.ENABLE on Rx queue %d not set within " - "the polling period\n", reg_idx); + "the polling period\n", reg_idx); } } void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, - struct ixgbe_ring *ring) + struct ixgbe_ring *ring) { struct ixgbe_hw *hw = &adapter->hw; union ixgbe_adv_rx_desc *rx_desc; @@ -4154,8 +4047,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, NULL)); xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); } else { - WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_PAGE_SHARED, NULL)); + xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, ring->pp); } /* disable queue to avoid use of these values while updating state */ @@ -4177,7 +4069,6 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RDT(reg_idx), 0); ring->tail = adapter->io_addr + IXGBE_RDT(reg_idx); - ixgbe_configure_srrctl(adapter, ring); ixgbe_configure_rscctl(adapter, ring); if (hw->mac.type == ixgbe_mac_82598EB) { @@ -4194,21 +4085,19 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, /* RXDCTL.RLPML does not work on 82599 */ } else if (hw->mac.type != ixgbe_mac_82599EB) { rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); + IXGBE_RXDCTL_RLPML_EN); } #endif if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { - u32 xsk_buf_len = xsk_pool_get_rx_frame_size(ring->xsk_pool); + u32 pkt_len = + READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; - rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | - IXGBE_RXDCTL_RLPML_EN); - rxdctl |= xsk_buf_len | IXGBE_RXDCTL_RLPML_EN; - - ring->rx_buf_len = xsk_buf_len; + rxdctl &= ~(IXGBE_RXDCTL_RLPMLMASK | IXGBE_RXDCTL_RLPML_EN); + if (pkt_len <= IXGBE_RXDCTL_RLPMLMASK) + rxdctl |= pkt_len | IXGBE_RXDCTL_RLPML_EN; } - /* initialize rx_buffer_info */ - memset(ring->rx_buffer_info, 0, - sizeof(struct ixgbe_rx_buffer) * ring->count); + + ixgbe_configure_srrctl(adapter, ring); /* initialize Rx descriptor 0 */ rx_desc = IXGBE_RX_DESC(ring, 0); rx_desc->wb.upper.length = 0; @@ -4232,10 +4121,10 @@ static void ixgbe_setup_psrtype(struct ixgbe_adapter *adapter) /* PSRTYPE must be initialized in non 82598 adapters */ u32 psrtype = IXGBE_PSRTYPE_TCPHDR | - IXGBE_PSRTYPE_UDPHDR | - IXGBE_PSRTYPE_IPV4HDR | - IXGBE_PSRTYPE_L2HDR | - IXGBE_PSRTYPE_IPV6HDR; + IXGBE_PSRTYPE_UDPHDR | + IXGBE_PSRTYPE_IPV4HDR | + IXGBE_PSRTYPE_L2HDR | + IXGBE_PSRTYPE_IPV6HDR; if (hw->mac.type == ixgbe_mac_82598EB) return; @@ -4320,40 +4209,6 @@ static void ixgbe_configure_virtualization(struct ixgbe_adapter *adapter) } } -static void ixgbe_set_rx_buffer_len(struct ixgbe_adapter *adapter) -{ - struct ixgbe_hw *hw = &adapter->hw; - struct net_device *netdev = adapter->netdev; - int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; - u32 mhadd, hlreg0; - -#ifdef IXGBE_FCOE - /* adjust max frame to be able to do baby jumbo for FCoE */ - if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && - (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) - max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; -#endif /* IXGBE_FCOE */ - - /* adjust max frame to be at least the size of a standard frame */ - if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) - max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); - - mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); - if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { - mhadd &= ~IXGBE_MHADD_MFS_MASK; - mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; - - IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); - } - - hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); - /* set jumbo enable since MHADD.MFS is keeping size locked at - * max_frame - */ - hlreg0 |= IXGBE_HLREG0_JUMBOEN; - IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); -} - static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; @@ -4407,8 +4262,12 @@ static void ixgbe_setup_rdrxctl(struct ixgbe_adapter *adapter) static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) { struct ixgbe_hw *hw = &adapter->hw; + struct net_device *netdev = adapter->netdev; + int max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN; + struct ixgbe_ring *rx_ring; int i; u32 rxctrl, rfctl; + u32 mhadd, hlreg0; /* disable receives while setting up the descriptors */ hw->mac.ops.disable_rx(hw); @@ -4429,15 +4288,38 @@ static void ixgbe_configure_rx(struct ixgbe_adapter *adapter) /* Program registers for the distribution of queues */ ixgbe_setup_mrqc(adapter); - /* set_rx_buffer_len must be called before ring initialization */ - ixgbe_set_rx_buffer_len(adapter); +#ifdef IXGBE_FCOE + /* adjust max frame to be able to do baby jumbo for FCoE */ + if ((adapter->flags & IXGBE_FLAG_FCOE_ENABLED) && + (max_frame < IXGBE_FCOE_JUMBO_FRAME_SIZE)) + max_frame = IXGBE_FCOE_JUMBO_FRAME_SIZE; +#endif /* IXGBE_FCOE */ + + /* adjust max frame to be at least the size of a standard frame */ + if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) + max_frame = (ETH_FRAME_LEN + ETH_FCS_LEN); + + mhadd = IXGBE_READ_REG(hw, IXGBE_MHADD); + if (max_frame != (mhadd >> IXGBE_MHADD_MFS_SHIFT)) { + mhadd &= ~IXGBE_MHADD_MFS_MASK; + mhadd |= max_frame << IXGBE_MHADD_MFS_SHIFT; + + IXGBE_WRITE_REG(hw, IXGBE_MHADD, mhadd); + } + + /* set jumbo enable since MHADD.MFS is keeping size locked at + * max_frame + */ + hlreg0 = IXGBE_READ_REG(hw, IXGBE_HLREG0); + hlreg0 |= IXGBE_HLREG0_JUMBOEN; + IXGBE_WRITE_REG(hw, IXGBE_HLREG0, hlreg0); /* * Setup the HW Rx Head and Tail Descriptor Pointers and * the Base and Length of the Rx Descriptor Ring */ for (i = 0; i < adapter->num_rx_queues; i++) { - struct ixgbe_ring *rx_ring = adapter->rx_ring[i]; + rx_ring = adapter->rx_ring[i]; clear_ring_rsc_enabled(rx_ring); if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) @@ -4708,7 +4590,7 @@ static void ixgbe_vlan_promisc_disable(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_VLNCTRL, vlnctrl); if (!(adapter->flags & IXGBE_FLAG_VMDQ_ENABLED) || - hw->mac.type == ixgbe_mac_82598EB) + hw->mac.type == ixgbe_mac_82598EB) return; /* We are not in VLAN promisc, nothing to do */ @@ -4773,9 +4655,9 @@ void ixgbe_full_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -4796,9 +4678,9 @@ static void ixgbe_sync_mac_table(struct ixgbe_adapter *adapter) if (mac_table->state & IXGBE_MAC_STATE_IN_USE) hw->mac.ops.set_rar(hw, i, - mac_table->addr, - mac_table->pool, - IXGBE_RAH_AV); + mac_table->addr, + mac_table->pool, + IXGBE_RAH_AV); else hw->mac.ops.clear_rar(hw, i); } @@ -4853,7 +4735,7 @@ static void ixgbe_mac_set_default_filter(struct ixgbe_adapter *adapter) mac_table->state = IXGBE_MAC_STATE_DEFAULT | IXGBE_MAC_STATE_IN_USE; hw->mac.ops.set_rar(hw, 0, mac_table->addr, mac_table->pool, - IXGBE_RAH_AV); + IXGBE_RAH_AV); } int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, @@ -4874,7 +4756,7 @@ int ixgbe_add_mac_filter(struct ixgbe_adapter *adapter, mac_table->pool = pool; mac_table->state |= IXGBE_MAC_STATE_MODIFIED | - IXGBE_MAC_STATE_IN_USE; + IXGBE_MAC_STATE_IN_USE; ixgbe_sync_mac_table(adapter); @@ -5165,7 +5047,7 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == ixgbe_fcoe_get_tc(adapter))) + (pb == ixgbe_fcoe_get_tc(adapter))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5199,8 +5081,8 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) */ if (marker < 0) { e_warn(drv, "Packet Buffer(%i) can not provide enough" - "headroom to support flow control." - "Decrease MTU or number of traffic classes\n", pb); + "headroom to support flow control." + "Decrease MTU or number of traffic classes\n", pb); marker = tc + 1; } @@ -5226,7 +5108,7 @@ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && - (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) + (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5280,7 +5162,7 @@ static void ixgbe_configure_pb(struct ixgbe_adapter *adapter) u8 tc = adapter->hw_tcs; if (adapter->flags & IXGBE_FLAG_FDIR_HASH_CAPABLE || - adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) + adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) hdrm = 32 << adapter->fdir_pballoc; else hdrm = 0; @@ -5311,13 +5193,13 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) if (!vf && (ring >= adapter->num_rx_queues)) { e_err(drv, "FDIR restore failed without VF, ring: %u\n", - ring); + ring); continue; } else if (vf && ((vf > adapter->num_vfs) || - ring >= adapter->num_rx_queues_per_pool)) { + ring >= adapter->num_rx_queues_per_pool)) { e_err(drv, "FDIR restore failed with VF, vf: %hhu, ring: %u\n", - vf, ring); + vf, ring); continue; } @@ -5342,9 +5224,6 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) **/ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { - u16 i = rx_ring->next_to_clean; - struct ixgbe_rx_buffer *rx_buffer = &rx_ring->rx_buffer_info[i]; - if (rx_ring->xsk_pool) { ixgbe_xsk_clean_rx_ring(rx_ring); goto skip_free; @@ -5356,29 +5235,12 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) rx_ring->skb = NULL; } - /* Free all the Rx ring pages */ - while (i != rx_ring->next_to_use) { - /* Invalidate cache lines that may have been written to by - * device so that we avoid corrupting memory. - */ - dma_sync_single_range_for_cpu(rx_ring->dev, - rx_buffer->dma, - rx_buffer->page_offset, - IXGBE_RXBUFFER_3K, - DMA_FROM_DEVICE); - - /* free resources associated with mapping */ - dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma, - PAGE_SIZE, DMA_FROM_DEVICE, - IXGBE_RX_DMA_ATTR); - __free_page(rx_buffer->page); + for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { + const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; - i++; - rx_buffer++; - if (i == rx_ring->count) { + libeth_rx_recycle_slow(rx_fqe->netmem); + if (unlikely(++i == rx_ring->count)) i = 0; - rx_buffer = rx_ring->rx_buffer_info; - } } skip_free: @@ -5387,7 +5249,7 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) } static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, - struct ixgbe_fwd_adapter *accel) + struct ixgbe_fwd_adapter *accel) { u16 rss_i = adapter->ring_feature[RING_F_RSS].indices; int num_tc = netdev_get_num_tc(adapter->netdev); @@ -5405,7 +5267,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, /* record configuration for macvlan interface in vdev */ for (i = 0; i < num_tc; i++) netdev_bind_sb_channel_queue(adapter->netdev, vdev, - i, rss_i, baseq + (rss_i * i)); + i, rss_i, baseq + (rss_i * i)); for (i = 0; i < adapter->num_rx_queues_per_pool; i++) adapter->rx_ring[baseq + i]->netdev = vdev; @@ -5442,7 +5304,7 @@ static int ixgbe_fwd_ring_up(struct ixgbe_adapter *adapter, } static int ixgbe_macvlan_up(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -5466,7 +5328,7 @@ static void ixgbe_configure_dfwd(struct ixgbe_adapter *adapter) }; netdev_walk_all_upper_dev_rcu(adapter->netdev, - ixgbe_macvlan_up, &priv); + ixgbe_macvlan_up, &priv); } static void ixgbe_configure(struct ixgbe_adapter *adapter) @@ -5501,7 +5363,7 @@ static void ixgbe_configure(struct ixgbe_adapter *adapter) adapter->fdir_pballoc); } else if (adapter->flags & IXGBE_FLAG_FDIR_PERFECT_CAPABLE) { ixgbe_init_fdir_perfect_82599(&adapter->hw, - adapter->fdir_pballoc); + adapter->fdir_pballoc); ixgbe_fdir_filter_restore(adapter); } @@ -5602,7 +5464,7 @@ static void ixgbe_sfp_link_config(struct ixgbe_adapter *adapter) static int ixgbe_non_sfp_link_config(struct ixgbe_hw *hw) { struct ixgbe_adapter *adapter = container_of(hw, struct ixgbe_adapter, - hw); + hw); u16 mask = ~((u16)(IXGBE_ACI_LINK_EVENT_UPDOWN | IXGBE_ACI_LINK_EVENT_MEDIA_NA | IXGBE_ACI_LINK_EVENT_MODULE_QUAL_FAIL | @@ -5726,7 +5588,7 @@ static void ixgbe_setup_gpie(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_MSIX_ENABLED) { gpie = IXGBE_GPIE_MSIX_MODE | IXGBE_GPIE_PBA_SUPPORT | - IXGBE_GPIE_OCD; + IXGBE_GPIE_OCD; gpie |= IXGBE_GPIE_EIAME; /* * use EIAM to auto-mask when MSI-X interrupt is asserted @@ -5960,7 +5822,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* Determine our minimum delay interval. We will increase this value @@ -6001,7 +5863,7 @@ void ixgbe_disable_rx(struct ixgbe_adapter *adapter) } e_err(drv, - "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "RXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); } void ixgbe_disable_tx(struct ixgbe_adapter *adapter) @@ -6082,7 +5944,7 @@ void ixgbe_disable_tx(struct ixgbe_adapter *adapter) } e_err(drv, - "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); + "TXDCTL.ENABLE for one or more queues not cleared within the polling period\n"); dma_engine_disable: /* Disable the Tx DMA engine on 82599 and later MAC */ @@ -6116,7 +5978,7 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) /* clear all SFP and link config related flags while holding SFP_INIT */ adapter->flags2 &= ~(IXGBE_FLAG2_SEARCH_FOR_SFP | - IXGBE_FLAG2_SFP_NEEDS_RESET); + IXGBE_FLAG2_SFP_NEEDS_RESET); adapter->flags &= ~IXGBE_FLAG_NEED_LINK_CONFIG; err = hw->mac.ops.init_hw(hw); @@ -6212,9 +6074,9 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) /* unmap any remaining paged data */ if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); } /* move us one more past the eop_desc for start of next pkt */ @@ -6428,7 +6290,7 @@ static void ixgbe_init_dcb(struct ixgbe_adapter *adapter) if (adapter->flags & IXGBE_FLAG_DCB_CAPABLE) adapter->dcbx_cap = DCB_CAP_DCBX_HOST | DCB_CAP_DCBX_VER_CEE; memcpy(&adapter->temp_dcb_cfg, &adapter->dcb_cfg, - sizeof(adapter->temp_dcb_cfg)); + sizeof(adapter->temp_dcb_cfg)); } #endif @@ -6500,8 +6362,8 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter, adapter->jump_tables[i] = NULL; adapter->mac_table = kcalloc(hw->mac.num_rar_entries, - sizeof(struct ixgbe_mac_addr), - GFP_KERNEL); + sizeof(struct ixgbe_mac_addr), + GFP_KERNEL); if (!adapter->mac_table) return -ENOMEM; @@ -6737,13 +6599,6 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } -static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) -{ - struct ixgbe_q_vector *q_vector = rx_ring->q_vector; - - return q_vector ? q_vector->napi.napi_id : 0; -} - /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) * @adapter: pointer to ixgbe_adapter @@ -6752,56 +6607,77 @@ static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) * Returns 0 on success, negative on failure **/ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) + struct ixgbe_ring *rx_ring) { - struct device *dev = rx_ring->dev; + struct libeth_fq fq = { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_MTU, + .xdp = !!rx_ring->xdp_prog, + .buf_len = IXGBE_RX_PAGE_LEN(rx_ring->xdp_prog ? + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), + }; + struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); int ring_node = NUMA_NO_NODE; - int size; - - size = sizeof(struct ixgbe_rx_buffer) * rx_ring->count; + int ret; - if (rx_ring->q_vector) + if (rx_ring->q_vector) { + fq.nid = rx_ring->q_vector->numa_node; ring_node = rx_ring->q_vector->numa_node; + } - rx_ring->rx_buffer_info = vmalloc_node(size, ring_node); - if (!rx_ring->rx_buffer_info) - rx_ring->rx_buffer_info = vmalloc(size); - if (!rx_ring->rx_buffer_info) - goto err; + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + return ret; + + rx_ring->pp = fq.pp; + rx_ring->rx_fqes = fq.fqes; + rx_ring->truesize = fq.truesize; + rx_ring->rx_buf_len = fq.buf_len; /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); set_dev_node(dev, ring_node); - rx_ring->desc = dma_alloc_coherent(dev, - rx_ring->size, - &rx_ring->dma, - GFP_KERNEL); + rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, + &rx_ring->dma, GFP_KERNEL); set_dev_node(dev, orig_node); if (!rx_ring->desc) rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size, - &rx_ring->dma, GFP_KERNEL); - if (!rx_ring->desc) - goto err; + &rx_ring->dma, + GFP_KERNEL); + if (!rx_ring->desc) { + ret = -ENOMEM; + goto err_destroy_fq; + } rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; /* XDP RX-queue info */ - if (xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring)) < 0) - goto err; + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, + rx_ring->queue_index, 0, rx_ring->truesize); + if (ret < 0) + goto err_free_desc; - rx_ring->xdp_prog = adapter->xdp_prog; + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; -err: - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; + +err_free_desc: + dma_free_coherent(dev, rx_ring->size, + rx_ring->desc, rx_ring->dma); + rx_ring->desc = NULL; +err_destroy_fq: dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); - return -ENOMEM; + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + return ret; } /** @@ -6888,21 +6764,32 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) **/ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { + struct libeth_fq fq = { + .fqes = rx_ring->rx_fqes, + .pp = rx_ring->pp, + }; + ixgbe_clean_rx_ring(rx_ring); - rx_ring->xdp_prog = NULL; + rcu_assign_pointer(rx_ring->xdp_prog, NULL); + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); xdp_rxq_info_unreg(&rx_ring->xdp_rxq); - vfree(rx_ring->rx_buffer_info); - rx_ring->rx_buffer_info = NULL; /* if not set, then don't free */ if (!rx_ring->desc) return; - dma_free_coherent(rx_ring->dev, rx_ring->size, - rx_ring->desc, rx_ring->dma); + dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + rx_ring->dma); rx_ring->desc = NULL; + + kvfree(rx_ring->rx_xsk_buffer_info); + rx_ring->rx_xsk_buffer_info = NULL; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; } /** @@ -6945,7 +6832,7 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); if (ixgbe_enabled_xdp_adapter(adapter)) { - int new_frame_size = new_mtu + IXGBE_PKT_HDR_PAD; + int new_frame_size = new_mtu + LIBETH_RX_LL_LEN; if (new_frame_size > ixgbe_max_xdp_frame_size(adapter)) { e_warn(probe, "Requested MTU size is not supported with XDP\n"); @@ -6959,8 +6846,8 @@ static int ixgbe_change_mtu(struct net_device *netdev, int new_mtu) * warning that legacy VFs will be disabled. */ if ((adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) && - (adapter->hw.mac.type == ixgbe_mac_82599EB) && - (new_mtu > ETH_DATA_LEN)) + (adapter->hw.mac.type == ixgbe_mac_82599EB) && + (new_mtu > ETH_DATA_LEN)) e_warn(probe, "Setting MTU > 1500 will disable legacy VFs\n"); netdev_dbg(netdev, "changing MTU from %d to %d\n", @@ -7258,7 +7145,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) u64 bytes = 0, packets = 0, hw_csum_rx_error = 0; if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_RESETTING, &adapter->state)) return; if (adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED) { @@ -7359,11 +7246,11 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) hwstats->qptc[i] += IXGBE_READ_REG(hw, IXGBE_QPTC(i)); hwstats->qprc[i] += IXGBE_READ_REG(hw, IXGBE_QPRC(i)); if (hw->mac.type == ixgbe_mac_82599EB || - hw->mac.type == ixgbe_mac_X540 || - hw->mac.type == ixgbe_mac_X550 || - hw->mac.type == ixgbe_mac_X550EM_x || - hw->mac.type == ixgbe_mac_x550em_a || - hw->mac.type == ixgbe_mac_e610) { + hw->mac.type == ixgbe_mac_X540 || + hw->mac.type == ixgbe_mac_X550 || + hw->mac.type == ixgbe_mac_X550EM_x || + hw->mac.type == ixgbe_mac_x550em_a || + hw->mac.type == ixgbe_mac_e610) { hwstats->qbtc[i] += IXGBE_READ_REG(hw, IXGBE_QBTC_L(i)); IXGBE_READ_REG(hw, IXGBE_QBTC_H(i)); /* to clear */ hwstats->qbrc[i] += IXGBE_READ_REG(hw, IXGBE_QBRC_L(i)); @@ -7399,7 +7286,7 @@ void ixgbe_update_stats(struct ixgbe_adapter *adapter) case ixgbe_mac_82599EB: for (i = 0; i < 16; i++) adapter->hw_rx_no_dma_resources += - IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); + IXGBE_READ_REG(hw, IXGBE_QPRDC(i)); hwstats->gorc += IXGBE_READ_REG(hw, IXGBE_GORCL); IXGBE_READ_REG(hw, IXGBE_GORCH); /* to clear */ hwstats->gotc += IXGBE_READ_REG(hw, IXGBE_GOTCL); @@ -7546,7 +7433,7 @@ static void ixgbe_fdir_reinit_subtask(struct ixgbe_adapter *adapter) IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMS_FLOW_DIR); } else { e_err(probe, "failed to finish FDIR re-initialization, " - "ignored adding FDIR ATR filters\n"); + "ignored adding FDIR ATR filters\n"); } } @@ -7567,8 +7454,8 @@ static void ixgbe_check_hang_subtask(struct ixgbe_adapter *adapter) /* If we're down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; /* Force detection of hung controller */ @@ -7628,7 +7515,7 @@ static void ixgbe_watchdog_update_link(struct ixgbe_adapter *adapter) } if (link_up || - time_after(jiffies, (adapter->link_check_timeout + + time_after(jiffies, (adapter->link_check_timeout + IXGBE_TRY_LINK_TIMEOUT))) { adapter->flags &= ~IXGBE_FLAG_NEED_LINK_UPDATE; IXGBE_WRITE_REG(hw, IXGBE_EIMS, IXGBE_EIMC_LSC); @@ -7644,9 +7531,9 @@ static void ixgbe_update_default_up(struct ixgbe_adapter *adapter) #ifdef CONFIG_IXGBE_DCB struct net_device *netdev = adapter->netdev; struct dcb_app app = { - .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, - .protocol = 0, - }; + .selector = IEEE_8021QAZ_APP_SEL_ETHERTYPE, + .protocol = 0, + }; u8 up = 0; if (adapter->dcbx_cap & DCB_CAP_DCBX_VER_IEEE) @@ -7730,9 +7617,9 @@ static void ixgbe_watchdog_link_is_up(struct ixgbe_adapter *adapter) break; } e_info(drv, "NIC Link is Up %s, Flow Control: %s\n", speed_str, - ((flow_rx && flow_tx) ? "RX/TX" : - (flow_rx ? "RX" : - (flow_tx ? "TX" : "None")))); + ((flow_rx && flow_tx) ? "RX/TX" : + (flow_rx ? "RX" : + (flow_tx ? "TX" : "None")))); netif_carrier_on(netdev); ixgbe_check_vf_rate_limit(adapter); @@ -7835,7 +7722,7 @@ static void ixgbe_watchdog_flush_tx(struct ixgbe_adapter *adapter) { if (!netif_carrier_ok(adapter->netdev)) { if (ixgbe_ring_tx_pending(adapter) || - ixgbe_vf_tx_pending(adapter)) { + ixgbe_vf_tx_pending(adapter)) { /* We've lost link, so the controller stops DMA, * but we've got queued Tx work that's never going * to get done, so reset controller to flush Tx. @@ -7853,18 +7740,18 @@ static void ixgbe_bad_vf_abort(struct ixgbe_adapter *adapter, u32 vf) struct ixgbe_hw *hw = &adapter->hw; if (adapter->hw.mac.type == ixgbe_mac_82599EB && - adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { + adapter->flags2 & IXGBE_FLAG2_AUTO_DISABLE_VF) { adapter->vfinfo[vf].primary_abort_count++; if (adapter->vfinfo[vf].primary_abort_count == - IXGBE_PRIMARY_ABORT_LIMIT) { + IXGBE_PRIMARY_ABORT_LIMIT) { ixgbe_set_vf_link_state(adapter, vf, IFLA_VF_LINK_STATE_DISABLE); adapter->vfinfo[vf].primary_abort_count = 0; e_info(drv, - "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", - hw->bus.func, vf, - adapter->vfinfo[vf].vf_mac_addresses); + "Malicious Driver Detection event detected on PF %d VF %d MAC: %pM mdd-disable-vf=on", + hw->bus.func, vf, + adapter->vfinfo[vf].vf_mac_addresses); } } } @@ -7900,7 +7787,7 @@ static void ixgbe_check_for_bad_vf(struct ixgbe_adapter *adapter) continue; pci_read_config_word(vfdev, PCI_STATUS, &status_reg); if (status_reg != IXGBE_FAILED_READ_CFG_WORD && - status_reg & PCI_STATUS_REC_MASTER_ABORT) { + status_reg & PCI_STATUS_REC_MASTER_ABORT) { ixgbe_bad_vf_abort(adapter, vf); pcie_flr(vfdev); } @@ -7913,7 +7800,7 @@ static void ixgbe_spoof_check(struct ixgbe_adapter *adapter) /* Do not perform spoof check for 82598 or if not in IOV mode */ if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) return; ssvpc = IXGBE_READ_REG(&adapter->hw, IXGBE_SSVPC); @@ -7947,8 +7834,8 @@ static void ixgbe_watchdog_subtask(struct ixgbe_adapter *adapter) { /* if interface is down, removing or resetting, do nothing */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) return; ixgbe_watchdog_update_link(adapter); @@ -7976,11 +7863,11 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) /* not searching for SFP so there is nothing to do here */ if (!(adapter->flags2 & IXGBE_FLAG2_SEARCH_FOR_SFP) && - !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) + !(adapter->flags2 & IXGBE_FLAG2_SFP_NEEDS_RESET)) return; if (adapter->sfp_poll_time && - time_after(adapter->sfp_poll_time, jiffies)) + time_after(adapter->sfp_poll_time, jiffies)) return; /* If not yet time to poll for SFP */ /* someone else is in init, wait until next service event */ @@ -8029,7 +7916,7 @@ static void ixgbe_sfp_detection_subtask(struct ixgbe_adapter *adapter) clear_bit(__IXGBE_IN_SFP_INIT, &adapter->state); if (err == -EOPNOTSUPP && - adapter->netdev->reg_state == NETREG_REGISTERED) { + adapter->netdev->reg_state == NETREG_REGISTERED) { e_dev_err("failed to initialize because an unsupported " "SFP+ module type was detected.\n"); e_dev_err("Reload the driver after installing a " @@ -8065,7 +7952,7 @@ static void ixgbe_sfp_link_config_subtask(struct ixgbe_adapter *adapter) speed = IXGBE_LINK_SPEED_10GB_FULL; else speed = cap_speed & (IXGBE_LINK_SPEED_10GB_FULL | - IXGBE_LINK_SPEED_1GB_FULL); + IXGBE_LINK_SPEED_1GB_FULL); if (hw->mac.ops.setup_link) hw->mac.ops.setup_link(hw, speed, true); @@ -8123,8 +8010,8 @@ static void ixgbe_reset_subtask(struct ixgbe_adapter *adapter) rtnl_lock(); /* If we're already down, removing or resetting, just bail */ if (test_bit(__IXGBE_DOWN, &adapter->state) || - test_bit(__IXGBE_REMOVING, &adapter->state) || - test_bit(__IXGBE_RESETTING, &adapter->state)) { + test_bit(__IXGBE_REMOVING, &adapter->state) || + test_bit(__IXGBE_RESETTING, &adapter->state)) { rtnl_unlock(); return; } @@ -8182,7 +8069,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) /* skip if E610's FW is reloading, warning in that case may be misleading */ if (fwsm & IXGBE_FWSM_EXT_ERR_IND_MASK || - (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) + (!(fwsm & IXGBE_FWSM_FW_VAL_BIT) && !(hw->mac.type == ixgbe_mac_e610))) e_dev_warn("Warning firmware error detected FWSM: 0x%08X\n", fwsm); @@ -8208,7 +8095,7 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) goto no_version; if (hw->mac.ops.get_nvm_ver && - hw->mac.ops.get_nvm_ver(hw, nvm_info)) + hw->mac.ops.get_nvm_ver(hw, nvm_info)) goto no_version; snprintf(ver_buff, sizeof(ver_buff), @@ -8228,8 +8115,8 @@ static bool ixgbe_check_fw_error(struct ixgbe_adapter *adapter) static void ixgbe_recovery_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); ixgbe_handle_fw_event(adapter); ixgbe_service_event_complete(adapter); @@ -8244,8 +8131,8 @@ static void ixgbe_recovery_service_task(struct work_struct *work) static void ixgbe_service_task(struct work_struct *work) { struct ixgbe_adapter *adapter = container_of(work, - struct ixgbe_adapter, - service_task); + struct ixgbe_adapter, + service_task); if (ixgbe_removed(adapter->hw.hw_addr)) { if (!test_bit(__IXGBE_DOWN, &adapter->state)) { rtnl_lock(); @@ -8291,9 +8178,9 @@ static void ixgbe_service_task(struct work_struct *work) } static int ixgbe_tso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, - u8 *hdr_len, - struct ixgbe_ipsec_tx_data *itd) + struct ixgbe_tx_buffer *first, + u8 *hdr_len, + struct ixgbe_ipsec_tx_data *itd) { u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; struct sk_buff *skb = first->skb; @@ -8329,7 +8216,7 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ? - IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; + IXGBE_ADVTXD_TUCMD_L4T_UDP : IXGBE_ADVTXD_TUCMD_L4T_TCP; /* initialize outer IP header fields */ if (ip.v4->version == 4) { @@ -8366,12 +8253,12 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, /* compute length of segmentation header */ *hdr_len = (l4.tcp->doff * 4) + l4_offset; csum_replace_by_diff(&l4.tcp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } else { /* compute length of segmentation header */ *hdr_len = sizeof(*l4.udp) + l4_offset; csum_replace_by_diff(&l4.udp->check, - (__force __wsum)htonl(paylen)); + (__force __wsum)htonl(paylen)); } /* update gso size and bytecount with header size */ @@ -8455,8 +8342,8 @@ static u32 ixgbe_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) { /* set type for advanced descriptor with frame checksum insertion */ u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS; /* set HW vlan bit if vlan is present */ cmd_type |= IXGBE_SET_FLAG(tx_flags, IXGBE_TX_FLAGS_HW_VLAN, @@ -8510,7 +8397,7 @@ static void ixgbe_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, static int __ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) { if (!netif_subqueue_try_stop(tx_ring->netdev, tx_ring->queue_index, - ixgbe_desc_unused(tx_ring), size)) + ixgbe_desc_unused(tx_ring), size)) return -EBUSY; ++tx_ring->tx_stats.restart_queue; @@ -8610,7 +8497,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, data_len -= size; dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size, - DMA_TO_DEVICE); + DMA_TO_DEVICE); tx_buffer = &tx_ring->tx_buffer_info[i]; } @@ -8660,9 +8547,9 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, tx_buffer = &tx_ring->tx_buffer_info[i]; if (dma_unmap_len(tx_buffer, len)) dma_unmap_page(tx_ring->dev, - dma_unmap_addr(tx_buffer, dma), - dma_unmap_len(tx_buffer, len), - DMA_TO_DEVICE); + dma_unmap_addr(tx_buffer, dma), + dma_unmap_len(tx_buffer, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buffer, len, 0); if (tx_buffer == first) break; @@ -8680,7 +8567,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, } static void ixgbe_atr(struct ixgbe_ring *ring, - struct ixgbe_tx_buffer *first) + struct ixgbe_tx_buffer *first) { struct ixgbe_q_vector *q_vector = ring->q_vector; union ixgbe_atr_hash_dword input = { .dword = 0 }; @@ -8708,7 +8595,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* currently only IPv4/IPv6 with TCP is supported */ if ((first->protocol != htons(ETH_P_IP)) && - (first->protocol != htons(ETH_P_IPV6))) + (first->protocol != htons(ETH_P_IPV6))) return; /* snag network header to get L4 type and address */ @@ -8717,21 +8604,21 @@ static void ixgbe_atr(struct ixgbe_ring *ring, if (unlikely(hdr.network <= skb->data)) return; if (skb->encapsulation && - first->protocol == htons(ETH_P_IP) && - hdr.ipv4->protocol == IPPROTO_UDP) { + first->protocol == htons(ETH_P_IP) && + hdr.ipv4->protocol == IPPROTO_UDP) { struct ixgbe_adapter *adapter = q_vector->adapter; if (unlikely(skb_tail_pointer(skb) < hdr.network + - vxlan_headroom(0))) + vxlan_headroom(0))) return; /* verify the port is recognized as VXLAN */ if (adapter->vxlan_port && - udp_hdr(skb)->dest == adapter->vxlan_port) + udp_hdr(skb)->dest == adapter->vxlan_port) hdr.network = skb_inner_network_header(skb); if (adapter->geneve_port && - udp_hdr(skb)->dest == adapter->geneve_port) + udp_hdr(skb)->dest == adapter->geneve_port) hdr.network = skb_inner_network_header(skb); } @@ -8761,7 +8648,7 @@ static void ixgbe_atr(struct ixgbe_ring *ring, return; if (unlikely(skb_tail_pointer(skb) < hdr.network + - hlen + sizeof(struct tcphdr))) + hlen + sizeof(struct tcphdr))) return; th = (struct tcphdr *)(hdr.network + hlen); @@ -8806,13 +8693,13 @@ static void ixgbe_atr(struct ixgbe_ring *ring, case 6: input.formatted.flow_type = IXGBE_ATR_FLOW_TYPE_TCPV6; common.ip ^= hdr.ipv6->saddr.s6_addr32[0] ^ - hdr.ipv6->saddr.s6_addr32[1] ^ - hdr.ipv6->saddr.s6_addr32[2] ^ - hdr.ipv6->saddr.s6_addr32[3] ^ - hdr.ipv6->daddr.s6_addr32[0] ^ - hdr.ipv6->daddr.s6_addr32[1] ^ - hdr.ipv6->daddr.s6_addr32[2] ^ - hdr.ipv6->daddr.s6_addr32[3]; + hdr.ipv6->saddr.s6_addr32[1] ^ + hdr.ipv6->saddr.s6_addr32[2] ^ + hdr.ipv6->saddr.s6_addr32[3] ^ + hdr.ipv6->daddr.s6_addr32[0] ^ + hdr.ipv6->daddr.s6_addr32[1] ^ + hdr.ipv6->daddr.s6_addr32[2] ^ + hdr.ipv6->daddr.s6_addr32[3]; break; default: break; @@ -8823,12 +8710,12 @@ static void ixgbe_atr(struct ixgbe_ring *ring, /* This assumes the Rx queue and Tx queue are bound to the same CPU */ ixgbe_fdir_add_signature_filter_82599(&q_vector->adapter->hw, - input, common, ring->queue_index); + input, common, ring->queue_index); } #ifdef IXGBE_FCOE static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, - struct net_device *sb_dev) + struct net_device *sb_dev) { struct ixgbe_adapter *adapter; struct ixgbe_ring_feature *f; @@ -8941,8 +8828,8 @@ int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, tx_buff = &ring->tx_buffer_info[index]; if (dma_unmap_len(tx_buff, len)) dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma), - dma_unmap_len(tx_buff, len), - DMA_TO_DEVICE); + dma_unmap_len(tx_buff, len), + DMA_TO_DEVICE); dma_unmap_len_set(tx_buff, len, 0); if (tx_buff == tx_head) break; @@ -9008,9 +8895,9 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, protocol = vlan_get_protocol(skb); if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) && - adapter->ptp_clock) { + adapter->ptp_clock) { if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && - !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, + !test_and_set_bit_lock(__IXGBE_PTP_TX_IN_PROGRESS, &adapter->state)) { skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; tx_flags |= IXGBE_TX_FLAGS_TSTAMP; @@ -9035,8 +8922,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #endif /* DCB maps skb priorities 0-7 onto 3 bit PCP of VLAN tag. */ if ((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || - (skb->priority != TC_PRIO_CONTROL))) { + ((tx_flags & (IXGBE_TX_FLAGS_HW_VLAN | IXGBE_TX_FLAGS_SW_VLAN)) || + (skb->priority != TC_PRIO_CONTROL))) { tx_flags &= ~IXGBE_TX_FLAGS_VLAN_PRIO_MASK; tx_flags |= (skb->priority & 0x7) << IXGBE_TX_FLAGS_VLAN_PRIO_SHIFT; @@ -9060,7 +8947,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef IXGBE_FCOE /* setup tx offload for FCoE */ if ((protocol == htons(ETH_P_FCOE)) && - (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { + (tx_ring->netdev->features & (NETIF_F_FSO | NETIF_F_FCOE_CRC))) { tso = ixgbe_fso(tx_ring, first, &hdr_len); if (tso < 0) goto out_drop; @@ -9072,7 +8959,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, #ifdef CONFIG_IXGBE_IPSEC if (xfrm_offload(skb) && - !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) + !ixgbe_ipsec_tx(tx_ring, first, &ipsec_tx)) goto out_drop; #endif tso = ixgbe_tso(tx_ring, first, &hdr_len, &ipsec_tx); @@ -9108,8 +8995,8 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, } static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev, - struct ixgbe_ring *ring) + struct net_device *netdev, + struct ixgbe_ring *ring) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_ring *tx_ring; @@ -9129,7 +9016,7 @@ static netdev_tx_t __ixgbe_xmit_frame(struct sk_buff *skb, } static netdev_tx_t ixgbe_xmit_frame(struct sk_buff *skb, - struct net_device *netdev) + struct net_device *netdev) { return __ixgbe_xmit_frame(skb, netdev, NULL); } @@ -9185,7 +9072,7 @@ ixgbe_mdio_read(struct net_device *netdev, int prtad, int devad, u16 addr) } static int ixgbe_mdio_write(struct net_device *netdev, int prtad, int devad, - u16 addr, u16 value) + u16 addr, u16 value) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); struct ixgbe_hw *hw = &adapter->hw; @@ -9282,7 +9169,7 @@ static void ixgbe_get_ring_stats64(struct rtnl_link_stats64 *stats, } static void ixgbe_get_stats64(struct net_device *netdev, - struct rtnl_link_stats64 *stats) + struct rtnl_link_stats64 *stats) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); int i; @@ -9406,7 +9293,7 @@ static void ixgbe_set_prio_tc_map(struct ixgbe_adapter *adapter) #endif /* CONFIG_IXGBE_DCB */ static int ixgbe_reassign_macvlan_pool(struct net_device *vdev, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { struct ixgbe_adapter *adapter = (struct ixgbe_adapter *)priv->data; struct ixgbe_fwd_adapter *accel; @@ -9454,7 +9341,7 @@ static void ixgbe_defrag_macvlan_pools(struct net_device *dev) /* walk through upper devices reassigning pools */ netdev_walk_all_upper_dev_rcu(dev, ixgbe_reassign_macvlan_pool, - &priv); + &priv); } /** @@ -9534,7 +9421,7 @@ int ixgbe_setup_tc(struct net_device *dev, u8 tc) } static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 hdl = cls->knode.handle; u32 uhtid = TC_U32_USERHTID(cls->knode.handle); @@ -9570,8 +9457,8 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, continue; spin_lock(&adapter->fdir_perfect_lock); err = ixgbe_update_ethtool_fdir_entry(adapter, - NULL, - j + 1); + NULL, + j + 1); spin_unlock(&adapter->fdir_perfect_lock); clear_bit(j, jump->child_loc_map); } @@ -9591,7 +9478,7 @@ static int ixgbe_delete_clsu32(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -9609,7 +9496,7 @@ static int ixgbe_configure_clsu32_add_hnode(struct ixgbe_adapter *adapter, } static int ixgbe_configure_clsu32_del_hnode(struct ixgbe_adapter *adapter, - struct tc_cls_u32_offload *cls) + struct tc_cls_u32_offload *cls) { u32 uhtid = TC_U32_USERHTID(cls->hnode.handle); @@ -9629,7 +9516,7 @@ struct upper_walk_data { }; static int get_macvlan_queue(struct net_device *upper, - struct netdev_nested_priv *priv) + struct netdev_nested_priv *priv) { if (netif_is_macvlan(upper)) { struct ixgbe_fwd_adapter *vadapter = macvlan_accel_priv(upper); @@ -9688,7 +9575,7 @@ static int handle_redirect_action(struct ixgbe_adapter *adapter, int ifindex, } static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { const struct tc_action *a; int i; @@ -9711,7 +9598,7 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, if (!dev) return -EINVAL; return handle_redirect_action(adapter, dev->ifindex, - queue, action); + queue, action); } return -EINVAL; @@ -9721,17 +9608,17 @@ static int parse_tc_actions(struct ixgbe_adapter *adapter, } #else static int parse_tc_actions(struct ixgbe_adapter *adapter, - struct tcf_exts *exts, u64 *action, u8 *queue) + struct tcf_exts *exts, u64 *action, u8 *queue) { return -EINVAL; } #endif /* CONFIG_NET_CLS_ACT */ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, - union ixgbe_atr_input *mask, - struct tc_cls_u32_offload *cls, - struct ixgbe_mat_field *field_ptr, - struct ixgbe_nexthdr *nexthdr) + union ixgbe_atr_input *mask, + struct tc_cls_u32_offload *cls, + struct ixgbe_mat_field *field_ptr, + struct ixgbe_nexthdr *nexthdr) { int i, j, off; __be32 val, m; @@ -9754,10 +9641,10 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, } if (nexthdr) { if (nexthdr->off == cls->knode.sel->keys[i].off && - nexthdr->val == - (__force u32)cls->knode.sel->keys[i].val && - nexthdr->mask == - (__force u32)cls->knode.sel->keys[i].mask) + nexthdr->val == + (__force u32)cls->knode.sel->keys[i].val && + nexthdr->mask == + (__force u32)cls->knode.sel->keys[i].mask) found_jump_field = true; else continue; @@ -9771,7 +9658,7 @@ static int ixgbe_clsu32_build_input(struct ixgbe_fdir_filter *input, return 0; mask->formatted.flow_type = IXGBE_ATR_L4TYPE_IPV6_MASK | - IXGBE_ATR_L4TYPE_MASK; + IXGBE_ATR_L4TYPE_MASK; if (input->filter.formatted.flow_type == IXGBE_ATR_FLOW_TYPE_IPV4) mask->formatted.flow_type &= IXGBE_ATR_L4TYPE_IPV6_MASK; @@ -9852,17 +9739,17 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, * and link to it. */ if (adapter->jump_tables[link_uhtid] && - (adapter->jump_tables[link_uhtid])->link_hdl) { + (adapter->jump_tables[link_uhtid])->link_hdl) { e_err(drv, "Link filter exists for link: %x\n", - link_uhtid); + link_uhtid); return err; } for (i = 0; nexthdr[i].jump; i++) { if (nexthdr[i].o != cls->knode.sel->offoff || - nexthdr[i].s != cls->knode.sel->offshift || - nexthdr[i].m != - (__force u32)cls->knode.sel->offmask) + nexthdr[i].s != cls->knode.sel->offshift || + nexthdr[i].m != + (__force u32)cls->knode.sel->offmask) return err; jump = kzalloc(sizeof(*jump), GFP_KERNEL); @@ -9883,7 +9770,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, jump->link_hdl = cls->knode.handle; err = ixgbe_clsu32_build_input(input, mask, cls, - field_ptr, &nexthdr[i]); + field_ptr, &nexthdr[i]); if (!err) { jump->mat = nexthdr[i].jump; adapter->jump_tables[link_uhtid] = jump; @@ -9909,10 +9796,10 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if ((uhtid != 0x800) && (adapter->jump_tables[uhtid])) { if ((adapter->jump_tables[uhtid])->input) memcpy(input, (adapter->jump_tables[uhtid])->input, - sizeof(*input)); + sizeof(*input)); if ((adapter->jump_tables[uhtid])->mask) memcpy(mask, (adapter->jump_tables[uhtid])->mask, - sizeof(*mask)); + sizeof(*mask)); /* Lookup in all child hash tables if this location is already * filled with a filter @@ -9922,7 +9809,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, if (link && (test_bit(loc - 1, link->child_loc_map))) { e_err(drv, "Filter exists in location: %x\n", - loc); + loc); err = -EINVAL; goto err_out; } @@ -9933,7 +9820,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, goto err_out; err = parse_tc_actions(adapter, cls->knode.exts, &input->action, - &queue); + &queue); if (err < 0) goto err_out; @@ -9953,7 +9840,7 @@ static int ixgbe_configure_clsu32(struct ixgbe_adapter *adapter, ixgbe_atr_compute_perfect_hash_82599(&input->filter, mask); err = ixgbe_fdir_write_perfect_filter_82599(hw, &input->filter, - input->sw_idx, queue); + input->sw_idx, queue); if (err) goto err_out_w_lock; @@ -10021,7 +9908,7 @@ static int ixgbe_setup_tc_mqprio(struct net_device *dev, static LIST_HEAD(ixgbe_block_cb_list); static int __ixgbe_setup_tc(struct net_device *dev, enum tc_setup_type type, - void *type_data) + void *type_data) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10060,7 +9947,7 @@ void ixgbe_do_reset(struct net_device *netdev) } static netdev_features_t ixgbe_fix_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); @@ -10088,7 +9975,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) /* go back to full RSS if we're not running SR-IOV */ if (!adapter->ring_feature[RING_F_VMDQ].offset) adapter->flags &= ~(IXGBE_FLAG_VMDQ_ENABLED | - IXGBE_FLAG_SRIOV_ENABLED); + IXGBE_FLAG_SRIOV_ENABLED); adapter->ring_feature[RING_F_RSS].limit = rss; adapter->ring_feature[RING_F_VMDQ].limit = 1; @@ -10097,7 +9984,7 @@ static void ixgbe_reset_l2fw_offload(struct ixgbe_adapter *adapter) } static int ixgbe_set_features(struct net_device *netdev, - netdev_features_t features) + netdev_features_t features) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(netdev); netdev_features_t changed = netdev->features ^ features; @@ -10111,12 +9998,12 @@ static int ixgbe_set_features(struct net_device *netdev, } else if ((adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) && !(adapter->flags2 & IXGBE_FLAG2_RSC_ENABLED)) { if (adapter->rx_itr_setting == 1 || - adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { + adapter->rx_itr_setting > IXGBE_MIN_RSC_ITR) { adapter->flags2 |= IXGBE_FLAG2_RSC_ENABLED; need_reset = true; } else if ((changed ^ features) & NETIF_F_LRO) { e_info(probe, "rx-usecs set too low, " - "disabling RSC\n"); + "disabling RSC\n"); } } @@ -10140,12 +10027,12 @@ static int ixgbe_set_features(struct net_device *netdev, /* We cannot enable ATR if SR-IOV is enabled */ if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED || - /* We cannot enable ATR if we have 2 or more tcs */ - (adapter->hw_tcs > 1) || - /* We cannot enable ATR if RSS is disabled */ - (adapter->ring_feature[RING_F_RSS].limit <= 1) || - /* A sample rate of 0 indicates ATR disabled */ - (!adapter->atr_sample_rate)) + /* We cannot enable ATR if we have 2 or more tcs */ + (adapter->hw_tcs > 1) || + /* We cannot enable ATR if RSS is disabled */ + (adapter->ring_feature[RING_F_RSS].limit <= 1) || + /* A sample rate of 0 indicates ATR disabled */ + (!adapter->atr_sample_rate)) ; /* do nothing not supported */ else /* otherwise supported and set the flag */ adapter->flags |= IXGBE_FLAG_FDIR_HASH_CAPABLE; @@ -10161,17 +10048,17 @@ static int ixgbe_set_features(struct net_device *netdev, else if (need_reset) ixgbe_do_reset(netdev); else if (changed & (NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_FILTER)) + NETIF_F_HW_VLAN_CTAG_FILTER)) ixgbe_set_rx_mode(netdev); return 1; } static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], - struct net_device *dev, - const unsigned char *addr, u16 vid, - u16 flags, bool *notified, - struct netlink_ext_ack *extack) + struct net_device *dev, + const unsigned char *addr, u16 vid, + u16 flags, bool *notified, + struct netlink_ext_ack *extack) { /* guarantee we can provide a unique filter for the unicast address */ if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) { @@ -10193,7 +10080,7 @@ static int ixgbe_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[], * Configure some settings require for various bridge modes. **/ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, - __u16 mode) + __u16 mode) { struct ixgbe_hw *hw = &adapter->hw; unsigned int p, num_pools; @@ -10219,8 +10106,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - true, - p); + true, + p); } break; case BRIDGE_MODE_VEB: @@ -10243,8 +10130,8 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, for (p = 0; p < num_pools; p++) { if (hw->mac.ops.set_source_address_pruning) hw->mac.ops.set_source_address_pruning(hw, - false, - p); + false, + p); } break; default: @@ -10254,14 +10141,14 @@ static int ixgbe_configure_bridge_mode(struct ixgbe_adapter *adapter, adapter->bridge_mode = mode; e_info(drv, "enabling bridge mode: %s\n", - mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); + mode == BRIDGE_MODE_VEPA ? "VEPA" : "VEB"); return 0; } static int ixgbe_ndo_bridge_setlink(struct net_device *dev, - struct nlmsghdr *nlh, u16 flags, - struct netlink_ext_ack *extack) + struct nlmsghdr *nlh, u16 flags, + struct netlink_ext_ack *extack) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct nlattr *attr, *br_spec; @@ -10288,8 +10175,8 @@ static int ixgbe_ndo_bridge_setlink(struct net_device *dev, } static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, - struct net_device *dev, - u32 filter_mask, int nlflags) + struct net_device *dev, + u32 filter_mask, int nlflags) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); @@ -10297,8 +10184,8 @@ static int ixgbe_ndo_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, return 0; return ndo_dflt_bridge_getlink(skb, pid, seq, dev, - adapter->bridge_mode, 0, 0, nlflags, - filter_mask, NULL); + adapter->bridge_mode, 0, 0, nlflags, + filter_mask, NULL); } static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) @@ -10333,8 +10220,8 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) u16 reserved_pools; if (((adapter->flags & IXGBE_FLAG_DCB_ENABLED) && - adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || - adapter->num_rx_pools > IXGBE_MAX_MACVLANS) + adapter->num_rx_pools >= (MAX_TX_QUEUES / tcs)) || + adapter->num_rx_pools > IXGBE_MAX_MACVLANS) return ERR_PTR(-EBUSY); /* Hardware has a limited number of available pools. Each VF, @@ -10354,12 +10241,12 @@ static void *ixgbe_fwd_add(struct net_device *pdev, struct net_device *vdev) */ if (used_pools < 32 && adapter->num_rx_pools < 16) reserved_pools = min_t(u16, - 32 - used_pools, - 16 - adapter->num_rx_pools); + 32 - used_pools, + 16 - adapter->num_rx_pools); else if (adapter->num_rx_pools < 32) reserved_pools = min_t(u16, - 64 - used_pools, - 32 - adapter->num_rx_pools); + 64 - used_pools, + 32 - adapter->num_rx_pools); else reserved_pools = 64 - used_pools; @@ -10406,7 +10293,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) /* delete unicast filter associated with offloaded interface */ ixgbe_del_mac_filter(adapter, accel->netdev->dev_addr, - VMDQ_P(accel->pool)); + VMDQ_P(accel->pool)); /* Allow remaining Rx packets to get flushed out of the * Rx FIFO before we drop the netdev for the ring. @@ -10438,7 +10325,7 @@ static void ixgbe_fwd_del(struct net_device *pdev, void *priv) static netdev_features_t ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, - netdev_features_t features) + netdev_features_t features) { unsigned int network_hdr_len, mac_hdr_len; @@ -10446,19 +10333,19 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, mac_hdr_len = skb_network_offset(skb); if (unlikely(mac_hdr_len > IXGBE_MAX_MAC_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_TSO | + NETIF_F_TSO6); network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); if (unlikely(network_hdr_len > IXGBE_MAX_NETWORK_HDR_LEN)) return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_GSO_UDP_L4 | - NETIF_F_TSO | - NETIF_F_TSO6); + NETIF_F_SCTP_CRC | + NETIF_F_GSO_UDP_L4 | + NETIF_F_TSO | + NETIF_F_TSO6); /* We can only support IPV4 TSO in tunnels if we can mangle the * inner IP ID field, so strip TSO if MANGLEID is not supported. @@ -10477,7 +10364,7 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) { - int i, frame_size = dev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + int i, frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct bpf_prog *old_prog; bool need_reset; @@ -10541,7 +10428,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) for (i = 0; i < num_queues; i++) if (adapter->xdp_ring[i]->xsk_pool) (void)ixgbe_xsk_wakeup(adapter->netdev, i, - XDP_WAKEUP_RX); + XDP_WAKEUP_RX); xdp_features_set_redirect_target(dev, true); } @@ -10557,7 +10444,7 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) return ixgbe_xdp_setup(dev, xdp->prog); case XDP_SETUP_XSK_POOL: return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool, - xdp->xsk.queue_id); + xdp->xsk.queue_id); default: return -EINVAL; @@ -10594,7 +10481,7 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, return -ENETDOWN; if (!netif_carrier_ok(adapter->netdev) || - !netif_running(adapter->netdev)) + !netif_running(adapter->netdev)) return -ENETDOWN; if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) @@ -10711,7 +10598,7 @@ static void ixgbe_disable_txr_hw(struct ixgbe_adapter *adapter, } static void ixgbe_disable_txr(struct ixgbe_adapter *adapter, - struct ixgbe_ring *tx_ring) + struct ixgbe_ring *tx_ring) { set_bit(__IXGBE_TX_DISABLED, &tx_ring->state); ixgbe_disable_txr_hw(adapter, tx_ring); @@ -10735,7 +10622,7 @@ static void ixgbe_disable_rxr_hw(struct ixgbe_adapter *adapter, /* RXDCTL.EN may not change on 82598 if link is down, so skip it */ if (hw->mac.type == ixgbe_mac_82598EB && - !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) + !(IXGBE_READ_REG(hw, IXGBE_LINKS) & IXGBE_LINKS_UP)) return; /* delay mechanism from ixgbe_disable_rx */ @@ -10915,7 +10802,7 @@ static inline int ixgbe_enumerate_functions(struct ixgbe_adapter *adapter) * this case, simply return -1 to indicate this. */ if ((entry->vendor != pdev->vendor) || - (entry->device != pdev->device)) + (entry->device != pdev->device)) return -1; physfns++; @@ -10947,8 +10834,8 @@ bool ixgbe_wol_supported(struct ixgbe_adapter *adapter, u16 device_id, /* check eeprom to see if WOL is enabled for X540 and newer */ if (hw->mac.type >= ixgbe_mac_X540) { if ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0_1) || - ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && - (hw->bus.func == 0))) + ((wol_cap == IXGBE_DEVICE_CAPS_WOL_PORT0) && + (hw->bus.func == 0))) return true; } @@ -11135,7 +11022,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) */ if (pdev->is_virtfn) { WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n", - pci_name(pdev), pdev->vendor, pdev->device); + pci_name(pdev), pdev->vendor, pdev->device); return -EINVAL; } @@ -11195,7 +11082,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE); hw->hw_addr = ioremap(pci_resource_start(pdev, 0), - pci_resource_len(pdev, 0)); + pci_resource_len(pdev, 0)); adapter->io_addr = hw->hw_addr; if (!hw->hw_addr) { err = -EIO; @@ -11338,15 +11225,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) NETIF_F_HW_CSUM; #define IXGBE_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \ - NETIF_F_GSO_GRE_CSUM | \ - NETIF_F_GSO_IPXIP4 | \ - NETIF_F_GSO_IPXIP6 | \ - NETIF_F_GSO_UDP_TUNNEL | \ - NETIF_F_GSO_UDP_TUNNEL_CSUM) + NETIF_F_GSO_GRE_CSUM | \ + NETIF_F_GSO_IPXIP4 | \ + NETIF_F_GSO_IPXIP6 | \ + NETIF_F_GSO_UDP_TUNNEL | \ + NETIF_F_GSO_UDP_TUNNEL_CSUM) netdev->gso_partial_features = IXGBE_GSO_PARTIAL_FEATURES; netdev->features |= NETIF_F_GSO_PARTIAL | - IXGBE_GSO_PARTIAL_FEATURES; + IXGBE_GSO_PARTIAL_FEATURES; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4; @@ -11361,15 +11248,15 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) #endif /* copy netdev features into list of user selectable features */ netdev->hw_features |= netdev->features | - NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_RXALL | - NETIF_F_HW_L2FW_DOFFLOAD; + NETIF_F_HW_VLAN_CTAG_FILTER | + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_RXALL | + NETIF_F_HW_L2FW_DOFFLOAD; if (hw->mac.type >= ixgbe_mac_82599EB) netdev->hw_features |= NETIF_F_NTUPLE | - NETIF_F_HW_TC; + NETIF_F_HW_TC; netdev->features |= NETIF_F_HIGHDMA; @@ -11383,14 +11270,14 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) /* set this bit last since it cannot be part of vlan_features */ netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER | - NETIF_F_HW_VLAN_CTAG_RX | - NETIF_F_HW_VLAN_CTAG_TX; + NETIF_F_HW_VLAN_CTAG_RX | + NETIF_F_HW_VLAN_CTAG_TX; netdev->priv_flags |= IFF_UNICAST_FLT; netdev->priv_flags |= IFF_SUPP_NOFCS; netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY; + NETDEV_XDP_ACT_XSK_ZEROCOPY; /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; @@ -11416,7 +11303,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) adapter->ring_feature[RING_F_FCOE].limit = fcoe_l; netdev->features |= NETIF_F_FSO | - NETIF_F_FCOE_CRC; + NETIF_F_FCOE_CRC; netdev->vlan_features |= NETIF_F_FSO | NETIF_F_FCOE_CRC; @@ -11435,7 +11322,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) } eth_platform_get_mac_address(&adapter->pdev->dev, - adapter->hw.mac.perm_addr); + adapter->hw.mac.perm_addr); eth_hw_addr_set(netdev, hw->mac.perm_addr); @@ -11753,7 +11640,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, u16 req_id, pf_func; if (adapter->hw.mac.type == ixgbe_mac_82598EB || - adapter->num_vfs == 0) + adapter->num_vfs == 0) goto skip_bad_vf_detection; bdev = pdev->bus->self; @@ -11818,7 +11705,7 @@ static pci_ers_result_t ixgbe_io_error_detected(struct pci_dev *pdev, if (vfdev->devfn == (req_id & 0xFF)) break; vfdev = pci_get_device(PCI_VENDOR_ID_INTEL, - device_id, vfdev); + device_id, vfdev); } /* * There's a slim chance the VF could have been hot plugged, @@ -12005,7 +11892,7 @@ static void __exit ixgbe_exit_module(void) #ifdef CONFIG_IXGBE_DCA static int ixgbe_notify_dca(struct notifier_block *nb, unsigned long event, - void *p) + void *p) { int ret_val; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index b1bfeb21537acc..825cf17fa1878d 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -2452,7 +2452,7 @@ enum { #define IXGBE_PSRTYPE_L2HDR 0x00001000 /* SRRCTL bit definitions */ -#define IXGBE_SRRCTL_BSIZEPKT_SHIFT 10 /* so many KBs */ +#define IXGBE_SRRCTL_BSIZEPKT_STEP 1024 #define IXGBE_SRRCTL_RDMTS_SHIFT 22 #define IXGBE_SRRCTL_RDMTS_MASK 0x01C00000 #define IXGBE_SRRCTL_DROP_EN 0x10000000 diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 7b941505a9d024..15425d3c367872 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -151,7 +151,7 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i = rx_ring->next_to_use; dma_addr_t dma; bool ok = true; @@ -161,7 +161,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) return true; rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; i -= rx_ring->count; do { @@ -183,7 +183,7 @@ bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) i++; if (unlikely(!i)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_buffer_info; + bi = rx_ring->rx_xsk_buffer_info; i -= rx_ring->count; } @@ -257,7 +257,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -279,17 +279,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - bi = &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + bi = &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; if (unlikely(!ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) { - struct ixgbe_rx_buffer *next_bi; + struct ixgbe_xsk_rx_buffer *next_bi; xsk_buff_free(bi->xdp); bi->xdp = NULL; ixgbe_inc_ntc(rx_ring); next_bi = - &rx_ring->rx_buffer_info[rx_ring->next_to_clean]; + &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; next_bi->discard = true; continue; } @@ -345,6 +345,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, total_rx_bytes += skb->len; total_rx_packets++; + skb->protocol = eth_type_trans(skb, rx_ring->netdev); ixgbe_process_skb_fields(rx_ring, rx_desc, skb); ixgbe_rx_skb(q_vector, skb); } @@ -374,11 +375,11 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) { - struct ixgbe_rx_buffer *bi; + struct ixgbe_xsk_rx_buffer *bi; u16 i; for (i = 0; i < rx_ring->count; i++) { - bi = &rx_ring->rx_buffer_info[i]; + bi = &rx_ring->rx_xsk_buffer_info[i]; if (!bi->xdp) continue; From aab8eaf3365f2558acdd54c48d778a4f996e1f42 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Thu, 28 May 2026 15:03:24 +0200 Subject: [PATCH 04/25] ixgbe: branch prediction and cleanup * Added branch prediction hints to the ixgbe Rx path and clean up the packet accounting. * Moved total_rx_bytes and total_rx_packets accounting together. Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 22 ++++++++++--------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index c50a5a1e73eb19..776fef0b4764cc 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1745,7 +1745,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) u16 ntu = rx_ring->next_to_use; /* nothing to do */ - if (!cleaned_count) + if (unlikely(!cleaned_count)) return; rx_desc = IXGBE_RX_DESC(rx_ring, ntu); @@ -1761,7 +1761,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) rx_desc++; ntu++; - if (unlikely(ntu == rx_ring->count)) { + if (unlikely(ntu == fq.count)) { rx_desc = IXGBE_RX_DESC(rx_ring, 0); ntu = 0; } @@ -2122,7 +2122,6 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* exit if we failed to retrieve a buffer */ if (unlikely(!xdp_res && !skb)) { - libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; cleaned_count++; break; @@ -2135,14 +2134,12 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, continue; /* verify the packet layout is correct */ - if (xdp_res || unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + if (xdp_res || + unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } - /* probably a little skewed due to removing CRC */ - total_rx_bytes += skb->len; - /* populate checksum, timestamp, VLAN, and protocol */ ixgbe_process_skb_fields(rx_ring, rx_desc, skb); @@ -2172,11 +2169,12 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, } #endif /* IXGBE_FCOE */ + /* probably a little skewed due to removing CRC */ + total_rx_bytes += skb->len; + total_rx_packets++; + ixgbe_rx_skb(q_vector, skb); skb = NULL; - - /* update budget accounting */ - total_rx_packets++; } rx_ring->skb = skb; @@ -6620,12 +6618,16 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, }; struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); + struct napi_struct *napi_dev; int ring_node = NUMA_NO_NODE; int ret; if (rx_ring->q_vector) { fq.nid = rx_ring->q_vector->numa_node; ring_node = rx_ring->q_vector->numa_node; + napi_dev = &rx_ring->q_vector->napi; + } else { + napi_dev = NULL; } ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); From f0e1ead1e433802c445aad83802deacb41706722 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 1 Jun 2026 09:51:36 +0200 Subject: [PATCH 05/25] ixgbe: support XDP multi-buffer on Rx path Implement XDP multi-buffer support for Rx fragmented packets using libeth_xdp. * Added MODULE_IMPORT_NS("LIBETH_XDP") and #include * Replaced struct sk_buff *skb in ixgbe_ring with struct libeth_xdp_buff_stash and ixgbe_clean_rx_ring skb cleanup with libeth_xdp_return_stash * Removed ixgbe_add_rx_frag() * Reorganized ixgbe_clean_rx_irq() and ixgbe_is_non_eop() * Updated ixgbe_xdp_setup() to allow multi-buffer XDP programs * Added NETDEV_XDP_ACT_RX_SG in xdp_features Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 3 +- drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c | 1 - drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 122 +++++++----------- 3 files changed, 52 insertions(+), 74 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index f5842f35d498bd..af801eacc45c65 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -34,6 +34,7 @@ #include #include +#include /* common prefix used by pr_<> macros */ #undef pr_fmt @@ -324,12 +325,12 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; - struct sk_buff *skb; /* partial Rx skb across non-EOP */ struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ u32 rx_buf_len; + struct libeth_xdp_buff_stash xdp_stash; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c index a1d04914fbbca8..2d139d2217fac4 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c @@ -965,7 +965,6 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, while (rxr_count) { /* assign generic ring traits */ - ring->dev = &adapter->pdev->dev; ring->netdev = adapter->netdev; /* configure backlink on ring */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 776fef0b4764cc..a59939ba094b2a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1865,7 +1865,7 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * ixgbe_is_non_eop - process handling of non-EOP buffers * @rx_ring: Rx ring being processed * @rx_desc: Rx descriptor for current buffer - * @skb: Current socket buffer containing buffer in progress + * @rsc_append_cnt: pointer to RSC append count accumulator * * This function updates next to clean. If the buffer is an EOP buffer * this function exits returning false, otherwise it will place the @@ -1873,8 +1873,8 @@ void ixgbe_rx_skb(struct ixgbe_q_vector *q_vector, * that this is in fact a non-EOP buffer. **/ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, - union ixgbe_adv_rx_desc *rx_desc, - struct sk_buff *skb) + union ixgbe_adv_rx_desc *rx_desc, + u32 *rsc_append_cnt) { u32 ntc = rx_ring->next_to_clean + 1; @@ -1893,7 +1893,7 @@ static bool ixgbe_is_non_eop(struct ixgbe_ring *rx_ring, u32 rsc_cnt = le32_to_cpu(rsc_enabled); rsc_cnt >>= IXGBE_RXDADV_RSCCNT_SHIFT; - IXGBE_CB(skb)->append_cnt += rsc_cnt - 1; + *rsc_append_cnt += rsc_cnt - 1; /* update ntc based on RSC value */ ntc = le32_to_cpu(rx_desc->wb.upper.status_error); @@ -1961,33 +1961,6 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; } -/** - * ixgbe_add_rx_frag - Add contents of Rx buffer to sk_buff - * @rx_ring: rx descriptor ring to transact packets on - * @rx_buffer: buffer containing page to add - * @skb: sk_buff to place the data into - * @size: size of data in rx_buffer - * - * This function will add the data contained in rx_buffer->page to the skb. - * This is done either through a direct copy if the data in the buffer is - * less than the skb header size, otherwise it will just attach the page as - * a frag to the skb. - * - * The function will then update the page offset if necessary and return - * true if the buffer can be reused by the adapter. - **/ -static void ixgbe_add_rx_frag(const struct libeth_fqe *rx_buffer, - struct sk_buff *skb, - unsigned int size) -{ - u32 hr = netmem_get_pp(rx_buffer->netmem)->p.offset; - - skb_add_rx_frag_netmem(skb, skb_shinfo(skb)->nr_frags, - rx_buffer->netmem, rx_buffer->offset + hr, - size, rx_buffer->truesize); -} - - static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, struct ixgbe_ring *rx_ring, struct libeth_xdp_buff *xdp) @@ -2069,16 +2042,16 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); LIBETH_XDP_ONSTACK_BUFF(xdp); - struct sk_buff *skb = rx_ring->skb; unsigned int xdp_xmit = 0; + u32 rsc_append_cnt = 0; int xdp_res = 0; - xdp->data = NULL; - xdp->base.rxq = &rx_ring->xdp_rxq; + libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; struct libeth_fqe *rx_buffer; + struct sk_buff *skb; unsigned int size; /* return some buffers to hardware, one at a time is too slow */ @@ -2099,43 +2072,43 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, dma_rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + libeth_xdp_process_buff(xdp, rx_buffer, size); - /* retrieve a buffer from the ring */ - if (!skb) { - libeth_xdp_process_buff(xdp, rx_buffer, size); - xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); - } else { - libeth_rx_sync_for_cpu(rx_buffer, size); - } + cleaned_count++; + /* fetch next buffer in frame if non-eop */ + if (ixgbe_is_non_eop(rx_ring, rx_desc, &rsc_append_cnt) || + unlikely(!xdp->data)) + continue; + xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); if (xdp_res) { if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) xdp_xmit |= xdp_res; total_rx_packets++; - total_rx_bytes += size; - } else if (skb) { - ixgbe_add_rx_frag(rx_buffer, skb, size); - } else { - skb = xdp_build_skb_from_buff(&xdp->base); + total_rx_bytes += xdp_get_buff_len(&xdp->base); + rsc_append_cnt = 0; + xdp->data = NULL; + continue; } + skb = xdp_build_skb_from_buff(&xdp->base); + /* exit if we failed to retrieve a buffer */ - if (unlikely(!xdp_res && !skb)) { + if (unlikely(!skb)) { + rsc_append_cnt = 0; + libeth_xdp_return_buff_slow(xdp); rx_ring->rx_stats.alloc_rx_buff_failed++; - cleaned_count++; break; } + xdp->data = NULL; - cleaned_count++; - - /* place incomplete frames back on ring for completion */ - if (ixgbe_is_non_eop(rx_ring, rx_desc, skb)) - continue; + /* apply RSC append count to skb if accumulated */ + IXGBE_CB(skb)->append_cnt = rsc_append_cnt; + rsc_append_cnt = 0; /* verify the packet layout is correct */ - if (xdp_res || - unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { + if (unlikely(ixgbe_cleanup_headers(rx_ring, rx_desc, skb))) { skb = NULL; continue; } @@ -2174,10 +2147,10 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, total_rx_packets++; ixgbe_rx_skb(q_vector, skb); - skb = NULL; } - rx_ring->skb = skb; + /* place incomplete frames back on ring for completion */ + libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -5227,11 +5200,8 @@ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) goto skip_free; } - /* Free Rx ring sk_buff */ - if (rx_ring->skb) { - dev_kfree_skb(rx_ring->skb); - rx_ring->skb = NULL; - } + /* Free Rx ring xdp stash */ + libeth_xdp_return_stash(&rx_ring->xdp_stash); for (u32 i = rx_ring->next_to_clean; i != rx_ring->next_to_use; ) { const struct libeth_fqe *rx_fqe = &rx_ring->rx_fqes[i]; @@ -6653,6 +6623,8 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, GFP_KERNEL); if (!rx_ring->desc) { ret = -ENOMEM; + dev_err(&adapter->pdev->dev, + "Unable to allocate memory for the Rx descriptor ring\n"); goto err_destroy_fq; } @@ -6675,7 +6647,6 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; err_destroy_fq: - dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -10364,13 +10335,15 @@ ixgbe_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) +static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) { - int i, frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; + u32 frame_size = READ_ONCE(dev->mtu) + LIBETH_RX_LL_LEN; struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); struct bpf_prog *old_prog; bool need_reset; int num_queues; + bool requires_mbuf; if (adapter->flags & IXGBE_FLAG_SRIOV_ENABLED) return -EINVAL; @@ -10379,15 +10352,19 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) return -EINVAL; /* verify ixgbe ring attributes are sufficient for XDP */ - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { struct ixgbe_ring *ring = adapter->rx_ring[i]; if (ring_is_rsc_enabled(ring)) return -EINVAL; } - if (frame_size > IXGBE_RXBUFFER_3K) - return -EINVAL; + requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { + NL_SET_ERR_MSG_MOD(extack, + "Configured MTU requires non-linear frames and XDP prog does not support frags"); + return -EOPNOTSUPP; + } /* if the number of cpus is much larger than the maximum of queues, * we should stop it and then return with ENOMEM like before. @@ -10412,7 +10389,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (!prog) xdp_features_clear_redirect_target(dev); } else { - for (i = 0; i < adapter->num_rx_queues; i++) { + for (int i = 0; i < adapter->num_rx_queues; i++) { WRITE_ONCE(adapter->rx_ring[i]->xdp_prog, adapter->xdp_prog); } @@ -10427,7 +10404,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog) if (need_reset && prog) { num_queues = min_t(int, adapter->num_rx_queues, adapter->num_xdp_queues); - for (i = 0; i < num_queues; i++) + for (int i = 0; i < num_queues; i++) if (adapter->xdp_ring[i]->xsk_pool) (void)ixgbe_xsk_wakeup(adapter->netdev, i, XDP_WAKEUP_RX); @@ -10443,7 +10420,7 @@ static int ixgbe_xdp(struct net_device *dev, struct netdev_bpf *xdp) switch (xdp->command) { case XDP_SETUP_PROG: - return ixgbe_xdp_setup(dev, xdp->prog); + return ixgbe_xdp_setup(dev, xdp->prog, xdp->extack); case XDP_SETUP_XSK_POOL: return ixgbe_xsk_pool_setup(adapter, xdp->xsk.pool, xdp->xsk.queue_id); @@ -11279,7 +11256,8 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->priv_flags |= IFF_SUPP_NOFCS; netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY; + NETDEV_XDP_ACT_XSK_ZEROCOPY | + NETDEV_XDP_ACT_RX_SG; /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; From a66f091496171a00c16874e5098083785f16aa23 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 12 Aug 2026 12:25:15 +0200 Subject: [PATCH 06/25] [SPLIT] Move descriptor definitions to a separate file + introduce and use libie_xg_ring --- drivers/net/ethernet/intel/ixgbevf/defines.h | 191 ------------------ include/linux/net/intel/ixgbe_desc.h | 200 +++++++++++++++++++ 2 files changed, 200 insertions(+), 191 deletions(-) create mode 100644 include/linux/net/intel/ixgbe_desc.h diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index afc927dd14381b..a67d1ecd6012a6 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -84,197 +84,6 @@ typedef u32 ixgbe_link_speed; #define IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000 #define IXGBE_SRRCTL_DESCTYPE_MASK 0x0E000000 -/* Receive Descriptor bit definitions */ -#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ -#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ -#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ -#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ -#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ -#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 -#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ -#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ -#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ -#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ -#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ -#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ -#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ -#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ -#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ -#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ -#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ -#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ -#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ -#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ -#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ -#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ -#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ -#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ -#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ -#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ -#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ -#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ -#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ -#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ -#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ -#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ -#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ -#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ -#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ -#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ -#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ -#define IXGBE_RXD_PRI_SHIFT 13 -#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ -#define IXGBE_RXD_CFI_SHIFT 12 - -#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ -#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ -#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ -#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ -#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ -#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ -#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ -#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ -#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ -#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ - -#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F -#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 -#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ -#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 -#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 -#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 -#define IXGBE_RXDADV_RSCCNT_SHIFT 17 -#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 -#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 -#define IXGBE_RXDADV_SPH 0x8000 - -/* RSS Hash results */ -#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 -#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 -#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 -#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 -#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 -#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 - -#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXD_ERR_CE | \ - IXGBE_RXD_ERR_LE | \ - IXGBE_RXD_ERR_PE | \ - IXGBE_RXD_ERR_OSE | \ - IXGBE_RXD_ERR_USE) - -#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXDADV_ERR_CE | \ - IXGBE_RXDADV_ERR_LE | \ - IXGBE_RXDADV_ERR_PE | \ - IXGBE_RXDADV_ERR_OSE | \ - IXGBE_RXDADV_ERR_USE) - -#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ -#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ -#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ -#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) - -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; - -/* Context descriptors */ -struct ixgbe_adv_tx_context_desc { - __le32 vlan_macip_lens; - __le32 fceof_saidx; - __le32 type_tucmd_mlhl; - __le32 mss_l4len_idx; -}; - -/* Adv Transmit Descriptor Config Masks */ -#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ -#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ -#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ -#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ -#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ -#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ -#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ -#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ -#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ -#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ -#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ -#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ -#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ -#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ -#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ -#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ -#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ -#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ -#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ -#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ - /* Interrupt register bitmasks */ #define IXGBE_EITR_CNT_WDIS 0x80000000 diff --git a/include/linux/net/intel/ixgbe_desc.h b/include/linux/net/intel/ixgbe_desc.h new file mode 100644 index 00000000000000..fdf8a35044f539 --- /dev/null +++ b/include/linux/net/intel/ixgbe_desc.h @@ -0,0 +1,200 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __IXGBE_DESC_H +#define __IXGBE_DESC_H + +#include + +/* Receive Descriptor bit definitions */ +#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ +#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ +#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ +#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ +#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ +#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 +#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ +#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ +#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ +#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ +#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ +#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ +#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ +#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ +#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ +#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ +#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ +#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ +#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ +#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ +#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ +#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ +#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ +#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ +#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ +#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ +#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ +#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ +#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ +#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ +#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ +#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ +#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ +#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ +#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ +#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ +#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ +#define IXGBE_RXD_PRI_SHIFT 13 +#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ +#define IXGBE_RXD_CFI_SHIFT 12 + +#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ +#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ +#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ +#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ +#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ +#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ +#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ +#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ +#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ +#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ + +#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F +#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 +#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ +#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 +#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 +#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 +#define IXGBE_RXDADV_RSCCNT_SHIFT 17 +#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 +#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 +#define IXGBE_RXDADV_SPH 0x8000 + +/* RSS Hash results */ +#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 +#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 +#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 +#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 +#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 +#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 + +#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXD_ERR_CE | \ + IXGBE_RXD_ERR_LE | \ + IXGBE_RXD_ERR_PE | \ + IXGBE_RXD_ERR_OSE | \ + IXGBE_RXD_ERR_USE) + +#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXDADV_ERR_CE | \ + IXGBE_RXDADV_ERR_LE | \ + IXGBE_RXDADV_ERR_PE | \ + IXGBE_RXDADV_ERR_OSE | \ + IXGBE_RXDADV_ERR_USE) + +#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ +#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ +#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ +#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ +#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ +#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ +#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ +#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ +#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) + +/* Transmit Descriptor - Advanced */ +union ixgbe_adv_tx_desc { + struct { + __le64 buffer_addr; /* Address of descriptor's data buf */ + __le32 cmd_type_len; + __le32 olinfo_status; + } read; + struct { + __le64 rsvd; /* Reserved */ + __le32 nxtseq_seed; + __le32 status; + } wb; +}; + +/* Receive Descriptor - Advanced */ +union ixgbe_adv_rx_desc { + struct { + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + } read; + struct { + struct { + union { + __le32 data; + struct { + __le16 pkt_info; /* RSS, Pkt type */ + __le16 hdr_info; /* Splithdr, hdrlen */ + } hs_rss; + } lo_dword; + union { + __le32 rss; /* RSS Hash */ + struct { + __le16 ip_id; /* IP id */ + __le16 csum; /* Packet Checksum */ + } csum_ip; + } hi_dword; + } lower; + struct { + __le32 status_error; /* ext status/error */ + __le16 length; /* Packet length */ + __le16 vlan; /* VLAN tag */ + } upper; + } wb; /* writeback */ +}; + +/* Context descriptors */ +struct ixgbe_adv_tx_context_desc { + __le32 vlan_macip_lens; + __le32 fceof_saidx; + __le32 type_tucmd_mlhl; + __le32 mss_l4len_idx; +}; + +/* Adv Transmit Descriptor Config Masks */ +#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ +#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ +#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ +#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ +#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ +#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ +#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ +#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ +#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ +#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ +#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ +#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ +#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ +#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ +#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ +#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ +#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ +#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ +#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ +#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ +#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ +#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ +#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ + +#endif From e2db87a2780f4282bc96c74b11a781b2b757cb0c Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 11:26:41 +0200 Subject: [PATCH 07/25] libie_xg_ring fixup --- include/linux/net/intel/libie/xg_ring.h | 82 +++++++++++++++++++++++++ 1 file changed, 82 insertions(+) create mode 100644 include/linux/net/intel/libie/xg_ring.h diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h new file mode 100644 index 00000000000000..768861c5e0688c --- /dev/null +++ b/include/linux/net/intel/libie/xg_ring.h @@ -0,0 +1,82 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __LIBIE_XG_RING_H +#define __LIBIE_XG_RING_H + +#include +#include +#include + +#include + +struct libie_xg_queue_stats { + u64 packets; + u64 bytes; +}; + +struct libie_xg_tx_queue_stats { + u64 restart_queue; + u64 tx_busy; + u64 tx_done_old; +}; + +struct libie_xg_rx_queue_stats { + /* u64 rsc_count; <- was in ixgbe + u64 rsc_flush; */ + u64 non_eop_descs; + u64 alloc_rx_page; + u64 alloc_rx_page_failed; + u64 alloc_rx_buff_failed; + u64 csum_err; +}; + +struct libie_xg_tx_buffer { + union ixgbe_adv_tx_desc *next_to_watch; + unsigned long time_stamp; + union { + struct sk_buff *skb; + /* XDP uses address ptr on irq_clean */ + void *data; + }; + unsigned int bytecount; + unsigned short gso_segs; + __be16 protocol; + DEFINE_DMA_UNMAP_ADDR(dma); + DEFINE_DMA_UNMAP_LEN(len); + u32 tx_flags; +}; + +struct libie_xg_ring { + struct device *dev; /* Tx ring */ + union { + struct page_pool *pp; /* Rx and XDP rings */ + struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */ + }; + void *desc; /* descriptor ring memory */ + u16 count; /* amount of descriptors */ + u16 next_to_clean; + u32 next_to_use; + u32 pending; /* Sent-not-completed descriptors */ + u32 cached_ntu; + + union { + struct libeth_fqe *rx_fqes; + struct libeth_xdp_buff **xsk_fqes; + struct libie_xg_tx_buffer *tx_buffer_info; + struct libeth_sqe *xdp_sqes; + }; + u8 __iomem *tail; + + struct libeth_xdp_buff *xsk_xdp_head; + struct libeth_xdpsq_lock xdpq_lock; +}; + +#define LIBIE_XG_RX_DESC(R, i) \ + (&(((union ixgbe_adv_rx_desc *)((R)->desc))[i])) +#define LIBIE_XG_TX_DESC(R, i) \ + (&(((union ixgbe_adv_tx_desc *)((R)->desc))[i])) +#define LIBIE_XG_TX_CTXTDESC(R, i) \ + (&(((struct ixgbe_adv_tx_context_desc *)((R)->desc))[i])) + +#endif \ No newline at end of file From cc5fca500b6065ca2b160ef23a721f20587f2263 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 11:04:40 +0200 Subject: [PATCH 08/25] ixgbevf, libie: separate ixgbevf_ring base and move to libie_xg In order to share some generation-specific non-trivial XDP code between ixgbevf and ixgbe, identify required ring fields and separate them into a base ring structure. Use the structure in a somewhat struct_group fashion, exposing a named .base structure (for shared code to use) as well as direct field access in a union (to not disturb the ancient code). But given that 2 different structures will need to include the base, instead of struct_group, utilize a relatively recently enabled -fms-extensions anonymous member structures feature. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbevf/ipsec.c | 2 +- drivers/net/ethernet/intel/ixgbevf/ixgbevf.h | 49 +++---------------- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 18 +++---- 3 files changed, 17 insertions(+), 52 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbevf/ipsec.c b/drivers/net/ethernet/intel/ixgbevf/ipsec.c index fce35924ff8b51..e40abea3eea4f5 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ipsec.c +++ b/drivers/net/ethernet/intel/ixgbevf/ipsec.c @@ -451,7 +451,7 @@ static const struct xfrmdev_ops ixgbevf_xfrmdev_ops = { * @itd: ipsec Tx data for later use in building context descriptor **/ int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd) { struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h index d8f841515ca62a..e1ef7b44ceecea 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -24,25 +25,6 @@ #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -/* wrapper around a pointer to a socket buffer, - * so a DMA handle can be stored along with the buffer - */ -struct ixgbevf_tx_buffer { - union ixgbe_adv_tx_desc *next_to_watch; - unsigned long time_stamp; - union { - struct sk_buff *skb; - /* XDP uses address ptr on irq_clean */ - void *data; - }; - unsigned int bytecount; - unsigned short gso_segs; - __be16 protocol; - DEFINE_DMA_UNMAP_ADDR(dma); - DEFINE_DMA_UNMAP_LEN(len); - u32 tx_flags; -}; - struct ixgbevf_stats { u64 packets; u64 bytes; @@ -84,30 +66,16 @@ enum ixgbevf_ring_state_t { clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) struct ixgbevf_ring { + union { + struct libie_xg_ring base; + struct libie_xg_ring; + }; struct ixgbevf_ring *next; struct ixgbevf_q_vector *q_vector; /* backpointer to q_vector */ struct net_device *netdev; struct bpf_prog __rcu *xdp_prog; - union { - struct page_pool *pp; /* Rx and XDP rings */ - struct device *dev; /* Tx ring */ - }; - void *desc; /* descriptor ring memory */ u32 truesize; /* Rx buffer full size */ u32 hdr_truesize; /* Rx header buffer full size */ - u16 count; /* amount of descriptors */ - u16 next_to_clean; - u32 next_to_use; - u32 pending; /* Sent-not-completed descriptors */ - - union { - struct libeth_fqe *rx_fqes; - struct libeth_xdp_buff **xsk_fqes; - struct ixgbevf_tx_buffer *tx_buffer_info; - struct libeth_sqe *xdp_sqes; - }; - struct libeth_xdpsq_lock xdpq_lock; - u32 cached_ntu; u32 thresh; unsigned long state; struct ixgbevf_stats stats; @@ -120,7 +88,6 @@ struct ixgbevf_ring { struct page_pool *hdr_pp; struct xdp_rxq_info xdp_rxq; u64 hw_csum_rx_error; - u8 __iomem *tail; /* holds the special value that gets the hardware register offset * associated with this ring, which is different for DCB and RSS modes @@ -129,10 +96,8 @@ struct ixgbevf_ring { int queue_index; /* needed for multiqueue queue management */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; - struct libeth_xdp_buff *xsk_xdp_head; unsigned int dma_size; /* length in bytes */ dma_addr_t dma; /* phys. address of descriptor ring */ - struct xsk_buff_pool *xsk_pool; /* AF_XDP ZC rings */ } ____cacheline_internodealigned_in_smp; /* How many Rx Buffers do we bundle into one write to the hardware ? */ @@ -444,7 +409,7 @@ void ixgbevf_ipsec_rx(struct ixgbevf_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd); #else static inline void ixgbevf_init_ipsec_offload(struct ixgbevf_adapter *adapter) @@ -456,7 +421,7 @@ static inline void ixgbevf_ipsec_rx(struct ixgbevf_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb) { } static inline int ixgbevf_ipsec_tx(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd) { return 0; } #endif /* CONFIG_IXGBEVF_IPSEC */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index 759592618549fb..c5fbde9b65d766 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -271,7 +271,7 @@ static bool ixgbevf_clean_tx_irq(struct ixgbevf_q_vector *q_vector, struct ixgbevf_ring *tx_ring, int napi_budget) { struct ixgbevf_adapter *adapter = q_vector->adapter; - struct ixgbevf_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0; unsigned int budget = tx_ring->count / 2; @@ -1215,7 +1215,7 @@ void ixgbevf_configure_tx_ring(struct ixgbevf_adapter *adapter, /* reinitialize tx_buffer_info */ if (!ring_is_xdp(ring)) { memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbevf_tx_buffer) * ring->count); + sizeof(struct libie_xg_tx_buffer) * ring->count); } else { memset(ring->xdp_sqes, 0, sizeof(struct libeth_sqe) * ring->count); @@ -2058,7 +2058,7 @@ void ixgbevf_clean_rx_ring(struct ixgbevf_ring *rx_ring) void ixgbevf_clean_tx_ring(struct ixgbevf_ring *tx_ring) { u16 i = tx_ring->next_to_clean; - struct ixgbevf_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; + struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; while (i != tx_ring->next_to_use) { union ixgbe_adv_tx_desc *eop_desc, *tx_desc; @@ -3072,7 +3072,7 @@ int ixgbevf_setup_tx_resources(struct ixgbevf_ring *tx_ring) struct ixgbevf_adapter *adapter = netdev_priv(tx_ring->netdev); int size; - size = (!ring_is_xdp(tx_ring) ? sizeof(struct ixgbevf_tx_buffer) : + size = (!ring_is_xdp(tx_ring) ? sizeof(struct libie_xg_tx_buffer) : sizeof(struct libeth_sqe)) * tx_ring->count; tx_ring->tx_buffer_info = vmalloc(size); @@ -3415,7 +3415,7 @@ static void ixgbevf_tx_ctxtdesc(struct ixgbevf_ring *tx_ring, } static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, u8 *hdr_len, struct ixgbevf_ipsec_tx_data *itd) { @@ -3512,7 +3512,7 @@ static int ixgbevf_tso(struct ixgbevf_ring *tx_ring, } static void ixgbevf_tx_csum(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbevf_ipsec_tx_data *itd) { struct sk_buff *skb = first->skb; @@ -3608,11 +3608,11 @@ static void ixgbevf_tx_olinfo_status(union ixgbe_adv_tx_desc *tx_desc, } static void ixgbevf_tx_map(struct ixgbevf_ring *tx_ring, - struct ixgbevf_tx_buffer *first, + struct libie_xg_tx_buffer *first, const u8 hdr_len) { struct sk_buff *skb = first->skb; - struct ixgbevf_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; skb_frag_t *frag; dma_addr_t dma; @@ -3776,7 +3776,7 @@ static int ixgbevf_maybe_stop_tx(struct ixgbevf_ring *tx_ring, int size) static int ixgbevf_xmit_frame_ring(struct sk_buff *skb, struct ixgbevf_ring *tx_ring) { - struct ixgbevf_tx_buffer *first; + struct libie_xg_tx_buffer *first; int tso; u32 tx_flags = 0; u16 count = TXD_USE_COUNT(skb_headlen(skb)); From de55c97e28bf0a7245065ec54b8f3cd2de615d5c Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 12 Aug 2026 12:26:06 +0200 Subject: [PATCH 09/25] ixgbevf, libie_xg: move selected XDP code to a libie_xg header Some functions used in ixgbevf XDP_TX and AF_XDP ZC implementation are at the same time non-trivial and trivially shareable with ixgbe. Move those functions to a new libie_xg header and rename them accordingly. Inline sharing was chosen, since a big chunk of those function are used in datapath, and a hypothetical small module would introduce more memory overhead than plain code duplication. Signed-off-by: Larysa Zaremba --- .../net/ethernet/intel/ixgbevf/ixgbevf_main.c | 47 +--- .../ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h | 102 +------ .../net/ethernet/intel/ixgbevf/ixgbevf_xsk.c | 121 +------- include/linux/net/intel/libie/xg_xdp.h | 261 ++++++++++++++++++ 4 files changed, 277 insertions(+), 254 deletions(-) create mode 100644 include/linux/net/intel/libie/xg_xdp.h diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index c5fbde9b65d766..e5bf50a82268a8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -503,54 +503,19 @@ static inline void ixgbevf_irq_enable_queues(struct ixgbevf_adapter *adapter, void ixgbevf_clean_xdp_ring(struct ixgbevf_ring *xdp_ring) { - ixgbevf_clean_xdp_num(xdp_ring, false, xdp_ring->pending); + libie_xg_clean_xdp_num(&xdp_ring->base, false, xdp_ring->pending, + ring_is_xsk(xdp_ring)); libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); } -static void ixgbevf_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, - const struct libeth_xdpsq *sq, - u64 priv) -{ - union ixgbe_adv_tx_desc *tx_desc = - &((union ixgbe_adv_tx_desc *)sq->descs)[i]; - - u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS | - desc.len; - - if (desc.flags & LIBETH_XDP_TX_LAST) - cmd_type |= IXGBE_TXD_CMD_EOP; - - if (desc.flags & LIBETH_XDP_TX_FIRST) { - struct libeth_sqe *sqe = &sq->sqes[i]; - struct skb_shared_info *sinfo; - u16 full_len = desc.len; - - if (desc.flags & LIBETH_XDP_TX_MULTI) { - sinfo = sqe->type == LIBETH_SQE_XDP_TX ? - sqe->sinfo : - xdp_get_shared_info_from_frame(sqe->xdpf); - full_len += sinfo->xdp_frags_size; - } - - tx_desc->read.olinfo_status = - cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - } - - tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); -} - LIBETH_XDP_DEFINE_START(); LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbevf_xdp_flush_tx, ixgbevf_prep_xdp_sq, - ixgbevf_xdp_xmit_desc); + libie_xg_xdp_xmit_desc); LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbevf_xdp_flush_xmit, ixgbevf_prep_xdp_sq, - ixgbevf_xdp_xmit_desc); + libie_xg_xdp_xmit_desc); LIBETH_XDP_DEFINE_RUN_PROG(static ixgbevf_xdp_run_prog, ixgbevf_xdp_flush_tx); LIBETH_XDP_DEFINE_FINALIZE(static ixgbevf_xdp_finalize_xdp_napi, - ixgbevf_xdp_flush_tx, ixgbevf_xdp_rs_and_bump); + ixgbevf_xdp_flush_tx, libie_xg_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); static u32 ixgbevf_rx_hsplit_wa(const struct libeth_fqe *hdr, @@ -713,7 +678,7 @@ static int ixgbevf_xdp_xmit(struct net_device *dev, int n, return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, adapter->xdp_ring, adapter->num_xdp_queues, ixgbevf_xdp_flush_xmit, - ixgbevf_xdp_rs_and_bump); + libie_xg_xdp_rs_and_bump); } /** diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h index ad1e09ae6aa1c3..13d2be98bd7fa8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xdp_lib.h @@ -4,80 +4,21 @@ #ifndef _IXGBEVF_XDP_LIB_H_ #define _IXGBEVF_XDP_LIB_H_ -#include +#include #include "ixgbevf.h" -static inline u16 ixgbevf_tx_get_num_sent(struct ixgbevf_ring *xdp_ring) -{ - u16 ntc = xdp_ring->next_to_clean; - u16 to_clean = 0; - - while (likely(to_clean < xdp_ring->pending)) { - u32 idx = xdp_ring->xdp_sqes[ntc].rs_idx; - union ixgbe_adv_tx_desc *rs_desc; - - if (!idx--) - break; - - rs_desc = IXGBEVF_TX_DESC(xdp_ring, idx); - - if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) - break; - - xdp_ring->xdp_sqes[ntc].rs_idx = 0; - - to_clean += - (idx >= ntc ? idx : idx + xdp_ring->count) - ntc + 1; - - ntc = (idx + 1 == xdp_ring->count) ? 0 : idx + 1; - } - - return to_clean; -} - -static inline void ixgbevf_clean_xdp_num(struct ixgbevf_ring *xdp_ring, - bool in_napi, u16 to_clean) -{ - struct libeth_xdpsq_napi_stats stats = { }; - bool xsk_ring = ring_is_xsk(xdp_ring); - u32 ntc = xdp_ring->next_to_clean; - struct xdp_frame_bulk cbulk; - struct libeth_cq_pp cp = { - .bq = &cbulk, - .dev = xdp_ring->dev, - .xss = &stats, - .napi = in_napi, - }; - u32 xsk_frames = 0; - - xdp_frame_bulk_init(&cbulk); - xdp_ring->pending -= to_clean; - - while (likely(to_clean--)) { - xsk_frames += xsk_ring && - likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0; - libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); - ntc++; - ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; - } - - xdp_ring->next_to_clean = ntc; - xdp_flush_frame_bulk(&cbulk); - if (xsk_frames) - xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); -} - static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) { struct ixgbevf_ring *xdp_ring = xdpsq; libeth_xdpsq_lock(&xdp_ring->xdpq_lock); if (unlikely(ixgbevf_desc_unused(xdp_ring) < xdp_ring->thresh)) { - u16 to_clean = ixgbevf_tx_get_num_sent(xdpsq); + u16 to_clean = libie_xg_tx_get_num_sent(xdpsq); if (likely(to_clean)) - ixgbevf_clean_xdp_num(xdp_ring, true, to_clean); + libie_xg_clean_xdp_num(&xdp_ring->base, true, to_clean, + ring_is_xsk(xdp_ring)); } if (unlikely(!test_bit(__IXGBEVF_TX_XDP_RING_PRIMED, @@ -117,39 +58,4 @@ static inline u32 ixgbevf_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) return ixgbevf_desc_unused(xdp_ring); } -static inline void ixgbevf_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) -{ - struct ixgbevf_ring *xdp_ring = xdpsq; - union ixgbe_adv_tx_desc *desc; - u32 ltu; - - libeth_xdpsq_lock(&xdp_ring->xdpq_lock); - - if ((!flush && xdp_ring->pending < xdp_ring->count - 1) || - xdp_ring->cached_ntu == xdp_ring->next_to_use) - goto unlock; - - ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; - - /* We will not get DD on a context descriptor */ - if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX)) - goto unlock; - - desc = IXGBEVF_TX_DESC(xdp_ring, ltu); - desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); - - xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; - xdp_ring->cached_ntu = xdp_ring->next_to_use; - - /* In case the packet was interrupted, discard it */ - xdp_ring->xdp_sqes[ltu].priv = 0; - - /* Finish descriptor writes before bumping tail */ - wmb(); - ixgbevf_write_tail(xdp_ring, xdp_ring->next_to_use); - -unlock: - libeth_xdpsq_unlock(&xdp_ring->xdpq_lock); -} - #endif /* _IXGBEVF_XDP_LIB_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c index 0ef43b62ed932e..f5d372127d1049 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_xsk.c @@ -86,131 +86,22 @@ int ixgbevf_setup_xsk_pool(struct ixgbevf_adapter *adapter, return err; } -static void ixgbevf_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) -{ - union ixgbe_adv_rx_desc *rx_desc = - &((union ixgbe_adv_rx_desc *)fq->descs)[i]; - - rx_desc->read.pkt_addr = - cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); - rx_desc->wb.upper.length = 0; -} - bool ixgbevf_xsk_alloc_rx_bufs(struct ixgbevf_ring *rx_ring, u32 num) { - struct libeth_xskfq_fp fq = { - .count = rx_ring->count, - .descs = rx_ring->desc, - .fqes = rx_ring->xsk_fqes, - .ntu = rx_ring->next_to_use, - .pool = rx_ring->xsk_pool, - }; - u32 done; - - done = libeth_xskfqe_alloc(&fq, num, ixgbevf_fill_rx_xsk_desc); - if (likely(done)) { - /* Finish descriptor writes before bumping tail */ - wmb(); - ixgbevf_write_tail(rx_ring, fq.ntu); - } - - /* Prevent ntc circling past ntu, e.g. in case of starvation */ - IXGBEVF_RX_DESC(rx_ring, fq.ntu)->wb.upper.length = 0; - - rx_ring->next_to_use = fq.ntu; - rx_ring->pending -= done; - - return done == num; + return libie_xg_xsk_alloc_rx_bufs(&rx_ring->base, num); } void ixgbevf_rx_xsk_ring_free_buffs(struct ixgbevf_ring *rx_ring) { - u32 ntc = rx_ring->next_to_clean; - - if (rx_ring->xsk_xdp_head) - xsk_buff_free(&rx_ring->xsk_xdp_head->base); - - rx_ring->xsk_xdp_head = NULL; - - while (ntc != rx_ring->next_to_use) { - xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base); - ntc++; - ntc = ntc == rx_ring->count ? 0 : ntc; - } -} - -struct ixgbevf_zc_sqe_priv { - u16 first_desc; - u16 len; -}; - -LIBETH_SQE_CHECK_PRIV(struct ixgbevf_zc_sqe_priv); - -static void ixgbevf_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, - const struct libeth_xdpsq *sq, u64 priv) -{ - union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i]; - u32 ltu = (i ? : sq->count) - 1; - - u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS | - desc.len; - - tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); - - if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) { - tx_desc->read.olinfo_status = - cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - tx_desc->read.cmd_type_len = - cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); - return; - } - - /* No previous packet */ - if (!sq->sqes[ltu].priv) { - struct ixgbevf_zc_sqe_priv *sqe_priv = - (void *)&sq->sqes[i].priv; - - sqe_priv->first_desc = i; - sqe_priv->len = desc.len; - - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - - return; - } - - if (sq->sqes[ltu].priv) { - struct ixgbevf_zc_sqe_priv *sqe_priv = - (void *)&sq->sqes[i].priv; - - sq->sqes[i].priv = sq->sqes[ltu].priv; - sq->sqes[ltu].priv = 0; - sqe_priv->len += desc.len; - - if (desc.flags & LIBETH_XDP_TX_LAST) { - union ixgbe_adv_tx_desc *first_desc = - &descs[sqe_priv->first_desc]; - - first_desc->read.olinfo_status = - cpu_to_le32((sqe_priv->len << - IXGBE_ADVTXD_PAYLEN_SHIFT) | - IXGBE_ADVTXD_CC); - cmd_type |= IXGBE_TXD_CMD_EOP; - sq->sqes[i].priv = 0; - } - - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - } + return libie_xg_rx_xsk_ring_free_buffs(&rx_ring->base); } LIBETH_XDP_DEFINE_START(); LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbevf_xsk_flush_tx, ixgbevf_prep_xdp_sq, - ixgbevf_xsk_xmit_desc); + libie_xg_xsk_xmit_desc); LIBETH_XSK_DEFINE_RUN_PROG(static ixgbevf_xsk_run_prog, ixgbevf_xsk_flush_tx); LIBETH_XSK_DEFINE_FINALIZE(static ixgbevf_xsk_finalize_xdp_napi, - ixgbevf_xsk_flush_tx, ixgbevf_xdp_rs_and_bump); + ixgbevf_xsk_flush_tx, libie_xg_xdp_rs_and_bump); LIBETH_XDP_DEFINE_END(); u32 ixgbevf_clean_xsk_rx_irq(struct ixgbevf_q_vector *q_vector, @@ -325,8 +216,8 @@ bool ixgbevf_clean_xsk_tx_irq(struct ixgbevf_q_vector *q_vector, return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, budget, NULL, ixgbevf_prep_xdp_sq, - ixgbevf_xsk_xmit_desc, - ixgbevf_xdp_rs_and_bump); + libie_xg_xsk_xmit_desc, + libie_xg_xdp_rs_and_bump); } int ixgbevf_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) diff --git a/include/linux/net/intel/libie/xg_xdp.h b/include/linux/net/intel/libie/xg_xdp.h new file mode 100644 index 00000000000000..749934ac01c51e --- /dev/null +++ b/include/linux/net/intel/libie/xg_xdp.h @@ -0,0 +1,261 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __LIBIE_XG_XDP_H +#define __LIBIE_XG_XDP_H + +#include +#include + +static inline void libie_xg_fill_rx_xsk_desc(const struct libeth_xskfq_fp *fq, u32 i) +{ + union ixgbe_adv_rx_desc *rx_desc = + &((union ixgbe_adv_rx_desc *)fq->descs)[i]; + + rx_desc->read.pkt_addr = + cpu_to_le64(libeth_xsk_buff_xdp_get_dma(fq->fqes[i])); + rx_desc->wb.upper.length = 0; +} + +static inline bool libie_xg_xsk_alloc_rx_bufs(struct libie_xg_ring *rx_ring, + u32 num) +{ + struct libeth_xskfq_fp fq = { + .count = rx_ring->count, + .descs = rx_ring->desc, + .fqes = rx_ring->xsk_fqes, + .ntu = rx_ring->next_to_use, + .pool = rx_ring->xsk_pool, + }; + u32 done; + + done = libeth_xskfqe_alloc(&fq, num, libie_xg_fill_rx_xsk_desc); + if (likely(done)) { + /* Finish descriptor writes before bumping tail */ + wmb(); + writel(fq.ntu, rx_ring->tail); + } + + /* Prevent ntc circling past ntu, e.g. in case of starvation */ + LIBIE_XG_RX_DESC(rx_ring, fq.ntu)->wb.upper.length = 0; + + rx_ring->next_to_use = fq.ntu; + rx_ring->pending -= done; + + return done == num; +} + +static inline void libie_xg_rx_xsk_ring_free_buffs(struct libie_xg_ring *rx_ring) +{ + u32 ntc = rx_ring->next_to_clean; + + if (rx_ring->xsk_xdp_head) + xsk_buff_free(&rx_ring->xsk_xdp_head->base); + + rx_ring->xsk_xdp_head = NULL; + + while (ntc != rx_ring->next_to_use) { + xsk_buff_free(&rx_ring->xsk_fqes[ntc]->base); + ntc++; + ntc = ntc == rx_ring->count ? 0 : ntc; + } +} + +struct libie_xg_zc_sqe_priv { + u16 first_desc; + u16 len; +}; + +LIBETH_SQE_CHECK_PRIV(struct libie_xg_zc_sqe_priv); + +static inline void libie_xg_xsk_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv) +{ + union ixgbe_adv_tx_desc *descs = sq->descs, *tx_desc = &descs[i]; + u32 ltu = (i ? : sq->count) - 1; + + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; + + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + + if (likely((desc.flags & LIBETH_XDP_TX_LAST) && !sq->sqes[ltu].priv)) { + tx_desc->read.olinfo_status = + cpu_to_le32((desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + tx_desc->read.cmd_type_len = + cpu_to_le32(cmd_type | IXGBE_TXD_CMD_EOP); + return; + } + + /* No previous packet */ + if (!sq->sqes[ltu].priv) { + struct libie_xg_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sqe_priv->first_desc = i; + sqe_priv->len = desc.len; + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + + return; + } + + if (sq->sqes[ltu].priv) { + struct libie_xg_zc_sqe_priv *sqe_priv = + (void *)&sq->sqes[i].priv; + + sq->sqes[i].priv = sq->sqes[ltu].priv; + sq->sqes[ltu].priv = 0; + sqe_priv->len += desc.len; + + if (desc.flags & LIBETH_XDP_TX_LAST) { + union ixgbe_adv_tx_desc *first_desc = + &descs[sqe_priv->first_desc]; + + first_desc->read.olinfo_status = + cpu_to_le32((sqe_priv->len << + IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + cmd_type |= IXGBE_TXD_CMD_EOP; + sq->sqes[i].priv = 0; + } + + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); + } +} + +static inline void libie_xg_xdp_xmit_desc(struct libeth_xdp_tx_desc desc, u32 i, + const struct libeth_xdpsq *sq, + u64 priv) +{ + union ixgbe_adv_tx_desc *tx_desc = + &((union ixgbe_adv_tx_desc *)sq->descs)[i]; + + u32 cmd_type = IXGBE_ADVTXD_DTYP_DATA | + IXGBE_ADVTXD_DCMD_DEXT | + IXGBE_ADVTXD_DCMD_IFCS | + desc.len; + + if (desc.flags & LIBETH_XDP_TX_LAST) + cmd_type |= IXGBE_TXD_CMD_EOP; + + if (desc.flags & LIBETH_XDP_TX_FIRST) { + struct libeth_sqe *sqe = &sq->sqes[i]; + struct skb_shared_info *sinfo; + u16 full_len = desc.len; + + if (desc.flags & LIBETH_XDP_TX_MULTI) { + sinfo = sqe->type == LIBETH_SQE_XDP_TX ? + sqe->sinfo : + xdp_get_shared_info_from_frame(sqe->xdpf); + full_len += sinfo->xdp_frags_size; + } + + tx_desc->read.olinfo_status = + cpu_to_le32((full_len << IXGBE_ADVTXD_PAYLEN_SHIFT) | + IXGBE_ADVTXD_CC); + } + + tx_desc->read.buffer_addr = cpu_to_le64(desc.addr); + tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); +} + +static inline u16 libie_xg_tx_get_num_sent(struct libie_xg_ring *xdp_ring) +{ + u16 ntc = xdp_ring->next_to_clean; + u16 to_clean = 0; + + while (likely(to_clean < xdp_ring->pending)) { + u32 idx = xdp_ring->xdp_sqes[ntc].rs_idx; + union ixgbe_adv_tx_desc *rs_desc; + + if (!idx--) + break; + + rs_desc = LIBIE_XG_TX_DESC(xdp_ring, idx); + + if (!(rs_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) + break; + + xdp_ring->xdp_sqes[ntc].rs_idx = 0; + + to_clean += + (idx >= ntc ? idx : idx + xdp_ring->count) - ntc + 1; + + ntc = (idx + 1 == xdp_ring->count) ? 0 : idx + 1; + } + + return to_clean; +} + +static inline void libie_xg_clean_xdp_num(struct libie_xg_ring *xdp_ring, + bool in_napi, u16 to_clean, bool xsk_ring) +{ + struct libeth_xdpsq_napi_stats stats = { }; + u32 ntc = xdp_ring->next_to_clean; + struct xdp_frame_bulk cbulk; + struct libeth_cq_pp cp = { + .bq = &cbulk, + .dev = xdp_ring->dev, + .xss = &stats, + .napi = in_napi, + }; + u32 xsk_frames = 0; + + xdp_frame_bulk_init(&cbulk); + xdp_ring->pending -= to_clean; + + while (likely(to_clean--)) { + xsk_frames += xsk_ring && + likely(!xdp_ring->xdp_sqes[ntc].type) ? 1 : 0; + libeth_xdp_complete_tx(&xdp_ring->xdp_sqes[ntc], &cp); + ntc++; + ntc = unlikely(ntc == xdp_ring->count) ? 0 : ntc; + } + + xdp_ring->next_to_clean = ntc; + xdp_flush_frame_bulk(&cbulk); + if (xsk_frames) + xsk_tx_completed(xdp_ring->xsk_pool, xsk_frames); +} + +static inline void libie_xg_xdp_rs_and_bump(void *xdpsq, bool sent, bool flush) +{ + struct libie_xg_ring *xdp_ring = xdpsq; + union ixgbe_adv_tx_desc *desc; + u32 ltu; + + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + + if ((!flush && xdp_ring->pending < xdp_ring->count - 1) || + xdp_ring->cached_ntu == xdp_ring->next_to_use) + goto unlock; + + ltu = (xdp_ring->next_to_use ? : xdp_ring->count) - 1; + + /* We will not get DD on a context descriptor */ + if (unlikely(xdp_ring->xdp_sqes[ltu].type == LIBETH_SQE_CTX)) + goto unlock; + + desc = LIBIE_XG_TX_DESC(xdp_ring, ltu); + desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); + + xdp_ring->xdp_sqes[xdp_ring->cached_ntu].rs_idx = ltu + 1; + xdp_ring->cached_ntu = xdp_ring->next_to_use; + + /* In case the packet was interrupted, discard it */ + xdp_ring->xdp_sqes[ltu].priv = 0; + + /* Finish descriptor writes before bumping tail */ + wmb(); + writel(xdp_ring->next_to_use, xdp_ring->tail); + +unlock: + libeth_xdpsq_unlock(&xdp_ring->xdpq_lock); +} + +#endif \ No newline at end of file From b2dc0067e55f1d2dc44b68205d0d2b40a2b30ae7 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 12:00:47 +0200 Subject: [PATCH 10/25] Revert "[SPLIT] Move descriptor definitions to a separate file + introduce and use libie_xg_ring" This reverts commit a66f091496171a00c16874e5098083785f16aa23. --- drivers/net/ethernet/intel/ixgbevf/defines.h | 191 ++++++++++++++++++ include/linux/net/intel/ixgbe_desc.h | 200 ------------------- 2 files changed, 191 insertions(+), 200 deletions(-) delete mode 100644 include/linux/net/intel/ixgbe_desc.h diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index a67d1ecd6012a6..afc927dd14381b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -84,6 +84,197 @@ typedef u32 ixgbe_link_speed; #define IXGBE_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS 0x0A000000 #define IXGBE_SRRCTL_DESCTYPE_MASK 0x0E000000 +/* Receive Descriptor bit definitions */ +#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ +#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ +#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ +#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ +#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ +#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 +#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ +#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ +#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ +#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ +#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ +#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ +#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ +#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ +#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ +#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ +#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ +#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ +#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ +#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ +#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ +#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ +#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ +#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ +#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ +#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ +#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ +#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ +#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ +#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ +#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ +#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ +#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ +#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ +#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ +#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ +#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ +#define IXGBE_RXD_PRI_SHIFT 13 +#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ +#define IXGBE_RXD_CFI_SHIFT 12 + +#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ +#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ +#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ +#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ +#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ +#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ +#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ +#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ +#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ +#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ +#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ + +#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F +#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 +#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ +#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ +#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 +#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 +#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 +#define IXGBE_RXDADV_RSCCNT_SHIFT 17 +#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 +#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 +#define IXGBE_RXDADV_SPH 0x8000 + +/* RSS Hash results */ +#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 +#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 +#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 +#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 +#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 +#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 +#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 +#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 + +#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXD_ERR_CE | \ + IXGBE_RXD_ERR_LE | \ + IXGBE_RXD_ERR_PE | \ + IXGBE_RXD_ERR_OSE | \ + IXGBE_RXD_ERR_USE) + +#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ + IXGBE_RXDADV_ERR_CE | \ + IXGBE_RXDADV_ERR_LE | \ + IXGBE_RXDADV_ERR_PE | \ + IXGBE_RXDADV_ERR_OSE | \ + IXGBE_RXDADV_ERR_USE) + +#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ +#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ +#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ +#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ +#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ +#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ +#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ +#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ +#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) + +/* Transmit Descriptor - Advanced */ +union ixgbe_adv_tx_desc { + struct { + __le64 buffer_addr; /* Address of descriptor's data buf */ + __le32 cmd_type_len; + __le32 olinfo_status; + } read; + struct { + __le64 rsvd; /* Reserved */ + __le32 nxtseq_seed; + __le32 status; + } wb; +}; + +/* Receive Descriptor - Advanced */ +union ixgbe_adv_rx_desc { + struct { + __le64 pkt_addr; /* Packet buffer address */ + __le64 hdr_addr; /* Header buffer address */ + } read; + struct { + struct { + union { + __le32 data; + struct { + __le16 pkt_info; /* RSS, Pkt type */ + __le16 hdr_info; /* Splithdr, hdrlen */ + } hs_rss; + } lo_dword; + union { + __le32 rss; /* RSS Hash */ + struct { + __le16 ip_id; /* IP id */ + __le16 csum; /* Packet Checksum */ + } csum_ip; + } hi_dword; + } lower; + struct { + __le32 status_error; /* ext status/error */ + __le16 length; /* Packet length */ + __le16 vlan; /* VLAN tag */ + } upper; + } wb; /* writeback */ +}; + +/* Context descriptors */ +struct ixgbe_adv_tx_context_desc { + __le32 vlan_macip_lens; + __le32 fceof_saidx; + __le32 type_tucmd_mlhl; + __le32 mss_l4len_idx; +}; + +/* Adv Transmit Descriptor Config Masks */ +#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ +#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ +#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ +#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ +#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ +#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ +#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ +#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ +#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ +#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ +#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ +#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ +#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ +#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ +#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ +#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ +#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ +#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ +#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ +#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ + IXGBE_ADVTXD_POPTS_SHIFT) +#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ +#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ +#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ +#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ +#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ + /* Interrupt register bitmasks */ #define IXGBE_EITR_CNT_WDIS 0x80000000 diff --git a/include/linux/net/intel/ixgbe_desc.h b/include/linux/net/intel/ixgbe_desc.h deleted file mode 100644 index fdf8a35044f539..00000000000000 --- a/include/linux/net/intel/ixgbe_desc.h +++ /dev/null @@ -1,200 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2026 Intel Corporation */ - -#ifndef __IXGBE_DESC_H -#define __IXGBE_DESC_H - -#include - -/* Receive Descriptor bit definitions */ -#define IXGBE_RXD_STAT_DD 0x01 /* Descriptor Done */ -#define IXGBE_RXD_STAT_EOP 0x02 /* End of Packet */ -#define IXGBE_RXD_STAT_FLM 0x04 /* FDir Match */ -#define IXGBE_RXD_STAT_VP 0x08 /* IEEE VLAN Packet */ -#define IXGBE_RXDADV_NEXTP_MASK 0x000FFFF0 /* Next Descriptor Index */ -#define IXGBE_RXDADV_NEXTP_SHIFT 0x00000004 -#define IXGBE_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ -#define IXGBE_RXD_STAT_L4CS 0x20 /* L4 xsum calculated */ -#define IXGBE_RXD_STAT_IPCS 0x40 /* IP xsum calculated */ -#define IXGBE_RXD_STAT_PIF 0x80 /* passed in-exact filter */ -#define IXGBE_RXD_STAT_CRCV 0x100 /* Speculative CRC Valid */ -#define IXGBE_RXD_STAT_VEXT 0x200 /* 1st VLAN found */ -#define IXGBE_RXD_STAT_UDPV 0x400 /* Valid UDP checksum */ -#define IXGBE_RXD_STAT_DYNINT 0x800 /* Pkt caused INT via DYNINT */ -#define IXGBE_RXD_STAT_TS 0x10000 /* Time Stamp */ -#define IXGBE_RXD_STAT_SECP 0x20000 /* Security Processing */ -#define IXGBE_RXD_STAT_LB 0x40000 /* Loopback Status */ -#define IXGBE_RXD_STAT_ACK 0x8000 /* ACK Packet indication */ -#define IXGBE_RXD_ERR_CE 0x01 /* CRC Error */ -#define IXGBE_RXD_ERR_LE 0x02 /* Length Error */ -#define IXGBE_RXD_ERR_PE 0x08 /* Packet Error */ -#define IXGBE_RXD_ERR_OSE 0x10 /* Oversize Error */ -#define IXGBE_RXD_ERR_USE 0x20 /* Undersize Error */ -#define IXGBE_RXD_ERR_TCPE 0x40 /* TCP/UDP Checksum Error */ -#define IXGBE_RXD_ERR_IPE 0x80 /* IP Checksum Error */ -#define IXGBE_RXDADV_ERR_MASK 0xFFF00000 /* RDESC.ERRORS mask */ -#define IXGBE_RXDADV_ERR_SHIFT 20 /* RDESC.ERRORS shift */ -#define IXGBE_RXDADV_ERR_HBO 0x00800000 /*Header Buffer Overflow */ -#define IXGBE_RXDADV_ERR_CE 0x01000000 /* CRC Error */ -#define IXGBE_RXDADV_ERR_LE 0x02000000 /* Length Error */ -#define IXGBE_RXDADV_ERR_PE 0x08000000 /* Packet Error */ -#define IXGBE_RXDADV_ERR_OSE 0x10000000 /* Oversize Error */ -#define IXGBE_RXDADV_ERR_USE 0x20000000 /* Undersize Error */ -#define IXGBE_RXDADV_ERR_TCPE 0x40000000 /* TCP/UDP Checksum Error */ -#define IXGBE_RXDADV_ERR_IPE 0x80000000 /* IP Checksum Error */ -#define IXGBE_RXD_VLAN_ID_MASK 0x0FFF /* VLAN ID is in lower 12 bits */ -#define IXGBE_RXD_PRI_MASK 0xE000 /* Priority is in upper 3 bits */ -#define IXGBE_RXD_PRI_SHIFT 13 -#define IXGBE_RXD_CFI_MASK 0x1000 /* CFI is bit 12 */ -#define IXGBE_RXD_CFI_SHIFT 12 - -#define IXGBE_RXDADV_STAT_DD IXGBE_RXD_STAT_DD /* Done */ -#define IXGBE_RXDADV_STAT_EOP IXGBE_RXD_STAT_EOP /* End of Packet */ -#define IXGBE_RXDADV_STAT_FLM IXGBE_RXD_STAT_FLM /* FDir Match */ -#define IXGBE_RXDADV_STAT_VP IXGBE_RXD_STAT_VP /* IEEE VLAN Pkt */ -#define IXGBE_RXDADV_STAT_MASK 0x000FFFFF /* Stat/NEXTP: bit 0-19 */ -#define IXGBE_RXDADV_STAT_FCEOFS 0x00000040 /* FCoE EOF/SOF Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT 0x00000030 /* FCoE Pkt Stat */ -#define IXGBE_RXDADV_STAT_FCSTAT_NOMTCH 0x00000000 /* 00: No Ctxt Match */ -#define IXGBE_RXDADV_STAT_FCSTAT_NODDP 0x00000010 /* 01: Ctxt w/o DDP */ -#define IXGBE_RXDADV_STAT_FCSTAT_FCPRSP 0x00000020 /* 10: Recv. FCP_RSP */ -#define IXGBE_RXDADV_STAT_FCSTAT_DDP 0x00000030 /* 11: Ctxt w/ DDP */ -#define IXGBE_RXDADV_STAT_SECP 0x00020000 /* IPsec/MACsec pkt found */ - -#define IXGBE_RXDADV_RSSTYPE_MASK 0x0000000F -#define IXGBE_RXDADV_PKTTYPE_MASK 0x0000FFF0 -#define IXGBE_RXDADV_PKTTYPE_IPV4 0x00000010 /* IPv4 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPV6 0x00000040 /* IPv6 hdr present */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_ESP 0x00001000 /* IPSec ESP */ -#define IXGBE_RXDADV_PKTTYPE_IPSEC_AH 0x00002000 /* IPSec AH */ -#define IXGBE_RXDADV_PKTTYPE_MASK_EX 0x0001FFF0 -#define IXGBE_RXDADV_HDRBUFLEN_MASK 0x00007FE0 -#define IXGBE_RXDADV_RSCCNT_MASK 0x001E0000 -#define IXGBE_RXDADV_RSCCNT_SHIFT 17 -#define IXGBE_RXDADV_HDRBUFLEN_SHIFT 5 -#define IXGBE_RXDADV_SPLITHEADER_EN 0x00001000 -#define IXGBE_RXDADV_SPH 0x8000 - -/* RSS Hash results */ -#define IXGBE_RXDADV_RSSTYPE_NONE 0x00000000 -#define IXGBE_RXDADV_RSSTYPE_IPV4_TCP 0x00000001 -#define IXGBE_RXDADV_RSSTYPE_IPV4 0x00000002 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP 0x00000003 -#define IXGBE_RXDADV_RSSTYPE_IPV6_EX 0x00000004 -#define IXGBE_RXDADV_RSSTYPE_IPV6 0x00000005 -#define IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX 0x00000006 -#define IXGBE_RXDADV_RSSTYPE_IPV4_UDP 0x00000007 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP 0x00000008 -#define IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX 0x00000009 - -#define IXGBE_RXD_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXD_ERR_CE | \ - IXGBE_RXD_ERR_LE | \ - IXGBE_RXD_ERR_PE | \ - IXGBE_RXD_ERR_OSE | \ - IXGBE_RXD_ERR_USE) - -#define IXGBE_RXDADV_ERR_FRAME_ERR_MASK ( \ - IXGBE_RXDADV_ERR_CE | \ - IXGBE_RXDADV_ERR_LE | \ - IXGBE_RXDADV_ERR_PE | \ - IXGBE_RXDADV_ERR_OSE | \ - IXGBE_RXDADV_ERR_USE) - -#define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ -#define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ -#define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ -#define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) - -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; - -/* Context descriptors */ -struct ixgbe_adv_tx_context_desc { - __le32 vlan_macip_lens; - __le32 fceof_saidx; - __le32 type_tucmd_mlhl; - __le32 mss_l4len_idx; -}; - -/* Adv Transmit Descriptor Config Masks */ -#define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ -#define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ -#define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ -#define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ -#define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ -#define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ -#define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ -#define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ -#define IXGBE_ADVTXD_TUCMD_IPV6 0x00000000 /* IP Packet Type: 0=IPv6 */ -#define IXGBE_ADVTXD_TUCMD_L4T_UDP 0x00000000 /* L4 Packet TYPE of UDP */ -#define IXGBE_ADVTXD_TUCMD_L4T_TCP 0x00000800 /* L4 Packet TYPE of TCP */ -#define IXGBE_ADVTXD_TUCMD_L4T_SCTP 0x00001000 /* L4 Packet TYPE of SCTP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ -#define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ -#define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ -#define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ -#define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ -#define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ - IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ -#define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ -#define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ -#define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ -#define IXGBE_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ - -#endif From 69b5631df814033e8c3e56824d11b7e819c71849 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 13:15:10 +0200 Subject: [PATCH 11/25] fixup! Revert "[SPLIT] Move descriptor definitions to a separate file + introduce and use libie_xg_ring" --- .../ethernet/intel/ixgbe/ixgbe_txrx_common.h | 3 - drivers/net/ethernet/intel/ixgbe/ixgbe_type.h | 56 +---------- drivers/net/ethernet/intel/ixgbevf/defines.h | 58 +---------- include/linux/net/intel/ixgbe_desc_common.h | 99 +++++++++++++++++++ 4 files changed, 102 insertions(+), 114 deletions(-) create mode 100644 include/linux/net/intel/ixgbe_desc_common.h diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h index 78deea5ec5369c..f681f95377980c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h @@ -10,9 +10,6 @@ #define IXGBE_XDP_REDIR BIT(2) #define IXGBE_XDP_EXIT BIT(3) -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | \ - IXGBE_TXD_CMD_RS) - int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, struct xdp_frame *xdpf); bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h index 825cf17fa1878d..914f10ce4b3367 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_type.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "ixgbe_type_e610.h" @@ -2358,13 +2359,8 @@ enum { #define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ #define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ #define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor extension (0 = legacy) */ #define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ /* Multiple Transmit Queue Command Register */ #define IXGBE_MTQC_RT_ENA 0x1 /* DCB Enable */ @@ -2884,51 +2880,6 @@ struct ixgbe_hic_phy_activity_resp { __be32 data[FW_PHY_ACT_DATA_COUNT]; }; -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; - /* Context descriptors */ struct ixgbe_adv_tx_context_desc { __le32 vlan_macip_lens; @@ -2952,19 +2903,15 @@ enum { #define IXGBE_ADVTXD_IPSEC_ESP_LEN_MASK 0x000001FF /* IPSec ESP length */ #define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ #define IXGBE_ADVTXD_DTYP_CTXT 0x2 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ #define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ #define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ #define IXGBE_ADVTXD_DCMD_DDTYP_ISCSI 0x10000000 /* DDP hdr type or iSCSI */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ #define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ #define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ #define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ #define IXGBE_ADVTXD_STAT_SN_CRC 0x00000002 /* NXTSEQ/SEED pres in WB */ #define IXGBE_ADVTXD_STAT_RSV 0x0000000C /* STA Reserved */ #define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ #define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ #define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ IXGBE_ADVTXD_POPTS_SHIFT) @@ -2976,7 +2923,6 @@ enum { #define IXGBE_ADVTXD_POPTS_ISCO_LAST 0x00001000 /* Last TSO of iSCSI PDU */ #define IXGBE_ADVTXD_POPTS_ISCO_FULL 0x00001800 /* 1st&Last TSO-full iSCSI PDU */ #define IXGBE_ADVTXD_POPTS_RSV 0x00002000 /* POPTS Reserved */ -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ #define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ #define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ #define IXGBE_ADVTXD_TUCMD_IPV4 0x00000400 /* IP Packet Type: 1=IPv4 */ diff --git a/drivers/net/ethernet/intel/ixgbevf/defines.h b/drivers/net/ethernet/intel/ixgbevf/defines.h index afc927dd14381b..b3554b75c591e8 100644 --- a/drivers/net/ethernet/intel/ixgbevf/defines.h +++ b/drivers/net/ethernet/intel/ixgbevf/defines.h @@ -4,6 +4,8 @@ #ifndef _IXGBEVF_DEFINES_H_ #define _IXGBEVF_DEFINES_H_ +#include + /* Device IDs */ #define IXGBE_DEV_ID_82599_VF 0x10ED #define IXGBE_DEV_ID_X540_VF 0x1515 @@ -181,59 +183,8 @@ typedef u32 ixgbe_link_speed; #define IXGBE_TXD_POPTS_IXSM 0x01 /* Insert IP checksum */ #define IXGBE_TXD_POPTS_TXSM 0x02 /* Insert TCP/UDP checksum */ -#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ -#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ #define IXGBE_TXD_CMD_IC 0x04000000 /* Insert Checksum */ -#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ -#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ #define IXGBE_TXD_CMD_VLE 0x40000000 /* Add VLAN tag */ -#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ -#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) - -/* Transmit Descriptor - Advanced */ -union ixgbe_adv_tx_desc { - struct { - __le64 buffer_addr; /* Address of descriptor's data buf */ - __le32 cmd_type_len; - __le32 olinfo_status; - } read; - struct { - __le64 rsvd; /* Reserved */ - __le32 nxtseq_seed; - __le32 status; - } wb; -}; - -/* Receive Descriptor - Advanced */ -union ixgbe_adv_rx_desc { - struct { - __le64 pkt_addr; /* Packet buffer address */ - __le64 hdr_addr; /* Header buffer address */ - } read; - struct { - struct { - union { - __le32 data; - struct { - __le16 pkt_info; /* RSS, Pkt type */ - __le16 hdr_info; /* Splithdr, hdrlen */ - } hs_rss; - } lo_dword; - union { - __le32 rss; /* RSS Hash */ - struct { - __le16 ip_id; /* IP id */ - __le16 csum; /* Packet Checksum */ - } csum_ip; - } hi_dword; - } lower; - struct { - __le32 status_error; /* ext status/error */ - __le16 length; /* Packet length */ - __le16 vlan; /* VLAN tag */ - } upper; - } wb; /* writeback */ -}; /* Context descriptors */ struct ixgbe_adv_tx_context_desc { @@ -246,11 +197,8 @@ struct ixgbe_adv_tx_context_desc { /* Adv Transmit Descriptor Config Masks */ #define IXGBE_ADVTXD_DTYP_MASK 0x00F00000 /* DTYP mask */ #define IXGBE_ADVTXD_DTYP_CTXT 0x00200000 /* Advanced Context Desc */ -#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ #define IXGBE_ADVTXD_DCMD_EOP IXGBE_TXD_CMD_EOP /* End of Packet */ -#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ #define IXGBE_ADVTXD_DCMD_RS IXGBE_TXD_CMD_RS /* Report Status */ -#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ #define IXGBE_ADVTXD_DCMD_VLE IXGBE_TXD_CMD_VLE /* VLAN pkt enable */ #define IXGBE_ADVTXD_DCMD_TSE 0x80000000 /* TCP Seg enable */ #define IXGBE_ADVTXD_STAT_DD IXGBE_TXD_STAT_DD /* Descriptor Done */ @@ -262,14 +210,12 @@ struct ixgbe_adv_tx_context_desc { #define IXGBE_ADVTXD_TUCMD_IPSEC_TYPE_ESP 0x00002000 /* IPSec Type ESP */ #define IXGBE_ADVTXD_TUCMD_IPSEC_ENCRYPT_EN 0x00004000 /* ESP Encrypt Enable */ #define IXGBE_ADVTXD_IDX_SHIFT 4 /* Adv desc Index shift */ -#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ #define IXGBE_ADVTXD_POPTS_SHIFT 8 /* Adv desc POPTS shift */ #define IXGBE_ADVTXD_POPTS_IPSEC 0x00000400 /* IPSec offload request */ #define IXGBE_ADVTXD_POPTS_IXSM (IXGBE_TXD_POPTS_IXSM << \ IXGBE_ADVTXD_POPTS_SHIFT) #define IXGBE_ADVTXD_POPTS_TXSM (IXGBE_TXD_POPTS_TXSM << \ IXGBE_ADVTXD_POPTS_SHIFT) -#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ #define IXGBE_ADVTXD_MACLEN_SHIFT 9 /* Adv ctxt desc mac len shift */ #define IXGBE_ADVTXD_VLAN_SHIFT 16 /* Adv ctxt vlan tag shift */ #define IXGBE_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ diff --git a/include/linux/net/intel/ixgbe_desc_common.h b/include/linux/net/intel/ixgbe_desc_common.h new file mode 100644 index 00000000000000..144d5333d798c3 --- /dev/null +++ b/include/linux/net/intel/ixgbe_desc_common.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2026 Intel Corporation */ + +#ifndef __IXGBE_DESC_COMMON_H +#define __IXGBE_DESC_COMMON_H + +#include + +/** + * union ixgbe_adv_rx_desc - Receive Descriptor - Advanced + * @read: read descriptor + * @read.pkt_addr: packet buffer address + * @read.hdr_addr: header buffer address + * @wb: writeback descriptor + * @wb.lower: lower 8 bytes + * @wb.lower.lo_dword: lowest 4 bytes + * @wb.lower.lo_dword.data: lower 4 bytes + * @wb.lower.lo_dword.hs_rss: RSS packet type and header split info + * @wb.lower.lo_dword.hs_rss.pkt_info: RSS Packet Type + * @wb.lower.lo_dword.hs_rss.hdr_info: header split info + * @wb.lower.hi_dword: higher 4 bytes of lower 8 bytes + * @wb.lower.hi_dword.rss: RSS hash + * @wb.lower.hi_dword.csum_ip: IP ID and packet checksum + * @wb.lower.hi_dword.csum_ip.ip_id: IP ID + * @wb.lower.hi_dword.csum_ip.csum: packet checksum + * @wb.upper: upper 8 bytes + * @wb.upper.status_error: reported status and errors + * @wb.upper.length: packet length + * @wb.upper.vlan: VLAN tag + */ +union ixgbe_adv_rx_desc { + struct { + __le64 pkt_addr; + __le64 hdr_addr; + } read; + struct { + struct { + union { + __le32 data; + struct { + __le16 pkt_info; + __le16 hdr_info; + } hs_rss; + } lo_dword; + union { + __le32 rss; + struct { + __le16 ip_id; + __le16 csum; + } csum_ip; + } hi_dword; + } lower; + struct { + __le32 status_error; + __le16 length; + __le16 vlan; + } upper; + } wb; +}; + +/** + * union ixgbe_adv_tx_desc - Transmit Descriptor - Advanced + * @read: read descriptor + * @read.buffer_addr: address of the descriptor's data buffer + * @read.cmd_type_len: length and basic frame flags (EoP, RS, etc.) + * @read.olinfo_status: per-packet offload information + * @wb: writeback descriptor + * @wb.rsvd1: reserved + * @wb.rsvd2: reserved + * @wb.status: completion status + */ +union ixgbe_adv_tx_desc { + struct { + __le64 buffer_addr; + __le32 cmd_type_len; + __le32 olinfo_status; + } read; + struct { + __le64 rsvd1; + __le32 rsvd2; + __le32 status; + } wb; +}; + +#define IXGBE_TXD_CMD_EOP 0x01000000 /* End of Packet */ +#define IXGBE_TXD_CMD_IFCS 0x02000000 /* Insert FCS (Ethernet CRC) */ +#define IXGBE_TXD_CMD_RS 0x08000000 /* Report Status */ +#define IXGBE_TXD_CMD_DEXT 0x20000000 /* Descriptor ext (0 = legacy) */ +#define IXGBE_TXD_STAT_DD 0x00000001 /* Descriptor Done */ +#define IXGBE_TXD_CMD (IXGBE_TXD_CMD_EOP | IXGBE_TXD_CMD_RS) + +#define IXGBE_ADVTXD_DTYP_DATA 0x00300000 /* Advanced Data Descriptor */ +#define IXGBE_ADVTXD_DCMD_IFCS IXGBE_TXD_CMD_IFCS /* Insert FCS */ +#define IXGBE_ADVTXD_DCMD_DEXT IXGBE_TXD_CMD_DEXT /* Desc ext (1=Adv) */ +#define IXGBE_ADVTXD_CC 0x00000080 /* Check Context */ + +#define IXGBE_ADVTXD_PAYLEN_SHIFT 14 /* Adv desc PAYLEN shift */ + +#endif /* __IXGBE_DESC_COMMON_H */ From a5cf86e2045ed9d8723b840e524900b587afa8ac Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 13:18:16 +0200 Subject: [PATCH 12/25] fixup! ixgbevf, libie: separate ixgbevf_ring base and move to libie_xg --- include/linux/net/intel/libie/xg_ring.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h index 768861c5e0688c..671780c85824e8 100644 --- a/include/linux/net/intel/libie/xg_ring.h +++ b/include/linux/net/intel/libie/xg_ring.h @@ -8,7 +8,7 @@ #include #include -#include +#include struct libie_xg_queue_stats { u64 packets; From 0a893243c0465563272f2e29757afd6c6f053578 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 15:06:49 +0200 Subject: [PATCH 13/25] fixup! ixgbevf, libie: separate ixgbevf_ring base and move to libie_xg --- include/linux/net/intel/libie/xg_ring.h | 29 ++----------------------- 1 file changed, 2 insertions(+), 27 deletions(-) diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h index 671780c85824e8..04c3814e531158 100644 --- a/include/linux/net/intel/libie/xg_ring.h +++ b/include/linux/net/intel/libie/xg_ring.h @@ -4,33 +4,10 @@ #ifndef __LIBIE_XG_RING_H #define __LIBIE_XG_RING_H -#include +#include #include #include -#include - -struct libie_xg_queue_stats { - u64 packets; - u64 bytes; -}; - -struct libie_xg_tx_queue_stats { - u64 restart_queue; - u64 tx_busy; - u64 tx_done_old; -}; - -struct libie_xg_rx_queue_stats { - /* u64 rsc_count; <- was in ixgbe - u64 rsc_flush; */ - u64 non_eop_descs; - u64 alloc_rx_page; - u64 alloc_rx_page_failed; - u64 alloc_rx_buff_failed; - u64 csum_err; -}; - struct libie_xg_tx_buffer { union ixgbe_adv_tx_desc *next_to_watch; unsigned long time_stamp; @@ -76,7 +53,5 @@ struct libie_xg_ring { (&(((union ixgbe_adv_rx_desc *)((R)->desc))[i])) #define LIBIE_XG_TX_DESC(R, i) \ (&(((union ixgbe_adv_tx_desc *)((R)->desc))[i])) -#define LIBIE_XG_TX_CTXTDESC(R, i) \ - (&(((struct ixgbe_adv_tx_context_desc *)((R)->desc))[i])) -#endif \ No newline at end of file +#endif /* __LIBIE_XG_RING_H */ From 2c742865da5263eaf346396d15eb21f54441b21e Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Tue, 18 Aug 2026 15:18:08 +0200 Subject: [PATCH 14/25] ixgbe, libie: use libie_xg_ring in ixgbe Replace overlapping fields in ixgbe_ring with libie_xg_ring, included both as anonymous and named structure. For now, add xdp_frame to the generic Tx buffer, but it will be rendered unnecessary by further commits. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 45 ++++--------------- .../net/ethernet/intel/ixgbe/ixgbe_ethtool.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c | 2 +- .../net/ethernet/intel/ixgbe/ixgbe_ipsec.c | 2 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 28 ++++++------ drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 8 ++-- include/linux/net/intel/libie/xg_ring.h | 2 +- 7 files changed, 31 insertions(+), 58 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index af801eacc45c65..ea69a056b3f5ae 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -32,6 +32,7 @@ #endif #include "ixgbe_ipsec.h" +#include #include #include #include @@ -196,23 +197,6 @@ struct vf_macvlans { #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -/* wrapper around a pointer to a socket buffer, - * so a DMA handle can be stored along with the buffer */ -struct ixgbe_tx_buffer { - union ixgbe_adv_tx_desc *next_to_watch; - unsigned long time_stamp; - union { - struct sk_buff *skb; - struct xdp_frame *xdpf; - }; - unsigned int bytecount; - unsigned short gso_segs; - __be16 protocol; - DEFINE_DMA_UNMAP_ADDR(dma); - DEFINE_DMA_UNMAP_LEN(len); - u32 tx_flags; -}; - struct ixgbe_xsk_rx_buffer { bool discard; struct xdp_buff *xdp; @@ -280,36 +264,25 @@ struct ixgbe_fwd_adapter { #define clear_ring_xdp(ring) \ clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) struct ixgbe_ring { + union { + struct libie_xg_ring base; + struct libie_xg_ring; + }; struct ixgbe_ring *next; /* pointer to next ring in q_vector */ struct ixgbe_q_vector *q_vector; /* backpointer to host q_vector */ struct net_device *netdev; /* netdev ring belongs to */ struct bpf_prog __rcu *xdp_prog; - union { - struct page_pool *pp; /* Rx ring */ - struct device *dev; /* Tx ring */ - }; - void *desc; /* descriptor ring memory */ - union { - struct libeth_fqe *rx_fqes; - struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; - struct ixgbe_tx_buffer *tx_buffer_info; - }; unsigned long state; - u8 __iomem *tail; dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ u32 truesize; - u16 count; /* amount of descriptors */ - u8 queue_index; /* needed for multiqueue queue management */ u8 reg_idx; /* holds the special value that gets * the hardware register offset * associated with this ring, which is * different for DCB and RSS modes */ - u16 next_to_use; - u16 next_to_clean; unsigned long last_rx_timestamp; @@ -327,10 +300,10 @@ struct ixgbe_ring { }; struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ - struct xsk_buff_pool *xsk_pool; u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; + struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { @@ -923,7 +896,7 @@ int ixgbe_sysfs_init(struct ixgbe_adapter *adapter); #endif /* CONFIG_IXGBE_HWMON */ #ifdef IXGBE_FCOE void ixgbe_configure_fcoe(struct ixgbe_adapter *adapter); -int ixgbe_fso(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, +int ixgbe_fso(struct ixgbe_ring *tx_ring, struct libie_xg_tx_buffer *first, u8 *hdr_len); int ixgbe_fcoe_ddp(struct ixgbe_adapter *adapter, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); @@ -1012,7 +985,7 @@ void ixgbe_ipsec_restore(struct ixgbe_adapter *adapter); void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); -int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, struct ixgbe_tx_buffer *first, +int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd); void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, u32 vf); int ixgbe_ipsec_vf_add_sa(struct ixgbe_adapter *adapter, u32 *mbuf, u32 vf); @@ -1025,7 +998,7 @@ static inline void ixgbe_ipsec_rx(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb) { } static inline int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd) { return 0; } static inline void ixgbe_ipsec_vf_clear(struct ixgbe_adapter *adapter, u32 vf) { } diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index a2cc9b39eb634c..903053c892a203 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -2087,8 +2087,8 @@ static u16 ixgbe_clean_test_rings(struct ixgbe_ring *rx_ring, rx_desc = IXGBE_RX_DESC(rx_ring, rx_ntc); while (tx_ntc != tx_ring->next_to_use) { + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; - struct ixgbe_tx_buffer *tx_buffer; tx_desc = IXGBE_TX_DESC(tx_ring, tx_ntc); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c index 011fda9c61939c..4660c62bb3e7e8 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c @@ -486,7 +486,7 @@ int ixgbe_fcoe_ddp(struct ixgbe_adapter *adapter, * Returns : 0 indicates success, < 0 for error */ int ixgbe_fso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, u8 *hdr_len) { struct sk_buff *skb = first->skb; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c index d1f4073b36f906..8e2c01daf7adb1 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ipsec.c @@ -1054,7 +1054,7 @@ int ixgbe_ipsec_vf_del_sa(struct ixgbe_adapter *adapter, u32 *msgbuf, u32 vf) * @itd: ipsec Tx data for later use in building context descriptor **/ int ixgbe_ipsec_tx(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(tx_ring->netdev); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index a59939ba094b2a..9ccff868a24040 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -569,7 +569,7 @@ static void ixgbe_regdump(struct ixgbe_hw *hw, struct ixgbe_reg_info *reginfo) static void ixgbe_print_buffer(struct ixgbe_ring *ring, int n) { - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; tx_buffer = &ring->tx_buffer_info[ring->next_to_clean]; pr_info(" %5d %5X %5X %016llX %08X %p %016llX\n", @@ -590,7 +590,7 @@ static void ixgbe_dump(struct ixgbe_adapter *adapter) struct ixgbe_reg_info *reginfo; int n = 0; struct ixgbe_ring *ring; - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; struct my_u0 { u64 a; u64 b; } *u0; struct ixgbe_ring *rx_ring; @@ -1348,7 +1348,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring, int napi_budget) { struct ixgbe_adapter *adapter = q_vector->adapter; - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; unsigned int total_bytes = 0, total_packets = 0, total_ipsec = 0; unsigned int budget = q_vector->tx.work_limit; @@ -3493,7 +3493,7 @@ void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, /* reinitialize tx_buffer_info */ memset(ring->tx_buffer_info, 0, - sizeof(struct ixgbe_tx_buffer) * ring->count); + sizeof(struct libie_xg_tx_buffer) * ring->count); /* enable queue */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), txdctl); @@ -6002,7 +6002,7 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) { u16 i = tx_ring->next_to_clean; - struct ixgbe_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; + struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; if (tx_ring->xsk_pool) { ixgbe_xsk_clean_tx_ring(tx_ring); @@ -6488,7 +6488,7 @@ int ixgbe_setup_tx_resources(struct ixgbe_ring *tx_ring) int ring_node = NUMA_NO_NODE; int size; - size = sizeof(struct ixgbe_tx_buffer) * tx_ring->count; + size = sizeof(struct libie_xg_tx_buffer) * tx_ring->count; if (tx_ring->q_vector) ring_node = tx_ring->q_vector->numa_node; @@ -8151,7 +8151,7 @@ static void ixgbe_service_task(struct work_struct *work) } static int ixgbe_tso(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, u8 *hdr_len, struct ixgbe_ipsec_tx_data *itd) { @@ -8257,7 +8257,7 @@ static int ixgbe_tso(struct ixgbe_ring *tx_ring, } static void ixgbe_tx_csum(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, struct ixgbe_ipsec_tx_data *itd) { struct sk_buff *skb = first->skb; @@ -8386,11 +8386,11 @@ static inline int ixgbe_maybe_stop_tx(struct ixgbe_ring *tx_ring, u16 size) } static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *first, + struct libie_xg_tx_buffer *first, const u8 hdr_len) { struct sk_buff *skb = first->skb; - struct ixgbe_tx_buffer *tx_buffer; + struct libie_xg_tx_buffer *tx_buffer; union ixgbe_adv_tx_desc *tx_desc; skb_frag_t *frag; dma_addr_t dma; @@ -8540,7 +8540,7 @@ static int ixgbe_tx_map(struct ixgbe_ring *tx_ring, } static void ixgbe_atr(struct ixgbe_ring *ring, - struct ixgbe_tx_buffer *first) + struct libie_xg_tx_buffer *first) { struct ixgbe_q_vector *q_vector = ring->q_vector; union ixgbe_atr_hash_dword input = { .dword = 0 }; @@ -8739,8 +8739,8 @@ int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; u16 i = 0, index = ring->next_to_use; - struct ixgbe_tx_buffer *tx_head = &ring->tx_buffer_info[index]; - struct ixgbe_tx_buffer *tx_buff = tx_head; + struct libie_xg_tx_buffer *tx_head = &ring->tx_buffer_info[index]; + struct libie_xg_tx_buffer *tx_buff = tx_head; union ixgbe_adv_tx_desc *tx_desc = IXGBE_TX_DESC(ring, index); u32 cmd_type, len = xdpf->len; void *data = xdpf->data; @@ -8819,7 +8819,7 @@ netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct ixgbe_adapter *adapter, struct ixgbe_ring *tx_ring) { - struct ixgbe_tx_buffer *first; + struct libie_xg_tx_buffer *first; int tso; u32 tx_flags = 0; unsigned short f; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 15425d3c367872..c84753e2bf904c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -393,7 +393,7 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) { struct xsk_buff_pool *pool = xdp_ring->xsk_pool; union ixgbe_adv_tx_desc *tx_desc = NULL; - struct ixgbe_tx_buffer *tx_bi; + struct libie_xg_tx_buffer *tx_bi; bool work_done = true; struct xdp_desc desc; dma_addr_t dma; @@ -447,7 +447,7 @@ static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) } static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring, - struct ixgbe_tx_buffer *tx_bi) + struct libie_xg_tx_buffer *tx_bi) { xdp_return_frame(tx_bi->xdpf); dma_unmap_single(tx_ring->dev, @@ -463,7 +463,7 @@ bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector, unsigned int total_packets = 0, total_bytes = 0; struct xsk_buff_pool *pool = tx_ring->xsk_pool; union ixgbe_adv_tx_desc *tx_desc; - struct ixgbe_tx_buffer *tx_bi; + struct libie_xg_tx_buffer *tx_bi; u32 xsk_frames = 0; tx_bi = &tx_ring->tx_buffer_info[ntc]; @@ -544,7 +544,7 @@ void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring) { u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use; struct xsk_buff_pool *pool = tx_ring->xsk_pool; - struct ixgbe_tx_buffer *tx_bi; + struct libie_xg_tx_buffer *tx_bi; u32 xsk_frames = 0; while (ntc != ntu) { diff --git a/include/linux/net/intel/libie/xg_ring.h b/include/linux/net/intel/libie/xg_ring.h index 04c3814e531158..3c9fde6b8d739c 100644 --- a/include/linux/net/intel/libie/xg_ring.h +++ b/include/linux/net/intel/libie/xg_ring.h @@ -13,8 +13,8 @@ struct libie_xg_tx_buffer { unsigned long time_stamp; union { struct sk_buff *skb; - /* XDP uses address ptr on irq_clean */ void *data; + struct xdp_frame *xdpf; }; unsigned int bytecount; unsigned short gso_segs; From eaa0d9ceb83d01f4cea53c5c8bd932b486e204f5 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 10:13:53 +0200 Subject: [PATCH 15/25] ixgbe: simplify xsk wakeup Wakeup is not necessarily tied to Tx processing, so Tx not being active is not a reason to skip on Rx buffer refill. Also, some checks are redundant. Rely on Rx rings to access the interrupt, and clear up the redundant code. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index c84753e2bf904c..cae45570d3e36a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -517,18 +517,12 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) if (test_bit(__IXGBE_DOWN, &adapter->state)) return -ENETDOWN; - if (!READ_ONCE(adapter->xdp_prog)) - return -EINVAL; - if (qid >= adapter->num_xdp_queues) return -EINVAL; - ring = adapter->xdp_ring[qid]; - - if (test_bit(__IXGBE_TX_DISABLED, &ring->state)) - return -ENETDOWN; + ring = adapter->rx_ring[qid]; - if (!ring->xsk_pool) + if (unlikely(!ring->xsk_pool)) return -EINVAL; if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) { From 3fd633cddfc23eebff0aebe9c6d24baefcfe233d Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 1 Jun 2026 15:34:38 +0200 Subject: [PATCH 16/25] ixgbe: add pseudo header split Introduce pseudo header split support in the ixgbe driver. * Added IXGBE_RX_SRRCTL_BUF_SIZE macro and IXGBE_FLAG2_HSPLIT flag * Added hdr_truesize, hdr_fqes, hdr_pp, hdr_buf_len to ixgbe_ring * Added ixgbe_rx_hsplit_wa() to copy header from data to header buffer * Added ixgbe_rx_create_pp()/ixgbe_rx_destroy_pp() helpers * Updated ixgbe_alloc_rx_buffers to allocate header buffers * Updated ixgbe_clean_rx_irq to process header split * Refactored ixgbe_setup_rx_resources to use ixgbe_rx_create_pp * Refactored ixgbe_free_rx_resources to use ixgbe_rx_destroy_pp * Set IXGBE_FLAG2_HSPLIT for 82599 in ixgbe_sw_init * Updated ixgbe_xdp_setup to require mbuf frags with hsplit Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 7 + drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 187 ++++++++++++++---- 2 files changed, 157 insertions(+), 37 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index ea69a056b3f5ae..991c846dcd778c 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -78,6 +78,8 @@ #define IXGBE_RX_PAGE_LEN(hr) (ALIGN_DOWN(LIBETH_RX_PAGE_LEN(hr), \ IXGBE_SRRCTL_BSIZEPKT_STEP)) +#define IXGBE_RX_SRRCTL_BUF_SIZE(mtu) (ALIGN((mtu) + LIBETH_RX_LL_LEN, \ + IXGBE_SRRCTL_BSIZEPKT_STEP)) /* Default HW header buffer size */ #define IXGBE_RX_HDR_SIZE IXGBE_RXBUFFER_256 @@ -276,6 +278,7 @@ struct ixgbe_ring { dma_addr_t dma; /* phys. address of descriptor ring */ unsigned int size; /* length in bytes */ u32 truesize; + u32 hdr_truesize; /* Rx header buffer full size */ u8 queue_index; /* needed for multiqueue queue management */ u8 reg_idx; /* holds the special value that gets @@ -298,9 +301,12 @@ struct ixgbe_ring { struct ixgbe_tx_queue_stats tx_stats; struct ixgbe_rx_queue_stats rx_stats; }; + struct libeth_fqe *hdr_fqes; + struct page_pool *hdr_pp; struct xdp_rxq_info xdp_rxq; spinlock_t tx_lock; /* used in XDP mode */ u16 ring_idx; /* {rx,tx,xdp}_ring back reference idx */ + u32 hdr_buf_len; u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; @@ -556,6 +562,7 @@ struct ixgbe_adapter { #define IXGBE_FLAG2_MOD_POWER_UNSUPPORTED BIT(22) #define IXGBE_FLAG2_API_MISMATCH BIT(23) #define IXGBE_FLAG2_FW_ROLLBACK BIT(24) +#define IXGBE_FLAG2_HSPLIT BIT(25) /* Tx fast path data */ int num_tx_queues; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 9ccff868a24040..40d3248297a098 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1742,6 +1742,12 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) .truesize = rx_ring->truesize, .count = rx_ring->count, }; + const struct libeth_fq_fp hdr_fq = { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + .truesize = rx_ring->hdr_truesize, + .count = rx_ring->count, + }; u16 ntu = rx_ring->next_to_use; /* nothing to do */ @@ -1758,6 +1764,14 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) rx_desc->read.pkt_addr = cpu_to_le64(addr); + if (hdr_fq.pp) { + addr = libeth_rx_alloc(&hdr_fq, ntu); + if (addr == DMA_MAPPING_ERROR) { + libeth_rx_recycle_slow(fq.fqes[ntu].netmem); + break; + } + } + rx_desc++; ntu++; @@ -2017,6 +2031,32 @@ static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, return result; } +static u32 ixgbe_rx_hsplit_wa(const struct libeth_fqe *hdr, + struct libeth_fqe *buf, u32 data_len) +{ + u32 copy = data_len <= L1_CACHE_BYTES ? data_len : ETH_HLEN; + struct page *hdr_page, *buf_page; + const void *src; + void *dst; + + if (unlikely(netmem_is_net_iov(buf->netmem)) || + !libeth_rx_sync_for_cpu(buf, copy)) + return 0; + + hdr_page = __netmem_to_page(hdr->netmem); + buf_page = __netmem_to_page(buf->netmem); + + dst = page_address(hdr_page) + hdr->offset + + pp_page_to_nmdesc(hdr_page)->pp->p.offset; + src = page_address(buf_page) + buf->offset + + pp_page_to_nmdesc(buf_page)->pp->p.offset; + + memcpy(dst, src, LARGEST_ALIGN(copy)); + buf->offset += copy; + + return copy; +} + /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2072,6 +2112,23 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, dma_rmb(); rx_buffer = &rx_ring->rx_fqes[rx_ring->next_to_clean]; + + if (unlikely(rx_ring->hdr_pp)) { + struct libeth_fqe *hdr_buff; + unsigned int hdr_size = 0; + + hdr_buff = &rx_ring->hdr_fqes[rx_ring->next_to_clean]; + + if (!xdp->data) { + hdr_size = ixgbe_rx_hsplit_wa(hdr_buff, + rx_buffer, + size); + size -= hdr_size ? : size; + } + + libeth_xdp_process_buff(xdp, hdr_buff, hdr_size); + } + libeth_xdp_process_buff(xdp, rx_buffer, size); cleaned_count++; @@ -6365,6 +6422,7 @@ static int ixgbe_sw_init(struct ixgbe_adapter *adapter, case ixgbe_mac_82599EB: if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) adapter->flags2 |= IXGBE_FLAG2_TEMP_SENSOR_CAPABLE; + adapter->flags2 |= IXGBE_FLAG2_HSPLIT; break; case ixgbe_mac_X540: fwsm = IXGBE_READ_REG(hw, IXGBE_FWSM(hw)); @@ -6567,38 +6625,53 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } -/** - * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) - * @adapter: pointer to ixgbe_adapter - * @rx_ring: rx descriptor ring (for a specific queue) to setup - * - * Returns 0 on success, negative on failure - **/ -int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring) +static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) +{ + struct libeth_fq fq = { + .pp = rx_ring->pp, + .fqes = rx_ring->rx_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->rx_fqes = NULL; + rx_ring->pp = NULL; + + if (!rx_ring->hdr_pp) + return; + + fq = (struct libeth_fq) { + .pp = rx_ring->hdr_pp, + .fqes = rx_ring->hdr_fqes, + }; + + libeth_rx_fq_destroy(&fq); + rx_ring->hdr_fqes = NULL; + rx_ring->hdr_pp = NULL; +} + +static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) { + struct ixgbe_adapter *adapter = netdev_priv(rx_ring->netdev); struct libeth_fq fq = { .count = rx_ring->count, .nid = NUMA_NO_NODE, .type = LIBETH_FQE_MTU, .xdp = !!rx_ring->xdp_prog, .buf_len = IXGBE_RX_PAGE_LEN(rx_ring->xdp_prog ? - LIBETH_XDP_HEADROOM : - LIBETH_SKB_HEADROOM), + LIBETH_XDP_HEADROOM : + LIBETH_SKB_HEADROOM), }; - struct device *dev = &adapter->pdev->dev; - int orig_node = dev_to_node(dev); - struct napi_struct *napi_dev; - int ring_node = NUMA_NO_NODE; + u32 frame_size; int ret; - if (rx_ring->q_vector) { - fq.nid = rx_ring->q_vector->numa_node; - ring_node = rx_ring->q_vector->numa_node; - napi_dev = &rx_ring->q_vector->napi; - } else { - napi_dev = NULL; - } + /* Some HW requires DMA write sizes to be aligned to 1K, + * which warrants fake header split usage, but this is + * not an issue if the frame size is at its maximum of 3K + */ + frame_size = + IXGBE_RX_SRRCTL_BUF_SIZE(READ_ONCE(rx_ring->netdev->mtu)); + fq.hsplit = (adapter->flags2 & IXGBE_FLAG2_HSPLIT) && + frame_size < fq.buf_len; ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); if (ret) @@ -6609,6 +6682,53 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->truesize = fq.truesize; rx_ring->rx_buf_len = fq.buf_len; + if (!fq.hsplit) + return 0; + + fq = (struct libeth_fq) { + .count = rx_ring->count, + .nid = NUMA_NO_NODE, + .type = LIBETH_FQE_HDR, + .xdp = !!rx_ring->xdp_prog, + }; + + ret = libeth_rx_fq_create(&fq, &rx_ring->q_vector->napi); + if (ret) + goto err; + + rx_ring->hdr_pp = fq.pp; + rx_ring->hdr_fqes = fq.fqes; + rx_ring->hdr_truesize = fq.truesize; + + return 0; + +err: + ixgbe_rx_destroy_pp(rx_ring); + return ret; +} + +/** + * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) + * @adapter: pointer to ixgbe_adapter + * @rx_ring: rx descriptor ring (for a specific queue) to setup + * + * Returns 0 on success, negative on failure + **/ +int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, + struct ixgbe_ring *rx_ring) +{ + struct device *dev = &adapter->pdev->dev; + int orig_node = dev_to_node(dev); + int ring_node = NUMA_NO_NODE; + int ret; + + if (rx_ring->q_vector) + ring_node = rx_ring->q_vector->numa_node; + + ret = ixgbe_rx_create_pp(rx_ring); + if (ret) + return ret; + /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); @@ -6637,7 +6757,7 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, if (ret < 0) goto err_free_desc; - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, fq.pp); + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); return 0; @@ -6647,9 +6767,8 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; err_destroy_fq: - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; + dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); + ixgbe_rx_destroy_pp(rx_ring); return ret; } @@ -6737,11 +6856,6 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) **/ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { - struct libeth_fq fq = { - .fqes = rx_ring->rx_fqes, - .pp = rx_ring->pp, - }; - ixgbe_clean_rx_ring(rx_ring); rcu_assign_pointer(rx_ring->xdp_prog, NULL); @@ -6752,7 +6866,7 @@ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) if (!rx_ring->desc) return; - dma_free_coherent(fq.pp->p.dev, rx_ring->size, rx_ring->desc, + dma_free_coherent(rx_ring->pp->p.dev, rx_ring->size, rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; @@ -6760,9 +6874,7 @@ void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) kvfree(rx_ring->rx_xsk_buffer_info); rx_ring->rx_xsk_buffer_info = NULL; - libeth_rx_fq_destroy(&fq); - rx_ring->rx_fqes = NULL; - rx_ring->pp = NULL; + ixgbe_rx_destroy_pp(rx_ring); } /** @@ -10359,10 +10471,11 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, return -EINVAL; } - requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM); + requires_mbuf = frame_size > IXGBE_RX_PAGE_LEN(LIBETH_XDP_HEADROOM) || + adapter->flags2 & IXGBE_FLAG2_HSPLIT; if (prog && !prog->aux->xdp_has_frags && requires_mbuf) { NL_SET_ERR_MSG_MOD(extack, - "Configured MTU requires non-linear frames and XDP prog does not support frags"); + "HW limitations or configured MTU requires non-linear frames and XDP prog does not support frags"); return -EOPNOTSUPP; } From 48623077507d3934c31169ea5b271ab91d67c458 Mon Sep 17 00:00:00 2001 From: Magdalena Pytel Date: Mon, 1 Jun 2026 16:06:54 +0200 Subject: [PATCH 17/25] ixgbe: reconfigure page pool when reallocating buffers * Moved page pool creation from ixgbe_setup_rx_resources() to ixgbe_configure_rx_ring() * Moved page pool destruction to ixgbe_clean_all_rx_rings() and ixgbe_free_rx_resources() * Added ixgbe_dma_dev_from_ring() helper to obtain DMA device without relying on page pool pointer. * Handled XSK path in ixgbe_configure_rx_ring() Signed-off-by: Magdalena Pytel --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 87 +++++++++++-------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 40d3248297a098..9941c48ae89a1a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -4031,6 +4031,10 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); } +static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring); +static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring); +static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring); + #define IXGBE_MAX_RX_DESC_POLL 10 static void ixgbe_rx_desc_queue_enable(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) @@ -4067,15 +4071,18 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, u32 rxdctl; u8 reg_idx = ring->reg_idx; - xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq); + ixgbe_rx_destroy_pp(ring); ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); if (ring->xsk_pool) { + __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, + ring->queue_index, + ixgbe_rx_napi_id(ring), 0); WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, MEM_TYPE_XSK_BUFF_POOL, NULL)); xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); } else { - xdp_rxq_info_attach_page_pool(&ring->xdp_rxq, ring->pp); + ixgbe_rx_create_pp(ring); } /* disable queue to avoid use of these values while updating state */ @@ -6131,8 +6138,10 @@ static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) { int i; - for (i = 0; i < adapter->num_rx_queues; i++) + for (i = 0; i < adapter->num_rx_queues; i++) { ixgbe_clean_rx_ring(adapter->rx_ring[i]); + ixgbe_rx_destroy_pp(adapter->rx_ring[i]); + } } /** @@ -6625,6 +6634,13 @@ static int ixgbe_setup_all_tx_resources(struct ixgbe_adapter *adapter) return err; } +static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) +{ + struct ixgbe_q_vector *q_vector = rx_ring->q_vector; + + return q_vector ? q_vector->napi.napi_id : 0; +} + static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) { struct libeth_fq fq = { @@ -6632,6 +6648,14 @@ static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) .fqes = rx_ring->rx_fqes, }; + if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq)) { + xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); + xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + } + + if (!fq.pp) + return; + libeth_rx_fq_destroy(&fq); rx_ring->rx_fqes = NULL; rx_ring->pp = NULL; @@ -6682,6 +6706,16 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) rx_ring->truesize = fq.truesize; rx_ring->rx_buf_len = fq.buf_len; + /* XDP RX-queue info */ + ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, + ixgbe_rx_napi_id(rx_ring), + rx_ring->truesize); + if (ret) + goto err; + + xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); + if (!fq.hsplit) return 0; @@ -6707,6 +6741,13 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) return ret; } +static struct device *ixgbe_dma_dev_from_ring(struct ixgbe_ring *ring) +{ + struct ixgbe_adapter *adapter = netdev_priv(ring->netdev); + + return &adapter->pdev->dev; +} + /** * ixgbe_setup_rx_resources - allocate Rx resources (Descriptors) * @adapter: pointer to ixgbe_adapter @@ -6720,15 +6761,10 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, struct device *dev = &adapter->pdev->dev; int orig_node = dev_to_node(dev); int ring_node = NUMA_NO_NODE; - int ret; if (rx_ring->q_vector) ring_node = rx_ring->q_vector->numa_node; - ret = ixgbe_rx_create_pp(rx_ring); - if (ret) - return ret; - /* Round up to nearest 4K */ rx_ring->size = rx_ring->count * sizeof(union ixgbe_adv_rx_desc); rx_ring->size = ALIGN(rx_ring->size, 4096); @@ -6742,34 +6778,16 @@ int ixgbe_setup_rx_resources(struct ixgbe_adapter *adapter, &rx_ring->dma, GFP_KERNEL); if (!rx_ring->desc) { - ret = -ENOMEM; - dev_err(&adapter->pdev->dev, - "Unable to allocate memory for the Rx descriptor ring\n"); - goto err_destroy_fq; + dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); + return -ENOMEM; } rx_ring->next_to_clean = 0; rx_ring->next_to_use = 0; - /* XDP RX-queue info */ - ret = __xdp_rxq_info_reg(&rx_ring->xdp_rxq, adapter->netdev, - rx_ring->queue_index, 0, rx_ring->truesize); - if (ret < 0) - goto err_free_desc; - - xdp_rxq_info_attach_page_pool(&rx_ring->xdp_rxq, rx_ring->pp); - rcu_assign_pointer(rx_ring->xdp_prog, adapter->xdp_prog); + WRITE_ONCE(rx_ring->xdp_prog, adapter->xdp_prog); return 0; - -err_free_desc: - dma_free_coherent(dev, rx_ring->size, - rx_ring->desc, rx_ring->dma); - rx_ring->desc = NULL; -err_destroy_fq: - dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n"); - ixgbe_rx_destroy_pp(rx_ring); - return ret; } /** @@ -6857,24 +6875,21 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { ixgbe_clean_rx_ring(rx_ring); + ixgbe_rx_destroy_pp(rx_ring); - rcu_assign_pointer(rx_ring->xdp_prog, NULL); - xdp_rxq_info_detach_mem_model(&rx_ring->xdp_rxq); - xdp_rxq_info_unreg(&rx_ring->xdp_rxq); + rx_ring->xdp_prog = NULL; /* if not set, then don't free */ if (!rx_ring->desc) return; - dma_free_coherent(rx_ring->pp->p.dev, rx_ring->size, rx_ring->desc, - rx_ring->dma); + dma_free_coherent(ixgbe_dma_dev_from_ring(rx_ring), rx_ring->size, + rx_ring->desc, rx_ring->dma); rx_ring->desc = NULL; kvfree(rx_ring->rx_xsk_buffer_info); rx_ring->rx_xsk_buffer_info = NULL; - - ixgbe_rx_destroy_pp(rx_ring); } /** From d7e23e943649c039f373330576990c8a6a0a60d7 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:06:43 +0200 Subject: [PATCH 18/25] fixup! ixgbe, libie: use libie_xg_ring in ixgbe --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 9941c48ae89a1a..956182d25cd6a7 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -4072,8 +4072,9 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, u8 reg_idx = ring->reg_idx; ixgbe_rx_destroy_pp(ring); - ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); - if (ring->xsk_pool) { + if (ixgbe_xsk_pool(adapter, ring)) { + ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); + set_ring_xsk(ring); __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->queue_index, ixgbe_rx_napi_id(ring), 0); From abad4ae9c5320477f16ddd0ed9a0943bfdac3d04 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:20:40 +0200 Subject: [PATCH 19/25] fixup! ixgbe, libie: use libie_xg_ring in ixgbe --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 7 +++++ drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 29 ++++++++++--------- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 2 +- 3 files changed, 24 insertions(+), 14 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 991c846dcd778c..d9f56f0d6fbe34 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -237,6 +237,7 @@ enum ixgbe_ring_state_t { __IXGBE_HANG_CHECK_ARMED, __IXGBE_TX_XDP_RING, __IXGBE_TX_DISABLED, + __IXGBEVF_RXTX_XSK_RING, }; struct ixgbe_fwd_adapter { @@ -265,6 +266,12 @@ struct ixgbe_fwd_adapter { set_bit(__IXGBE_TX_XDP_RING, &(ring)->state) #define clear_ring_xdp(ring) \ clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) +#define ring_is_xsk(ring) \ + test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define set_ring_xsk(ring) \ + set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) +#define clear_ring_xsk(ring) \ + clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) struct ixgbe_ring { union { struct libie_xg_ring base; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 956182d25cd6a7..336b669a3caa50 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -3168,7 +3168,7 @@ int ixgbe_poll(struct napi_struct *napi, int budget) #endif ixgbe_for_each_ring(ring, q_vector->tx) { - bool wd = ring->xsk_pool ? + bool wd = ring_is_xsk(ring) ? ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) : ixgbe_clean_tx_irq(q_vector, ring, budget); @@ -3188,7 +3188,7 @@ int ixgbe_poll(struct napi_struct *napi, int budget) per_ring_budget = budget; ixgbe_for_each_ring(ring, q_vector->rx) { - int cleaned = ring->xsk_pool ? + int cleaned = ring_is_xsk(ring) ? ixgbe_clean_rx_irq_zc(q_vector, ring, per_ring_budget) : ixgbe_clean_rx_irq(q_vector, ring, @@ -3482,15 +3482,18 @@ static void ixgbe_configure_msi_and_legacy(struct ixgbe_adapter *adapter) void ixgbe_configure_tx_ring(struct ixgbe_adapter *adapter, struct ixgbe_ring *ring) { + struct xsk_buff_pool *xsk_pool = NULL; struct ixgbe_hw *hw = &adapter->hw; - u64 tdba = ring->dma; - int wait_loop = 10; u32 txdctl = IXGBE_TXDCTL_ENABLE; u8 reg_idx = ring->reg_idx; + u64 tdba = ring->dma; + int wait_loop = 10; - ring->xsk_pool = NULL; - if (ring_is_xdp(ring)) - ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); + xsk_pool = ixgbe_xsk_pool(adapter, ring); + if (xsk_pool && ring_is_xdp(ring)) { + ring->xsk_pool = xsk_pool; + set_ring_xsk(ring); + } /* disable queue to avoid issues while updating state */ IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(reg_idx), 0); @@ -3736,7 +3739,7 @@ static void ixgbe_configure_srrctl(struct ixgbe_adapter *adapter, /* configure header buffer length, needed for RSC */ srrctl = IXGBE_RX_HDR_SIZE << IXGBE_SRRCTL_BSIZEHDRSIZE_SHIFT; - if (ring->xsk_pool) + if (ring_is_xsk(ring)) srrctl |= DIV_ROUND_UP(xsk_pool_get_rx_frame_size(ring->xsk_pool), IXGBE_SRRCTL_BSIZEPKT_STEP); else @@ -4124,7 +4127,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_RXDCTL_RLPML_EN); } #endif - if (ring->xsk_pool && hw->mac.type != ixgbe_mac_82599EB) { + if (ring_is_xsk(ring) && hw->mac.type != ixgbe_mac_82599EB) { u32 pkt_len = READ_ONCE(adapter->netdev->mtu) + LIBETH_RX_LL_LEN; @@ -4143,7 +4146,7 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(reg_idx), rxdctl); ixgbe_rx_desc_queue_enable(adapter, ring); - if (ring->xsk_pool) + if (ring_is_xsk(ring)) ixgbe_alloc_rx_buffers_zc(ring, ixgbe_desc_unused(ring)); else ixgbe_alloc_rx_buffers(ring, ixgbe_desc_unused(ring)); @@ -5260,7 +5263,7 @@ static void ixgbe_fdir_filter_restore(struct ixgbe_adapter *adapter) **/ static void ixgbe_clean_rx_ring(struct ixgbe_ring *rx_ring) { - if (rx_ring->xsk_pool) { + if (ring_is_xsk(rx_ring)) { ixgbe_xsk_clean_rx_ring(rx_ring); goto skip_free; } @@ -6069,7 +6072,7 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) u16 i = tx_ring->next_to_clean; struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; - if (tx_ring->xsk_pool) { + if (ring_is_xsk(tx_ring)) { ixgbe_xsk_clean_tx_ring(tx_ring); goto out; } @@ -10534,7 +10537,7 @@ static int ixgbe_xdp_setup(struct net_device *dev, struct bpf_prog *prog, num_queues = min_t(int, adapter->num_rx_queues, adapter->num_xdp_queues); for (int i = 0; i < num_queues; i++) - if (adapter->xdp_ring[i]->xsk_pool) + if (ring_is_xsk(adapter->xdp_ring[i])) (void)ixgbe_xsk_wakeup(adapter->netdev, i, XDP_WAKEUP_RX); xdp_features_set_redirect_target(dev, true); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index cae45570d3e36a..bc19470c0adaaa 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -522,7 +522,7 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) ring = adapter->rx_ring[qid]; - if (unlikely(!ring->xsk_pool)) + if (unlikely(!ring_is_xsk(ring))) return -EINVAL; if (!napi_if_scheduled_mark_missed(&ring->q_vector->napi)) { From d3cadbb8f6dad875d279ccf76cc315cb94649e54 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:24:12 +0200 Subject: [PATCH 20/25] fixup! ixgbe: reconfigure page pool when reallocating buffers --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 336b669a3caa50..f3d16810be3a8a 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -1751,7 +1751,7 @@ void ixgbe_alloc_rx_buffers(struct ixgbe_ring *rx_ring, u16 cleaned_count) u16 ntu = rx_ring->next_to_use; /* nothing to do */ - if (unlikely(!cleaned_count)) + if (unlikely(!cleaned_count) || unlikely(!fq.pp)) return; rx_desc = IXGBE_RX_DESC(rx_ring, ntu); @@ -4034,8 +4034,8 @@ static void ixgbe_configure_rscctl(struct ixgbe_adapter *adapter, IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(reg_idx), rscctrl); } -static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring); -static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring); +static int ixgbe_rx_create_fq(struct ixgbe_ring *rx_ring); +static void ixgbe_rx_destroy_fq(struct ixgbe_ring *rx_ring); static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring); #define IXGBE_MAX_RX_DESC_POLL 10 @@ -4073,20 +4073,14 @@ void ixgbe_configure_rx_ring(struct ixgbe_adapter *adapter, u64 rdba = ring->dma; u32 rxdctl; u8 reg_idx = ring->reg_idx; + int err; - ixgbe_rx_destroy_pp(ring); - if (ixgbe_xsk_pool(adapter, ring)) { - ring->xsk_pool = ixgbe_xsk_pool(adapter, ring); - set_ring_xsk(ring); - __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, - ring->queue_index, - ixgbe_rx_napi_id(ring), 0); - WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq, - MEM_TYPE_XSK_BUFF_POOL, - NULL)); - xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq); - } else { - ixgbe_rx_create_pp(ring); + err = ixgbe_rx_create_fq(ring); + if (err) { + netdev_err(ring->netdev, + "Failed to create Page Pool for buffer allocation: (%pe), RxQ %d is disabled, driver reload may be needed\n", + ERR_PTR(err), ring->queue_index); + return; } /* disable queue to avoid use of these values while updating state */ @@ -6144,7 +6138,7 @@ static void ixgbe_clean_all_rx_rings(struct ixgbe_adapter *adapter) for (i = 0; i < adapter->num_rx_queues; i++) { ixgbe_clean_rx_ring(adapter->rx_ring[i]); - ixgbe_rx_destroy_pp(adapter->rx_ring[i]); + ixgbe_rx_destroy_fq(adapter->rx_ring[i]); } } @@ -6645,7 +6639,7 @@ static int ixgbe_rx_napi_id(struct ixgbe_ring *rx_ring) return q_vector ? q_vector->napi.napi_id : 0; } -static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) +static void ixgbe_rx_destroy_fq(struct ixgbe_ring *rx_ring) { struct libeth_fq fq = { .pp = rx_ring->pp, @@ -6657,6 +6651,9 @@ static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) xdp_rxq_info_unreg(&rx_ring->xdp_rxq); } + if (ring_is_xsk(rx_ring)) + return; + if (!fq.pp) return; @@ -6677,9 +6674,10 @@ static void ixgbe_rx_destroy_pp(struct ixgbe_ring *rx_ring) rx_ring->hdr_pp = NULL; } -static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) +static int ixgbe_rx_create_fq(struct ixgbe_ring *rx_ring) { struct ixgbe_adapter *adapter = netdev_priv(rx_ring->netdev); + struct xsk_buff_pool *xsk_pool; struct libeth_fq fq = { .count = rx_ring->count, .nid = NUMA_NO_NODE, @@ -6692,6 +6690,24 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) u32 frame_size; int ret; + xsk_pool = ixgbe_xsk_pool(adapter, rx_ring); + if (xsk_pool) { + rx_ring->xsk_pool = xsk_pool; + set_ring_xsk(rx_ring); + __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, + rx_ring->queue_index, + ixgbe_rx_napi_id(rx_ring), 0); + ret = xdp_rxq_info_reg_mem_model(&rx_ring->xdp_rxq, + MEM_TYPE_XSK_BUFF_POOL, + NULL); + if (ret) + return ret; + + xsk_pool_set_rxq_info(rx_ring->xsk_pool, &rx_ring->xdp_rxq); + + return 0; + } + /* Some HW requires DMA write sizes to be aligned to 1K, * which warrants fake header split usage, but this is * not an issue if the frame size is at its maximum of 3K @@ -6741,7 +6757,7 @@ static int ixgbe_rx_create_pp(struct ixgbe_ring *rx_ring) return 0; err: - ixgbe_rx_destroy_pp(rx_ring); + ixgbe_rx_destroy_fq(rx_ring); return ret; } @@ -6879,7 +6895,7 @@ static void ixgbe_free_all_tx_resources(struct ixgbe_adapter *adapter) void ixgbe_free_rx_resources(struct ixgbe_ring *rx_ring) { ixgbe_clean_rx_ring(rx_ring); - ixgbe_rx_destroy_pp(rx_ring); + ixgbe_rx_destroy_fq(rx_ring); rx_ring->xdp_prog = NULL; @@ -10837,6 +10853,7 @@ void ixgbe_txrx_ring_disable(struct ixgbe_adapter *adapter, int ring) if (xdp_ring) ixgbe_clean_tx_ring(xdp_ring); ixgbe_clean_rx_ring(rx_ring); + ixgbe_rx_destroy_fq(rx_ring); ixgbe_reset_txr_stats(tx_ring); if (xdp_ring) From dc4e8c4c7e6e1bc4cf0cc26166a4d753c71c171a Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:24:42 +0200 Subject: [PATCH 21/25] ixgbevf TODO --- drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c index e5bf50a82268a8..494449e844e71b 100644 --- a/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c +++ b/drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c @@ -1422,6 +1422,7 @@ void ixgbevf_rx_destroy_pp(struct ixgbevf_ring *rx_ring) rx_ring->hdr_pp = NULL; } +// TODO : rename to create_fq static int ixgbevf_rx_create_pp(struct ixgbevf_ring *rx_ring) { u32 adapter_flags = rx_ring->q_vector->adapter->flags; From 4a7d55059334e9802f09002f2faae0903e4803b5 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 14:45:34 +0200 Subject: [PATCH 22/25] fixup! ixgbe, libie: use libie_xg_ring in ixgbe --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index d9f56f0d6fbe34..0ad93645aa1cfc 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -237,7 +237,7 @@ enum ixgbe_ring_state_t { __IXGBE_HANG_CHECK_ARMED, __IXGBE_TX_XDP_RING, __IXGBE_TX_DISABLED, - __IXGBEVF_RXTX_XSK_RING, + __IXGBE_RXTX_XSK_RING, }; struct ixgbe_fwd_adapter { @@ -267,11 +267,11 @@ struct ixgbe_fwd_adapter { #define clear_ring_xdp(ring) \ clear_bit(__IXGBE_TX_XDP_RING, &(ring)->state) #define ring_is_xsk(ring) \ - test_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + test_bit(__IXGBE_RXTX_XSK_RING, &(ring)->state) #define set_ring_xsk(ring) \ - set_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + set_bit(__IXGBE_RXTX_XSK_RING, &(ring)->state) #define clear_ring_xsk(ring) \ - clear_bit(__IXGBEVF_RXTX_XSK_RING, &(ring)->state) + clear_bit(__IXGBE_RXTX_XSK_RING, &(ring)->state) struct ixgbe_ring { union { struct libie_xg_ring base; From ddfe684c37300d37b82a69c78053201b2a9390bf Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 17:32:17 +0200 Subject: [PATCH 23/25] [SQUASH SMWR] enable ZC frags in features --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index f3d16810be3a8a..094523b90004fa 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -11404,9 +11404,7 @@ static int ixgbe_probe(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->priv_flags |= IFF_UNICAST_FLT; netdev->priv_flags |= IFF_SUPP_NOFCS; - netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | - NETDEV_XDP_ACT_XSK_ZEROCOPY | - NETDEV_XDP_ACT_RX_SG; + libeth_xdp_set_features(netdev, NULL, MAX_SKB_FRAGS); /* MTU range: 68 - 9710 */ netdev->min_mtu = ETH_MIN_MTU; From 0189623eaee87956cf2f9b7f2b9d29d553012b2c Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 17:33:31 +0200 Subject: [PATCH 24/25] ixgbe: support AF_XDP multi-buffer via libeth and libie_xg Utilize libeth and libie_xg libraries to support segments in AF_XDP ZC mode. Since Tx path supports frags, it does not need updating for now. Signed-off-by: Larysa Zaremba --- drivers/net/ethernet/intel/ixgbe/ixgbe.h | 6 +- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 29 ++- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 186 ++++-------------- 3 files changed, 63 insertions(+), 158 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe.h b/drivers/net/ethernet/intel/ixgbe/ixgbe.h index 0ad93645aa1cfc..7d784662f494e8 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe.h @@ -199,11 +199,6 @@ struct vf_macvlans { #define TXD_USE_COUNT(S) DIV_ROUND_UP((S), IXGBE_MAX_DATA_PER_TXD) #define DESC_NEEDED (MAX_SKB_FRAGS + 4) -struct ixgbe_xsk_rx_buffer { - bool discard; - struct xdp_buff *xdp; -}; - struct ixgbe_queue_stats { u64 packets; u64 bytes; @@ -317,6 +312,7 @@ struct ixgbe_ring { u32 rx_buf_len; struct libeth_xdp_buff_stash xdp_stash; struct ixgbe_xsk_rx_buffer *rx_xsk_buffer_info; + u32 thresh; } ____cacheline_internodealigned_in_smp; enum ixgbe_ring_f_enum { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 094523b90004fa..e977c2218194ca 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -31,7 +31,7 @@ #include #include #include -#include +#include #include #include #include @@ -6651,8 +6651,19 @@ static void ixgbe_rx_destroy_fq(struct ixgbe_ring *rx_ring) xdp_rxq_info_unreg(&rx_ring->xdp_rxq); } - if (ring_is_xsk(rx_ring)) - return; + // TODO : previous code was not clearing xsk bit + if (test_and_clear_bit(__IXGBE_RXTX_XSK_RING, &rx_ring->state)) { + struct libeth_xskfq xskfq = { + .fqes = rx_ring->xsk_fqes, + }; + + libeth_xskfq_destroy(&xskfq); + rx_ring->xsk_fqes = NULL; + rx_ring->pending = xskfq.pending; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; + rx_ring->xsk_pool = NULL; + } if (!fq.pp) return; @@ -6692,8 +6703,20 @@ static int ixgbe_rx_create_fq(struct ixgbe_ring *rx_ring) xsk_pool = ixgbe_xsk_pool(adapter, rx_ring); if (xsk_pool) { + struct libeth_xskfq xskfq = { + .nid = numa_node_id(), + .count = rx_ring->count, + .pool = xsk_pool, + }; + + libeth_xskfq_create(&xskfq); rx_ring->xsk_pool = xsk_pool; + rx_ring->xsk_fqes = xskfq.fqes; + rx_ring->pending = xskfq.count - 1; + rx_ring->thresh = xskfq.thresh; + rx_ring->rx_buf_len = xskfq.buf_len; set_ring_xsk(rx_ring); + __xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev, rx_ring->queue_index, ixgbe_rx_napi_id(rx_ring), 0); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index bc19470c0adaaa..526610c76d83d9 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -2,6 +2,7 @@ /* Copyright(c) 2018 Intel Corporation. */ #include +#include #include #include @@ -150,98 +151,24 @@ static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { - union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_xsk_rx_buffer *bi; - u16 i = rx_ring->next_to_use; - dma_addr_t dma; - bool ok = true; - - /* nothing to do */ - if (!count) - return true; - - rx_desc = IXGBE_RX_DESC(rx_ring, i); - bi = &rx_ring->rx_xsk_buffer_info[i]; - i -= rx_ring->count; - - do { - bi->xdp = xsk_buff_alloc(rx_ring->xsk_pool); - if (!bi->xdp) { - ok = false; - break; - } - - dma = xsk_buff_xdp_get_dma(bi->xdp); - - /* Refresh the desc even if buffer_addrs didn't change - * because each write-back erases this info. - */ - rx_desc->read.pkt_addr = cpu_to_le64(dma); - - rx_desc++; - bi++; - i++; - if (unlikely(!i)) { - rx_desc = IXGBE_RX_DESC(rx_ring, 0); - bi = rx_ring->rx_xsk_buffer_info; - i -= rx_ring->count; - } - - /* clear the length for the next_to_use descriptor */ - rx_desc->wb.upper.length = 0; - - count--; - } while (count); - - i += rx_ring->count; - - if (rx_ring->next_to_use != i) { - rx_ring->next_to_use = i; - - /* Force memory writes to complete before letting h/w - * know there are new descriptors to fetch. (Only - * applicable for weak-ordered memory model archs, - * such as IA-64). - */ - wmb(); - writel(i, rx_ring->tail); - } - - return ok; -} - -static struct sk_buff *ixgbe_construct_skb_zc(struct ixgbe_ring *rx_ring, - const struct xdp_buff *xdp) -{ - unsigned int totalsize = xdp->data_end - xdp->data_meta; - unsigned int metasize = xdp->data - xdp->data_meta; - struct sk_buff *skb; - - net_prefetch(xdp->data_meta); - - /* allocate a skb to store the frags */ - skb = napi_alloc_skb(&rx_ring->q_vector->napi, totalsize); - if (unlikely(!skb)) - return NULL; - - memcpy(__skb_put(skb, totalsize), xdp->data_meta, - ALIGN(totalsize, sizeof(long))); - - if (metasize) { - skb_metadata_set(skb, metasize); - __skb_pull(skb, metasize); - } - - return skb; + return libie_xg_xsk_alloc_rx_bufs(&rx_ring->base, count); } -static void ixgbe_inc_ntc(struct ixgbe_ring *rx_ring) +static bool ixgbe_xsk_is_non_eop(struct ixgbe_ring *rx_ring, + union ixgbe_adv_rx_desc *rx_desc) { u32 ntc = rx_ring->next_to_clean + 1; ntc = (ntc < rx_ring->count) ? ntc : 0; rx_ring->next_to_clean = ntc; + rx_ring->pending++; + prefetch(IXGBE_RX_DESC(rx_ring, ntc)); + + if (likely(ixgbe_test_staterr(rx_desc, IXGBE_RXD_STAT_EOP))) + return false; + + return true; } int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, @@ -250,27 +177,19 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, { unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; - u16 cleaned_count = ixgbe_desc_unused(rx_ring); unsigned int xdp_res, xdp_xmit = 0; + struct libeth_xdp_buff *head_xdp; bool failure = false; struct sk_buff *skb; while (likely(total_rx_packets < budget)) { + struct libeth_xdp_buff *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; - struct ixgbe_xsk_rx_buffer *bi; unsigned int size; - /* return some buffers to hardware, one at a time is too slow */ - if (cleaned_count >= IXGBE_RX_BUFFER_WRITE) { - failure = failure || - !ixgbe_alloc_rx_buffers_zc(rx_ring, - cleaned_count); - cleaned_count = 0; - } - rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); - if (!size) + if (unlikely(!size)) break; /* This memory barrier is needed to keep us from reading @@ -279,77 +198,55 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, */ dma_rmb(); - bi = &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; - - if (unlikely(!ixgbe_test_staterr(rx_desc, - IXGBE_RXD_STAT_EOP))) { - struct ixgbe_xsk_rx_buffer *next_bi; - - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - ixgbe_inc_ntc(rx_ring); - next_bi = - &rx_ring->rx_xsk_buffer_info[rx_ring->next_to_clean]; - next_bi->discard = true; + rx_buffer = rx_ring->xsk_fqes[rx_ring->next_to_clean]; + head_xdp = libeth_xsk_process_buff(head_xdp, rx_buffer, size); + if (ixgbe_xsk_is_non_eop(rx_ring, rx_desc) || + unlikely(!head_xdp)) continue; - } - if (unlikely(bi->discard)) { - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - bi->discard = false; - ixgbe_inc_ntc(rx_ring); - continue; - } - - bi->xdp->data_end = bi->xdp->data + size; - xsk_buff_dma_sync_for_cpu(bi->xdp); - xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, bi->xdp); + total_rx_packets++; + total_rx_bytes += xdp_get_buff_len(&head_xdp->base); + xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, &head_xdp->base); if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) { xdp_xmit |= xdp_res; } else if (xdp_res == IXGBE_XDP_EXIT) { failure = true; break; } else if (xdp_res == IXGBE_XDP_CONSUMED) { - xsk_buff_free(bi->xdp); + libeth_xdp_return_buff(head_xdp); } else if (xdp_res == IXGBE_XDP_PASS) { goto construct_skb; } - bi->xdp = NULL; - total_rx_packets++; - total_rx_bytes += size; + head_xdp = NULL; - cleaned_count++; - ixgbe_inc_ntc(rx_ring); continue; construct_skb: /* XDP_PASS path */ - skb = ixgbe_construct_skb_zc(rx_ring, bi->xdp); - if (!skb) { + skb = xdp_build_skb_from_zc(&head_xdp->base); + if (unlikely(!skb)) { + libeth_xdp_return_buff_slow(head_xdp); + head_xdp = NULL; rx_ring->rx_stats.alloc_rx_buff_failed++; break; } - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - - cleaned_count++; - ixgbe_inc_ntc(rx_ring); - - if (eth_skb_pad(skb)) - continue; - - total_rx_bytes += skb->len; - total_rx_packets++; + head_xdp = NULL; skb->protocol = eth_type_trans(skb, rx_ring->netdev); ixgbe_process_skb_fields(rx_ring, rx_desc, skb); ixgbe_rx_skb(q_vector, skb); } + if (rx_ring->pending >= rx_ring->thresh) + failure |= !ixgbe_alloc_rx_buffers_zc(rx_ring, + rx_ring->pending); + + /* place incomplete frames back on ring for completion */ + rx_ring->xsk_xdp_head = head_xdp; + if (xdp_xmit & IXGBE_XDP_REDIR) xdp_do_flush(); @@ -375,18 +272,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) { - struct ixgbe_xsk_rx_buffer *bi; - u16 i; - - for (i = 0; i < rx_ring->count; i++) { - bi = &rx_ring->rx_xsk_buffer_info[i]; - - if (!bi->xdp) - continue; - - xsk_buff_free(bi->xdp); - bi->xdp = NULL; - } + return libie_xg_rx_xsk_ring_free_buffs(&rx_ring->base); } static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) From 510fcde47637379e3a9c189410967bc3323a7405 Mon Sep 17 00:00:00 2001 From: Larysa Zaremba Date: Wed, 19 Aug 2026 11:31:44 +0200 Subject: [PATCH 25/25] [REWORD] ixgbe: refactor .ndo_xdp_xmit and XDP_TX (also ZC Rx) --- drivers/net/ethernet/intel/ixgbe/ixgbe_main.c | 250 ++++-------------- .../ethernet/intel/ixgbe/ixgbe_txrx_common.h | 5 +- drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c | 245 ++--------------- 3 files changed, 85 insertions(+), 415 deletions(-) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index e977c2218194ca..e64e7c1619ebb8 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -1386,10 +1387,7 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, total_ipsec++; /* free the skb */ - if (ring_is_xdp(tx_ring)) - xdp_return_frame(tx_buffer->xdpf); - else - napi_consume_skb(tx_buffer->skb, napi_budget); + napi_consume_skb(tx_buffer->skb, napi_budget); /* unmap skb header data */ dma_unmap_single(tx_ring->dev, @@ -1444,9 +1442,6 @@ static bool ixgbe_clean_tx_irq(struct ixgbe_q_vector *q_vector, total_bytes); adapter->tx_ipsec += total_ipsec; - if (ring_is_xdp(tx_ring)) - return !!budget; - if (check_for_tx_hang(tx_ring) && ixgbe_check_tx_hang(tx_ring)) { if (adapter->hw.mac.type == ixgbe_mac_e610) ixgbe_handle_mdd_event(adapter, tx_ring); @@ -1975,60 +1970,30 @@ bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, return false; } -static int ixgbe_run_xdp(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring, - struct libeth_xdp_buff *xdp) +u32 ixgbe_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq) { - int err, result = IXGBE_XDP_PASS; - struct bpf_prog *xdp_prog; - struct ixgbe_ring *ring; - struct xdp_frame *xdpf; - u32 act; - - xdp_prog = READ_ONCE(rx_ring->xdp_prog); - - if (!xdp_prog) - goto xdp_out; - - prefetchw(xdp->base.data_hard_start); /* xdp_frame write */ - - act = bpf_prog_run_xdp(xdp_prog, &xdp->base); - switch (act) { - case XDP_PASS: - break; - case XDP_TX: - xdpf = xdp_convert_buff_to_frame(&xdp->base); - if (unlikely(!xdpf)) - goto out_failure; - ring = ixgbe_determine_xdp_ring(adapter); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_lock(&ring->tx_lock); - result = ixgbe_xmit_xdp_ring(ring, xdpf); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_unlock(&ring->tx_lock); - if (result == IXGBE_XDP_CONSUMED) - goto out_failure; - break; - case XDP_REDIRECT: - err = xdp_do_redirect(adapter->netdev, &xdp->base, xdp_prog); - if (err) - goto out_failure; - result = IXGBE_XDP_REDIR; - break; - default: - bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); - fallthrough; - case XDP_ABORTED: -out_failure: - trace_xdp_exception(rx_ring->netdev, xdp_prog, act); - fallthrough; /* handle aborts by dropping packet */ - case XDP_DROP: - result = IXGBE_XDP_CONSUMED; - libeth_xdp_return_buff(xdp); - break; + struct ixgbe_ring *xdp_ring = xdpsq; + + libeth_xdpsq_lock(&xdp_ring->xdpq_lock); + if (unlikely(ixgbe_desc_unused(xdp_ring) < xdp_ring->thresh)) { + u16 to_clean = libie_xg_tx_get_num_sent(xdpsq); + + if (likely(to_clean)) + libie_xg_clean_xdp_num(&xdp_ring->base, true, to_clean, + false); } -xdp_out: - return result; + + *sq = (struct libeth_xdpsq) { + .count = xdp_ring->count, + .descs = xdp_ring->desc, + .lock = &xdp_ring->xdpq_lock, + .ntu = &xdp_ring->next_to_use, + .pending = &xdp_ring->pending, + .pool = xdp_ring->xsk_pool, + .sqes = xdp_ring->xdp_sqes, + }; + + return ixgbe_desc_unused(xdp_ring); } static u32 ixgbe_rx_hsplit_wa(const struct libeth_fqe *hdr, @@ -2057,6 +2022,16 @@ static u32 ixgbe_rx_hsplit_wa(const struct libeth_fqe *hdr, return copy; } +LIBETH_XDP_DEFINE_START(); +LIBETH_XDP_DEFINE_FLUSH_TX(static ixgbe_xdp_flush_tx, ixgbe_prep_xdp_sq, + libie_xg_xdp_xmit_desc); +LIBETH_XDP_DEFINE_FLUSH_XMIT(static ixgbe_xdp_flush_xmit, ixgbe_prep_xdp_sq, + libie_xg_xdp_xmit_desc); +LIBETH_XDP_DEFINE_RUN_PROG(static ixgbe_xdp_run_prog, ixgbe_xdp_flush_tx); +LIBETH_XDP_DEFINE_FINALIZE(static ixgbe_xdp_finalize_xdp_napi, + ixgbe_xdp_flush_tx, libie_xg_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); + /** * ixgbe_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf * @q_vector: structure containing interrupt and ring information @@ -2081,12 +2056,14 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unsigned int mss = 0; #endif /* IXGBE_FCOE */ u16 cleaned_count = ixgbe_desc_unused(rx_ring); + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); LIBETH_XDP_ONSTACK_BUFF(xdp); - unsigned int xdp_xmit = 0; u32 rsc_append_cnt = 0; - int xdp_res = 0; libeth_xdp_init_buff(xdp, &rx_ring->xdp_stash, &rx_ring->xdp_rxq); + libeth_xdp_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + rx_ring->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); while (likely(total_rx_packets < budget)) { union ixgbe_adv_rx_desc *rx_desc; @@ -2137,17 +2114,8 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, unlikely(!xdp->data)) continue; - xdp_res = ixgbe_run_xdp(adapter, rx_ring, xdp); - if (xdp_res) { - if (xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR)) - xdp_xmit |= xdp_res; - - total_rx_packets++; - total_rx_bytes += xdp_get_buff_len(&xdp->base); - rsc_append_cnt = 0; - xdp->data = NULL; - continue; - } + if (xdp_tx_bulk.prog && !ixgbe_xdp_run_prog(xdp, &xdp_tx_bulk)) + continue; skb = xdp_build_skb_from_buff(&xdp->base); @@ -2209,14 +2177,7 @@ static int ixgbe_clean_rx_irq(struct ixgbe_q_vector *q_vector, /* place incomplete frames back on ring for completion */ libeth_xdp_save_buff(&rx_ring->xdp_stash, xdp); - if (xdp_xmit & IXGBE_XDP_REDIR) - xdp_do_flush(); - - if (xdp_xmit & IXGBE_XDP_TX) { - struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter); - - ixgbe_xdp_ring_update_tail_locked(ring); - } + ixgbe_xdp_finalize_xdp_napi(&xdp_tx_bulk); ixgbe_update_rx_ring_stats(rx_ring, q_vector, total_rx_packets, total_rx_bytes); @@ -3170,6 +3131,7 @@ int ixgbe_poll(struct napi_struct *napi, int budget) ixgbe_for_each_ring(ring, q_vector->tx) { bool wd = ring_is_xsk(ring) ? ixgbe_clean_xdp_tx_irq(q_vector, ring, budget) : + ring_is_xdp(ring) ? true : ixgbe_clean_tx_irq(q_vector, ring, budget); if (!wd) @@ -6057,6 +6019,13 @@ void ixgbe_reset(struct ixgbe_adapter *adapter) } } +static void ixgbe_clean_xdp_ring(struct ixgbe_ring *xdp_ring) +{ + libie_xg_clean_xdp_num(&xdp_ring->base, false, xdp_ring->pending, + !!xdp_ring->xsk_pool); + libeth_xdpsq_put(&xdp_ring->xdpq_lock, xdp_ring->netdev); +} + /** * ixgbe_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned @@ -6066,8 +6035,8 @@ static void ixgbe_clean_tx_ring(struct ixgbe_ring *tx_ring) u16 i = tx_ring->next_to_clean; struct libie_xg_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i]; - if (ring_is_xsk(tx_ring)) { - ixgbe_xsk_clean_tx_ring(tx_ring); + if (ring_is_xdp(tx_ring)) { + ixgbe_clean_xdp_ring(tx_ring); goto out; } @@ -8903,87 +8872,6 @@ static u16 ixgbe_select_queue(struct net_device *dev, struct sk_buff *skb, } #endif -int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, - struct xdp_frame *xdpf) -{ - struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); - u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; - u16 i = 0, index = ring->next_to_use; - struct libie_xg_tx_buffer *tx_head = &ring->tx_buffer_info[index]; - struct libie_xg_tx_buffer *tx_buff = tx_head; - union ixgbe_adv_tx_desc *tx_desc = IXGBE_TX_DESC(ring, index); - u32 cmd_type, len = xdpf->len; - void *data = xdpf->data; - - if (unlikely(ixgbe_desc_unused(ring) < 1 + nr_frags)) - return IXGBE_XDP_CONSUMED; - - tx_head->bytecount = xdp_get_frame_len(xdpf); - tx_head->gso_segs = 1; - tx_head->xdpf = xdpf; - - tx_desc->read.olinfo_status = - cpu_to_le32(tx_head->bytecount << IXGBE_ADVTXD_PAYLEN_SHIFT); - - for (;;) { - dma_addr_t dma; - - dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE); - if (dma_mapping_error(ring->dev, dma)) - goto unmap; - - dma_unmap_len_set(tx_buff, len, len); - dma_unmap_addr_set(tx_buff, dma, dma); - - cmd_type = IXGBE_ADVTXD_DTYP_DATA | IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS | len; - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - tx_desc->read.buffer_addr = cpu_to_le64(dma); - tx_buff->protocol = 0; - - if (++index == ring->count) - index = 0; - - if (i == nr_frags) - break; - - tx_buff = &ring->tx_buffer_info[index]; - tx_desc = IXGBE_TX_DESC(ring, index); - tx_desc->read.olinfo_status = 0; - - data = skb_frag_address(&sinfo->frags[i]); - len = skb_frag_size(&sinfo->frags[i]); - i++; - } - /* put descriptor type bits */ - tx_desc->read.cmd_type_len |= cpu_to_le32(IXGBE_TXD_CMD); - - /* Avoid any potential race with xdp_xmit and cleanup */ - smp_wmb(); - - tx_head->next_to_watch = tx_desc; - ring->next_to_use = index; - - return IXGBE_XDP_TX; - -unmap: - for (;;) { - tx_buff = &ring->tx_buffer_info[index]; - if (dma_unmap_len(tx_buff, len)) - dma_unmap_page(ring->dev, dma_unmap_addr(tx_buff, dma), - dma_unmap_len(tx_buff, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buff, len, 0); - if (tx_buff == tx_head) - break; - - if (!index) - index += ring->count; - index--; - } - - return IXGBE_XDP_CONSUMED; -} netdev_tx_t ixgbe_xmit_frame_ring(struct sk_buff *skb, struct ixgbe_adapter *adapter, @@ -10623,9 +10511,6 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames, u32 flags) { struct ixgbe_adapter *adapter = ixgbe_from_netdev(dev); - struct ixgbe_ring *ring; - int nxmit = 0; - int i; if (unlikely(test_bit(__IXGBE_DOWN, &adapter->state))) return -ENETDOWN; @@ -10637,36 +10522,13 @@ static int ixgbe_xdp_xmit(struct net_device *dev, int n, if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) return -EINVAL; - /* During program transitions its possible adapter->xdp_prog is assigned - * but ring has not been configured yet. In this case simply abort xmit. - */ - ring = adapter->xdp_prog ? ixgbe_determine_xdp_ring(adapter) : NULL; - if (unlikely(!ring)) + if (unlikely(!adapter->num_xdp_queues)) return -ENXIO; - if (unlikely(test_bit(__IXGBE_TX_DISABLED, &ring->state))) - return -ENXIO; - - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_lock(&ring->tx_lock); - - for (i = 0; i < n; i++) { - struct xdp_frame *xdpf = frames[i]; - int err; - - err = ixgbe_xmit_xdp_ring(ring, xdpf); - if (err != IXGBE_XDP_TX) - break; - nxmit++; - } - - if (unlikely(flags & XDP_XMIT_FLUSH)) - ixgbe_xdp_ring_update_tail(ring); - - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_unlock(&ring->tx_lock); - - return nxmit; + return libeth_xdp_xmit_do_bulk(dev, n, frames, flags, adapter->xdp_ring, + adapter->num_xdp_queues, + ixgbe_xdp_flush_xmit, + libie_xg_xdp_rs_and_bump); } static const struct net_device_ops ixgbe_netdev_ops = { diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h index f681f95377980c..85eb4e442d9482 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_txrx_common.h @@ -10,8 +10,6 @@ #define IXGBE_XDP_REDIR BIT(2) #define IXGBE_XDP_EXIT BIT(3) -int ixgbe_xmit_xdp_ring(struct ixgbe_ring *ring, - struct xdp_frame *xdpf); bool ixgbe_cleanup_headers(struct ixgbe_ring *rx_ring, union ixgbe_adv_rx_desc *rx_desc, struct sk_buff *skb); @@ -41,7 +39,6 @@ void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring); bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring, int napi_budget); int ixgbe_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags); -void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring); void ixgbe_update_tx_ring_stats(struct ixgbe_ring *tx_ring, struct ixgbe_q_vector *q_vector, u64 pkts, @@ -50,4 +47,6 @@ void ixgbe_update_rx_ring_stats(struct ixgbe_ring *rx_ring, struct ixgbe_q_vector *q_vector, u64 pkts, u64 bytes); +u32 ixgbe_prep_xdp_sq(void *xdpsq, struct libeth_xdpsq *sq); + #endif /* #define _IXGBE_TXRX_COMMON_H_ */ diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c index 526610c76d83d9..902540624e6282 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_xsk.c @@ -95,60 +95,6 @@ int ixgbe_xsk_pool_setup(struct ixgbe_adapter *adapter, ixgbe_xsk_pool_disable(adapter, qid); } -static int ixgbe_run_xdp_zc(struct ixgbe_adapter *adapter, - struct ixgbe_ring *rx_ring, - struct xdp_buff *xdp) -{ - int err, result = IXGBE_XDP_PASS; - struct bpf_prog *xdp_prog; - struct ixgbe_ring *ring; - struct xdp_frame *xdpf; - u32 act; - - xdp_prog = READ_ONCE(rx_ring->xdp_prog); - act = bpf_prog_run_xdp(xdp_prog, xdp); - - if (likely(act == XDP_REDIRECT)) { - err = xdp_do_redirect(rx_ring->netdev, xdp, xdp_prog); - if (!err) - return IXGBE_XDP_REDIR; - if (xsk_uses_need_wakeup(rx_ring->xsk_pool) && err == -ENOBUFS) - result = IXGBE_XDP_EXIT; - else - result = IXGBE_XDP_CONSUMED; - goto out_failure; - } - - switch (act) { - case XDP_PASS: - break; - case XDP_TX: - xdpf = xdp_convert_buff_to_frame(xdp); - if (unlikely(!xdpf)) - goto out_failure; - ring = ixgbe_determine_xdp_ring(adapter); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_lock(&ring->tx_lock); - result = ixgbe_xmit_xdp_ring(ring, xdpf); - if (static_branch_unlikely(&ixgbe_xdp_locking_key)) - spin_unlock(&ring->tx_lock); - if (result == IXGBE_XDP_CONSUMED) - goto out_failure; - break; - case XDP_DROP: - result = IXGBE_XDP_CONSUMED; - break; - default: - bpf_warn_invalid_xdp_action(rx_ring->netdev, xdp_prog, act); - fallthrough; - case XDP_ABORTED: - result = IXGBE_XDP_CONSUMED; -out_failure: - trace_xdp_exception(rx_ring->netdev, xdp_prog, act); - } - return result; -} - bool ixgbe_alloc_rx_buffers_zc(struct ixgbe_ring *rx_ring, u16 count) { return libie_xg_xsk_alloc_rx_bufs(&rx_ring->base, count); @@ -171,21 +117,35 @@ static bool ixgbe_xsk_is_non_eop(struct ixgbe_ring *rx_ring, return true; } +LIBETH_XDP_DEFINE_START(); +LIBETH_XSK_DEFINE_FLUSH_TX(static ixgbe_xsk_flush_tx, ixgbe_prep_xdp_sq, + libie_xg_xsk_xmit_desc); +LIBETH_XSK_DEFINE_RUN_PROG(static ixgbe_xsk_run_prog, ixgbe_xsk_flush_tx); +LIBETH_XSK_DEFINE_FINALIZE(static ixgbe_xsk_finalize_xdp_napi, + ixgbe_xsk_flush_tx, libie_xg_xdp_rs_and_bump); +LIBETH_XDP_DEFINE_END(); + int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *rx_ring, const int budget) { unsigned int total_rx_bytes = 0, total_rx_packets = 0; struct ixgbe_adapter *adapter = q_vector->adapter; - unsigned int xdp_res, xdp_xmit = 0; + LIBETH_XDP_ONSTACK_BULK(xdp_tx_bulk); struct libeth_xdp_buff *head_xdp; bool failure = false; struct sk_buff *skb; + head_xdp = rx_ring->xsk_xdp_head; + libeth_xsk_tx_init_bulk(&xdp_tx_bulk, rx_ring->xdp_prog, + adapter->netdev, adapter->xdp_ring, + adapter->num_xdp_queues); + while (likely(total_rx_packets < budget)) { struct libeth_xdp_buff *rx_buffer; union ixgbe_adv_rx_desc *rx_desc; unsigned int size; + u32 xdp_result; rx_desc = IXGBE_RX_DESC(rx_ring, rx_ring->next_to_clean); size = le16_to_cpu(rx_desc->wb.upper.length); @@ -207,26 +167,17 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, total_rx_packets++; total_rx_bytes += xdp_get_buff_len(&head_xdp->base); - xdp_res = ixgbe_run_xdp_zc(adapter, rx_ring, &head_xdp->base); - if (likely(xdp_res & (IXGBE_XDP_TX | IXGBE_XDP_REDIR))) { - xdp_xmit |= xdp_res; - } else if (xdp_res == IXGBE_XDP_EXIT) { + xdp_result = ixgbe_xsk_run_prog(head_xdp, &xdp_tx_bulk); + if (xdp_result) { + head_xdp = NULL; + if (likely(xdp_result != LIBETH_XDP_ABORTED)) + continue; failure = true; break; - } else if (xdp_res == IXGBE_XDP_CONSUMED) { - libeth_xdp_return_buff(head_xdp); - } else if (xdp_res == IXGBE_XDP_PASS) { - goto construct_skb; } - head_xdp = NULL; - - continue; - -construct_skb: - /* XDP_PASS path */ skb = xdp_build_skb_from_zc(&head_xdp->base); - if (unlikely(!skb)) { + if (!skb) { libeth_xdp_return_buff_slow(head_xdp); head_xdp = NULL; rx_ring->rx_stats.alloc_rx_buff_failed++; @@ -247,14 +198,7 @@ int ixgbe_clean_rx_irq_zc(struct ixgbe_q_vector *q_vector, /* place incomplete frames back on ring for completion */ rx_ring->xsk_xdp_head = head_xdp; - if (xdp_xmit & IXGBE_XDP_REDIR) - xdp_do_flush(); - - if (xdp_xmit & IXGBE_XDP_TX) { - struct ixgbe_ring *ring = ixgbe_determine_xdp_ring(adapter); - - ixgbe_xdp_ring_update_tail_locked(ring); - } + ixgbe_xsk_finalize_xdp_napi(&xdp_tx_bulk); ixgbe_update_rx_ring_stats(rx_ring, q_vector, total_rx_packets, total_rx_bytes); @@ -275,124 +219,15 @@ void ixgbe_xsk_clean_rx_ring(struct ixgbe_ring *rx_ring) return libie_xg_rx_xsk_ring_free_buffs(&rx_ring->base); } -static bool ixgbe_xmit_zc(struct ixgbe_ring *xdp_ring, unsigned int budget) -{ - struct xsk_buff_pool *pool = xdp_ring->xsk_pool; - union ixgbe_adv_tx_desc *tx_desc = NULL; - struct libie_xg_tx_buffer *tx_bi; - bool work_done = true; - struct xdp_desc desc; - dma_addr_t dma; - u32 cmd_type; - - while (likely(budget)) { - if (unlikely(!ixgbe_desc_unused(xdp_ring))) { - work_done = false; - break; - } - - if (!netif_carrier_ok(xdp_ring->netdev)) - break; - - if (!xsk_tx_peek_desc(pool, &desc)) - break; - - dma = xsk_buff_raw_get_dma(pool, desc.addr); - xsk_buff_raw_dma_sync_for_device(pool, dma, desc.len); - - tx_bi = &xdp_ring->tx_buffer_info[xdp_ring->next_to_use]; - tx_bi->bytecount = desc.len; - tx_bi->xdpf = NULL; - tx_bi->gso_segs = 1; - - tx_desc = IXGBE_TX_DESC(xdp_ring, xdp_ring->next_to_use); - tx_desc->read.buffer_addr = cpu_to_le64(dma); - - /* put descriptor type bits */ - cmd_type = IXGBE_ADVTXD_DTYP_DATA | - IXGBE_ADVTXD_DCMD_DEXT | - IXGBE_ADVTXD_DCMD_IFCS; - cmd_type |= desc.len | IXGBE_TXD_CMD; - tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type); - tx_desc->read.olinfo_status = - cpu_to_le32(desc.len << IXGBE_ADVTXD_PAYLEN_SHIFT); - - xdp_ring->next_to_use++; - if (xdp_ring->next_to_use == xdp_ring->count) - xdp_ring->next_to_use = 0; - - budget--; - } - - if (tx_desc) { - ixgbe_xdp_ring_update_tail(xdp_ring); - xsk_tx_release(pool); - } - - return !!budget && work_done; -} - -static void ixgbe_clean_xdp_tx_buffer(struct ixgbe_ring *tx_ring, - struct libie_xg_tx_buffer *tx_bi) -{ - xdp_return_frame(tx_bi->xdpf); - dma_unmap_single(tx_ring->dev, - dma_unmap_addr(tx_bi, dma), - dma_unmap_len(tx_bi, len), DMA_TO_DEVICE); - dma_unmap_len_set(tx_bi, len, 0); -} - bool ixgbe_clean_xdp_tx_irq(struct ixgbe_q_vector *q_vector, struct ixgbe_ring *tx_ring, int napi_budget) { - u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use; - unsigned int total_packets = 0, total_bytes = 0; - struct xsk_buff_pool *pool = tx_ring->xsk_pool; - union ixgbe_adv_tx_desc *tx_desc; - struct libie_xg_tx_buffer *tx_bi; - u32 xsk_frames = 0; - - tx_bi = &tx_ring->tx_buffer_info[ntc]; - tx_desc = IXGBE_TX_DESC(tx_ring, ntc); - - while (ntc != ntu) { - if (!(tx_desc->wb.status & cpu_to_le32(IXGBE_TXD_STAT_DD))) - break; - - total_bytes += tx_bi->bytecount; - total_packets += tx_bi->gso_segs; - - if (tx_bi->xdpf) - ixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi); - else - xsk_frames++; - - tx_bi->xdpf = NULL; - - tx_bi++; - tx_desc++; - ntc++; - if (unlikely(ntc == tx_ring->count)) { - ntc = 0; - tx_bi = tx_ring->tx_buffer_info; - tx_desc = IXGBE_TX_DESC(tx_ring, 0); - } + u32 budget = min_t(u32, napi_budget, tx_ring->thresh); - /* issue prefetch for next Tx descriptor */ - prefetch(tx_desc); - } - - tx_ring->next_to_clean = ntc; - ixgbe_update_tx_ring_stats(tx_ring, q_vector, total_packets, - total_bytes); - - if (xsk_frames) - xsk_tx_completed(pool, xsk_frames); - - if (xsk_uses_need_wakeup(pool)) - xsk_set_tx_need_wakeup(pool); - - return ixgbe_xmit_zc(tx_ring, q_vector->tx.work_limit); + return libeth_xsk_xmit_do_bulk(tx_ring->xsk_pool, tx_ring, budget, + NULL, ixgbe_prep_xdp_sq, + libie_xg_xsk_xmit_desc, + libie_xg_xdp_rs_and_bump); } int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) @@ -419,29 +254,3 @@ int ixgbe_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) return 0; } - -void ixgbe_xsk_clean_tx_ring(struct ixgbe_ring *tx_ring) -{ - u16 ntc = tx_ring->next_to_clean, ntu = tx_ring->next_to_use; - struct xsk_buff_pool *pool = tx_ring->xsk_pool; - struct libie_xg_tx_buffer *tx_bi; - u32 xsk_frames = 0; - - while (ntc != ntu) { - tx_bi = &tx_ring->tx_buffer_info[ntc]; - - if (tx_bi->xdpf) - ixgbe_clean_xdp_tx_buffer(tx_ring, tx_bi); - else - xsk_frames++; - - tx_bi->xdpf = NULL; - - ntc++; - if (ntc == tx_ring->count) - ntc = 0; - } - - if (xsk_frames) - xsk_tx_completed(pool, xsk_frames); -}