diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 8d54ca19a047c..82e00c724e594 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -49,7 +49,6 @@ #define GENET_Q0_TX_BD_CNT \ (TOTAL_DESC - priv->hw_params->tx_queues * priv->hw_params->tx_bds_per_q) -#define RX_BUF_LENGTH 2048 #define SKB_ALIGNMENT 32 /* Tx/Rx DMA register offset, skip 256 descriptors */ @@ -744,8 +743,17 @@ static void bcmgenet_hfb_init(struct bcmgenet_priv *priv) INIT_LIST_HEAD(&priv->rxnfc_rules[i].list); priv->rxnfc_rules[i].state = BCMGENET_RXNFC_STATE_UNUSED; } +} + +static void bcmgenet_hfb_restore(struct bcmgenet_priv *priv) +{ + struct bcmgenet_rxnfc_rule *rule; bcmgenet_hfb_clear(priv); + + list_for_each_entry(rule, &priv->rxnfc_list, list) + if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) + bcmgenet_hfb_create_rxnfc_filter(priv, rule); } static int bcmgenet_begin(struct net_device *dev) @@ -2137,6 +2145,20 @@ static netdev_tx_t bcmgenet_xmit(struct sk_buff *skb, struct net_device *dev) goto out; } + /* The MAC holds a frame to insert its checksum, but only up to the + * packet ready threshold. Longer frames are dropped silently. + */ + if (unlikely(skb->len > priv->tx_csum_max_len) && + skb->ip_summed == CHECKSUM_PARTIAL) { + if (skb_checksum_help(skb)) { + BCMGENET_STATS64_INC((&ring->stats64), dropped); + dev_kfree_skb_any(skb); + ret = NETDEV_TX_OK; + goto out; + } + nr_frags = skb_shinfo(skb)->nr_frags; + } + /* Retain how many bytes will be sent on the wire, without TSB inserted * by transmit checksum offload */ @@ -2279,6 +2301,16 @@ static struct sk_buff *bcmgenet_rx_refill(struct bcmgenet_priv *priv, return rx_skb; } +static void bcmgenet_discard_frags(struct bcmgenet_rx_ring *ring) +{ + if (!ring->frag_head) + return; + + dev_kfree_skb_any(ring->frag_head); + ring->frag_head = NULL; + ring->frag_tail = NULL; +} + /* bcmgenet_desc_rx - descriptor based rx process. * this could be called from bottom half, or from NAPI polling method. */ @@ -2335,6 +2367,7 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, if (unlikely(!skb)) { BCMGENET_STATS64_INC(stats, dropped); + bcmgenet_discard_frags(ring); goto next; } @@ -2359,16 +2392,21 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, __func__, p_index, ring->c_index, ring->read_ptr, dma_length_status); - if (unlikely(len > RX_BUF_LENGTH)) { + if (unlikely(len > priv->rx_buf_len)) { netif_err(priv, rx_status, dev, "oversized packet\n"); BCMGENET_STATS64_INC(stats, length_errors); + bcmgenet_discard_frags(ring); dev_kfree_skb_any(skb); goto next; } - if (unlikely(!(dma_flag & DMA_EOP) || !(dma_flag & DMA_SOP))) { - netif_err(priv, rx_status, dev, - "dropping fragmented packet!\n"); + /* A new SOP resynchronizes after an incomplete frame */ + if (dma_flag & DMA_SOP) { + if (ring->frag_head) { + BCMGENET_STATS64_INC(stats, fragmented_errors); + bcmgenet_discard_frags(ring); + } + } else if (unlikely(!ring->frag_head)) { BCMGENET_STATS64_INC(stats, fragmented_errors); dev_kfree_skb_any(skb); goto next; @@ -2398,15 +2436,51 @@ static unsigned int bcmgenet_desc_rx(struct bcmgenet_rx_ring *ring, DMA_RX_RXER)) == DMA_RX_RXER) u64_stats_inc(&stats->errors); u64_stats_update_end(&stats->syncp); + bcmgenet_discard_frags(ring); dev_kfree_skb_any(skb); goto next; } /* error packet */ skb_put(skb, len); - /* remove RSB and hardware 2bytes added for IP alignment */ - skb_pull(skb, 66); - len -= 66; + /* Every fragment has a status block, only the first one also + * has the alignment bytes. + */ + if (dma_flag & DMA_SOP) { + skb_pull(skb, ENET_RX_OFFSET); + len -= ENET_RX_OFFSET; + } else { + skb_pull(skb, ENET_RSB_LEN); + len -= ENET_RSB_LEN; + } + + if (!(dma_flag & DMA_SOP)) { + /* chain onto the frame already being collected */ + if (!skb_shinfo(ring->frag_head)->frag_list) + skb_shinfo(ring->frag_head)->frag_list = skb; + else + ring->frag_tail->next = skb; + ring->frag_tail = skb; + + ring->frag_head->len += len; + ring->frag_head->data_len += len; + ring->frag_head->truesize += skb->truesize; + } + + if (!(dma_flag & DMA_EOP)) { + if (dma_flag & DMA_SOP) { + ring->frag_head = skb; + ring->frag_tail = skb; + } + goto next; + } + + if (!(dma_flag & DMA_SOP)) { + skb = ring->frag_head; + len = skb->len; + ring->frag_head = NULL; + ring->frag_tail = NULL; + } if (priv->crc_fwd_en) { skb_trim(skb, len - ETH_FCS_LEN); @@ -2588,6 +2662,44 @@ static void bcmgenet_link_intr_enable(struct bcmgenet_priv *priv) bcmgenet_intrl2_0_writel(priv, int0_enable, INTRL2_CPU_MASK_CLEAR); } +/* Threshold in register units. Covers the alignment bytes and the frame, but + * not the status block, which the hardware adds on top. + */ +static unsigned int bcmgenet_pkt_rdy_thld(unsigned int mtu) +{ + unsigned int len = ENET_RBUF_ALIGN + mtu + ETH_HLEN + VLAN_HLEN; + + len = round_up(len, ENET_THLD_BURST) / ENET_THLD_UNIT; + + /* Keep the reset default for the common MTUs */ + return clamp_t(unsigned int, len, ENET_THLD_DEFAULT, ENET_THLD_MAX); +} + +/* A buffer has to hold everything the threshold lets the hardware deliver */ +static unsigned int bcmgenet_rx_buf_len(unsigned int mtu) +{ + return ENET_RSB_LEN + bcmgenet_pkt_rdy_thld(mtu) * ENET_THLD_UNIT; +} + +/* Program the MTU dependent registers. Call with the MAC disabled. */ +static void bcmgenet_set_mtu_regs(struct bcmgenet_priv *priv, unsigned int mtu) +{ + u32 thld; + + bcmgenet_umac_writel(priv, ENET_MAX_FRAME_LEN(mtu), UMAC_MAX_FRAME_LEN); + + thld = bcmgenet_pkt_rdy_thld(mtu); + priv->tx_csum_max_len = thld * ENET_THLD_UNIT; + + /* GENET v1 maps other registers at these offsets */ + if (GENET_IS_V1(priv)) + return; + + bcmgenet_rbuf_writel(priv, thld, RBUF_PKT_RDY_THLD); + bcmgenet_writel(thld, priv->base + priv->hw_params->tbuf_offset + + TBUF_PKT_RDY_THLD); +} + static void init_umac(struct bcmgenet_priv *priv) { struct device *kdev = &priv->pdev->dev; @@ -2604,7 +2716,7 @@ static void init_umac(struct bcmgenet_priv *priv) UMAC_MIB_CTRL); bcmgenet_umac_writel(priv, 0, UMAC_MIB_CTRL); - bcmgenet_umac_writel(priv, ENET_MAX_MTU_SIZE, UMAC_MAX_FRAME_LEN); + bcmgenet_set_mtu_regs(priv, priv->dev->mtu); /* init tx registers, enable TSB */ reg = bcmgenet_tbuf_ctrl_get(priv); @@ -2710,7 +2822,7 @@ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, /* Set flow period for ring != 0 */ if (index) - flow_period_val = ENET_MAX_MTU_SIZE << 16; + flow_period_val = ENET_MAX_FRAME_LEN(priv->dev->mtu) << 16; bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_PROD_INDEX); bcmgenet_tdma_ring_writel(priv, index, 0, TDMA_CONS_INDEX); @@ -2720,7 +2832,7 @@ static void bcmgenet_init_tx_ring(struct bcmgenet_priv *priv, TDMA_FLOW_PERIOD); bcmgenet_tdma_ring_writel(priv, index, ((size << DMA_RING_SIZE_SHIFT) | - RX_BUF_LENGTH), DMA_RING_BUF_SIZE); + priv->rx_buf_len), DMA_RING_BUF_SIZE); /* Set start and end address, read and write pointers */ bcmgenet_tdma_ring_writel(priv, index, start_ptr * words_per_bd, @@ -2768,7 +2880,7 @@ static int bcmgenet_init_rx_ring(struct bcmgenet_priv *priv, bcmgenet_rdma_ring_writel(priv, index, 0, RDMA_CONS_INDEX); bcmgenet_rdma_ring_writel(priv, index, ((size << DMA_RING_SIZE_SHIFT) | - RX_BUF_LENGTH), DMA_RING_BUF_SIZE); + priv->rx_buf_len), DMA_RING_BUF_SIZE); bcmgenet_rdma_ring_writel(priv, index, (DMA_FC_THRESH_LO << DMA_XOFF_THRESHOLD_SHIFT) | @@ -3013,6 +3125,10 @@ static void bcmgenet_fini_dma(struct bcmgenet_priv *priv) struct netdev_queue *txq; int i; + /* An MTU change can fail with the rings already freed */ + if (!priv->rx_cbs) + return; + bcmgenet_fini_rx_napi(priv); bcmgenet_fini_tx_napi(priv); @@ -3021,9 +3137,14 @@ static void bcmgenet_fini_dma(struct bcmgenet_priv *priv) netdev_tx_reset_queue(txq); } + for (i = 0; i <= priv->hw_params->rx_queues; i++) + bcmgenet_discard_frags(&priv->rx_rings[i]); + bcmgenet_free_rx_buffers(priv); kfree(priv->rx_cbs); + priv->rx_cbs = NULL; kfree(priv->tx_cbs); + priv->tx_cbs = NULL; } /* init_edma: Initialize DMA control register */ @@ -3083,6 +3204,7 @@ static int bcmgenet_init_dma(struct bcmgenet_priv *priv, bool flush_rx) GFP_KERNEL); if (!priv->tx_cbs) { kfree(priv->rx_cbs); + priv->rx_cbs = NULL; return -ENOMEM; } @@ -3101,7 +3223,9 @@ static int bcmgenet_init_dma(struct bcmgenet_priv *priv, bool flush_rx) netdev_err(priv->dev, "failed to initialize Rx queues\n"); bcmgenet_free_rx_buffers(priv); kfree(priv->rx_cbs); + priv->rx_cbs = NULL; kfree(priv->tx_cbs); + priv->tx_cbs = NULL; return ret; } @@ -3270,7 +3394,7 @@ static void bcmgenet_get_hw_addr(struct bcmgenet_priv *priv, put_unaligned_be16(addr_tmp, &addr[4]); } -static void bcmgenet_netif_start(struct net_device *dev) +static void bcmgenet_netif_start(struct net_device *dev, bool start_phy) { struct bcmgenet_priv *priv = netdev_priv(dev); @@ -3287,7 +3411,8 @@ static void bcmgenet_netif_start(struct net_device *dev) /* Monitor link interrupts now */ bcmgenet_link_intr_enable(priv); - phy_start(dev->phydev); + if (start_phy) + phy_start(dev->phydev); } static int bcmgenet_open(struct net_device *dev) @@ -3318,8 +3443,8 @@ static int bcmgenet_open(struct net_device *dev) bcmgenet_set_hw_addr(priv, dev->dev_addr); - /* HFB init */ - bcmgenet_hfb_init(priv); + /* Restore the filters, the MAC was reset above */ + bcmgenet_hfb_restore(priv); /* Reinitialize TDMA and RDMA and SW housekeeping */ ret = bcmgenet_init_dma(priv, true); @@ -3350,8 +3475,9 @@ static int bcmgenet_open(struct net_device *dev) bcmgenet_phy_pause_set(dev, priv->rx_pause, priv->tx_pause); - bcmgenet_netif_start(dev); + bcmgenet_netif_start(dev, true); + priv->datapath_up = true; netif_tx_start_all_queues(dev); return 0; @@ -3374,6 +3500,8 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) { struct bcmgenet_priv *priv = netdev_priv(dev); + /* Stop completion polling before it can wake a stopped queue */ + bcmgenet_disable_tx_napi(priv); netif_tx_disable(dev); /* Disable MAC receive */ @@ -3388,7 +3516,6 @@ static void bcmgenet_netif_stop(struct net_device *dev, bool stop_phy) /* Disable MAC transmit. TX DMA disabled must be done before this */ umac_enable_set(priv, CMD_TX_EN, false); - bcmgenet_disable_tx_napi(priv); bcmgenet_disable_rx_napi(priv); bcmgenet_intr_disable(priv); @@ -3409,7 +3536,11 @@ static int bcmgenet_close(struct net_device *dev) netif_dbg(priv, ifdown, dev, "bcmgenet_close\n"); - bcmgenet_netif_stop(dev, false); + /* A failed MTU change can have torn the datapath down already */ + if (priv->datapath_up) { + bcmgenet_netif_stop(dev, false); + priv->datapath_up = false; + } /* Really kill the PHY state machine and disconnect from it */ phy_disconnect(dev->phydev); @@ -3654,6 +3785,66 @@ static int bcmgenet_change_carrier(struct net_device *dev, bool new_carrier) return 0; } +static int bcmgenet_change_mtu(struct net_device *dev, int new_mtu) +{ + struct bcmgenet_priv *priv = netdev_priv(dev); + unsigned int old_mtu = dev->mtu; + int ret; + + if (!netif_running(dev)) { + WRITE_ONCE(dev->mtu, new_mtu); + priv->rx_buf_len = bcmgenet_rx_buf_len(new_mtu); + return 0; + } + + /* The watchdog trips on an idle queue once the rings are gone */ + netif_device_detach(dev); + + /* Only the buffers and the MTU registers change, leave the PHY up */ + bcmgenet_netif_stop(dev, false); + priv->datapath_up = false; + + WRITE_ONCE(dev->mtu, new_mtu); + priv->rx_buf_len = bcmgenet_rx_buf_len(new_mtu); + bcmgenet_set_mtu_regs(priv, new_mtu); + + ret = bcmgenet_init_dma(priv, true); + if (ret) { + /* Retry the size that was allocated a moment ago */ + WRITE_ONCE(dev->mtu, old_mtu); + priv->rx_buf_len = bcmgenet_rx_buf_len(old_mtu); + bcmgenet_set_mtu_regs(priv, old_mtu); + if (bcmgenet_init_dma(priv, true)) { + /* Nothing left to run on. Take the interface down so + * that close and suspend do not tear it down twice. + */ + netdev_err(dev, "failed to restore MTU %u, closing\n", + old_mtu); + netif_close(dev); + + /* Mark the device present again, __dev_open() + * refuses a detached one. The queues stay stopped + * because the interface is down by now. + */ + netif_device_attach(dev); + return ret; + } + } + + bcmgenet_hfb_restore(priv); + bcmgenet_netif_start(dev, false); + + /* bcmgenet_netif_start() only restores the link interrupt */ + if (bcmgenet_has_mdio_intr(priv)) + bcmgenet_intrl2_0_writel(priv, UMAC_IRQ_MDIO_EVENT, + INTRL2_CPU_MASK_CLEAR); + + priv->datapath_up = true; + netif_device_attach(dev); + + return ret; +} + static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_open = bcmgenet_open, .ndo_stop = bcmgenet_close, @@ -3665,6 +3856,7 @@ static const struct net_device_ops bcmgenet_netdev_ops = { .ndo_set_features = bcmgenet_set_features, .ndo_get_stats64 = bcmgenet_get_stats64, .ndo_change_carrier = bcmgenet_change_carrier, + .ndo_change_mtu = bcmgenet_change_mtu, }; /* GENET hardware parameters/characteristics */ @@ -4021,8 +4213,12 @@ static int bcmgenet_probe(struct platform_device *pdev) /* Mii wait queue */ init_waitqueue_head(&priv->wq); - /* Always use RX_BUF_LENGTH (2KB) buffer for all chips */ - priv->rx_buf_len = RX_BUF_LENGTH; + bcmgenet_hfb_init(priv); + + /* v1 cannot program the thresholds, so it stays at the default MTU */ + priv->rx_buf_len = bcmgenet_rx_buf_len(dev->mtu); + if (!GENET_IS_V1(priv)) + dev->max_mtu = ENET_MAX_JUMBO_MTU; INIT_WORK(&priv->bcmgenet_irq_work, bcmgenet_irq_task); priv->clk_wol = devm_clk_get_optional(&priv->pdev->dev, "enet-wol"); @@ -4225,10 +4421,7 @@ static int bcmgenet_resume(struct device *d) bcmgenet_set_hw_addr(priv, dev->dev_addr); /* Restore hardware filters */ - bcmgenet_hfb_clear(priv); - list_for_each_entry(rule, &priv->rxnfc_list, list) - if (rule->state != BCMGENET_RXNFC_STATE_UNUSED) - bcmgenet_hfb_create_rxnfc_filter(priv, rule); + bcmgenet_hfb_restore(priv); /* Reinitialize TDMA and RDMA and SW housekeeping */ ret = bcmgenet_init_dma(priv, false); @@ -4240,7 +4433,7 @@ static int bcmgenet_resume(struct device *d) if (!device_may_wakeup(d)) phy_resume(dev->phydev); - bcmgenet_netif_start(dev); + bcmgenet_netif_start(dev, true); netif_device_attach(dev); diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 9e4110c7fdf6f..754f72e07740d 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -27,13 +27,41 @@ /* which ring is descriptor based */ #define DESC_INDEX 16 -/* Body(1500) + EH_SIZE(14) + VLANTAG(4) + BRCMTAG(6) + FCS(4) = 1528. - * 1536 is multiple of 256 bytes - */ #define ENET_BRCM_TAG_LEN 6 #define ENET_PAD 8 -#define ENET_MAX_MTU_SIZE (ETH_DATA_LEN + ETH_HLEN + VLAN_HLEN + \ - ENET_BRCM_TAG_LEN + ETH_FCS_LEN + ENET_PAD) + +/* The hardware writes a status block and two alignment bytes ahead of the + * frame. + */ +#define ENET_RSB_LEN 64 +#define ENET_RBUF_ALIGN 2 +#define ENET_RX_OFFSET (ENET_RSB_LEN + ENET_RBUF_ALIGN) + +/* Longest frame the MAC must accept for a given MTU */ +#define ENET_FRAME_OVERHEAD (ETH_HLEN + VLAN_HLEN + ENET_BRCM_TAG_LEN + \ + ETH_FCS_LEN + ENET_PAD) +#define ENET_MAX_FRAME_LEN(mtu) ((mtu) + ENET_FRAME_OVERHEAD) + +/* RBUF and TBUF hand a frame to the DMA once the threshold is reached, so a + * longer frame arrives without an end of packet marker and is dropped. Both + * registers are 8 bit in units of 16 bytes and want a multiple of the 256 + * byte burst size, so 0xf0 is the largest usable value. + */ +#define ENET_THLD_UNIT 16 +#define ENET_THLD_BURST 256 +#define ENET_THLD_DEFAULT 0x80 +#define ENET_THLD_MAX 0xf0 +#define ENET_THLD_MAX_LEN (ENET_THLD_MAX * ENET_THLD_UNIT) + +/* Largest MTU that fits one descriptor, with room for a VLAN tag so a VLAN + * interface can use the parent MTU. Longer frames are split at the threshold + * and reassembled in bcmgenet_desc_rx(). + */ +#define ENET_MAX_MTU (ENET_THLD_MAX_LEN - ENET_RBUF_ALIGN - \ + ETH_HLEN - VLAN_HLEN) + +/* UMAC_MAX_FRAME_LEN is 14 bits wide and counts the FCS */ +#define ENET_MAX_JUMBO_MTU (GENMASK(13, 0) - ENET_FRAME_OVERHEAD) #define DMA_MAX_BURST_LENGTH 0x10 /* misc. configuration */ @@ -219,6 +247,8 @@ struct bcmgenet_rx_stats64 { #define RBUF_ALIGN_2B (1 << 1) #define RBUF_BAD_DIS (1 << 2) +#define RBUF_PKT_RDY_THLD 0x08 + #define RBUF_STATUS 0x0C #define RBUF_STATUS_WOL (1 << 0) #define RBUF_STATUS_MPD_INTR_ACTIVE (1 << 1) @@ -249,6 +279,7 @@ struct bcmgenet_rx_stats64 { #define TBUF_CTRL 0x00 #define TBUF_64B_EN (1 << 0) #define TBUF_BP_MC 0x0C +#define TBUF_PKT_RDY_THLD 0x10 #define TBUF_ENERGY_CTRL 0x14 #define TBUF_EEE_EN (1 << 0) #define TBUF_PM_EN (1 << 1) @@ -572,6 +603,8 @@ struct bcmgenet_rx_ring { unsigned int cb_ptr; /* Rx ring initial CB ptr */ unsigned int end_ptr; /* Rx ring end CB ptr */ unsigned int old_discards; + struct sk_buff *frag_head; /* frame being reassembled */ + struct sk_buff *frag_tail; /* its last fragment */ struct bcmgenet_net_dim dim; u32 rx_max_coalesced_frames; u32 rx_coalesce_usecs; @@ -610,12 +643,14 @@ struct bcmgenet_priv { struct enet_cb *rx_cbs; unsigned int num_rx_bds; unsigned int rx_buf_len; + unsigned int tx_csum_max_len; struct bcmgenet_rxnfc_rule rxnfc_rules[MAX_NUM_OF_FS_RULES]; struct list_head rxnfc_list; struct bcmgenet_rx_ring rx_rings[GENET_MAX_MQ_CNT + 1]; /* other misc variables */ + bool datapath_up; const struct bcmgenet_hw_params *hw_params; u32 flags; unsigned autoneg_pause:1;