From a522fd432f9e4b9400758dddf50ac0b2576fed39 Mon Sep 17 00:00:00 2001 From: meehien Date: Tue, 7 Jul 2026 14:34:03 +0100 Subject: [PATCH 1/4] When the mt7996e endpoint suffers a PCIe surprise-down (AER Uncorrectable Fatal / Surprise Down Error), the firmware stops answering and the MCU message watchdog escalates to mt7996_mac_reset_work(). That worker (and the mt7996_reset() path that queues it) issues MMIO read-modify-writes against the register window of a device that is no longer on the bus. On arm64 with the mtk-pcie-gen3 controller a read from an offline endpoint comes back as an asynchronous external abort -> SError, which is an unconditional kernel panic (no oops, no recovery): SError Interrupt on CPU0, code 0x00000000bf000002 -- SError Kernel panic - not syncing: Asynchronous SError Interrupt mt76_mmio_rmw+0x58/0x120 [mt76] mt7996_rr+0xf0/0x14c [mt7996e] mt7996_update_channel+0x9c/0x178 [mt7996e] mt76_update_survey+0x28/0xf4 [mt76] mt7996_mac_work+0x30/0xe0 [mt7996e] Add a mt7996_dev_gone() helper that reports the endpoint unreachable when either the driver has flagged it removed (MT76_REMOVED) or the AER core has taken the PCIe channel offline, and gate both recovery entry points on it. The pci_channel_offline() check is the load-bearing one for the panic above: the AER core freezes the channel at the start of pcie_do_recovery(), well before the watchdog-triggered reset worker runs, so the guard trips and the worker returns instead of touching dead MMIO. --- .../net/wireless/mediatek/mt76/mt7996/mac.c | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c index 1be2d8cbc3c1..cc0052a38351 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mac.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mac.c @@ -2136,6 +2136,26 @@ void mt7996_update_channel(struct mt76_phy *mphy) state->noise = -(phy->noise >> 4); } +/* True when the PCIe endpoint is no longer reachable: either the driver has + * flagged it removed (PCIe .error_detected handler / unplug) or the AER core + * has taken the channel offline. Any MMIO access past this point would read + * from a dead register window and raise an asynchronous external abort / + * SError on arm64 (mtk-pcie-gen3), which is an unconditional kernel panic. + */ +static bool mt7996_dev_gone(struct mt7996_dev *dev) +{ + struct mt76_dev *mdev = &dev->mt76; + + if (test_bit(MT76_REMOVED, &dev->mphy.state)) + return true; + + if (dev_is_pci(mdev->dev) && + pci_channel_offline(to_pci_dev(mdev->dev))) + return true; + + return false; +} + static bool mt7996_wait_reset_state(struct mt7996_dev *dev, u32 state) { @@ -2469,6 +2489,13 @@ void mt7996_mac_reset_work(struct work_struct *work) dev = container_of(work, struct mt7996_dev, reset_work); hw = mt76_hw(dev); + if (mt7996_dev_gone(dev)) { + dev_warn(dev->mt76.dev, + "%s: PCIe endpoint unreachable, skipping MAC reset\n", + wiphy_name(hw->wiphy)); + return; + } + /* chip full reset */ if (dev->recovery.restart) { /* disable WA/WM WDT */ @@ -2700,6 +2727,14 @@ void mt7996_reset(struct mt7996_dev *dev) if (dev->recovery.hw_full_reset) return; + /* Endpoint surprise-removed (e.g. SFP hot-pull browning out the WiFi + * rail on BPI-R4): driving the recovery worker would touch the now- + * unreachable MMIO window and raise an SError, turning a WiFi hiccup + * into a kernel panic. Bail out instead. + */ + if (mt7996_dev_gone(dev)) + return; + /* wm/wa exception: do full recovery */ if (READ_ONCE(dev->recovery.state) & MT_MCU_CMD_WDT_MASK) { dev->recovery.restart = true; From 5584aa664f56c1621482acada3c49fc5f59c7058 Mon Sep 17 00:00:00 2001 From: meehien Date: Tue, 7 Jul 2026 14:35:02 +0100 Subject: [PATCH 2/4] mt7996 registers no pci_error_handlers, so a fatal AER event (surprise down) gives the driver no notification that the endpoint has vanished; the log shows 'mt7996e: AER: can't recover (no error_detected callback)'. The reset worker is guarded against an offline channel by the previous patch, but relying solely on pci_channel_offline() misses controllers that do not update pci_dev->error_state, and it leaves the device wedged rather than disconnected. Add an .error_detected handler to mt7996_pci_driver. It sets MT76_REMOVED - the same idiom the mt792x PCIe driver uses on unplug - so all present and future liveness checks (including mt7996_dev_gone()) short-circuit, wakes any thread blocked in the MCU wait so it fails fast, and returns PCI_ERS_RESULT_DISCONNECT. Disconnect is the honest result: the MAC and firmware state cannot survive a PCIe link loss and this driver has no slot_reset re-init path, so an orderly removal is preferable to a doomed recovery attempt. The handler is attached to mt7996_pci_driver only. The hif2 device (second PCIe function) stores a struct mt7996_hif in its drvdata rather than the mt76_dev, so it needs a back-pointer to the owning device before the same handler can be reused there; that is left as a follow-up. --- .../net/wireless/mediatek/mt76/mt7996/pci.c | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c index 12523ddba630..ecd8ec92161d 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c @@ -241,6 +241,38 @@ static void mt7996_pci_remove(struct pci_dev *pdev) mt7996_unregister_device(dev); } +static pci_ers_result_t +mt7996_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + + dev_err(&pdev->dev, "PCIe error detected (state=%u), marking device removed\n", + state); + + /* Flag the device gone so every liveness check - including + * mt7996_dev_gone() guarding the reset worker - short-circuits before + * it can touch the now-unreachable MMIO window. This is the same idiom + * the mt792x PCIe driver uses on surprise removal, and it does not rely + * on the controller updating pci_dev->error_state. + */ + set_bit(MT76_REMOVED, &mdev->phy.state); + + /* Wake anyone blocked on an MCU response so it fails fast instead of + * stalling to timeout against a dead endpoint. + */ + wake_up(&mdev->mcu.wait); + + /* The MAC and firmware state cannot survive a PCIe link loss and this + * driver has no slot_reset re-init path, so request a clean disconnect + * rather than a doomed recovery attempt. + */ + return PCI_ERS_RESULT_DISCONNECT; +} + +static const struct pci_error_handlers mt7996_pci_err_handler = { + .error_detected = mt7996_pci_error_detected, +}; + struct pci_driver mt7996_hif_driver = { .name = KBUILD_MODNAME "_hif", .id_table = mt7996_hif_device_table, @@ -253,6 +285,7 @@ struct pci_driver mt7996_pci_driver = { .id_table = mt7996_pci_device_table, .probe = mt7996_pci_probe, .remove = mt7996_pci_remove, + .err_handler = &mt7996_pci_err_handler, }; MODULE_DEVICE_TABLE(pci, mt7996_pci_device_table); From 6bb85d769e5eacb9358e202540fc055e3f48fe30 Mon Sep 17 00:00:00 2001 From: meehien Date: Tue, 7 Jul 2026 14:35:25 +0100 Subject: [PATCH 3/4] The previous patch added an .error_detected handler to mt7996_pci_driver only. The second PCIe function (hif2, bound by mt7996_hif_driver) is the same physical chip, so a fatal AER event on its link downs the whole device just the same - but it had no handler, and its drvdata is a struct mt7996_hif rather than the mt76_dev, so the primary handler cannot be reused as-is. Give struct mt7996_hif a back-pointer to the owning mt7996_dev, populated when the primary function claims the hif during probe, and add an .error_detected handler for mt7996_hif_driver that follows it to the mt76 device and marks it removed. The shared marking logic (set MT76_REMOVED, wake the MCU wait, return PCI_ERS_RESULT_DISCONNECT) is factored into mt7996_pci_mark_removed() and used by both handlers. The hif handler tolerates a not-yet-claimed hif (back-pointer still NULL if the error races the primary function's probe) by reporting the disconnect without dereferencing it. --- .../wireless/mediatek/mt76/mt7996/mt7996.h | 6 +++ .../net/wireless/mediatek/mt76/mt7996/pci.c | 46 ++++++++++++++++--- 2 files changed, 45 insertions(+), 7 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index bdcf72457954..f3f74582f991 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -145,6 +145,7 @@ #define MT7996_RRO_MSDU_PG_CR_CNT 8 #define MT7996_RRO_MSDU_PG_SIZE_PER_CR 0x10000 +struct mt7996_dev; struct mt7996_vif; struct mt7996_sta; struct mt7996_dfs_pulse; @@ -313,6 +314,11 @@ struct mt7996_hif { enum pci_bus_speed speed; enum pcie_link_width width; + + /* Owning device, set when the primary PCIe function claims this hif. + * Lets the hif2 .error_detected handler reach the mt76 device. + */ + struct mt7996_dev *mt7996; }; #define WED_RRO_ADDR_SIGNATURE_MASK GENMASK(31, 24) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c index ecd8ec92161d..347a8a29b717 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c @@ -139,6 +139,8 @@ static int mt7996_pci_probe(struct pci_dev *pdev, mt7996_wfsys_reset(dev); hif2 = mt7996_pci_init_hif2(pdev); dev->hif2 = hif2; + if (hif2) + hif2->mt7996 = dev; mt76_npu_init(mdev, pci_resource_start(pdev, 0), pdev->bus && pci_domain_nr(pdev->bus) ? 3 : 2); @@ -241,14 +243,8 @@ static void mt7996_pci_remove(struct pci_dev *pdev) mt7996_unregister_device(dev); } -static pci_ers_result_t -mt7996_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) +static pci_ers_result_t mt7996_pci_mark_removed(struct mt76_dev *mdev) { - struct mt76_dev *mdev = pci_get_drvdata(pdev); - - dev_err(&pdev->dev, "PCIe error detected (state=%u), marking device removed\n", - state); - /* Flag the device gone so every liveness check - including * mt7996_dev_gone() guarding the reset worker - short-circuits before * it can touch the now-unreachable MMIO window. This is the same idiom @@ -269,15 +265,51 @@ mt7996_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) return PCI_ERS_RESULT_DISCONNECT; } +static pci_ers_result_t +mt7996_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + + dev_err(&pdev->dev, "PCIe error detected (state=%u), marking device removed\n", + state); + + return mt7996_pci_mark_removed(mdev); +} + +static pci_ers_result_t +mt7996_hif_error_detected(struct pci_dev *pdev, pci_channel_state_t state) +{ + struct mt7996_hif *hif = pci_get_drvdata(pdev); + struct mt7996_dev *dev = hif ? hif->mt7996 : NULL; + + dev_err(&pdev->dev, "PCIe error detected (state=%u) on hif2\n", state); + + /* hif2 is the second PCIe function of the same physical chip; a link + * loss here downs the whole device. Mark the owning mt76 device removed + * so its reset worker bails out too. The primary function may not have + * claimed this hif yet (still probing), in which case there is nothing + * to mark - just report the disconnect. + */ + if (!dev) + return PCI_ERS_RESULT_DISCONNECT; + + return mt7996_pci_mark_removed(&dev->mt76); +} + static const struct pci_error_handlers mt7996_pci_err_handler = { .error_detected = mt7996_pci_error_detected, }; +static const struct pci_error_handlers mt7996_hif_err_handler = { + .error_detected = mt7996_hif_error_detected, +}; + struct pci_driver mt7996_hif_driver = { .name = KBUILD_MODNAME "_hif", .id_table = mt7996_hif_device_table, .probe = mt7996_pci_probe, .remove = mt7996_hif_remove, + .err_handler = &mt7996_hif_err_handler, }; struct pci_driver mt7996_pci_driver = { From ba9f07e9faa30ed68f128aa25ef34bbfbfff87df Mon Sep 17 00:00:00 2001 From: meehien Date: Tue, 7 Jul 2026 14:36:09 +0100 Subject: [PATCH 4/4] The previous patches convert a post-error MMIO access from a kernel panic into a clean disconnect, but the WiFi stays down until a manual power cycle. Implement actual PCIe error recovery so the device can come back on its own. error_detected() now returns PCI_ERS_RESULT_NEED_RESET (unless the channel has permanently failed), asking the PCIe core to perform a secondary bus reset - which asserts PERST# to the endpoint and retrains the link - and then call slot_reset(). The crash logs show this bus reset already succeeds ('Root Port link has been reset', 'ready 0ms after bus reset'); recovery previously failed only because the driver registered no callbacks. slot_reset() restores config space (clobbered by the bus reset), optionally hard power-cycles the card through a board reset-gpios line - needed when PERST# cannot clear an MCU wedged by a supply brownout, a no-op when the GPIO is absent - and confirms the chip answers MMIO (MT_HW_REV) before clearing MT76_REMOVED and driving the existing full chip reset (mt7996_mac_full_reset() reloads firmware and rebuilds DMA/NAPI/token state). If the chip does not answer, it returns DISCONNECT, leaving the safe WiFi-down state of the earlier patches. pci_save_state() is added at the end of probe so slot_reset() has valid state to restore. The hif2 function gets a minimal slot_reset that restores its own config space; the primary function drives the shared chip reinit. --- .../wireless/mediatek/mt76/mt7996/mt7996.h | 6 + .../net/wireless/mediatek/mt76/mt7996/pci.c | 129 +++++++++++++++--- 2 files changed, 115 insertions(+), 20 deletions(-) diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h index f3f74582f991..91af95ea7f14 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h +++ b/drivers/net/wireless/mediatek/mt76/mt7996/mt7996.h @@ -419,6 +419,12 @@ struct mt7996_dev { struct wiphy_radio_freq_range radio_freqs[MT7996_MAX_RADIOS]; struct mt7996_hif *hif2; + + /* Optional board GPIO wired to the WiFi card reset/power-enable (in + * parallel with the manual switch), used for hard recovery after a PCIe + * error when PERST# alone does not revive a brownout-wedged MCU. + */ + struct gpio_desc *reset_gpio; struct mt7996_reg_desc reg; u8 q_id[MT7996_MAX_QUEUE]; u32 q_int_mask[MT7996_MAX_QUEUE]; diff --git a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c index 347a8a29b717..ad2f734e7ba6 100644 --- a/drivers/net/wireless/mediatek/mt76/mt7996/pci.c +++ b/drivers/net/wireless/mediatek/mt76/mt7996/pci.c @@ -3,6 +3,8 @@ * Copyright (C) 2022 MediaTek Inc. */ +#include +#include #include #include #include @@ -142,6 +144,12 @@ static int mt7996_pci_probe(struct pci_dev *pdev, if (hif2) hif2->mt7996 = dev; + /* Optional reset line for slot_reset() recovery; absent on stock DT. */ + dev->reset_gpio = devm_gpiod_get_optional(&pdev->dev, "reset", + GPIOD_OUT_LOW); + if (IS_ERR(dev->reset_gpio)) + dev->reset_gpio = NULL; + mt76_npu_init(mdev, pci_resource_start(pdev, 0), pdev->bus && pci_domain_nr(pdev->bus) ? 3 : 2); @@ -198,6 +206,11 @@ static int mt7996_pci_probe(struct pci_dev *pdev, if (ret) goto free_hif2_irq; + /* Snapshot config space so slot_reset() can restore BARs and the + * command register after a PCIe bus reset clobbers them. + */ + pci_save_state(pdev); + return 0; free_hif2_irq: @@ -243,26 +256,15 @@ static void mt7996_pci_remove(struct pci_dev *pdev) mt7996_unregister_device(dev); } -static pci_ers_result_t mt7996_pci_mark_removed(struct mt76_dev *mdev) +static void mt7996_pci_mark_removed(struct mt76_dev *mdev) { /* Flag the device gone so every liveness check - including * mt7996_dev_gone() guarding the reset worker - short-circuits before - * it can touch the now-unreachable MMIO window. This is the same idiom - * the mt792x PCIe driver uses on surprise removal, and it does not rely - * on the controller updating pci_dev->error_state. + * it can touch the now-unreachable MMIO window, and wake anyone blocked + * on an MCU response so it fails fast instead of stalling to timeout. */ set_bit(MT76_REMOVED, &mdev->phy.state); - - /* Wake anyone blocked on an MCU response so it fails fast instead of - * stalling to timeout against a dead endpoint. - */ wake_up(&mdev->mcu.wait); - - /* The MAC and firmware state cannot survive a PCIe link loss and this - * driver has no slot_reset re-init path, so request a clean disconnect - * rather than a doomed recovery attempt. - */ - return PCI_ERS_RESULT_DISCONNECT; } static pci_ers_result_t @@ -270,10 +272,20 @@ mt7996_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state) { struct mt76_dev *mdev = pci_get_drvdata(pdev); - dev_err(&pdev->dev, "PCIe error detected (state=%u), marking device removed\n", + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + dev_err(&pdev->dev, "PCIe error detected (state=%u), requesting slot reset\n", state); - return mt7996_pci_mark_removed(mdev); + mt7996_pci_mark_removed(mdev); + + /* Ask the PCIe core to perform a secondary bus reset (PERST#) and then + * call .slot_reset, where the chip is reinitialised. If recovery there + * fails, slot_reset returns DISCONNECT and we end up in the same safe + * state as before - WiFi down, the rest of the system alive. + */ + return PCI_ERS_RESULT_NEED_RESET; } static pci_ers_result_t @@ -282,26 +294,103 @@ mt7996_hif_error_detected(struct pci_dev *pdev, pci_channel_state_t state) struct mt7996_hif *hif = pci_get_drvdata(pdev); struct mt7996_dev *dev = hif ? hif->mt7996 : NULL; + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + dev_err(&pdev->dev, "PCIe error detected (state=%u) on hif2\n", state); /* hif2 is the second PCIe function of the same physical chip; a link * loss here downs the whole device. Mark the owning mt76 device removed - * so its reset worker bails out too. The primary function may not have - * claimed this hif yet (still probing), in which case there is nothing - * to mark - just report the disconnect. + * so its reset worker bails out too, then request the shared slot reset. + * If the primary function has not claimed this hif yet, there is nothing + * to reinitialise from here - just disconnect this function. */ if (!dev) return PCI_ERS_RESULT_DISCONNECT; - return mt7996_pci_mark_removed(&dev->mt76); + mt7996_pci_mark_removed(&dev->mt76); + + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t mt7996_pci_slot_reset(struct pci_dev *pdev) +{ + struct mt76_dev *mdev = pci_get_drvdata(pdev); + struct mt7996_dev *dev = container_of(mdev, struct mt7996_dev, mt76); + + dev_info(&pdev->dev, "PCIe slot reset, reinitialising device\n"); + + /* The secondary bus reset clobbered config space; restore BARs and the + * command register before any MMIO, and re-enable bus mastering. + */ + pci_restore_state(pdev); + pci_set_master(pdev); + + /* Optional hard power-cycle of the card via a board GPIO wired in + * parallel with the manual power switch. PERST# from the bus reset above + * may not clear an MCU wedged by a supply brownout; cycling the rail + * does. No-op when no reset-gpios is described in DT. + */ + if (dev->reset_gpio) { + gpiod_set_value_cansleep(dev->reset_gpio, 1); + msleep(20); + gpiod_set_value_cansleep(dev->reset_gpio, 0); + msleep(50); + } + + /* Confirm the chip is actually back on the bus before promising the + * core a recovery; a dead read returns all-ones. + */ + if (mt76_rr(dev, MT_HW_REV) == 0xffffffff) { + dev_err(&pdev->dev, "device unreachable after slot reset\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + /* Config space is valid and the channel is online again, so the + * mt7996_dev_gone() guard must stop tripping for the reinit to run. + */ + clear_bit(MT76_REMOVED, &mdev->phy.state); + + /* Reload firmware and rebuild DMA/NAPI/token state through the existing + * full chip reset path (mt7996_mac_reset_work -> mt7996_mac_full_reset). + */ + dev->recovery.restart = true; + queue_work(mdev->wq, &dev->reset_work); + + return PCI_ERS_RESULT_RECOVERED; +} + +static pci_ers_result_t mt7996_hif_slot_reset(struct pci_dev *pdev) +{ + /* hif2 shares the chip with the primary function, which drives the + * actual reinit from its own slot_reset. Here we only need this + * function's own config space restored so the reinit can reach the + * hif2 register window. + */ + pci_restore_state(pdev); + pci_set_master(pdev); + + return PCI_ERS_RESULT_RECOVERED; +} + +static void mt7996_pci_io_resume(struct pci_dev *pdev) +{ + /* The chip reinit queued from slot_reset ends in ieee80211_restart_hw(), + * which restarts the queues, so nothing extra is required here. + */ + dev_info(&pdev->dev, "PCIe recovery complete\n"); } static const struct pci_error_handlers mt7996_pci_err_handler = { .error_detected = mt7996_pci_error_detected, + .slot_reset = mt7996_pci_slot_reset, + .resume = mt7996_pci_io_resume, }; static const struct pci_error_handlers mt7996_hif_err_handler = { .error_detected = mt7996_hif_error_detected, + .slot_reset = mt7996_hif_slot_reset, + .resume = mt7996_pci_io_resume, }; struct pci_driver mt7996_hif_driver = {