From 06636fd770aad50023b415c2d8729d5af010d01f Mon Sep 17 00:00:00 2001 From: Kenneth Cheung Date: Mon, 12 Jan 2026 19:48:32 +0000 Subject: [PATCH] [202405] PCIe AER printk ratelimiting backport Backported patches that implement ratelimiting for AER report printks. This reduces log spam for devices that regularly report corrected issues on its link. kernel 6.1 version of https://github.com/sonic-net/sonic-linux-kernel/pull/520 applied to 202405. This is similar to https://github.com/sonic-net/sonic-linux-kernel/pull/521 applied to 202505. Upstream discussion: https://lore.kernel.org/linux-pci/20250522232339.1525671-1-helgaas@kernel.org/T/ Signed-off-by: Kenneth Cheung --- ...OR-UNCOR-error-handling-out-from-aer.patch | 104 +++++++ ...-Fix-NULL-pointer-access-by-aer_info.patch | 36 +++ .../0001-PCI-AER-Simplify-pci_print_aer.patch | 69 +++++ ...LL-pointer-dereference-in-aer_rateli.patch | 61 ++++ ...pdate-statistics-before-ratelimiting.patch | 68 +++++ ...race-error-event-before-ratelimiting.patch | 69 +++++ ...04-PCI-AER-Simplify-add_error_device.patch | 49 ++++ ...Check-log-level-once-and-remember-it.patch | 128 +++++++++ ...aer_get_device_error_info-aer_print_.patch | 150 ++++++++++ ...t-correctable-and-non-fatal-error-lo.patch | 221 ++++++++++++++ ...-sysfs-attributes-for-log-ratelimits.patch | 272 ++++++++++++++++++ ...-aer-ratelimiting-sysfs-output-units.patch | 35 +++ patch/series | 14 + 13 files changed, 1276 insertions(+) create mode 100644 patch/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch create mode 100644 patch/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch create mode 100644 patch/0001-PCI-AER-Simplify-pci_print_aer.patch create mode 100644 patch/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch create mode 100644 patch/0002-PCI-AER-Update-statistics-before-ratelimiting.patch create mode 100644 patch/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch create mode 100644 patch/0004-PCI-AER-Simplify-add_error_device.patch create mode 100644 patch/0005-PCI-AER-Check-log-level-once-and-remember-it.patch create mode 100644 patch/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch create mode 100644 patch/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch create mode 100644 patch/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch create mode 100644 patch/fix-aer-ratelimiting-sysfs-output-units.patch diff --git a/patch/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch b/patch/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch new file mode 100644 index 000000000..a5ec4c52c --- /dev/null +++ b/patch/0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch @@ -0,0 +1,104 @@ +From 8aec28caf011788522c1fd2dea92959477016053 Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:09 -0500 +Subject: [PATCH] PCI/AER: Factor COR/UNCOR error handling out from + aer_isr_one_error() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +aer_isr_one_error() duplicates the Error Source ID logging and AER error +processing for Correctable Errors and Uncorrectable Errors. Factor out the +duplicated code to aer_isr_one_error_type(). + +aer_isr_one_error() doesn't need the struct aer_rpc pointer, so pass it the +Root Port or RCEC pci_dev pointer instead. + +Signed-off-by: Bjorn Helgaas +Reviewed-by: Ilpo Järvinen +Reviewed-by: Jonathan Cameron +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-4-helgaas@kernel.org +(cherry picked from commit 6fc4dae74afcf29ef82afbaaa9b082893871eda4) +--- + drivers/pci/pcie/aer.c | 36 +++++++++++++++++++++++------------- + 1 file changed, 23 insertions(+), 13 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -1287,17 +1287,32 @@ static inline void aer_process_err_devices(struct aer_err_info *e_info) + } + + /** +- * aer_isr_one_error - consume an error detected by root port +- * @rpc: pointer to the root port which holds an error ++ * aer_isr_one_error_type - consume a Correctable or Uncorrectable Error ++ * detected by Root Port or RCEC ++ * @root: pointer to Root Port or RCEC that signaled AER interrupt ++ * @info: pointer to AER error info ++ */ ++static void aer_isr_one_error_type(struct pci_dev *root, ++ struct aer_err_info *info) ++{ ++ aer_print_port_info(root, info); ++ ++ if (find_source_device(root, info)) ++ aer_process_err_devices(info); ++} ++ ++/** ++ * aer_isr_one_error - consume error(s) signaled by an AER interrupt from ++ * Root Port or RCEC ++ * @root: pointer to Root Port or RCEC that signaled AER interrupt + * @e_src: pointer to an error source + */ +-static void aer_isr_one_error(struct aer_rpc *rpc, ++static void aer_isr_one_error(struct pci_dev *root, + struct aer_err_source *e_src) + { +- struct pci_dev *pdev = rpc->rpd; + struct aer_err_info e_info; + +- pci_rootport_aer_stats_incr(pdev, e_src); ++ pci_rootport_aer_stats_incr(root, e_src); + + /* + * There is a possibility that both correctable error and +@@ -1312,10 +1327,8 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + e_info.multi_error_valid = 1; + else + e_info.multi_error_valid = 0; +- aer_print_port_info(pdev, &e_info); + +- if (find_source_device(pdev, &e_info)) +- aer_process_err_devices(&e_info); ++ aer_isr_one_error_type(root, &e_info); + } + + if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { +@@ -1332,10 +1345,7 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + else + e_info.multi_error_valid = 0; + +- aer_print_port_info(pdev, &e_info); +- +- if (find_source_device(pdev, &e_info)) +- aer_process_err_devices(&e_info); ++ aer_isr_one_error_type(root, &e_info); + } + } + +@@ -1356,7 +1366,7 @@ static irqreturn_t aer_isr(int irq, void *context) + return IRQ_NONE; + + while (kfifo_get(&rpc->aer_fifo, &e_src)) +- aer_isr_one_error(rpc, &e_src); ++ aer_isr_one_error(rpc->rpd, &e_src); + return IRQ_HANDLED; + } + +-- +2.47.0 + diff --git a/patch/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch b/patch/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch new file mode 100644 index 000000000..5089ff321 --- /dev/null +++ b/patch/0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch @@ -0,0 +1,36 @@ +From cf5770619326108794a72ca7b3500ad9e3aefa90 Mon Sep 17 00:00:00 2001 +From: Vernon Yang +Date: Fri, 5 Sep 2025 02:25:27 +0800 +Subject: [PATCH 1/2] PCI/AER: Fix NULL pointer access by aer_info + +The kzalloc(GFP_KERNEL) may return NULL, so all accesses to aer_info->xxx +will result in kernel panic. Fix it. + +Signed-off-by: Vernon Yang +Signed-off-by: Bjorn Helgaas +Link: https://patch.msgid.link/20250904182527.67371-1-vernon2gm@gmail.com +(cherry picked from commit 0a27bdb14b028fed30a10cec2f945c38cb5ca4fa) +--- + drivers/pci/pcie/aer.c | 6 +++++- + 1 file changed, 5 insertions(+), 1 deletion(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -385,7 +385,11 @@ void pci_aer_init(struct pci_dev *dev) + if (!dev->aer_cap) + return; + +- dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL); ++ dev->aer_stats = kzalloc(sizeof(*dev->aer_stats), GFP_KERNEL); ++ if (!dev->aer_stats) { ++ dev->aer_cap = 0; ++ return; ++ } + + ratelimit_state_init(&dev->aer_stats->correctable_ratelimit, + DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); +-- +2.39.5 (Apple Git-154) + diff --git a/patch/0001-PCI-AER-Simplify-pci_print_aer.patch b/patch/0001-PCI-AER-Simplify-pci_print_aer.patch new file mode 100644 index 000000000..4e028f95b --- /dev/null +++ b/patch/0001-PCI-AER-Simplify-pci_print_aer.patch @@ -0,0 +1,69 @@ +From fed119ab131c202e7677ce0228d17f5cb74baa29 Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:15 -0500 +Subject: [PATCH 1/8] PCI/AER: Simplify pci_print_aer() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Simplify pci_print_aer() by initializing the struct aer_err_info "info" +with a designated initializer list (it was previously initialized with +memset()) and using pci_name(). + +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Jonathan Cameron +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-10-helgaas@kernel.org +(cherry picked from commit ad9839137cf9fb0f0c2d531bd04bc4382e6f2de9) +--- + drivers/pci/pcie/aer.c | 16 ++++++++-------- + 1 file changed, 8 insertions(+), 8 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -774,7 +774,10 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + { + int layer, agent, tlp_header_valid = 0; + u32 status, mask; +- struct aer_err_info info; ++ struct aer_err_info info = { ++ .severity = aer_severity, ++ .first_error = PCI_ERR_CAP_FEP(aer->cap_control), ++ }; + + if (aer_severity == AER_CORRECTABLE) { + status = aer->cor_status; +@@ -785,14 +788,11 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + tlp_header_valid = status & AER_LOG_TLP_MASKS; + } + +- layer = AER_GET_LAYER_ERROR(aer_severity, status); +- agent = AER_GET_AGENT(aer_severity, status); +- +- memset(&info, 0, sizeof(info)); +- info.severity = aer_severity; + info.status = status; + info.mask = mask; +- info.first_error = PCI_ERR_CAP_FEP(aer->cap_control); ++ ++ layer = AER_GET_LAYER_ERROR(aer_severity, status); ++ agent = AER_GET_AGENT(aer_severity, status); + + pci_err(dev, "aer_status: 0x%08x, aer_mask: 0x%08x\n", status, mask); + __aer_print_error(dev, &info); +@@ -806,7 +806,7 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + if (tlp_header_valid) + __print_tlp_header(dev, &aer->header_log); + +- trace_aer_event(dev_name(&dev->dev), (status & ~mask), ++ trace_aer_event(pci_name(dev), (status & ~mask), + aer_severity, tlp_header_valid, &aer->header_log); + } + #endif +-- +2.47.0 + diff --git a/patch/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch b/patch/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch new file mode 100644 index 000000000..b196154f0 --- /dev/null +++ b/patch/0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch @@ -0,0 +1,61 @@ +From 23d8218139853dda49859e3041f17111cdc47400 Mon Sep 17 00:00:00 2001 +From: Breno Leitao +Date: Mon, 29 Sep 2025 02:15:47 -0700 +Subject: [PATCH 2/2] PCI/AER: Avoid NULL pointer dereference in + aer_ratelimit() + +When platform firmware supplies error information to the OS, e.g., via the +ACPI APEI GHES mechanism, it may identify an error source device that +doesn't advertise an AER Capability and therefore dev->aer_info, which +contains AER stats and ratelimiting data, is NULL. + +pci_dev_aer_stats_incr() already checks dev->aer_info for NULL, but +aer_ratelimit() did not, leading to NULL pointer dereferences like this one +from the URL below: + + {1}[Hardware Error]: Hardware error from APEI Generic Hardware Error Source: 0 + {1}[Hardware Error]: event severity: corrected + {1}[Hardware Error]: device_id: 0000:00:00.0 + {1}[Hardware Error]: vendor_id: 0x8086, device_id: 0x2020 + {1}[Hardware Error]: aer_cor_status: 0x00001000, aer_cor_mask: 0x00002000 + BUG: kernel NULL pointer dereference, address: 0000000000000264 + RIP: 0010:___ratelimit+0xc/0x1b0 + pci_print_aer+0x141/0x360 + aer_recover_work_func+0xb5/0x130 + +[8086:2020] is an Intel "Sky Lake-E DMI3 Registers" device that claims to +be a Root Port but does not advertise an AER Capability. + +Add a NULL check in aer_ratelimit() to avoid the NULL pointer dereference. +Note that this also prevents ratelimiting these events from GHES. + +Fixes: a57f2bfb4a5863 ("PCI/AER: Ratelimit correctable and non-fatal error logging") +Link: https://lore.kernel.org/r/buduna6darbvwfg3aogl5kimyxkggu3n4romnmq6sozut6axeu@clnx7sfsy457/ +Signed-off-by: Breno Leitao +[bhelgaas: add crash details to commit log] +Signed-off-by: Bjorn Helgaas +Reviewed-by: Kuppuswamy Sathyanarayanan +Cc: stable@vger.kernel.org +Link: https://patch.msgid.link/20250929-aer_crash_2-v1-1-68ec4f81c356@debian.org +(cherry picked from commit deb2f228388ff3a9d0623e3b59a053e9235c341d) +--- + drivers/pci/pcie/aer.c | 3 +++ + 1 file changed, 3 insertions(+) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -800,6 +800,9 @@ static void __print_tlp_header(struct pci_dev *dev, + + static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) + { ++ if (!dev->aer_stats) ++ return 1; ++ + switch (severity) { + case AER_NONFATAL: + return __ratelimit(&dev->aer_stats->nonfatal_ratelimit); +-- +2.39.5 (Apple Git-154) + diff --git a/patch/0002-PCI-AER-Update-statistics-before-ratelimiting.patch b/patch/0002-PCI-AER-Update-statistics-before-ratelimiting.patch new file mode 100644 index 000000000..60d97218f --- /dev/null +++ b/patch/0002-PCI-AER-Update-statistics-before-ratelimiting.patch @@ -0,0 +1,68 @@ +From 8bbcbe849d91b64da2850797482bf0915772898b Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:16 -0500 +Subject: [PATCH 2/8] PCI/AER: Update statistics before ratelimiting +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +There are two AER logging entry points: + + - aer_print_error() is used by DPC (dpc_process_error()) and native AER + handling (aer_process_err_devices()). + + - pci_print_aer() is used by GHES (aer_recover_work_func()) and CXL + (cxl_handle_rdport_errors()) + +Both use __aer_print_error() to print the AER error bits. Previously +__aer_print_error() also incremented the AER statistics via +pci_dev_aer_stats_incr(). + +Call pci_dev_aer_stats_incr() early in the entry points instead of in +__aer_print_error() so we update the statistics even if the actual printing +of error bits is rate limited by a future change. + +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Reviewed-by: Jonathan Cameron +Link: https://patch.msgid.link/20250522232339.1525671-11-helgaas@kernel.org +(cherry picked from commit 88a7765e62b9e4c79c7ca2c7b749ae04f54a5668) +--- + drivers/pci/pcie/aer.c | 5 ++++- + 1 file changed, 4 insertions(+), 1 deletion(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -703,7 +703,6 @@ static void __aer_print_error(struct pci_dev *dev, + pci_printk(level, dev, " [%2d] %-22s%s\n", i, errmsg, + info->first_error == i ? " (First)" : ""); + } +- pci_dev_aer_stats_incr(dev, info); + } + + void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) +@@ -712,6 +711,8 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + int id = ((dev->bus->number << 8) | dev->devfn); + const char *level; + ++ pci_dev_aer_stats_incr(dev, info); ++ + if (!info->status) { + pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", + aer_error_severity_string[info->severity]); +@@ -791,6 +792,8 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + info.status = status; + info.mask = mask; + ++ pci_dev_aer_stats_incr(dev, &info); ++ + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); + +-- +2.47.0 + diff --git a/patch/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch b/patch/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch new file mode 100644 index 000000000..9b2defc83 --- /dev/null +++ b/patch/0003-PCI-AER-Trace-error-event-before-ratelimiting.patch @@ -0,0 +1,69 @@ +From 9232650afd9ebf430cdf85f75bf1bf4ecce03c0f Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:17 -0500 +Subject: [PATCH 3/8] PCI/AER: Trace error event before ratelimiting +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +As with the AER statistics, we always want to emit trace events, even if +the actual dmesg logging is rate limited. + +Call trace_aer_event() immediately after pci_dev_aer_stats_incr() so both +happen before ratelimiting. + +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-12-helgaas@kernel.org +(cherry picked from commit 6bb4befbd65fa7f99688fb707e376637e5acfe36) +--- + drivers/pci/pcie/aer.c | 10 ++++------ + 1 file changed, 4 insertions(+), 6 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -712,6 +712,8 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + const char *level; + + pci_dev_aer_stats_incr(dev, info); ++ trace_aer_event(pci_name(dev), (info->status & ~info->mask), ++ info->severity, info->tlp_header_valid, &info->tlp); + + if (!info->status) { + pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", +@@ -739,9 +741,6 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + out: + if (info->id && info->error_dev_num > 1 && info->id == id) + pci_err(dev, " Error of this Agent is reported first\n"); +- +- trace_aer_event(dev_name(&dev->dev), (info->status & ~info->mask), +- info->severity, info->tlp_header_valid, &info->tlp); + } + + static void aer_print_port_info(struct pci_dev *dev, struct aer_err_info *info) +@@ -793,6 +792,8 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + info.mask = mask; + + pci_dev_aer_stats_incr(dev, &info); ++ trace_aer_event(pci_name(dev), (status & ~mask), ++ aer_severity, tlp_header_valid, &aer->header_log); + + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); +@@ -808,9 +809,6 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + + if (tlp_header_valid) + __print_tlp_header(dev, &aer->header_log); +- +- trace_aer_event(pci_name(dev), (status & ~mask), +- aer_severity, tlp_header_valid, &aer->header_log); + } + #endif + +-- +2.47.0 + diff --git a/patch/0004-PCI-AER-Simplify-add_error_device.patch b/patch/0004-PCI-AER-Simplify-add_error_device.patch new file mode 100644 index 000000000..3cf6cd40e --- /dev/null +++ b/patch/0004-PCI-AER-Simplify-add_error_device.patch @@ -0,0 +1,49 @@ +From 68b5f15d70cabfc8c9706f313d51fb82f766e4d6 Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:23 -0500 +Subject: [PATCH 4/8] PCI/AER: Simplify add_error_device() +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Return -ENOSPC error early so the usual path through add_error_device() is +the straightline code. + +Signed-off-by: Bjorn Helgaas +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-18-helgaas@kernel.org +(cherry picked from commit d72bae423004aa7b4d94c34a7fd0b48b64305a08) +--- + drivers/pci/pcie/aer.c | 15 +++++++++------ + 1 file changed, 9 insertions(+), 6 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -819,12 +819,15 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + */ + static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) + { +- if (e_info->error_dev_num < AER_MAX_MULTI_ERR_DEVICES) { +- e_info->dev[e_info->error_dev_num] = pci_dev_get(dev); +- e_info->error_dev_num++; +- return 0; +- } +- return -ENOSPC; ++ int i = e_info->error_dev_num; ++ ++ if (i >= AER_MAX_MULTI_ERR_DEVICES) ++ return -ENOSPC; ++ ++ e_info->dev[i] = pci_dev_get(dev); ++ e_info->error_dev_num++; ++ ++ return 0; + } + + /** +-- +2.47.0 + diff --git a/patch/0005-PCI-AER-Check-log-level-once-and-remember-it.patch b/patch/0005-PCI-AER-Check-log-level-once-and-remember-it.patch new file mode 100644 index 000000000..c04606d49 --- /dev/null +++ b/patch/0005-PCI-AER-Check-log-level-once-and-remember-it.patch @@ -0,0 +1,128 @@ +From 826b9ac84463a15f4cb8e9cdb82ded0695fc5b4a Mon Sep 17 00:00:00 2001 +From: Karolina Stolarek +Date: Thu, 22 May 2025 18:21:18 -0500 +Subject: [PATCH 5/8] PCI/AER: Check log level once and remember it +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +When reporting an AER error, we check its type multiple times to determine +the log level for each message. Do this check only in the top-level +functions (aer_isr_one_error(), pci_print_aer()) and save the level in +struct aer_err_info. + +[bhelgaas: save log level in struct aer_err_info instead of passing it +as a parameter] + +Signed-off-by: Karolina Stolarek +Signed-off-by: Bjorn Helgaas +Tested-by: Krzysztof Wilczyński +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Reviewed-by: Jonathan Cameron +Link: https://patch.msgid.link/20250522232339.1525671-13-helgaas@kernel.org +(cherry picked from commit c8f6791e33a7757025285db26f3b382cdcb7f7cd) +--- + drivers/pci/pci.h | 1 + + drivers/pci/pcie/aer.c | 18 +++++++++--------- + drivers/pci/pcie/dpc.c | 1 + + 3 files changed, 11 insertions(+), 9 deletions(-) + +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -374,6 +374,7 @@ static inline bool pci_dev_is_added(const struct pci_dev *dev) + struct aer_err_info { + struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES]; + int error_dev_num; ++ const char *level; /* printk level */ + + unsigned int id:16; + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -684,16 +684,14 @@ static void __aer_print_error(struct pci_dev *dev, + { + const char **strings; + unsigned long status = info->status & ~info->mask; +- const char *level, *errmsg; ++ const char *level = info->level; ++ const char *errmsg; + int i; + +- if (info->severity == AER_CORRECTABLE) { ++ if (info->severity == AER_CORRECTABLE) + strings = aer_correctable_error_string; +- level = KERN_WARNING; +- } else { ++ else + strings = aer_uncorrectable_error_string; +- level = KERN_ERR; +- } + + for_each_set_bit(i, &status, 32) { + errmsg = strings[i]; +@@ -709,7 +707,7 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + { + int layer, agent; + int id = ((dev->bus->number << 8) | dev->devfn); +- const char *level; ++ const char *level = info->level; + + pci_dev_aer_stats_incr(dev, info); + trace_aer_event(pci_name(dev), (info->status & ~info->mask), +@@ -724,8 +722,6 @@ void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) + layer = AER_GET_LAYER_ERROR(info->severity, info->status); + agent = AER_GET_AGENT(info->severity, info->status); + +- level = (info->severity == AER_CORRECTABLE) ? KERN_WARNING : KERN_ERR; +- + pci_printk(level, dev, "PCIe Bus Error: severity=%s, type=%s, (%s)\n", + aer_error_severity_string[info->severity], + aer_error_layer[layer], aer_agent_string[agent]); +@@ -782,9 +778,11 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + if (aer_severity == AER_CORRECTABLE) { + status = aer->cor_status; + mask = aer->cor_mask; ++ info.level = KERN_WARNING; + } else { + status = aer->uncor_status; + mask = aer->uncor_mask; ++ info.level = KERN_ERR; + tlp_header_valid = status & AER_LOG_TLP_MASKS; + } + +@@ -1147,6 +1145,7 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + if (e_src->status & PCI_ERR_ROOT_COR_RCV) { + e_info.id = ERR_COR_ID(e_src->id); + e_info.severity = AER_CORRECTABLE; ++ e_info.level = KERN_WARNING; + + if (e_src->status & PCI_ERR_ROOT_MULTI_COR_RCV) + e_info.multi_error_valid = 1; +@@ -1160,6 +1159,7 @@ static void aer_isr_one_error(struct aer_rpc *rpc, + + if (e_src->status & PCI_ERR_ROOT_UNCOR_RCV) { + e_info.id = ERR_UNCOR_ID(e_src->id); ++ e_info.level = KERN_ERR; + + if (e_src->status & PCI_ERR_ROOT_FATAL_RCV) + e_info.severity = AER_FATAL; +diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/dpc.c ++++ b/drivers/pci/pcie/dpc.c +@@ -257,6 +257,7 @@ static int dpc_get_aer_uncorrect_severity(struct pci_dev *dev, + else + info->severity = AER_NONFATAL; + ++ info->level = KERN_ERR; + return 1; + } + +-- +2.47.0 + diff --git a/patch/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch b/patch/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch new file mode 100644 index 000000000..a9d0b6e80 --- /dev/null +++ b/patch/0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch @@ -0,0 +1,150 @@ +From bf2793977c116b8125fab34b1ce9136eaa057e7e Mon Sep 17 00:00:00 2001 +From: Bjorn Helgaas +Date: Thu, 22 May 2025 18:21:22 -0500 +Subject: [PATCH 6/8] PCI/AER: Convert aer_get_device_error_info(), + aer_print_error() to index +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Previously aer_get_device_error_info() and aer_print_error() took a pointer +to struct aer_err_info and a pointer to a pci_dev. Typically the pci_dev +was one of the elements of the aer_err_info.dev[] array (DPC was an +exception, where the dev[] array was unused). + +Convert aer_get_device_error_info() and aer_print_error() to take an index +into the aer_err_info.dev[] array instead. A future patch will add +per-device ratelimit information, so the index makes it convenient to find +the ratelimit associated with the device. + +To accommodate DPC, set info->dev[0] to the DPC port before using these +interfaces. + +Signed-off-by: Bjorn Helgaas +Reviewed-by: Ilpo Järvinen +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-17-helgaas@kernel.org +(cherry picked from commit 94bc15c3484aa1dbbd01aee8f9eaa5dc347a01a8) +--- + drivers/pci/pci.h | 5 +++-- + drivers/pci/pcie/aer.c | 33 +++++++++++++++++++++++---------- + drivers/pci/pcie/dpc.c | 8 ++++++-- + 3 files changed, 32 insertions(+), 14 deletions(-) + +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -391,8 +391,9 @@ struct aer_err_info { + struct aer_header_log_regs tlp; /* TLP Header */ + }; + +-int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info); +-void aer_print_error(struct pci_dev *dev, struct aer_err_info *info); ++int aer_get_device_error_info(struct aer_err_info *info, int i); ++void aer_print_error(struct aer_err_info *info, int i); ++ + #endif /* CONFIG_PCIEAER */ + + #ifdef CONFIG_PCIEPORTBUS +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -703,12 +703,18 @@ static void __aer_print_error(struct pci_dev *dev, + } + } + +-void aer_print_error(struct pci_dev *dev, struct aer_err_info *info) ++void aer_print_error(struct aer_err_info *info, int i) + { +- int layer, agent; +- int id = ((dev->bus->number << 8) | dev->devfn); ++ struct pci_dev *dev; ++ int layer, agent, id; + const char *level = info->level; + ++ if (WARN_ON_ONCE(i >= AER_MAX_MULTI_ERR_DEVICES)) ++ return; ++ ++ dev = info->dev[i]; ++ id = pci_dev_id(dev); ++ + pci_dev_aer_stats_incr(dev, info); + trace_aer_event(pci_name(dev), (info->status & ~info->mask), + info->severity, info->tlp_header_valid, &info->tlp); +@@ -1049,19 +1055,26 @@ EXPORT_SYMBOL_GPL(aer_recover_queue); + + /** + * aer_get_device_error_info - read error status from dev and store it to info +- * @dev: pointer to the device expected to have a error record + * @info: pointer to structure to store the error record ++ * @i: index into info->dev[] + * + * Return 1 on success, 0 on error. + * + * Note that @info is reused among all error devices. Clear fields properly. + */ +-int aer_get_device_error_info(struct pci_dev *dev, struct aer_err_info *info) ++int aer_get_device_error_info(struct aer_err_info *info, int i) + { +- int type = pci_pcie_type(dev); +- int aer = dev->aer_cap; ++ struct pci_dev *dev; ++ int type, aer; + int temp; + ++ if (i >= AER_MAX_MULTI_ERR_DEVICES) ++ return 0; ++ ++ dev = info->dev[i]; ++ aer = dev->aer_cap; ++ type = pci_pcie_type(dev); ++ + /* Must reset in this function */ + info->status = 0; + info->tlp_header_valid = 0; +@@ -1116,11 +1129,11 @@ static inline void aer_process_err_devices(struct aer_err_info *e_info) + + /* Report all before handle them, not to lost records by reset etc. */ + for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { +- if (aer_get_device_error_info(e_info->dev[i], e_info)) +- aer_print_error(e_info->dev[i], e_info); ++ if (aer_get_device_error_info(e_info, i)) ++ aer_print_error(e_info, i); + } + for (i = 0; i < e_info->error_dev_num && e_info->dev[i]; i++) { +- if (aer_get_device_error_info(e_info->dev[i], e_info)) ++ if (aer_get_device_error_info(e_info, i)) + handle_error_source(e_info->dev[i], e_info); + } + } +diff --git a/drivers/pci/pcie/dpc.c b/drivers/pci/pcie/dpc.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/dpc.c ++++ b/drivers/pci/pcie/dpc.c +@@ -258,6 +258,10 @@ static int dpc_get_aer_uncorrect_severity(struct pci_dev *dev, + info->severity = AER_NONFATAL; + + info->level = KERN_ERR; ++ ++ info->dev[0] = dev; ++ info->error_dev_num = 1; ++ + return 1; + } + +@@ -287,8 +291,8 @@ void dpc_process_error(struct pci_dev *pdev) + dpc_process_rp_pio_error(pdev); + else if (reason == 0 && + dpc_get_aer_uncorrect_severity(pdev, &info) && +- aer_get_device_error_info(pdev, &info)) { +- aer_print_error(pdev, &info); ++ aer_get_device_error_info(&info, 0)) { ++ aer_print_error(&info, 0); + pci_aer_clear_nonfatal_status(pdev); + pci_aer_clear_fatal_status(pdev); + } +-- +2.39.5 + diff --git a/patch/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch b/patch/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch new file mode 100644 index 000000000..6a8c2f75c --- /dev/null +++ b/patch/0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch @@ -0,0 +1,221 @@ +From 8a2d949e1349ea20ce77a1a42920f67c39597e19 Mon Sep 17 00:00:00 2001 +From: Jon Pan-Doh +Date: Thu, 22 May 2025 18:21:24 -0500 +Subject: [PATCH 7/8] PCI/AER: Ratelimit correctable and non-fatal error + logging + +Spammy devices can flood kernel logs with AER errors and slow/stall +execution. Add per-device ratelimits for AER correctable and non-fatal +uncorrectable errors that use the kernel defaults (10 per 5s). Logging of +fatal errors is not ratelimited. + +There are two AER logging entry points: + + - aer_print_error() is used by DPC and native AER + + - pci_print_aer() is used by GHES and CXL + +The native AER aer_print_error() case includes a loop that may log details +from multiple devices, which are ratelimited individually. If we log +details for any device, we also log the Error Source ID from the Root Port +or RCEC. + +If no such device details are found, we still log the Error Source from the +ERR_* Message, ratelimited by the Root Port or RCEC that received it. + +The DPC aer_print_error() case is not ratelimited, since this only happens +for fatal errors. + +The CXL pci_print_aer() case is ratelimited by the Error Source device. + +The GHES pci_print_aer() case is via aer_recover_work_func(), which +searches for the Error Source device. If the device is not found, there's +no per-device ratelimit, so we use a system-wide ratelimit that covers all +error types (correctable, non-fatal, and fatal). + +Sargun at Meta reported internally that a flood of AER errors causes RCU +CPU stall warnings and CSD-lock warnings. + +Tested using aer-inject[1]. Sent 11 AER errors. Observed 10 errors logged +while AER stats (cat /sys/bus/pci/devices//aer_dev_correctable) show +true count of 11. + +[1] https://git.kernel.org/pub/scm/linux/kernel/git/gong.chen/aer-inject.git + +[bhelgaas: commit log, factor out trace_aer_event() and aer_print_rp_info() +changes to previous patches, enable Error Source logging if any downstream +detail will be printed, don't ratelimit fatal errors, "aer_report" -> +"aer_info", "cor_log_ratelimit" -> "correctable_ratelimit", +"uncor_log_ratelimit" -> "nonfatal_ratelimit"] + +Reported-by: Sargun Dhillon +Signed-off-by: Jon Pan-Doh +Signed-off-by: Bjorn Helgaas +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-19-helgaas@kernel.org +(cherry picked from commit a57f2bfb4a5863f83087867c0e671f2418212d23) +--- + drivers/pci/pci.h | 4 +++- + drivers/pci/pcie/aer.c | 47 +++++++++++++++++++++++++++++++++++++++--- + 2 files changed, 47 insertions(+), 4 deletions(-) + +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -373,13 +373,15 @@ static inline bool pci_dev_is_added(const struct pci_dev *dev) + + struct aer_err_info { + struct pci_dev *dev[AER_MAX_MULTI_ERR_DEVICES]; ++ int ratelimit_print[AER_MAX_MULTI_ERR_DEVICES]; + int error_dev_num; + const char *level; /* printk level */ + + unsigned int id:16; + + unsigned int severity:2; /* 0:NONFATAL | 1:FATAL | 2:COR */ +- unsigned int __pad1:5; ++ unsigned int root_ratelimit_print:1; /* 0=skip, 1=print */ ++ unsigned int __pad1:4; + unsigned int multi_error_valid:1; + + unsigned int first_error:5; +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -27,6 +27,7 @@ + #include + #include + #include ++#include + #include + #include + #include +@@ -83,6 +84,10 @@ struct aer_stats { + u64 rootport_total_cor_errs; + u64 rootport_total_fatal_errs; + u64 rootport_total_nonfatal_errs; ++ ++ /* Ratelimits for errors */ ++ struct ratelimit_state correctable_ratelimit; ++ struct ratelimit_state nonfatal_ratelimit; + }; + + #define AER_LOG_TLP_MASKS (PCI_ERR_UNC_POISON_TLP| \ +@@ -382,6 +387,11 @@ void pci_aer_init(struct pci_dev *dev) + + dev->aer_stats = kzalloc(sizeof(struct aer_stats), GFP_KERNEL); + ++ ratelimit_state_init(&dev->aer_stats->correctable_ratelimit, ++ DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); ++ ratelimit_state_init(&dev->aer_stats->nonfatal_ratelimit, ++ DEFAULT_RATELIMIT_INTERVAL, DEFAULT_RATELIMIT_BURST); ++ + /* + * We save/restore PCI_ERR_UNCOR_MASK, PCI_ERR_UNCOR_SEVER, + * PCI_ERR_COR_MASK, and PCI_ERR_CAP. Root and Root Complex Event +@@ -679,6 +689,18 @@ static void __print_tlp_header(struct pci_dev *dev, + t->dw0, t->dw1, t->dw2, t->dw3); + } + ++static int aer_ratelimit(struct pci_dev *dev, unsigned int severity) ++{ ++ switch (severity) { ++ case AER_NONFATAL: ++ return __ratelimit(&dev->aer_stats->nonfatal_ratelimit); ++ case AER_CORRECTABLE: ++ return __ratelimit(&dev->aer_stats->correctable_ratelimit); ++ default: ++ return 1; /* Don't ratelimit fatal errors */ ++ } ++} ++ + static void __aer_print_error(struct pci_dev *dev, + struct aer_err_info *info) + { +@@ -720,6 +742,9 @@ void aer_print_error(struct aer_err_info *info, int i) + trace_aer_event(pci_name(dev), (info->status & ~info->mask), + info->severity, info->tlp_header_valid, &info->tlp); + ++ if (!info->ratelimit_print[i]) ++ return; ++ + if (!info->status) { + pci_err(dev, "PCIe Bus Error: severity=%s, type=Inaccessible, (Unregistered Agent ID)\n", + aer_error_severity_string[info->severity]); +@@ -800,6 +825,9 @@ void cper_print_aer(struct pci_dev *dev, int aer_severity, + trace_aer_event(pci_name(dev), (status & ~mask), + aer_severity, tlp_header_valid, &aer->header_log); + ++ if (!aer_ratelimit(dev, info.severity)) ++ return; ++ + layer = AER_GET_LAYER_ERROR(aer_severity, status); + agent = AER_GET_AGENT(aer_severity, status); + +@@ -832,6 +860,18 @@ static int add_error_device(struct aer_err_info *e_info, struct pci_dev *dev) + e_info->dev[i] = pci_dev_get(dev); + e_info->error_dev_num++; + ++ /* ++ * Ratelimit AER log messages. "dev" is either the source ++ * identified by the root's Error Source ID or it has an unmasked ++ * error logged in its own AER Capability. Messages are emitted ++ * when "ratelimit_print[i]" is non-zero. If we will print detail ++ * for a downstream device, make sure we print the Error Source ID ++ * from the root as well. ++ */ ++ if (aer_ratelimit(dev, e_info->severity)) { ++ e_info->ratelimit_print[i] = 1; ++ e_info->root_ratelimit_print = 1; ++ } + return 0; + } + +@@ -1009,9 +1049,10 @@ static void aer_recover_work_func(struct work_struct *work) + pdev = pci_get_domain_bus_and_slot(entry.domain, entry.bus, + entry.devfn); + if (!pdev) { +- pr_err("no pci_dev for %04x:%02x:%02x.%x\n", +- entry.domain, entry.bus, +- PCI_SLOT(entry.devfn), PCI_FUNC(entry.devfn)); ++ pr_err_ratelimited("%04x:%02x:%02x.%x: no pci_dev found\n", ++ entry.domain, entry.bus, ++ PCI_SLOT(entry.devfn), ++ PCI_FUNC(entry.devfn)); + continue; + } + cper_print_aer(pdev, entry.severity, entry.regs); +@@ -1147,9 +1147,26 @@ static inline void aer_process_err_devices(struct aer_err_info *e_info) + static void aer_isr_one_error_type(struct pci_dev *root, + struct aer_err_info *info) + { +- aer_print_port_info(root, info); ++ bool found; + +- if (find_source_device(root, info)) ++ found = find_source_device(root, info); ++ ++ /* ++ * If we're going to log error messages, we've already set ++ * "info->root_ratelimit_print" and "info->ratelimit_print[i]" to ++ * non-zero (which enables printing) because this is either an ++ * ERR_FATAL or we found a device with an error logged in its AER ++ * Capability. ++ * ++ * If we didn't find the Error Source device, at least log the ++ * Requester ID from the ERR_* Message received by the Root Port or ++ * RCEC, ratelimited by the RP or RCEC. ++ */ ++ if (info->root_ratelimit_print || ++ (!found && aer_ratelimit(root, info->severity))) ++ aer_print_port_info(root, info); ++ ++ if (found) + aer_process_err_devices(info); + } + +-- +2.47.0 + diff --git a/patch/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch b/patch/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch new file mode 100644 index 000000000..49060f895 --- /dev/null +++ b/patch/0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch @@ -0,0 +1,272 @@ +From c687a25894edec70b2e663447704c10ba2f58a87 Mon Sep 17 00:00:00 2001 +From: Jon Pan-Doh +Date: Thu, 22 May 2025 18:21:26 -0500 +Subject: [PATCH 8/8] PCI/AER: Add sysfs attributes for log ratelimits + +Allow userspace to read/write log ratelimits per device (including +enable/disable). Create aer/ sysfs directory to store them and any +future AER configs. + +The new sysfs files are: + + /sys/bus/pci/devices/*/aer/correctable_ratelimit_burst + /sys/bus/pci/devices/*/aer/correctable_ratelimit_interval_ms + /sys/bus/pci/devices/*/aer/nonfatal_ratelimit_burst + /sys/bus/pci/devices/*/aer/nonfatal_ratelimit_interval_ms + +The default values are ratelimit_burst=10, ratelimit_interval_ms=5000, so +if we try to emit more than 10 messages in a 5 second period, some are +suppressed. + +Update AER sysfs ABI filename to reflect the broader scope of AER sysfs +attributes (e.g. stats and ratelimits). + + Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats -> + sysfs-bus-pci-devices-aer + +Tested using aer-inject[1]. Configured correctable log ratelimit to 5. +Sent 6 AER errors. Observed 5 errors logged while AER stats +(cat /sys/bus/pci/devices//aer_dev_correctable) shows 6. + +Disabled ratelimiting and sent 6 more AER errors. Observed all 6 errors +logged and accounted in AER stats (12 total errors). + +[1] https://git.kernel.org/pub/scm/linux/kernel/git/gong.chen/aer-inject.git + +[bhelgaas: note fatal errors are not ratelimited, "aer_report" -> +"aer_info", replace ratelimit_log_enable toggle with *_ratelimit_interval_ms] + +Signed-off-by: Karolina Stolarek +Signed-off-by: Jon Pan-Doh +Signed-off-by: Bjorn Helgaas +Reviewed-by: Kuppuswamy Sathyanarayanan +Link: https://patch.msgid.link/20250522232339.1525671-21-helgaas@kernel.org +(cherry picked from commit b4fe7398def6df344442b884d2288f80205cbd2d) +--- + ...es-aer_stats => sysfs-bus-pci-devices-aer} | 44 ++++++++ + Documentation/PCI/pcieaer-howto.rst | 5 +- + drivers/pci/pci-sysfs.c | 1 + + drivers/pci/pci.h | 1 + + drivers/pci/pcie/aer.c | 105 ++++++++++++++++++ + 5 files changed, 155 insertions(+), 1 deletion(-) + rename Documentation/ABI/testing/{sysfs-bus-pci-devices-aer_stats => sysfs-bus-pci-devices-aer} (72%) + +diff --git a/Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats b/Documentation/ABI/testing/sysfs-bus-pci-devices-aer +similarity index 72% +rename from Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats +rename to Documentation/ABI/testing/sysfs-bus-pci-devices-aer +index 01234567..89abcdef 100644 +--- a/Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats ++++ b/Documentation/ABI/testing/sysfs-bus-pci-devices-aer +@@ -117,3 +117,47 @@ Date: July 2018 + KernelVersion: 4.19.0 + Contact: linux-pci@vger.kernel.org, rajatja@google.com + Description: Total number of ERR_NONFATAL messages reported to rootport. ++ ++PCIe AER ratelimits ++------------------- ++ ++These attributes show up under all the devices that are AER capable. ++They represent configurable ratelimits of logs per error type. ++ ++See Documentation/PCI/pcieaer-howto.rst for more info on ratelimits. ++ ++What: /sys/bus/pci/devices//aer/correctable_ratelimit_interval_ms ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Writing 0 disables AER correctable error log ratelimiting. ++ Writing a positive value sets the ratelimit interval in ms. ++ Default is DEFAULT_RATELIMIT_INTERVAL (5000 ms). ++ ++What: /sys/bus/pci/devices//aer/correctable_ratelimit_burst ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Ratelimit burst for correctable error logs. Writing a value ++ changes the number of errors (burst) allowed per interval ++ before ratelimiting. Reading gets the current ratelimit ++ burst. Default is DEFAULT_RATELIMIT_BURST (10). ++ ++What: /sys/bus/pci/devices//aer/nonfatal_ratelimit_interval_ms ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Writing 0 disables AER non-fatal uncorrectable error log ++ ratelimiting. Writing a positive value sets the ratelimit ++ interval in ms. Default is DEFAULT_RATELIMIT_INTERVAL ++ (5000 ms). ++ ++What: /sys/bus/pci/devices//aer/nonfatal_ratelimit_burst ++Date: May 2025 ++KernelVersion: 6.16.0 ++Contact: linux-pci@vger.kernel.org ++Description: Ratelimit burst for non-fatal uncorrectable error logs. ++ Writing a value changes the number of errors (burst) ++ allowed per interval before ratelimiting. Reading gets the ++ current ratelimit burst. Default is DEFAULT_RATELIMIT_BURST ++ (10). +diff --git a/Documentation/PCI/pcieaer-howto.rst b/Documentation/PCI/pcieaer-howto.rst +index 01234567..89abcdef 100644 +--- a/Documentation/PCI/pcieaer-howto.rst ++++ b/Documentation/PCI/pcieaer-howto.rst +@@ -86,12 +86,15 @@ In the example, 'Requester ID' means the ID of the device who sends + the error message to root port. Pls. refer to pci express specs for + other fields. + ++Ratelimits are exposed in the form of sysfs attributes and configurable. ++See Documentation/ABI/testing/sysfs-bus-pci-devices-aer. ++ + AER Statistics / Counters + ------------------------- + + When PCIe AER errors are captured, the counters / statistics are also exposed + in the form of sysfs attributes which are documented at +-Documentation/ABI/testing/sysfs-bus-pci-devices-aer_stats ++Documentation/ABI/testing/sysfs-bus-pci-devices-aer. + + Developer Guide + =============== +diff --git a/drivers/pci/pci-sysfs.c b/drivers/pci/pci-sysfs.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci-sysfs.c ++++ b/drivers/pci/pci-sysfs.c +@@ -1670,6 +1670,7 @@ static const struct attribute_group *pci_dev_attr_groups[] = { + &pcie_dev_attr_group, + #ifdef CONFIG_PCIEAER + &aer_stats_attr_group, ++ &aer_attr_group, + #endif + #ifdef CONFIG_PCIEASPM + &aspm_ctrl_attr_group, +diff --git a/drivers/pci/pci.h b/drivers/pci/pci.h +index 01234567..89abcdef 100644 +--- a/drivers/pci/pci.h ++++ b/drivers/pci/pci.h +@@ -673,6 +673,7 @@ void pci_no_aer(void); + void pci_aer_init(struct pci_dev *dev); + void pci_aer_exit(struct pci_dev *dev); + extern const struct attribute_group aer_stats_attr_group; ++extern const struct attribute_group aer_attr_group; + void pci_aer_clear_fatal_status(struct pci_dev *dev); + int pci_aer_clear_status(struct pci_dev *dev); + int pci_aer_raw_clear_status(struct pci_dev *dev); +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -630,6 +630,111 @@ const struct attribute_group aer_stats_attr_group = { + .is_visible = aer_stats_attrs_are_visible, + }; + ++/* ++ * Ratelimit interval ++ * <=0: disabled with ratelimit.interval = 0 ++ * >0: enabled with ratelimit.interval in ms ++ */ ++#define aer_ratelimit_interval_attr(name, ratelimit) \ ++ static ssize_t \ ++ name##_show(struct device *dev, struct device_attribute *attr, \ ++ char *buf) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ \ ++ return sysfs_emit(buf, "%d\n", \ ++ pdev->aer_stats->ratelimit.interval); \ ++ } \ ++ \ ++ static ssize_t \ ++ name##_store(struct device *dev, struct device_attribute *attr, \ ++ const char *buf, size_t count) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ int interval; \ ++ \ ++ if (!capable(CAP_SYS_ADMIN)) \ ++ return -EPERM; \ ++ \ ++ if (kstrtoint(buf, 0, &interval) < 0) \ ++ return -EINVAL; \ ++ \ ++ if (interval <= 0) \ ++ interval = 0; \ ++ else \ ++ interval = msecs_to_jiffies(interval); \ ++ \ ++ pdev->aer_stats->ratelimit.interval = interval; \ ++ \ ++ return count; \ ++ } \ ++ static DEVICE_ATTR_RW(name); ++ ++#define aer_ratelimit_burst_attr(name, ratelimit) \ ++ static ssize_t \ ++ name##_show(struct device *dev, struct device_attribute *attr, \ ++ char *buf) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ \ ++ return sysfs_emit(buf, "%d\n", \ ++ pdev->aer_stats->ratelimit.burst); \ ++ } \ ++ \ ++ static ssize_t \ ++ name##_store(struct device *dev, struct device_attribute *attr, \ ++ const char *buf, size_t count) \ ++ { \ ++ struct pci_dev *pdev = to_pci_dev(dev); \ ++ int burst; \ ++ \ ++ if (!capable(CAP_SYS_ADMIN)) \ ++ return -EPERM; \ ++ \ ++ if (kstrtoint(buf, 0, &burst) < 0) \ ++ return -EINVAL; \ ++ \ ++ pdev->aer_stats->ratelimit.burst = burst; \ ++ \ ++ return count; \ ++ } \ ++ static DEVICE_ATTR_RW(name); ++ ++#define aer_ratelimit_attrs(name) \ ++ aer_ratelimit_interval_attr(name##_ratelimit_interval_ms, \ ++ name##_ratelimit) \ ++ aer_ratelimit_burst_attr(name##_ratelimit_burst, \ ++ name##_ratelimit) ++ ++aer_ratelimit_attrs(correctable) ++aer_ratelimit_attrs(nonfatal) ++ ++static struct attribute *aer_attrs[] = { ++ &dev_attr_correctable_ratelimit_interval_ms.attr, ++ &dev_attr_correctable_ratelimit_burst.attr, ++ &dev_attr_nonfatal_ratelimit_interval_ms.attr, ++ &dev_attr_nonfatal_ratelimit_burst.attr, ++ NULL ++}; ++ ++static umode_t aer_attrs_are_visible(struct kobject *kobj, ++ struct attribute *a, int n) ++{ ++ struct device *dev = kobj_to_dev(kobj); ++ struct pci_dev *pdev = to_pci_dev(dev); ++ ++ if (!pdev->aer_stats) ++ return 0; ++ ++ return a->mode; ++} ++ ++const struct attribute_group aer_attr_group = { ++ .name = "aer", ++ .attrs = aer_attrs, ++ .is_visible = aer_attrs_are_visible, ++}; ++ + static void pci_dev_aer_stats_incr(struct pci_dev *pdev, + struct aer_err_info *info) + { +-- +2.47.0 + diff --git a/patch/fix-aer-ratelimiting-sysfs-output-units.patch b/patch/fix-aer-ratelimiting-sysfs-output-units.patch new file mode 100644 index 000000000..5cb928713 --- /dev/null +++ b/patch/fix-aer-ratelimiting-sysfs-output-units.patch @@ -0,0 +1,35 @@ +From 02209e4c38f2471d74e82775272096379c44a68f Mon Sep 17 00:00:00 2001 +From: jamessewart +Date: Mon, 10 Nov 2025 13:45:50 +0000 +Subject: [PATCH 4/4] fix aer ratelimiting sysfs output units + +aer statistics include an interval value that determines the time period over which +to limit error logging. This is stored in jiffies, but the sysfs file configuring it +is supposed to expose milliseconds, the store function converts the input to jiffies +and this change ensures the show implementation converts back to milliseconds + +Signed-off-by: James Sewart + +--- + drivers/pci/pcie/aer.c | 5 +++-- + 1 file changed, 3 insertions(+), 2 deletions(-) + +diff --git a/drivers/pci/pcie/aer.c b/drivers/pci/pcie/aer.c +index 01234567..89abcdef 100644 +--- a/drivers/pci/pcie/aer.c ++++ b/drivers/pci/pcie/aer.c +@@ -642,8 +642,9 @@ const struct attribute_group aer_stats_attr_group = { + { \ + struct pci_dev *pdev = to_pci_dev(dev); \ + \ +- return sysfs_emit(buf, "%d\n", \ +- pdev->aer_stats->ratelimit.interval); \ ++ int interval = pdev->aer_stats->ratelimit.interval; \ ++ interval = jiffies_to_msecs(interval); \ ++ return sysfs_emit(buf, "%d\n", interval); \ + } \ + \ + static ssize_t \ +-- +2.39.5 + diff --git a/patch/series b/patch/series index f721dcecb..3a260a96d 100755 --- a/patch/series +++ b/patch/series @@ -219,6 +219,20 @@ cisco-npu-disable-other-bars.patch # https://github.com/sonic-net/sonic-buildimage/issues/20901 revert-456d8aa-to-fix-pcie_aspm_exit_link_status.patch +# PCIe AER printk ratelimiting +0001-PCI-AER-Simplify-pci_print_aer.patch +0002-PCI-AER-Update-statistics-before-ratelimiting.patch +0003-PCI-AER-Trace-error-event-before-ratelimiting.patch +0004-PCI-AER-Simplify-add_error_device.patch +0005-PCI-AER-Check-log-level-once-and-remember-it.patch +0006-PCI-AER-Convert-aer_get_device_error_info-aer_print_.patch +0001-PCI-AER-Factor-COR-UNCOR-error-handling-out-from-aer.patch +0007-PCI-AER-Ratelimit-correctable-and-non-fatal-error-lo.patch +0008-PCI-AER-Add-sysfs-attributes-for-log-ratelimits.patch +fix-aer-ratelimiting-sysfs-output-units.patch +0001-PCI-AER-Fix-NULL-pointer-access-by-aer_info.patch +0002-PCI-AER-Avoid-NULL-pointer-dereference-in-aer_rateli.patch + # # ############################################################