Subject: Re: [PATCH 2/6] powerpc/eeh: Check PCIe link after reset

From: Benjamin Herrenschmidt <[email-masked]>
Date: Tue, 25 Jun 2013 21:58:40 +1000

On Tue, 2013-06-25 at 18:00 +0800, Gavin Shan wrote:
> After reset (e.g. complete reset) in order to bring the fenced PHB
> back, the PCIe link might not be ready yet. The patch intends to
> make sure the PCIe link is ready before accessing its subordinate
> PCI devices. The patch also fixes that wrong values restored to
> PCI_COMMAND register for PCI bridges.

This should also help if we end up doing a full reset for ER cases
right ?

IE, in a setup with PHB -> device (no switch), if the device driver
requests a fundamental reset, we should do a PERST at the PHB level (are
we ?) and thus restore things in a similar way.

> Signed-off-by: Gavin Shan <[email-masked]>
> ---
>  arch/powerpc/kernel/eeh_pe.c |  157 ++++++++++++++++++++++++++++++++++++++----
>  1 files changed, 144 insertions(+), 13 deletions(-)
> 
> diff --git a/arch/powerpc/kernel/eeh_pe.c b/arch/powerpc/kernel/eeh_pe.c
> index 55943fc..016588a 100644
> --- a/arch/powerpc/kernel/eeh_pe.c
> +++ b/arch/powerpc/kernel/eeh_pe.c
> @@ -22,6 +22,7 @@
>   * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307 USA
>   */
>  
> +#include <linux/delay.h>
>  #include <linux/export.h>
>  #include <linux/gfp.h>
>  #include <linux/init.h>
> @@ -567,30 +568,132 @@ void eeh_pe_state_clear(struct eeh_pe *pe, int state)
>  	eeh_pe_traverse(pe, __eeh_pe_state_clear, &state);
>  }
>  
> -/**
> - * eeh_restore_one_device_bars - Restore the Base Address Registers for one device
> - * @data: EEH device
> - * @flag: Unused
> +/*
> + * Some PCI bridges (e.g. PLX bridges) have primary/secondary
> + * buses assigned explicitly by firmware, and we probably have
> + * lost that after reset. So we have to delay the check until
> + * the PCI-CFG registers have been restored for the parent
> + * bridge.
>   *
> - * Loads the PCI configuration space base address registers,
> - * the expansion ROM base address, the latency timer, and etc.
> - * from the saved values in the device node.
> + * Don't use normal PCI-CFG accessors, which probably has been
> + * blocked on normal path during the stage. So we need utilize
> + * eeh operations, which is always permitted.
>   */
> -static void *eeh_restore_one_device_bars(void *data, void *flag)
> +static void eeh_bridge_check_link(struct pci_dev *pdev,
> +				  struct device_node *dn)
> +{
> +	int cap;
> +	uint32_t val;
> +	int timeout = 0;
> +
> +	/*
> +	 * We only check root port and downstream ports of
> +	 * PCIe switches
> +	 */
> +	if (!pci_is_pcie(pdev) ||
> +	    (pci_pcie_type(pdev) != PCI_EXP_TYPE_ROOT_PORT &&
> +	     pci_pcie_type(pdev) != PCI_EXP_TYPE_DOWNSTREAM))
> +		return;
> +
> +	pr_debug("%s: Check PCIe link for %s ...\n",
> +		 __func__, pci_name(pdev));
> +
> +	/* Check slot status */
> +	cap = pdev->pcie_cap;
> +	eeh_ops->read_config(dn, cap + PCI_EXP_SLTSTA, 2, &val);
> +	if (!(val & PCI_EXP_SLTSTA_PDS)) {
> +		pr_debug("  No card in the slot (0x%04x) !\n", val);
> +		return;
> +	}
> +
> +	/* Check power status if we have the capability */
> +	eeh_ops->read_config(dn, cap + PCI_EXP_SLTCAP, 2, &val);
> +	if (val & PCI_EXP_SLTCAP_PCP) {
> +		eeh_ops->read_config(dn, cap + PCI_EXP_SLTCTL, 2, &val);
> +		if (val & PCI_EXP_SLTCTL_PCC) {
> +			pr_debug("  In power-off state, power it on ...\n");
> +			val &= ~(PCI_EXP_SLTCTL_PCC | PCI_EXP_SLTCTL_PIC);
> +			val |= (0x0100 & PCI_EXP_SLTCTL_PIC);
> +			eeh_ops->write_config(dn, cap + PCI_EXP_SLTCTL, 2, val);
> +			msleep(2 * 1000);
> +		}
> +	}
> +
> +	/* Enable link */
> +	eeh_ops->read_config(dn, cap + PCI_EXP_LNKCTL, 2, &val);
> +	val &= ~PCI_EXP_LNKCTL_LD;
> +	eeh_ops->write_config(dn, cap + PCI_EXP_LNKCTL, 2, val);
> +
> +	/* Check link */
> +	eeh_ops->read_config(dn, cap + PCI_EXP_LNKCAP, 4, &val);
> +	if (!(val & PCI_EXP_LNKCAP_DLLLARC)) {
> +		pr_debug("  No link reporting capability (0x%08x) \n", val);
> +		msleep(1000);
> +		return;
> +	}
> +
> +	/* Wait the link is up until timeout (5s) */
> +	timeout = 0;
> +	while (timeout < 5000) {
> +		msleep(20);
> +		timeout += 20;
> +
> +		eeh_ops->read_config(dn, cap + PCI_EXP_LNKSTA, 2, &val);
> +		if (val & PCI_EXP_LNKSTA_DLLLA)
> +			break;
> +	}
> +
> +	if (val & PCI_EXP_LNKSTA_DLLLA)
> +		pr_debug("  Link up (%s)\n",
> +			 (val & PCI_EXP_LNKSTA_CLS_2_5GB) ? "2.5GB" : "5GB");
> +	else
> +		pr_debug("  Link not ready (0x%04x)\n", val);
> +}
> +
> +#define BYTE_SWAP(OFF)	(8*((OFF)/4)+3-(OFF))
> +#define SAVED_BYTE(OFF)	(((u8 *)(edev->config_space))[BYTE_SWAP(OFF)])
> +
> +static void eeh_restore_bridge_bars(struct pci_dev *pdev,
> +				    struct eeh_dev *edev,
> +				    struct device_node *dn)
> +{
> +	int i;
> +
> +	/*
> +	 * Device BARs: 0x10 - 0x18
> +	 * Bus numbers and windows: 0x18 - 0x30
> +	 */
> +	for (i = 4; i < 13; i++)
> +		eeh_ops->write_config(dn, i*4, 4, edev->config_space[i]);
> +	/* Rom: 0x38 */
> +	eeh_ops->write_config(dn, 14*4, 4, edev->config_space[14]);
> +
> +	/* Cache line & Latency timer: 0xC 0xD */
> +	eeh_ops->write_config(dn, PCI_CACHE_LINE_SIZE, 1,
> +                SAVED_BYTE(PCI_CACHE_LINE_SIZE));
> +        eeh_ops->write_config(dn, PCI_LATENCY_TIMER, 1,
> +                SAVED_BYTE(PCI_LATENCY_TIMER));
> +	/* Max latency, min grant, interrupt ping and line: 0x3C */
> +	eeh_ops->write_config(dn, 15*4, 4, edev->config_space[15]);
> +
> +	/* PCI Command: 0x4 */
> +	eeh_ops->write_config(dn, PCI_COMMAND, 4, edev->config_space[1]);
> +
> +	/* Check the PCIe link is ready */
> +	eeh_bridge_check_link(pdev, dn);
> +}
> +
> +static void eeh_restore_device_bars(struct eeh_dev *edev,
> +				    struct device_node *dn)
>  {
>  	int i;
>  	u32 cmd;
> -	struct eeh_dev *edev = (struct eeh_dev *)data;
> -	struct device_node *dn = eeh_dev_to_of_node(edev);
>  
>  	for (i = 4; i < 10; i++)
>  		eeh_ops->write_config(dn, i*4, 4, edev->config_space[i]);
>  	/* 12 == Expansion ROM Address */
>  	eeh_ops->write_config(dn, 12*4, 4, edev->config_space[12]);
>  
> -#define BYTE_SWAP(OFF) (8*((OFF)/4)+3-(OFF))
> -#define SAVED_BYTE(OFF) (((u8 *)(edev->config_space))[BYTE_SWAP(OFF)])
> -
>  	eeh_ops->write_config(dn, PCI_CACHE_LINE_SIZE, 1,
>  		SAVED_BYTE(PCI_CACHE_LINE_SIZE));
>  	eeh_ops->write_config(dn, PCI_LATENCY_TIMER, 1,
> @@ -613,6 +716,34 @@ static void *eeh_restore_one_device_bars(void *data, void *flag)
>  	else
>  		cmd &= ~PCI_COMMAND_SERR;
>  	eeh_ops->write_config(dn, PCI_COMMAND, 4, cmd);
> +}
> +
> +/**
> + * eeh_restore_one_device_bars - Restore the Base Address Registers for one device
> + * @data: EEH device
> + * @flag: Unused
> + *
> + * Loads the PCI configuration space base address registers,
> + * the expansion ROM base address, the latency timer, and etc.
> + * from the saved values in the device node.
> + */
> +static void *eeh_restore_one_device_bars(void *data, void *flag)
> +{
> +	struct pci_dev *pdev = NULL;
> +	struct eeh_dev *edev = (struct eeh_dev *)data;
> +	struct device_node *dn = eeh_dev_to_of_node(edev);
> +
> +	/* Trace the PCI bridge */
> +	if (eeh_probe_mode_dev()) {
> +		pdev = eeh_dev_to_pci_dev(edev);
> +		if (pdev->hdr_type != PCI_HEADER_TYPE_BRIDGE)
> +                        pdev = NULL;
> +        }
> +
> +	if (pdev)
> +		eeh_restore_bridge_bars(pdev, edev, dn);
> +	else
> +		eeh_restore_device_bars(edev, dn);
>  
>  	return NULL;
>  }