Subject: Re: [PATCH v3 03/10] VAS: Define vas_init() and vas_exit()

From: Michael Neuling <[email-masked]>
Date: Fri, 24 Mar 2017 15:45:05 +1100

On Thu, 2017-03-16 at 20:33 -0700, Sukadev Bhattiprolu wrote:
> Implement vas_init() and vas_exit() functions for a new VAS module.
> This VAS module is essentially a library for other device drivers
> and kernel users of the NX coprocessors like NX-842 and NX-GZIP.
> 
> Signed-off-by: Sukadev Bhattiprolu <[email-masked]>
> ---
> Changelog[v3]:
> 	- Zero vas_instances memory on allocation
> 	- [Haren Myneni] Fix description in Kconfig
> Changelog[v2]:
> 	- Get HVWC, UWC and window address parameters from device tree.
> ---
>  MAINTAINERS                     |   8 ++-
>  arch/powerpc/include/asm/reg.h  |   1 +
>  drivers/misc/Kconfig            |   1 +
>  drivers/misc/Makefile           |   1 +
>  drivers/misc/vas/Kconfig        |  21 ++++++
>  drivers/misc/vas/Makefile       |   3 +
>  drivers/misc/vas/vas-internal.h |   3 +
>  drivers/misc/vas/vas-window.c   |  19 +++++
>  drivers/misc/vas/vas.c          | 155
> ++++++++++++++++++++++++++++++++++++++++
>  9 files changed, 210 insertions(+), 2 deletions(-)
>  create mode 100644 drivers/misc/vas/Kconfig
>  create mode 100644 drivers/misc/vas/Makefile
>  create mode 100644 drivers/misc/vas/vas-window.c
>  create mode 100644 drivers/misc/vas/vas.c
> 
> diff --git a/MAINTAINERS b/MAINTAINERS
> index 2a910c9..4037252 100644
> --- a/MAINTAINERS
> +++ b/MAINTAINERS
> @@ -3673,8 +3673,6 @@ F:	arch/powerpc/platforms/powernv/pci-cxl.c
>  F:	drivers/misc/cxl/
>  F:	include/misc/cxl*
>  F:	include/uapi/misc/cxl.h
> -F:	Documentation/powerpc/cxl.txt
> -F:	Documentation/ABI/testing/sysfs-class-cxl

err?

>  CXLFLASH (IBM Coherent Accelerator Processor Interface CAPI Flash) SCSI
> DRIVER
>  M:	Manoj N. Kumar <[email-masked]>
> @@ -3686,6 +3684,12 @@ F:	drivers/scsi/cxlflash/
>  F:	include/uapi/scsi/cxlflash_ioctls.h
>  F:	Documentation/powerpc/cxlflash.txt
>  
> +VAS (IBM Virtual Accelerator Switch) DRIVER
> +M:	Sukadev Bhattiprolu <[email-masked]>
> +L:	[email-masked]
> +S:	Supported
> +F:	drivers/misc/vas/
> +

This was already added in patch 1.

>  STMMAC ETHERNET DRIVER
>  M:	Giuseppe Cavallaro <[email-masked]>
>  M:	Alexandre Torgue <[email-masked]>
> diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h
> index fc879fd..7a45ff7 100644
> --- a/arch/powerpc/include/asm/reg.h
> +++ b/arch/powerpc/include/asm/reg.h
> @@ -1225,6 +1225,7 @@
>  #define PVR_POWER8E	0x004B
>  #define PVR_POWER8NVL	0x004C
>  #define PVR_POWER8	0x004D
> +#define PVR_POWER9	0x004E

Can you send this up separately?  

>  #define PVR_BE		0x0070
>  #define PVR_PA6T	0x0090
>  
> diff --git a/drivers/misc/Kconfig b/drivers/misc/Kconfig
> index c290990..97d652e 100644
> --- a/drivers/misc/Kconfig
> +++ b/drivers/misc/Kconfig
> @@ -783,4 +783,5 @@ source "drivers/misc/mic/Kconfig"
>  source "drivers/misc/genwqe/Kconfig"
>  source "drivers/misc/echo/Kconfig"
>  source "drivers/misc/cxl/Kconfig"
> +source "drivers/misc/vas/Kconfig"
>  endmenu
> diff --git a/drivers/misc/Makefile b/drivers/misc/Makefile
> index 7a3ea89..5201ffd 100644
> --- a/drivers/misc/Makefile
> +++ b/drivers/misc/Makefile
> @@ -53,6 +53,7 @@ obj-$(CONFIG_GENWQE)		+= genwqe/
>  obj-$(CONFIG_ECHO)		+= echo/
>  obj-$(CONFIG_VEXPRESS_SYSCFG)	+= vexpress-syscfg.o
>  obj-$(CONFIG_CXL_BASE)		+= cxl/
> +obj-$(CONFIG_VAS)		+= vas/
>  obj-$(CONFIG_PANEL)             += panel.o
>  
>  lkdtm-$(CONFIG_LKDTM)		+= lkdtm_core.o
> diff --git a/drivers/misc/vas/Kconfig b/drivers/misc/vas/Kconfig
> new file mode 100644
> index 0000000..43cedda
> --- /dev/null
> +++ b/drivers/misc/vas/Kconfig
> @@ -0,0 +1,21 @@
> +#
> +# IBM Virtual Accelarator Switchboard (VAS) compatible devices
> +#depends on PPC_POWERNV && PCI_MSI && EEH
> +#
> +
> +config VAS
> +	tristate "Support for IBM Virtual Accelerator Switchboard (VAS)"
> +	depends on PPC_POWERNV
> +	default n
> +	help
> +	  Select this option to enable driver support for IBM Virtual
> +	  Accelerator Switchboard (VAS).
> +
> +	  VAS allows accelerators in co processors like NX-842 to be
> +	  directly available to a user process. This driver enables
> +	  userspace programs to access these accelerators via device
> +	  nodes like /dev/crypto/nx-gzip.

I though this was kernel only users for now?

> +
> +	  VAS adapters are found in POWER9 based systems.
> +
> +	  If unsure, say N.
> diff --git a/drivers/misc/vas/Makefile b/drivers/misc/vas/Makefile
> new file mode 100644
> index 0000000..7dd7139
> --- /dev/null
> +++ b/drivers/misc/vas/Makefile
> @@ -0,0 +1,3 @@
> +ccflags-y			:= $(call cc-disable-warning, unused-const-
> variable)
> +ccflags-$(CONFIG_PPC_WERROR)	+= -Werror
> +obj-$(CONFIG_VAS)		+= vas.o vas-window.o
> diff --git a/drivers/misc/vas/vas-internal.h b/drivers/misc/vas/vas-internal.h
> index ce48f14..15b62e0 100644
> --- a/drivers/misc/vas/vas-internal.h
> +++ b/drivers/misc/vas/vas-internal.h

Just call this file vas.h. 

Internal one is here.  Kernel api one lives in include/misc.  User API lives in
./include/uapi/misc.  

eg. CXL does this:

% find . -name cxl.h
./include/uapi/misc/cxl.h
./include/misc/cxl.h
./drivers/misc/cxl/cxl.h

> @@ -389,4 +389,7 @@ struct vas_winctx {
>  	enum vas_notify_after_count notify_after_count;
>  };
>  
> +extern int vas_initialized;

I would probably make this a function.

> +extern int vas_window_reset(struct vas_instance *vinst, int winid);
> +extern struct vas_instance *find_vas_instance(int vasid);
>  #endif
> diff --git a/drivers/misc/vas/vas-window.c b/drivers/misc/vas/vas-window.c
> new file mode 100644
> index 0000000..468f3bf
> --- /dev/null
> +++ b/drivers/misc/vas/vas-window.c
> @@ -0,0 +1,19 @@
> +/*
> + * Copyright 2016 IBM Corp.
> + *
> + * This program is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU General Public License
> + * as published by the Free Software Foundation; either version
> + * 2 of the License, or (at your option) any later version.
> + */
> +
> +#include <linux/types.h>
> +#include <linux/mutex.h>
> +#include <asm/vas.h>
> +#include "vas-internal.h"
> +
> +/* stub for now */
> +int vas_window_reset(struct vas_instance *vinst, int winid)
> +{
> +	return 0;

No biggy, but best to return an error if it's just a stub in case sometime tries
to use it.

> +}
> diff --git a/drivers/misc/vas/vas.c b/drivers/misc/vas/vas.c
> new file mode 100644
> index 0000000..75e55f9
> --- /dev/null
> +++ b/drivers/misc/vas/vas.c
> @@ -0,0 +1,155 @@
> +/*
> + * Copyright 2016 IBM Corp.
> + *
> + * This program is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU General Public License
> + * as published by the Free Software Foundation; either version
> + * 2 of the License, or (at your option) any later version.
> + */
> +
> +#include <linux/module.h>
> +#include <linux/kernel.h>
> +#include <linux/export.h>
> +#include <linux/types.h>
> +#include <linux/slab.h>
> +#include <linux/of.h>
> +#include <asm/vas.h>
> +#include "vas-internal.h"
> +
> +int vas_initialized;
> +int vas_num_instances;
> +struct vas_instance *vas_instances;
> +
> +static void init_vas_chip(struct vas_instance *vinst)
> +{
> +	int i;
> +
> +	for (i = 0; i < VAS_MAX_WINDOWS_PER_CHIP; i++)
> +		vas_window_reset(vinst, i);
> +}
> +
> +static int init_vas_instance(struct device_node *dn,
> +				struct vas_instance *vinst)
> +{
> +	int rc;
> +	const __be32 *p;
> +
> +	ida_init(&vinst->ida);
> +	mutex_init(&vinst->mutex);
> +
> +	p = of_get_property(dn, "vas-id", NULL);
> +	if (!p) {
> +		pr_err("VAS: NULL vas-id? %p\n", p);
> +		return -ENODEV;
> +	}
> +
> +	vinst->vas_id = of_read_number(p, 1);
> +
> +	rc = of_property_read_u64(dn, "hvwc-bar-start", &vinst-
> >hvwc_bar_start);
> +	if (rc)
> +		return rc;
> +
> +	rc = of_property_read_u64(dn, "hvwc-bar-size", &vinst->hvwc_bar_len);
> +	if (rc)
> +		return rc;
> +
> +	rc = of_property_read_u64(dn, "uwc-bar-start", &vinst-
> >uwc_bar_start);
> +	if (rc)
> +		return rc;
> +
> +	rc = of_property_read_u64(dn, "uwc-bar-size", &vinst->uwc_bar_len);
> +	if (rc)
> +		return rc;
> +
> +	rc = of_property_read_u64(dn, "window-base", &vinst->win_base_addr);
> +	if (rc)
> +		return rc;
> +
> +	rc = of_property_read_u64(dn, "window-shift", &vinst->win_id_shift);
> +	if (rc)
> +		return rc;
> +
> +	init_vas_chip(vinst);
> +
> +	return 0;
> +}
> +
> +/*
> + * Although this is read/used multiple times, it is written to only
> + * during initialization.
> + */
> +struct vas_instance *find_vas_instance(int vasid)
> +{
> +	int i;
> +	struct vas_instance *vinst;
> +
> +	for (i = 0; i < vas_num_instances; i++) {
> +		vinst = &vas_instances[i];
> +		if (vinst->vas_id == vasid)
> +			return vinst;
> +	}
> +	pr_err("VAS instance for vas-id %d not found\n", vasid);
> +	WARN_ON_ONCE(1);
> +	return NULL;
> +}
> +
> +
> +int vas_init(void)
> +{
> +	int rc;
> +	struct device_node *dn;
> +	struct vas_instance *vinst;
> +
> +	if (!pvr_version_is(PVR_POWER9))
> +		return -ENODEV;
> +
> +	vas_num_instances = 0;
> +	for_each_node_by_name(dn, "vas")
> +		vas_num_instances++;
> +
> +	if (!vas_num_instances)
> +		return -ENODEV;
> +
> +	vas_instances = kcalloc(vas_num_instances, sizeof(*vinst),
> GFP_KERNEL);
> +	if (!vas_instances)
> +		return -ENOMEM;
> +
> +	vinst = &vas_instances[0];
> +	for_each_node_by_name(dn, "vas") {
> +		rc = init_vas_instance(dn, vinst);
> +		if (rc) {
> +			pr_err("Error %d initializing VAS instance %ld\n",
> rc,
> +					(vinst-&vas_instances[0]));
> +			goto cleanup;
> +		}
> +		vinst++;
> +	}
> +
> +	rc = -ENODEV;
> +	if (vinst == &vas_instances[0]) {
> +		/* Should not happen as we saw some above. */
> +		pr_err("VAS: Did not find any VAS DT nodes now!\n");
> +		goto cleanup;
> +	}
> +
> +	pr_devel("VAS: Initialized %d instances\n", vas_num_instances);
> +	vas_initialized = 1;
> +
> +	return 0;
> +
> +cleanup:
> +	kfree(vas_instances);
> +	return rc;
> +}
> +
> +void vas_exit(void)
> +{
> +	vas_initialized = 0;
> +	kfree(vas_instances);
> +}
> +
> +module_init(vas_init);
> +module_exit(vas_exit);
> +MODULE_DESCRIPTION("IBM Virtual Accelerator Switchboard");
> +MODULE_AUTHOR("Sukadev Bhattiprolu <[email-masked]>");
> +MODULE_LICENSE("GPL");