From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 16:30:29 +0800
Message-ID: <4AF924A5.1050303@gmail.com>
Reply-To: xiaosuo@gmail.com
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: 7bit
Cc: netdev@vger.kernel.org, xiaosuo <xiaosuo@gmail.com>,
	Tom Herbert <therbert@google.com>
To: "David S. Miller" <davem@davemloft.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:42998 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752549AbZKJIa3 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 03:30:29 -0500
Received: by pwi3 with SMTP id 3so1162554pwi.21
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 00:30:34 -0800 (PST)
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

ifb: add multi-queue support

Add multi-queue support, and one kernel thread is created for per queue.
It can used to emulate multi-queue NIC in software, and distribute work
among CPUs.
gentux linux # modprobe ifb numtxqs=2
gentux linux # ifconfig ifb0 up
gentux linux # pgrep ifb0
18508
18509
gentux linux # taskset -p 1 18508
pid 18508's current affinity mask: 3
pid 18508's new affinity mask: 1
gentux linux # taskset -p 2 18509
pid 18509's current affinity mask: 3
pid 18509's new affinity mask: 2
gentux linux # tc qdisc add dev br0 ingress
gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
action mirred egress redirect dev ifb0

Signed-off-by: Changli Gao <xiaosuo@gmail.com>
----
drivers/net/ifb.c | 309
++++++++++++++++++++++++++++++++----------------------
1 file changed, 186 insertions(+), 123 deletions(-)

diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 030913f..6e04188 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -33,139 +33,101 @@
 #include <linux/etherdevice.h>
 #include <linux/init.h>
 #include <linux/moduleparam.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <net/ip.h>
 #include <net/pkt_sched.h>
 #include <net/net_namespace.h>
 
-#define TX_TIMEOUT  (2*HZ)
-
 #define TX_Q_LIMIT    32
+
 struct ifb_private {
-	struct tasklet_struct   ifb_tasklet;
-	int     tasklet_pending;
-	/* mostly debug stats leave in for now */
-	unsigned long   st_task_enter; /* tasklet entered */
-	unsigned long   st_txq_refl_try; /* transmit queue refill attempt */
-	unsigned long   st_rxq_enter; /* receive queue entered */
-	unsigned long   st_rx2tx_tran; /* receive to trasmit transfers */
-	unsigned long   st_rxq_notenter; /*receiveQ not entered, resched */
-	unsigned long   st_rx_frm_egr; /* received from egress path */
-	unsigned long   st_rx_frm_ing; /* received from ingress path */
-	unsigned long   st_rxq_check;
-	unsigned long   st_rxq_rsch;
-	struct sk_buff_head     rq;
-	struct sk_buff_head     tq;
+	struct net_device	*dev;
+	struct sk_buff_head	rq;
+	struct sk_buff_head	tq;
+	wait_queue_head_t	wq;
+	struct task_struct	*task;
 };
 
+/* Number of ifb devices to be set up by this module. */
 static int numifbs = 2;
+module_param(numifbs, int, 0444);
+MODULE_PARM_DESC(numifbs, "Number of ifb devices");
 
-static void ri_tasklet(unsigned long dev);
-static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
-static int ifb_open(struct net_device *dev);
-static int ifb_close(struct net_device *dev);
+/* Number of TX queues per ifb */
+static int numtxqs = 1;
+module_param(numtxqs, int, 0444);
+MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
 
-static void ri_tasklet(unsigned long dev)
+static int ifb_thread(void *priv)
 {
-
-	struct net_device *_dev = (struct net_device *)dev;
-	struct ifb_private *dp = netdev_priv(_dev);
-	struct net_device_stats *stats = &_dev->stats;
-	struct netdev_queue *txq;
+	struct ifb_private *dp = (struct ifb_private*)priv;
+	struct net_device *dev = dp->dev;
+	struct net_device_stats *stats = &dev->stats;
+	unsigned int num = dp - (struct ifb_private*)netdev_priv(dev);
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 	struct sk_buff *skb;
-
-	txq = netdev_get_tx_queue(_dev, 0);
-	dp->st_task_enter++;
-	if ((skb = skb_peek(&dp->tq)) == NULL) {
-		dp->st_txq_refl_try++;
-		if (__netif_tx_trylock(txq)) {
-			dp->st_rxq_enter++;
-			while ((skb = skb_dequeue(&dp->rq)) != NULL) {
+	DEFINE_WAIT(wait);
+
+	while (1) {
+		/* move skb from rq to tq */
+		while (1) {
+			prepare_to_wait(&dp->wq, &wait, TASK_UNINTERRUPTIBLE);
+			while (!__netif_tx_trylock(txq))
+				yield();
+			while ((skb = skb_dequeue(&dp->rq)) != NULL)
 				skb_queue_tail(&dp->tq, skb);
-				dp->st_rx2tx_tran++;
-			}
+			if (netif_queue_stopped(dev))
+				netif_wake_queue(dev);
 			__netif_tx_unlock(txq);
-		} else {
-			/* reschedule */
-			dp->st_rxq_notenter++;
-			goto resched;
+			if (kthread_should_stop() || !skb_queue_empty(&dp->tq))
+				break;
+			schedule();
 		}
-	}
-
-	while ((skb = skb_dequeue(&dp->tq)) != NULL) {
-		u32 from = G_TC_FROM(skb->tc_verd);
-
-		skb->tc_verd = 0;
-		skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
-		stats->tx_packets++;
-		stats->tx_bytes +=skb->len;
-
-		skb->dev = dev_get_by_index(&init_net, skb->iif);
-		if (!skb->dev) {
-			dev_kfree_skb(skb);
-			stats->tx_dropped++;
+		finish_wait(&dp->wq, &wait);
+		if (kthread_should_stop())
 			break;
-		}
-		dev_put(skb->dev);
-		skb->iif = _dev->ifindex;
-
-		if (from & AT_EGRESS) {
-			dp->st_rx_frm_egr++;
-			dev_queue_xmit(skb);
-		} else if (from & AT_INGRESS) {
-			dp->st_rx_frm_ing++;
-			skb_pull(skb, skb->dev->hard_header_len);
-			netif_rx(skb);
-		} else
-			BUG();
-	}
 
-	if (__netif_tx_trylock(txq)) {
-		dp->st_rxq_check++;
-		if ((skb = skb_peek(&dp->rq)) == NULL) {
-			dp->tasklet_pending = 0;
-			if (netif_queue_stopped(_dev))
-				netif_wake_queue(_dev);
-		} else {
-			dp->st_rxq_rsch++;
-			__netif_tx_unlock(txq);
-			goto resched;
+		/* transfer packets */
+		while ((skb = skb_dequeue(&dp->tq)) != NULL) {
+			u32 from = G_TC_FROM(skb->tc_verd);
+	
+			skb->tc_verd = 0;
+			skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
+			stats->tx_packets++;
+			stats->tx_bytes +=skb->len;
+	
+			skb->dev = dev_get_by_index(&init_net, skb->iif);
+			if (!skb->dev) {
+				dev_kfree_skb(skb);
+				stats->tx_dropped++;
+				break;
+			}
+			dev_put(skb->dev);
+			skb->iif = dev->ifindex;
+	
+			if (from & AT_EGRESS) {
+				dev_queue_xmit(skb);
+			} else if (from & AT_INGRESS) {
+				skb_pull(skb, skb->dev->hard_header_len);
+				netif_rx_ni(skb);
+			} else
+				BUG();
 		}
-		__netif_tx_unlock(txq);
-	} else {
-resched:
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
 	}
 
-}
-
-static const struct net_device_ops ifb_netdev_ops = {
-	.ndo_open	= ifb_open,
-	.ndo_stop	= ifb_close,
-	.ndo_start_xmit	= ifb_xmit,
-	.ndo_validate_addr = eth_validate_addr,
-};
-
-static void ifb_setup(struct net_device *dev)
-{
-	/* Initialize the device structure. */
-	dev->destructor = free_netdev;
-	dev->netdev_ops = &ifb_netdev_ops;
-
-	/* Fill in device structure with ethernet-generic values. */
-	ether_setup(dev);
-	dev->tx_queue_len = TX_Q_LIMIT;
-
-	dev->flags |= IFF_NOARP;
-	dev->flags &= ~IFF_MULTICAST;
-	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
-	random_ether_addr(dev->dev_addr);
+	return 0;
 }
 
 static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 {
-	struct ifb_private *dp = netdev_priv(dev);
 	struct net_device_stats *stats = &dev->stats;
 	u32 from = G_TC_FROM(skb->tc_verd);
+	int num = skb_get_queue_mapping(skb);
+	struct ifb_private *dp = ((struct ifb_private*)netdev_priv(dev)) + num;
 
 	stats->rx_packets++;
 	stats->rx_bytes+=skb->len;
@@ -182,10 +144,8 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 
 	dev->trans_start = jiffies;
 	skb_queue_tail(&dp->rq, skb);
-	if (!dp->tasklet_pending) {
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
-	}
+	if (skb_queue_len(&dp->rq) == 1)
+		wake_up(&dp->wq);
 
 	return NETDEV_TX_OK;
 }
@@ -193,26 +153,132 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 static int ifb_close(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
+	int i;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		kthread_stop(dp[i].task);
+		skb_queue_purge(&dp[i].tq);
+		skb_queue_purge(&dp[i].rq);
+	}
 
-	tasklet_kill(&dp->ifb_tasklet);
 	netif_stop_queue(dev);
-	skb_queue_purge(&dp->rq);
-	skb_queue_purge(&dp->tq);
+
 	return 0;
 }
 
 static int ifb_open(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
+	int i;
+	
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		dp[i].dev = dev;
+		skb_queue_head_init(&dp[i].rq);
+		skb_queue_head_init(&dp[i].tq);
+		init_waitqueue_head(&dp[i].wq);
+		dp[i].task = kthread_run(ifb_thread, &dp[i], "%s/%d", dev->name,
+					i);
+		if (IS_ERR(dp[i].task)) {
+			int err = PTR_ERR(dp[i].task);
+			while (--i >= 0)
+				kthread_stop(dp[i].task);
+			return err;
+		}
+	}
 
-	tasklet_init(&dp->ifb_tasklet, ri_tasklet, (unsigned long)dev);
-	skb_queue_head_init(&dp->rq);
-	skb_queue_head_init(&dp->tq);
 	netif_start_queue(dev);
 
 	return 0;
 }
 
+static u32 simple_tx_hashrnd;
+
+static u16 ifb_select_queue(struct net_device *dev, struct sk_buff *skb)
+{
+	u32 addr1, addr2;
+	u32 hash, ihl;
+	union {
+		u16 in16[2];
+		u32 in32;
+	} ports;
+	u8 ip_proto;
+
+	if ((hash = skb_rx_queue_recorded(skb))) {
+		while (hash >= dev->real_num_tx_queues)
+			hash -= dev->real_num_tx_queues;
+		return hash;
+	}
+
+	switch (skb->protocol) {
+	case __constant_htons(ETH_P_IP):
+		if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)))
+			ip_proto = ip_hdr(skb)->protocol;
+		else
+			ip_proto = 0;
+		addr1 = ip_hdr(skb)->saddr;
+		addr2 = ip_hdr(skb)->daddr;
+		ihl = ip_hdr(skb)->ihl << 2;
+		break;
+	case __constant_htons(ETH_P_IPV6):
+		ip_proto = ipv6_hdr(skb)->nexthdr;
+		addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
+		addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
+		ihl = 10;
+		break;
+	default:
+		return 0;
+	}
+	if (addr1 > addr2)
+		swap(addr1, addr2);
+
+	switch (ip_proto) {
+	case IPPROTO_TCP:
+	case IPPROTO_UDP:
+	case IPPROTO_DCCP:
+	case IPPROTO_ESP:
+	case IPPROTO_AH:
+	case IPPROTO_SCTP:
+	case IPPROTO_UDPLITE:
+		ports.in32 = *((u32 *) (skb_network_header(skb) + ihl));
+		if (ports.in16[0] > ports.in16[1])
+			swap(ports.in16[0], ports.in16[1]);
+		break;
+
+	default:
+		ports.in32 = 0;
+		break;
+	}
+
+	hash = jhash_3words(addr1, addr2, ports.in32,
+			    simple_tx_hashrnd ^ ip_proto);
+
+	return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
+}
+
+static const struct net_device_ops ifb_netdev_ops = {
+	.ndo_open		= ifb_open,
+	.ndo_stop		= ifb_close,
+	.ndo_start_xmit		= ifb_xmit,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_select_queue	= ifb_select_queue,
+};
+
+static void ifb_setup(struct net_device *dev)
+{
+	/* Initialize the device structure. */
+	dev->destructor = free_netdev;
+	dev->netdev_ops = &ifb_netdev_ops;
+
+	/* Fill in device structure with ethernet-generic values. */
+	ether_setup(dev);
+	dev->tx_queue_len = TX_Q_LIMIT;
+
+	dev->flags |= IFF_NOARP;
+	dev->flags &= ~IFF_MULTICAST;
+	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+	random_ether_addr(dev->dev_addr);
+}
+
 static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 {
 	if (tb[IFLA_ADDRESS]) {
@@ -231,17 +297,13 @@ static struct rtnl_link_ops ifb_link_ops __read_mostly = {
 	.validate	= ifb_validate,
 };
 
-/* Number of ifb devices to be set up by this module. */
-module_param(numifbs, int, 0);
-MODULE_PARM_DESC(numifbs, "Number of ifb devices");
-
 static int __init ifb_init_one(int index)
 {
 	struct net_device *dev_ifb;
 	int err;
 
-	dev_ifb = alloc_netdev(sizeof(struct ifb_private),
-				 "ifb%d", ifb_setup);
+	dev_ifb = alloc_netdev_mq(sizeof(struct ifb_private) * numtxqs, "ifb%d",
+				  ifb_setup, numtxqs);
 
 	if (!dev_ifb)
 		return -ENOMEM;
@@ -266,6 +328,7 @@ static int __init ifb_init_module(void)
 {
 	int i, err;
 
+	get_random_bytes(&simple_tx_hashrnd, 4);
 	rtnl_lock();
 	err = __rtnl_link_register(&ifb_link_ops);
 



From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 10:07:57 +0100
Message-ID: <4AF92D6D.8060300@gmail.com>
References: <4AF924A5.1050303@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: xiaosuo@gmail.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:54733 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1753021AbZKJJIA (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 04:08:00 -0500
In-Reply-To: <4AF924A5.1050303@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =A8=A6crit :
> ifb: add multi-queue support
>=20
> Add multi-queue support, and one kernel thread is created for per que=
ue.
> It can used to emulate multi-queue NIC in software, and distribute wo=
rk
> among CPUs.
> gentux linux # modprobe ifb numtxqs=3D2
> gentux linux # ifconfig ifb0 up
> gentux linux # pgrep ifb0
> 18508
> 18509
> gentux linux # taskset -p 1 18508
> pid 18508's current affinity mask: 3
> pid 18508's new affinity mask: 1
> gentux linux # taskset -p 2 18509
> pid 18509's current affinity mask: 3
> pid 18509's new affinity mask: 2
> gentux linux # tc qdisc add dev br0 ingress
> gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
> action mirred egress redirect dev ifb0

Seems pretty cool !

I have some comments=20

>=20
> Signed-off-by: Changli Gao <xiaosuo@gmail.com>
> ----
> drivers/net/ifb.c | 309
> ++++++++++++++++++++++++++++++++----------------------
> 1 file changed, 186 insertions(+), 123 deletions(-)
>=20
> diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
> index 030913f..6e04188 100644
> --- a/drivers/net/ifb.c
> +++ b/drivers/net/ifb.c
> @@ -33,139 +33,101 @@
>  #include <linux/etherdevice.h>
>  #include <linux/init.h>
>  #include <linux/moduleparam.h>
> +#include <linux/wait.h>
> +#include <linux/sched.h>
> +#include <linux/kthread.h>
> +#include <linux/ip.h>
> +#include <linux/ipv6.h>
> +#include <net/ip.h>
>  #include <net/pkt_sched.h>
>  #include <net/net_namespace.h>
> =20
> -#define TX_TIMEOUT  (2*HZ)
> -
>  #define TX_Q_LIMIT    32
> +
>  struct ifb_private {
> -	struct tasklet_struct   ifb_tasklet;
> -	int     tasklet_pending;
> -	/* mostly debug stats leave in for now */
> -	unsigned long   st_task_enter; /* tasklet entered */
> -	unsigned long   st_txq_refl_try; /* transmit queue refill attempt *=
/
> -	unsigned long   st_rxq_enter; /* receive queue entered */
> -	unsigned long   st_rx2tx_tran; /* receive to trasmit transfers */
> -	unsigned long   st_rxq_notenter; /*receiveQ not entered, resched */
> -	unsigned long   st_rx_frm_egr; /* received from egress path */
> -	unsigned long   st_rx_frm_ing; /* received from ingress path */
> -	unsigned long   st_rxq_check;
> -	unsigned long   st_rxq_rsch;
> -	struct sk_buff_head     rq;
> -	struct sk_buff_head     tq;
> +	struct net_device	*dev;
> +	struct sk_buff_head	rq;
> +	struct sk_buff_head	tq;
> +	wait_queue_head_t	wq;
> +	struct task_struct	*task;
>  };

Maybe you should allocate true per_cpu structure, to avoid cache line s=
haring
and get appropriate NUMA properties.

At least use a __cacheline_aligned_in_smp ...

> =20
> +/* Number of ifb devices to be set up by this module. */
>  static int numifbs =3D 2;
> +module_param(numifbs, int, 0444);
> +MODULE_PARM_DESC(numifbs, "Number of ifb devices");
> =20
> -static void ri_tasklet(unsigned long dev);
> -static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *=
dev);
> -static int ifb_open(struct net_device *dev);
> -static int ifb_close(struct net_device *dev);
> +/* Number of TX queues per ifb */
> +static int numtxqs =3D 1;
> +module_param(numtxqs, int, 0444);
> +MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
> =20
> -static void ri_tasklet(unsigned long dev)
> +static int ifb_thread(void *priv)
>  {
> -
> -	struct net_device *_dev =3D (struct net_device *)dev;
> -	struct ifb_private *dp =3D netdev_priv(_dev);
> -	struct net_device_stats *stats =3D &_dev->stats;
> -	struct netdev_queue *txq;

> +	struct ifb_private *dp =3D (struct ifb_private*)priv;

A space is required before * : (struct ifb_private *)priv;
(in many places in your patch)

> +	struct net_device *dev =3D dp->dev;


> +	struct net_device_stats *stats =3D &dev->stats;

Here you use a net_device_stats that is shared by all your queues,
your updates wont be protected and some will be lost.
You should use txq->tx_ counters.

> +	unsigned int num =3D dp - (struct ifb_private*)netdev_priv(dev);
> +	struct netdev_queue *txq =3D netdev_get_tx_queue(dev, num);
>  	struct sk_buff *skb;
> -
> -	txq =3D netdev_get_tx_queue(_dev, 0);
> -	dp->st_task_enter++;
> -	if ((skb =3D skb_peek(&dp->tq)) =3D=3D NULL) {
> -		dp->st_txq_refl_try++;
> -		if (__netif_tx_trylock(txq)) {
> -			dp->st_rxq_enter++;
> -			while ((skb =3D skb_dequeue(&dp->rq)) !=3D NULL) {
> +	DEFINE_WAIT(wait);
> +
> +	while (1) {
> +		/* move skb from rq to tq */
> +		while (1) {
> +			prepare_to_wait(&dp->wq, &wait, TASK_UNINTERRUPTIBLE);
> +			while (!__netif_tx_trylock(txq))
> +				yield();
> +			while ((skb =3D skb_dequeue(&dp->rq)) !=3D NULL)
>  				skb_queue_tail(&dp->tq, skb);
> -				dp->st_rx2tx_tran++;
> -			}
> +			if (netif_queue_stopped(dev))
> +				netif_wake_queue(dev);
>  			__netif_tx_unlock(txq);
> -		} else {
> -			/* reschedule */
> -			dp->st_rxq_notenter++;
> -			goto resched;
> +			if (kthread_should_stop() || !skb_queue_empty(&dp->tq))
> +				break;
> +			schedule();
>  		}
> -	}
> -
> -	while ((skb =3D skb_dequeue(&dp->tq)) !=3D NULL) {
> -		u32 from =3D G_TC_FROM(skb->tc_verd);
> -
> -		skb->tc_verd =3D 0;
> -		skb->tc_verd =3D SET_TC_NCLS(skb->tc_verd);
> -		stats->tx_packets++;
> -		stats->tx_bytes +=3Dskb->len;
> -
> -		skb->dev =3D dev_get_by_index(&init_net, skb->iif);
> -		if (!skb->dev) {
> -			dev_kfree_skb(skb);
> -			stats->tx_dropped++;
> +		finish_wait(&dp->wq, &wait);
> +		if (kthread_should_stop())
>  			break;
> -		}
> -		dev_put(skb->dev);
> -		skb->iif =3D _dev->ifindex;
> -
> -		if (from & AT_EGRESS) {
> -			dp->st_rx_frm_egr++;
> -			dev_queue_xmit(skb);
> -		} else if (from & AT_INGRESS) {
> -			dp->st_rx_frm_ing++;
> -			skb_pull(skb, skb->dev->hard_header_len);
> -			netif_rx(skb);
> -		} else
> -			BUG();
> -	}
> =20
> -	if (__netif_tx_trylock(txq)) {
> -		dp->st_rxq_check++;
> -		if ((skb =3D skb_peek(&dp->rq)) =3D=3D NULL) {
> -			dp->tasklet_pending =3D 0;
> -			if (netif_queue_stopped(_dev))
> -				netif_wake_queue(_dev);
> -		} else {
> -			dp->st_rxq_rsch++;
> -			__netif_tx_unlock(txq);
> -			goto resched;
> +		/* transfer packets */
> +		while ((skb =3D skb_dequeue(&dp->tq)) !=3D NULL) {
> +			u32 from =3D G_TC_FROM(skb->tc_verd);
> +=09
> +			skb->tc_verd =3D 0;
> +			skb->tc_verd =3D SET_TC_NCLS(skb->tc_verd);

> +			stats->tx_packets++;
> +			stats->tx_bytes +=3Dskb->len;

Use :			txq->tx_packets++
			txq->tx_bytes +=3D skb->len;
> +=09
> +			skb->dev =3D dev_get_by_index(&init_net, skb->iif);
> +			if (!skb->dev) {
> +				dev_kfree_skb(skb);

> +				stats->tx_dropped++;
			txq->tx_dropped ?

> +				break;
> +			}
> +			dev_put(skb->dev);
> +			skb->iif =3D dev->ifindex;
> +=09
> +			if (from & AT_EGRESS) {
> +				dev_queue_xmit(skb);
> +			} else if (from & AT_INGRESS) {
> +				skb_pull(skb, skb->dev->hard_header_len);
> +				netif_rx_ni(skb);
> +			} else
> +				BUG();
>  		}
> -		__netif_tx_unlock(txq);
> -	} else {
> -resched:
> -		dp->tasklet_pending =3D 1;
> -		tasklet_schedule(&dp->ifb_tasklet);
>  	}
> =20
> -}
> -
> -static const struct net_device_ops ifb_netdev_ops =3D {
> -	.ndo_open	=3D ifb_open,
> -	.ndo_stop	=3D ifb_close,
> -	.ndo_start_xmit	=3D ifb_xmit,
> -	.ndo_validate_addr =3D eth_validate_addr,
> -};
> -
> -static void ifb_setup(struct net_device *dev)
> -{
> -	/* Initialize the device structure. */
> -	dev->destructor =3D free_netdev;
> -	dev->netdev_ops =3D &ifb_netdev_ops;
> -
> -	/* Fill in device structure with ethernet-generic values. */
> -	ether_setup(dev);
> -	dev->tx_queue_len =3D TX_Q_LIMIT;
> -
> -	dev->flags |=3D IFF_NOARP;
> -	dev->flags &=3D ~IFF_MULTICAST;
> -	dev->priv_flags &=3D ~IFF_XMIT_DST_RELEASE;
> -	random_ether_addr(dev->dev_addr);
> +	return 0;
>  }
> =20
>  static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *=
dev)
>  {
> -	struct ifb_private *dp =3D netdev_priv(dev);
>  	struct net_device_stats *stats =3D &dev->stats;
>  	u32 from =3D G_TC_FROM(skb->tc_verd);
> +	int num =3D skb_get_queue_mapping(skb);
> +	struct ifb_private *dp =3D ((struct ifb_private*)netdev_priv(dev)) =
+ num;
> =20

>  	stats->rx_packets++;
>  	stats->rx_bytes+=3Dskb->len;

Not sure how to solve this problem (several cpus can updates counter in=
 //)

Thanks

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 17:43:58 +0800
Message-ID: <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f188.google.com ([209.85.222.188]:40403 "EHLO
	mail-pz0-f188.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755841AbZKJJnw convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 04:43:52 -0500
Received: by pzk26 with SMTP id 26so2729162pzk.4
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 01:43:58 -0800 (PST)
In-Reply-To: <4AF92D6D.8060300@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
>
> Not sure how to solve this problem (several cpus can updates counter =
in //)
>
Thanks, and follow your suggestions. I can maintain the counter per TX
queue, and update it in a timer handler like ixgbe or implement our
own struct net_device_stats* (*ndo_get_stats)(struct net_device *dev),
and update the counters when it gets called.



--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 11:29:00 +0100
Message-ID: <4AF9406C.8010603@trash.net>
References: <4AF924A5.1050303@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=ISO-8859-15
Content-Transfer-Encoding: 7bit
Cc: "David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: xiaosuo@gmail.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:63679 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1750811AbZKJK3C (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 05:29:02 -0500
In-Reply-To: <4AF924A5.1050303@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> ifb: add multi-queue support
> 
> Add multi-queue support, and one kernel thread is created for per queue.
> It can used to emulate multi-queue NIC in software, and distribute work
> among CPUs.
> gentux linux # modprobe ifb numtxqs=2
> gentux linux # ifconfig ifb0 up
> gentux linux # pgrep ifb0
> 18508
> 18509
> gentux linux # taskset -p 1 18508
> pid 18508's current affinity mask: 3
> pid 18508's new affinity mask: 1
> gentux linux # taskset -p 2 18509
> pid 18509's current affinity mask: 3
> pid 18509's new affinity mask: 2
> gentux linux # tc qdisc add dev br0 ingress
> gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
> action mirred egress redirect dev ifb0

I'm not sure how this will help, ifb device transmission is
still serialized by the ingress queue lock and the mirred lock.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 18:48:15 +0800
Message-ID: <412e6f7f0911100248mbc14287p26c43141d6ab202c@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF9406C.8010603@trash.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:49834 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1750881AbZKJKsK convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 05:48:10 -0500
Received: by pwi3 with SMTP id 3so1218857pwi.21
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 02:48:15 -0800 (PST)
In-Reply-To: <4AF9406C.8010603@trash.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, Nov 10, 2009 at 6:29 PM, Patrick McHardy <kaber@trash.net> wrot=
e:
>
> I'm not sure how this will help, ifb device transmission is
> still serialized by the ingress queue lock and the mirred lock.
>

But not in ifb_thread. For firewall the main routine is after
netif_rx_ni(). For server applications, the packets belong to the same
flow are always routed to the same CPU, and the applications can
benefit with better localization.

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 11:55:59 +0100
Message-ID: <4AF946BF.30800@gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF9406C.8010603@trash.net> <412e6f7f0911100248mbc14287p26c43141d6ab202c@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Patrick McHardy <kaber@trash.net>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:35215 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1753222AbZKJK4B (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 05:56:01 -0500
In-Reply-To: <412e6f7f0911100248mbc14287p26c43141d6ab202c@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =C3=A9crit :
> On Tue, Nov 10, 2009 at 6:29 PM, Patrick McHardy <kaber@trash.net> wr=
ote:
>> I'm not sure how this will help, ifb device transmission is
>> still serialized by the ingress queue lock and the mirred lock.
>>
>=20
> But not in ifb_thread. For firewall the main routine is after
> netif_rx_ni(). For server applications, the packets belong to the sam=
e
> flow are always routed to the same CPU, and the applications can
> benefit with better localization.
>=20

Hmm, you know many cache lines already were bringed into cpu receiving
the original softirq ? But yes, this is a possible way to go / try :)

Please submit your future patch on top of net-next-2.6, because
I see you still use dev_get_by_index() :-(

commit 05e8689c9a3a208bf75b60662778d81e23eac460
Author: Eric Dumazet <eric.dumazet@gmail.com>
Date:   Sun Nov 1 19:45:16 2009 +0000

    ifb: RCU locking avoids touching dev refcount
   =20
    Avoids touching dev refcount in hotpath
   =20
    Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
    Signed-off-by: David S. Miller <davem@davemloft.net>

commit db519144243de6b17ff0c56c26f06059743110a7
Author: Eric Dumazet <eric.dumazet@gmail.com>
Date:   Tue Oct 20 02:35:50 2009 +0000

    ifb: should not use __dev_get_by_index() without locks
   =20
    At this point (ri_tasklet()), RTNL or dev_base_lock are not held,
    we must use dev_get_by_index() instead of __dev_get_by_index()
   =20
    Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
    Signed-off-by: David S. Miller <davem@davemloft.net>



From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 11:57:32 +0100
Message-ID: <4AF9471C.5080606@gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com> <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:35225 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751214AbZKJK5e (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 05:57:34 -0500
In-Reply-To: <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =A8=A6crit :
> 2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
>> Not sure how to solve this problem (several cpus can updates counter=
 in //)
>>
> Thanks, and follow your suggestions. I can maintain the counter per T=
X
> queue, and update it in a timer handler like ixgbe or implement our
> own struct net_device_stats* (*ndo_get_stats)(struct net_device *dev)=
,
> and update the counters when it gets called.

Please no timer stuff :)


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 19:14:32 +0800
Message-ID: <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>
	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>
	 <4AF9471C.5080606@gmail.com>
Mime-Version: 1.0
Content-Type: multipart/mixed; boundary=00504502c9fd1e34190478026ab6
Cc: "David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:44982 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1750756AbZKJLO1 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 06:14:27 -0500
Received: by pwi3 with SMTP id 3so1229630pwi.21
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 03:14:32 -0800 (PST)
In-Reply-To: <4AF9471C.5080606@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

--00504502c9fd1e34190478026ab6
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: quoted-printable

2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
> Changli Gao a =A8=A6crit :
>> 2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
>>> Not sure how to solve this problem (several cpus can updates counter in=
 //)
>>>
>> Thanks, and follow your suggestions. I can maintain the counter per TX
>> queue, and update it in a timer handler like ixgbe or implement our
>> own struct net_device_stats* (*ndo_get_stats)(struct net_device *dev),
>> and update the counters when it gets called.
>
> Please no timer stuff :)

The whole ifb.c file is attached, please review and test it. Thanks!


--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

--00504502c9fd1e34190478026ab6
Content-Type: application/octet-stream; name="ifb.c"
Content-Disposition: attachment; filename="ifb.c"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_g1ukecdw0

LyogZHJpdmVycy9uZXQvaWZiLmM6CgoJVGhlIHB1cnBvc2Ugb2YgdGhpcyBkcml2ZXIgaXMgdG8g
cHJvdmlkZSBhIGRldmljZSB0aGF0IGFsbG93cwoJZm9yIHNoYXJpbmcgb2YgcmVzb3VyY2VzOgoK
CTEpIHFkaXNjcy9wb2xpY2llcyB0aGF0IGFyZSBwZXIgZGV2aWNlIGFzIG9wcG9zZWQgdG8gc3lz
dGVtIHdpZGUuCglpZmIgYWxsb3dzIGZvciBhIGRldmljZSB3aGljaCBjYW4gYmUgcmVkaXJlY3Rl
ZCB0byB0aHVzIHByb3ZpZGluZwoJYW4gaW1wcmVzc2lvbiBvZiBzaGFyaW5nLgoKCTIpIEFsbG93
cyBmb3IgcXVldWVpbmcgaW5jb21pbmcgdHJhZmZpYyBmb3Igc2hhcGluZyBpbnN0ZWFkIG9mCglk
cm9wcGluZy4KCglUaGUgb3JpZ2luYWwgY29uY2VwdCBpcyBiYXNlZCBvbiB3aGF0IGlzIGtub3du
IGFzIHRoZSBJTVEKCWRyaXZlciBpbml0aWFsbHkgd3JpdHRlbiBieSBNYXJ0aW4gRGV2ZXJhLCBs
YXRlciByZXdyaXR0ZW4KCWJ5IFBhdHJpY2sgTWNIYXJkeSBhbmQgdGhlbiBtYWludGFpbmVkIGJ5
IEFuZHJlIENvcnJlYS4KCglZb3UgbmVlZCB0aGUgdGMgYWN0aW9uICBtaXJyb3Igb3IgcmVkaXJl
Y3QgdG8gZmVlZCB0aGlzIGRldmljZQogICAgICAgCXBhY2tldHMuCgoJVGhpcyBwcm9ncmFtIGlz
IGZyZWUgc29mdHdhcmU7IHlvdSBjYW4gcmVkaXN0cmlidXRlIGl0IGFuZC9vcgoJbW9kaWZ5IGl0
IHVuZGVyIHRoZSB0ZXJtcyBvZiB0aGUgR05VIEdlbmVyYWwgUHVibGljIExpY2Vuc2UKCWFzIHB1
Ymxpc2hlZCBieSB0aGUgRnJlZSBTb2Z0d2FyZSBGb3VuZGF0aW9uOyBlaXRoZXIgdmVyc2lvbgoJ
MiBvZiB0aGUgTGljZW5zZSwgb3IgKGF0IHlvdXIgb3B0aW9uKSBhbnkgbGF0ZXIgdmVyc2lvbi4K
CiAgCUF1dGhvcnM6CUphbWFsIEhhZGkgU2FsaW0gKDIwMDUpCgoqLwoKCiNpbmNsdWRlIDxsaW51
eC9tb2R1bGUuaD4KI2luY2x1ZGUgPGxpbnV4L2tlcm5lbC5oPgojaW5jbHVkZSA8bGludXgvbmV0
ZGV2aWNlLmg+CiNpbmNsdWRlIDxsaW51eC9ldGhlcmRldmljZS5oPgojaW5jbHVkZSA8bGludXgv
aW5pdC5oPgojaW5jbHVkZSA8bGludXgvbW9kdWxlcGFyYW0uaD4KI2luY2x1ZGUgPGxpbnV4L3dh
aXQuaD4KI2luY2x1ZGUgPGxpbnV4L3NjaGVkLmg+CiNpbmNsdWRlIDxsaW51eC9rdGhyZWFkLmg+
CiNpbmNsdWRlIDxsaW51eC9pcC5oPgojaW5jbHVkZSA8bGludXgvaXB2Ni5oPgojaW5jbHVkZSA8
bmV0L2lwLmg+CiNpbmNsdWRlIDxuZXQvcGt0X3NjaGVkLmg+CiNpbmNsdWRlIDxuZXQvbmV0X25h
bWVzcGFjZS5oPgoKI2RlZmluZSBUWF9RX0xJTUlUICAgIDMyCgpzdHJ1Y3QgaWZiX3ByaXZhdGUg
ewoJc3RydWN0IG5ldF9kZXZpY2UJKmRldjsKCXN0cnVjdCBza19idWZmX2hlYWQJcnE7CglzdHJ1
Y3Qgc2tfYnVmZl9oZWFkCXRxOwoJd2FpdF9xdWV1ZV9oZWFkX3QJd3E7CglzdHJ1Y3QgdGFza19z
dHJ1Y3QJKnRhc2s7Cgl1bnNpZ25lZCBsb25nCQlyeF9wYWNrZXRzOwoJdW5zaWduZWQgbG9uZwkJ
cnhfYnl0ZXM7Cgl1bnNpZ25lZCBsb25nCQlyeF9kcm9wcGVkOwp9IF9fX19jYWNoZWxpbmVfYWxp
Z25lZF9pbl9zbXA7CgovKiBOdW1iZXIgb2YgaWZiIGRldmljZXMgdG8gYmUgc2V0IHVwIGJ5IHRo
aXMgbW9kdWxlLiAqLwpzdGF0aWMgaW50IG51bWlmYnMgPSAyOwptb2R1bGVfcGFyYW0obnVtaWZi
cywgaW50LCAwNDQ0KTsKTU9EVUxFX1BBUk1fREVTQyhudW1pZmJzLCAiTnVtYmVyIG9mIGlmYiBk
ZXZpY2VzIik7CgovKiBOdW1iZXIgb2YgVFggcXVldWVzIHBlciBpZmIgKi8Kc3RhdGljIGludCBu
dW10eHFzID0gMTsKbW9kdWxlX3BhcmFtKG51bXR4cXMsIGludCwgMDQ0NCk7Ck1PRFVMRV9QQVJN
X0RFU0MobnVtdHhxcywgIk51bWJlciBvZiBUWCBxdWV1ZXMgcGVyIGlmYiIpOwoKc3RhdGljIGlu
dCBpZmJfdGhyZWFkKHZvaWQgKnByaXYpCnsKCXN0cnVjdCBpZmJfcHJpdmF0ZSAqZHAgPSAoc3Ry
dWN0IGlmYl9wcml2YXRlICopcHJpdjsKCXN0cnVjdCBuZXRfZGV2aWNlICpkZXYgPSBkcC0+ZGV2
OwoJdW5zaWduZWQgaW50IG51bSA9IGRwIC0gKHN0cnVjdCBpZmJfcHJpdmF0ZSAqKW5ldGRldl9w
cml2KGRldik7CglzdHJ1Y3QgbmV0ZGV2X3F1ZXVlICp0eHEgPSBuZXRkZXZfZ2V0X3R4X3F1ZXVl
KGRldiwgbnVtKTsKCXN0cnVjdCBza19idWZmICpza2I7CglERUZJTkVfV0FJVCh3YWl0KTsKCgl3
aGlsZSAoMSkgewoJCS8qIG1vdmUgc2tiIGZyb20gcnEgdG8gdHEgKi8KCQl3aGlsZSAoMSkgewoJ
CQlwcmVwYXJlX3RvX3dhaXQoJmRwLT53cSwgJndhaXQsIFRBU0tfVU5JTlRFUlJVUFRJQkxFKTsK
CQkJd2hpbGUgKCFfX25ldGlmX3R4X3RyeWxvY2sodHhxKSkKCQkJCXlpZWxkKCk7CgkJCXdoaWxl
ICgoc2tiID0gc2tiX2RlcXVldWUoJmRwLT5ycSkpICE9IE5VTEwpCgkJCQlza2JfcXVldWVfdGFp
bCgmZHAtPnRxLCBza2IpOwoJCQlpZiAobmV0aWZfcXVldWVfc3RvcHBlZChkZXYpKQoJCQkJbmV0
aWZfd2FrZV9xdWV1ZShkZXYpOwoJCQlfX25ldGlmX3R4X3VubG9jayh0eHEpOwoJCQlpZiAoa3Ro
cmVhZF9zaG91bGRfc3RvcCgpIHx8ICFza2JfcXVldWVfZW1wdHkoJmRwLT50cSkpCgkJCQlicmVh
azsKCQkJc2NoZWR1bGUoKTsKCQl9CgkJZmluaXNoX3dhaXQoJmRwLT53cSwgJndhaXQpOwoJCWlm
IChrdGhyZWFkX3Nob3VsZF9zdG9wKCkpCgkJCWJyZWFrOwoKCQkvKiB0cmFuc2ZlciBwYWNrZXRz
ICovCgkJd2hpbGUgKChza2IgPSBza2JfZGVxdWV1ZSgmZHAtPnRxKSkgIT0gTlVMTCkgewoJCQl1
MzIgZnJvbSA9IEdfVENfRlJPTShza2ItPnRjX3ZlcmQpOwoJCgkJCXNrYi0+dGNfdmVyZCA9IDA7
CgkJCXNrYi0+dGNfdmVyZCA9IFNFVF9UQ19OQ0xTKHNrYi0+dGNfdmVyZCk7CgkJCXR4cS0+dHhf
cGFja2V0cysrOwoJCQl0eHEtPnR4X2J5dGVzICs9c2tiLT5sZW47CgkKCQkJcmN1X3JlYWRfbG9j
aygpOwoJCQlza2ItPmRldiA9IGRldl9nZXRfYnlfaW5kZXhfcmN1KCZpbml0X25ldCwgc2tiLT5p
aWYpOwoJCQlpZiAoIXNrYi0+ZGV2KSB7CgkJCQlyY3VfcmVhZF91bmxvY2soKTsKCQkJCWRldl9r
ZnJlZV9za2Ioc2tiKTsKCQkJCXR4cS0+dHhfZHJvcHBlZCsrOwoJCQkJYnJlYWs7CgkJCX0KCQkJ
cmN1X3JlYWRfdW5sb2NrKCk7CgkJCXNrYi0+aWlmID0gZGV2LT5pZmluZGV4OwoJCgkJCWlmIChm
cm9tICYgQVRfRUdSRVNTKSB7CgkJCQlkZXZfcXVldWVfeG1pdChza2IpOwoJCQl9IGVsc2UgaWYg
KGZyb20gJiBBVF9JTkdSRVNTKSB7CgkJCQlza2JfcHVsbChza2IsIHNrYi0+ZGV2LT5oYXJkX2hl
YWRlcl9sZW4pOwoJCQkJbmV0aWZfcnhfbmkoc2tiKTsKCQkJfSBlbHNlCgkJCQlCVUcoKTsKCQl9
Cgl9CgoJcmV0dXJuIDA7Cn0KCnN0cnVjdCBuZXRfZGV2aWNlX3N0YXRzKiBpZmJfZ2V0X3N0YXRz
KHN0cnVjdCBuZXRfZGV2aWNlICpkZXYpCnsKCXN0cnVjdCBuZXRfZGV2aWNlX3N0YXRzICpzdGF0
cyA9ICZkZXYtPnN0YXRzOwoJc3RydWN0IGlmYl9wcml2YXRlICpkcCA9IG5ldGRldl9wcml2KGRl
dik7CglzdHJ1Y3QgbmV0ZGV2X3F1ZXVlICp0eHE7CglpbnQgaTsKCXVuc2lnbmVkIGxvbmcgcnhf
cGFja2V0cyA9IDAsIHJ4X2J5dGVzID0gMCwgcnhfZHJvcHBlZCA9IDA7Cgl1bnNpZ25lZCBsb25n
IHR4X3BhY2tldHMgPSAwLCB0eF9ieXRlcyA9IDAsIHR4X2Ryb3BwZWQgPSAwOwoKCWZvciAoaSA9
IDA7IGkgPCBkZXYtPnJlYWxfbnVtX3R4X3F1ZXVlczsgaSsrKSB7CgkJcnhfcGFja2V0cyArPSBk
cFtpXS5yeF9wYWNrZXRzOwoJCXJ4X2J5dGVzICs9IGRwW2ldLnJ4X2J5dGVzOwoJCXJ4X2Ryb3Bw
ZWQgKz0gZHBbaV0ucnhfZHJvcHBlZDsKCQl0eHEgPSBuZXRkZXZfZ2V0X3R4X3F1ZXVlKGRldiwg
aSk7CgkJdHhfcGFja2V0cyA9IHR4cS0+dHhfcGFja2V0czsKCQl0eF9ieXRlcyA9IHR4cS0+dHhf
Ynl0ZXM7CgkJdHhfZHJvcHBlZCArPSB0eHEtPnR4X2Ryb3BwZWQ7Cgl9CgoJc3RhdHMtPnJ4X3Bh
Y2tldHMgPSByeF9wYWNrZXRzOwoJc3RhdHMtPnJ4X2J5dGVzID0gcnhfYnl0ZXM7CglzdGF0cy0+
cnhfZHJvcHBlZCA9IHJ4X2Ryb3BwZWQ7CglzdGF0cy0+dHhfcGFja2V0cyA9IHR4X3BhY2tldHM7
CglzdGF0cy0+dHhfYnl0ZXMgPSB0eF9ieXRlczsKCXN0YXRzLT50eF9kcm9wcGVkID0gdHhfZHJv
cHBlZDsKCglyZXR1cm4gc3RhdHM7Cn0KCnN0YXRpYyBuZXRkZXZfdHhfdCBpZmJfeG1pdChzdHJ1
Y3Qgc2tfYnVmZiAqc2tiLCBzdHJ1Y3QgbmV0X2RldmljZSAqZGV2KQp7Cgl1MzIgZnJvbSA9IEdf
VENfRlJPTShza2ItPnRjX3ZlcmQpOwoJaW50IG51bSA9IHNrYl9nZXRfcXVldWVfbWFwcGluZyhz
a2IpOwoJc3RydWN0IGlmYl9wcml2YXRlICpkcCA9ICgoc3RydWN0IGlmYl9wcml2YXRlICopbmV0
ZGV2X3ByaXYoZGV2KSkgKyBudW07CglzdHJ1Y3QgbmV0ZGV2X3F1ZXVlICp0eHEgPSBuZXRkZXZf
Z2V0X3R4X3F1ZXVlKGRldiwgbnVtKTsKCglkcC0+cnhfcGFja2V0cysrOwoJZHAtPnJ4X2J5dGVz
Kz1za2ItPmxlbjsKCglpZiAoIShmcm9tICYgKEFUX0lOR1JFU1N8QVRfRUdSRVNTKSkgfHwgIXNr
Yi0+aWlmKSB7CgkJZGV2X2tmcmVlX3NrYihza2IpOwoJCWRwLT5yeF9kcm9wcGVkKys7CgkJcmV0
dXJuIE5FVERFVl9UWF9PSzsKCX0KCglpZiAoc2tiX3F1ZXVlX2xlbigmZHAtPnJxKSA+PSBkZXYt
PnR4X3F1ZXVlX2xlbikgewoJCW5ldGlmX3N0b3BfcXVldWUoZGV2KTsKCX0KCgl0eHEtPnRyYW5z
X3N0YXJ0ID0gamlmZmllczsKCXNrYl9xdWV1ZV90YWlsKCZkcC0+cnEsIHNrYik7CglpZiAoc2ti
X3F1ZXVlX2xlbigmZHAtPnJxKSA9PSAxKQoJCXdha2VfdXAoJmRwLT53cSk7CgoJcmV0dXJuIE5F
VERFVl9UWF9PSzsKfQoKc3RhdGljIGludCBpZmJfY2xvc2Uoc3RydWN0IG5ldF9kZXZpY2UgKmRl
dikKewoJc3RydWN0IGlmYl9wcml2YXRlICpkcCA9IG5ldGRldl9wcml2KGRldik7CglpbnQgaTsK
Cglmb3IgKGkgPSAwOyBpIDwgZGV2LT5yZWFsX251bV90eF9xdWV1ZXM7IGkrKykgewoJCWt0aHJl
YWRfc3RvcChkcFtpXS50YXNrKTsKCQlza2JfcXVldWVfcHVyZ2UoJmRwW2ldLnRxKTsKCQlza2Jf
cXVldWVfcHVyZ2UoJmRwW2ldLnJxKTsKCX0KCgluZXRpZl9zdG9wX3F1ZXVlKGRldik7CgoJcmV0
dXJuIDA7Cn0KCnN0YXRpYyBpbnQgaWZiX29wZW4oc3RydWN0IG5ldF9kZXZpY2UgKmRldikKewoJ
c3RydWN0IGlmYl9wcml2YXRlICpkcCA9IG5ldGRldl9wcml2KGRldik7CglpbnQgaTsKCQoJZm9y
IChpID0gMDsgaSA8IGRldi0+cmVhbF9udW1fdHhfcXVldWVzOyBpKyspIHsKCQlkcFtpXS5kZXYg
PSBkZXY7CgkJc2tiX3F1ZXVlX2hlYWRfaW5pdCgmZHBbaV0ucnEpOwoJCXNrYl9xdWV1ZV9oZWFk
X2luaXQoJmRwW2ldLnRxKTsKCQlpbml0X3dhaXRxdWV1ZV9oZWFkKCZkcFtpXS53cSk7CgkJZHBb
aV0udGFzayA9IGt0aHJlYWRfcnVuKGlmYl90aHJlYWQsICZkcFtpXSwgIiVzLyVkIiwgZGV2LT5u
YW1lLAoJCQkJCWkpOwoJCWlmIChJU19FUlIoZHBbaV0udGFzaykpIHsKCQkJaW50IGVyciA9IFBU
Ul9FUlIoZHBbaV0udGFzayk7CgkJCXdoaWxlICgtLWkgPj0gMCkKCQkJCWt0aHJlYWRfc3RvcChk
cFtpXS50YXNrKTsKCQkJcmV0dXJuIGVycjsKCQl9Cgl9CgoJbmV0aWZfc3RhcnRfcXVldWUoZGV2
KTsKCglyZXR1cm4gMDsKfQoKc3RhdGljIHUzMiBzaW1wbGVfdHhfaGFzaHJuZDsKCnN0YXRpYyB1
MTYgaWZiX3NlbGVjdF9xdWV1ZShzdHJ1Y3QgbmV0X2RldmljZSAqZGV2LCBzdHJ1Y3Qgc2tfYnVm
ZiAqc2tiKQp7Cgl1MzIgYWRkcjEsIGFkZHIyOwoJdTMyIGhhc2gsIGlobDsKCXVuaW9uIHsKCQl1
MTYgaW4xNlsyXTsKCQl1MzIgaW4zMjsKCX0gcG9ydHM7Cgl1OCBpcF9wcm90bzsKCglpZiAoKGhh
c2ggPSBza2JfcnhfcXVldWVfcmVjb3JkZWQoc2tiKSkpIHsKCQl3aGlsZSAoaGFzaCA+PSBkZXYt
PnJlYWxfbnVtX3R4X3F1ZXVlcykKCQkJaGFzaCAtPSBkZXYtPnJlYWxfbnVtX3R4X3F1ZXVlczsK
CQlyZXR1cm4gaGFzaDsKCX0KCglzd2l0Y2ggKHNrYi0+cHJvdG9jb2wpIHsKCWNhc2UgX19jb25z
dGFudF9odG9ucyhFVEhfUF9JUCk6CgkJaWYgKCEoaXBfaGRyKHNrYiktPmZyYWdfb2ZmICYgaHRv
bnMoSVBfTUYgfCBJUF9PRkZTRVQpKSkKCQkJaXBfcHJvdG8gPSBpcF9oZHIoc2tiKS0+cHJvdG9j
b2w7CgkJZWxzZQoJCQlpcF9wcm90byA9IDA7CgkJYWRkcjEgPSBpcF9oZHIoc2tiKS0+c2FkZHI7
CgkJYWRkcjIgPSBpcF9oZHIoc2tiKS0+ZGFkZHI7CgkJaWhsID0gaXBfaGRyKHNrYiktPmlobCA8
PCAyOwoJCWJyZWFrOwoJY2FzZSBfX2NvbnN0YW50X2h0b25zKEVUSF9QX0lQVjYpOgoJCWlwX3By
b3RvID0gaXB2Nl9oZHIoc2tiKS0+bmV4dGhkcjsKCQlhZGRyMSA9IGlwdjZfaGRyKHNrYiktPnNh
ZGRyLnM2X2FkZHIzMlszXTsKCQlhZGRyMiA9IGlwdjZfaGRyKHNrYiktPmRhZGRyLnM2X2FkZHIz
MlszXTsKCQlpaGwgPSAxMDsKCQlicmVhazsKCWRlZmF1bHQ6CgkJcmV0dXJuIDA7Cgl9CglpZiAo
YWRkcjEgPiBhZGRyMikKCQlzd2FwKGFkZHIxLCBhZGRyMik7CgoJc3dpdGNoIChpcF9wcm90bykg
ewoJY2FzZSBJUFBST1RPX1RDUDoKCWNhc2UgSVBQUk9UT19VRFA6CgljYXNlIElQUFJPVE9fREND
UDoKCWNhc2UgSVBQUk9UT19FU1A6CgljYXNlIElQUFJPVE9fQUg6CgljYXNlIElQUFJPVE9fU0NU
UDoKCWNhc2UgSVBQUk9UT19VRFBMSVRFOgoJCXBvcnRzLmluMzIgPSAqKCh1MzIgKikgKHNrYl9u
ZXR3b3JrX2hlYWRlcihza2IpICsgaWhsKSk7CgkJaWYgKHBvcnRzLmluMTZbMF0gPiBwb3J0cy5p
bjE2WzFdKQoJCQlzd2FwKHBvcnRzLmluMTZbMF0sIHBvcnRzLmluMTZbMV0pOwoJCWJyZWFrOwoK
CWRlZmF1bHQ6CgkJcG9ydHMuaW4zMiA9IDA7CgkJYnJlYWs7Cgl9CgoJaGFzaCA9IGpoYXNoXzN3
b3JkcyhhZGRyMSwgYWRkcjIsIHBvcnRzLmluMzIsCgkJCSAgICBzaW1wbGVfdHhfaGFzaHJuZCBe
IGlwX3Byb3RvKTsKCglyZXR1cm4gKHUxNikgKCgodTY0KSBoYXNoICogZGV2LT5yZWFsX251bV90
eF9xdWV1ZXMpID4+IDMyKTsKfQoKc3RhdGljIGNvbnN0IHN0cnVjdCBuZXRfZGV2aWNlX29wcyBp
ZmJfbmV0ZGV2X29wcyA9IHsKCS5uZG9fb3BlbgkJPSBpZmJfb3BlbiwKCS5uZG9fc3RvcAkJPSBp
ZmJfY2xvc2UsCgkubmRvX3N0YXJ0X3htaXQJCT0gaWZiX3htaXQsCgkubmRvX3ZhbGlkYXRlX2Fk
ZHIJPSBldGhfdmFsaWRhdGVfYWRkciwKCS5uZG9fc2VsZWN0X3F1ZXVlCT0gaWZiX3NlbGVjdF9x
dWV1ZSwKCS5uZG9fZ2V0X3N0YXRzCQk9IGlmYl9nZXRfc3RhdHMsCn07CgpzdGF0aWMgdm9pZCBp
ZmJfc2V0dXAoc3RydWN0IG5ldF9kZXZpY2UgKmRldikKewoJLyogSW5pdGlhbGl6ZSB0aGUgZGV2
aWNlIHN0cnVjdHVyZS4gKi8KCWRldi0+ZGVzdHJ1Y3RvciA9IGZyZWVfbmV0ZGV2OwoJZGV2LT5u
ZXRkZXZfb3BzID0gJmlmYl9uZXRkZXZfb3BzOwoKCS8qIEZpbGwgaW4gZGV2aWNlIHN0cnVjdHVy
ZSB3aXRoIGV0aGVybmV0LWdlbmVyaWMgdmFsdWVzLiAqLwoJZXRoZXJfc2V0dXAoZGV2KTsKCWRl
di0+dHhfcXVldWVfbGVuID0gVFhfUV9MSU1JVDsKCglkZXYtPmZsYWdzIHw9IElGRl9OT0FSUDsK
CWRldi0+ZmxhZ3MgJj0gfklGRl9NVUxUSUNBU1Q7CglkZXYtPnByaXZfZmxhZ3MgJj0gfklGRl9Y
TUlUX0RTVF9SRUxFQVNFOwoJcmFuZG9tX2V0aGVyX2FkZHIoZGV2LT5kZXZfYWRkcik7Cn0KCnN0
YXRpYyBpbnQgaWZiX3ZhbGlkYXRlKHN0cnVjdCBubGF0dHIgKnRiW10sIHN0cnVjdCBubGF0dHIg
KmRhdGFbXSkKewoJaWYgKHRiW0lGTEFfQUREUkVTU10pIHsKCQlpZiAobmxhX2xlbih0YltJRkxB
X0FERFJFU1NdKSAhPSBFVEhfQUxFTikKCQkJcmV0dXJuIC1FSU5WQUw7CgkJaWYgKCFpc192YWxp
ZF9ldGhlcl9hZGRyKG5sYV9kYXRhKHRiW0lGTEFfQUREUkVTU10pKSkKCQkJcmV0dXJuIC1FQURE
Uk5PVEFWQUlMOwoJfQoJcmV0dXJuIDA7Cn0KCnN0YXRpYyBzdHJ1Y3QgcnRubF9saW5rX29wcyBp
ZmJfbGlua19vcHMgX19yZWFkX21vc3RseSA9IHsKCS5raW5kCQk9ICJpZmIiLAoJLnByaXZfc2l6
ZQk9IHNpemVvZihzdHJ1Y3QgaWZiX3ByaXZhdGUpLAoJLnNldHVwCQk9IGlmYl9zZXR1cCwKCS52
YWxpZGF0ZQk9IGlmYl92YWxpZGF0ZSwKfTsKCnN0YXRpYyBpbnQgX19pbml0IGlmYl9pbml0X29u
ZShpbnQgaW5kZXgpCnsKCXN0cnVjdCBuZXRfZGV2aWNlICpkZXZfaWZiOwoJaW50IGVycjsKCglk
ZXZfaWZiID0gYWxsb2NfbmV0ZGV2X21xKHNpemVvZihzdHJ1Y3QgaWZiX3ByaXZhdGUpICogbnVt
dHhxcywgImlmYiVkIiwKCQkJCSAgaWZiX3NldHVwLCBudW10eHFzKTsKCglpZiAoIWRldl9pZmIp
CgkJcmV0dXJuIC1FTk9NRU07CgoJZXJyID0gZGV2X2FsbG9jX25hbWUoZGV2X2lmYiwgZGV2X2lm
Yi0+bmFtZSk7CglpZiAoZXJyIDwgMCkKCQlnb3RvIGVycjsKCglkZXZfaWZiLT5ydG5sX2xpbmtf
b3BzID0gJmlmYl9saW5rX29wczsKCWVyciA9IHJlZ2lzdGVyX25ldGRldmljZShkZXZfaWZiKTsK
CWlmIChlcnIgPCAwKQoJCWdvdG8gZXJyOwoKCXJldHVybiAwOwoKZXJyOgoJZnJlZV9uZXRkZXYo
ZGV2X2lmYik7CglyZXR1cm4gZXJyOwp9CgpzdGF0aWMgaW50IF9faW5pdCBpZmJfaW5pdF9tb2R1
bGUodm9pZCkKewoJaW50IGksIGVycjsKCglnZXRfcmFuZG9tX2J5dGVzKCZzaW1wbGVfdHhfaGFz
aHJuZCwgNCk7CglydG5sX2xvY2soKTsKCWVyciA9IF9fcnRubF9saW5rX3JlZ2lzdGVyKCZpZmJf
bGlua19vcHMpOwoKCWZvciAoaSA9IDA7IGkgPCBudW1pZmJzICYmICFlcnI7IGkrKykKCQllcnIg
PSBpZmJfaW5pdF9vbmUoaSk7CglpZiAoZXJyKQoJCV9fcnRubF9saW5rX3VucmVnaXN0ZXIoJmlm
Yl9saW5rX29wcyk7CglydG5sX3VubG9jaygpOwoKCXJldHVybiBlcnI7Cn0KCnN0YXRpYyB2b2lk
IF9fZXhpdCBpZmJfY2xlYW51cF9tb2R1bGUodm9pZCkKewoJcnRubF9saW5rX3VucmVnaXN0ZXIo
JmlmYl9saW5rX29wcyk7Cn0KCm1vZHVsZV9pbml0KGlmYl9pbml0X21vZHVsZSk7Cm1vZHVsZV9l
eGl0KGlmYl9jbGVhbnVwX21vZHVsZSk7Ck1PRFVMRV9MSUNFTlNFKCJHUEwiKTsKTU9EVUxFX0FV
VEhPUigiSmFtYWwgSGFkaSBTYWxpbSIpOwpNT0RVTEVfQUxJQVNfUlROTF9MSU5LKCJpZmIiKTsK
--00504502c9fd1e34190478026ab6--

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 12:41:40 +0100
Message-ID: <4AF95174.3060104@trash.net>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>	 <4AF9471C.5080606@gmail.com> <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=ISO-8859-15
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:65283 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752220AbZKJLlm (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 06:41:42 -0500
In-Reply-To: <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> 2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
>> Changli Gao a =E9crit :
>>> 2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
>>>> Not sure how to solve this problem (several cpus can updates count=
er in //)
>>>>
>>> Thanks, and follow your suggestions. I can maintain the counter per=
 TX
>>> queue, and update it in a timer handler like ixgbe or implement our
>>> own struct net_device_stats* (*ndo_get_stats)(struct net_device *de=
v),
>>> and update the counters when it gets called.
>> Please no timer stuff :)
>=20
> The whole ifb.c file is attached, please review and test it. Thanks!

> /* Number of TX queues per ifb */
> static int numtxqs =3D 1;
> module_param(numtxqs, int, 0444);
> MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");

Module parameters suck as a configuration API. The existing numifbs
option exists purely for compatibility reasons, I'd prefer if you'd
this to the netlink interface.

> static int __init ifb_init_one(int index)
> {
> 	struct net_device *dev_ifb;
> 	int err;
>=20
> 	dev_ifb =3D alloc_netdev_mq(sizeof(struct ifb_private) * numtxqs, "i=
fb%d",
> 				  ifb_setup, numtxqs);
>=20

This won't work for the rtnl_link setup since the size must
be constant.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 20:14:09 +0800
Message-ID: <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>
	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>
	 <4AF9471C.5080606@gmail.com>
	 <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
	 <4AF95174.3060104@trash.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f188.google.com ([209.85.222.188]:48634 "EHLO
	mail-pz0-f188.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752220AbZKJMOE convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 07:14:04 -0500
Received: by pzk26 with SMTP id 26so2796193pzk.4
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 04:14:09 -0800 (PST)
In-Reply-To: <4AF95174.3060104@trash.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, Nov 10, 2009 at 7:41 PM, Patrick McHardy <kaber@trash.net> wrot=
e:
> Changli Gao wrote:
>> 2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
>>
>> The whole ifb.c file is attached, please review and test it. Thanks!
>
>> /* Number of TX queues per ifb */
>> static int numtxqs =3D 1;
>> module_param(numtxqs, int, 0444);
>> MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
>
> Module parameters suck as a configuration API. The existing numifbs
> option exists purely for compatibility reasons, I'd prefer if you'd
> this to the netlink interface.

How to do that? I haven't found any examples. Is there a interface to
config the number of the TX queues of a NIC.

>
>> static int __init ifb_init_one(int index)
>> {
>> =C2=A0 =C2=A0 =C2=A0 struct net_device *dev_ifb;
>> =C2=A0 =C2=A0 =C2=A0 int err;
>>
>> =C2=A0 =C2=A0 =C2=A0 dev_ifb =3D alloc_netdev_mq(sizeof(struct ifb_p=
rivate) * numtxqs, "ifb%d",
>> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ifb_setup, numtxqs);
>>
>
> This won't work for the rtnl_link setup since the size must
> be constant.
>

Does this work?

        ifb_link_ops.priv_size =3D sizeof(struct ifb_private) * numtxqs=
;
        rtnl_lock();
        err =3D __rtnl_link_register(&ifb_link_ops);

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 13:19:09 +0100
Message-ID: <4AF95A3D.5060602@trash.net>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>	 <4AF9471C.5080606@gmail.com>	 <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>	 <4AF95174.3060104@trash.net> <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=ISO-8859-15
Content-Transfer-Encoding: 7bit
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:33341 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751386AbZKJMTJ (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 07:19:09 -0500
In-Reply-To: <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> On Tue, Nov 10, 2009 at 7:41 PM, Patrick McHardy <kaber@trash.net> wrote:
>> Changli Gao wrote:
>>> 2009/11/10 Eric Dumazet <eric.dumazet@gmail.com>:
>>>
>>> The whole ifb.c file is attached, please review and test it. Thanks!
>>> /* Number of TX queues per ifb */
>>> static int numtxqs = 1;
>>> module_param(numtxqs, int, 0444);
>>> MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
>> Module parameters suck as a configuration API. The existing numifbs
>> option exists purely for compatibility reasons, I'd prefer if you'd
>> this to the netlink interface.
> 
> How to do that? I haven't found any examples. Is there a interface to
> config the number of the TX queues of a NIC.

You have to add a get_tx_queues() callback to the rtnl_link_ops.
Additionally you need a new attribute (IFLA_NTXQ or something like
that) that contains the number of queues. The callback has to parse
the attribute and set the number of queues accordingly.

>>> static int __init ifb_init_one(int index)
>>> {
>>>       struct net_device *dev_ifb;
>>>       int err;
>>>
>>>       dev_ifb = alloc_netdev_mq(sizeof(struct ifb_private) * numtxqs, "ifb%d",
>>>                                 ifb_setup, numtxqs);
>>>
>> This won't work for the rtnl_link setup since the size must
>> be constant.
>>
> 
> Does this work?
> 
>         ifb_link_ops.priv_size = sizeof(struct ifb_private) * numtxqs;
>         rtnl_lock();
>         err = __rtnl_link_register(&ifb_link_ops);

Only for the module parameter. For rtnl_link you need to either
allocate the private space seperately or turn priv_size into
a callback that returns the required space based on the number
of queues.


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 20:37:02 +0800
Message-ID: <412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>
	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>
	 <4AF9471C.5080606@gmail.com>
	 <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
	 <4AF95174.3060104@trash.net>
	 <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
	 <4AF95A3D.5060602@trash.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f188.google.com ([209.85.222.188]:63455 "EHLO
	mail-pz0-f188.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751669AbZKJMg4 convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 07:36:56 -0500
Received: by pzk26 with SMTP id 26so2806767pzk.4
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 04:37:02 -0800 (PST)
In-Reply-To: <4AF95A3D.5060602@trash.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, Nov 10, 2009 at 8:19 PM, Patrick McHardy <kaber@trash.net> wrot=
e:
>
> You have to add a get_tx_queues() callback to the rtnl_link_ops.

It is used to get the current real_num_tx_queues and num_tx_queues,
not to setting.

> Additionally you need a new attribute (IFLA_NTXQ or something like
> that) that contains the number of queues. The callback has to parse
> the attribute and set the number of queues accordingly.

It seems another patch is needed first.

>> Does this work?
>>
>> =C2=A0 =C2=A0 =C2=A0 =C2=A0 ifb_link_ops.priv_size =3D sizeof(struct=
 ifb_private) * numtxqs;
>> =C2=A0 =C2=A0 =C2=A0 =C2=A0 rtnl_lock();
>> =C2=A0 =C2=A0 =C2=A0 =C2=A0 err =3D __rtnl_link_register(&ifb_link_o=
ps);
>
> Only for the module parameter. For rtnl_link you need to either
> allocate the private space seperately or turn priv_size into
> a callback that returns the required space based on the number
> of queues.

Do you means that if module ifb is loaded automatically, parameters
won't be set correctly?

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 13:45:09 +0100
Message-ID: <4AF96055.90509@trash.net>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>	 <4AF9471C.5080606@gmail.com>	 <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>	 <4AF95174.3060104@trash.net>	 <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>	 <4AF95A3D.5060602@trash.net> <412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 7bit
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:34030 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1756166AbZKJMpJ (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 07:45:09 -0500
In-Reply-To: <412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> On Tue, Nov 10, 2009 at 8:19 PM, Patrick McHardy <kaber@trash.net> wrote:
>   
>> You have to add a get_tx_queues() callback to the rtnl_link_ops.
>>     
>
> It is used to get the current real_num_tx_queues and num_tx_queues,
> not to setting.
>   

Yes, it gets the number from a user-supplied parameter for device
allocation.

>> Additionally you need a new attribute (IFLA_NTXQ or something like
>> that) that contains the number of queues. The callback has to parse
>> the attribute and set the number of queues accordingly.
>>     
>
> It seems another patch is needed first.
>   

Its a trivial change, you can put them in the same patch:

static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
                             unsigned int *num_tx_queues,
                             unsigned int *real_num_tx_queues)
{
    unsigned int n = 1;

    if (tb[IFLA_NTXQ])
       n = nla_get_u32(tb[IFLA_NTXQ]);

    *num_tx_queues = n;
    *real_num_tx_queues = n;
    return 0;
}

>>> Does this work?
>>>
>>>         ifb_link_ops.priv_size = sizeof(struct ifb_private) * numtxqs;
>>>         rtnl_lock();
>>>         err = __rtnl_link_register(&ifb_link_ops);
>>>       
>> Only for the module parameter. For rtnl_link you need to either
>> allocate the private space seperately or turn priv_size into
>> a callback that returns the required space based on the number
>> of queues.
>>     
>
> Do you means that if module ifb is loaded automatically, parameters
> won't be set correctly?

No, I mean if you add a proper interface for this, you have to deal
with different interfaces using a different amount of queues.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 21:06:58 +0800
Message-ID: <412e6f7f0911100506r6a922b8aoabd359e7e4290ebe@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>
	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>
	 <4AF9471C.5080606@gmail.com>
	 <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
	 <4AF95174.3060104@trash.net>
	 <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
	 <4AF95A3D.5060602@trash.net>
	 <412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>
	 <4AF96055.90509@trash.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-px0-f173.google.com ([209.85.216.173]:46851 "EHLO
	mail-px0-f173.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752693AbZKJNGx convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 08:06:53 -0500
Received: by pxi3 with SMTP id 3so2583925pxi.22
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 05:06:58 -0800 (PST)
In-Reply-To: <4AF96055.90509@trash.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, Nov 10, 2009 at 8:45 PM, Patrick McHardy <kaber@trash.net> wrot=
e:
>
> Its a trivial change, you can put them in the same patch:
>
> static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 unsigned int *num_tx_queues,
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 unsigned int *real_num_tx_queues)
> {
> =C2=A0 =C2=A0unsigned int n =3D 1;
>
> =C2=A0 =C2=A0if (tb[IFLA_NTXQ])
> =C2=A0 =C2=A0 =C2=A0 n =3D nla_get_u32(tb[IFLA_NTXQ]);
>
> =C2=A0 =C2=A0*num_tx_queues =3D n;
> =C2=A0 =C2=A0*real_num_tx_queues =3D n;
> =C2=A0 =C2=A0return 0;
> }
>

Is IFLA_NTXQ added? I didn't find it in any branch. And I think both
num_tx_queueus and real_num_tx_queues are needed.

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 14:34:41 +0100
Message-ID: <4AF96BF1.5070902@gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF92D6D.8060300@gmail.com>	 <412e6f7f0911100143k68284faes92e39d8ca94aa9d4@mail.gmail.com>	 <4AF9471C.5080606@gmail.com>	 <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>	 <4AF95174.3060104@trash.net>	 <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>	 <4AF95A3D.5060602@trash.net>	 <412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>	 <4AF96055.90509@trash.net> <412e6f7f0911100506r6a922b8aoabd359e7e4290ebe@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Patrick McHardy <kaber@trash.net>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:35403 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752891AbZKJNeo (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 08:34:44 -0500
In-Reply-To: <412e6f7f0911100506r6a922b8aoabd359e7e4290ebe@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =C3=A9crit :
> On Tue, Nov 10, 2009 at 8:45 PM, Patrick McHardy <kaber@trash.net> wr=
ote:
>> Its a trivial change, you can put them in the same patch:
>>
>> static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
>>                             unsigned int *num_tx_queues,
>>                             unsigned int *real_num_tx_queues)
>> {
>>    unsigned int n =3D 1;
>>
>>    if (tb[IFLA_NTXQ])
>>       n =3D nla_get_u32(tb[IFLA_NTXQ]);
>>
>>    *num_tx_queues =3D n;
>>    *real_num_tx_queues =3D n;
>>    return 0;
>> }
>>
>=20
> Is IFLA_NTXQ added? I didn't find it in any branch. And I think both
> num_tx_queueus and real_num_tx_queues are needed.
>=20

I believe Patrick was referring to vlan get_tx_queues() implementation,
but it actually gets values from real device (tb[IFLA_LINK])

In your case you'll need to add a new IFLA_NTXQ attribute, and
change iproute2 to pass this new attribute at link creation.
(check include/linux/if_link.h)

ip link add link .....  ntxq 2

static int vlan_get_tx_queues(struct net *net,
                              struct nlattr *tb[],
                              unsigned int *num_tx_queues,
                              unsigned int *real_num_tx_queues)
{
        struct net_device *real_dev;

        if (!tb[IFLA_LINK])
                return -EINVAL;

        real_dev =3D __dev_get_by_index(net, nla_get_u32(tb[IFLA_LINK])=
);
        if (!real_dev)
                return -ENODEV;

        *num_tx_queues      =3D real_dev->num_tx_queues;
        *real_num_tx_queues =3D real_dev->real_num_tx_queues;
        return 0;
}



From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 21:49:31 +0800
Message-ID: <412e6f7f0911100549q68a57ba7s83b11be3a77323a2@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF9471C.5080606@gmail.com>
	 <412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
	 <4AF95174.3060104@trash.net>
	 <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
	 <4AF95A3D.5060602@trash.net>
	 <412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>
	 <4AF96055.90509@trash.net>
	 <412e6f7f0911100506r6a922b8aoabd359e7e4290ebe@mail.gmail.com>
	 <4AF96BF1.5070902@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Patrick McHardy <kaber@trash.net>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f188.google.com ([209.85.222.188]:64495 "EHLO
	mail-pz0-f188.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1750923AbZKJNtZ convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 08:49:25 -0500
Received: by pzk26 with SMTP id 26so16937pzk.4
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 05:49:31 -0800 (PST)
In-Reply-To: <4AF96BF1.5070902@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, Nov 10, 2009 at 9:34 PM, Eric Dumazet <eric.dumazet@gmail.com> =
wrote:
> Changli Gao a =C3=A9crit :
>
> I believe Patrick was referring to vlan get_tx_queues() implementatio=
n,
> but it actually gets values from real device (tb[IFLA_LINK])
>
> In your case you'll need to add a new IFLA_NTXQ attribute, and
> change iproute2 to pass this new attribute at link creation.
> (check include/linux/if_link.h)
>
> ip link add link ..... =C2=A0ntxq 2
>
> static int vlan_get_tx_queues(struct net *net,
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0struct nlattr *tb[],
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *num_tx_queues,
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *real_num_tx_queues)
> {
> =C2=A0 =C2=A0 =C2=A0 =C2=A0struct net_device *real_dev;
>
> =C2=A0 =C2=A0 =C2=A0 =C2=A0if (!tb[IFLA_LINK])
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return -EINVAL=
;
>
> =C2=A0 =C2=A0 =C2=A0 =C2=A0real_dev =3D __dev_get_by_index(net, nla_g=
et_u32(tb[IFLA_LINK]));
> =C2=A0 =C2=A0 =C2=A0 =C2=A0if (!real_dev)
> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return -ENODEV=
;
>
> =C2=A0 =C2=A0 =C2=A0 =C2=A0*num_tx_queues =C2=A0 =C2=A0 =C2=A0=3D rea=
l_dev->num_tx_queues;
> =C2=A0 =C2=A0 =C2=A0 =C2=A0*real_num_tx_queues =3D real_dev->real_num=
_tx_queues;
> =C2=A0 =C2=A0 =C2=A0 =C2=A0return 0;
> }
>
>
>

got it. Thanks. BTW: why not merge iproute2 into linux, just like perf.

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Stephen Hemminger <shemminger@vyatta.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 10 Nov 2009 08:45:16 -0800
Message-ID: <20091110084516.15659fcc@nehalam>
References: <4AF924A5.1050303@gmail.com>
	<4AF9471C.5080606@gmail.com>
	<412e6f7f0911100314i33d6a49dl5163b5e8472babcf@mail.gmail.com>
	<4AF95174.3060104@trash.net>
	<412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
	<4AF95A3D.5060602@trash.net>
	<412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>
	<4AF96055.90509@trash.net>
	<412e6f7f0911100506r6a922b8aoabd359e7e4290ebe@mail.gmail.com>
	<4AF96BF1.5070902@gmail.com>
	<412e6f7f0911100549q68a57ba7s83b11be3a77323a2@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	Patrick McHardy <kaber@trash.net>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail.vyatta.com ([76.74.103.46]:47745 "EHLO mail.vyatta.com"
	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
	id S1757195AbZKJQpm convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 10 Nov 2009 11:45:42 -0500
In-Reply-To: <412e6f7f0911100549q68a57ba7s83b11be3a77323a2@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, 10 Nov 2009 21:49:31 +0800
Changli Gao <xiaosuo@gmail.com> wrote:

> On Tue, Nov 10, 2009 at 9:34 PM, Eric Dumazet <eric.dumazet@gmail.com=
> wrote:
> > Changli Gao a =C3=A9crit :
> >
> > I believe Patrick was referring to vlan get_tx_queues() implementat=
ion,
> > but it actually gets values from real device (tb[IFLA_LINK])
> >
> > In your case you'll need to add a new IFLA_NTXQ attribute, and
> > change iproute2 to pass this new attribute at link creation.
> > (check include/linux/if_link.h)
> >
> > ip link add link ..... =C2=A0ntxq 2
> >
> > static int vlan_get_tx_queues(struct net *net,
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0struct nlattr *tb[],
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *num_tx_queues,
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *real_num_tx_queues)
> > {
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0struct net_device *real_dev;
> >
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0if (!tb[IFLA_LINK])
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return -EINV=
AL;
> >
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0real_dev =3D __dev_get_by_index(net, nla=
_get_u32(tb[IFLA_LINK]));
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0if (!real_dev)
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0return -ENOD=
EV;
> >
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0*num_tx_queues =C2=A0 =C2=A0 =C2=A0=3D r=
eal_dev->num_tx_queues;
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0*real_num_tx_queues =3D real_dev->real_n=
um_tx_queues;
> > =C2=A0 =C2=A0 =C2=A0 =C2=A0return 0;
> > }
> >
> >
> >
>=20
> got it. Thanks. BTW: why not merge iproute2 into linux, just like per=
f.
>=20

Distro's would find that hard, also it would cause people to forgot bin=
ary
API compatibility.

--=20

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Wed, 11 Nov 2009 14:30:37 +0800
Message-ID: <412e6f7f0911102230t40cc125bq6e9311cd034e864d@mail.gmail.com>
References: <4AF924A5.1050303@gmail.com> <4AF95174.3060104@trash.net>
	 <412e6f7f0911100414w7b6f45aap39f6568b7af2c1a7@mail.gmail.com>
	 <4AF95A3D.5060602@trash.net>
	 <412e6f7f0911100437g2759eb6dg509af4edb8848291@mail.gmail.com>
	 <4AF96055.90509@trash.net>
	 <412e6f7f0911100506r6a922b8aoabd359e7e4290ebe@mail.gmail.com>
	 <4AF96BF1.5070902@gmail.com>
	 <412e6f7f0911100549q68a57ba7s83b11be3a77323a2@mail.gmail.com>
	 <20091110084516.15659fcc@nehalam>
Mime-Version: 1.0
Content-Type: multipart/mixed; boundary=00504502cb24a0476b04781290a6
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	Patrick McHardy <kaber@trash.net>,
	"David S. Miller" <davem@davemloft.net>, netdev@vger.kernel.org,
	Tom Herbert <therbert@google.com>
To: Stephen Hemminger <shemminger@vyatta.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f188.google.com ([209.85.222.188]:44491 "EHLO
	mail-pz0-f188.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751964AbZKKGac (ORCPT
	<rfc822;netdev@vger.kernel.org>); Wed, 11 Nov 2009 01:30:32 -0500
Received: by pzk26 with SMTP id 26so557487pzk.4
        for <netdev@vger.kernel.org>; Tue, 10 Nov 2009 22:30:38 -0800 (PST)
In-Reply-To: <20091110084516.15659fcc@nehalam>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

--00504502cb24a0476b04781290a6
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: quoted-printable

On Wed, Nov 11, 2009 at 12:45 AM, Stephen Hemminger
<shemminger@vyatta.com> wrote:
> On Tue, 10 Nov 2009 21:49:31 +0800
> Changli Gao <xiaosuo@gmail.com> wrote:
>
>
> Distro's would find that hard, also it would cause people to forgot binar=
y
> API compatibility.
>

Get it. Thanks.

The new version of ifb is attached:
1. fixbug: a deadlock.
2. fixbug: statistics data corruption.
3. fixbug: can't work with ip link command.

Dear Patrick:

The support for specifying the number of TX queues with ip link
command is comming soon. I do think the corresponding kernel module
parameter is needed for the default option and the ifbs created when
module loading, so I preserve it.

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

--00504502cb24a0476b04781290a6
Content-Type: application/octet-stream; name="ifb.c"
Content-Disposition: attachment; filename="ifb.c"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_g1vpojai0

LyogZHJpdmVycy9uZXQvaWZiLmM6CgoJVGhlIHB1cnBvc2Ugb2YgdGhpcyBkcml2ZXIgaXMgdG8g
cHJvdmlkZSBhIGRldmljZSB0aGF0IGFsbG93cwoJZm9yIHNoYXJpbmcgb2YgcmVzb3VyY2VzOgoK
CTEpIHFkaXNjcy9wb2xpY2llcyB0aGF0IGFyZSBwZXIgZGV2aWNlIGFzIG9wcG9zZWQgdG8gc3lz
dGVtIHdpZGUuCglpZmIgYWxsb3dzIGZvciBhIGRldmljZSB3aGljaCBjYW4gYmUgcmVkaXJlY3Rl
ZCB0byB0aHVzIHByb3ZpZGluZwoJYW4gaW1wcmVzc2lvbiBvZiBzaGFyaW5nLgoKCTIpIEFsbG93
cyBmb3IgcXVldWVpbmcgaW5jb21pbmcgdHJhZmZpYyBmb3Igc2hhcGluZyBpbnN0ZWFkIG9mCglk
cm9wcGluZy4KCglUaGUgb3JpZ2luYWwgY29uY2VwdCBpcyBiYXNlZCBvbiB3aGF0IGlzIGtub3du
IGFzIHRoZSBJTVEKCWRyaXZlciBpbml0aWFsbHkgd3JpdHRlbiBieSBNYXJ0aW4gRGV2ZXJhLCBs
YXRlciByZXdyaXR0ZW4KCWJ5IFBhdHJpY2sgTWNIYXJkeSBhbmQgdGhlbiBtYWludGFpbmVkIGJ5
IEFuZHJlIENvcnJlYS4KCglZb3UgbmVlZCB0aGUgdGMgYWN0aW9uICBtaXJyb3Igb3IgcmVkaXJl
Y3QgdG8gZmVlZCB0aGlzIGRldmljZQogICAgICAgCXBhY2tldHMuCgoJVGhpcyBwcm9ncmFtIGlz
IGZyZWUgc29mdHdhcmU7IHlvdSBjYW4gcmVkaXN0cmlidXRlIGl0IGFuZC9vcgoJbW9kaWZ5IGl0
IHVuZGVyIHRoZSB0ZXJtcyBvZiB0aGUgR05VIEdlbmVyYWwgUHVibGljIExpY2Vuc2UKCWFzIHB1
Ymxpc2hlZCBieSB0aGUgRnJlZSBTb2Z0d2FyZSBGb3VuZGF0aW9uOyBlaXRoZXIgdmVyc2lvbgoJ
MiBvZiB0aGUgTGljZW5zZSwgb3IgKGF0IHlvdXIgb3B0aW9uKSBhbnkgbGF0ZXIgdmVyc2lvbi4K
CiAgCUF1dGhvcnM6CUphbWFsIEhhZGkgU2FsaW0gKDIwMDUpCgoqLwoKCiNpbmNsdWRlIDxsaW51
eC9tb2R1bGUuaD4KI2luY2x1ZGUgPGxpbnV4L2tlcm5lbC5oPgojaW5jbHVkZSA8bGludXgvbmV0
ZGV2aWNlLmg+CiNpbmNsdWRlIDxsaW51eC9ldGhlcmRldmljZS5oPgojaW5jbHVkZSA8bGludXgv
aW5pdC5oPgojaW5jbHVkZSA8bGludXgvbW9kdWxlcGFyYW0uaD4KI2luY2x1ZGUgPGxpbnV4L3dh
aXQuaD4KI2luY2x1ZGUgPGxpbnV4L3NjaGVkLmg+CiNpbmNsdWRlIDxsaW51eC9rdGhyZWFkLmg+
CiNpbmNsdWRlIDxsaW51eC9pcC5oPgojaW5jbHVkZSA8bGludXgvaXB2Ni5oPgojaW5jbHVkZSA8
bmV0L2lwLmg+CiNpbmNsdWRlIDxuZXQvcGt0X3NjaGVkLmg+CiNpbmNsdWRlIDxuZXQvbmV0X25h
bWVzcGFjZS5oPgoKI2RlZmluZSBUWF9RX0xJTUlUICAgIDMyCgpzdHJ1Y3QgaWZiX3ByaXZhdGVf
cSB7CglzdHJ1Y3QgbmV0X2RldmljZQkqZGV2OwoJc3RydWN0IHNrX2J1ZmZfaGVhZAlycTsKCXN0
cnVjdCBza19idWZmX2hlYWQJdHE7Cgl3YWl0X3F1ZXVlX2hlYWRfdAl3cTsKCXN0cnVjdCB0YXNr
X3N0cnVjdAkqdGFzazsKCXVuc2lnbmVkIGxvbmcJCXJ4X3BhY2tldHM7Cgl1bnNpZ25lZCBsb25n
CQlyeF9ieXRlczsKCXVuc2lnbmVkIGxvbmcJCXJ4X2Ryb3BwZWQ7Cn0gX19fX2NhY2hlbGluZV9h
bGlnbmVkX2luX3NtcDsKCnN0cnVjdCBpZmJfcHJpdmF0ZSB7CglzdHJ1Y3QgaWZiX3ByaXZhdGVf
cQkqcHE7Cn07CgovKiBOdW1iZXIgb2YgaWZiIGRldmljZXMgdG8gYmUgc2V0IHVwIGJ5IHRoaXMg
bW9kdWxlLiAqLwpzdGF0aWMgaW50IG51bWlmYnMgPSAyOwptb2R1bGVfcGFyYW0obnVtaWZicywg
aW50LCAwNDQ0KTsKTU9EVUxFX1BBUk1fREVTQyhudW1pZmJzLCAiTnVtYmVyIG9mIGlmYiBkZXZp
Y2VzIik7CgovKiBOdW1iZXIgb2YgVFggcXVldWVzIHBlciBpZmIgKi8Kc3RhdGljIGludCBudW10
eHFzID0gMTsKbW9kdWxlX3BhcmFtKG51bXR4cXMsIGludCwgMDQ0NCk7Ck1PRFVMRV9QQVJNX0RF
U0MobnVtdHhxcywgIk51bWJlciBvZiBUWCBxdWV1ZXMgcGVyIGlmYiIpOwoKc3RhdGljIGludCBp
ZmJfdGhyZWFkKHZvaWQgKnByaXYpCnsKCXN0cnVjdCBpZmJfcHJpdmF0ZV9xICpwcSA9IHByaXY7
CglzdHJ1Y3QgbmV0X2RldmljZSAqZGV2ID0gcHEtPmRldjsKCWludCBudW0gPSBwcSAtICgoc3Ry
dWN0IGlmYl9wcml2YXRlICopbmV0ZGV2X3ByaXYoZGV2KSktPnBxOwoJc3RydWN0IG5ldGRldl9x
dWV1ZSAqdHhxID0gbmV0ZGV2X2dldF90eF9xdWV1ZShkZXYsIG51bSk7CglzdHJ1Y3Qgc2tfYnVm
ZiAqc2tiOwoJREVGSU5FX1dBSVQod2FpdCk7CgoJd2hpbGUgKDEpIHsKCQkvKiBtb3ZlIHNrYiBm
cm9tIHJxIHRvIHRxICovCgkJd2hpbGUgKDEpIHsKCQkJcHJlcGFyZV90b193YWl0KCZwcS0+d3Es
ICZ3YWl0LCBUQVNLX1VOSU5URVJSVVBUSUJMRSk7CgkJCV9fbmV0aWZfdHhfbG9ja19iaCh0eHEp
OwoJCQl3aGlsZSAoKHNrYiA9IHNrYl9kZXF1ZXVlKCZwcS0+cnEpKSAhPSBOVUxMKQoJCQkJc2ti
X3F1ZXVlX3RhaWwoJnBxLT50cSwgc2tiKTsKCQkJaWYgKG5ldGlmX3F1ZXVlX3N0b3BwZWQoZGV2
KSkKCQkJCW5ldGlmX3dha2VfcXVldWUoZGV2KTsKCQkJX19uZXRpZl90eF91bmxvY2tfYmgodHhx
KTsKCQkJaWYgKGt0aHJlYWRfc2hvdWxkX3N0b3AoKSB8fCAhc2tiX3F1ZXVlX2VtcHR5KCZwcS0+
dHEpKQoJCQkJYnJlYWs7CgkJCXNjaGVkdWxlKCk7CgkJfQoJCWZpbmlzaF93YWl0KCZwcS0+d3Es
ICZ3YWl0KTsKCQlpZiAoa3RocmVhZF9zaG91bGRfc3RvcCgpKQoJCQlicmVhazsKCQlpZiAobmVl
ZF9yZXNjaGVkKCkpCgkJCXNjaGVkdWxlKCk7CgoJCS8qIHRyYW5zZmVyIHBhY2tldHMgKi8KCQl3
aGlsZSAoKHNrYiA9IHNrYl9kZXF1ZXVlKCZwcS0+dHEpKSAhPSBOVUxMKSB7CgkJCXUzMiBmcm9t
ID0gR19UQ19GUk9NKHNrYi0+dGNfdmVyZCk7CgkKCQkJc2tiLT50Y192ZXJkID0gMDsKCQkJc2ti
LT50Y192ZXJkID0gU0VUX1RDX05DTFMoc2tiLT50Y192ZXJkKTsKCQkJdHhxLT50eF9wYWNrZXRz
Kys7CgkJCXR4cS0+dHhfYnl0ZXMgKz1za2ItPmxlbjsKCgkJCXJjdV9yZWFkX2xvY2soKTsKCQkJ
c2tiLT5kZXYgPSBkZXZfZ2V0X2J5X2luZGV4X3JjdSgmaW5pdF9uZXQsIHNrYi0+aWlmKTsKCQkJ
aWYgKCFza2ItPmRldikgewoJCQkJcmN1X3JlYWRfdW5sb2NrKCk7CgkJCQlkZXZfa2ZyZWVfc2ti
KHNrYik7CgkJCQl0eHEtPnR4X2Ryb3BwZWQrKzsKCQkJCWJyZWFrOwoJCQl9CgkJCXJjdV9yZWFk
X3VubG9jaygpOwoJCQlza2ItPmlpZiA9IGRldi0+aWZpbmRleDsKCQoJCQlpZiAoZnJvbSAmIEFU
X0VHUkVTUykgewoJCQkJZGV2X3F1ZXVlX3htaXQoc2tiKTsKCQkJfSBlbHNlIGlmIChmcm9tICYg
QVRfSU5HUkVTUykgewoJCQkJc2tiX3B1bGwoc2tiLCBza2ItPmRldi0+aGFyZF9oZWFkZXJfbGVu
KTsKCQkJCW5ldGlmX3J4X25pKHNrYik7CgkJCX0gZWxzZQoJCQkJQlVHKCk7CgkJfQoJfQoKCXJl
dHVybiAwOwp9CgpzdHJ1Y3QgbmV0X2RldmljZV9zdGF0cyogaWZiX2dldF9zdGF0cyhzdHJ1Y3Qg
bmV0X2RldmljZSAqZGV2KQp7CglzdHJ1Y3QgbmV0X2RldmljZV9zdGF0cyAqc3RhdHMgPSAmZGV2
LT5zdGF0czsKCXN0cnVjdCBpZmJfcHJpdmF0ZSAqZHAgPSBuZXRkZXZfcHJpdihkZXYpOwoJc3Ry
dWN0IGlmYl9wcml2YXRlX3EgKnBxID0gZHAtPnBxOwoJc3RydWN0IG5ldGRldl9xdWV1ZSAqdHhx
OwoJaW50IGk7Cgl1bnNpZ25lZCBsb25nIHJ4X3BhY2tldHMgPSAwLCByeF9ieXRlcyA9IDAsIHJ4
X2Ryb3BwZWQgPSAwOwoJdW5zaWduZWQgbG9uZyB0eF9wYWNrZXRzID0gMCwgdHhfYnl0ZXMgPSAw
LCB0eF9kcm9wcGVkID0gMDsKCglmb3IgKGkgPSAwOyBpIDwgZGV2LT5yZWFsX251bV90eF9xdWV1
ZXM7IGkrKykgewoJCXJ4X3BhY2tldHMgKz0gcHFbaV0ucnhfcGFja2V0czsKCQlyeF9ieXRlcyAr
PSBwcVtpXS5yeF9ieXRlczsKCQlyeF9kcm9wcGVkICs9IHBxW2ldLnJ4X2Ryb3BwZWQ7CgkJdHhx
ID0gbmV0ZGV2X2dldF90eF9xdWV1ZShkZXYsIGkpOwoJCXR4X3BhY2tldHMgKz0gdHhxLT50eF9w
YWNrZXRzOwoJCXR4X2J5dGVzICs9IHR4cS0+dHhfYnl0ZXM7CgkJdHhfZHJvcHBlZCArPSB0eHEt
PnR4X2Ryb3BwZWQ7Cgl9CgoJc3RhdHMtPnJ4X3BhY2tldHMgPSByeF9wYWNrZXRzOwoJc3RhdHMt
PnJ4X2J5dGVzID0gcnhfYnl0ZXM7CglzdGF0cy0+cnhfZHJvcHBlZCA9IHJ4X2Ryb3BwZWQ7Cglz
dGF0cy0+dHhfcGFja2V0cyA9IHR4X3BhY2tldHM7CglzdGF0cy0+dHhfYnl0ZXMgPSB0eF9ieXRl
czsKCXN0YXRzLT50eF9kcm9wcGVkID0gdHhfZHJvcHBlZDsKCglyZXR1cm4gc3RhdHM7Cn0KCnN0
YXRpYyBuZXRkZXZfdHhfdCBpZmJfeG1pdChzdHJ1Y3Qgc2tfYnVmZiAqc2tiLCBzdHJ1Y3QgbmV0
X2RldmljZSAqZGV2KQp7Cgl1MzIgZnJvbSA9IEdfVENfRlJPTShza2ItPnRjX3ZlcmQpOwoJaW50
IG51bSA9IHNrYl9nZXRfcXVldWVfbWFwcGluZyhza2IpOwoJc3RydWN0IGlmYl9wcml2YXRlICpk
cCA9IG5ldGRldl9wcml2KGRldik7CglzdHJ1Y3QgaWZiX3ByaXZhdGVfcSAqcHEgPSBkcC0+cHEg
KyBudW07CglzdHJ1Y3QgbmV0ZGV2X3F1ZXVlICp0eHEgPSBuZXRkZXZfZ2V0X3R4X3F1ZXVlKGRl
diwgbnVtKTsKCglwcS0+cnhfcGFja2V0cysrOwoJcHEtPnJ4X2J5dGVzICs9IHNrYi0+bGVuOwoK
CWlmICghKGZyb20gJiAoQVRfSU5HUkVTU3xBVF9FR1JFU1MpKSB8fCAhc2tiLT5paWYpIHsKCQlk
ZXZfa2ZyZWVfc2tiKHNrYik7CgkJcHEtPnJ4X2Ryb3BwZWQrKzsKCQlyZXR1cm4gTkVUREVWX1RY
X09LOwoJfQoKCXR4cS0+dHJhbnNfc3RhcnQgPSBqaWZmaWVzOwoJc2tiX3F1ZXVlX3RhaWwoJnBx
LT5ycSwgc2tiKTsKCWlmIChza2JfcXVldWVfbGVuKCZwcS0+cnEpID49IGRldi0+dHhfcXVldWVf
bGVuKQoJCW5ldGlmX3N0b3BfcXVldWUoZGV2KTsKCWlmIChza2JfcXVldWVfbGVuKCZwcS0+cnEp
ID09IDEpCgkJd2FrZV91cCgmcHEtPndxKTsKCglyZXR1cm4gTkVUREVWX1RYX09LOwp9CgpzdGF0
aWMgaW50IGlmYl9jbG9zZShzdHJ1Y3QgbmV0X2RldmljZSAqZGV2KQp7CglzdHJ1Y3QgaWZiX3By
aXZhdGUgKmRwID0gbmV0ZGV2X3ByaXYoZGV2KTsKCXN0cnVjdCBpZmJfcHJpdmF0ZV9xICpwcSA9
IGRwLT5wcTsKCWludCBpOwoKCW5ldGlmX3N0b3BfcXVldWUoZGV2KTsKCWZvciAoaSA9IDA7IGkg
PCBkZXYtPnJlYWxfbnVtX3R4X3F1ZXVlczsgaSsrKSB7CgkJa3RocmVhZF9zdG9wKHBxW2ldLnRh
c2spOwoJCXNrYl9xdWV1ZV9wdXJnZSgmcHFbaV0udHEpOwoJCXNrYl9xdWV1ZV9wdXJnZSgmcHFb
aV0ucnEpOwoJfQoKCXJldHVybiAwOwp9CgpzdGF0aWMgaW50IGlmYl9vcGVuKHN0cnVjdCBuZXRf
ZGV2aWNlICpkZXYpCnsKCXN0cnVjdCBpZmJfcHJpdmF0ZSAqZHAgPSBuZXRkZXZfcHJpdihkZXYp
OwoJc3RydWN0IGlmYl9wcml2YXRlX3EgKnBxID0gZHAtPnBxOwoJaW50IGk7CgkKCWZvciAoaSA9
IDA7IGkgPCBkZXYtPnJlYWxfbnVtX3R4X3F1ZXVlczsgaSsrKSB7CgkJcHFbaV0udGFzayA9IGt0
aHJlYWRfcnVuKGlmYl90aHJlYWQsICZwcVtpXSwgIiVzLyVkIiwgZGV2LT5uYW1lLAoJCQkJCSBp
KTsKCQlpZiAoSVNfRVJSKHBxW2ldLnRhc2spKSB7CgkJCWludCBlcnIgPSBQVFJfRVJSKHBxW2ld
LnRhc2spOwoJCQl3aGlsZSAoLS1pID49IDApCgkJCQlrdGhyZWFkX3N0b3AocHFbaV0udGFzayk7
CgkJCXJldHVybiBlcnI7CgkJfQoJfQoJbmV0aWZfc3RhcnRfcXVldWUoZGV2KTsKCglyZXR1cm4g
MDsKfQoKc3RhdGljIHUzMiBzaW1wbGVfdHhfaGFzaHJuZDsKCnN0YXRpYyB1MTYgaWZiX3NlbGVj
dF9xdWV1ZShzdHJ1Y3QgbmV0X2RldmljZSAqZGV2LCBzdHJ1Y3Qgc2tfYnVmZiAqc2tiKQp7Cgl1
MzIgYWRkcjEsIGFkZHIyOwoJdTMyIGhhc2gsIGlobDsKCXVuaW9uIHsKCQl1MTYgaW4xNlsyXTsK
CQl1MzIgaW4zMjsKCX0gcG9ydHM7Cgl1OCBpcF9wcm90bzsKCglpZiAoKGhhc2ggPSBza2Jfcnhf
cXVldWVfcmVjb3JkZWQoc2tiKSkpIHsKCQl3aGlsZSAoaGFzaCA+PSBkZXYtPnJlYWxfbnVtX3R4
X3F1ZXVlcykKCQkJaGFzaCAtPSBkZXYtPnJlYWxfbnVtX3R4X3F1ZXVlczsKCQlyZXR1cm4gaGFz
aDsKCX0KCglzd2l0Y2ggKHNrYi0+cHJvdG9jb2wpIHsKCWNhc2UgX19jb25zdGFudF9odG9ucyhF
VEhfUF9JUCk6CgkJaWYgKCEoaXBfaGRyKHNrYiktPmZyYWdfb2ZmICYgaHRvbnMoSVBfTUYgfCBJ
UF9PRkZTRVQpKSkKCQkJaXBfcHJvdG8gPSBpcF9oZHIoc2tiKS0+cHJvdG9jb2w7CgkJZWxzZQoJ
CQlpcF9wcm90byA9IDA7CgkJYWRkcjEgPSBpcF9oZHIoc2tiKS0+c2FkZHI7CgkJYWRkcjIgPSBp
cF9oZHIoc2tiKS0+ZGFkZHI7CgkJaWhsID0gaXBfaGRyKHNrYiktPmlobCA8PCAyOwoJCWJyZWFr
OwoJY2FzZSBfX2NvbnN0YW50X2h0b25zKEVUSF9QX0lQVjYpOgoJCWlwX3Byb3RvID0gaXB2Nl9o
ZHIoc2tiKS0+bmV4dGhkcjsKCQlhZGRyMSA9IGlwdjZfaGRyKHNrYiktPnNhZGRyLnM2X2FkZHIz
MlszXTsKCQlhZGRyMiA9IGlwdjZfaGRyKHNrYiktPmRhZGRyLnM2X2FkZHIzMlszXTsKCQlpaGwg
PSAxMDsKCQlicmVhazsKCWRlZmF1bHQ6CgkJcmV0dXJuIDA7Cgl9CglpZiAoYWRkcjEgPiBhZGRy
MikKCQlzd2FwKGFkZHIxLCBhZGRyMik7CgoJc3dpdGNoIChpcF9wcm90bykgewoJY2FzZSBJUFBS
T1RPX1RDUDoKCWNhc2UgSVBQUk9UT19VRFA6CgljYXNlIElQUFJPVE9fRENDUDoKCWNhc2UgSVBQ
Uk9UT19FU1A6CgljYXNlIElQUFJPVE9fQUg6CgljYXNlIElQUFJPVE9fU0NUUDoKCWNhc2UgSVBQ
Uk9UT19VRFBMSVRFOgoJCXBvcnRzLmluMzIgPSAqKCh1MzIgKikgKHNrYl9uZXR3b3JrX2hlYWRl
cihza2IpICsgaWhsKSk7CgkJaWYgKHBvcnRzLmluMTZbMF0gPiBwb3J0cy5pbjE2WzFdKQoJCQlz
d2FwKHBvcnRzLmluMTZbMF0sIHBvcnRzLmluMTZbMV0pOwoJCWJyZWFrOwoKCWRlZmF1bHQ6CgkJ
cG9ydHMuaW4zMiA9IDA7CgkJYnJlYWs7Cgl9CgoJaGFzaCA9IGpoYXNoXzN3b3JkcyhhZGRyMSwg
YWRkcjIsIHBvcnRzLmluMzIsCgkJCSAgICBzaW1wbGVfdHhfaGFzaHJuZCBeIGlwX3Byb3RvKTsK
CglyZXR1cm4gKHUxNikgKCgodTY0KSBoYXNoICogZGV2LT5yZWFsX251bV90eF9xdWV1ZXMpID4+
IDMyKTsKfQoKc3RhdGljIGludCBpZmJfaW5pdChzdHJ1Y3QgbmV0X2RldmljZSAqZGV2KQp7Cglz
dHJ1Y3QgaWZiX3ByaXZhdGUgKmRwID0gbmV0ZGV2X3ByaXYoZGV2KTsKCXN0cnVjdCBpZmJfcHJp
dmF0ZV9xICpwcSA9IGRwLT5wcTsKCWludCBpOwoKCXBxID0ga21hbGxvYyhzaXplb2YoKnBxKSAq
IGRldi0+cmVhbF9udW1fdHhfcXVldWVzLCBHRlBfS0VSTkVMKTsKCWlmIChwcSA9PSBOVUxMKQoJ
CXJldHVybiAtRU5PTUVNOwoJZHAtPnBxID0gcHE7CgoJZm9yIChpID0gMDsgaSA8IGRldi0+cmVh
bF9udW1fdHhfcXVldWVzOyBpKyspIHsKCQlwcVtpXS5kZXYgPSBkZXY7CgkJc2tiX3F1ZXVlX2hl
YWRfaW5pdCgmcHFbaV0ucnEpOwoJCXNrYl9xdWV1ZV9oZWFkX2luaXQoJnBxW2ldLnRxKTsKCQlp
bml0X3dhaXRxdWV1ZV9oZWFkKCZwcVtpXS53cSk7CgkJcHFbaV0ucnhfcGFja2V0cyA9IDA7CgkJ
cHFbaV0ucnhfYnl0ZXMgPSAwOwoJCXBxW2ldLnJ4X2Ryb3BwZWQgPSAwOwoJfQoKCXJldHVybiAw
Owp9CgpzdGF0aWMgdm9pZCBpZmJfdW5pbml0KHN0cnVjdCBuZXRfZGV2aWNlICpkZXYpCnsKCXN0
cnVjdCBpZmJfcHJpdmF0ZSAqZHAgPSBuZXRkZXZfcHJpdihkZXYpOwoKCWtmcmVlKGRwLT5wcSk7
Cn0KCnN0YXRpYyBjb25zdCBzdHJ1Y3QgbmV0X2RldmljZV9vcHMgaWZiX25ldGRldl9vcHMgPSB7
CgkubmRvX2luaXQJCT0gaWZiX2luaXQsCgkubmRvX3VuaW5pdAkJPSBpZmJfdW5pbml0LAoJLm5k
b19vcGVuCQk9IGlmYl9vcGVuLAoJLm5kb19zdG9wCQk9IGlmYl9jbG9zZSwKCS5uZG9fc3RhcnRf
eG1pdAkJPSBpZmJfeG1pdCwKCS5uZG9fdmFsaWRhdGVfYWRkcgk9IGV0aF92YWxpZGF0ZV9hZGRy
LAoJLm5kb19zZWxlY3RfcXVldWUJPSBpZmJfc2VsZWN0X3F1ZXVlLAoJLm5kb19nZXRfc3RhdHMJ
CT0gaWZiX2dldF9zdGF0cywKfTsKCnN0YXRpYyB2b2lkIGlmYl9zZXR1cChzdHJ1Y3QgbmV0X2Rl
dmljZSAqZGV2KQp7CgkvKiBJbml0aWFsaXplIHRoZSBkZXZpY2Ugc3RydWN0dXJlLiAqLwoJZGV2
LT5kZXN0cnVjdG9yID0gZnJlZV9uZXRkZXY7CglkZXYtPm5ldGRldl9vcHMgPSAmaWZiX25ldGRl
dl9vcHM7CgoJLyogRmlsbCBpbiBkZXZpY2Ugc3RydWN0dXJlIHdpdGggZXRoZXJuZXQtZ2VuZXJp
YyB2YWx1ZXMuICovCglldGhlcl9zZXR1cChkZXYpOwoJZGV2LT50eF9xdWV1ZV9sZW4gPSBUWF9R
X0xJTUlUOwoKCWRldi0+ZmxhZ3MgfD0gSUZGX05PQVJQOwoJZGV2LT5mbGFncyAmPSB+SUZGX01V
TFRJQ0FTVDsKCWRldi0+cHJpdl9mbGFncyAmPSB+SUZGX1hNSVRfRFNUX1JFTEVBU0U7CglyYW5k
b21fZXRoZXJfYWRkcihkZXYtPmRldl9hZGRyKTsKfQoKc3RhdGljIGludCBpZmJfdmFsaWRhdGUo
c3RydWN0IG5sYXR0ciAqdGJbXSwgc3RydWN0IG5sYXR0ciAqZGF0YVtdKQp7CglpZiAodGJbSUZM
QV9BRERSRVNTXSkgewoJCWlmIChubGFfbGVuKHRiW0lGTEFfQUREUkVTU10pICE9IEVUSF9BTEVO
KQoJCQlyZXR1cm4gLUVJTlZBTDsKCQlpZiAoIWlzX3ZhbGlkX2V0aGVyX2FkZHIobmxhX2RhdGEo
dGJbSUZMQV9BRERSRVNTXSkpKQoJCQlyZXR1cm4gLUVBRERSTk9UQVZBSUw7Cgl9CglyZXR1cm4g
MDsKfQoKc3RhdGljIGludCBpZmJfZ2V0X3R4X3F1ZXVlcyhzdHJ1Y3QgbmV0ICpuZXQsIHN0cnVj
dCBubGF0dHIgKnRiW10sCgkJCSAgICAgdW5zaWduZWQgaW50ICpudW1fdHhfcXVldWVzLAoJCQkg
ICAgIHVuc2lnbmVkIGludCAqcmVhbF9udW1fdHhfcXVldWVzKQp7CgkqbnVtX3R4X3F1ZXVlcyA9
IG51bXR4cXM7CgkqcmVhbF9udW1fdHhfcXVldWVzID0gbnVtdHhxczsKCglyZXR1cm4gMDsKfQoK
c3RhdGljIHN0cnVjdCBydG5sX2xpbmtfb3BzIGlmYl9saW5rX29wcyBfX3JlYWRfbW9zdGx5ID0g
ewoJLmtpbmQJCT0gImlmYiIsCgkuc2V0dXAJCT0gaWZiX3NldHVwLAoJLnZhbGlkYXRlCT0gaWZi
X3ZhbGlkYXRlLAoJLmdldF90eF9xdWV1ZXMJPSBpZmJfZ2V0X3R4X3F1ZXVlcywKCS5wcml2X3Np
emUJPSBzaXplb2Yoc3RydWN0IGlmYl9wcml2YXRlKSwKfTsKCnN0YXRpYyBpbnQgX19pbml0IGlm
Yl9pbml0X29uZShpbnQgaW5kZXgpCnsKCXN0cnVjdCBuZXRfZGV2aWNlICpkZXZfaWZiOwoJaW50
IGVycjsKCglkZXZfaWZiID0gYWxsb2NfbmV0ZGV2X21xKHNpemVvZihzdHJ1Y3QgaWZiX3ByaXZh
dGUpLCAiaWZiJWQiLAoJCQkJICBpZmJfc2V0dXAsIG51bXR4cXMpOwoKCWlmICghZGV2X2lmYikK
CQlyZXR1cm4gLUVOT01FTTsKCgllcnIgPSBkZXZfYWxsb2NfbmFtZShkZXZfaWZiLCBkZXZfaWZi
LT5uYW1lKTsKCWlmIChlcnIgPCAwKQoJCWdvdG8gZXJyOwoKCWRldl9pZmItPnJ0bmxfbGlua19v
cHMgPSAmaWZiX2xpbmtfb3BzOwoJZXJyID0gcmVnaXN0ZXJfbmV0ZGV2aWNlKGRldl9pZmIpOwoJ
aWYgKGVyciA8IDApCgkJZ290byBlcnI7CgoJcmV0dXJuIDA7CgplcnI6CglmcmVlX25ldGRldihk
ZXZfaWZiKTsKCXJldHVybiBlcnI7Cn0KCnN0YXRpYyBpbnQgX19pbml0IGlmYl9pbml0X21vZHVs
ZSh2b2lkKQp7CglpbnQgaSwgZXJyOwoKCWdldF9yYW5kb21fYnl0ZXMoJnNpbXBsZV90eF9oYXNo
cm5kLCA0KTsKCXJ0bmxfbG9jaygpOwoJZXJyID0gX19ydG5sX2xpbmtfcmVnaXN0ZXIoJmlmYl9s
aW5rX29wcyk7Cglmb3IgKGkgPSAwOyBpIDwgbnVtaWZicyAmJiAhZXJyOyBpKyspCgkJZXJyID0g
aWZiX2luaXRfb25lKGkpOwoJaWYgKGVycikKCQlfX3J0bmxfbGlua191bnJlZ2lzdGVyKCZpZmJf
bGlua19vcHMpOwoJcnRubF91bmxvY2soKTsKCglyZXR1cm4gZXJyOwp9CgpzdGF0aWMgdm9pZCBf
X2V4aXQgaWZiX2NsZWFudXBfbW9kdWxlKHZvaWQpCnsKCXJ0bmxfbGlua191bnJlZ2lzdGVyKCZp
ZmJfbGlua19vcHMpOwp9Cgptb2R1bGVfaW5pdChpZmJfaW5pdF9tb2R1bGUpOwptb2R1bGVfZXhp
dChpZmJfY2xlYW51cF9tb2R1bGUpOwpNT0RVTEVfTElDRU5TRSgiR1BMIik7Ck1PRFVMRV9BVVRI
T1IoIkphbWFsIEhhZGkgU2FsaW0iKTsKTU9EVUxFX0FMSUFTX1JUTkxfTElOSygiaWZiIik7Cg==
--00504502cb24a0476b04781290a6--

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: [PATCH] ifb: add multi-queue support
Date: Wed, 11 Nov 2009 17:51:13 +0800
Message-ID: <4AFA8911.7050204@gmail.com>
Reply-To: xiaosuo@gmail.com
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: 7bit
Cc: Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:32911 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752464AbZKKJvM (ORCPT
	<rfc822;netdev@vger.kernel.org>); Wed, 11 Nov 2009 04:51:12 -0500
Received: by pwi3 with SMTP id 3so591516pwi.21
        for <netdev@vger.kernel.org>; Wed, 11 Nov 2009 01:51:18 -0800 (PST)
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

ifb: add multi-queue support

Add multi-queue support, and one kernel thread is created for per queue.
It can used to emulate multi-queue NIC in software, and distribute work
among CPUs.
gentux linux # modprobe ifb numtxqs=2
gentux linux # ifconfig ifb0 up
gentux linux # pgrep ifb0
18508
18509
gentux linux # taskset -p 1 18508
pid 18508's current affinity mask: 3
pid 18508's new affinity mask: 1
gentux linux # taskset -p 2 18509
pid 18509's current affinity mask: 3
pid 18509's new affinity mask: 2
gentux linux # tc qdisc add dev br0 ingress
gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
action mirred egress redirect dev ifb0

This patch also introduces a ip link option "numtxqs" for specifying the
number of the TX queues. so you can add a new ifb with the command:

ip link add numtxqs 4 type ifb

Signed-off-by: Changli Gao <xiaosuo@gmail.com>
----
drivers/net/ifb.c | 414 ++++++++++++++++++++++++++++++++----------------
include/linux/if_link.h | 1
net/core/rtnetlink.c | 3
3 files changed, 283 insertions(+), 135 deletions(-)

diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 69c2566..ac04e85 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -33,161 +33,157 @@
 #include <linux/etherdevice.h>
 #include <linux/init.h>
 #include <linux/moduleparam.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <net/ip.h>
 #include <net/pkt_sched.h>
 #include <net/net_namespace.h>
 
-#define TX_TIMEOUT  (2*HZ)
-
 #define TX_Q_LIMIT    32
+
+struct ifb_private_q {
+	struct net_device	*dev;
+	struct sk_buff_head	rq;
+	struct sk_buff_head	tq;
+	wait_queue_head_t	wq;
+	struct task_struct	*task;
+	unsigned long		rx_packets;
+	unsigned long		rx_bytes;
+	unsigned long		rx_dropped;
+} ____cacheline_aligned_in_smp;
+
 struct ifb_private {
-	struct tasklet_struct   ifb_tasklet;
-	int     tasklet_pending;
-	/* mostly debug stats leave in for now */
-	unsigned long   st_task_enter; /* tasklet entered */
-	unsigned long   st_txq_refl_try; /* transmit queue refill attempt */
-	unsigned long   st_rxq_enter; /* receive queue entered */
-	unsigned long   st_rx2tx_tran; /* receive to trasmit transfers */
-	unsigned long   st_rxq_notenter; /*receiveQ not entered, resched */
-	unsigned long   st_rx_frm_egr; /* received from egress path */
-	unsigned long   st_rx_frm_ing; /* received from ingress path */
-	unsigned long   st_rxq_check;
-	unsigned long   st_rxq_rsch;
-	struct sk_buff_head     rq;
-	struct sk_buff_head     tq;
+	struct ifb_private_q	*pq;
 };
 
+/* Number of ifb devices to be set up by this module. */
 static int numifbs = 2;
+module_param(numifbs, int, 0444);
+MODULE_PARM_DESC(numifbs, "Number of ifb devices");
 
-static void ri_tasklet(unsigned long dev);
-static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
-static int ifb_open(struct net_device *dev);
-static int ifb_close(struct net_device *dev);
+/* Number of TX queues per ifb */
+static int numtxqs = 1;
+module_param(numtxqs, int, 0444);
+MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
 
-static void ri_tasklet(unsigned long dev)
+static int ifb_thread(void *priv)
 {
-
-	struct net_device *_dev = (struct net_device *)dev;
-	struct ifb_private *dp = netdev_priv(_dev);
-	struct net_device_stats *stats = &_dev->stats;
-	struct netdev_queue *txq;
+	struct ifb_private_q *pq = priv;
+	struct net_device *dev = pq->dev;
+	int num = pq - ((struct ifb_private *)netdev_priv(dev))->pq;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 	struct sk_buff *skb;
-
-	txq = netdev_get_tx_queue(_dev, 0);
-	dp->st_task_enter++;
-	if ((skb = skb_peek(&dp->tq)) == NULL) {
-		dp->st_txq_refl_try++;
-		if (__netif_tx_trylock(txq)) {
-			dp->st_rxq_enter++;
-			while ((skb = skb_dequeue(&dp->rq)) != NULL) {
-				skb_queue_tail(&dp->tq, skb);
-				dp->st_rx2tx_tran++;
-			}
-			__netif_tx_unlock(txq);
-		} else {
-			/* reschedule */
-			dp->st_rxq_notenter++;
-			goto resched;
+	DEFINE_WAIT(wait);
+
+	while (1) {
+		/* move skb from rq to tq */
+		while (1) {
+			prepare_to_wait(&pq->wq, &wait, TASK_UNINTERRUPTIBLE);
+			__netif_tx_lock_bh(txq);
+			while ((skb = skb_dequeue(&pq->rq)) != NULL)
+				skb_queue_tail(&pq->tq, skb);
+			if (netif_queue_stopped(dev))
+				netif_wake_queue(dev);
+			__netif_tx_unlock_bh(txq);
+			if (kthread_should_stop() || !skb_queue_empty(&pq->tq))
+				break;
+			schedule();
 		}
-	}
-
-	while ((skb = skb_dequeue(&dp->tq)) != NULL) {
-		u32 from = G_TC_FROM(skb->tc_verd);
-
-		skb->tc_verd = 0;
-		skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
-		stats->tx_packets++;
-		stats->tx_bytes +=skb->len;
-
-		rcu_read_lock();
-		skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
-		if (!skb->dev) {
-			rcu_read_unlock();
-			dev_kfree_skb(skb);
-			stats->tx_dropped++;
+		finish_wait(&pq->wq, &wait);
+		if (kthread_should_stop())
 			break;
+		if (need_resched())
+			schedule();
+
+		/* transfer packets */
+		while ((skb = skb_dequeue(&pq->tq)) != NULL) {
+			u32 from = G_TC_FROM(skb->tc_verd);
+	
+			skb->tc_verd = 0;
+			skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
+			txq->tx_packets++;
+			txq->tx_bytes +=skb->len;
+
+			rcu_read_lock();
+			skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
+			if (!skb->dev) {
+				rcu_read_unlock();
+				dev_kfree_skb(skb);
+				txq->tx_dropped++;
+				break;
+			}
+			rcu_read_unlock();
+			skb->iif = dev->ifindex;
+	
+			if (from & AT_EGRESS) {
+				dev_queue_xmit(skb);
+			} else if (from & AT_INGRESS) {
+				skb_pull(skb, skb->dev->hard_header_len);
+				netif_rx_ni(skb);
+			} else
+				BUG();
 		}
-		rcu_read_unlock();
-		skb->iif = _dev->ifindex;
-
-		if (from & AT_EGRESS) {
-			dp->st_rx_frm_egr++;
-			dev_queue_xmit(skb);
-		} else if (from & AT_INGRESS) {
-			dp->st_rx_frm_ing++;
-			skb_pull(skb, skb->dev->hard_header_len);
-			netif_rx(skb);
-		} else
-			BUG();
-	}
-
-	if (__netif_tx_trylock(txq)) {
-		dp->st_rxq_check++;
-		if ((skb = skb_peek(&dp->rq)) == NULL) {
-			dp->tasklet_pending = 0;
-			if (netif_queue_stopped(_dev))
-				netif_wake_queue(_dev);
-		} else {
-			dp->st_rxq_rsch++;
-			__netif_tx_unlock(txq);
-			goto resched;
-		}
-		__netif_tx_unlock(txq);
-	} else {
-resched:
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
 	}
 
+	return 0;
 }
 
-static const struct net_device_ops ifb_netdev_ops = {
-	.ndo_open	= ifb_open,
-	.ndo_stop	= ifb_close,
-	.ndo_start_xmit	= ifb_xmit,
-	.ndo_validate_addr = eth_validate_addr,
-};
-
-static void ifb_setup(struct net_device *dev)
+struct net_device_stats* ifb_get_stats(struct net_device *dev)
 {
-	/* Initialize the device structure. */
-	dev->destructor = free_netdev;
-	dev->netdev_ops = &ifb_netdev_ops;
+	struct net_device_stats *stats = &dev->stats;
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	struct netdev_queue *txq;
+	int i;
+	unsigned long rx_packets = 0, rx_bytes = 0, rx_dropped = 0;
+	unsigned long tx_packets = 0, tx_bytes = 0, tx_dropped = 0;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		rx_packets += pq[i].rx_packets;
+		rx_bytes += pq[i].rx_bytes;
+		rx_dropped += pq[i].rx_dropped;
+		txq = netdev_get_tx_queue(dev, i);
+		tx_packets += txq->tx_packets;
+		tx_bytes += txq->tx_bytes;
+		tx_dropped += txq->tx_dropped;
+	}
 
-	/* Fill in device structure with ethernet-generic values. */
-	ether_setup(dev);
-	dev->tx_queue_len = TX_Q_LIMIT;
+	stats->rx_packets = rx_packets;
+	stats->rx_bytes = rx_bytes;
+	stats->rx_dropped = rx_dropped;
+	stats->tx_packets = tx_packets;
+	stats->tx_bytes = tx_bytes;
+	stats->tx_dropped = tx_dropped;
 
-	dev->flags |= IFF_NOARP;
-	dev->flags &= ~IFF_MULTICAST;
-	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
-	random_ether_addr(dev->dev_addr);
+	return stats;
 }
 
 static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 {
-	struct ifb_private *dp = netdev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
 	u32 from = G_TC_FROM(skb->tc_verd);
+	int num = skb_get_queue_mapping(skb);
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq + num;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 
-	stats->rx_packets++;
-	stats->rx_bytes+=skb->len;
+	pq->rx_packets++;
+	pq->rx_bytes += skb->len;
 
 	if (!(from & (AT_INGRESS|AT_EGRESS)) || !skb->iif) {
 		dev_kfree_skb(skb);
-		stats->rx_dropped++;
+		pq->rx_dropped++;
 		return NETDEV_TX_OK;
 	}
 
-	if (skb_queue_len(&dp->rq) >= dev->tx_queue_len) {
+	txq->trans_start = jiffies;
+	skb_queue_tail(&pq->rq, skb);
+	if (skb_queue_len(&pq->rq) >= dev->tx_queue_len)
 		netif_stop_queue(dev);
-	}
-
-	dev->trans_start = jiffies;
-	skb_queue_tail(&dp->rq, skb);
-	if (!dp->tasklet_pending) {
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
-	}
+	if (skb_queue_len(&pq->rq) == 1)
+		wake_up(&pq->wq);
 
 	return NETDEV_TX_OK;
 }
@@ -195,26 +191,162 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 static int ifb_close(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
 
-	tasklet_kill(&dp->ifb_tasklet);
 	netif_stop_queue(dev);
-	skb_queue_purge(&dp->rq);
-	skb_queue_purge(&dp->tq);
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		kthread_stop(pq[i].task);
+		skb_queue_purge(&pq[i].tq);
+		skb_queue_purge(&pq[i].rq);
+	}
+
 	return 0;
 }
 
 static int ifb_open(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
-
-	tasklet_init(&dp->ifb_tasklet, ri_tasklet, (unsigned long)dev);
-	skb_queue_head_init(&dp->rq);
-	skb_queue_head_init(&dp->tq);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
+	
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		pq[i].task = kthread_run(ifb_thread, &pq[i], "%s/%d", dev->name,
+					 i);
+		if (IS_ERR(pq[i].task)) {
+			int err = PTR_ERR(pq[i].task);
+			while (--i >= 0)
+				kthread_stop(pq[i].task);
+			return err;
+		}
+	}
 	netif_start_queue(dev);
 
 	return 0;
 }
 
+static u32 simple_tx_hashrnd;
+
+static u16 ifb_select_queue(struct net_device *dev, struct sk_buff *skb)
+{
+	u32 addr1, addr2;
+	u32 hash, ihl;
+	union {
+		u16 in16[2];
+		u32 in32;
+	} ports;
+	u8 ip_proto;
+
+	if ((hash = skb_rx_queue_recorded(skb))) {
+		while (hash >= dev->real_num_tx_queues)
+			hash -= dev->real_num_tx_queues;
+		return hash;
+	}
+
+	switch (skb->protocol) {
+	case __constant_htons(ETH_P_IP):
+		if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)))
+			ip_proto = ip_hdr(skb)->protocol;
+		else
+			ip_proto = 0;
+		addr1 = ip_hdr(skb)->saddr;
+		addr2 = ip_hdr(skb)->daddr;
+		ihl = ip_hdr(skb)->ihl << 2;
+		break;
+	case __constant_htons(ETH_P_IPV6):
+		ip_proto = ipv6_hdr(skb)->nexthdr;
+		addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
+		addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
+		ihl = 10;
+		break;
+	default:
+		return 0;
+	}
+	if (addr1 > addr2)
+		swap(addr1, addr2);
+
+	switch (ip_proto) {
+	case IPPROTO_TCP:
+	case IPPROTO_UDP:
+	case IPPROTO_DCCP:
+	case IPPROTO_ESP:
+	case IPPROTO_AH:
+	case IPPROTO_SCTP:
+	case IPPROTO_UDPLITE:
+		ports.in32 = *((u32 *) (skb_network_header(skb) + ihl));
+		if (ports.in16[0] > ports.in16[1])
+			swap(ports.in16[0], ports.in16[1]);
+		break;
+
+	default:
+		ports.in32 = 0;
+		break;
+	}
+
+	hash = jhash_3words(addr1, addr2, ports.in32,
+			    simple_tx_hashrnd ^ ip_proto);
+
+	return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
+}
+
+static int ifb_init(struct net_device *dev)
+{
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
+
+	pq = kmalloc(sizeof(*pq) * dev->real_num_tx_queues, GFP_KERNEL);
+	if (pq == NULL)
+		return -ENOMEM;
+	dp->pq = pq;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		pq[i].dev = dev;
+		skb_queue_head_init(&pq[i].rq);
+		skb_queue_head_init(&pq[i].tq);
+		init_waitqueue_head(&pq[i].wq);
+		pq[i].rx_packets = 0;
+		pq[i].rx_bytes = 0;
+		pq[i].rx_dropped = 0;
+	}
+
+	return 0;
+}
+
+static void ifb_uninit(struct net_device *dev)
+{
+	struct ifb_private *dp = netdev_priv(dev);
+
+	kfree(dp->pq);
+}
+
+static const struct net_device_ops ifb_netdev_ops = {
+	.ndo_init		= ifb_init,
+	.ndo_uninit		= ifb_uninit,
+	.ndo_open		= ifb_open,
+	.ndo_stop		= ifb_close,
+	.ndo_start_xmit		= ifb_xmit,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_select_queue	= ifb_select_queue,
+	.ndo_get_stats		= ifb_get_stats,
+};
+
+static void ifb_setup(struct net_device *dev)
+{
+	/* Initialize the device structure. */
+	dev->destructor = free_netdev;
+	dev->netdev_ops = &ifb_netdev_ops;
+
+	/* Fill in device structure with ethernet-generic values. */
+	ether_setup(dev);
+	dev->tx_queue_len = TX_Q_LIMIT;
+
+	dev->flags |= IFF_NOARP;
+	dev->flags &= ~IFF_MULTICAST;
+	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+	random_ether_addr(dev->dev_addr);
+}
+
 static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 {
 	if (tb[IFLA_ADDRESS]) {
@@ -226,24 +358,36 @@ static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 	return 0;
 }
 
+static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
+			     unsigned int *num_tx_queues,
+			     unsigned int *real_num_tx_queues)
+{
+	if (tb[IFLA_NTXQ]) {
+		*num_tx_queues = nla_get_u16(tb[IFLA_NTXQ]);
+		*real_num_tx_queues = *num_tx_queues;
+	} else {
+		*num_tx_queues = numtxqs;
+		*real_num_tx_queues = numtxqs;
+	}
+
+	return 0;
+}
+
 static struct rtnl_link_ops ifb_link_ops __read_mostly = {
 	.kind		= "ifb",
-	.priv_size	= sizeof(struct ifb_private),
 	.setup		= ifb_setup,
 	.validate	= ifb_validate,
+	.get_tx_queues	= ifb_get_tx_queues,
+	.priv_size	= sizeof(struct ifb_private),
 };
 
-/* Number of ifb devices to be set up by this module. */
-module_param(numifbs, int, 0);
-MODULE_PARM_DESC(numifbs, "Number of ifb devices");
-
 static int __init ifb_init_one(int index)
 {
 	struct net_device *dev_ifb;
 	int err;
 
-	dev_ifb = alloc_netdev(sizeof(struct ifb_private),
-				 "ifb%d", ifb_setup);
+	dev_ifb = alloc_netdev_mq(sizeof(struct ifb_private), "ifb%d",
+				  ifb_setup, numtxqs);
 
 	if (!dev_ifb)
 		return -ENOMEM;
@@ -268,9 +412,9 @@ static int __init ifb_init_module(void)
 {
 	int i, err;
 
+	get_random_bytes(&simple_tx_hashrnd, 4);
 	rtnl_lock();
 	err = __rtnl_link_register(&ifb_link_ops);
-
 	for (i = 0; i < numifbs && !err; i++)
 		err = ifb_init_one(i);
 	if (err)
diff --git a/include/linux/if_link.h b/include/linux/if_link.h
index 1d3b242..99da411 100644
--- a/include/linux/if_link.h
+++ b/include/linux/if_link.h
@@ -78,6 +78,7 @@ enum {
 #define IFLA_LINKINFO IFLA_LINKINFO
 	IFLA_NET_NS_PID,
 	IFLA_IFALIAS,
+	IFLA_NTXQ,
 	__IFLA_MAX
 };
 
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 33148a5..7b94fd7 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -597,6 +597,7 @@ static inline size_t if_nlmsg_size(const struct net_device *dev)
 	       + nla_total_size(4) /* IFLA_MASTER */
 	       + nla_total_size(1) /* IFLA_OPERSTATE */
 	       + nla_total_size(1) /* IFLA_LINKMODE */
+	       + nla_total_size(2) /* IFLA_NTXQ */
 	       + rtnl_link_get_size(dev); /* IFLA_LINKINFO */
 }
 
@@ -627,6 +628,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
 		   netif_running(dev) ? dev->operstate : IF_OPER_DOWN);
 	NLA_PUT_U8(skb, IFLA_LINKMODE, dev->link_mode);
 	NLA_PUT_U32(skb, IFLA_MTU, dev->mtu);
+	NLA_PUT_U16(skb, IFLA_NTXQ, dev->real_num_tx_queues);
 
 	if (dev->ifindex != dev->iflink)
 		NLA_PUT_U32(skb, IFLA_LINK, dev->iflink);
@@ -725,6 +727,7 @@ const struct nla_policy ifla_policy[IFLA_MAX+1] = {
 	[IFLA_LINKINFO]		= { .type = NLA_NESTED },
 	[IFLA_NET_NS_PID]	= { .type = NLA_U32 },
 	[IFLA_IFALIAS]	        = { .type = NLA_STRING, .len = IFALIASZ-1 },
+	[IFLA_NTXQ]		= { .type = NLA_U16 },
 };
 EXPORT_SYMBOL(ifla_policy);
 



From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Wed, 11 Nov 2009 17:56:59 +0800
Message-ID: <412e6f7f0911110156l350dbb8ewdb1d669c30555770@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com>
Mime-Version: 1.0
Content-Type: multipart/mixed; boundary=000e0cd2dff09b9a1b0478157272
Cc: Patrick McHardy <kaber@trash.net>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:49838 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752536AbZKKJ4x (ORCPT
	<rfc822;netdev@vger.kernel.org>); Wed, 11 Nov 2009 04:56:53 -0500
Received: by pwi3 with SMTP id 3so593643pwi.21
        for <netdev@vger.kernel.org>; Wed, 11 Nov 2009 01:56:59 -0800 (PST)
In-Reply-To: <4AFA8911.7050204@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

--000e0cd2dff09b9a1b0478157272
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: quoted-printable

2009/11/11 Changli Gao <xiaosuo@gmail.com>:
> ifb: add multi-queue support
>
> Add multi-queue support, and one kernel thread is created for per queue.
> It can used to emulate multi-queue NIC in software, and distribute work
> among CPUs.
> gentux linux # modprobe ifb numtxqs=3D2
> gentux linux # ifconfig ifb0 up
> gentux linux # pgrep ifb0
> 18508
> 18509
> gentux linux # taskset -p 1 18508
> pid 18508's current affinity mask: 3
> pid 18508's new affinity mask: 1
> gentux linux # taskset -p 2 18509
> pid 18509's current affinity mask: 3
> pid 18509's new affinity mask: 2
> gentux linux # tc qdisc add dev br0 ingress
> gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
> action mirred egress redirect dev ifb0
>
> This patch also introduces a ip link option "numtxqs" for specifying the
> number of the TX queues. so you can add a new ifb with the command:
>
> ip link add numtxqs 4 type ifb
>

The corresponding patch for iproute2 is attached too. And you need to
update the linux header files too.


--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

--000e0cd2dff09b9a1b0478157272
Content-Type: application/octet-stream; name="iproute2-support-numtxqs.diff"
Content-Disposition: attachment; filename="iproute2-support-numtxqs.diff"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_g1vx20xy0

LS0tIGlwL2lwbGluay5jLm9yaWcJMjAwOS0xMS0xMSAxNjo1Njo0MS4wMDAwMDAwMDAgKzA4MDAK
KysrIGlwL2lwbGluay5jCTIwMDktMTEtMTEgMTc6MzI6NTIuMDAwMDAwMDAwICswODAwCkBAIC00
OCw2ICs0OCw3IEBACiAJCWZwcmludGYoc3RkZXJyLCAiICAgICAgICAgICAgICAgICAgIFsgYWRk
cmVzcyBMTEFERFIgXVxuIik7CiAJCWZwcmludGYoc3RkZXJyLCAiICAgICAgICAgICAgICAgICAg
IFsgYnJvYWRjYXN0IExMQUREUiBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIsICIgICAgICAgICAg
ICAgICAgICAgWyBtdHUgTVRVIF1cbiIpOworCQlmcHJpbnRmKHN0ZGVyciwgIiAgICAgICAgICAg
ICAgICAgICBbIG51bXR4cXMgTlVNVFhRUyBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIsICIgICAg
ICAgICAgICAgICAgICAgdHlwZSBUWVBFIFsgQVJHUyBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIs
ICIgICAgICAgaXAgbGluayBkZWxldGUgREVWIHR5cGUgVFlQRSBbIEFSR1MgXVxuIik7CiAJCWZw
cmludGYoc3RkZXJyLCAiXG4iKTsKQEAgLTE4MCw2ICsxODEsNyBAQAogCWludCBxbGVuID0gLTE7
CiAJaW50IG10dSA9IC0xOwogCWludCBuZXRucyA9IC0xOworCWludCBudW10eHFzID0gLTE7CiAK
IAlyZXQgPSBhcmdjOwogCkBAIC0yMjEsNiArMjIzLDEzIEBACiAJCQlpZiAoZ2V0X2ludGVnZXIo
Jm10dSwgKmFyZ3YsIDApKQogCQkJCWludmFyZygiSW52YWxpZCBcIm10dVwiIHZhbHVlXG4iLCAq
YXJndik7CiAJCQlhZGRhdHRyX2woJnJlcS0+biwgc2l6ZW9mKCpyZXEpLCBJRkxBX01UVSwgJm10
dSwgNCk7CisJCX0gZWxzZSBpZiAoc3RyY21wKCphcmd2LCAibnVtdHhxcyIpID09IDApIHsKKwkJ
CU5FWFRfQVJHKCk7CisJCQlpZiAobnVtdHhxcyAhPSAtMSkKKwkJCQlkdXBhcmcoIm51bXR4cXMi
LCAqYXJndik7CisJCQlpZiAoZ2V0X2ludGVnZXIoJm51bXR4cXMsICphcmd2LCAwKSkKKwkJCQlp
bnZhcmcoIkludmFsaWQgXCJudW10eHFzXCIgdmFsdWVcbiIsICphcmd2KTsKKwkJCWFkZGF0dHJf
bCgmcmVxLT5uLCBzaXplb2YoKnJlcSksIElGTEFfTlRYUSwgJm51bXR4cXMsIDIpOwogICAgICAg
ICAgICAgICAgIH0gZWxzZSBpZiAoc3RyY21wKCphcmd2LCAibmV0bnMiKSA9PSAwKSB7CiAgICAg
ICAgICAgICAgICAgICAgICAgICBORVhUX0FSRygpOwogICAgICAgICAgICAgICAgICAgICAgICAg
aWYgKG5ldG5zICE9IC0xKQo=
--000e0cd2dff09b9a1b0478157272--

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Wed, 11 Nov 2009 11:30:13 +0100
Message-ID: <4AFA9235.1030004@gmail.com>
References: <4AFA8911.7050204@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: xiaosuo@gmail.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:48347 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1753380AbZKKKaS (ORCPT
	<rfc822;netdev@vger.kernel.org>); Wed, 11 Nov 2009 05:30:18 -0500
In-Reply-To: <4AFA8911.7050204@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =A8=A6crit :
> ifb: add multi-queue support
>=20
> Add multi-queue support, and one kernel thread is created for per que=
ue.
> It can used to emulate multi-queue NIC in software, and distribute wo=
rk
> among CPUs.
> gentux linux # modprobe ifb numtxqs=3D2
> gentux linux # ifconfig ifb0 up
> gentux linux # pgrep ifb0
> 18508
> 18509
> gentux linux # taskset -p 1 18508
> pid 18508's current affinity mask: 3
> pid 18508's new affinity mask: 1
> gentux linux # taskset -p 2 18509
> pid 18509's current affinity mask: 3
> pid 18509's new affinity mask: 2
> gentux linux # tc qdisc add dev br0 ingress
> gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
> action mirred egress redirect dev ifb0
>=20
> This patch also introduces a ip link option "numtxqs" for specifying =
the
> number of the TX queues. so you can add a new ifb with the command:
>=20
> ip link add numtxqs 4 type ifb
>=20
> Signed-off-by: Changli Gao <xiaosuo@gmail.com>
> ----
> drivers/net/ifb.c | 414 ++++++++++++++++++++++++++++++++-------------=
---
> include/linux/if_link.h | 1
> net/core/rtnetlink.c | 3
> 3 files changed, 283 insertions(+), 135 deletions(-)
>=20
> diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
> index 69c2566..ac04e85 100644
> --- a/drivers/net/ifb.c
> +++ b/drivers/net/ifb.c
> @@ -33,161 +33,157 @@
>  #include <linux/etherdevice.h>
>  #include <linux/init.h>
>  #include <linux/moduleparam.h>
> +#include <linux/wait.h>
> +#include <linux/sched.h>
> +#include <linux/kthread.h>
> +#include <linux/ip.h>
> +#include <linux/ipv6.h>
> +#include <net/ip.h>
>  #include <net/pkt_sched.h>
>  #include <net/net_namespace.h>
> =20
> -#define TX_TIMEOUT  (2*HZ)
> -
>  #define TX_Q_LIMIT    32
> +
> +struct ifb_private_q {
> +	struct net_device	*dev;
> +	struct sk_buff_head	rq;
> +	struct sk_buff_head	tq;
> +	wait_queue_head_t	wq;
> +	struct task_struct	*task;
> +	unsigned long		rx_packets;
> +	unsigned long		rx_bytes;
> +	unsigned long		rx_dropped;
> +} ____cacheline_aligned_in_smp;
> +

Could you split this struct in two parts, one used by ifb_xmit(),
one used by the ifb_thread() ?
This would reduce number of cache line ping pongs...


struct ifb_private_q {

	struct net_device	*dev;
	struct task_struct	*task;

/* used by ifb_xmit() */

	struct sk_buff_head	rq;
	unsigned long		rx_packets;
	unsigned long		rx_bytes;
	unsigned long		rx_dropped;
	wait_queue_head_t	wq;

/* used by ifb_thread() */

	struct sk_buff_head	tq ____cacheline_aligned_in_smp;
} ____cacheline_aligned_in_smp;


Or even better, tq could be local to ifb_xmit()

ifb_xmit() would purge it itself, and could use __skb_dequeue() and
other lockless primitives.

The whole :

while ((skb =3D skb_dequeue(&pq->rq)) !=3D NULL)
	skb_queue_tail(&pq->tq, skb);

could be optimized a lot IMHO, its almost a skb_queue_splice() thing...


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Wed, 11 Nov 2009 18:57:08 +0800
Message-ID: <412e6f7f0911110257s2bbca7aei648e324832ae455d@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFA9235.1030004@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-px0-f177.google.com ([209.85.216.177]:53757 "EHLO
	mail-px0-f177.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755417AbZKKK5D convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Wed, 11 Nov 2009 05:57:03 -0500
Received: by pxi7 with SMTP id 7so700812pxi.17
        for <netdev@vger.kernel.org>; Wed, 11 Nov 2009 02:57:08 -0800 (PST)
In-Reply-To: <4AFA9235.1030004@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

2009/11/11 Eric Dumazet <eric.dumazet@gmail.com>:
>
> Or even better, tq could be local to ifb_xmit()
>
> ifb_xmit() would purge it itself, and could use __skb_dequeue() and
> other lockless primitives.
>
> The whole :
>
> while ((skb =3D skb_dequeue(&pq->rq)) !=3D NULL)
>        skb_queue_tail(&pq->tq, skb);
>
> could be optimized a lot IMHO, its almost a skb_queue_splice() thing.=
=2E.
>

Thanks, it is a good idea. Any others?


--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Wed, 11 Nov 2009 16:59:32 +0100
Message-ID: <4AFADF64.8070601@trash.net>
References: <4AFA8911.7050204@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=ISO-8859-15
Content-Transfer-Encoding: 7bit
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: xiaosuo@gmail.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:34052 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1754149AbZKKP7c (ORCPT
	<rfc822;netdev@vger.kernel.org>); Wed, 11 Nov 2009 10:59:32 -0500
In-Reply-To: <4AFA8911.7050204@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
> index 69c2566..ac04e85 100644
> --- a/drivers/net/ifb.c
> +++ b/drivers/net/ifb.c
> ...
> +/* Number of ifb devices to be set up by this module. */
>  static int numifbs = 2;
> +module_param(numifbs, int, 0444);
> +MODULE_PARM_DESC(numifbs, "Number of ifb devices");
>  
> -static void ri_tasklet(unsigned long dev);
> -static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
> -static int ifb_open(struct net_device *dev);
> -static int ifb_close(struct net_device *dev);
> +/* Number of TX queues per ifb */
> +static int numtxqs = 1;
> +module_param(numtxqs, int, 0444);
> +MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");

unsigned?

> +		while ((skb = skb_dequeue(&pq->tq)) != NULL) {
> +			u32 from = G_TC_FROM(skb->tc_verd);
> +	
> +			skb->tc_verd = 0;
> +			skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
> +			txq->tx_packets++;
> +			txq->tx_bytes +=skb->len;
> +
> +			rcu_read_lock();
> +			skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
> +			if (!skb->dev) {
> +				rcu_read_unlock();
> +				dev_kfree_skb(skb);
> +				txq->tx_dropped++;
> +				break;
> +			}
> +			rcu_read_unlock();
> +			skb->iif = dev->ifindex;

What protects the device from disappearing here and below during
dev_queue_xmit() and netif_rx_ni()?

> +	
> +			if (from & AT_EGRESS) {
> +				dev_queue_xmit(skb);
> +			} else if (from & AT_INGRESS) {
> +				skb_pull(skb, skb->dev->hard_header_len);
> +				netif_rx_ni(skb);
> +			} else
> +				BUG();
>  		}

> +static u32 simple_tx_hashrnd;
> +
> +static u16 ifb_select_queue(struct net_device *dev, struct sk_buff *skb)
> +{
> +	u32 addr1, addr2;
> +	u32 hash, ihl;
> +	union {
> +		u16 in16[2];
> +		u32 in32;
> +	} ports;
> +	u8 ip_proto;
> +
> +	if ((hash = skb_rx_queue_recorded(skb))) {
> +		while (hash >= dev->real_num_tx_queues)
> +			hash -= dev->real_num_tx_queues;
> +		return hash;
> +	}
> +
> +	switch (skb->protocol) {
> +	case __constant_htons(ETH_P_IP):
> +		if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)))
> +			ip_proto = ip_hdr(skb)->protocol;
> +		else
> +			ip_proto = 0;

So fragments will get reordered?

> +		addr1 = ip_hdr(skb)->saddr;
> +		addr2 = ip_hdr(skb)->daddr;
> +		ihl = ip_hdr(skb)->ihl << 2;

ip_hdrlen()?

> +		break;
> +	case __constant_htons(ETH_P_IPV6):
> +		ip_proto = ipv6_hdr(skb)->nexthdr;
> +		addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
> +		addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
> +		ihl = 10;

Where does 10 come from?

> +		break;
> +	default:
> +		return 0;

Perhaps hash on skb->protocol here.

> +	}
> +	if (addr1 > addr2)
> +		swap(addr1, addr2);
> +
> +	switch (ip_proto) {
> +	case IPPROTO_TCP:
> +	case IPPROTO_UDP:
> +	case IPPROTO_DCCP:
> +	case IPPROTO_ESP:
> +	case IPPROTO_AH:
> +	case IPPROTO_SCTP:
> +	case IPPROTO_UDPLITE:
> +		ports.in32 = *((u32 *) (skb_network_header(skb) + ihl));
> +		if (ports.in16[0] > ports.in16[1])
> +			swap(ports.in16[0], ports.in16[1]);
> +		break;
> +
> +	default:
> +		ports.in32 = 0;
> +		break;
> +	}
> +
> +	hash = jhash_3words(addr1, addr2, ports.in32,
> +			    simple_tx_hashrnd ^ ip_proto);
> +
> +	return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
> +}
> +
> +static int ifb_init(struct net_device *dev)
> +{
> +	struct ifb_private *dp = netdev_priv(dev);
> +	struct ifb_private_q *pq = dp->pq;
> +	int i;
> +
> +	pq = kmalloc(sizeof(*pq) * dev->real_num_tx_queues, GFP_KERNEL);

kcalloc()?

> +	if (pq == NULL)
> +		return -ENOMEM;
> +	dp->pq = pq;
> +
> +	for (i = 0; i < dev->real_num_tx_queues; i++) {
> +		pq[i].dev = dev;
> +		skb_queue_head_init(&pq[i].rq);
> +		skb_queue_head_init(&pq[i].tq);
> +		init_waitqueue_head(&pq[i].wq);
> +		pq[i].rx_packets = 0;
> +		pq[i].rx_bytes = 0;
> +		pq[i].rx_dropped = 0;
> +	}
> +
> +	return 0;
> +}

> +static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
> +			     unsigned int *num_tx_queues,
> +			     unsigned int *real_num_tx_queues)
> +{
> +	if (tb[IFLA_NTXQ]) {
> +		*num_tx_queues = nla_get_u16(tb[IFLA_NTXQ]);

We currently use unsigned ints for the queue number, so please
use an u32 for the attribute as well.

> +		*real_num_tx_queues = *num_tx_queues;
> +	} else {
> +		*num_tx_queues = numtxqs;
> +		*real_num_tx_queues = numtxqs;
> +	}
> +
> +	return 0;
> +}
> +

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 11:12:17 +0800
Message-ID: <412e6f7f0911111912q27f2b0aate56c637349292c3f@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFADF64.8070601@trash.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:60657 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1760042AbZKLDMM convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Wed, 11 Nov 2009 22:12:12 -0500
Received: by pzk1 with SMTP id 1so222100pzk.33
        for <netdev@vger.kernel.org>; Wed, 11 Nov 2009 19:12:17 -0800 (PST)
In-Reply-To: <4AFADF64.8070601@trash.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Wed, Nov 11, 2009 at 11:59 PM, Patrick McHardy <kaber@trash.net> wro=
te:
> Changli Gao wrote:
>> diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
>> index 69c2566..ac04e85 100644
>> --- a/drivers/net/ifb.c
>> +++ b/drivers/net/ifb.c
>> ...
>> +/* Number of ifb devices to be set up by this module. */
>> =C2=A0static int numifbs =3D 2;
>> +module_param(numifbs, int, 0444);
>> +MODULE_PARM_DESC(numifbs, "Number of ifb devices");
>>
>> -static void ri_tasklet(unsigned long dev);
>> -static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device =
*dev);
>> -static int ifb_open(struct net_device *dev);
>> -static int ifb_close(struct net_device *dev);
>> +/* Number of TX queues per ifb */
>> +static int numtxqs =3D 1;
>> +module_param(numtxqs, int, 0444);
>> +MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
>
> unsigned?

Yea, unsigned is better, and I need to check whether its value is
smaller than 1 somewhere. The same will be done with IFLA_NTXQ.

>
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 while ((skb =3D skb_dequ=
eue(&pq->tq)) !=3D NULL) {
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 u32 from =3D G_TC_FROM(skb->tc_verd);
>> +
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 skb->tc_verd =3D 0;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 skb->tc_verd =3D SET_TC_NCLS(skb->tc_verd);
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 txq->tx_packets++;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 txq->tx_bytes +=3Dskb->len;
>> +
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 rcu_read_lock();
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 skb->dev =3D dev_get_by_index_rcu(&init_net, skb->iif);
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 if (!skb->dev) {
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 rcu_read_unlock();
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 dev_kfree_skb(skb);
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 txq->tx_dropped++;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 }
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 rcu_read_unlock();
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 skb->iif =3D dev->ifindex;
>
> What protects the device from disappearing here and below during
> dev_queue_xmit() and netif_rx_ni()?

=46or dev_queue_xmit(), dev is holded by skb->_dst, so there is no
problem. But for netif_rx_ni(), I don't know how to prevent the device
disappearing, and it seems that all the NIC drivers have this problem.
Maybe there was the assumption about the execution context of
netif_rx() before. Now softirq can't be executed by softirqd, so the
packet receiving path maybe interleaved. I don't know how to prevent
it happening.


>> +
>> +static u16 ifb_select_queue(struct net_device *dev, struct sk_buff =
*skb)
>> +{
>> + =C2=A0 =C2=A0 u32 addr1, addr2;
>> + =C2=A0 =C2=A0 u32 hash, ihl;
>> + =C2=A0 =C2=A0 union {
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 u16 in16[2];
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 u32 in32;
>> + =C2=A0 =C2=A0 } ports;
>> + =C2=A0 =C2=A0 u8 ip_proto;
>> +
>> + =C2=A0 =C2=A0 if ((hash =3D skb_rx_queue_recorded(skb))) {
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 while (hash >=3D dev->re=
al_num_tx_queues)
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 hash -=3D dev->real_num_tx_queues;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return hash;
>> + =C2=A0 =C2=A0 }
>> +
>> + =C2=A0 =C2=A0 switch (skb->protocol) {
>> + =C2=A0 =C2=A0 case __constant_htons(ETH_P_IP):
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if (!(ip_hdr(skb)->frag_=
off & htons(IP_MF | IP_OFFSET)))
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 ip_proto =3D ip_hdr(skb)->protocol;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 else
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 ip_proto =3D 0;
>
> So fragments will get reordered?

Yea. It is OK, as the number of fragments isn't large. If we decide to
avoid this, we have to reasm them as netfilter does. and it isn't
efficiency.

>
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 addr1 =3D ip_hdr(skb)->s=
addr;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 addr2 =3D ip_hdr(skb)->d=
addr;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ihl =3D ip_hdr(skb)->ihl=
 << 2;
>
> ip_hdrlen()?

Yea, I'll use ip_hdrlen() instead.

>
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
>> + =C2=A0 =C2=A0 case __constant_htons(ETH_P_IPV6):
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ip_proto =3D ipv6_hdr(sk=
b)->nexthdr;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 addr1 =3D ipv6_hdr(skb)-=
>saddr.s6_addr32[3];
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 addr2 =3D ipv6_hdr(skb)-=
>daddr.s6_addr32[3];
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ihl =3D 10;
>
> Where does 10 come from?

It should be 40, after reviewing IPV6, I found that I need to loop
until finding the right protocol value.

>
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
>> + =C2=A0 =C2=A0 default:
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return 0;
>
> Perhaps hash on skb->protocol here.

use return skb->protocol % dev->real_num_tx_queues; instead.

>
>> + =C2=A0 =C2=A0 }
>> + =C2=A0 =C2=A0 if (addr1 > addr2)
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 swap(addr1, addr2);
>> +
>> + =C2=A0 =C2=A0 switch (ip_proto) {
>> + =C2=A0 =C2=A0 case IPPROTO_TCP:
>> + =C2=A0 =C2=A0 case IPPROTO_UDP:
>> + =C2=A0 =C2=A0 case IPPROTO_DCCP:
>> + =C2=A0 =C2=A0 case IPPROTO_ESP:
>> + =C2=A0 =C2=A0 case IPPROTO_AH:
>> + =C2=A0 =C2=A0 case IPPROTO_SCTP:
>> + =C2=A0 =C2=A0 case IPPROTO_UDPLITE:
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ports.in32 =3D *((u32 *)=
 (skb_network_header(skb) + ihl));
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 if (ports.in16[0] > port=
s.in16[1])
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 swap(ports.in16[0], ports.in16[1]);
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
>> +
>> + =C2=A0 =C2=A0 default:
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ports.in32 =3D 0;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
>> + =C2=A0 =C2=A0 }
>> +
>> + =C2=A0 =C2=A0 hash =3D jhash_3words(addr1, addr2, ports.in32,
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 simple_tx_hashrnd ^ ip_proto);
>> +
>> + =C2=A0 =C2=A0 return (u16) (((u64) hash * dev->real_num_tx_queues)=
 >> 32);
>> +}
>> +
>> +static int ifb_init(struct net_device *dev)
>> +{
>> + =C2=A0 =C2=A0 struct ifb_private *dp =3D netdev_priv(dev);
>> + =C2=A0 =C2=A0 struct ifb_private_q *pq =3D dp->pq;
>> + =C2=A0 =C2=A0 int i;
>> +
>> + =C2=A0 =C2=A0 pq =3D kmalloc(sizeof(*pq) * dev->real_num_tx_queues=
, GFP_KERNEL);
>
> kcalloc()?

OK.

>> + =C2=A0 =C2=A0 if (pq =3D=3D NULL)
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 return -ENOMEM;
>> + =C2=A0 =C2=A0 dp->pq =3D pq;
>> +
>> + =C2=A0 =C2=A0 for (i =3D 0; i < dev->real_num_tx_queues; i++) {
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 pq[i].dev =3D dev;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 skb_queue_head_init(&pq[=
i].rq);
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 skb_queue_head_init(&pq[=
i].tq);
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 init_waitqueue_head(&pq[=
i].wq);
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 pq[i].rx_packets =3D 0;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 pq[i].rx_bytes =3D 0;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 pq[i].rx_dropped =3D 0;
>> + =C2=A0 =C2=A0 }
>> +
>> + =C2=A0 =C2=A0 return 0;
>> +}
>
>> +static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *num_tx_queues,
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *real_num_tx_queues)
>> +{
>> + =C2=A0 =C2=A0 if (tb[IFLA_NTXQ]) {
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *num_tx_queues =3D nla_g=
et_u16(tb[IFLA_NTXQ]);
>
> We currently use unsigned ints for the queue number, so please
> use an u32 for the attribute as well.
>
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *real_num_tx_queues =3D =
*num_tx_queues;
>> + =C2=A0 =C2=A0 } else {
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *num_tx_queues =3D numtx=
qs;
>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *real_num_tx_queues =3D =
numtxqs;
>> + =C2=A0 =C2=A0 }
>> +
>> + =C2=A0 =C2=A0 return 0;
>> +}
>> +
>
        u16                     (*ndo_select_queue)(struct net_device *=
dev,
                                                    struct sk_buff *skb=
);
use u16 as the return value so ..., and I think u16 is big enough. If
you insist on this, I'll use u32 instead.


--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Jarek Poplawski <jarkao2@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 08:52:50 +0000
Message-ID: <20091112085250.GA7112@ff.dom.local>
References: <412e6f7f0911111912q27f2b0aate56c637349292c3f@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Cc: Patrick McHardy <kaber@trash.net>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-bw0-f227.google.com ([209.85.218.227]:60628 "EHLO
	mail-bw0-f227.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752585AbZKLIwz (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 03:52:55 -0500
Received: by bwz27 with SMTP id 27so1938327bwz.21
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 00:52:59 -0800 (PST)
Content-Disposition: inline
In-Reply-To: <412e6f7f0911111912q27f2b0aate56c637349292c3f@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On 12-11-2009 04:12, Changli Gao wrote:
> On Wed, Nov 11, 2009 at 11:59 PM, Patrick McHardy <kaber@trash.net> wrote:
...
>> What protects the device from disappearing here and below during
>> dev_queue_xmit() and netif_rx_ni()?
> 
> For dev_queue_xmit(), dev is holded by skb->_dst, so there is no
> problem. But for netif_rx_ni(), I don't know how to prevent the device
> disappearing, and it seems that all the NIC drivers have this problem.
> Maybe there was the assumption about the execution context of
> netif_rx() before. Now softirq can't be executed by softirqd, so the
> packet receiving path maybe interleaved. I don't know how to prevent
> it happening.

Btw, maybe I miss something, but ifb with act_mirred is for many users
on the fast path. I'm not sure you proved enough why moving it to the
process context and additional multiqueue functionality (which seems
"pretty cool", but probably mostly for testing purposes) don't harm
its main use?

Regards,
Jarek P.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 17:32:13 +0800
Message-ID: <412e6f7f0911120132j5442a1ffp790277597f563953@mail.gmail.com>
References: <412e6f7f0911111912q27f2b0aate56c637349292c3f@mail.gmail.com>
	 <20091112085250.GA7112@ff.dom.local>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Patrick McHardy <kaber@trash.net>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Jarek Poplawski <jarkao2@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:42453 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751911AbZKLJcH convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 04:32:07 -0500
Received: by pwi3 with SMTP id 3so1240700pwi.21
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 01:32:13 -0800 (PST)
In-Reply-To: <20091112085250.GA7112@ff.dom.local>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Thu, Nov 12, 2009 at 4:52 PM, Jarek Poplawski <jarkao2@gmail.com> wr=
ote:
> On 12-11-2009 04:12, Changli Gao wrote:
>> On Wed, Nov 11, 2009 at 11:59 PM, Patrick McHardy <kaber@trash.net> =
wrote:
> ...
>>> What protects the device from disappearing here and below during
>>> dev_queue_xmit() and netif_rx_ni()?
>>
>> For dev_queue_xmit(), dev is holded by skb->_dst, so there is no
>> problem.

Oh, I'm wrong, and not all the skbs using dev_queue_xmit() has a valid
dst, such as AF_PACKET. As skb->dev doesn't own dev, so I think it is
AF_PACKETs fault. Too bad!

>> But for netif_rx_ni(), I don't know how to prevent the device
>> disappearing, and it seems that all the NIC drivers have this proble=
m.
>> Maybe there was the assumption about the execution context of
>> netif_rx() before. Now softirq can't be executed by softirqd, so the
>> packet receiving path maybe interleaved. I don't know how to prevent
>> it happening.
>
> Btw, maybe I miss something, but ifb with act_mirred is for many user=
s
> on the fast path. I'm not sure you proved enough why moving it to the
> process context and additional multiqueue functionality (which seems
> "pretty cool", but probably mostly for testing purposes) don't harm
> its main use?

Because process is more flexible than other activities, and you can
specify its priority and bind it to some CPU, and so on. If you have a
SMP system, and a NIC which doesn't support MQ. When you use it as a
firewall, you'll find all the work is nearly on a CPU. As throughput
increase, that CPU will be full and no more throughput can gain,
though the other CPUs are still idle. With IFB-MQ, you can distribute
the packets among all the CPUs, and archive good throughput.

I know you worry about the latency. From my experiments, It just OK.
No much explicit latency is added.

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 17:44:49 +0800
Message-ID: <4AFBD911.6000900@gmail.com>
References: <4AFA8911.7050204@gmail.com>
Reply-To: xiaosuo@gmail.com
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: 7bit
Cc: xiaosuo@gmail.com, Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-px0-f179.google.com ([209.85.216.179]:61309 "EHLO
	mail-px0-f179.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751736AbZKLJou (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 04:44:50 -0500
Received: by pxi9 with SMTP id 9so348258pxi.16
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 01:44:55 -0800 (PST)
In-Reply-To: <4AFA8911.7050204@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

ifb: add multi-queue support

Add multi-queue support, and one kernel thread is created for per queue.
It can used to emulate multi-queue NIC in software, and distribute work
among CPUs.
gentux linux # modprobe ifb numtxqs=2
gentux linux # ifconfig ifb0 up
gentux linux # pgrep ifb0
18508
18509
gentux linux # taskset -p 1 18508
pid 18508's current affinity mask: 3
pid 18508's new affinity mask: 1
gentux linux # taskset -p 2 18509
pid 18509's current affinity mask: 3
pid 18509's new affinity mask: 2
gentux linux # tc qdisc add dev br0 ingress
gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
action mirred egress redirect dev ifb0

This patch also introduces a ip link option "numtxqs" for specifying the
number of the TX queues. so you can add a new ifb with the command:

ip link add numtxqs 4 type ifb

Signed-off-by: Changli Gao <xiaosuo@gmail.com>
----
drivers/net/ifb.c | 506 +++++++++++++++++++++++++++++++++++-------------
include/linux/if_link.h | 1
net/core/rtnetlink.c | 3
3 files changed, 375 insertions(+), 135 deletions(-)

diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 69c2566..7116953 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -33,161 +33,166 @@
 #include <linux/etherdevice.h>
 #include <linux/init.h>
 #include <linux/moduleparam.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/if_vlan.h>
+#include <linux/if_pppox.h>
+#include <net/ip.h>
+#include <net/ipv6.h>
 #include <net/pkt_sched.h>
 #include <net/net_namespace.h>
 
-#define TX_TIMEOUT  (2*HZ)
-
 #define TX_Q_LIMIT    32
+
+struct ifb_private_q {
+	struct net_device	*dev;
+	struct sk_buff_head	rq;
+	wait_queue_head_t	wq;
+	struct task_struct	*task;
+	unsigned long		rx_packets;
+	unsigned long		rx_bytes;
+	unsigned long		rx_dropped;
+} ____cacheline_aligned_in_smp;
+
 struct ifb_private {
-	struct tasklet_struct   ifb_tasklet;
-	int     tasklet_pending;
-	/* mostly debug stats leave in for now */
-	unsigned long   st_task_enter; /* tasklet entered */
-	unsigned long   st_txq_refl_try; /* transmit queue refill attempt */
-	unsigned long   st_rxq_enter; /* receive queue entered */
-	unsigned long   st_rx2tx_tran; /* receive to trasmit transfers */
-	unsigned long   st_rxq_notenter; /*receiveQ not entered, resched */
-	unsigned long   st_rx_frm_egr; /* received from egress path */
-	unsigned long   st_rx_frm_ing; /* received from ingress path */
-	unsigned long   st_rxq_check;
-	unsigned long   st_rxq_rsch;
-	struct sk_buff_head     rq;
-	struct sk_buff_head     tq;
+	struct ifb_private_q	*pq;
 };
 
+/* Number of ifb devices to be set up by this module. */
 static int numifbs = 2;
+module_param(numifbs, int, 0444);
+MODULE_PARM_DESC(numifbs, "Number of ifb devices");
 
-static void ri_tasklet(unsigned long dev);
-static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
-static int ifb_open(struct net_device *dev);
-static int ifb_close(struct net_device *dev);
+/* Number of TX queues per ifb */
+static unsigned int numtxqs = 1;
+module_param(numtxqs, uint, 0444);
+MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
 
-static void ri_tasklet(unsigned long dev)
+static int ifb_thread(void *priv)
 {
-
-	struct net_device *_dev = (struct net_device *)dev;
-	struct ifb_private *dp = netdev_priv(_dev);
-	struct net_device_stats *stats = &_dev->stats;
-	struct netdev_queue *txq;
+	struct ifb_private_q *pq = priv;
+	struct net_device *dev = pq->dev, *_dev;
+	int num = pq - ((struct ifb_private *)netdev_priv(dev))->pq;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 	struct sk_buff *skb;
-
-	txq = netdev_get_tx_queue(_dev, 0);
-	dp->st_task_enter++;
-	if ((skb = skb_peek(&dp->tq)) == NULL) {
-		dp->st_txq_refl_try++;
-		if (__netif_tx_trylock(txq)) {
-			dp->st_rxq_enter++;
-			while ((skb = skb_dequeue(&dp->rq)) != NULL) {
-				skb_queue_tail(&dp->tq, skb);
-				dp->st_rx2tx_tran++;
-			}
-			__netif_tx_unlock(txq);
-		} else {
-			/* reschedule */
-			dp->st_rxq_notenter++;
-			goto resched;
+	DEFINE_WAIT(wait);
+	struct sk_buff_head tq;
+
+	__skb_queue_head_init(&tq);
+	while (1) {
+		/* move skb from rq to tq */
+		while (1) {
+			prepare_to_wait(&pq->wq, &wait, TASK_UNINTERRUPTIBLE);
+			__netif_tx_lock_bh(txq);
+			skb_queue_splice_tail_init(&pq->rq, &tq);
+			if (netif_queue_stopped(dev))
+				netif_wake_queue(dev);
+			__netif_tx_unlock_bh(txq);
+			if (kthread_should_stop() || !skb_queue_empty(&tq))
+				break;
+			schedule();
 		}
-	}
-
-	while ((skb = skb_dequeue(&dp->tq)) != NULL) {
-		u32 from = G_TC_FROM(skb->tc_verd);
-
-		skb->tc_verd = 0;
-		skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
-		stats->tx_packets++;
-		stats->tx_bytes +=skb->len;
-
-		rcu_read_lock();
-		skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
-		if (!skb->dev) {
-			rcu_read_unlock();
-			dev_kfree_skb(skb);
-			stats->tx_dropped++;
+		finish_wait(&pq->wq, &wait);
+		if (kthread_should_stop()) {
+			__skb_queue_purge(&tq);
 			break;
 		}
-		rcu_read_unlock();
-		skb->iif = _dev->ifindex;
-
-		if (from & AT_EGRESS) {
-			dp->st_rx_frm_egr++;
-			dev_queue_xmit(skb);
-		} else if (from & AT_INGRESS) {
-			dp->st_rx_frm_ing++;
-			skb_pull(skb, skb->dev->hard_header_len);
-			netif_rx(skb);
-		} else
-			BUG();
-	}
-
-	if (__netif_tx_trylock(txq)) {
-		dp->st_rxq_check++;
-		if ((skb = skb_peek(&dp->rq)) == NULL) {
-			dp->tasklet_pending = 0;
-			if (netif_queue_stopped(_dev))
-				netif_wake_queue(_dev);
-		} else {
-			dp->st_rxq_rsch++;
-			__netif_tx_unlock(txq);
-			goto resched;
+		if (need_resched())
+			schedule();
+
+		/* transfer packets */
+		while ((skb = __skb_dequeue(&tq)) != NULL) {
+			u32 from = G_TC_FROM(skb->tc_verd);
+	
+			skb->tc_verd = 0;
+			skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
+			txq->tx_packets++;
+			txq->tx_bytes +=skb->len;
+
+			rcu_read_lock();
+			_dev = dev_get_by_index_rcu(&init_net, skb->iif);
+			if (!_dev) {
+				rcu_read_unlock();
+				dev_kfree_skb(skb);
+				txq->tx_dropped++;
+				continue;
+			}
+			if (from & AT_INGRESS)
+				dev_hold(_dev);
+			rcu_read_unlock();
+			skb->dev = _dev;
+			skb->iif = dev->ifindex;
+	
+			if (from & AT_EGRESS) {
+				dev_queue_xmit(skb);
+			} else if (from & AT_INGRESS) {
+				skb_pull(skb, skb->dev->hard_header_len);
+				netif_rx_ni(skb);
+				dev_put(_dev);
+			} else
+				BUG();
 		}
-		__netif_tx_unlock(txq);
-	} else {
-resched:
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
 	}
 
+	return 0;
 }
 
-static const struct net_device_ops ifb_netdev_ops = {
-	.ndo_open	= ifb_open,
-	.ndo_stop	= ifb_close,
-	.ndo_start_xmit	= ifb_xmit,
-	.ndo_validate_addr = eth_validate_addr,
-};
-
-static void ifb_setup(struct net_device *dev)
+struct net_device_stats* ifb_get_stats(struct net_device *dev)
 {
-	/* Initialize the device structure. */
-	dev->destructor = free_netdev;
-	dev->netdev_ops = &ifb_netdev_ops;
+	struct net_device_stats *stats = &dev->stats;
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	struct netdev_queue *txq;
+	int i;
+	unsigned long rx_packets = 0, rx_bytes = 0, rx_dropped = 0;
+	unsigned long tx_packets = 0, tx_bytes = 0, tx_dropped = 0;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		rx_packets += pq[i].rx_packets;
+		rx_bytes += pq[i].rx_bytes;
+		rx_dropped += pq[i].rx_dropped;
+		txq = netdev_get_tx_queue(dev, i);
+		tx_packets += txq->tx_packets;
+		tx_bytes += txq->tx_bytes;
+		tx_dropped += txq->tx_dropped;
+	}
 
-	/* Fill in device structure with ethernet-generic values. */
-	ether_setup(dev);
-	dev->tx_queue_len = TX_Q_LIMIT;
+	stats->rx_packets = rx_packets;
+	stats->rx_bytes = rx_bytes;
+	stats->rx_dropped = rx_dropped;
+	stats->tx_packets = tx_packets;
+	stats->tx_bytes = tx_bytes;
+	stats->tx_dropped = tx_dropped;
 
-	dev->flags |= IFF_NOARP;
-	dev->flags &= ~IFF_MULTICAST;
-	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
-	random_ether_addr(dev->dev_addr);
+	return stats;
 }
 
 static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 {
-	struct ifb_private *dp = netdev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
 	u32 from = G_TC_FROM(skb->tc_verd);
+	int num = skb_get_queue_mapping(skb);
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq + num;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 
-	stats->rx_packets++;
-	stats->rx_bytes+=skb->len;
+	pq->rx_packets++;
+	pq->rx_bytes += skb->len;
 
 	if (!(from & (AT_INGRESS|AT_EGRESS)) || !skb->iif) {
 		dev_kfree_skb(skb);
-		stats->rx_dropped++;
+		pq->rx_dropped++;
 		return NETDEV_TX_OK;
 	}
 
-	if (skb_queue_len(&dp->rq) >= dev->tx_queue_len) {
+	txq->trans_start = jiffies;
+	__skb_queue_tail(&pq->rq, skb);
+	if (skb_queue_len(&pq->rq) >= dev->tx_queue_len)
 		netif_stop_queue(dev);
-	}
-
-	dev->trans_start = jiffies;
-	skb_queue_tail(&dp->rq, skb);
-	if (!dp->tasklet_pending) {
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
-	}
+	if (skb_queue_len(&pq->rq) == 1)
+		wake_up(&pq->wq);
 
 	return NETDEV_TX_OK;
 }
@@ -195,26 +200,241 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 static int ifb_close(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
 
-	tasklet_kill(&dp->ifb_tasklet);
 	netif_stop_queue(dev);
-	skb_queue_purge(&dp->rq);
-	skb_queue_purge(&dp->tq);
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		kthread_stop(pq[i].task);
+		__skb_queue_purge(&pq[i].rq);
+	}
+
 	return 0;
 }
 
 static int ifb_open(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
-
-	tasklet_init(&dp->ifb_tasklet, ri_tasklet, (unsigned long)dev);
-	skb_queue_head_init(&dp->rq);
-	skb_queue_head_init(&dp->tq);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
+	
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		pq[i].task = kthread_run(ifb_thread, &pq[i], "%s/%d", dev->name,
+					 i);
+		if (IS_ERR(pq[i].task)) {
+			int err = PTR_ERR(pq[i].task);
+			while (--i >= 0)
+				kthread_stop(pq[i].task);
+			return err;
+		}
+	}
 	netif_start_queue(dev);
 
 	return 0;
 }
 
+static u32 simple_tx_hashrnd;
+
+static inline __be16 pppoe_proto(const struct sk_buff *skb)
+{
+	return *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
+			sizeof(struct pppoe_hdr)));
+}
+
+static u8 ifb_ipv6_ihl(struct sk_buff *skb, u32 *pihl)
+{
+	struct ipv6hdr *hdr = ipv6_hdr(skb);
+	u32 ihl = sizeof(*hdr);
+	u8 nexthdr = hdr->nexthdr;
+
+	while (1) {
+		switch (nexthdr) {
+		case NEXTHDR_HOP:
+		case NEXTHDR_ROUTING:
+		case NEXTHDR_DEST:
+			if (!pskb_may_pull(skb, ihl + 8) ||
+			    !pskb_may_pull(skb, ihl +
+			    	((skb_network_header(skb)[ihl + 1] + 1) << 3)))
+			return 0;
+			nexthdr = skb_network_header(skb)[ihl];
+			ihl += (skb_network_header(skb)[ihl + 1] + 1) << 3;
+			break;
+		case NEXTHDR_AUTH:
+		case NEXTHDR_ESP:
+		case NEXTHDR_NONE:
+		case NEXTHDR_FRAGMENT:
+			return 0;
+		default:
+			*pihl = ihl;
+			return nexthdr;
+		}
+	}
+}
+
+static u16 ifb_select_queue(struct net_device *dev, struct sk_buff *skb)
+{
+	u32 addr1, addr2;
+	u32 hash, ihl = 0;
+	union {
+		u16 in16[2];
+		u32 in32;
+	} ports;
+	u8 ip_proto;
+	unsigned int pull_len;
+
+	if ((hash = skb_rx_queue_recorded(skb))) {
+		while (hash >= dev->real_num_tx_queues)
+			hash -= dev->real_num_tx_queues;
+		return hash;
+	}
+
+	pull_len = 0;
+	switch (skb->protocol) {
+	case __constant_htons(ETH_P_8021Q):
+		if (unlikely(skb_pull(skb, VLAN_HLEN) == NULL))
+			goto process_other;
+		pull_len = VLAN_HLEN;
+		skb->network_header += pull_len;
+		switch (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto) {
+		case __constant_htons(ETH_P_IP):
+			goto process_ip;
+		case __constant_htons(ETH_P_IPV6):
+			goto process_ipv6;
+		default:
+			goto process_other;
+		}
+		break;
+	case __constant_htons(ETH_P_PPP_SES):
+		if (unlikely(skb_pull(skb, PPPOE_SES_HLEN) == NULL))
+			goto process_other;
+		pull_len = PPPOE_SES_HLEN;
+		skb->network_header += pull_len;
+		switch (pppoe_proto(skb)) {
+		case __constant_htons(ETH_P_IP):
+			goto process_ip;
+		case __constant_htons(ETH_P_IPV6):
+			goto process_ipv6;
+		default:
+			goto process_other;
+		}
+		break;
+	case __constant_htons(ETH_P_IP):
+process_ip:
+		if (unlikely(!pskb_may_pull(skb, sizeof(struct iphdr))))
+			goto process_other;
+		if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)))
+			ip_proto = ip_hdr(skb)->protocol;
+		else
+			ip_proto = 0;
+		addr1 = ip_hdr(skb)->saddr;
+		addr2 = ip_hdr(skb)->daddr;
+		ihl = ip_hdrlen(skb);
+		break;
+	case __constant_htons(ETH_P_IPV6):
+process_ipv6:
+		if (unlikely(!pskb_may_pull(skb, sizeof(struct ipv6hdr))))
+			goto process_other;
+		addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
+		addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
+		ip_proto = ifb_ipv6_ihl(skb, &ihl);
+		break;
+	default:
+process_other:
+		if (pull_len != 0) {
+			skb_push(skb, pull_len);
+			skb->network_header -= pull_len;
+		}
+		return skb->protocol % dev->real_num_tx_queues;
+	}
+	if (addr1 > addr2)
+		swap(addr1, addr2);
+
+	switch (ip_proto) {
+	case IPPROTO_TCP:
+	case IPPROTO_UDP:
+	case IPPROTO_DCCP:
+	case IPPROTO_ESP:
+	case IPPROTO_AH:
+	case IPPROTO_SCTP:
+	case IPPROTO_UDPLITE:
+		if (unlikely(!pskb_may_pull(skb, ihl + 4)))
+			goto process_other_trans;
+		ports.in32 = *((u32 *) (skb_network_header(skb) + ihl));
+		if (ports.in16[0] > ports.in16[1])
+			swap(ports.in16[0], ports.in16[1]);
+		break;
+
+	default:
+process_other_trans:
+		ports.in32 = 0;
+		break;
+	}
+
+	if (pull_len != 0) {
+		skb_push(skb, pull_len);
+		skb->network_header -= pull_len;
+	}
+
+	hash = jhash_3words(addr1, addr2, ports.in32,
+			    simple_tx_hashrnd ^ ip_proto);
+
+	return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
+}
+
+static int ifb_init(struct net_device *dev)
+{
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
+
+	pq = kcalloc(dev->real_num_tx_queues, sizeof(*pq), GFP_KERNEL);
+	if (pq == NULL)
+		return -ENOMEM;
+	dp->pq = pq;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		pq[i].dev = dev;
+		__skb_queue_head_init(&pq[i].rq);
+		init_waitqueue_head(&pq[i].wq);
+	}
+
+	return 0;
+}
+
+static void ifb_uninit(struct net_device *dev)
+{
+	struct ifb_private *dp = netdev_priv(dev);
+
+	kfree(dp->pq);
+}
+
+static const struct net_device_ops ifb_netdev_ops = {
+	.ndo_init		= ifb_init,
+	.ndo_uninit		= ifb_uninit,
+	.ndo_open		= ifb_open,
+	.ndo_stop		= ifb_close,
+	.ndo_start_xmit		= ifb_xmit,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_select_queue	= ifb_select_queue,
+	.ndo_get_stats		= ifb_get_stats,
+};
+
+static void ifb_setup(struct net_device *dev)
+{
+	/* Initialize the device structure. */
+	dev->destructor = free_netdev;
+	dev->netdev_ops = &ifb_netdev_ops;
+
+	/* Fill in device structure with ethernet-generic values. */
+	ether_setup(dev);
+	dev->tx_queue_len = TX_Q_LIMIT;
+
+	dev->flags |= IFF_NOARP;
+	dev->flags &= ~IFF_MULTICAST;
+	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+	random_ether_addr(dev->dev_addr);
+}
+
 static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 {
 	if (tb[IFLA_ADDRESS]) {
@@ -226,25 +446,38 @@ static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 	return 0;
 }
 
+static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
+			     unsigned int *num_tx_queues,
+			     unsigned int *real_num_tx_queues)
+{
+	if (tb[IFLA_NTXQ]) {
+		*num_tx_queues = nla_get_u32(tb[IFLA_NTXQ]);
+		if (*num_tx_queues < 1)
+			return -EINVAL;
+		*real_num_tx_queues = *num_tx_queues;
+	} else {
+		*num_tx_queues = numtxqs;
+		*real_num_tx_queues = numtxqs;
+	}
+
+	return 0;
+}
+
 static struct rtnl_link_ops ifb_link_ops __read_mostly = {
 	.kind		= "ifb",
-	.priv_size	= sizeof(struct ifb_private),
 	.setup		= ifb_setup,
 	.validate	= ifb_validate,
+	.get_tx_queues	= ifb_get_tx_queues,
+	.priv_size	= sizeof(struct ifb_private),
 };
 
-/* Number of ifb devices to be set up by this module. */
-module_param(numifbs, int, 0);
-MODULE_PARM_DESC(numifbs, "Number of ifb devices");
-
 static int __init ifb_init_one(int index)
 {
 	struct net_device *dev_ifb;
 	int err;
 
-	dev_ifb = alloc_netdev(sizeof(struct ifb_private),
-				 "ifb%d", ifb_setup);
-
+	dev_ifb = alloc_netdev_mq(sizeof(struct ifb_private), "ifb%d",
+				  ifb_setup, numtxqs);
 	if (!dev_ifb)
 		return -ENOMEM;
 
@@ -268,9 +501,12 @@ static int __init ifb_init_module(void)
 {
 	int i, err;
 
+	if (numtxqs < 1)
+		return -EINVAL;
+
+	get_random_bytes(&simple_tx_hashrnd, 4);
 	rtnl_lock();
 	err = __rtnl_link_register(&ifb_link_ops);
-
 	for (i = 0; i < numifbs && !err; i++)
 		err = ifb_init_one(i);
 	if (err)
diff --git a/include/linux/if_link.h b/include/linux/if_link.h
index 1d3b242..99da411 100644
--- a/include/linux/if_link.h
+++ b/include/linux/if_link.h
@@ -78,6 +78,7 @@ enum {
 #define IFLA_LINKINFO IFLA_LINKINFO
 	IFLA_NET_NS_PID,
 	IFLA_IFALIAS,
+	IFLA_NTXQ,
 	__IFLA_MAX
 };
 
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 33148a5..1cbe555 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -597,6 +597,7 @@ static inline size_t if_nlmsg_size(const struct net_device *dev)
 	       + nla_total_size(4) /* IFLA_MASTER */
 	       + nla_total_size(1) /* IFLA_OPERSTATE */
 	       + nla_total_size(1) /* IFLA_LINKMODE */
+	       + nla_total_size(4) /* IFLA_NTXQ */
 	       + rtnl_link_get_size(dev); /* IFLA_LINKINFO */
 }
 
@@ -627,6 +628,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
 		   netif_running(dev) ? dev->operstate : IF_OPER_DOWN);
 	NLA_PUT_U8(skb, IFLA_LINKMODE, dev->link_mode);
 	NLA_PUT_U32(skb, IFLA_MTU, dev->mtu);
+	NLA_PUT_U32(skb, IFLA_NTXQ, dev->real_num_tx_queues);
 
 	if (dev->ifindex != dev->iflink)
 		NLA_PUT_U32(skb, IFLA_LINK, dev->iflink);
@@ -725,6 +727,7 @@ const struct nla_policy ifla_policy[IFLA_MAX+1] = {
 	[IFLA_LINKINFO]		= { .type = NLA_NESTED },
 	[IFLA_NET_NS_PID]	= { .type = NLA_U32 },
 	[IFLA_IFALIAS]	        = { .type = NLA_STRING, .len = IFALIASZ-1 },
+	[IFLA_NTXQ]		= { .type = NLA_U32 },
 };
 EXPORT_SYMBOL(ifla_policy);
 



From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 17:48:05 +0800
Message-ID: <412e6f7f0911120148r48fa553bt610e013d96522a3d@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>
Mime-Version: 1.0
Content-Type: multipart/mixed; boundary=001636e0a6c09b801c0478297036
Cc: xiaosuo@gmail.com, Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:44615 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752039AbZKLJr7 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 04:47:59 -0500
Received: by pzk1 with SMTP id 1so421988pzk.33
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 01:48:05 -0800 (PST)
In-Reply-To: <4AFBD911.6000900@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

--001636e0a6c09b801c0478297036
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: quoted-printable

The corresponding iprout2 patch is attached.

--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

--001636e0a6c09b801c0478297036
Content-Type: application/octet-stream; name="iproute2-support-numtxqs.diff"
Content-Disposition: attachment; filename="iproute2-support-numtxqs.diff"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_g1xc6slg0

LS0tIGlwL2lwbGluay5jLm9yaWcJMjAwOS0xMS0xMSAxNjo1Njo0MS4wMDAwMDAwMDAgKzA4MDAK
KysrIGlwL2lwbGluay5jCTIwMDktMTEtMTIgMTQ6NTc6MjUuMDAwMDAwMDAwICswODAwCkBAIC00
OCw2ICs0OCw3IEBACiAJCWZwcmludGYoc3RkZXJyLCAiICAgICAgICAgICAgICAgICAgIFsgYWRk
cmVzcyBMTEFERFIgXVxuIik7CiAJCWZwcmludGYoc3RkZXJyLCAiICAgICAgICAgICAgICAgICAg
IFsgYnJvYWRjYXN0IExMQUREUiBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIsICIgICAgICAgICAg
ICAgICAgICAgWyBtdHUgTVRVIF1cbiIpOworCQlmcHJpbnRmKHN0ZGVyciwgIiAgICAgICAgICAg
ICAgICAgICBbIG51bXR4cXMgTlVNVFhRUyBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIsICIgICAg
ICAgICAgICAgICAgICAgdHlwZSBUWVBFIFsgQVJHUyBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIs
ICIgICAgICAgaXAgbGluayBkZWxldGUgREVWIHR5cGUgVFlQRSBbIEFSR1MgXVxuIik7CiAJCWZw
cmludGYoc3RkZXJyLCAiXG4iKTsKQEAgLTE4MCw2ICsxODEsNyBAQAogCWludCBxbGVuID0gLTE7
CiAJaW50IG10dSA9IC0xOwogCWludCBuZXRucyA9IC0xOworCWludCBudW10eHFzID0gLTE7CiAK
IAlyZXQgPSBhcmdjOwogCkBAIC0yMjEsNiArMjIzLDEzIEBACiAJCQlpZiAoZ2V0X2ludGVnZXIo
Jm10dSwgKmFyZ3YsIDApKQogCQkJCWludmFyZygiSW52YWxpZCBcIm10dVwiIHZhbHVlXG4iLCAq
YXJndik7CiAJCQlhZGRhdHRyX2woJnJlcS0+biwgc2l6ZW9mKCpyZXEpLCBJRkxBX01UVSwgJm10
dSwgNCk7CisJCX0gZWxzZSBpZiAoc3RyY21wKCphcmd2LCAibnVtdHhxcyIpID09IDApIHsKKwkJ
CU5FWFRfQVJHKCk7CisJCQlpZiAobnVtdHhxcyAhPSAtMSkKKwkJCQlkdXBhcmcoIm51bXR4cXMi
LCAqYXJndik7CisJCQlpZiAoZ2V0X2ludGVnZXIoJm51bXR4cXMsICphcmd2LCAwKSkKKwkJCQlp
bnZhcmcoIkludmFsaWQgXCJudW10eHFzXCIgdmFsdWVcbiIsICphcmd2KTsKKwkJCWFkZGF0dHJf
bCgmcmVxLT5uLCBzaXplb2YoKnJlcSksIElGTEFfTlRYUSwgJm51bXR4cXMsIDQpOwogICAgICAg
ICAgICAgICAgIH0gZWxzZSBpZiAoc3RyY21wKCphcmd2LCAibmV0bnMiKSA9PSAwKSB7CiAgICAg
ICAgICAgICAgICAgICAgICAgICBORVhUX0FSRygpOwogICAgICAgICAgICAgICAgICAgICAgICAg
aWYgKG5ldG5zICE9IC0xKQo=
--001636e0a6c09b801c0478297036--

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 13:48:54 +0100
Message-ID: <4AFC0436.9090906@gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: xiaosuo@gmail.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:39329 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751954AbZKLMs5 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 07:48:57 -0500
In-Reply-To: <4AFBD911.6000900@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =A8=A6crit :
> ifb: add multi-queue support
>=20
> Add multi-queue support, and one kernel thread is created for per que=
ue.
> It can used to emulate multi-queue NIC in software, and distribute wo=
rk
> among CPUs.
> gentux linux # modprobe ifb numtxqs=3D2
> gentux linux # ifconfig ifb0 up
> gentux linux # pgrep ifb0
> 18508
> 18509
> gentux linux # taskset -p 1 18508
> pid 18508's current affinity mask: 3
> pid 18508's new affinity mask: 1
> gentux linux # taskset -p 2 18509
> pid 18509's current affinity mask: 3
> pid 18509's new affinity mask: 2
> gentux linux # tc qdisc add dev br0 ingress
> gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
> action mirred egress redirect dev ifb0
>=20
> This patch also introduces a ip link option "numtxqs" for specifying =
the
> number of the TX queues. so you can add a new ifb with the command:
>=20
> ip link add numtxqs 4 type ifb
>=20
> Signed-off-by: Changli Gao <xiaosuo@gmail.com>
> ----

I believe this patch is fine, but maybe Jarek concern about workqueue=20
vs tasklet should be addressed...

We could use the previous handling in case numtxqs=3D=3D1 , ie use a ta=
sklet
instead of a work queue ?


Sorry for this late idea...

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 16:10:27 +0100
Message-ID: <4AFC2563.1020902@trash.net>
References: <4AFA8911.7050204@gmail.com> <4AFADF64.8070601@trash.net> <412e6f7f0911111912q27f2b0aate56c637349292c3f@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=ISO-8859-15
Content-Transfer-Encoding: 7bit
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:59345 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752750AbZKLPHD (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 10:07:03 -0500
In-Reply-To: <412e6f7f0911111912q27f2b0aate56c637349292c3f@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> On Wed, Nov 11, 2009 at 11:59 PM, Patrick McHardy <kaber@trash.net> wrote:
>>> +static int numtxqs = 1;
>>> +module_param(numtxqs, int, 0444);
>>> +MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
>> unsigned?
> 
> Yea, unsigned is better, and I need to check whether its value is
> smaller than 1 somewhere. The same will be done with IFLA_NTXQ.

Good point.

>>> +                     rcu_read_lock();
>>> +                     skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
>>> +                     if (!skb->dev) {
>>> +                             rcu_read_unlock();
>>> +                             dev_kfree_skb(skb);
>>> +                             txq->tx_dropped++;
>>> +                             break;
>>> +                     }
>>> +                     rcu_read_unlock();
>>> +                     skb->iif = dev->ifindex;
>> What protects the device from disappearing here and below during
>> dev_queue_xmit() and netif_rx_ni()?
> 
> For dev_queue_xmit(), dev is holded by skb->_dst, so there is no
> problem. But for netif_rx_ni(), I don't know how to prevent the device
> disappearing, and it seems that all the NIC drivers have this problem.
> Maybe there was the assumption about the execution context of
> netif_rx() before. Now softirq can't be executed by softirqd, so the
> packet receiving path maybe interleaved. I don't know how to prevent
> it happening.

You can either take a reference of move the rcu_read_unlock()
after the skb submission.

>>> +             break;
>>> +     case __constant_htons(ETH_P_IPV6):
>>> +             ip_proto = ipv6_hdr(skb)->nexthdr;
>>> +             addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
>>> +             addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
>>> +             ihl = 10;
>> Where does 10 come from?
> 
> It should be 40, after reviewing IPV6, I found that I need to loop
> until finding the right protocol value.

There is a helper for this (ipv6_skip_exthdr).

>>> +static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
>>> +                          unsigned int *num_tx_queues,
>>> +                          unsigned int *real_num_tx_queues)
>>> +{
>>> +     if (tb[IFLA_NTXQ]) {
>>> +             *num_tx_queues = nla_get_u16(tb[IFLA_NTXQ]);
>> We currently use unsigned ints for the queue number, so please
>> use an u32 for the attribute as well.
>>
>>> +             *real_num_tx_queues = *num_tx_queues;
>>> +     } else {
>>> +             *num_tx_queues = numtxqs;
>>> +             *real_num_tx_queues = numtxqs;
>>> +     }
>>> +
>>> +     return 0;
>>> +}
>>> +
>         u16                     (*ndo_select_queue)(struct net_device *dev,
>                                                     struct sk_buff *skb);
> use u16 as the return value so ..., and I think u16 is big enough. If
> you insist on this, I'll use u32 instead.

Well, get_tx_queues() uses unsigned int, as does struct net_device.
I agree that it probably won't ever be needed, but there's no harm
in using the correct type, the attribute encoding doesn't even need
more room.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Thu, 12 Nov 2009 16:11:35 +0100
Message-ID: <4AFC25A7.10301@trash.net>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com> <412e6f7f0911120148r48fa553bt610e013d96522a3d@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=ISO-8859-15
Content-Transfer-Encoding: 7bit
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:59373 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1753092AbZKLPIG (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 10:08:06 -0500
In-Reply-To: <412e6f7f0911120148r48fa553bt610e013d96522a3d@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> The corresponding iprout2 patch is attached.

Printing the value during dumps is missing from this patch.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 09:26:18 +0800
Message-ID: <412e6f7f0911121726t47874807g2223d2f1141818f5@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>
	 <4AFC0436.9090906@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:61088 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1754642AbZKMB0M convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 20:26:12 -0500
Received: by pzk1 with SMTP id 1so950812pzk.33
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 17:26:18 -0800 (PST)
In-Reply-To: <4AFC0436.9090906@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

2009/11/12 Eric Dumazet <eric.dumazet@gmail.com>:
>
> I believe this patch is fine, but maybe Jarek concern about workqueue
> vs tasklet should be addressed...
>
> We could use the previous handling in case numtxqs=3D=3D1 , ie use a =
tasklet
> instead of a work queue ?

I don't think it is a good idea. If we do so, the code will be messy,
and lost the flexibility of process. In fact, latency isn't a problem
when system load isn't high, and when system load is high (due to too
many NIC IRQs), throughput and interaction is more important, and the
current linux networking subsystem just dose so through the softirqd
mechanism.

>
>
> Sorry for this late idea...
>



--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 09:28:45 +0800
Message-ID: <412e6f7f0911121728m76f76754jab8f4d9a0d360f6d@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFADF64.8070601@trash.net>
	 <412e6f7f0911111912q27f2b0aate56c637349292c3f@mail.gmail.com>
	 <4AFC2563.1020902@trash.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:58838 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1754746AbZKMB2k convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 20:28:40 -0500
Received: by pwi3 with SMTP id 3so1703567pwi.21
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 17:28:45 -0800 (PST)
In-Reply-To: <4AFC2563.1020902@trash.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Thu, Nov 12, 2009 at 11:10 PM, Patrick McHardy <kaber@trash.net> wro=
te:
> Changli Gao wrote:
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 rcu_read_lock();
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 skb->dev =3D dev_get_by_index_rcu(&init_net, skb->iif);
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 if (!skb->dev) {
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 rcu_read_unlock();
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 dev_kfree_skb(skb);
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 txq->tx_dropped++;
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 }
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 rcu_read_unlock();
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 skb->iif =3D dev->ifindex;
>>> What protects the device from disappearing here and below during
>>> dev_queue_xmit() and netif_rx_ni()?
>>
>> For dev_queue_xmit(), dev is holded by skb->_dst, so there is no
>> problem. But for netif_rx_ni(), I don't know how to prevent the devi=
ce
>> disappearing, and it seems that all the NIC drivers have this proble=
m.
>> Maybe there was the assumption about the execution context of
>> netif_rx() before. Now softirq can't be executed by softirqd, so the
>> packet receiving path maybe interleaved. I don't know how to prevent
>> it happening.
>
> You can either take a reference of move the rcu_read_unlock()
> after the skb submission.

good idea!

>
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 break;
>>>> + =C2=A0 =C2=A0 case __constant_htons(ETH_P_IPV6):
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ip_proto =3D ipv6_hdr(=
skb)->nexthdr;
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 addr1 =3D ipv6_hdr(skb=
)->saddr.s6_addr32[3];
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 addr2 =3D ipv6_hdr(skb=
)->daddr.s6_addr32[3];
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 ihl =3D 10;
>>> Where does 10 come from?
>>
>> It should be 40, after reviewing IPV6, I found that I need to loop
>> until finding the right protocol value.
>
> There is a helper for this (ipv6_skip_exthdr).
>

It is a little late. Thanks all the same, and I'll use it.

>>>> +static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[]=
,
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *num_tx_queues,
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0unsigned int *real_num_tx_queues)
>>>> +{
>>>> + =C2=A0 =C2=A0 if (tb[IFLA_NTXQ]) {
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *num_tx_queues =3D nla=
_get_u16(tb[IFLA_NTXQ]);
>>> We currently use unsigned ints for the queue number, so please
>>> use an u32 for the attribute as well.
>>>
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *real_num_tx_queues =3D=
 *num_tx_queues;
>>>> + =C2=A0 =C2=A0 } else {
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *num_tx_queues =3D num=
txqs;
>>>> + =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 *real_num_tx_queues =3D=
 numtxqs;
>>>> + =C2=A0 =C2=A0 }
>>>> +
>>>> + =C2=A0 =C2=A0 return 0;
>>>> +}
>>>> +
>> =C2=A0 =C2=A0 =C2=A0 =C2=A0 u16 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=
=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 (*ndo_select_queue)(struct net_device *=
dev,
>> =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0=
 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =
=C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 =C2=A0 struct sk_buff *skb);
>> use u16 as the return value so ..., and I think u16 is big enough. I=
f
>> you insist on this, I'll use u32 instead.
>
> Well, get_tx_queues() uses unsigned int, as does struct net_device.
> I agree that it probably won't ever be needed, but there's no harm
> in using the correct type, the attribute encoding doesn't even need
> more room.
>

So, in the last patch, I use it instead. :)

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 09:32:36 +0800
Message-ID: <412e6f7f0911121732j106f96a2rb77f68098b7fd47e@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>
	 <412e6f7f0911120148r48fa553bt610e013d96522a3d@mail.gmail.com>
	 <4AFC25A7.10301@trash.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:42500 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1754067AbZKMBcb convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 20:32:31 -0500
Received: by pzk1 with SMTP id 1so953745pzk.33
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 17:32:37 -0800 (PST)
In-Reply-To: <4AFC25A7.10301@trash.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Thu, Nov 12, 2009 at 11:11 PM, Patrick McHardy <kaber@trash.net> wro=
te:
> Changli Gao wrote:
>> The corresponding iprout2 patch is attached.
>
> Printing the value during dumps is missing from this patch.
>

I doesn't know the rules about adding more dumps. Can I add it in the
end of the corresponding line?

ifb0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN qlen 32 ntxqs 4

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 12:37:07 +0800
Message-ID: <4AFCE273.7010901@gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>
Reply-To: xiaosuo@gmail.com
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: 7bit
Cc: xiaosuo@gmail.com, Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-px0-f180.google.com ([209.85.216.180]:61466 "EHLO
	mail-px0-f180.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755698AbZKMEhH (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 23:37:07 -0500
Received: by pxi10 with SMTP id 10so2217694pxi.33
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 20:37:13 -0800 (PST)
In-Reply-To: <4AFBD911.6000900@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

ifb: add multi-queue support

Add multi-queue support, and one kernel thread is created for per queue.
It can used to emulate multi-queue NIC in software, and distribute work
among CPUs.
gentux linux # modprobe ifb numtxqs=2
gentux linux # ifconfig ifb0 up
gentux linux # pgrep ifb0
18508
18509
gentux linux # taskset -p 1 18508
pid 18508's current affinity mask: 3
pid 18508's new affinity mask: 1
gentux linux # taskset -p 2 18509
pid 18509's current affinity mask: 3
pid 18509's new affinity mask: 2
gentux linux # tc qdisc add dev br0 ingress
gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
action mirred egress redirect dev ifb0

This patch also introduces a ip link option "numtxqs" for specifying the
number of the TX queues. so you can add a new ifb with the command:

ip link add numtxqs 4 type ifb

Signed-off-by: Changli Gao <xiaosuo@gmail.com>
----
drivers/net/ifb.c | 491 ++++++++++++++++++++++++++++++++++--------------
include/linux/if_link.h | 1
net/core/rtnetlink.c | 3
3 files changed, 360 insertions(+), 135 deletions(-)
diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 69c2566..96ab20a 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -33,161 +33,162 @@
#include <linux/etherdevice.h>
#include <linux/init.h>
#include <linux/moduleparam.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/if_vlan.h>
+#include <linux/if_pppox.h>
+#include <net/ip.h>
+#include <net/ipv6.h>
#include <net/pkt_sched.h>
#include <net/net_namespace.h>

-#define TX_TIMEOUT (2*HZ)
-
#define TX_Q_LIMIT 32
+
+struct ifb_private_q {
+ struct net_device *dev;
+ struct sk_buff_head rq;
+ wait_queue_head_t wq;
+ struct task_struct *task;
+ unsigned long rx_packets;
+ unsigned long rx_bytes;
+ unsigned long rx_dropped;
+} ____cacheline_aligned_in_smp;
+
struct ifb_private {
- struct tasklet_struct ifb_tasklet;
- int tasklet_pending;
- /* mostly debug stats leave in for now */
- unsigned long st_task_enter; /* tasklet entered */
- unsigned long st_txq_refl_try; /* transmit queue refill attempt */
- unsigned long st_rxq_enter; /* receive queue entered */
- unsigned long st_rx2tx_tran; /* receive to trasmit transfers */
- unsigned long st_rxq_notenter; /*receiveQ not entered, resched */
- unsigned long st_rx_frm_egr; /* received from egress path */
- unsigned long st_rx_frm_ing; /* received from ingress path */
- unsigned long st_rxq_check;
- unsigned long st_rxq_rsch;
- struct sk_buff_head rq;
- struct sk_buff_head tq;
+ struct ifb_private_q *pq;
};

+/* Number of ifb devices to be set up by this module. */
static int numifbs = 2;
+module_param(numifbs, int, 0444);
+MODULE_PARM_DESC(numifbs, "Number of ifb devices");

-static void ri_tasklet(unsigned long dev);
-static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
-static int ifb_open(struct net_device *dev);
-static int ifb_close(struct net_device *dev);
+/* Number of TX queues per ifb */
+static unsigned int numtxqs = 1;
+module_param(numtxqs, uint, 0444);
+MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");

-static void ri_tasklet(unsigned long dev)
+static int ifb_thread(void *priv)
{
-
- struct net_device *_dev = (struct net_device *)dev;
- struct ifb_private *dp = netdev_priv(_dev);
- struct net_device_stats *stats = &_dev->stats;
- struct netdev_queue *txq;
+ struct ifb_private_q *pq = priv;
+ struct net_device *dev = pq->dev, *_dev;
+ int num = pq - ((struct ifb_private *)netdev_priv(dev))->pq;
+ struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
struct sk_buff *skb;
-
- txq = netdev_get_tx_queue(_dev, 0);
- dp->st_task_enter++;
- if ((skb = skb_peek(&dp->tq)) == NULL) {
- dp->st_txq_refl_try++;
- if (__netif_tx_trylock(txq)) {
- dp->st_rxq_enter++;
- while ((skb = skb_dequeue(&dp->rq)) != NULL) {
- skb_queue_tail(&dp->tq, skb);
- dp->st_rx2tx_tran++;
- }
- __netif_tx_unlock(txq);
- } else {
- /* reschedule */
- dp->st_rxq_notenter++;
- goto resched;
+ DEFINE_WAIT(wait);
+ struct sk_buff_head tq;
+
+ __skb_queue_head_init(&tq);
+ while (1) {
+ /* move skb from rq to tq */
+ while (1) {
+ prepare_to_wait(&pq->wq, &wait, TASK_UNINTERRUPTIBLE);
+ __netif_tx_lock_bh(txq);
+ skb_queue_splice_tail_init(&pq->rq, &tq);
+ if (netif_queue_stopped(dev))
+ netif_wake_queue(dev);
+ __netif_tx_unlock_bh(txq);
+ if (kthread_should_stop() || !skb_queue_empty(&tq))
+ break;
+ schedule();
}
- }
-
- while ((skb = skb_dequeue(&dp->tq)) != NULL) {
- u32 from = G_TC_FROM(skb->tc_verd);
-
- skb->tc_verd = 0;
- skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
- stats->tx_packets++;
- stats->tx_bytes +=skb->len;
-
- rcu_read_lock();
- skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
- if (!skb->dev) {
- rcu_read_unlock();
- dev_kfree_skb(skb);
- stats->tx_dropped++;
+ finish_wait(&pq->wq, &wait);
+ if (kthread_should_stop()) {
+ __skb_queue_purge(&tq);
break;
}
- rcu_read_unlock();
- skb->iif = _dev->ifindex;
-
- if (from & AT_EGRESS) {
- dp->st_rx_frm_egr++;
- dev_queue_xmit(skb);
- } else if (from & AT_INGRESS) {
- dp->st_rx_frm_ing++;
- skb_pull(skb, skb->dev->hard_header_len);
- netif_rx(skb);
- } else
- BUG();
- }
-
- if (__netif_tx_trylock(txq)) {
- dp->st_rxq_check++;
- if ((skb = skb_peek(&dp->rq)) == NULL) {
- dp->tasklet_pending = 0;
- if (netif_queue_stopped(_dev))
- netif_wake_queue(_dev);
- } else {
- dp->st_rxq_rsch++;
- __netif_tx_unlock(txq);
- goto resched;
+ if (need_resched())
+ schedule();
+
+ /* transfer packets */
+ while ((skb = __skb_dequeue(&tq)) != NULL) {
+ u32 from = G_TC_FROM(skb->tc_verd);
+
+ skb->tc_verd = 0;
+ skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
+ txq->tx_packets++;
+ txq->tx_bytes +=skb->len;
+
+ rcu_read_lock();
+ _dev = dev_get_by_index_rcu(&init_net, skb->iif);
+ if (_dev != NULL) {
+ skb->dev = _dev;
+ skb->iif = dev->ifindex;
+ if (from & AT_EGRESS) {
+ dev_queue_xmit(skb);
+ } else if (from & AT_INGRESS) {
+ skb_pull(skb, _dev->hard_header_len);
+ netif_rx_ni(skb);
+ } else {
+ BUG();
+ }
+ } else {
+ dev_kfree_skb(skb);
+ txq->tx_dropped++;
+ }
+ rcu_read_unlock();
}
- __netif_tx_unlock(txq);
- } else {
-resched:
- dp->tasklet_pending = 1;
- tasklet_schedule(&dp->ifb_tasklet);
}

+ return 0;
}

-static const struct net_device_ops ifb_netdev_ops = {
- .ndo_open = ifb_open,
- .ndo_stop = ifb_close,
- .ndo_start_xmit = ifb_xmit,
- .ndo_validate_addr = eth_validate_addr,
-};
-
-static void ifb_setup(struct net_device *dev)
+struct net_device_stats* ifb_get_stats(struct net_device *dev)
{
- /* Initialize the device structure. */
- dev->destructor = free_netdev;
- dev->netdev_ops = &ifb_netdev_ops;
+ struct net_device_stats *stats = &dev->stats;
+ struct ifb_private *dp = netdev_priv(dev);
+ struct ifb_private_q *pq = dp->pq;
+ struct netdev_queue *txq;
+ int i;
+ unsigned long rx_packets = 0, rx_bytes = 0, rx_dropped = 0;
+ unsigned long tx_packets = 0, tx_bytes = 0, tx_dropped = 0;
+
+ for (i = 0; i < dev->real_num_tx_queues; i++) {
+ rx_packets += pq[i].rx_packets;
+ rx_bytes += pq[i].rx_bytes;
+ rx_dropped += pq[i].rx_dropped;
+ txq = netdev_get_tx_queue(dev, i);
+ tx_packets += txq->tx_packets;
+ tx_bytes += txq->tx_bytes;
+ tx_dropped += txq->tx_dropped;
+ }

- /* Fill in device structure with ethernet-generic values. */
- ether_setup(dev);
- dev->tx_queue_len = TX_Q_LIMIT;
+ stats->rx_packets = rx_packets;
+ stats->rx_bytes = rx_bytes;
+ stats->rx_dropped = rx_dropped;
+ stats->tx_packets = tx_packets;
+ stats->tx_bytes = tx_bytes;
+ stats->tx_dropped = tx_dropped;

- dev->flags |= IFF_NOARP;
- dev->flags &= ~IFF_MULTICAST;
- dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
- random_ether_addr(dev->dev_addr);
+ return stats;
}

static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
{
- struct ifb_private *dp = netdev_priv(dev);
- struct net_device_stats *stats = &dev->stats;
u32 from = G_TC_FROM(skb->tc_verd);
+ int num = skb_get_queue_mapping(skb);
+ struct ifb_private *dp = netdev_priv(dev);
+ struct ifb_private_q *pq = dp->pq + num;
+ struct netdev_queue *txq = netdev_get_tx_queue(dev, num);

- stats->rx_packets++;
- stats->rx_bytes+=skb->len;
+ pq->rx_packets++;
+ pq->rx_bytes += skb->len;

if (!(from & (AT_INGRESS|AT_EGRESS)) || !skb->iif) {
dev_kfree_skb(skb);
- stats->rx_dropped++;
+ pq->rx_dropped++;
return NETDEV_TX_OK;
}

- if (skb_queue_len(&dp->rq) >= dev->tx_queue_len) {
+ txq->trans_start = jiffies;
+ __skb_queue_tail(&pq->rq, skb);
+ if (skb_queue_len(&pq->rq) >= dev->tx_queue_len)
netif_stop_queue(dev);
- }
-
- dev->trans_start = jiffies;
- skb_queue_tail(&dp->rq, skb);
- if (!dp->tasklet_pending) {
- dp->tasklet_pending = 1;
- tasklet_schedule(&dp->ifb_tasklet);
- }
+ if (skb_queue_len(&pq->rq) == 1)
+ wake_up(&pq->wq);

return NETDEV_TX_OK;
}
@@ -195,26 +196,230 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb,
struct net_device *dev)
static int ifb_close(struct net_device *dev)
{
struct ifb_private *dp = netdev_priv(dev);
+ struct ifb_private_q *pq = dp->pq;
+ int i;

- tasklet_kill(&dp->ifb_tasklet);
netif_stop_queue(dev);
- skb_queue_purge(&dp->rq);
- skb_queue_purge(&dp->tq);
+ for (i = 0; i < dev->real_num_tx_queues; i++) {
+ kthread_stop(pq[i].task);
+ __skb_queue_purge(&pq[i].rq);
+ }
+
return 0;
}

static int ifb_open(struct net_device *dev)
{
struct ifb_private *dp = netdev_priv(dev);
-
- tasklet_init(&dp->ifb_tasklet, ri_tasklet, (unsigned long)dev);
- skb_queue_head_init(&dp->rq);
- skb_queue_head_init(&dp->tq);
+ struct ifb_private_q *pq = dp->pq;
+ int i;
+
+ for (i = 0; i < dev->real_num_tx_queues; i++) {
+ pq[i].task = kthread_run(ifb_thread, &pq[i], "%s/%d", dev->name,
+ i);
+ if (IS_ERR(pq[i].task)) {
+ int err = PTR_ERR(pq[i].task);
+ while (--i >= 0)
+ kthread_stop(pq[i].task);
+ return err;
+ }
+ }
netif_start_queue(dev);

return 0;
}

+static u32 simple_tx_hashrnd;
+
+static inline __be16 pppoe_proto(const struct sk_buff *skb)
+{
+ return *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
+ sizeof(struct pppoe_hdr)));
+}
+
+static u16 ifb_select_queue(struct net_device *dev, struct sk_buff *skb)
+{
+ u32 addr1, addr2;
+ u32 hash;
+ union {
+ u16 in16[2];
+ u32 in32;
+ } ports;
+ u8 ip_proto;
+ unsigned int pull_len, proto_pull_len;
+ int ihl;
+
+ if ((hash = skb_rx_queue_recorded(skb))) {
+ while (hash >= dev->real_num_tx_queues)
+ hash -= dev->real_num_tx_queues;
+ return hash;
+ }
+
+ /* act_mirred pushed the skb before calling dev_queue_xmit() */
+ proto_pull_len = 0;
+ pull_len = skb_network_header(skb) - skb->data;
+ if (unlikely(skb_pull(skb, pull_len) == NULL)) {
+ pull_len = 0;
+ goto process_other;
+ }
+
+ ihl = 0;
+ switch (skb->protocol) {
+ case __constant_htons(ETH_P_8021Q):
+ if (unlikely(skb_pull(skb, VLAN_HLEN) == NULL))
+ goto process_other;
+ pull_len += VLAN_HLEN;
+ skb->network_header += VLAN_HLEN;
+ proto_pull_len = VLAN_HLEN;
+ switch (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto) {
+ case __constant_htons(ETH_P_IP):
+ goto process_ip;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+ case __constant_htons(ETH_P_IPV6):
+ goto process_ipv6;
+#endif
+ default:
+ goto process_other;
+ }
+ break;
+ case __constant_htons(ETH_P_PPP_SES):
+ if (unlikely(skb_pull(skb, PPPOE_SES_HLEN) == NULL))
+ goto process_other;
+ pull_len += PPPOE_SES_HLEN;
+ skb->network_header += PPPOE_SES_HLEN;
+ proto_pull_len = PPPOE_SES_HLEN;
+ switch (pppoe_proto(skb)) {
+ case __constant_htons(ETH_P_IP):
+ goto process_ip;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+ case __constant_htons(ETH_P_IPV6):
+ goto process_ipv6;
+#endif
+ default:
+ goto process_other;
+ }
+ break;
+ case __constant_htons(ETH_P_IP):
+process_ip:
+ if (unlikely(!pskb_may_pull(skb, sizeof(struct iphdr))))
+ goto process_other;
+ if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)))
+ ip_proto = ip_hdr(skb)->protocol;
+ else
+ ip_proto = 0;
+ addr1 = ip_hdr(skb)->saddr;
+ addr2 = ip_hdr(skb)->daddr;
+ ihl = ip_hdrlen(skb);
+ break;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+ case __constant_htons(ETH_P_IPV6):
+process_ipv6:
+ if (unlikely(!pskb_may_pull(skb, sizeof(struct ipv6hdr))))
+ goto process_other;
+ addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
+ addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
+ ihl = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &ip_proto);
+ if (unlikely(ihl < 0))
+ goto process_other_trans;
+ break;
+#endif
+ default:
+process_other:
+ if (pull_len != 0) {
+ skb_push(skb, pull_len);
+ if (proto_pull_len != 0)
+ skb->network_header -= proto_pull_len;
+ }
+ return skb->protocol % dev->real_num_tx_queues;
+ }
+ if (addr1 > addr2)
+ swap(addr1, addr2);
+
+ switch (ip_proto) {
+ case IPPROTO_TCP:
+ case IPPROTO_UDP:
+ case IPPROTO_DCCP:
+ case IPPROTO_ESP:
+ case IPPROTO_AH:
+ case IPPROTO_SCTP:
+ case IPPROTO_UDPLITE:
+ if (unlikely(skb_copy_bits(skb, ihl, &ports.in32, 4) < 0))
+ goto process_other_trans;
+ if (ports.in16[0] > ports.in16[1])
+ swap(ports.in16[0], ports.in16[1]);
+ break;
+ default:
+process_other_trans:
+ ports.in32 = 0;
+ break;
+ }
+
+ if (pull_len != 0) {
+ skb_push(skb, pull_len);
+ if (proto_pull_len != 0)
+ skb->network_header -= proto_pull_len;
+ }
+
+ hash = jhash_3words(addr1, addr2, ports.in32,
+ simple_tx_hashrnd ^ ip_proto);
+
+ return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
+}
+
+static int ifb_init(struct net_device *dev)
+{
+ struct ifb_private *dp = netdev_priv(dev);
+ struct ifb_private_q *pq = dp->pq;
+ int i;
+
+ pq = kcalloc(dev->real_num_tx_queues, sizeof(*pq), GFP_KERNEL);
+ if (pq == NULL)
+ return -ENOMEM;
+ dp->pq = pq;
+
+ for (i = 0; i < dev->real_num_tx_queues; i++) {
+ pq[i].dev = dev;
+ __skb_queue_head_init(&pq[i].rq);
+ init_waitqueue_head(&pq[i].wq);
+ }
+
+ return 0;
+}
+
+static void ifb_uninit(struct net_device *dev)
+{
+ struct ifb_private *dp = netdev_priv(dev);
+
+ kfree(dp->pq);
+}
+
+static const struct net_device_ops ifb_netdev_ops = {
+ .ndo_init = ifb_init,
+ .ndo_uninit = ifb_uninit,
+ .ndo_open = ifb_open,
+ .ndo_stop = ifb_close,
+ .ndo_start_xmit = ifb_xmit,
+ .ndo_validate_addr = eth_validate_addr,
+ .ndo_select_queue = ifb_select_queue,
+ .ndo_get_stats = ifb_get_stats,
+};
+
+static void ifb_setup(struct net_device *dev)
+{
+ /* Initialize the device structure. */
+ dev->destructor = free_netdev;
+ dev->netdev_ops = &ifb_netdev_ops;
+
+ /* Fill in device structure with ethernet-generic values. */
+ ether_setup(dev);
+ dev->tx_queue_len = TX_Q_LIMIT;
+
+ dev->flags |= IFF_NOARP;
+ dev->flags &= ~IFF_MULTICAST;
+ dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+ random_ether_addr(dev->dev_addr);
+}
+
static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
{
if (tb[IFLA_ADDRESS]) {
@@ -226,25 +431,38 @@ static int ifb_validate(struct nlattr *tb[],
struct nlattr *data[])
return 0;
}

+static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
+ unsigned int *num_tx_queues,
+ unsigned int *real_num_tx_queues)
+{
+ if (tb[IFLA_NTXQ]) {
+ *num_tx_queues = nla_get_u32(tb[IFLA_NTXQ]);
+ if (*num_tx_queues < 1)
+ return -EINVAL;
+ *real_num_tx_queues = *num_tx_queues;
+ } else {
+ *num_tx_queues = numtxqs;
+ *real_num_tx_queues = numtxqs;
+ }
+
+ return 0;
+}
+
static struct rtnl_link_ops ifb_link_ops __read_mostly = {
.kind = "ifb",
- .priv_size = sizeof(struct ifb_private),
.setup = ifb_setup,
.validate = ifb_validate,
+ .get_tx_queues = ifb_get_tx_queues,
+ .priv_size = sizeof(struct ifb_private),
};

-/* Number of ifb devices to be set up by this module. */
-module_param(numifbs, int, 0);
-MODULE_PARM_DESC(numifbs, "Number of ifb devices");
-
static int __init ifb_init_one(int index)
{
struct net_device *dev_ifb;
int err;

- dev_ifb = alloc_netdev(sizeof(struct ifb_private),
- "ifb%d", ifb_setup);
-
+ dev_ifb = alloc_netdev_mq(sizeof(struct ifb_private), "ifb%d",
+ ifb_setup, numtxqs);
if (!dev_ifb)
return -ENOMEM;

@@ -268,9 +486,12 @@ static int __init ifb_init_module(void)
{
int i, err;

+ if (numtxqs < 1)
+ return -EINVAL;
+
+ get_random_bytes(&simple_tx_hashrnd, 4);
rtnl_lock();
err = __rtnl_link_register(&ifb_link_ops);
-
for (i = 0; i < numifbs && !err; i++)
err = ifb_init_one(i);
if (err)
diff --git a/include/linux/if_link.h b/include/linux/if_link.h
index 1d3b242..99da411 100644
--- a/include/linux/if_link.h
+++ b/include/linux/if_link.h
@@ -78,6 +78,7 @@ enum {
#define IFLA_LINKINFO IFLA_LINKINFO
IFLA_NET_NS_PID,
IFLA_IFALIAS,
+ IFLA_NTXQ,
__IFLA_MAX
};

diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 33148a5..1cbe555 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -597,6 +597,7 @@ static inline size_t if_nlmsg_size(const struct
net_device *dev)
+ nla_total_size(4) /* IFLA_MASTER */
+ nla_total_size(1) /* IFLA_OPERSTATE */
+ nla_total_size(1) /* IFLA_LINKMODE */
+ + nla_total_size(4) /* IFLA_NTXQ */
+ rtnl_link_get_size(dev); /* IFLA_LINKINFO */
}

@@ -627,6 +628,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb,
struct net_device *dev,
netif_running(dev) ? dev->operstate : IF_OPER_DOWN);
NLA_PUT_U8(skb, IFLA_LINKMODE, dev->link_mode);
NLA_PUT_U32(skb, IFLA_MTU, dev->mtu);
+ NLA_PUT_U32(skb, IFLA_NTXQ, dev->real_num_tx_queues);

if (dev->ifindex != dev->iflink)
NLA_PUT_U32(skb, IFLA_LINK, dev->iflink);
@@ -725,6 +727,7 @@ const struct nla_policy ifla_policy[IFLA_MAX+1] = {
[IFLA_LINKINFO] = { .type = NLA_NESTED },
[IFLA_NET_NS_PID] = { .type = NLA_U32 },
[IFLA_IFALIAS] = { .type = NLA_STRING, .len = IFALIASZ-1 },
+ [IFLA_NTXQ] = { .type = NLA_U32 },
};
EXPORT_SYMBOL(ifla_policy);


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 12:42:51 +0800
Message-ID: <4AFCE3CB.5030201@gmail.com>
Reply-To: xiaosuo@gmail.com
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: 7bit
Cc: xiaosuo <xiaosuo@gmail.com>, Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-px0-f180.google.com ([209.85.216.180]:48653 "EHLO
	mail-px0-f180.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755475AbZKMEm5 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 23:42:57 -0500
Received: by pxi10 with SMTP id 10so2220759pxi.33
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 20:43:02 -0800 (PST)
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

ifb: add multi-queue support

Add multi-queue support, and one kernel thread is created for per queue.
It can used to emulate multi-queue NIC in software, and distribute work
among CPUs.
gentux linux # modprobe ifb numtxqs=2
gentux linux # ifconfig ifb0 up
gentux linux # pgrep ifb0
18508
18509
gentux linux # taskset -p 1 18508
pid 18508's current affinity mask: 3
pid 18508's new affinity mask: 1
gentux linux # taskset -p 2 18509
pid 18509's current affinity mask: 3
pid 18509's new affinity mask: 2
gentux linux # tc qdisc add dev br0 ingress
gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
action mirred egress redirect dev ifb0

This patch also introduces a ip link option "numtxqs" for specifying the
number of the TX queues. so you can add a new ifb with the command:

ip link add numtxqs 4 type ifb

Signed-off-by: Changli Gao <xiaosuo@gmail.com>
----
drivers/net/ifb.c | 491 ++++++++++++++++++++++++++++++++++--------------
include/linux/if_link.h | 1
net/core/rtnetlink.c | 3
3 files changed, 360 insertions(+), 135 deletions(-)

diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 69c2566..96ab20a 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -33,161 +33,162 @@
 #include <linux/etherdevice.h>
 #include <linux/init.h>
 #include <linux/moduleparam.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/if_vlan.h>
+#include <linux/if_pppox.h>
+#include <net/ip.h>
+#include <net/ipv6.h>
 #include <net/pkt_sched.h>
 #include <net/net_namespace.h>
 
-#define TX_TIMEOUT  (2*HZ)
-
 #define TX_Q_LIMIT    32
+
+struct ifb_private_q {
+	struct net_device	*dev;
+	struct sk_buff_head	rq;
+	wait_queue_head_t	wq;
+	struct task_struct	*task;
+	unsigned long		rx_packets;
+	unsigned long		rx_bytes;
+	unsigned long		rx_dropped;
+} ____cacheline_aligned_in_smp;
+
 struct ifb_private {
-	struct tasklet_struct   ifb_tasklet;
-	int     tasklet_pending;
-	/* mostly debug stats leave in for now */
-	unsigned long   st_task_enter; /* tasklet entered */
-	unsigned long   st_txq_refl_try; /* transmit queue refill attempt */
-	unsigned long   st_rxq_enter; /* receive queue entered */
-	unsigned long   st_rx2tx_tran; /* receive to trasmit transfers */
-	unsigned long   st_rxq_notenter; /*receiveQ not entered, resched */
-	unsigned long   st_rx_frm_egr; /* received from egress path */
-	unsigned long   st_rx_frm_ing; /* received from ingress path */
-	unsigned long   st_rxq_check;
-	unsigned long   st_rxq_rsch;
-	struct sk_buff_head     rq;
-	struct sk_buff_head     tq;
+	struct ifb_private_q	*pq;
 };
 
+/* Number of ifb devices to be set up by this module. */
 static int numifbs = 2;
+module_param(numifbs, int, 0444);
+MODULE_PARM_DESC(numifbs, "Number of ifb devices");
 
-static void ri_tasklet(unsigned long dev);
-static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
-static int ifb_open(struct net_device *dev);
-static int ifb_close(struct net_device *dev);
+/* Number of TX queues per ifb */
+static unsigned int numtxqs = 1;
+module_param(numtxqs, uint, 0444);
+MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
 
-static void ri_tasklet(unsigned long dev)
+static int ifb_thread(void *priv)
 {
-
-	struct net_device *_dev = (struct net_device *)dev;
-	struct ifb_private *dp = netdev_priv(_dev);
-	struct net_device_stats *stats = &_dev->stats;
-	struct netdev_queue *txq;
+	struct ifb_private_q *pq = priv;
+	struct net_device *dev = pq->dev, *_dev;
+	int num = pq - ((struct ifb_private *)netdev_priv(dev))->pq;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 	struct sk_buff *skb;
-
-	txq = netdev_get_tx_queue(_dev, 0);
-	dp->st_task_enter++;
-	if ((skb = skb_peek(&dp->tq)) == NULL) {
-		dp->st_txq_refl_try++;
-		if (__netif_tx_trylock(txq)) {
-			dp->st_rxq_enter++;
-			while ((skb = skb_dequeue(&dp->rq)) != NULL) {
-				skb_queue_tail(&dp->tq, skb);
-				dp->st_rx2tx_tran++;
-			}
-			__netif_tx_unlock(txq);
-		} else {
-			/* reschedule */
-			dp->st_rxq_notenter++;
-			goto resched;
+	DEFINE_WAIT(wait);
+	struct sk_buff_head tq;
+
+	__skb_queue_head_init(&tq);
+	while (1) {
+		/* move skb from rq to tq */
+		while (1) {
+			prepare_to_wait(&pq->wq, &wait, TASK_UNINTERRUPTIBLE);
+			__netif_tx_lock_bh(txq);
+			skb_queue_splice_tail_init(&pq->rq, &tq);
+			if (netif_queue_stopped(dev))
+				netif_wake_queue(dev);
+			__netif_tx_unlock_bh(txq);
+			if (kthread_should_stop() || !skb_queue_empty(&tq))
+				break;
+			schedule();
 		}
-	}
-
-	while ((skb = skb_dequeue(&dp->tq)) != NULL) {
-		u32 from = G_TC_FROM(skb->tc_verd);
-
-		skb->tc_verd = 0;
-		skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
-		stats->tx_packets++;
-		stats->tx_bytes +=skb->len;
-
-		rcu_read_lock();
-		skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
-		if (!skb->dev) {
-			rcu_read_unlock();
-			dev_kfree_skb(skb);
-			stats->tx_dropped++;
+		finish_wait(&pq->wq, &wait);
+		if (kthread_should_stop()) {
+			__skb_queue_purge(&tq);
 			break;
 		}
-		rcu_read_unlock();
-		skb->iif = _dev->ifindex;
-
-		if (from & AT_EGRESS) {
-			dp->st_rx_frm_egr++;
-			dev_queue_xmit(skb);
-		} else if (from & AT_INGRESS) {
-			dp->st_rx_frm_ing++;
-			skb_pull(skb, skb->dev->hard_header_len);
-			netif_rx(skb);
-		} else
-			BUG();
-	}
-
-	if (__netif_tx_trylock(txq)) {
-		dp->st_rxq_check++;
-		if ((skb = skb_peek(&dp->rq)) == NULL) {
-			dp->tasklet_pending = 0;
-			if (netif_queue_stopped(_dev))
-				netif_wake_queue(_dev);
-		} else {
-			dp->st_rxq_rsch++;
-			__netif_tx_unlock(txq);
-			goto resched;
+		if (need_resched())
+			schedule();
+
+		/* transfer packets */
+		while ((skb = __skb_dequeue(&tq)) != NULL) {
+			u32 from = G_TC_FROM(skb->tc_verd);
+	
+			skb->tc_verd = 0;
+			skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
+			txq->tx_packets++;
+			txq->tx_bytes +=skb->len;
+
+			rcu_read_lock();
+			_dev = dev_get_by_index_rcu(&init_net, skb->iif);
+			if (_dev != NULL) {
+				skb->dev = _dev;
+				skb->iif = dev->ifindex;
+				if (from & AT_EGRESS) {
+					dev_queue_xmit(skb);
+				} else if (from & AT_INGRESS) {
+					skb_pull(skb, _dev->hard_header_len);
+					netif_rx_ni(skb);
+				} else {
+					BUG();
+				}
+			} else {
+				dev_kfree_skb(skb);
+				txq->tx_dropped++;
+			}
+			rcu_read_unlock();
 		}
-		__netif_tx_unlock(txq);
-	} else {
-resched:
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
 	}
 
+	return 0;
 }
 
-static const struct net_device_ops ifb_netdev_ops = {
-	.ndo_open	= ifb_open,
-	.ndo_stop	= ifb_close,
-	.ndo_start_xmit	= ifb_xmit,
-	.ndo_validate_addr = eth_validate_addr,
-};
-
-static void ifb_setup(struct net_device *dev)
+struct net_device_stats* ifb_get_stats(struct net_device *dev)
 {
-	/* Initialize the device structure. */
-	dev->destructor = free_netdev;
-	dev->netdev_ops = &ifb_netdev_ops;
+	struct net_device_stats *stats = &dev->stats;
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	struct netdev_queue *txq;
+	int i;
+	unsigned long rx_packets = 0, rx_bytes = 0, rx_dropped = 0;
+	unsigned long tx_packets = 0, tx_bytes = 0, tx_dropped = 0;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		rx_packets += pq[i].rx_packets;
+		rx_bytes += pq[i].rx_bytes;
+		rx_dropped += pq[i].rx_dropped;
+		txq = netdev_get_tx_queue(dev, i);
+		tx_packets += txq->tx_packets;
+		tx_bytes += txq->tx_bytes;
+		tx_dropped += txq->tx_dropped;
+	}
 
-	/* Fill in device structure with ethernet-generic values. */
-	ether_setup(dev);
-	dev->tx_queue_len = TX_Q_LIMIT;
+	stats->rx_packets = rx_packets;
+	stats->rx_bytes = rx_bytes;
+	stats->rx_dropped = rx_dropped;
+	stats->tx_packets = tx_packets;
+	stats->tx_bytes = tx_bytes;
+	stats->tx_dropped = tx_dropped;
 
-	dev->flags |= IFF_NOARP;
-	dev->flags &= ~IFF_MULTICAST;
-	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
-	random_ether_addr(dev->dev_addr);
+	return stats;
 }
 
 static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 {
-	struct ifb_private *dp = netdev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
 	u32 from = G_TC_FROM(skb->tc_verd);
+	int num = skb_get_queue_mapping(skb);
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq + num;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 
-	stats->rx_packets++;
-	stats->rx_bytes+=skb->len;
+	pq->rx_packets++;
+	pq->rx_bytes += skb->len;
 
 	if (!(from & (AT_INGRESS|AT_EGRESS)) || !skb->iif) {
 		dev_kfree_skb(skb);
-		stats->rx_dropped++;
+		pq->rx_dropped++;
 		return NETDEV_TX_OK;
 	}
 
-	if (skb_queue_len(&dp->rq) >= dev->tx_queue_len) {
+	txq->trans_start = jiffies;
+	__skb_queue_tail(&pq->rq, skb);
+	if (skb_queue_len(&pq->rq) >= dev->tx_queue_len)
 		netif_stop_queue(dev);
-	}
-
-	dev->trans_start = jiffies;
-	skb_queue_tail(&dp->rq, skb);
-	if (!dp->tasklet_pending) {
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
-	}
+	if (skb_queue_len(&pq->rq) == 1)
+		wake_up(&pq->wq);
 
 	return NETDEV_TX_OK;
 }
@@ -195,26 +196,230 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 static int ifb_close(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
 
-	tasklet_kill(&dp->ifb_tasklet);
 	netif_stop_queue(dev);
-	skb_queue_purge(&dp->rq);
-	skb_queue_purge(&dp->tq);
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		kthread_stop(pq[i].task);
+		__skb_queue_purge(&pq[i].rq);
+	}
+
 	return 0;
 }
 
 static int ifb_open(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
-
-	tasklet_init(&dp->ifb_tasklet, ri_tasklet, (unsigned long)dev);
-	skb_queue_head_init(&dp->rq);
-	skb_queue_head_init(&dp->tq);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
+	
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		pq[i].task = kthread_run(ifb_thread, &pq[i], "%s/%d", dev->name,
+					 i);
+		if (IS_ERR(pq[i].task)) {
+			int err = PTR_ERR(pq[i].task);
+			while (--i >= 0)
+				kthread_stop(pq[i].task);
+			return err;
+		}
+	}
 	netif_start_queue(dev);
 
 	return 0;
 }
 
+static u32 simple_tx_hashrnd;
+
+static inline __be16 pppoe_proto(const struct sk_buff *skb)
+{
+	return *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
+			sizeof(struct pppoe_hdr)));
+}
+
+static u16 ifb_select_queue(struct net_device *dev, struct sk_buff *skb)
+{
+	u32 addr1, addr2;
+	u32 hash;
+	union {
+		u16 in16[2];
+		u32 in32;
+	} ports;
+	u8 ip_proto;
+	unsigned int pull_len, proto_pull_len;
+	int ihl;
+
+	if ((hash = skb_rx_queue_recorded(skb))) {
+		while (hash >= dev->real_num_tx_queues)
+			hash -= dev->real_num_tx_queues;
+		return hash;
+	}
+
+	/* act_mirred pushed the skb before calling dev_queue_xmit() */
+	proto_pull_len = 0;
+	pull_len = skb_network_header(skb) - skb->data;
+	if (unlikely(skb_pull(skb, pull_len) == NULL)) {
+		pull_len = 0;
+		goto process_other;
+	}
+
+	ihl = 0;
+	switch (skb->protocol) {
+	case __constant_htons(ETH_P_8021Q):
+		if (unlikely(skb_pull(skb, VLAN_HLEN) == NULL))
+			goto process_other;
+		pull_len += VLAN_HLEN;
+		skb->network_header += VLAN_HLEN;
+		proto_pull_len = VLAN_HLEN;
+		switch (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto) {
+		case __constant_htons(ETH_P_IP):
+			goto process_ip;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+		case __constant_htons(ETH_P_IPV6):
+			goto process_ipv6;
+#endif
+		default:
+			goto process_other;
+		}
+		break;
+	case __constant_htons(ETH_P_PPP_SES):
+		if (unlikely(skb_pull(skb, PPPOE_SES_HLEN) == NULL))
+			goto process_other;
+		pull_len += PPPOE_SES_HLEN;
+		skb->network_header += PPPOE_SES_HLEN;
+		proto_pull_len = PPPOE_SES_HLEN;
+		switch (pppoe_proto(skb)) {
+		case __constant_htons(ETH_P_IP):
+			goto process_ip;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+		case __constant_htons(ETH_P_IPV6):
+			goto process_ipv6;
+#endif
+		default:
+			goto process_other;
+		}
+		break;
+	case __constant_htons(ETH_P_IP):
+process_ip:
+		if (unlikely(!pskb_may_pull(skb, sizeof(struct iphdr))))
+			goto process_other;
+		if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)))
+			ip_proto = ip_hdr(skb)->protocol;
+		else
+			ip_proto = 0;
+		addr1 = ip_hdr(skb)->saddr;
+		addr2 = ip_hdr(skb)->daddr;
+		ihl = ip_hdrlen(skb);
+		break;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+	case __constant_htons(ETH_P_IPV6):
+process_ipv6:
+		if (unlikely(!pskb_may_pull(skb, sizeof(struct ipv6hdr))))
+			goto process_other;
+		addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
+		addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
+		ihl = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &ip_proto);
+		if (unlikely(ihl < 0))
+			goto process_other_trans;
+		break;
+#endif
+	default:
+process_other:
+		if (pull_len != 0) {
+			skb_push(skb, pull_len);
+			if (proto_pull_len != 0)
+				skb->network_header -= proto_pull_len;
+		}
+		return skb->protocol % dev->real_num_tx_queues;
+	}
+	if (addr1 > addr2)
+		swap(addr1, addr2);
+
+	switch (ip_proto) {
+	case IPPROTO_TCP:
+	case IPPROTO_UDP:
+	case IPPROTO_DCCP:
+	case IPPROTO_ESP:
+	case IPPROTO_AH:
+	case IPPROTO_SCTP:
+	case IPPROTO_UDPLITE:
+		if (unlikely(skb_copy_bits(skb, ihl, &ports.in32, 4) < 0))
+			goto process_other_trans;
+		if (ports.in16[0] > ports.in16[1])
+			swap(ports.in16[0], ports.in16[1]);
+		break;
+	default:
+process_other_trans:
+		ports.in32 = 0;
+		break;
+	}
+
+	if (pull_len != 0) {
+		skb_push(skb, pull_len);
+		if (proto_pull_len != 0)
+			skb->network_header -= proto_pull_len;
+	}
+
+	hash = jhash_3words(addr1, addr2, ports.in32,
+			    simple_tx_hashrnd ^ ip_proto);
+
+	return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
+}
+
+static int ifb_init(struct net_device *dev)
+{
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
+
+	pq = kcalloc(dev->real_num_tx_queues, sizeof(*pq), GFP_KERNEL);
+	if (pq == NULL)
+		return -ENOMEM;
+	dp->pq = pq;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		pq[i].dev = dev;
+		__skb_queue_head_init(&pq[i].rq);
+		init_waitqueue_head(&pq[i].wq);
+	}
+
+	return 0;
+}
+
+static void ifb_uninit(struct net_device *dev)
+{
+	struct ifb_private *dp = netdev_priv(dev);
+
+	kfree(dp->pq);
+}
+
+static const struct net_device_ops ifb_netdev_ops = {
+	.ndo_init		= ifb_init,
+	.ndo_uninit		= ifb_uninit,
+	.ndo_open		= ifb_open,
+	.ndo_stop		= ifb_close,
+	.ndo_start_xmit		= ifb_xmit,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_select_queue	= ifb_select_queue,
+	.ndo_get_stats		= ifb_get_stats,
+};
+
+static void ifb_setup(struct net_device *dev)
+{
+	/* Initialize the device structure. */
+	dev->destructor = free_netdev;
+	dev->netdev_ops = &ifb_netdev_ops;
+
+	/* Fill in device structure with ethernet-generic values. */
+	ether_setup(dev);
+	dev->tx_queue_len = TX_Q_LIMIT;
+
+	dev->flags |= IFF_NOARP;
+	dev->flags &= ~IFF_MULTICAST;
+	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+	random_ether_addr(dev->dev_addr);
+}
+
 static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 {
 	if (tb[IFLA_ADDRESS]) {
@@ -226,25 +431,38 @@ static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 	return 0;
 }
 
+static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
+			     unsigned int *num_tx_queues,
+			     unsigned int *real_num_tx_queues)
+{
+	if (tb[IFLA_NTXQ]) {
+		*num_tx_queues = nla_get_u32(tb[IFLA_NTXQ]);
+		if (*num_tx_queues < 1)
+			return -EINVAL;
+		*real_num_tx_queues = *num_tx_queues;
+	} else {
+		*num_tx_queues = numtxqs;
+		*real_num_tx_queues = numtxqs;
+	}
+
+	return 0;
+}
+
 static struct rtnl_link_ops ifb_link_ops __read_mostly = {
 	.kind		= "ifb",
-	.priv_size	= sizeof(struct ifb_private),
 	.setup		= ifb_setup,
 	.validate	= ifb_validate,
+	.get_tx_queues	= ifb_get_tx_queues,
+	.priv_size	= sizeof(struct ifb_private),
 };
 
-/* Number of ifb devices to be set up by this module. */
-module_param(numifbs, int, 0);
-MODULE_PARM_DESC(numifbs, "Number of ifb devices");
-
 static int __init ifb_init_one(int index)
 {
 	struct net_device *dev_ifb;
 	int err;
 
-	dev_ifb = alloc_netdev(sizeof(struct ifb_private),
-				 "ifb%d", ifb_setup);
-
+	dev_ifb = alloc_netdev_mq(sizeof(struct ifb_private), "ifb%d",
+				  ifb_setup, numtxqs);
 	if (!dev_ifb)
 		return -ENOMEM;
 
@@ -268,9 +486,12 @@ static int __init ifb_init_module(void)
 {
 	int i, err;
 
+	if (numtxqs < 1)
+		return -EINVAL;
+
+	get_random_bytes(&simple_tx_hashrnd, 4);
 	rtnl_lock();
 	err = __rtnl_link_register(&ifb_link_ops);
-
 	for (i = 0; i < numifbs && !err; i++)
 		err = ifb_init_one(i);
 	if (err)
diff --git a/include/linux/if_link.h b/include/linux/if_link.h
index 1d3b242..99da411 100644
--- a/include/linux/if_link.h
+++ b/include/linux/if_link.h
@@ -78,6 +78,7 @@ enum {
 #define IFLA_LINKINFO IFLA_LINKINFO
 	IFLA_NET_NS_PID,
 	IFLA_IFALIAS,
+	IFLA_NTXQ,
 	__IFLA_MAX
 };
 
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 33148a5..1cbe555 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -597,6 +597,7 @@ static inline size_t if_nlmsg_size(const struct net_device *dev)
 	       + nla_total_size(4) /* IFLA_MASTER */
 	       + nla_total_size(1) /* IFLA_OPERSTATE */
 	       + nla_total_size(1) /* IFLA_LINKMODE */
+	       + nla_total_size(4) /* IFLA_NTXQ */
 	       + rtnl_link_get_size(dev); /* IFLA_LINKINFO */
 }
 
@@ -627,6 +628,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
 		   netif_running(dev) ? dev->operstate : IF_OPER_DOWN);
 	NLA_PUT_U8(skb, IFLA_LINKMODE, dev->link_mode);
 	NLA_PUT_U32(skb, IFLA_MTU, dev->mtu);
+	NLA_PUT_U32(skb, IFLA_NTXQ, dev->real_num_tx_queues);
 
 	if (dev->ifindex != dev->iflink)
 		NLA_PUT_U32(skb, IFLA_LINK, dev->iflink);
@@ -725,6 +727,7 @@ const struct nla_policy ifla_policy[IFLA_MAX+1] = {
 	[IFLA_LINKINFO]		= { .type = NLA_NESTED },
 	[IFLA_NET_NS_PID]	= { .type = NLA_U32 },
 	[IFLA_IFALIAS]	        = { .type = NLA_STRING, .len = IFALIASZ-1 },
+	[IFLA_NTXQ]		= { .type = NLA_U32 },
 };
 EXPORT_SYMBOL(ifla_policy);
 



From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 12:46:28 +0800
Message-ID: <412e6f7f0911122046t667226a7xf9e5383243951a59@mail.gmail.com>
References: <4AFCE3CB.5030201@gmail.com>
Mime-Version: 1.0
Content-Type: multipart/mixed; boundary=000e0cd28b1cd636b6047839575f
Cc: xiaosuo <xiaosuo@gmail.com>, Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:40708 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755896AbZKMEq0 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Thu, 12 Nov 2009 23:46:26 -0500
Received: by pwi3 with SMTP id 3so1780309pwi.21
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 20:46:29 -0800 (PST)
In-Reply-To: <4AFCE3CB.5030201@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

--000e0cd28b1cd636b6047839575f
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: quoted-printable

2009/11/13 Changli Gao <xiaosuo@gmail.com>:
> ifb: add multi-queue support
>
> Add multi-queue support, and one kernel thread is created for per queue.
> It can used to emulate multi-queue NIC in software, and distribute work
> among CPUs.
> gentux linux # modprobe ifb numtxqs=3D2
> gentux linux # ifconfig ifb0 up
> gentux linux # pgrep ifb0
> 18508
> 18509
> gentux linux # taskset -p 1 18508
> pid 18508's current affinity mask: 3
> pid 18508's new affinity mask: 1
> gentux linux # taskset -p 2 18509
> pid 18509's current affinity mask: 3
> pid 18509's new affinity mask: 2
> gentux linux # tc qdisc add dev br0 ingress
> gentux linux # tc filter add dev br0 parent ffff: protocol ip basic
> action mirred egress redirect dev ifb0
>
> This patch also introduces a ip link option "numtxqs" for specifying the
> number of the TX queues. so you can add a new ifb with the command:
>
> ip link add numtxqs 4 type ifb
>
> Signed-off-by: Changli Gao <xiaosuo@gmail.com>
> ----

the corresponding iproute2 patch is attached.

--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

--000e0cd28b1cd636b6047839575f
Content-Type: application/octet-stream; name="iproute2-support-numtxqs.diff"
Content-Disposition: attachment; filename="iproute2-support-numtxqs.diff"
Content-Transfer-Encoding: base64
X-Attachment-Id: f_g1ygu3hz0

LS0tIGlwL2lwbGluay5jLm9yaWcJMjAwOS0xMS0xMSAxNjo1Njo0MS4wMDAwMDAwMDAgKzA4MDAK
KysrIGlwL2lwbGluay5jCTIwMDktMTEtMTIgMTQ6NTc6MjUuMDAwMDAwMDAwICswODAwCkBAIC00
OCw2ICs0OCw3IEBACiAJCWZwcmludGYoc3RkZXJyLCAiICAgICAgICAgICAgICAgICAgIFsgYWRk
cmVzcyBMTEFERFIgXVxuIik7CiAJCWZwcmludGYoc3RkZXJyLCAiICAgICAgICAgICAgICAgICAg
IFsgYnJvYWRjYXN0IExMQUREUiBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIsICIgICAgICAgICAg
ICAgICAgICAgWyBtdHUgTVRVIF1cbiIpOworCQlmcHJpbnRmKHN0ZGVyciwgIiAgICAgICAgICAg
ICAgICAgICBbIG51bXR4cXMgTlVNVFhRUyBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIsICIgICAg
ICAgICAgICAgICAgICAgdHlwZSBUWVBFIFsgQVJHUyBdXG4iKTsKIAkJZnByaW50ZihzdGRlcnIs
ICIgICAgICAgaXAgbGluayBkZWxldGUgREVWIHR5cGUgVFlQRSBbIEFSR1MgXVxuIik7CiAJCWZw
cmludGYoc3RkZXJyLCAiXG4iKTsKQEAgLTE4MCw2ICsxODEsNyBAQAogCWludCBxbGVuID0gLTE7
CiAJaW50IG10dSA9IC0xOwogCWludCBuZXRucyA9IC0xOworCWludCBudW10eHFzID0gLTE7CiAK
IAlyZXQgPSBhcmdjOwogCkBAIC0yMjEsNiArMjIzLDEzIEBACiAJCQlpZiAoZ2V0X2ludGVnZXIo
Jm10dSwgKmFyZ3YsIDApKQogCQkJCWludmFyZygiSW52YWxpZCBcIm10dVwiIHZhbHVlXG4iLCAq
YXJndik7CiAJCQlhZGRhdHRyX2woJnJlcS0+biwgc2l6ZW9mKCpyZXEpLCBJRkxBX01UVSwgJm10
dSwgNCk7CisJCX0gZWxzZSBpZiAoc3RyY21wKCphcmd2LCAibnVtdHhxcyIpID09IDApIHsKKwkJ
CU5FWFRfQVJHKCk7CisJCQlpZiAobnVtdHhxcyAhPSAtMSkKKwkJCQlkdXBhcmcoIm51bXR4cXMi
LCAqYXJndik7CisJCQlpZiAoZ2V0X2ludGVnZXIoJm51bXR4cXMsICphcmd2LCAwKSkKKwkJCQlp
bnZhcmcoIkludmFsaWQgXCJudW10eHFzXCIgdmFsdWVcbiIsICphcmd2KTsKKwkJCWFkZGF0dHJf
bCgmcmVxLT5uLCBzaXplb2YoKnJlcSksIElGTEFfTlRYUSwgJm51bXR4cXMsIDQpOwogICAgICAg
ICAgICAgICAgIH0gZWxzZSBpZiAoc3RyY21wKCphcmd2LCAibmV0bnMiKSA9PSAwKSB7CiAgICAg
ICAgICAgICAgICAgICAgICAgICBORVhUX0FSRygpOwogICAgICAgICAgICAgICAgICAgICAgICAg
aWYgKG5ldG5zICE9IC0xKQotLS0gaXAvaXBhZGRyZXNzLmMub3JpZwkyMDA5LTExLTEzIDEyOjAw
OjMzLjAwMDAwMDAwMCArMDgwMAorKysgaXAvaXBhZGRyZXNzLmMJMjAwOS0xMS0xMyAxMjowNDox
NC4wMDAwMDAwMDAgKzA4MDAKQEAgLTI1NSw2ICsyNTUsOSBAQAogCWlmIChmaWx0ZXIuc2hvd3F1
ZXVlKQogCQlwcmludF9xdWV1ZWxlbihmcCwgKGNoYXIqKVJUQV9EQVRBKHRiW0lGTEFfSUZOQU1F
XSkpOwogCisJaWYgKHRiW0lGTEFfTlRYUV0pCisJCWZwcmludGYoZnAsICIgbnVtdHhxcyAlZCIs
ICoodW5zaWduZWQgaW50KilSVEFfREFUQSh0YltJRkxBX05UWFFdKSk7CisKIAlpZiAoIWZpbHRl
ci5mYW1pbHkgfHwgZmlsdGVyLmZhbWlseSA9PSBBRl9QQUNLRVQpIHsKIAkJU1BSSU5UX0JVRihi
MSk7CiAJCWZwcmludGYoZnAsICIlcyIsIF9TTF8pOwo=
--000e0cd28b1cd636b6047839575f--

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 06:56:29 +0100
Message-ID: <4AFCF50D.6040106@gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>	 <4AFC0436.9090906@gmail.com> <412e6f7f0911121726t47874807g2223d2f1141818f5@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:36243 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752063AbZKMF4f (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 00:56:35 -0500
In-Reply-To: <412e6f7f0911121726t47874807g2223d2f1141818f5@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =A8=A6crit :
> 2009/11/12 Eric Dumazet <eric.dumazet@gmail.com>:
>> I believe this patch is fine, but maybe Jarek concern about workqueu=
e
>> vs tasklet should be addressed...
>>
>> We could use the previous handling in case numtxqs=3D=3D1 , ie use a=
 tasklet
>> instead of a work queue ?
>=20
> I don't think it is a good idea. If we do so, the code will be messy,
> and lost the flexibility of process. In fact, latency isn't a problem
> when system load isn't high, and when system load is high (due to too
> many NIC IRQs), throughput and interaction is more important, and the
> current linux networking subsystem just dose so through the softirqd
> mechanism.
>=20

Messy ? Because of few tests added in code, and branches always
correctly predicted ?

Still some people might rely on tasklet instead of workqueues
and added scheduler stress and latency penalty. Tasklet are softirq
and normally are processed a few nanosecs later than RX softirq,=20
on the same CPU, while with your workqueue, I guess the scheduler will
try to not migrate it, so we add a penalty for light to moderate load.

I guess this new ifb mode would be a regression for them ?

If you dont want to maintain a compatibility mode, maybe you
should introduce a complete new driver, drivers/net/ifbmq.c or ifbwq.c

(multiqueue or workqueue references)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 14:16:40 +0800
Message-ID: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>
	 <4AFC0436.9090906@gmail.com>
	 <412e6f7f0911121726t47874807g2223d2f1141818f5@mail.gmail.com>
	 <4AFCF50D.6040106@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:57553 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752631AbZKMGQf convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 01:16:35 -0500
Received: by pzk1 with SMTP id 1so1085158pzk.33
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 22:16:41 -0800 (PST)
In-Reply-To: <4AFCF50D.6040106@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

2009/11/13 Eric Dumazet <eric.dumazet@gmail.com>:
>
> Messy ? Because of few tests added in code, and branches always
> correctly predicted ?
>
> Still some people might rely on tasklet instead of workqueues
> and added scheduler stress and latency penalty. Tasklet are softirq
> and normally are processed a few nanosecs later than RX softirq,
> on the same CPU, while with your workqueue, I guess the scheduler wil=
l
> try to not migrate it, so we add a penalty for light to moderate load=
=2E
>
> I guess this new ifb mode would be a regression for them ?
>
> If you dont want to maintain a compatibility mode, maybe you
> should introduce a complete new driver, drivers/net/ifbmq.c or ifbwq.=
c
>
> (multiqueue or workqueue references)
>

It sounds a good idea.

--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Patrick McHardy <kaber@trash.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 08:18:32 +0100
Message-ID: <4AFD0848.6010708@trash.net>
References: <4AFA8911.7050204@gmail.com> <4AFBD911.6000900@gmail.com>	 <412e6f7f0911120148r48fa553bt610e013d96522a3d@mail.gmail.com>	 <4AFC25A7.10301@trash.net> <412e6f7f0911121732j106f96a2rb77f68098b7fd47e@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=ISO-8859-15
Content-Transfer-Encoding: 7bit
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from stinky.trash.net ([213.144.137.162]:43421 "EHLO
	stinky.trash.net" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1753463AbZKMHS2 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 02:18:28 -0500
In-Reply-To: <412e6f7f0911121732j106f96a2rb77f68098b7fd47e@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao wrote:
> On Thu, Nov 12, 2009 at 11:11 PM, Patrick McHardy <kaber@trash.net> wrote:
>> Changli Gao wrote:
>>> The corresponding iprout2 patch is attached.
>> Printing the value during dumps is missing from this patch.
>>
> 
> I doesn't know the rules about adding more dumps. Can I add it in the
> end of the corresponding line?
> 
> ifb0: <BROADCAST,NOARP> mtu 1500 qdisc noop state DOWN qlen 32 ntxqs 4

Yes, sounds fine.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Jarek Poplawski <jarkao2@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 07:45:08 +0000
Message-ID: <20091113074508.GA6605@ff.dom.local>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-bw0-f227.google.com ([209.85.218.227]:57746 "EHLO
	mail-bw0-f227.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752828AbZKMHpK (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 02:45:10 -0500
Received: by bwz27 with SMTP id 27so3107881bwz.21
        for <netdev@vger.kernel.org>; Thu, 12 Nov 2009 23:45:14 -0800 (PST)
Content-Disposition: inline
In-Reply-To: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On 13-11-2009 07:16, Changli Gao wrote:
> 2009/11/13 Eric Dumazet <eric.dumazet@gmail.com>:
>> Messy ? Because of few tests added in code, and branches always
>> correctly predicted ?
>>
>> Still some people might rely on tasklet instead of workqueues
>> and added scheduler stress and latency penalty. Tasklet are softirq
>> and normally are processed a few nanosecs later than RX softirq,
>> on the same CPU, while with your workqueue, I guess the scheduler will
>> try to not migrate it, so we add a penalty for light to moderate load.
>>
>> I guess this new ifb mode would be a regression for them ?
>>
>> If you dont want to maintain a compatibility mode, maybe you
>> should introduce a complete new driver, drivers/net/ifbmq.c or ifbwq.c
>>
>> (multiqueue or workqueue references)
>>
> 
> It sounds a good idea.

I don't think so. There would be a lot of code duplication and later
maintenance problems only because of the scheduling method. The main
question is to establish if there is really no performance difference
(which I doubt) - unless Changli can show some tests for various
setups now. On the other hand, if there is a difference, why keep
ineffective solution - similar thing should be possible to do in the
softirq context as well.

So it should not be a big problem to do it a bit messy for some
testing time. Since we can use separate ->ndo_start_xmit() etc. it
shouldn't be too messy, I guess.

Jarek P.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 16:54:50 +0800
Message-ID: <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	 <20091113074508.GA6605@ff.dom.local>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Jarek Poplawski <jarkao2@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:65011 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1754820AbZKMIyo convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 03:54:44 -0500
Received: by pzk1 with SMTP id 1so1159772pzk.33
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 00:54:50 -0800 (PST)
In-Reply-To: <20091113074508.GA6605@ff.dom.local>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, Nov 13, 2009 at 3:45 PM, Jarek Poplawski <jarkao2@gmail.com> wr=
ote:
> On 13-11-2009 07:16, Changli Gao wrote:
>
> I don't think so. There would be a lot of code duplication and later
> maintenance problems only because of the scheduling method. The main
> question is to establish if there is really no performance difference
> (which I doubt) - unless Changli can show some tests for various
> setups now. On the other hand, if there is a difference, why keep
> ineffective solution - similar thing should be possible to do in the
> softirq context as well.
>
> So it should not be a big problem to do it a bit messy for some
> testing time. Since we can use separate ->ndo_start_xmit() etc. it
> shouldn't be too messy, I guess.
>

I have done a simple test. I run a simple program on computer A, which
sends SYN packets with random source ports to Computer B's 80 port (No
socket listens on that port, so tcp reset packets will be sent) in
90kpps. On computer B, I redirect the traffic to IFB. At the same
time, I ping from B to A to get the RTT between them. I can't see any
difference between the original IFB and my MQ version. They are both:

CPU idle: 50%
Latency: 0.3-0.4ms, burst 2ms.


--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Jarek Poplawski <jarkao2@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 09:18:25 +0000
Message-ID: <20091113091825.GA7449@ff.dom.local>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com> <20091113074508.GA6605@ff.dom.local> <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-fx0-f221.google.com ([209.85.220.221]:42121 "EHLO
	mail-fx0-f221.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755660AbZKMJS1 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 04:18:27 -0500
Received: by fxm21 with SMTP id 21so209581fxm.21
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 01:18:30 -0800 (PST)
Content-Disposition: inline
In-Reply-To: <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, Nov 13, 2009 at 04:54:50PM +0800, Changli Gao wrote:
> On Fri, Nov 13, 2009 at 3:45 PM, Jarek Poplawski <jarkao2@gmail.com> wrote:
> > On 13-11-2009 07:16, Changli Gao wrote:
> >
> > I don't think so. There would be a lot of code duplication and later
> > maintenance problems only because of the scheduling method. The main
> > question is to establish if there is really no performance difference
> > (which I doubt) - unless Changli can show some tests for various
> > setups now. On the other hand, if there is a difference, why keep
> > ineffective solution - similar thing should be possible to do in the
> > softirq context as well.
> >
> > So it should not be a big problem to do it a bit messy for some
> > testing time. Since we can use separate ->ndo_start_xmit() etc. it
> > shouldn't be too messy, I guess.
> >
> 
> I have done a simple test. I run a simple program on computer A, which
> sends SYN packets with random source ports to Computer B's 80 port (No
> socket listens on that port, so tcp reset packets will be sent) in
> 90kpps. On computer B, I redirect the traffic to IFB. At the same
> time, I ping from B to A to get the RTT between them. I can't see any
> difference between the original IFB and my MQ version. They are both:
> 
> CPU idle: 50%
> Latency: 0.3-0.4ms, burst 2ms.
> 

I'm mostly concerned with routers doing forwarding with 1Gb or 10Gb
NICs (including multiqueue). Alas/happily I don't have such a problem,
but can't help you with testing either.

Regards,
Jarek P.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 17:38:56 +0800
Message-ID: <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	 <20091113074508.GA6605@ff.dom.local>
	 <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
	 <20091113091825.GA7449@ff.dom.local>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Jarek Poplawski <jarkao2@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:64897 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755631AbZKMJiu convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 04:38:50 -0500
Received: by pzk1 with SMTP id 1so1181141pzk.33
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 01:38:56 -0800 (PST)
In-Reply-To: <20091113091825.GA7449@ff.dom.local>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, Nov 13, 2009 at 5:18 PM, Jarek Poplawski <jarkao2@gmail.com> wr=
ote:
> On Fri, Nov 13, 2009 at 04:54:50PM +0800, Changli Gao wrote:
>> On Fri, Nov 13, 2009 at 3:45 PM, Jarek Poplawski <jarkao2@gmail.com>=
 wrote:
>>
>> I have done a simple test. I run a simple program on computer A, whi=
ch
>> sends SYN packets with random source ports to Computer B's 80 port (=
No
>> socket listens on that port, so tcp reset packets will be sent) in
>> 90kpps. On computer B, I redirect the traffic to IFB. At the same
>> time, I ping from B to A to get the RTT between them. I can't see an=
y
>> difference between the original IFB and my MQ version. They are both=
:
>>
>> CPU idle: 50%
>> Latency: 0.3-0.4ms, burst 2ms.
>>
>
> I'm mostly concerned with routers doing forwarding with 1Gb or 10Gb
> NICs (including multiqueue). Alas/happily I don't have such a problem=
,
> but can't help you with testing either.
>

Oh, :) . I know more than one companies use kernel threads to forward
packets, and there isn't explicit extra overhead at all. And as you
know, as throughput increases, NAPI will bind the NIC to a CPU, and
softirqd will be waked up to do the work, which should be done in
SoftIRQ context. At that time, there isn't any difference between my
approach and the current kernel's.


--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Jarek Poplawski <jarkao2@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 09:57:31 +0000
Message-ID: <20091113095731.GA7749@ff.dom.local>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com> <20091113074508.GA6605@ff.dom.local> <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com> <20091113091825.GA7449@ff.dom.local> <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-fx0-f221.google.com ([209.85.220.221]:61559 "EHLO
	mail-fx0-f221.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1755606AbZKMJ5b (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 04:57:31 -0500
Received: by fxm21 with SMTP id 21so243649fxm.21
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 01:57:35 -0800 (PST)
Content-Disposition: inline
In-Reply-To: <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, Nov 13, 2009 at 05:38:56PM +0800, Changli Gao wrote:
> On Fri, Nov 13, 2009 at 5:18 PM, Jarek Poplawski <jarkao2@gmail.com> wrote:
> > I'm mostly concerned with routers doing forwarding with 1Gb or 10Gb
> > NICs (including multiqueue). Alas/happily I don't have such a problem,
> > but can't help you with testing either.
> >
> 
> Oh, :) . I know more than one companies use kernel threads to forward
> packets, and there isn't explicit extra overhead at all. And as you
> know, as throughput increases, NAPI will bind the NIC to a CPU, and
> softirqd will be waked up to do the work, which should be done in
> SoftIRQ context. At that time, there isn't any difference between my
> approach and the current kernel's.

I'm not against your solution at all. It only needs more proof... You
seem to forget the main networking paths now are just softirq, and
it's probably for some reason. If kernel threads are good enough, it
seems we should do more such changes.

Jarek P.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 19:25:33 +0800
Message-ID: <412e6f7f0911130325r20e2fbbbg21e2e384de1f7c0f@mail.gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	 <20091113074508.GA6605@ff.dom.local>
	 <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
	 <20091113091825.GA7449@ff.dom.local>
	 <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
	 <20091113095731.GA7749@ff.dom.local>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Jarek Poplawski <jarkao2@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:41551 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1753921AbZKMLZ2 convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 06:25:28 -0500
Received: by pzk1 with SMTP id 1so1229692pzk.33
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 03:25:33 -0800 (PST)
In-Reply-To: <20091113095731.GA7749@ff.dom.local>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, Nov 13, 2009 at 5:57 PM, Jarek Poplawski <jarkao2@gmail.com> wr=
ote:
>
> I'm not against your solution at all. It only needs more proof...

I know. :)

> You seem to forget the main networking paths now are just softirq, an=
d
> it's probably for some reason. If kernel threads are good enough, it
> seems we should do more such changes.
>

I find there is still a softirqd kernel thread for each online CPU,
and these threads will be waked up if there are more SoftIRQs need to
be done after restarting the SoftIRQ processing many times, to keep
the whole system responsible. Did I miss sth.? And at the other side,
real time branch just wants to make all the activities based on kernel
threads, even ISR.


--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Jarek Poplawski <jarkao2@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 12:32:03 +0000
Message-ID: <20091113123203.GA8648@ff.dom.local>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com> <20091113074508.GA6605@ff.dom.local> <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com> <20091113091825.GA7449@ff.dom.local> <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com> <20091113095731.GA7749@ff.dom.local> <412e6f7f0911130325r20e2fbbbg21e2e384de1f7c0f@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=us-ascii
Cc: Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from fg-out-1718.google.com ([72.14.220.156]:42406 "EHLO
	fg-out-1718.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751337AbZKMMcD (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 07:32:03 -0500
Received: by fg-out-1718.google.com with SMTP id d23so207775fga.1
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 04:32:08 -0800 (PST)
Content-Disposition: inline
In-Reply-To: <412e6f7f0911130325r20e2fbbbg21e2e384de1f7c0f@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, Nov 13, 2009 at 07:25:33PM +0800, Changli Gao wrote:
> I find there is still a softirqd kernel thread for each online CPU,
> and these threads will be waked up if there are more SoftIRQs need to
> be done after restarting the SoftIRQ processing many times, to keep
> the whole system responsible. Did I miss sth.? And at the other side,
> real time branch just wants to make all the activities based on kernel
> threads, even ISR.

I wish them well! But again: for some strange reason network admins,
who sometimes care to complain here, rarely (very rarely) mention this
RT branch... (Actually, I recall only one case, and it probably wasn't
even real admin ;-)

Jarek P.


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 14:10:16 +0100
Message-ID: <4AFD5AB8.2070902@gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>	 <20091113074508.GA6605@ff.dom.local>	 <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>	 <20091113091825.GA7449@ff.dom.local>	 <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>	 <20091113095731.GA7749@ff.dom.local> <412e6f7f0911130325r20e2fbbbg21e2e384de1f7c0f@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Jarek Poplawski <jarkao2@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:36307 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751800AbZKMNKS (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 08:10:18 -0500
In-Reply-To: <412e6f7f0911130325r20e2fbbbg21e2e384de1f7c0f@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =C3=A9crit :
> On Fri, Nov 13, 2009 at 5:57 PM, Jarek Poplawski <jarkao2@gmail.com> =
wrote:
>> I'm not against your solution at all. It only needs more proof...
>=20
> I know. :)
>=20
>> You seem to forget the main networking paths now are just softirq, a=
nd
>> it's probably for some reason. If kernel threads are good enough, it
>> seems we should do more such changes.
>>
>=20
> I find there is still a softirqd kernel thread for each online CPU,
> and these threads will be waked up if there are more SoftIRQs need to
> be done after restarting the SoftIRQ processing many times, to keep
> the whole system responsible. Did I miss sth.? And at the other side,
> real time branch just wants to make all the activities based on kerne=
l
> threads, even ISR.
>=20
>=20


You seem to focus on stress loads, where we enter ksoftirqd more to get
more throughput (and cpu fairness), at latency expense.

But in normal situations, ksoftirq is not started at all.


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 14:55:50 +0100
Message-ID: <4AFD6566.2040007@gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>	 <20091113074508.GA6605@ff.dom.local> <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Jarek Poplawski <jarkao2@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:54425 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1754359AbZKMNzw (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 08:55:52 -0500
In-Reply-To: <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =C3=A9crit :

> I have done a simple test. I run a simple program on computer A, whic=
h
> sends SYN packets with random source ports to Computer B's 80 port (N=
o
> socket listens on that port, so tcp reset packets will be sent) in
> 90kpps. On computer B, I redirect the traffic to IFB. At the same
> time, I ping from B to A to get the RTT between them. I can't see any
> difference between the original IFB and my MQ version. They are both:
>=20
> CPU idle: 50%
> Latency: 0.3-0.4ms, burst 2ms.

Yes, but redo your test with normal load (including several user thread=
s
fighting for scheduler/cpu use) :)

softirq preempts user threads instantly (they are interrupts after all =
:) ),
while workqueue do compete with other workqueues/threads.

Tom Herbert RPS solution is still using softirqs.

I like to test workqueue based solution with IFB, but some people might=
 still
keep previous pure softirq solution for production environment, not ben=
chmarks...


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Stephen Hemminger <shemminger@vyatta.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 08:15:53 -0800
Message-ID: <20091113081553.0568296c@s6510>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	<20091113074508.GA6605@ff.dom.local>
	<412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
	<20091113091825.GA7449@ff.dom.local>
	<412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=US-ASCII
Content-Transfer-Encoding: 7bit
Cc: Jarek Poplawski <jarkao2@gmail.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail.vyatta.com ([76.74.103.46]:35969 "EHLO mail.vyatta.com"
	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
	id S1755729AbZKMQQP (ORCPT <rfc822;netdev@vger.kernel.org>);
	Fri, 13 Nov 2009 11:16:15 -0500
In-Reply-To: <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, 13 Nov 2009 17:38:56 +0800
Changli Gao <xiaosuo@gmail.com> wrote:

> On Fri, Nov 13, 2009 at 5:18 PM, Jarek Poplawski <jarkao2@gmail.com> wrote:
> > On Fri, Nov 13, 2009 at 04:54:50PM +0800, Changli Gao wrote:
> >> On Fri, Nov 13, 2009 at 3:45 PM, Jarek Poplawski <jarkao2@gmail.com> wrote:
> >>
> >> I have done a simple test. I run a simple program on computer A, which
> >> sends SYN packets with random source ports to Computer B's 80 port (No
> >> socket listens on that port, so tcp reset packets will be sent) in
> >> 90kpps. On computer B, I redirect the traffic to IFB. At the same
> >> time, I ping from B to A to get the RTT between them. I can't see any
> >> difference between the original IFB and my MQ version. They are both:
> >>
> >> CPU idle: 50%
> >> Latency: 0.3-0.4ms, burst 2ms.
> >>
> >
> > I'm mostly concerned with routers doing forwarding with 1Gb or 10Gb
> > NICs (including multiqueue). Alas/happily I don't have such a problem,
> > but can't help you with testing either.
> >
> 
> Oh, :) . I know more than one companies use kernel threads to forward
> packets, and there isn't explicit extra overhead at all. And as you
> know, as throughput increases, NAPI will bind the NIC to a CPU, and
> softirqd will be waked up to do the work, which should be done in
> SoftIRQ context. At that time, there isn't any difference between my
> approach and the current kernel's.
> 
> 

Why not make IFB a NAPI device. This would get rid of the extra soft-irq
round trip from going through netif_rx().  It would also behave like
regular multi-queue recieive device, and eliminate need for seperate
tasklets or threads.


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Sat, 14 Nov 2009 07:28:42 +0800
Message-ID: <412e6f7f0911131528i3105648y49213143e12f2522@mail.gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	 <20091113074508.GA6605@ff.dom.local>
	 <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
	 <20091113091825.GA7449@ff.dom.local>
	 <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
	 <20091113081553.0568296c@s6510>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Jarek Poplawski <jarkao2@gmail.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Stephen Hemminger <shemminger@vyatta.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-px0-f180.google.com ([209.85.216.180]:47413 "EHLO
	mail-px0-f180.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S932614AbZKMX2i convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 18:28:38 -0500
Received: by pxi10 with SMTP id 10so2776378pxi.33
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 15:28:42 -0800 (PST)
In-Reply-To: <20091113081553.0568296c@s6510>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Sat, Nov 14, 2009 at 12:15 AM, Stephen Hemminger
<shemminger@vyatta.com> wrote:
> On Fri, 13 Nov 2009 17:38:56 +0800
> Changli Gao <xiaosuo@gmail.com> wrote:
>>
>> Oh, :) . I know more than one companies use kernel threads to forwar=
d
>> packets, and there isn't explicit extra overhead at all. And as you
>> know, as throughput increases, NAPI will bind the NIC to a CPU, and
>> softirqd will be waked up to do the work, which should be done in
>> SoftIRQ context. At that time, there isn't any difference between my
>> approach and the current kernel's.
>>
> Why not make IFB a NAPI device. This would get rid of the extra soft-=
irq
> round trip from going through netif_rx(). =C2=A0It would also behave =
like
> regular multi-queue recieive device, and eliminate need for seperate
> tasklets or threads.
>

It needs to send remote SoftIRQ, as Receiving Packet Steering, and we
must support a extra interface to map load to CPUs.

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Stephen Hemminger <shemminger@vyatta.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Fri, 13 Nov 2009 15:32:29 -0800
Message-ID: <20091113153229.408c889f@nehalam>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	<20091113074508.GA6605@ff.dom.local>
	<412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
	<20091113091825.GA7449@ff.dom.local>
	<412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
	<20091113081553.0568296c@s6510>
	<412e6f7f0911131528i3105648y49213143e12f2522@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Jarek Poplawski <jarkao2@gmail.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail.vyatta.com ([76.74.103.46]:57657 "EHLO mail.vyatta.com"
	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
	id S932450AbZKMXcp convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 18:32:45 -0500
In-Reply-To: <412e6f7f0911131528i3105648y49213143e12f2522@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Sat, 14 Nov 2009 07:28:42 +0800
Changli Gao <xiaosuo@gmail.com> wrote:

> On Sat, Nov 14, 2009 at 12:15 AM, Stephen Hemminger
> <shemminger@vyatta.com> wrote:
> > On Fri, 13 Nov 2009 17:38:56 +0800
> > Changli Gao <xiaosuo@gmail.com> wrote:
> >>
> >> Oh, :) . I know more than one companies use kernel threads to forw=
ard
> >> packets, and there isn't explicit extra overhead at all. And as yo=
u
> >> know, as throughput increases, NAPI will bind the NIC to a CPU, an=
d
> >> softirqd will be waked up to do the work, which should be done in
> >> SoftIRQ context. At that time, there isn't any difference between =
my
> >> approach and the current kernel's.
> >>
> > Why not make IFB a NAPI device. This would get rid of the extra sof=
t-irq
> > round trip from going through netif_rx(). =C2=A0It would also behav=
e like
> > regular multi-queue recieive device, and eliminate need for seperat=
e
> > tasklets or threads.
> >
>=20
> It needs to send remote SoftIRQ, as Receiving Packet Steering, and we
> must support a extra interface to map load to CPUs.
>=20

But it still could use NAPI to avoid causing excess packet overhead.
The softirq could be equivalent of hardirq in network device.

--=20

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Sat, 14 Nov 2009 07:42:12 +0800
Message-ID: <412e6f7f0911131542w1029893cmd07599660b810ed5@mail.gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	 <20091113074508.GA6605@ff.dom.local>
	 <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
	 <20091113091825.GA7449@ff.dom.local>
	 <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
	 <20091113081553.0568296c@s6510>
	 <412e6f7f0911131528i3105648y49213143e12f2522@mail.gmail.com>
	 <20091113153229.408c889f@nehalam>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Jarek Poplawski <jarkao2@gmail.com>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Stephen Hemminger <shemminger@vyatta.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:64949 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S932549AbZKMXmI convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Fri, 13 Nov 2009 18:42:08 -0500
Received: by pwi3 with SMTP id 3so2276196pwi.21
        for <netdev@vger.kernel.org>; Fri, 13 Nov 2009 15:42:12 -0800 (PST)
In-Reply-To: <20091113153229.408c889f@nehalam>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Sat, Nov 14, 2009 at 7:32 AM, Stephen Hemminger
<shemminger@vyatta.com> wrote:
> On Sat, 14 Nov 2009 07:28:42 +0800
> Changli Gao <xiaosuo@gmail.com> wrote:
>
>> On Sat, Nov 14, 2009 at 12:15 AM, Stephen Hemminger
>>
>> It needs to send remote SoftIRQ, as Receiving Packet Steering, and w=
e
>> must support a extra interface to map load to CPUs.
>>
>
> But it still could use NAPI to avoid causing excess packet overhead.
> The softirq could be equivalent of hardirq in network device.
>

Yea, the overhead of SoftIRQ is less than kernel threads, and I'll try
to find a way to solve both flexibility and efficiency. Maybe I need
some real NIC drivers as examples. Is there a standard API to bind RQs
of NIC to CPUs, such as ioctl or setsockopt?


--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Sat, 14 Nov 2009 13:53:04 +0100
Message-ID: <4AFEA830.3000101@gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>	 <20091113074508.GA6605@ff.dom.local>	 <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>	 <20091113091825.GA7449@ff.dom.local>	 <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>	 <20091113081553.0568296c@s6510>	 <412e6f7f0911131528i3105648y49213143e12f2522@mail.gmail.com>	 <20091113153229.408c889f@nehalam> <412e6f7f0911131542w1029893cmd07599660b810ed5@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Stephen Hemminger <shemminger@vyatta.com>,
	Jarek Poplawski <jarkao2@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:37383 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751362AbZKNMxH (ORCPT
	<rfc822;netdev@vger.kernel.org>); Sat, 14 Nov 2009 07:53:07 -0500
In-Reply-To: <412e6f7f0911131542w1029893cmd07599660b810ed5@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =C3=A9crit :
> Yea, the overhead of SoftIRQ is less than kernel threads, and I'll tr=
y
> to find a way to solve both flexibility and efficiency. Maybe I need
> some real NIC drivers as examples. Is there a standard API to bind RQ=
s
> of NIC to CPUs, such as ioctl or setsockopt?

Thats a good question... napi is bound to a cpu, and you'll need
things that were done by Tom Herbert in its RPS patch, to
be able to deleguate work to remote cpus napi contexts.

But if we consider RPS being close to be committed, shouldnt
IFB use its, in order to not duplicate changes ?

Or, if you prefer, once RPS is in, we dont need to change IFB, since
RPS will already split the load to multiple cpus before entering IFB.


From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Sat, 14 Nov 2009 21:30:52 +0800
Message-ID: <412e6f7f0911140530h1fac3053v5d5fe30ceac303a1@mail.gmail.com>
References: <412e6f7f0911122216u6880e855g6a15dac29ad6a100@mail.gmail.com>
	 <20091113074508.GA6605@ff.dom.local>
	 <412e6f7f0911130054i7a508a6ah16368f11bdc7353d@mail.gmail.com>
	 <20091113091825.GA7449@ff.dom.local>
	 <412e6f7f0911130138td181935w36cab3119972753e@mail.gmail.com>
	 <20091113081553.0568296c@s6510>
	 <412e6f7f0911131528i3105648y49213143e12f2522@mail.gmail.com>
	 <20091113153229.408c889f@nehalam>
	 <412e6f7f0911131542w1029893cmd07599660b810ed5@mail.gmail.com>
	 <4AFEA830.3000101@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: Stephen Hemminger <shemminger@vyatta.com>,
	Jarek Poplawski <jarkao2@gmail.com>,
	"David S. Miller" <davem@davemloft.net>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:40767 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752609AbZKNNaq convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Sat, 14 Nov 2009 08:30:46 -0500
Received: by pwi3 with SMTP id 3so2499960pwi.21
        for <netdev@vger.kernel.org>; Sat, 14 Nov 2009 05:30:52 -0800 (PST)
In-Reply-To: <4AFEA830.3000101@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Sat, Nov 14, 2009 at 8:53 PM, Eric Dumazet <eric.dumazet@gmail.com> =
wrote:
> Changli Gao a =C3=A9crit :
>> Yea, the overhead of SoftIRQ is less than kernel threads, and I'll t=
ry
>> to find a way to solve both flexibility and efficiency. Maybe I need
>> some real NIC drivers as examples. Is there a standard API to bind R=
Qs
>> of NIC to CPUs, such as ioctl or setsockopt?
>
> Thats a good question... napi is bound to a cpu, and you'll need
> things that were done by Tom Herbert in its RPS patch, to
> be able to deleguate work to remote cpus napi contexts.
>
> But if we consider RPS being close to be committed, shouldnt
> IFB use its, in order to not duplicate changes ?
>
> Or, if you prefer, once RPS is in, we dont need to change IFB, since
> RPS will already split the load to multiple cpus before entering IFB.
>
>

I had known RPS before I began my IFB work, so I added Tom to the CC
list, but I don't know if RPS will be in the mainline. Since there
isn't any core change in IFB as is in RPS, I think it is easier to
merge IFB into the mainline. If RPS is merged before IFB-MQ, I'll give
IFB-MQ up, and vice versa I think.

--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: [PATCH] ifb: add multi-queue support
Date: Mon, 16 Nov 2009 15:31:36 +0800
Message-ID: <4B00FFD8.4040508@gmail.com>
Reply-To: xiaosuo@gmail.com
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: 7bit
Cc: xiaosuo <xiaosuo@gmail.com>, Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:64523 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1752388AbZKPHbl (ORCPT
	<rfc822;netdev@vger.kernel.org>); Mon, 16 Nov 2009 02:31:41 -0500
Received: by pwi3 with SMTP id 3so3113391pwi.21
        for <netdev@vger.kernel.org>; Sun, 15 Nov 2009 23:31:46 -0800 (PST)
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

ifb: add multi-queue support.

Add multi-queue support, through more than one tasklets. One tasklet per
queue always steers on the same CPU, and if the number of the
tasklets(queues) is lager than the number of CPUs, more than one
tasklets will be assigned to the same CPU.

Signed-off-by: Changli Gao <xiaosuo@gmail.com>
----
drivers/net/ifb.c | 453 ++++++++++++++++++++++++++++++++--------------
include/linux/if_link.h | 1
include/linux/interrupt.h | 16 +
net/core/rtnetlink.c | 3
4 files changed, 342 insertions(+), 131 deletions(-)

diff --git a/drivers/net/ifb.c b/drivers/net/ifb.c
index 69c2566..0639822 100644
--- a/drivers/net/ifb.c
+++ b/drivers/net/ifb.c
@@ -33,161 +33,145 @@
 #include <linux/etherdevice.h>
 #include <linux/init.h>
 #include <linux/moduleparam.h>
+#include <linux/wait.h>
+#include <linux/sched.h>
+#include <linux/kthread.h>
+#include <linux/ip.h>
+#include <linux/ipv6.h>
+#include <linux/if_vlan.h>
+#include <linux/if_pppox.h>
+#include <net/ip.h>
+#include <net/ipv6.h>
 #include <net/pkt_sched.h>
 #include <net/net_namespace.h>
 
-#define TX_TIMEOUT  (2*HZ)
-
 #define TX_Q_LIMIT    32
+
+struct ifb_private_q {
+	struct net_device	*dev;
+	struct sk_buff_head	rq;
+	struct tasklet_struct	task;
+	unsigned long		rx_packets;
+	unsigned long		rx_bytes;
+	unsigned long		rx_dropped;
+} ____cacheline_aligned_in_smp;
+
 struct ifb_private {
-	struct tasklet_struct   ifb_tasklet;
-	int     tasklet_pending;
-	/* mostly debug stats leave in for now */
-	unsigned long   st_task_enter; /* tasklet entered */
-	unsigned long   st_txq_refl_try; /* transmit queue refill attempt */
-	unsigned long   st_rxq_enter; /* receive queue entered */
-	unsigned long   st_rx2tx_tran; /* receive to trasmit transfers */
-	unsigned long   st_rxq_notenter; /*receiveQ not entered, resched */
-	unsigned long   st_rx_frm_egr; /* received from egress path */
-	unsigned long   st_rx_frm_ing; /* received from ingress path */
-	unsigned long   st_rxq_check;
-	unsigned long   st_rxq_rsch;
-	struct sk_buff_head     rq;
-	struct sk_buff_head     tq;
+	struct ifb_private_q	*pq;
 };
 
+/* Number of ifb devices to be set up by this module. */
 static int numifbs = 2;
+module_param(numifbs, int, 0444);
+MODULE_PARM_DESC(numifbs, "Number of ifb devices");
 
-static void ri_tasklet(unsigned long dev);
-static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev);
-static int ifb_open(struct net_device *dev);
-static int ifb_close(struct net_device *dev);
+/* Number of TX queues per ifb */
+static unsigned int numtxqs = 1;
+module_param(numtxqs, uint, 0444);
+MODULE_PARM_DESC(numtxqs, "Number of TX queues per ifb");
 
-static void ri_tasklet(unsigned long dev)
+static void ifb_tasklet(void *priv)
 {
-
-	struct net_device *_dev = (struct net_device *)dev;
-	struct ifb_private *dp = netdev_priv(_dev);
-	struct net_device_stats *stats = &_dev->stats;
-	struct netdev_queue *txq;
+	struct ifb_private_q *pq = priv;
+	struct net_device *dev = pq->dev, *_dev;
+	int num = pq - ((struct ifb_private *)netdev_priv(dev))->pq;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 	struct sk_buff *skb;
-
-	txq = netdev_get_tx_queue(_dev, 0);
-	dp->st_task_enter++;
-	if ((skb = skb_peek(&dp->tq)) == NULL) {
-		dp->st_txq_refl_try++;
-		if (__netif_tx_trylock(txq)) {
-			dp->st_rxq_enter++;
-			while ((skb = skb_dequeue(&dp->rq)) != NULL) {
-				skb_queue_tail(&dp->tq, skb);
-				dp->st_rx2tx_tran++;
-			}
-			__netif_tx_unlock(txq);
-		} else {
-			/* reschedule */
-			dp->st_rxq_notenter++;
-			goto resched;
-		}
-	}
-
-	while ((skb = skb_dequeue(&dp->tq)) != NULL) {
+	struct sk_buff_head tq;
+
+	__skb_queue_head_init(&tq);
+	/* move skb from rq to tq */
+	__netif_tx_lock(txq, smp_processor_id());
+	skb_queue_splice_tail_init(&pq->rq, &tq);
+	if (netif_queue_stopped(dev))
+		netif_wake_queue(dev);
+	__netif_tx_unlock(txq);
+	if (skb_queue_empty(&tq))
+		return;
+
+	/* transfer packets */
+	while ((skb = __skb_dequeue(&tq)) != NULL) {
 		u32 from = G_TC_FROM(skb->tc_verd);
 
 		skb->tc_verd = 0;
 		skb->tc_verd = SET_TC_NCLS(skb->tc_verd);
-		stats->tx_packets++;
-		stats->tx_bytes +=skb->len;
+		txq->tx_packets++;
+		txq->tx_bytes +=skb->len;
 
 		rcu_read_lock();
-		skb->dev = dev_get_by_index_rcu(&init_net, skb->iif);
-		if (!skb->dev) {
-			rcu_read_unlock();
+		_dev = dev_get_by_index_rcu(&init_net, skb->iif);
+		if (_dev != NULL) {
+			skb->dev = _dev;
+			skb->iif = dev->ifindex;
+			if (from & AT_EGRESS) {
+				dev_queue_xmit(skb);
+			} else if (from & AT_INGRESS) {
+				skb_pull(skb, _dev->hard_header_len);
+				netif_rx_ni(skb);
+			} else {
+				BUG();
+			}
+		} else {
 			dev_kfree_skb(skb);
-			stats->tx_dropped++;
-			break;
+			txq->tx_dropped++;
 		}
 		rcu_read_unlock();
-		skb->iif = _dev->ifindex;
-
-		if (from & AT_EGRESS) {
-			dp->st_rx_frm_egr++;
-			dev_queue_xmit(skb);
-		} else if (from & AT_INGRESS) {
-			dp->st_rx_frm_ing++;
-			skb_pull(skb, skb->dev->hard_header_len);
-			netif_rx(skb);
-		} else
-			BUG();
 	}
-
-	if (__netif_tx_trylock(txq)) {
-		dp->st_rxq_check++;
-		if ((skb = skb_peek(&dp->rq)) == NULL) {
-			dp->tasklet_pending = 0;
-			if (netif_queue_stopped(_dev))
-				netif_wake_queue(_dev);
-		} else {
-			dp->st_rxq_rsch++;
-			__netif_tx_unlock(txq);
-			goto resched;
-		}
-		__netif_tx_unlock(txq);
-	} else {
-resched:
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
-	}
-
 }
 
-static const struct net_device_ops ifb_netdev_ops = {
-	.ndo_open	= ifb_open,
-	.ndo_stop	= ifb_close,
-	.ndo_start_xmit	= ifb_xmit,
-	.ndo_validate_addr = eth_validate_addr,
-};
-
-static void ifb_setup(struct net_device *dev)
+struct net_device_stats* ifb_get_stats(struct net_device *dev)
 {
-	/* Initialize the device structure. */
-	dev->destructor = free_netdev;
-	dev->netdev_ops = &ifb_netdev_ops;
+	struct net_device_stats *stats = &dev->stats;
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	struct netdev_queue *txq;
+	int i;
+	unsigned long rx_packets = 0, rx_bytes = 0, rx_dropped = 0;
+	unsigned long tx_packets = 0, tx_bytes = 0, tx_dropped = 0;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		rx_packets += pq[i].rx_packets;
+		rx_bytes += pq[i].rx_bytes;
+		rx_dropped += pq[i].rx_dropped;
+		txq = netdev_get_tx_queue(dev, i);
+		tx_packets += txq->tx_packets;
+		tx_bytes += txq->tx_bytes;
+		tx_dropped += txq->tx_dropped;
+	}
 
-	/* Fill in device structure with ethernet-generic values. */
-	ether_setup(dev);
-	dev->tx_queue_len = TX_Q_LIMIT;
+	stats->rx_packets = rx_packets;
+	stats->rx_bytes = rx_bytes;
+	stats->rx_dropped = rx_dropped;
+	stats->tx_packets = tx_packets;
+	stats->tx_bytes = tx_bytes;
+	stats->tx_dropped = tx_dropped;
 
-	dev->flags |= IFF_NOARP;
-	dev->flags &= ~IFF_MULTICAST;
-	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
-	random_ether_addr(dev->dev_addr);
+	return stats;
 }
 
 static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 {
-	struct ifb_private *dp = netdev_priv(dev);
-	struct net_device_stats *stats = &dev->stats;
 	u32 from = G_TC_FROM(skb->tc_verd);
+	int num = skb_get_queue_mapping(skb);
+	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq + num;
+	struct netdev_queue *txq = netdev_get_tx_queue(dev, num);
 
-	stats->rx_packets++;
-	stats->rx_bytes+=skb->len;
+	pq->rx_packets++;
+	pq->rx_bytes += skb->len;
 
 	if (!(from & (AT_INGRESS|AT_EGRESS)) || !skb->iif) {
 		dev_kfree_skb(skb);
-		stats->rx_dropped++;
+		pq->rx_dropped++;
 		return NETDEV_TX_OK;
 	}
 
-	if (skb_queue_len(&dp->rq) >= dev->tx_queue_len) {
+	txq->trans_start = jiffies;
+	__skb_queue_tail(&pq->rq, skb);
+	if (skb_queue_len(&pq->rq) >= dev->tx_queue_len)
 		netif_stop_queue(dev);
-	}
-
-	dev->trans_start = jiffies;
-	skb_queue_tail(&dp->rq, skb);
-	if (!dp->tasklet_pending) {
-		dp->tasklet_pending = 1;
-		tasklet_schedule(&dp->ifb_tasklet);
-	}
+	if (skb_queue_len(&pq->rq) == 1)
+		tasklet_schedule_on(&pq->task, num % num_online_cpus());
 
 	return NETDEV_TX_OK;
 }
@@ -195,26 +179,217 @@ static netdev_tx_t ifb_xmit(struct sk_buff *skb, struct net_device *dev)
 static int ifb_close(struct net_device *dev)
 {
 	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
 
-	tasklet_kill(&dp->ifb_tasklet);
 	netif_stop_queue(dev);
-	skb_queue_purge(&dp->rq);
-	skb_queue_purge(&dp->tq);
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		tasklet_kill(&pq[i].task);
+		__skb_queue_purge(&pq[i].rq);
+	}
+
 	return 0;
 }
 
 static int ifb_open(struct net_device *dev)
 {
+	netif_start_queue(dev);
+
+	return 0;
+}
+
+static u32 simple_tx_hashrnd;
+
+static inline __be16 pppoe_proto(const struct sk_buff *skb)
+{
+	return *((__be16 *)(skb_mac_header(skb) + ETH_HLEN +
+			sizeof(struct pppoe_hdr)));
+}
+
+static u16 ifb_select_queue(struct net_device *dev, struct sk_buff *skb)
+{
+	u32 addr1, addr2;
+	u32 hash;
+	union {
+		u16 in16[2];
+		u32 in32;
+	} ports;
+	u8 ip_proto;
+	unsigned int pull_len, proto_pull_len;
+	int ihl;
+
+	if ((hash = skb_rx_queue_recorded(skb))) {
+		while (hash >= dev->real_num_tx_queues)
+			hash -= dev->real_num_tx_queues;
+		return hash;
+	}
+
+	/* act_mirred pushed the skb before calling dev_queue_xmit() */
+	proto_pull_len = 0;
+	pull_len = skb_network_header(skb) - skb->data;
+	if (unlikely(skb_pull(skb, pull_len) == NULL)) {
+		pull_len = 0;
+		goto process_other;
+	}
+
+	ihl = 0;
+	switch (skb->protocol) {
+	case __constant_htons(ETH_P_8021Q):
+		if (unlikely(skb_pull(skb, VLAN_HLEN) == NULL))
+			goto process_other;
+		pull_len += VLAN_HLEN;
+		skb->network_header += VLAN_HLEN;
+		proto_pull_len = VLAN_HLEN;
+		switch (vlan_eth_hdr(skb)->h_vlan_encapsulated_proto) {
+		case __constant_htons(ETH_P_IP):
+			goto process_ip;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+		case __constant_htons(ETH_P_IPV6):
+			goto process_ipv6;
+#endif
+		default:
+			goto process_other;
+		}
+		break;
+	case __constant_htons(ETH_P_PPP_SES):
+		if (unlikely(skb_pull(skb, PPPOE_SES_HLEN) == NULL))
+			goto process_other;
+		pull_len += PPPOE_SES_HLEN;
+		skb->network_header += PPPOE_SES_HLEN;
+		proto_pull_len = PPPOE_SES_HLEN;
+		switch (pppoe_proto(skb)) {
+		case __constant_htons(ETH_P_IP):
+			goto process_ip;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+		case __constant_htons(ETH_P_IPV6):
+			goto process_ipv6;
+#endif
+		default:
+			goto process_other;
+		}
+		break;
+	case __constant_htons(ETH_P_IP):
+process_ip:
+		if (unlikely(!pskb_may_pull(skb, sizeof(struct iphdr))))
+			goto process_other;
+		if (!(ip_hdr(skb)->frag_off & htons(IP_MF | IP_OFFSET)))
+			ip_proto = ip_hdr(skb)->protocol;
+		else
+			ip_proto = 0;
+		addr1 = ip_hdr(skb)->saddr;
+		addr2 = ip_hdr(skb)->daddr;
+		ihl = ip_hdrlen(skb);
+		break;
+#if defined(CONFIG_IPV6) || defined(CONFIG_IPV6_MODULE)
+	case __constant_htons(ETH_P_IPV6):
+process_ipv6:
+		if (unlikely(!pskb_may_pull(skb, sizeof(struct ipv6hdr))))
+			goto process_other;
+		addr1 = ipv6_hdr(skb)->saddr.s6_addr32[3];
+		addr2 = ipv6_hdr(skb)->daddr.s6_addr32[3];
+		ihl = ipv6_skip_exthdr(skb, sizeof(struct ipv6hdr), &ip_proto);
+		if (unlikely(ihl < 0))
+			goto process_other_trans;
+		break;
+#endif
+	default:
+process_other:
+		if (pull_len != 0) {
+			skb_push(skb, pull_len);
+			if (proto_pull_len != 0)
+				skb->network_header -= proto_pull_len;
+		}
+		return skb->protocol % dev->real_num_tx_queues;
+	}
+	if (addr1 > addr2)
+		swap(addr1, addr2);
+
+	switch (ip_proto) {
+	case IPPROTO_TCP:
+	case IPPROTO_UDP:
+	case IPPROTO_DCCP:
+	case IPPROTO_ESP:
+	case IPPROTO_AH:
+	case IPPROTO_SCTP:
+	case IPPROTO_UDPLITE:
+		if (unlikely(skb_copy_bits(skb, ihl, &ports.in32, 4) < 0))
+			goto process_other_trans;
+		if (ports.in16[0] > ports.in16[1])
+			swap(ports.in16[0], ports.in16[1]);
+		break;
+	default:
+process_other_trans:
+		ports.in32 = 0;
+		break;
+	}
+
+	if (pull_len != 0) {
+		skb_push(skb, pull_len);
+		if (proto_pull_len != 0)
+			skb->network_header -= proto_pull_len;
+	}
+
+	hash = jhash_3words(addr1, addr2, ports.in32,
+			    simple_tx_hashrnd ^ ip_proto);
+
+	return (u16) (((u64) hash * dev->real_num_tx_queues) >> 32);
+}
+
+static int ifb_init(struct net_device *dev)
+{
 	struct ifb_private *dp = netdev_priv(dev);
+	struct ifb_private_q *pq = dp->pq;
+	int i;
 
-	tasklet_init(&dp->ifb_tasklet, ri_tasklet, (unsigned long)dev);
-	skb_queue_head_init(&dp->rq);
-	skb_queue_head_init(&dp->tq);
-	netif_start_queue(dev);
+	pq = kcalloc(dev->real_num_tx_queues, sizeof(*pq), GFP_KERNEL);
+	if (pq == NULL)
+		return -ENOMEM;
+	dp->pq = pq;
+
+	for (i = 0; i < dev->real_num_tx_queues; i++) {
+		pq[i].dev = dev;
+		__skb_queue_head_init(&pq[i].rq);
+		tasklet_init(&pq[i].task, (void(*)(unsigned long))ifb_tasklet,
+			     (unsigned long)&pq[i]);
+	}
 
 	return 0;
 }
 
+static void ifb_uninit(struct net_device *dev)
+{
+	struct ifb_private *dp = netdev_priv(dev);
+
+	kfree(dp->pq);
+}
+
+static const struct net_device_ops ifb_netdev_ops = {
+	.ndo_init		= ifb_init,
+	.ndo_uninit		= ifb_uninit,
+	.ndo_open		= ifb_open,
+	.ndo_stop		= ifb_close,
+	.ndo_start_xmit		= ifb_xmit,
+	.ndo_validate_addr	= eth_validate_addr,
+	.ndo_select_queue	= ifb_select_queue,
+	.ndo_get_stats		= ifb_get_stats,
+};
+
+static void ifb_setup(struct net_device *dev)
+{
+	/* Initialize the device structure. */
+	dev->destructor = free_netdev;
+	dev->netdev_ops = &ifb_netdev_ops;
+
+	/* Fill in device structure with ethernet-generic values. */
+	ether_setup(dev);
+	dev->tx_queue_len = TX_Q_LIMIT;
+
+	dev->flags |= IFF_NOARP;
+	dev->flags &= ~IFF_MULTICAST;
+	dev->priv_flags &= ~IFF_XMIT_DST_RELEASE;
+	random_ether_addr(dev->dev_addr);
+}
+
 static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 {
 	if (tb[IFLA_ADDRESS]) {
@@ -226,25 +401,38 @@ static int ifb_validate(struct nlattr *tb[], struct nlattr *data[])
 	return 0;
 }
 
+static int ifb_get_tx_queues(struct net *net, struct nlattr *tb[],
+			     unsigned int *num_tx_queues,
+			     unsigned int *real_num_tx_queues)
+{
+	if (tb[IFLA_NTXQ]) {
+		*num_tx_queues = nla_get_u32(tb[IFLA_NTXQ]);
+		if (*num_tx_queues < 1)
+			return -EINVAL;
+		*real_num_tx_queues = *num_tx_queues;
+	} else {
+		*num_tx_queues = numtxqs;
+		*real_num_tx_queues = numtxqs;
+	}
+
+	return 0;
+}
+
 static struct rtnl_link_ops ifb_link_ops __read_mostly = {
 	.kind		= "ifb",
-	.priv_size	= sizeof(struct ifb_private),
 	.setup		= ifb_setup,
 	.validate	= ifb_validate,
+	.get_tx_queues	= ifb_get_tx_queues,
+	.priv_size	= sizeof(struct ifb_private),
 };
 
-/* Number of ifb devices to be set up by this module. */
-module_param(numifbs, int, 0);
-MODULE_PARM_DESC(numifbs, "Number of ifb devices");
-
 static int __init ifb_init_one(int index)
 {
 	struct net_device *dev_ifb;
 	int err;
 
-	dev_ifb = alloc_netdev(sizeof(struct ifb_private),
-				 "ifb%d", ifb_setup);
-
+	dev_ifb = alloc_netdev_mq(sizeof(struct ifb_private), "ifb%d",
+				  ifb_setup, numtxqs);
 	if (!dev_ifb)
 		return -ENOMEM;
 
@@ -268,9 +456,12 @@ static int __init ifb_init_module(void)
 {
 	int i, err;
 
+	if (numtxqs < 1)
+		return -EINVAL;
+
+	get_random_bytes(&simple_tx_hashrnd, 4);
 	rtnl_lock();
 	err = __rtnl_link_register(&ifb_link_ops);
-
 	for (i = 0; i < numifbs && !err; i++)
 		err = ifb_init_one(i);
 	if (err)
diff --git a/include/linux/if_link.h b/include/linux/if_link.h
index 1d3b242..99da411 100644
--- a/include/linux/if_link.h
+++ b/include/linux/if_link.h
@@ -78,6 +78,7 @@ enum {
 #define IFLA_LINKINFO IFLA_LINKINFO
 	IFLA_NET_NS_PID,
 	IFLA_IFALIAS,
+	IFLA_NTXQ,
 	__IFLA_MAX
 };
 
diff --git a/include/linux/interrupt.h b/include/linux/interrupt.h
index 7ca72b7..0507c33 100644
--- a/include/linux/interrupt.h
+++ b/include/linux/interrupt.h
@@ -469,6 +469,14 @@ static inline void tasklet_schedule(struct tasklet_struct *t)
 		__tasklet_schedule(t);
 }
 
+static inline void tasklet_schedule_on(struct tasklet_struct *t, int cpu)
+{
+	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state) &&
+	    unlikely(smp_call_function_single(cpu,
+				(void(*)(void*))__tasklet_schedule, t, 0)))
+		__tasklet_schedule(t);
+}
+
 extern void __tasklet_hi_schedule(struct tasklet_struct *t);
 
 static inline void tasklet_hi_schedule(struct tasklet_struct *t)
@@ -477,6 +485,14 @@ static inline void tasklet_hi_schedule(struct tasklet_struct *t)
 		__tasklet_hi_schedule(t);
 }
 
+static inline void tasklet_hi_schedule_on(struct tasklet_struct *t, int cpu)
+{
+	if (!test_and_set_bit(TASKLET_STATE_SCHED, &t->state) &&
+	    unlikely(smp_call_function_single(cpu,
+				(void(*)(void*))__tasklet_hi_schedule, t, 0)))
+		__tasklet_hi_schedule(t);
+}
+
 extern void __tasklet_hi_schedule_first(struct tasklet_struct *t);
 
 /*
diff --git a/net/core/rtnetlink.c b/net/core/rtnetlink.c
index 33148a5..1cbe555 100644
--- a/net/core/rtnetlink.c
+++ b/net/core/rtnetlink.c
@@ -597,6 +597,7 @@ static inline size_t if_nlmsg_size(const struct net_device *dev)
 	       + nla_total_size(4) /* IFLA_MASTER */
 	       + nla_total_size(1) /* IFLA_OPERSTATE */
 	       + nla_total_size(1) /* IFLA_LINKMODE */
+	       + nla_total_size(4) /* IFLA_NTXQ */
 	       + rtnl_link_get_size(dev); /* IFLA_LINKINFO */
 }
 
@@ -627,6 +628,7 @@ static int rtnl_fill_ifinfo(struct sk_buff *skb, struct net_device *dev,
 		   netif_running(dev) ? dev->operstate : IF_OPER_DOWN);
 	NLA_PUT_U8(skb, IFLA_LINKMODE, dev->link_mode);
 	NLA_PUT_U32(skb, IFLA_MTU, dev->mtu);
+	NLA_PUT_U32(skb, IFLA_NTXQ, dev->real_num_tx_queues);
 
 	if (dev->ifindex != dev->iflink)
 		NLA_PUT_U32(skb, IFLA_LINK, dev->iflink);
@@ -725,6 +727,7 @@ const struct nla_policy ifla_policy[IFLA_MAX+1] = {
 	[IFLA_LINKINFO]		= { .type = NLA_NESTED },
 	[IFLA_NET_NS_PID]	= { .type = NLA_U32 },
 	[IFLA_IFALIAS]	        = { .type = NLA_STRING, .len = IFALIASZ-1 },
+	[IFLA_NTXQ]		= { .type = NLA_U32 },
 };
 EXPORT_SYMBOL(ifla_policy);
 



From mboxrd@z Thu Jan  1 00:00:00 1970
From: Eric Dumazet <eric.dumazet@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Mon, 16 Nov 2009 09:19:21 +0100
Message-ID: <4B010B09.9020101@gmail.com>
References: <4B00FFD8.4040508@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: xiaosuo@gmail.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from gw1.cosmosbay.com ([212.99.114.194]:46598 "EHLO
	gw1.cosmosbay.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1750779AbZKPIT1 (ORCPT
	<rfc822;netdev@vger.kernel.org>); Mon, 16 Nov 2009 03:19:27 -0500
In-Reply-To: <4B00FFD8.4040508@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

Changli Gao a =A8=A6crit :
> ifb: add multi-queue support.
>=20
> Add multi-queue support, through more than one tasklets. One tasklet =
per
> queue always steers on the same CPU, and if the number of the
> tasklets(queues) is lager than the number of CPUs, more than one
> tasklets will be assigned to the same CPU.
>=20
> Signed-off-by: Changli Gao <xiaosuo@gmail.com>
> ----

>=20

I must say I fail to see how multiple tasklets per cpu can be of any us=
e.

And :

+	if (skb_queue_len(&pq->rq) =3D=3D 1)
+		tasklet_schedule_on(&pq->task, num % num_online_cpus());

Is probably not what you want, because :

A) num_online_cpus() can be expensive to compute,
B) and you can have such configs :

Three online cpus, CPU0 and CPU4, CPU5
-> you call tasklet_schedule_on(... , num =3D {0|1|2})

To avoid packets reorder, if your hash function selects an offline cpu,
you must forward the packet to a particular cpu, regardless of cpu curr=
ently running.
(even if real device is not multiqueue, its RX interrupts can be handle=
d by any online cpu)

You maybe need to maintain a mapping table with cpu hotplug notifiers.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Mon, 16 Nov 2009 16:43:46 +0800
Message-ID: <412e6f7f0911160043n3a84f79ayd382135bea4c5d2d@mail.gmail.com>
References: <4B00FFD8.4040508@gmail.com> <4B010B09.9020101@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=GB2312
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: "David S. Miller" <davem@davemloft.net>,
	Stephen Hemminger <shemminger@vyatta.com>,
	Patrick McHardy <kaber@trash.net>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: Eric Dumazet <eric.dumazet@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pz0-f171.google.com ([209.85.222.171]:36595 "EHLO
	mail-pz0-f171.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751885AbZKPInk convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Mon, 16 Nov 2009 03:43:40 -0500
Received: by pzk1 with SMTP id 1so2639026pzk.33
        for <netdev@vger.kernel.org>; Mon, 16 Nov 2009 00:43:46 -0800 (PST)
In-Reply-To: <4B010B09.9020101@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

2009/11/16 Eric Dumazet <eric.dumazet@gmail.com>:
> Changli Gao a =A8=A6crit :
>> ifb: add multi-queue support.
>>
>> Add multi-queue support, through more than one tasklets. One tasklet=
 per
>> queue always steers on the same CPU, and if the number of the
>> tasklets(queues) is lager than the number of CPUs, more than one
>> tasklets will be assigned to the same CPU.
>>
>> Signed-off-by: Changli Gao <xiaosuo@gmail.com>
>> ----
>
>>
>
> I must say I fail to see how multiple tasklets per cpu can be of any =
use.
>
> And :
>
> +       if (skb_queue_len(&pq->rq) =3D=3D 1)
> +               tasklet_schedule_on(&pq->task, num % num_online_cpus(=
));
>
> Is probably not what you want, because :
>
> A) num_online_cpus() can be expensive to compute,
> B) and you can have such configs :
>
> Three online cpus, CPU0 and CPU4, CPU5
> -> you call tasklet_schedule_on(... , num =3D {0|1|2})
>
> To avoid packets reorder, if your hash function selects an offline cp=
u,
> you must forward the packet to a particular cpu, regardless of cpu cu=
rrently running.
> (even if real device is not multiqueue, its RX interrupts can be hand=
led by any online cpu)
>
> You maybe need to maintain a mapping table with cpu hotplug notifiers=
=2E
>

Yea, a mapping table is needed in any way. But I don't find a suitable
user interface to do that. And I think many people maybe interested in
the two new APIs: tasklet_schedule_on() and tasklet_hi_schedule_on().
Any comment?

--=20
Regards=A3=AC
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Stephen Hemminger <shemminger@vyatta.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Mon, 16 Nov 2009 08:39:05 -0800
Message-ID: <20091116083905.634b8f56@nehalam>
References: <4AFA8911.7050204@gmail.com>
	<4AFBD911.6000900@gmail.com>
	<4AFCE273.7010901@gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=US-ASCII
Content-Transfer-Encoding: 7bit
Cc: "David S. Miller" <davem@davemloft.net>,
	Patrick McHardy <kaber@trash.net>,
	Eric Dumazet <eric.dumazet@gmail.com>,
	Tom Herbert <therbert@google.com>, netdev@vger.kernel.org
To: xiaosuo@gmail.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail.vyatta.com ([76.74.103.46]:57885 "EHLO mail.vyatta.com"
	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
	id S1752620AbZKPQjZ (ORCPT <rfc822;netdev@vger.kernel.org>);
	Mon, 16 Nov 2009 11:39:25 -0500
In-Reply-To: <4AFCE273.7010901@gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Fri, 13 Nov 2009 12:37:07 +0800
Changli Gao <xiaosuo@gmail.com> wrote:

> ifb: add multi-queue support
> 
> Add multi-queue support, and one kernel thread is created for per queue.
> It can used to emulate multi-queue NIC in software, and distribute work
> among CPUs.

My $.02 is that receive packet steering RPS should be done generically at
receive layer. Then all the CPU, mapping and configuration issues can be
done once, not just for IFB, Bridge, VLAN, ... The number of users of IFB
is small, and setup is complex. Steering packets in IFB is optimizing only
a rarely used corner.

Layered link services like IFB need to be multi-threaded lockless to maintain
the advantages of multi-queue and RPS.

-- 

From mboxrd@z Thu Jan  1 00:00:00 1970
From: David Miller <davem@davemloft.net>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Mon, 16 Nov 2009 19:10:54 -0800 (PST)
Message-ID: <20091116.191054.131722163.davem@davemloft.net>
References: <4AFBD911.6000900@gmail.com>
	<4AFCE273.7010901@gmail.com>
	<20091116083905.634b8f56@nehalam>
Mime-Version: 1.0
Content-Type: Text/Plain; charset=us-ascii
Content-Transfer-Encoding: 7bit
Cc: xiaosuo@gmail.com, kaber@trash.net, eric.dumazet@gmail.com,
	therbert@google.com, netdev@vger.kernel.org
To: shemminger@vyatta.com
Return-path: <netdev-owner@vger.kernel.org>
Received: from 74-93-104-97-Washington.hfc.comcastbusiness.net ([74.93.104.97]:53684
	"EHLO sunset.davemloft.net" rhost-flags-OK-OK-OK-OK)
	by vger.kernel.org with ESMTP id S1752085AbZKQDKi (ORCPT
	<rfc822;netdev@vger.kernel.org>); Mon, 16 Nov 2009 22:10:38 -0500
In-Reply-To: <20091116083905.634b8f56@nehalam>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

From: Stephen Hemminger <shemminger@vyatta.com>
Date: Mon, 16 Nov 2009 08:39:05 -0800

> My $.02 is that receive packet steering RPS should be done generically at
> receive layer. Then all the CPU, mapping and configuration issues can be
> done once, not just for IFB, Bridge, VLAN, ... The number of users of IFB
> is small, and setup is complex. Steering packets in IFB is optimizing only
> a rarely used corner.
> 
> Layered link services like IFB need to be multi-threaded lockless to maintain
> the advantages of multi-queue and RPS.

You're probably right.

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Changli Gao <xiaosuo@gmail.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Tue, 17 Nov 2009 13:38:29 +0800
Message-ID: <412e6f7f0911162138k6537afd1l5f8eeeaa6c636289@mail.gmail.com>
References: <4AFBD911.6000900@gmail.com> <4AFCE273.7010901@gmail.com>
	 <20091116083905.634b8f56@nehalam>
	 <20091116.191054.131722163.davem@davemloft.net>
Mime-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: QUOTED-PRINTABLE
Cc: shemminger@vyatta.com, kaber@trash.net, eric.dumazet@gmail.com,
	therbert@google.com, netdev@vger.kernel.org
To: David Miller <davem@davemloft.net>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail-pw0-f42.google.com ([209.85.160.42]:33552 "EHLO
	mail-pw0-f42.google.com" rhost-flags-OK-OK-OK-OK) by vger.kernel.org
	with ESMTP id S1751740AbZKQFiY convert rfc822-to-8bit (ORCPT
	<rfc822;netdev@vger.kernel.org>); Tue, 17 Nov 2009 00:38:24 -0500
Received: by pwi3 with SMTP id 3so3750544pwi.21
        for <netdev@vger.kernel.org>; Mon, 16 Nov 2009 21:38:30 -0800 (PST)
In-Reply-To: <20091116.191054.131722163.davem@davemloft.net>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, Nov 17, 2009 at 11:10 AM, David Miller <davem@davemloft.net> wr=
ote:
> From: Stephen Hemminger <shemminger@vyatta.com>
> Date: Mon, 16 Nov 2009 08:39:05 -0800
>
>> My $.02 is that receive packet steering RPS should be done generical=
ly at
>> receive layer. Then all the CPU, mapping and configuration issues ca=
n be
>> done once, not just for IFB, Bridge, VLAN, ... The number of users o=
f IFB
>> is small, and setup is complex. Steering packets in IFB is optimizin=
g only
>> a rarely used corner.
>>
>> Layered link services like IFB need to be multi-threaded lockless to=
 maintain
>> the advantages of multi-queue and RPS.
>
> You're probably right.
>

So I have to wait RPS, and after it gets merged, I'll back with
multi-threaded lockless IFB.


--=20
Regards=EF=BC=8C
Changli Gao(xiaosuo@gmail.com)

From mboxrd@z Thu Jan  1 00:00:00 1970
From: Stephen Hemminger <shemminger@vyatta.com>
Subject: Re: [PATCH] ifb: add multi-queue support
Date: Mon, 16 Nov 2009 22:02:47 -0800
Message-ID: <20091116220247.564fb0b6@nehalam>
References: <4AFBD911.6000900@gmail.com>
	<4AFCE273.7010901@gmail.com>
	<20091116083905.634b8f56@nehalam>
	<20091116.191054.131722163.davem@davemloft.net>
	<412e6f7f0911162138k6537afd1l5f8eeeaa6c636289@mail.gmail.com>
Mime-Version: 1.0
Content-Type: text/plain; charset=US-ASCII
Content-Transfer-Encoding: 7bit
Cc: David Miller <davem@davemloft.net>, kaber@trash.net,
	eric.dumazet@gmail.com, therbert@google.com, netdev@vger.kernel.org
To: Changli Gao <xiaosuo@gmail.com>
Return-path: <netdev-owner@vger.kernel.org>
Received: from mail.vyatta.com ([76.74.103.46]:38178 "EHLO mail.vyatta.com"
	rhost-flags-OK-OK-OK-OK) by vger.kernel.org with ESMTP
	id S1753227AbZKQGC5 (ORCPT <rfc822;netdev@vger.kernel.org>);
	Tue, 17 Nov 2009 01:02:57 -0500
In-Reply-To: <412e6f7f0911162138k6537afd1l5f8eeeaa6c636289@mail.gmail.com>
Sender: netdev-owner@vger.kernel.org
List-ID: <netdev.vger.kernel.org>

On Tue, 17 Nov 2009 13:38:29 +0800
Changli Gao <xiaosuo@gmail.com> wrote:

> On Tue, Nov 17, 2009 at 11:10 AM, David Miller <davem@davemloft.net> wrote:
> > From: Stephen Hemminger <shemminger@vyatta.com>
> > Date: Mon, 16 Nov 2009 08:39:05 -0800
> >
> >> My $.02 is that receive packet steering RPS should be done generically at
> >> receive layer. Then all the CPU, mapping and configuration issues can be
> >> done once, not just for IFB, Bridge, VLAN, ... The number of users of IFB
> >> is small, and setup is complex. Steering packets in IFB is optimizing only
> >> a rarely used corner.
> >>
> >> Layered link services like IFB need to be multi-threaded lockless to maintain
> >> the advantages of multi-queue and RPS.
> >
> > You're probably right.
> >
> 
> So I have to wait RPS, and after it gets merged, I'll back with
> multi-threaded lockless IFB.

Please don't wait... Help, test it make sure it works.

