Subject: Re: [PATCH 6/13] bridge: Add core IGMP snooping support

From: Arnd Bergmann <[email-masked]>
Date: Thu, 11 Mar 2010 19:49:52 +0100

Following up on the earlier discussion,

On Monday 08 March 2010, Arnd Bergmann wrote:
> > Arnd, would it be reasonable to extend your RCU-sparse changes to have
> > four different pointer namespaces, one for each flavor of RCU?  (RCU,
> > RCU-bh, RCU-sched, and SRCU)?  Always a fan of making the computer do
> > the auditing where reasonable.  ;-)
> 
> Yes, I guess that would be possible. I'd still leave out the rculist
> from any annotations for now, as this would get even more complex then.
> 
> One consequence will be the need for new rcu_assign_pointer{,_bh,_sched}
> macros that check the address space of the first argument, otherwise
> you'd be able to stick anything in there, including non-__rcu pointers.

I've tested this out now, see the patch below. I needed to add a number
of interfaces, but it still seems ok. Doing it for all the rculist
functions most likely would be less so.

This is currently the head of my rcu-annotate branch of playground.git.
Paul, before I split it up and merge this with the per-subsystem patches,
can you tell me if this is what you had in mind?

> > This could potentially catch the mismatched call_rcu()s, at least if the
> > rcu_head could be labeled.
> ...
> #define rcu_exchange_call(ptr, new, member, func) \
> ({ \
>         typeof(new) old = rcu_exchange((ptr),(new)); \
>         if (old) \
>                 call_rcu(&(old)->member, (func));       \
>         old; \
> })
 
Unfortunately, this did not work out at all. Almost every user follows
a slightly different pattern for call_rcu, so I did not find a way
to match the call_rcu calls with the pointers. In particular, the functions
calling call_rcu() sometimes no longer have access to the 'old' data,
e.g. in case of synchronize_rcu.

My current take is that static annotations won't help us here.

	Arnd

---

rcu: split up __rcu annotations
    
This adds separate name spaces for the four distinct types of RCU
that we use in the kernel, namely __rcu, __rcu_bh, __rcu_sched and
__srcu.
    
Signed-off-by: Arnd Bergmann <[email-masked]>

---
 arch/x86/kvm/mmu.c         |    6 ++--
 arch/x86/kvm/vmx.c         |    2 +-
 drivers/net/macvlan.c      |    8 ++--
 include/linux/compiler.h   |    6 ++++
 include/linux/kvm_host.h   |    4 +-
 include/linux/netdevice.h  |    2 +-
 include/linux/rcupdate.h   |   68 +++++++++++++++++++++++++++++++++----------
 include/linux/srcu.h       |    5 ++-
 include/net/dst.h          |    4 +-
 include/net/llc.h          |    3 +-
 include/net/sock.h         |    2 +-
 include/trace/events/kvm.h |    4 +-
 kernel/cgroup.c            |   10 +++---
 kernel/perf_event.c        |    8 ++--
 kernel/sched.c             |    6 ++--
 kernel/sched_fair.c        |    2 +-
 lib/radix-tree.c           |    8 ++--
 net/core/filter.c          |    4 +-
 net/core/sock.c            |    6 ++--
 net/decnet/dn_route.c      |    2 +-
 net/ipv4/route.c           |   60 +++++++++++++++++++-------------------
 net/ipv4/tcp.c             |    4 +-
 net/llc/llc_core.c         |    6 ++--
 net/llc/llc_input.c        |    2 +-
 virt/kvm/iommu.c           |    4 +-
 virt/kvm/kvm_main.c        |   56 +++++++++++++++++++-----------------
 26 files changed, 171 insertions(+), 121 deletions(-)

diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 741373e..45877ca 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -793,7 +793,7 @@ static int kvm_handle_hva(struct kvm *kvm, unsigned long hva,
 	int retval = 0;
 	struct kvm_memslots *slots;
 
-	slots = rcu_dereference(kvm->memslots);
+	slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 
 	for (i = 0; i < slots->nmemslots; i++) {
 		struct kvm_memory_slot *memslot = &slots->memslots[i];
@@ -3007,7 +3007,7 @@ unsigned int kvm_mmu_calculate_mmu_pages(struct kvm *kvm)
 	unsigned int  nr_pages = 0;
 	struct kvm_memslots *slots;
 
-	slots = rcu_dereference(kvm->memslots);
+	slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 	for (i = 0; i < slots->nmemslots; i++)
 		nr_pages += slots->memslots[i].npages;
 
@@ -3282,7 +3282,7 @@ static int count_rmaps(struct kvm_vcpu *vcpu)
 	int i, j, k, idx;
 
 	idx = srcu_read_lock(&kvm->srcu);
-	slots = rcu_dereference(kvm->memslots);
+	slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
 		struct kvm_memory_slot *m = &slots->memslots[i];
 		struct kvm_rmap_desc *d;
diff --git a/arch/x86/kvm/vmx.c b/arch/x86/kvm/vmx.c
index 0aec1f3..d0c82ed 100644
--- a/arch/x86/kvm/vmx.c
+++ b/arch/x86/kvm/vmx.c
@@ -1513,7 +1513,7 @@ static gva_t rmode_tss_base(struct kvm *kvm)
 		struct kvm_memslots *slots;
 		gfn_t base_gfn;
 
-		slots = rcu_dereference(kvm->memslots);
+		slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 		base_gfn = slots->memslots[0].base_gfn +
 				 slots->memslots[0].npages - 3;
 		return base_gfn << PAGE_SHIFT;
diff --git a/drivers/net/macvlan.c b/drivers/net/macvlan.c
index 95e1bcc..b958d5a 100644
--- a/drivers/net/macvlan.c
+++ b/drivers/net/macvlan.c
@@ -531,15 +531,15 @@ static int macvlan_port_create(struct net_device *dev)
 	INIT_LIST_HEAD(&port->vlans);
 	for (i = 0; i < MACVLAN_HASH_SIZE; i++)
 		INIT_HLIST_HEAD(&port->vlan_hash[i]);
-	rcu_assign_pointer(dev->macvlan_port, port);
+	rcu_assign_pointer_bh(dev->macvlan_port, port);
 	return 0;
 }
 
 static void macvlan_port_destroy(struct net_device *dev)
 {
-	struct macvlan_port *port = rcu_dereference_const(dev->macvlan_port);
+	struct macvlan_port *port = rcu_dereference_bh_const(dev->macvlan_port);
 
-	rcu_assign_pointer(dev->macvlan_port, NULL);
+	rcu_assign_pointer_bh(dev->macvlan_port, NULL);
 	synchronize_rcu();
 	kfree(port);
 }
@@ -624,7 +624,7 @@ int macvlan_common_newlink(struct net *src_net, struct net_device *dev,
 		if (err < 0)
 			return err;
 	}
-	port = rcu_dereference(lowerdev->macvlan_port);
+	port = rcu_dereference_bh(lowerdev->macvlan_port);
 
 	vlan->lowerdev = lowerdev;
 	vlan->dev      = dev;
diff --git a/include/linux/compiler.h b/include/linux/compiler.h
index 0ab21c2..d5756d4 100644
--- a/include/linux/compiler.h
+++ b/include/linux/compiler.h
@@ -17,6 +17,9 @@
 # define __cond_lock(x,c)	((c) ? ({ __acquire(x); 1; }) : 0)
 # define __percpu	__attribute__((noderef, address_space(3)))
 # define __rcu		__attribute__((noderef, address_space(4)))
+# define __rcu_bh	__attribute__((noderef, address_space(5)))
+# define __rcu_sched	__attribute__((noderef, address_space(6)))
+# define __srcu		__attribute__((noderef, address_space(7)))
 extern void __chk_user_ptr(const volatile void __user *);
 extern void __chk_io_ptr(const volatile void __iomem *);
 #else
@@ -36,6 +39,9 @@ extern void __chk_io_ptr(const volatile void __iomem *);
 # define __cond_lock(x,c) (c)
 # define __percpu
 # define __rcu
+# define __rcu_bh
+# define __rcu_sched
+# define __srcu
 #endif
 
 #ifdef __KERNEL__
diff --git a/include/linux/kvm_host.h b/include/linux/kvm_host.h
index 9eb0f9c..bad1787 100644
--- a/include/linux/kvm_host.h
+++ b/include/linux/kvm_host.h
@@ -164,7 +164,7 @@ struct kvm {
 	raw_spinlock_t requests_lock;
 	struct mutex slots_lock;
 	struct mm_struct *mm; /* userspace tied to this vm */
-	struct kvm_memslots __rcu *memslots;
+	struct kvm_memslots __srcu *memslots;
 	struct srcu_struct srcu;
 #ifdef CONFIG_KVM_APIC_ARCHITECTURE
 	u32 bsp_vcpu_id;
@@ -174,7 +174,7 @@ struct kvm {
 	atomic_t online_vcpus;
 	struct list_head vm_list;
 	struct mutex lock;
-	struct kvm_io_bus __rcu *buses[KVM_NR_BUSES];
+	struct kvm_io_bus __srcu *buses[KVM_NR_BUSES];
 #ifdef CONFIG_HAVE_KVM_EVENTFD
 	struct {
 		spinlock_t        lock;
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h
index fd7e8de..1b72188 100644
--- a/include/linux/netdevice.h
+++ b/include/linux/netdevice.h
@@ -949,7 +949,7 @@ struct net_device {
 	/* bridge stuff */
 	void __rcu		*br_port;
 	/* macvlan */
-	struct macvlan_port __rcu *macvlan_port;
+	struct macvlan_port __rcu_bh *macvlan_port;
 	/* GARP */
 	struct garp_port __rcu	*garp_port;
 
diff --git a/include/linux/rcupdate.h b/include/linux/rcupdate.h
index 03702cc..b4c6f39 100644
--- a/include/linux/rcupdate.h
+++ b/include/linux/rcupdate.h
@@ -183,19 +183,33 @@ static inline int rcu_read_lock_sched_held(void)
  * read-side critical section.  It is also possible to check for
  * locks being held, for example, by using lockdep_is_held().
  */
-#define rcu_dereference_check(p, c) \
+#define __rcu_dereference_check(p, c, space) \
 	({ \
 		if (debug_locks && !(c)) \
 			lockdep_rcu_dereference(__FILE__, __LINE__); \
-		rcu_dereference_raw(p); \
+		__rcu_dereference_raw(p, space); \
 	})
 
+
 #else /* #ifdef CONFIG_PROVE_RCU */
 
-#define rcu_dereference_check(p, c)	rcu_dereference_raw(p)
+#define __rcu_dereference_check(p, c, space)	\
+	__rcu_dereference_raw(p, space)
 
 #endif /* #else #ifdef CONFIG_PROVE_RCU */
 
+#define rcu_dereference_check(p, c) \
+	__rcu_dereference_check(p, c, __rcu)
+
+#define rcu_dereference_bh_check(p, c) \
+	__rcu_dereference_check(p, rcu_read_lock_bh_held() || (c), __rcu_bh)
+
+#define rcu_dereference_sched_check(p, c) \
+	__rcu_dereference_check(p, rcu_read_lock_sched_held() || (c), __rcu_sched)
+
+#define srcu_dereference_check(p, c) \
+	__rcu_dereference_check(p, srcu_read_lock_held() || (c), __srcu)
+
 /**
  * rcu_read_lock - mark the beginning of an RCU read-side critical section.
  *
@@ -341,13 +355,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  * exactly which pointers are protected by RCU and checks that
  * the pointer is annotated as __rcu.
  */
-#define rcu_dereference_raw(p)  ({ \
+#define __rcu_dereference_raw(p, space)  ({ \
 				typeof(*p) *_________p1 = (typeof(*p)*__force )ACCESS_ONCE(p); \
-				(void) (((typeof (*p) __rcu *)p) == p); \
+				(void) (((typeof (*p) space *)p) == p); \
 				smp_read_barrier_depends(); \
 				((typeof(*p) __force __kernel *)(_________p1)); \
 				})
 
+#define rcu_dereference_raw(p) __rcu_dereference_raw(p, __rcu)
+
 /**
  * rcu_dereference_const - fetch an __rcu pointer outside of a
  * read-side critical section.
@@ -360,18 +376,22 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  * or in an RCU call.
  */
 
-#define rcu_dereference_const(p)     ({ \
-				(void) (((typeof (*p) __rcu *)p) == p); \
+#define __rcu_dereference_const(p, space)     ({ \
+				(void) (((typeof (*p) space *)p) == p); \
 				((typeof(*p) __force __kernel *)(p)); \
 				})
 
+#define rcu_dereference_const(p)  __rcu_dereference_const(p, __rcu)
+#define rcu_dereference_bh_const(p)  __rcu_dereference_const(p, __rcu_bh)
+#define rcu_dereference_sched_const(p)  __rcu_dereference_const(p, __rcu_sched)
+
 /**
  * rcu_dereference - fetch an RCU-protected pointer, checking for RCU
  *
  * Makes rcu_dereference_check() do the dirty work.
  */
 #define rcu_dereference(p) \
-	rcu_dereference_check(p, rcu_read_lock_held())
+	__rcu_dereference_check(p, rcu_read_lock_held(), __rcu)
 
 /**
  * rcu_dereference_bh - fetch an RCU-protected pointer, checking for RCU-bh
@@ -379,7 +399,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  * Makes rcu_dereference_check() do the dirty work.
  */
 #define rcu_dereference_bh(p) \
-		rcu_dereference_check(p, rcu_read_lock_bh_held())
+	__rcu_dereference_check(p, rcu_read_lock_bh_held(), __rcu_bh)
 
 /**
  * rcu_dereference_sched - fetch RCU-protected pointer, checking for RCU-sched
@@ -387,7 +407,7 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  * Makes rcu_dereference_check() do the dirty work.
  */
 #define rcu_dereference_sched(p) \
-		rcu_dereference_check(p, rcu_read_lock_sched_held())
+	__rcu_dereference_check(p, rcu_read_lock_sched_held(), __rcu_sched)
 
 /**
  * rcu_assign_pointer - assign (publicize) a pointer to a newly
@@ -402,12 +422,12 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  * code.
  */
 
-#define rcu_assign_pointer(p, v) \
+#define __rcu_assign_pointer(p, v, space) \
 	({ \
 		if (!__builtin_constant_p(v) || \
 		    ((v) != NULL)) \
 			smp_wmb(); \
-		(p) = (typeof(*v) __force __rcu *)(v); \
+		(p) = (typeof(*v) __force space *)(v); \
 	})
 
 /**
@@ -415,10 +435,17 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
  * without barriers.
  * Using this is almost always a bug.
  */
-#define __rcu_assign_pointer(p, v) \
-	({ \
-		(p) = (typeof(*v) __force __rcu *)(v); \
-	})
+#define rcu_assign_pointer(p, v) \
+	__rcu_assign_pointer(p, v, __rcu)
+
+#define rcu_assign_pointer_bh(p, v) \
+	__rcu_assign_pointer(p, v, __rcu_bh)
+
+#define rcu_assign_pointer_sched(p, v) \
+	__rcu_assign_pointer(p, v, __rcu_sched)
+
+#define srcu_assign_pointer(p, v) \
+	__rcu_assign_pointer(p, v, __srcu)
 
 /**
  * RCU_INIT_POINTER - initialize an RCU protected member
@@ -427,6 +454,15 @@ static inline notrace void rcu_read_unlock_sched_notrace(void)
 #define RCU_INIT_POINTER(p, v) \
 		p = (typeof(*v) __force __rcu *)(v)
 
+#define RCU_INIT_POINTER_BH(p, v) \
+		p = (typeof(*v) __force __rcu_bh *)(v)
+
+#define RCU_INIT_POINTER_SCHED(p, v) \
+		p = (typeof(*v) __force __rcu_sched *)(v)
+
+#define SRCU_INIT_POINTER(p, v) \
+		p = (typeof(*v) __force __srcu *)(v)
+
 /* Infrastructure to implement the synchronize_() primitives. */
 
 struct rcu_synchronize {
diff --git a/include/linux/srcu.h b/include/linux/srcu.h
index 4d5ecb2..feaf661 100644
--- a/include/linux/srcu.h
+++ b/include/linux/srcu.h
@@ -111,7 +111,10 @@ static inline int srcu_read_lock_held(struct srcu_struct *sp)
  * Makes rcu_dereference_check() do the dirty work.
  */
 #define srcu_dereference(p, sp) \
-		rcu_dereference_check(p, srcu_read_lock_held(sp))
+		__rcu_dereference_check(p, srcu_read_lock_held(sp), __srcu)
+
+#define srcu_dereference_const(p) \
+		__rcu_dereference_const(p, __srcu)
 
 /**
  * srcu_read_lock - register a new reader for an SRCU-protected structure.
diff --git a/include/net/dst.h b/include/net/dst.h
index 5f839aa..bbeaba2 100644
--- a/include/net/dst.h
+++ b/include/net/dst.h
@@ -94,9 +94,9 @@ struct dst_entry {
 	unsigned long		lastuse;
 	union {
 		struct dst_entry *next;
-		struct rtable   __rcu *rt_next;
+		struct rtable   __rcu_bh *rt_next;
 		struct rt6_info   *rt6_next;
-		struct dn_route  *dn_next;
+		struct dn_route  __rcu_bh *dn_next;
 	};
 };
 
diff --git a/include/net/llc.h b/include/net/llc.h
index 8299cb2..5700082 100644
--- a/include/net/llc.h
+++ b/include/net/llc.h
@@ -59,7 +59,8 @@ struct llc_sap {
 	int		 (* rcv_func)(struct sk_buff *skb,
 				     struct net_device *dev,
 				     struct packet_type *pt,
-				     struct net_device *orig_dev) __rcu;
+				     struct net_device *orig_dev)
+							 __rcu_bh;
 	struct llc_addr	 laddr;
 	struct list_head node;
 	spinlock_t sk_lock;
diff --git a/include/net/sock.h b/include/net/sock.h
index e07cd78..66d5e09 100644
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -290,7 +290,7 @@ struct sock {
 	struct ucred		sk_peercred;
 	long			sk_rcvtimeo;
 	long			sk_sndtimeo;
-	struct sk_filter __rcu	*sk_filter;
+	struct sk_filter __rcu_bh *sk_filter;
 	void			*sk_protinfo;
 	struct timer_list	sk_timer;
 	ktime_t			sk_stamp;
diff --git a/kernel/cgroup.c b/kernel/cgroup.c
index fc45694..db3e502 100644
--- a/kernel/cgroup.c
+++ b/kernel/cgroup.c
@@ -1392,7 +1392,7 @@ static int cgroup_get_sb(struct file_system_type *fs_type,
 		root_count++;
 
 		sb->s_root->d_fsdata = root_cgrp;
-		__rcu_assign_pointer(root->top_cgroup.dentry, sb->s_root);
+		rcu_assign_pointer(root->top_cgroup.dentry, sb->s_root);
 
 		/* Link the top cgroup in this hierarchy into all
 		 * the css_set objects */
@@ -3243,7 +3243,7 @@ int __init cgroup_init_early(void)
 	css_set_count = 1;
 	init_cgroup_root(&rootnode);
 	root_count = 1;
-	__rcu_assign_pointer(init_task.cgroups, &init_css_set);
+	rcu_assign_pointer(init_task.cgroups, &init_css_set);
 
 	init_css_set_link.cg = &init_css_set;
 	init_css_set_link.cgrp = dummytop;
@@ -3551,7 +3551,7 @@ void cgroup_exit(struct task_struct *tsk, int run_callbacks)
 	/* Reassign the task to the init_css_set. */
 	task_lock(tsk);
 	cg = rcu_dereference_const(tsk->cgroups);
-	__rcu_assign_pointer(tsk->cgroups, &init_css_set);
+	rcu_assign_pointer(tsk->cgroups, &init_css_set);
 	task_unlock(tsk);
 	if (cg)
 		put_css_set_taskexit(cg);
@@ -3959,8 +3959,8 @@ static int __init cgroup_subsys_init_idr(struct cgroup_subsys *ss)
 		return PTR_ERR(newid);
 
 	newid->stack[0] = newid->id;
-	__rcu_assign_pointer(newid->css, rootcss);
-	__rcu_assign_pointer(rootcss->id, newid);
+	rcu_assign_pointer(newid->css, rootcss);
+	rcu_assign_pointer(rootcss->id, newid);
 	return 0;
 }
 
diff --git a/kernel/perf_event.c b/kernel/perf_event.c
index ac8bcbd..e1b65b2 100644
--- a/kernel/perf_event.c
+++ b/kernel/perf_event.c
@@ -1223,8 +1223,8 @@ void perf_event_task_sched_out(struct task_struct *task,
 			 * XXX do we need a memory barrier of sorts
 			 * wrt to rcu_dereference() of perf_event_ctxp
 			 */
-			__rcu_assign_pointer(task->perf_event_ctxp, next_ctx);
-			__rcu_assign_pointer(next->perf_event_ctxp, ctx);
+			rcu_assign_pointer(task->perf_event_ctxp, next_ctx);
+			rcu_assign_pointer(next->perf_event_ctxp, ctx);
 			ctx->task = next;
 			next_ctx->task = task;
 			do_switch = 0;
@@ -5376,10 +5376,10 @@ int perf_event_init_task(struct task_struct *child)
 		 */
 		cloned_ctx = rcu_dereference(parent_ctx->parent_ctx);
 		if (cloned_ctx) {
-			__rcu_assign_pointer(child_ctx->parent_ctx, cloned_ctx);
+			rcu_assign_pointer(child_ctx->parent_ctx, cloned_ctx);
 			child_ctx->parent_gen = parent_ctx->parent_gen;
 		} else {
-			__rcu_assign_pointer(child_ctx->parent_ctx, parent_ctx);
+			rcu_assign_pointer(child_ctx->parent_ctx, parent_ctx);
 			child_ctx->parent_gen = parent_ctx->generation;
 		}
 		get_ctx(rcu_dereference_const(child_ctx->parent_ctx));
diff --git a/kernel/sched.c b/kernel/sched.c
index 05fd61e..83744d6 100644
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -528,7 +528,7 @@ struct rq {
 
 #ifdef CONFIG_SMP
 	struct root_domain *rd;
-	struct sched_domain __rcu *sd;
+	struct sched_domain __rcu_sched *sd;
 
 	unsigned char idle_at_tick;
 	/* For active balancing */
@@ -603,7 +603,7 @@ static inline int cpu_of(struct rq *rq)
 }
 
 #define rcu_dereference_check_sched_domain(p) \
-	rcu_dereference_check((p), \
+	rcu_dereference_sched_check((p), \
 			      rcu_read_lock_sched_held() || \
 			      lockdep_is_held(&sched_domains_mutex))
 
@@ -6323,7 +6323,7 @@ cpu_attach_domain(struct sched_domain *sd, struct root_domain *rd, int cpu)
 	sched_domain_debug(sd, cpu);
 
 	rq_attach_root(rq, rd);
-	rcu_assign_pointer(rq->sd, sd);
+	rcu_assign_pointer_sched(rq->sd, sd);
 }
 
 /* cpus with isolated domains */
diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index 3e1fd96..5a5ea2c 100644
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -3476,7 +3476,7 @@ static void run_rebalance_domains(struct softirq_action *h)
 
 static inline int on_null_domain(int cpu)
 {
-	return !rcu_dereference(cpu_rq(cpu)->sd);
+	return !rcu_dereference_sched(cpu_rq(cpu)->sd);
 }
 
 /*
diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index f6ae74c..4c6f149 100644
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -264,7 +264,7 @@ static int radix_tree_extend(struct radix_tree_root *root, unsigned long index)
 			return -ENOMEM;
 
 		/* Increase the height.  */
-		__rcu_assign_pointer(node->slots[0],
+		rcu_assign_pointer(node->slots[0],
 			radix_tree_indirect_to_ptr(rcu_dereference_const(root->rnode)));
 
 		/* Propagate the aggregated tag info into the new root */
@@ -1090,7 +1090,7 @@ static inline void radix_tree_shrink(struct radix_tree_root *root)
 		newptr = rcu_dereference_const(to_free->slots[0]);
 		if (root->height > 1)
 			newptr = radix_tree_ptr_to_indirect(newptr);
-		__rcu_assign_pointer(root->rnode, newptr);
+		rcu_assign_pointer(root->rnode, newptr);
 		root->height--;
 		radix_tree_node_free(to_free);
 	}
@@ -1125,7 +1125,7 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
 	slot = rcu_dereference_const(root->rnode);
 	if (height == 0) {
 		root_tag_clear_all(root);
-		__rcu_assign_pointer(root->rnode, NULL);
+		rcu_assign_pointer(root->rnode, NULL);
 		goto out;
 	}
 	slot = radix_tree_indirect_to_ptr(slot);
@@ -1183,7 +1183,7 @@ void *radix_tree_delete(struct radix_tree_root *root, unsigned long index)
 	}
 	root_tag_clear_all(root);
 	root->height = 0;
-	__rcu_assign_pointer(root->rnode, NULL);
+	rcu_assign_pointer(root->rnode, NULL);
 	if (to_free)
 		radix_tree_node_free(to_free);
 
diff --git a/net/core/filter.c b/net/core/filter.c
index d38ef7f..b88675b 100644
--- a/net/core/filter.c
+++ b/net/core/filter.c
@@ -522,7 +522,7 @@ int sk_attach_filter(struct sock_fprog *fprog, struct sock *sk)
 
 	rcu_read_lock_bh();
 	old_fp = rcu_dereference_bh(sk->sk_filter);
-	rcu_assign_pointer(sk->sk_filter, fp);
+	rcu_assign_pointer_bh(sk->sk_filter, fp);
 	rcu_read_unlock_bh();
 
 	if (old_fp)
@@ -539,7 +539,7 @@ int sk_detach_filter(struct sock *sk)
 	rcu_read_lock_bh();
 	filter = rcu_dereference_bh(sk->sk_filter);
 	if (filter) {
-		rcu_assign_pointer(sk->sk_filter, NULL);
+		rcu_assign_pointer_bh(sk->sk_filter, NULL);
 		sk_filter_delayed_uncharge(sk, filter);
 		ret = 0;
 	}
diff --git a/net/core/sock.c b/net/core/sock.c
index 74242e2..8549387 100644
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -1073,11 +1073,11 @@ static void __sk_free(struct sock *sk)
 	if (sk->sk_destruct)
 		sk->sk_destruct(sk);
 
-	filter = rcu_dereference_check(sk->sk_filter,
+	filter = rcu_dereference_bh_check(sk->sk_filter,
 				       atomic_read(&sk->sk_wmem_alloc) == 0);
 	if (filter) {
 		sk_filter_uncharge(sk, filter);
-		rcu_assign_pointer(sk->sk_filter, NULL);
+		rcu_assign_pointer_bh(sk->sk_filter, NULL);
 	}
 
 	sock_disable_timestamp(sk, SOCK_TIMESTAMP);
@@ -1167,7 +1167,7 @@ struct sock *sk_clone(const struct sock *sk, const gfp_t priority)
 		sock_reset_flag(newsk, SOCK_DONE);
 		skb_queue_head_init(&newsk->sk_error_queue);
 
-		filter = rcu_dereference_const(newsk->sk_filter);
+		filter = rcu_dereference_bh_const(newsk->sk_filter);
 		if (filter != NULL)
 			sk_filter_charge(newsk, filter);
 
diff --git a/net/decnet/dn_route.c b/net/decnet/dn_route.c
index a7bf03c..22ec1d1 100644
--- a/net/decnet/dn_route.c
+++ b/net/decnet/dn_route.c
@@ -92,7 +92,7 @@
 
 struct dn_rt_hash_bucket
 {
-	struct dn_route *chain;
+	struct dn_route __rcu_bh *chain;
 	spinlock_t lock;
 };
 
diff --git a/net/ipv4/route.c b/net/ipv4/route.c
index 37bf0d9..99cef80 100644
--- a/net/ipv4/route.c
+++ b/net/ipv4/route.c
@@ -200,7 +200,7 @@ const __u8 ip_tos2prio[16] = {
  */
 
 struct rt_hash_bucket {
-	struct rtable __rcu *chain;
+	struct rtable __rcu_bh *chain;
 };
 
 #if defined(CONFIG_SMP) || defined(CONFIG_DEBUG_SPINLOCK) || \
@@ -731,26 +731,26 @@ static void rt_do_flush(int process_context)
 		spin_lock_bh(rt_hash_lock_addr(i));
 #ifdef CONFIG_NET_NS
 		{
-		struct rtable __rcu ** prev;
+		struct rtable __rcu_bh ** prev;
 		struct rtable * p;
 
-		rth = rcu_dereference_const(rt_hash_table[i].chain);
+		rth = rcu_dereference_bh(rt_hash_table[i].chain);
 
 		/* defer releasing the head of the list after spin_unlock */
-		for (tail = rth; tail; tail = rcu_dereference_const(tail->u.dst.rt_next))
+		for (tail = rth; tail; tail = rcu_dereference_bh(tail->u.dst.rt_next))
 			if (!rt_is_expired(tail))
 				break;
 		if (rth != tail)
-			__rcu_assign_pointer(rt_hash_table[i].chain, tail);
+			rcu_assign_pointer_bh(rt_hash_table[i].chain, tail);
 
 		/* call rt_free on entries after the tail requiring flush */
 		prev = &rt_hash_table[i].chain;
-		for (p = rcu_dereference_const(*prev); p; p = next) {
-			next = rcu_dereference_const(p->u.dst.rt_next);
+		for (p = rcu_dereference_bh(*prev); p; p = next) {
+			next = rcu_dereference_bh(p->u.dst.rt_next);
 			if (!rt_is_expired(p)) {
 				prev = &p->u.dst.rt_next;
 			} else {
-				__rcu_assign_pointer(*prev, next);
+				rcu_assign_pointer_bh(*prev, next);
 				rt_free(p);
 			}
 		}
@@ -763,7 +763,7 @@ static void rt_do_flush(int process_context)
 		spin_unlock_bh(rt_hash_lock_addr(i));
 
 		for (; rth != tail; rth = next) {
-			next = rcu_dereference_const(rth->u.dst.rt_next);
+			next = rcu_dereference_bh(rth->u.dst.rt_next);
 			rt_free(rth);
 		}
 	}
@@ -785,7 +785,7 @@ static void rt_check_expire(void)
 	static unsigned int rover;
 	unsigned int i = rover, goal;
 	struct rtable *rth, *aux;
-	struct rtable __rcu **rthp;
+	struct rtable __rcu_bh **rthp;
 	unsigned long samples = 0;
 	unsigned long sum = 0, sum2 = 0;
 	unsigned long delta;
@@ -815,8 +815,8 @@ static void rt_check_expire(void)
 			continue;
 		length = 0;
 		spin_lock_bh(rt_hash_lock_addr(i));
-		while ((rth = rcu_dereference_const(*rthp)) != NULL) {
-			prefetch(rcu_dereference_const(rth->u.dst.rt_next));
+		while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
+			prefetch(rcu_dereference_bh(rth->u.dst.rt_next));
 			if (rt_is_expired(rth)) {
 				*rthp = rth->u.dst.rt_next;
 				rt_free(rth);
@@ -836,14 +836,14 @@ nofree:
 					 * attributes don't unfairly skew
 					 * the length computation
 					 */
-					for (aux = rcu_dereference_const(rt_hash_table[i].chain);;) {
+					for (aux = rcu_dereference_bh(rt_hash_table[i].chain);;) {
 						if (aux == rth) {
 							length += ONE;
 							break;
 						}
 						if (compare_hash_inputs(&aux->fl, &rth->fl))
 							break;
-						aux = rcu_dereference_const(aux->u.dst.rt_next);
+						aux = rcu_dereference_bh(aux->u.dst.rt_next);
 					}
 					continue;
 				}
@@ -959,7 +959,7 @@ static int rt_garbage_collect(struct dst_ops *ops)
 	static int rover;
 	static int equilibrium;
 	struct rtable *rth;
-	struct rtable __rcu **rthp;
+	struct rtable __rcu_bh **rthp;
 	unsigned long now = jiffies;
 	int goal;
 
@@ -1012,7 +1012,7 @@ static int rt_garbage_collect(struct dst_ops *ops)
 			k = (k + 1) & rt_hash_mask;
 			rthp = &rt_hash_table[k].chain;
 			spin_lock_bh(rt_hash_lock_addr(k));
-			while ((rth = rcu_dereference_const(*rthp)) != NULL) {
+			while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
 				if (!rt_is_expired(rth) &&
 					!rt_may_expire(rth, tmo, expire)) {
 					tmo >>= 1;
@@ -1079,10 +1079,10 @@ static int rt_intern_hash(unsigned hash, struct rtable *rt,
 			  struct rtable **rp, struct sk_buff *skb)
 {
 	struct rtable	*rth;
-	struct rtable __rcu **rthp;
+	struct rtable __rcu_bh **rthp;
 	unsigned long	now;
 	struct rtable *cand;
-	struct rtable __rcu **candp;
+	struct rtable __rcu_bh **candp;
 	u32 		min_score;
 	int		chain_length;
 	int attempts = !in_softirq();
@@ -1129,7 +1129,7 @@ restart:
 	rthp = &rt_hash_table[hash].chain;
 
 	spin_lock_bh(rt_hash_lock_addr(hash));
-	while ((rth = rcu_dereference_const(*rthp)) != NULL) {
+	while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
 		if (rt_is_expired(rth)) {
 			*rthp = rth->u.dst.rt_next;
 			rt_free(rth);
@@ -1143,13 +1143,13 @@ restart:
 			 * must be visible to another weakly ordered CPU before
 			 * the insertion at the start of the hash chain.
 			 */
-			rcu_assign_pointer(rth->u.dst.rt_next,
+			rcu_assign_pointer_bh(rth->u.dst.rt_next,
 					   rt_hash_table[hash].chain);
 			/*
 			 * Since lookup is lockfree, the update writes
 			 * must be ordered for consistency on SMP.
 			 */
-			rcu_assign_pointer(rt_hash_table[hash].chain, rth);
+			rcu_assign_pointer_bh(rt_hash_table[hash].chain, rth);
 
 			dst_use(&rth->u.dst, now);
 			spin_unlock_bh(rt_hash_lock_addr(hash));
@@ -1252,7 +1252,7 @@ restart:
 	 * previous writes to rt are comitted to memory
 	 * before making rt visible to other CPUS.
 	 */
-	rcu_assign_pointer(rt_hash_table[hash].chain, rt);
+	rcu_assign_pointer_bh(rt_hash_table[hash].chain, rt);
 
 	spin_unlock_bh(rt_hash_lock_addr(hash));
 
@@ -1325,13 +1325,13 @@ void __ip_select_ident(struct iphdr *iph, struct dst_entry *dst, int more)
 
 static void rt_del(unsigned hash, struct rtable *rt)
 {
-	struct rtable __rcu **rthp;
+	struct rtable __rcu_bh **rthp;
 	struct rtable *aux;
 
 	rthp = &rt_hash_table[hash].chain;
 	spin_lock_bh(rt_hash_lock_addr(hash));
 	ip_rt_put(rt);
-	while ((aux = rcu_dereference_const(*rthp)) != NULL) {
+	while ((aux = rcu_dereference_bh(*rthp)) != NULL) {
 		if (aux == rt || rt_is_expired(aux)) {
 			*rthp = aux->u.dst.rt_next;
 			rt_free(aux);
@@ -1348,7 +1348,7 @@ void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
 	int i, k;
 	struct in_device *in_dev = in_dev_get(dev);
 	struct rtable *rth;
-	struct rtable __rcu **rthp;
+	struct rtable __rcu_bh **rthp;
 	__be32  skeys[2] = { saddr, 0 };
 	int  ikeys[2] = { dev->ifindex, 0 };
 	struct netevent_redirect netevent;
@@ -1384,7 +1384,7 @@ void ip_rt_redirect(__be32 old_gw, __be32 daddr, __be32 new_gw,
 			rthp=&rt_hash_table[hash].chain;
 
 			rcu_read_lock();
-			while ((rth = rcu_dereference(*rthp)) != NULL) {
+			while ((rth = rcu_dereference_bh(*rthp)) != NULL) {
 				struct rtable *rt;
 
 				if (rth->fl.fl4_dst != daddr ||
@@ -1646,8 +1646,8 @@ unsigned short ip_rt_frag_needed(struct net *net, struct iphdr *iph,
 						rt_genid(net));
 
 			rcu_read_lock();
-			for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
-			     rth = rcu_dereference(rth->u.dst.rt_next)) {
+			for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
+			     rth = rcu_dereference_bh(rth->u.dst.rt_next)) {
 				unsigned short mtu = new_mtu;
 
 				if (rth->fl.fl4_dst != daddr ||
@@ -2287,8 +2287,8 @@ int ip_route_input(struct sk_buff *skb, __be32 daddr, __be32 saddr,
 	hash = rt_hash(daddr, saddr, iif, rt_genid(net));
 
 	rcu_read_lock();
-	for (rth = rcu_dereference(rt_hash_table[hash].chain); rth;
-	     rth = rcu_dereference(rth->u.dst.rt_next)) {
+	for (rth = rcu_dereference_bh(rt_hash_table[hash].chain); rth;
+	     rth = rcu_dereference_bh(rth->u.dst.rt_next)) {
 		if (((rth->fl.fl4_dst ^ daddr) |
 		     (rth->fl.fl4_src ^ saddr) |
 		     (rth->fl.iif ^ iif) |
diff --git a/net/ipv4/tcp.c b/net/ipv4/tcp.c
index d8ce05b..003d54f 100644
--- a/net/ipv4/tcp.c
+++ b/net/ipv4/tcp.c
@@ -3252,9 +3252,9 @@ void __init tcp_init(void)
 	memset(&tcp_secret_two.secrets[0], 0, sizeof(tcp_secret_two.secrets));
 	tcp_secret_one.expires = jiffy; /* past due */
 	tcp_secret_two.expires = jiffy; /* past due */
-	__rcu_assign_pointer(tcp_secret_generating, &tcp_secret_one);
+	rcu_assign_pointer(tcp_secret_generating, &tcp_secret_one);
 	tcp_secret_primary = &tcp_secret_one;
-	__rcu_assign_pointer(tcp_secret_retiring, &tcp_secret_two);
+	rcu_assign_pointer(tcp_secret_retiring, &tcp_secret_two);
 	tcp_secret_secondary = &tcp_secret_two;
 }
 
diff --git a/net/llc/llc_core.c b/net/llc/llc_core.c
index ed7f424..8696677 100644
--- a/net/llc/llc_core.c
+++ b/net/llc/llc_core.c
@@ -50,7 +50,7 @@ static struct llc_sap *__llc_sap_find(unsigned char sap_value)
 {
 	struct llc_sap* sap;
 
-	list_for_each_entry(sap, &llc_sap_list, node)
+	list_for_each_entry_rcu(sap, &llc_sap_list, node)
 		if (sap->laddr.lsap == sap_value)
 			goto out;
 	sap = NULL;
@@ -103,7 +103,7 @@ struct llc_sap *llc_sap_open(unsigned char lsap,
 	if (!sap)
 		goto out;
 	sap->laddr.lsap = lsap;
-	rcu_assign_pointer(sap->rcv_func, func);
+	rcu_assign_pointer_bh(sap->rcv_func, func);
 	list_add_tail_rcu(&sap->node, &llc_sap_list);
 out:
 	spin_unlock_bh(&llc_sap_list_lock);
@@ -127,7 +127,7 @@ void llc_sap_close(struct llc_sap *sap)
 	list_del_rcu(&sap->node);
 	spin_unlock_bh(&llc_sap_list_lock);
 
-	synchronize_rcu();
+	synchronize_rcu_bh();
 
 	kfree(sap);
 }
diff --git a/net/llc/llc_input.c b/net/llc/llc_input.c
index 57ad974..b775530 100644
--- a/net/llc/llc_input.c
+++ b/net/llc/llc_input.c
@@ -179,7 +179,7 @@ int llc_rcv(struct sk_buff *skb, struct net_device *dev,
 	 * First the upper layer protocols that don't need the full
 	 * LLC functionality
 	 */
-	rcv = rcu_dereference(sap->rcv_func);
+	rcv = rcu_dereference_bh(sap->rcv_func);
 	if (rcv) {
 		struct sk_buff *cskb = skb_clone(skb, GFP_ATOMIC);
 		if (cskb)
diff --git a/virt/kvm/iommu.c b/virt/kvm/iommu.c
index 80fd3ad..2ba7048 100644
--- a/virt/kvm/iommu.c
+++ b/virt/kvm/iommu.c
@@ -78,7 +78,7 @@ static int kvm_iommu_map_memslots(struct kvm *kvm)
 	int i, r = 0;
 	struct kvm_memslots *slots;
 
-	slots = rcu_dereference(kvm->memslots);
+	slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 
 	for (i = 0; i < slots->nmemslots; i++) {
 		r = kvm_iommu_map_pages(kvm, &slots->memslots[i]);
@@ -217,7 +217,7 @@ static int kvm_iommu_unmap_memslots(struct kvm *kvm)
 	int i;
 	struct kvm_memslots *slots;
 
-	slots = rcu_dereference(kvm->memslots);
+	slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 
 	for (i = 0; i < slots->nmemslots; i++) {
 		kvm_iommu_put_pages(kvm, slots->memslots[i].base_gfn,
diff --git a/virt/kvm/kvm_main.c b/virt/kvm/kvm_main.c
index 548f925..ae28c71 100644
--- a/virt/kvm/kvm_main.c
+++ b/virt/kvm/kvm_main.c
@@ -372,6 +372,8 @@ static struct kvm *kvm_create_vm(void)
 {
 	int r = 0, i;
 	struct kvm *kvm = kvm_arch_create_vm();
+	struct kvm_io_bus *buses[KVM_NR_BUSES];
+	struct kvm_memslots *memslots;
 
 	if (IS_ERR(kvm))
 		goto out;
@@ -386,14 +388,15 @@ static struct kvm *kvm_create_vm(void)
 #endif
 
 	r = -ENOMEM;
-	kvm->memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
+	memslots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
+	srcu_assign_pointer(kvm->memslots, memslots);
 	if (!kvm->memslots)
 		goto out_err;
 	if (init_srcu_struct(&kvm->srcu))
 		goto out_err;
 	for (i = 0; i < KVM_NR_BUSES; i++) {
-		kvm->buses[i] = kzalloc(sizeof(struct kvm_io_bus),
-					GFP_KERNEL);
+		buses[i] = kzalloc(sizeof(struct kvm_io_bus), GFP_KERNEL);
+		srcu_assign_pointer(kvm->buses[i], buses[i]);
 		if (!kvm->buses[i]) {
 			cleanup_srcu_struct(&kvm->srcu);
 			goto out_err;
@@ -428,8 +431,8 @@ out_err:
 	hardware_disable_all();
 out_err_nodisable:
 	for (i = 0; i < KVM_NR_BUSES; i++)
-		kfree(kvm->buses[i]);
-	kfree(kvm->memslots);
+		kfree(buses[i]);
+	kfree(memslots);
 	kfree(kvm);
 	return ERR_PTR(r);
 }
@@ -464,12 +467,12 @@ static void kvm_free_physmem_slot(struct kvm_memory_slot *free,
 void kvm_free_physmem(struct kvm *kvm)
 {
 	int i;
-	struct kvm_memslots *slots = kvm->memslots;
+	struct kvm_memslots *slots = srcu_dereference_const(kvm->memslots);
 
 	for (i = 0; i < slots->nmemslots; ++i)
 		kvm_free_physmem_slot(&slots->memslots[i], NULL);
 
-	kfree(kvm->memslots);
+	kfree(slots);
 }
 
 static void kvm_destroy_vm(struct kvm *kvm)
@@ -483,7 +486,7 @@ static void kvm_destroy_vm(struct kvm *kvm)
 	spin_unlock(&kvm_lock);
 	kvm_free_irq_routing(kvm);
 	for (i = 0; i < KVM_NR_BUSES; i++)
-		kvm_io_bus_destroy(kvm->buses[i]);
+		kvm_io_bus_destroy(srcu_dereference_const(kvm->buses[i]));
 	kvm_coalesced_mmio_free(kvm);
 #if defined(CONFIG_MMU_NOTIFIER) && defined(KVM_ARCH_WANT_MMU_NOTIFIER)
 	mmu_notifier_unregister(&kvm->mmu_notifier, kvm->mm);
@@ -552,7 +555,8 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	if (mem->guest_phys_addr + mem->memory_size < mem->guest_phys_addr)
 		goto out;
 
-	memslot = &kvm->memslots->memslots[mem->slot];
+	old_memslots = srcu_dereference(kvm->memslots, &kvm->srcu);
+	memslot = &old_memslots->memslots[mem->slot];
 	base_gfn = mem->guest_phys_addr >> PAGE_SHIFT;
 	npages = mem->memory_size >> PAGE_SHIFT;
 
@@ -573,7 +577,7 @@ int __kvm_set_memory_region(struct kvm *kvm,
 	/* Check for overlaps */
 	r = -EEXIST;
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
-		struct kvm_memory_slot *s = &kvm->memslots->memslots[i];
+		struct kvm_memory_slot *s = &old_memslots->memslots[i];
 
 		if (s == memslot || !s->npages)
 			continue;
@@ -669,13 +673,13 @@ skip_lpage:
 		slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 		if (!slots)
 			goto out_free;
-		memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
+		old_memslots = srcu_dereference_const(kvm->memslots);
+		memcpy(slots, old_memslots, sizeof(struct kvm_memslots));
 		if (mem->slot >= slots->nmemslots)
 			slots->nmemslots = mem->slot + 1;
 		slots->memslots[mem->slot].flags |= KVM_MEMSLOT_INVALID;
 
-		old_memslots = kvm->memslots;
-		rcu_assign_pointer(kvm->memslots, slots);
+		srcu_assign_pointer(kvm->memslots, slots);
 		synchronize_srcu_expedited(&kvm->srcu);
 		/* From this point no new shadow pages pointing to a deleted
 		 * memslot will be created.
@@ -705,7 +709,8 @@ skip_lpage:
 	slots = kzalloc(sizeof(struct kvm_memslots), GFP_KERNEL);
 	if (!slots)
 		goto out_free;
-	memcpy(slots, kvm->memslots, sizeof(struct kvm_memslots));
+	old_memslots = srcu_dereference_const(kvm->memslots);
+	memcpy(slots, old_memslots, sizeof(struct kvm_memslots));
 	if (mem->slot >= slots->nmemslots)
 		slots->nmemslots = mem->slot + 1;
 
@@ -718,8 +723,7 @@ skip_lpage:
 	}
 
 	slots->memslots[mem->slot] = new;
-	old_memslots = kvm->memslots;
-	rcu_assign_pointer(kvm->memslots, slots);
+	srcu_assign_pointer(kvm->memslots, slots);
 	synchronize_srcu_expedited(&kvm->srcu);
 
 	kvm_arch_commit_memory_region(kvm, mem, old, user_alloc);
@@ -775,7 +779,7 @@ int kvm_get_dirty_log(struct kvm *kvm,
 	if (log->slot >= KVM_MEMORY_SLOTS)
 		goto out;
 
-	memslot = &kvm->memslots->memslots[log->slot];
+	memslot = &srcu_dereference(kvm->memslots, &kvm->srcu)->memslots[log->slot];
 	r = -ENOENT;
 	if (!memslot->dirty_bitmap)
 		goto out;
@@ -829,7 +833,7 @@ EXPORT_SYMBOL_GPL(kvm_is_error_hva);
 struct kvm_memory_slot *gfn_to_memslot_unaliased(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
-	struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
+	struct kvm_memslots *slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 
 	for (i = 0; i < slots->nmemslots; ++i) {
 		struct kvm_memory_slot *memslot = &slots->memslots[i];
@@ -851,7 +855,7 @@ struct kvm_memory_slot *gfn_to_memslot(struct kvm *kvm, gfn_t gfn)
 int kvm_is_visible_gfn(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
-	struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
+	struct kvm_memslots *slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 
 	gfn = unalias_gfn_instantiation(kvm, gfn);
 	for (i = 0; i < KVM_MEMORY_SLOTS; ++i) {
@@ -895,7 +899,7 @@ out:
 int memslot_id(struct kvm *kvm, gfn_t gfn)
 {
 	int i;
-	struct kvm_memslots *slots = rcu_dereference(kvm->memslots);
+	struct kvm_memslots *slots = srcu_dereference(kvm->memslots, &kvm->srcu);
 	struct kvm_memory_slot *memslot = NULL;
 
 	gfn = unalias_gfn(kvm, gfn);
@@ -1984,7 +1988,7 @@ int kvm_io_bus_write(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 		     int len, const void *val)
 {
 	int i;
-	struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
+	struct kvm_io_bus *bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
 	for (i = 0; i < bus->dev_count; i++)
 		if (!kvm_iodevice_write(bus->devs[i], addr, len, val))
 			return 0;
@@ -1996,7 +2000,7 @@ int kvm_io_bus_read(struct kvm *kvm, enum kvm_bus bus_idx, gpa_t addr,
 		    int len, void *val)
 {
 	int i;
-	struct kvm_io_bus *bus = rcu_dereference(kvm->buses[bus_idx]);
+	struct kvm_io_bus *bus = srcu_dereference(kvm->buses[bus_idx], &kvm->srcu);
 
 	for (i = 0; i < bus->dev_count; i++)
 		if (!kvm_iodevice_read(bus->devs[i], addr, len, val))
@@ -2010,7 +2014,7 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 {
 	struct kvm_io_bus *new_bus, *bus;
 
-	bus = kvm->buses[bus_idx];
+	bus = srcu_dereference_const(kvm->buses[bus_idx]);
 	if (bus->dev_count > NR_IOBUS_DEVS-1)
 		return -ENOSPC;
 
@@ -2019,7 +2023,7 @@ int kvm_io_bus_register_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 		return -ENOMEM;
 	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
 	new_bus->devs[new_bus->dev_count++] = dev;
-	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
+	srcu_assign_pointer(kvm->buses[bus_idx], new_bus);
 	synchronize_srcu_expedited(&kvm->srcu);
 	kfree(bus);
 
@@ -2037,7 +2041,7 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 	if (!new_bus)
 		return -ENOMEM;
 
-	bus = kvm->buses[bus_idx];
+	bus = srcu_dereference_const(kvm->buses[bus_idx]);
 	memcpy(new_bus, bus, sizeof(struct kvm_io_bus));
 
 	r = -ENOENT;
@@ -2053,7 +2057,7 @@ int kvm_io_bus_unregister_dev(struct kvm *kvm, enum kvm_bus bus_idx,
 		return r;
 	}
 
-	rcu_assign_pointer(kvm->buses[bus_idx], new_bus);
+	srcu_assign_pointer(kvm->buses[bus_idx], new_bus);
 	synchronize_srcu_expedited(&kvm->srcu);
 	kfree(bus);
 	return r;