Subject: [PATCH v3 19/19] arm64: KVM: vgic: add GICv3 world switch

From: [email-masked] (Marc Zyngier)
Date: Thu, 15 May 2014 09:31:14 +0100

On Fri, May 09 2014 at  3:07:38 pm BST, Christoffer Dall <[email-masked]> wrote:
> On Wed, Apr 16, 2014 at 02:39:51PM +0100, Marc Zyngier wrote:
>> Introduce the GICv3 world switch code and helper functions, enabling
>> GICv2 emulation on GICv3 hardware.
>>
>> Acked-by: Catalin Marinas <[email-masked]>
>> Signed-off-by: Marc Zyngier <[email-masked]>
>> ---
>>  arch/arm64/include/asm/kvm_asm.h  |   4 +
>>  arch/arm64/include/asm/kvm_host.h |   7 +
>>  arch/arm64/kernel/asm-offsets.c   |   8 ++
>>  arch/arm64/kvm/Makefile           |   2 +
>>  arch/arm64/kvm/vgic-v3-switch.S   | 279 ++++++++++++++++++++++++++++++++++++++
>>  5 files changed, 300 insertions(+)
>>  create mode 100644 arch/arm64/kvm/vgic-v3-switch.S
>>
>> diff --git a/arch/arm64/include/asm/kvm_asm.h b/arch/arm64/include/asm/kvm_asm.h
>> index 6515a52..270ea13 100644
>> --- a/arch/arm64/include/asm/kvm_asm.h
>> +++ b/arch/arm64/include/asm/kvm_asm.h
>> @@ -105,8 +105,12 @@ extern void __kvm_tlb_flush_vmid_ipa(struct kvm *kvm, phys_addr_t ipa);
>>
>>  extern int __kvm_vcpu_run(struct kvm_vcpu *vcpu);
>>
>> +extern u64 __vgic_v3_get_ich_vtr_el2(void);
>> +
>>  extern char __save_vgic_v2_state[];
>>  extern char __restore_vgic_v2_state[];
>> +extern char __save_vgic_v3_state[];
>> +extern char __restore_vgic_v3_state[];
>>
>>  #endif
>>
>> diff --git a/arch/arm64/include/asm/kvm_host.h b/arch/arm64/include/asm/kvm_host.h
>> index 65f0c43..a10803c 100644
>> --- a/arch/arm64/include/asm/kvm_host.h
>> +++ b/arch/arm64/include/asm/kvm_host.h
>> @@ -216,6 +216,13 @@ static inline void vgic_arch_setup(const struct vgic_params *vgic)
>>               __vgic_sr_vectors.restore_vgic  = __restore_vgic_v2_state;
>>               break;
>>
>> +#ifdef CONFIG_ARM_GIC_V3
>> +     case VGIC_V3:
>> +             __vgic_sr_vectors.save_vgic     = __save_vgic_v3_state;
>> +             __vgic_sr_vectors.restore_vgic  = __restore_vgic_v3_state;
>> +             break;
>> +#endif
>> +
>>       default:
>>               BUG();
>>       }
>> diff --git a/arch/arm64/kernel/asm-offsets.c b/arch/arm64/kernel/asm-offsets.c
>> index dafc415..e74654c 100644
>> --- a/arch/arm64/kernel/asm-offsets.c
>> +++ b/arch/arm64/kernel/asm-offsets.c
>> @@ -139,6 +139,14 @@ int main(void)
>>    DEFINE(VGIC_V2_CPU_ELRSR,  offsetof(struct vgic_cpu, vgic_v2.vgic_elrsr));
>>    DEFINE(VGIC_V2_CPU_APR,    offsetof(struct vgic_cpu, vgic_v2.vgic_apr));
>>    DEFINE(VGIC_V2_CPU_LR,     offsetof(struct vgic_cpu, vgic_v2.vgic_lr));
>> +  DEFINE(VGIC_V3_CPU_HCR,    offsetof(struct vgic_cpu, vgic_v3.vgic_hcr));
>> +  DEFINE(VGIC_V3_CPU_VMCR,   offsetof(struct vgic_cpu, vgic_v3.vgic_vmcr));
>> +  DEFINE(VGIC_V3_CPU_MISR,   offsetof(struct vgic_cpu, vgic_v3.vgic_misr));
>> +  DEFINE(VGIC_V3_CPU_EISR,   offsetof(struct vgic_cpu, vgic_v3.vgic_eisr));
>> +  DEFINE(VGIC_V3_CPU_ELRSR,  offsetof(struct vgic_cpu, vgic_v3.vgic_elrsr));
>> +  DEFINE(VGIC_V3_CPU_AP0R,   offsetof(struct vgic_cpu, vgic_v3.vgic_ap0r));
>> +  DEFINE(VGIC_V3_CPU_AP1R,   offsetof(struct vgic_cpu, vgic_v3.vgic_ap1r));
>> +  DEFINE(VGIC_V3_CPU_LR,     offsetof(struct vgic_cpu, vgic_v3.vgic_lr));
>>    DEFINE(VGIC_CPU_NR_LR,     offsetof(struct vgic_cpu, nr_lr));
>>    DEFINE(KVM_VTTBR,          offsetof(struct kvm, arch.vttbr));
>>    DEFINE(KVM_VGIC_VCTRL,     offsetof(struct kvm, arch.vgic.vctrl_base));
>> diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile
>> index daf24dc..32a0961 100644
>> --- a/arch/arm64/kvm/Makefile
>> +++ b/arch/arm64/kvm/Makefile
>> @@ -22,4 +22,6 @@ kvm-$(CONFIG_KVM_ARM_HOST) += guest.o reset.o sys_regs.o sys_regs_generic_v8.o
>>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic.o
>>  kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v2.o
>>  kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v2-switch.o
>> +kvm-$(CONFIG_KVM_ARM_VGIC) += $(KVM)/arm/vgic-v3.o
>> +kvm-$(CONFIG_KVM_ARM_VGIC) += vgic-v3-switch.o
>>  kvm-$(CONFIG_KVM_ARM_TIMER) += $(KVM)/arm/arch_timer.o
>> diff --git a/arch/arm64/kvm/vgic-v3-switch.S b/arch/arm64/kvm/vgic-v3-switch.S
>> new file mode 100644
>> index 0000000..7d2bc86
>> --- /dev/null
>> +++ b/arch/arm64/kvm/vgic-v3-switch.S
>> @@ -0,0 +1,279 @@
>> +/*
>> + * Copyright (C) 2012,2013 - ARM Ltd
>> + * Author: Marc Zyngier <[email-masked]>
>> + *
>> + * This program is free software; you can redistribute it and/or modify
>> + * it under the terms of the GNU General Public License version 2 as
>> + * published by the Free Software Foundation.
>> + *
>> + * This program is distributed in the hope that it will be useful,
>> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
>> + * GNU General Public License for more details.
>> + *
>> + * You should have received a copy of the GNU General Public License
>> + * along with this program.  If not, see <http://www.gnu.org/licenses/>.
>> + */
>> +
>> +#include <linux/linkage.h>
>> +#include <linux/irqchip/arm-gic-v3.h>
>> +
>> +#include <asm/assembler.h>
>> +#include <asm/memory.h>
>> +#include <asm/asm-offsets.h>
>> +#include <asm/kvm.h>
>> +#include <asm/kvm_asm.h>
>> +#include <asm/kvm_arm.h>
>> +
>> +     .text
>> +     .pushsection    .hyp.text, "ax"
>> +
>> +/*
>> + * Save the VGIC CPU state into memory
>> + * x0: Register pointing to VCPU struct
>> + * Do not corrupt x1!!!
>> + */
>> +.macro       save_vgic_v3_state
>> +     // Compute the address of struct vgic_cpu
>> +     add     x3, x0, #VCPU_VGIC_CPU
>> +
>> +     // Make sure stores to the GIC via the memory mapped interface
>> +     // are now visible to the system register interface
>> +     dsb     sy
>> +
>> +     // Save all interesting registers
>> +     mrs     x4, ICH_HCR_EL2
>> +     mrs     x5, ICH_VMCR_EL2
>> +     mrs     x6, ICH_MISR_EL2
>> +     mrs     x7, ICH_EISR_EL2
>> +     mrs     x8, ICH_ELSR_EL2
>> +
>> +     str     w4, [x3, #VGIC_V3_CPU_HCR]
>> +     str     w5, [x3, #VGIC_V3_CPU_VMCR]
>> +     str     w6, [x3, #VGIC_V3_CPU_MISR]
>> +     str     w7, [x3, #VGIC_V3_CPU_EISR]
>> +     str     w8, [x3, #VGIC_V3_CPU_ELRSR]
>> +
>> +     msr     ICH_HCR_EL2, xzr
>> +
>> +     mrs     x21, ICH_VTR_EL2
>> +     and     w22, w21, #0xf
>> +     mov     w23, #0xf
>> +     sub     w23, w23, w22   // How many regs we have to skip
>> +
>> +     adr     x24, 1f
>> +     add     x24, x24, x23, lsl #2
>> +     br      x24
>> +
>> +1:
>> +     mrs     x20, ICH_LR15_EL2
>> +     mrs     x19, ICH_LR14_EL2
>> +     mrs     x18, ICH_LR13_EL2
>> +     mrs     x17, ICH_LR12_EL2
>> +     mrs     x16, ICH_LR11_EL2
>> +     mrs     x15, ICH_LR10_EL2
>> +     mrs     x14, ICH_LR9_EL2
>> +     mrs     x13, ICH_LR8_EL2
>> +     mrs     x12, ICH_LR7_EL2
>> +     mrs     x11, ICH_LR6_EL2
>> +     mrs     x10, ICH_LR5_EL2
>> +     mrs     x9, ICH_LR4_EL2
>> +     mrs     x8, ICH_LR3_EL2
>> +     mrs     x7, ICH_LR2_EL2
>> +     mrs     x6, ICH_LR1_EL2
>> +     mrs     x5, ICH_LR0_EL2
>> +
>> +     adr     x24, 1f
>> +     add     x24, x24, x23, lsl #2   // adr(1f) + unimp_nr*4
>> +     br      x24
>> +
>> +1:
>> +     str     x20, [x3, #(VGIC_V3_CPU_LR + 15*8)]
>> +     str     x19, [x3, #(VGIC_V3_CPU_LR + 14*8)]
>> +     str     x18, [x3, #(VGIC_V3_CPU_LR + 13*8)]
>> +     str     x17, [x3, #(VGIC_V3_CPU_LR + 12*8)]
>> +     str     x16, [x3, #(VGIC_V3_CPU_LR + 11*8)]
>> +     str     x15, [x3, #(VGIC_V3_CPU_LR + 10*8)]
>> +     str     x14, [x3, #(VGIC_V3_CPU_LR + 9*8)]
>> +     str     x13, [x3, #(VGIC_V3_CPU_LR + 8*8)]
>> +     str     x12, [x3, #(VGIC_V3_CPU_LR + 7*8)]
>> +     str     x11, [x3, #(VGIC_V3_CPU_LR + 6*8)]
>> +     str     x10, [x3, #(VGIC_V3_CPU_LR + 5*8)]
>> +     str     x9, [x3, #(VGIC_V3_CPU_LR + 4*8)]
>> +     str     x8, [x3, #(VGIC_V3_CPU_LR + 3*8)]
>> +     str     x7, [x3, #(VGIC_V3_CPU_LR + 2*8)]
>> +     str     x6, [x3, #(VGIC_V3_CPU_LR + 1*8)]
>> +     str     x5, [x3, #VGIC_V3_CPU_LR]
>> +
>> +     lsr     w22, w21, #29   // Get PRIbits
>> +     cmp     w22, #4         // 5 bits
>> +     b.eq    5f
>> +     cmp     w22, #5         // 6 bits
>> +     b.eq    6f
>> +                             // 7 bits
>> +     mrs     x20, ICH_AP0R3_EL2
>> +     str     w20, [x3, #(VGIC_V3_CPU_AP0R + 3*4)]
>> +     mrs     x19, ICH_AP0R2_EL2
>> +     str     w19, [x3, #(VGIC_V3_CPU_AP0R + 2*4)]
>> +6:   mrs     x18, ICH_AP0R1_EL2
>> +     str     w18, [x3, #(VGIC_V3_CPU_AP0R + 1*4)]
>> +5:   mrs     x17, ICH_AP0R0_EL2
>> +     str     w17, [x3, #VGIC_V3_CPU_AP0R]
>> +
>> +     cmp     w22, #4         // 5 bits
>> +     b.eq    5f
>> +     cmp     w22, #5         // 6 bits
>> +     b.eq    6f
>> +                             // 7 bits
>> +     mrs     x20, ICH_AP1R3_EL2
>> +     str     w20, [x3, #(VGIC_V3_CPU_AP1R + 3*4)]
>> +     mrs     x19, ICH_AP1R2_EL2
>> +     str     w19, [x3, #(VGIC_V3_CPU_AP1R + 2*4)]
>> +6:   mrs     x18, ICH_AP1R1_EL2
>> +     str     w18, [x3, #(VGIC_V3_CPU_AP1R + 1*4)]
>> +5:   mrs     x17, ICH_AP1R0_EL2
>> +     str     w17, [x3, #VGIC_V3_CPU_AP1R]
>> +
>> +     // Restore SRE_EL1 access and re-enable SRE at EL1.
>> +     mrs     x5, ICC_SRE_EL2
>> +     orr     x5, x5, #(1 << 3)
>
> couldn't we define ICC_SRE_ENABLE (1 << 3)?

We sure can.

>> +     msr     ICC_SRE_EL2, x5
>> +     isb
>> +     mov     x5, #1
>> +     msr     ICC_SRE_EL1, x5
>
> the other bits are always read-only (WI), so you can safely just
> overwrite all other bits here?

Essentially, yes. The only case where the other bits are writable is
when EL2 is not present, and that's obviously not the case if we're
running KVM.

>> +
>> +     mov     x2, #HCR_RW
>> +     msr     hcr_el2, x2
>> +.endm
>> +
>> +/*
>> + * Restore the VGIC CPU state from memory
>> + * x0: Register pointing to VCPU struct
>> + */
>> +.macro       restore_vgic_v3_state
>> +     ldr     x2, [x0, #VCPU_IRQ_LINES]
>
> again, what can this be used for with aarch64?

Yes it can.

>> +     ldr     x1, [x0, #VCPU_HCR_EL2]
>> +     orr     x2, x2, x1
>> +     msr     hcr_el2, x2
>> +
>> +     // Disable SRE_EL1 access. Necessary, otherwise
>> +     // ICH_VMCR_EL2.VFIQEn becomes one, and FIQ happens...
>> +     msr     ICC_SRE_EL1, xzr
>> +     isb
>> +
>> +     // Compute the address of struct vgic_cpu
>> +     add     x3, x0, #VCPU_VGIC_CPU
>> +
>> +     // Restore all interesting registers
>> +     ldr     w4, [x3, #VGIC_V3_CPU_HCR]
>> +     ldr     w5, [x3, #VGIC_V3_CPU_VMCR]
>> +
>> +     msr     ICH_HCR_EL2, x4
>> +     msr     ICH_VMCR_EL2, x5
>> +
>> +     mrs     x21, ICH_VTR_EL2
>> +
>> +     lsr     w22, w21, #29   // Get PRIbits
>> +     cmp     w22, #4         // 5 bits
>> +     b.eq    5f
>> +     cmp     w22, #5         // 6 bits
>> +     b.eq    6f
>> +                             // 7 bits
>> +     ldr     w20, [x3, #(VGIC_V3_CPU_AP1R + 3*4)]
>> +     msr     ICH_AP1R3_EL2, x20
>> +     ldr     w19, [x3, #(VGIC_V3_CPU_AP1R + 2*4)]
>> +     msr     ICH_AP1R2_EL2, x19
>> +6:   ldr     w18, [x3, #(VGIC_V3_CPU_AP1R + 1*4)]
>> +     msr     ICH_AP1R1_EL2, x18
>> +5:   ldr     w17, [x3, #VGIC_V3_CPU_AP1R]
>> +     msr     ICH_AP1R0_EL2, x17
>> +
>> +     cmp     w22, #4         // 5 bits
>> +     b.eq    5f
>> +     cmp     w22, #5         // 6 bits
>> +     b.eq    6f
>> +                             // 7 bits
>> +     ldr     w20, [x3, #(VGIC_V3_CPU_AP0R + 3*4)]
>> +     msr     ICH_AP0R3_EL2, x20
>> +     ldr     w19, [x3, #(VGIC_V3_CPU_AP0R + 2*4)]
>> +     msr     ICH_AP0R2_EL2, x19
>> +6:   ldr     w18, [x3, #(VGIC_V3_CPU_AP0R + 1*4)]
>> +     msr     ICH_AP0R1_EL2, x18
>> +5:   ldr     w17, [x3, #VGIC_V3_CPU_AP0R]
>> +     msr     ICH_AP0R0_EL2, x17
>> +
>> +     and     w22, w21, #0xf
>> +     mov     w23, #0xf
>> +     sub     w23, w23, w22   // How many regs we have to skip
>> +
>> +     adr     x24, 1f
>> +     add     x24, x24, x23, lsl #2   // adr(1f) + unimp_nr*4
>> +     br      x24
>> +
>> +1:
>> +     ldr     x20, [x3, #(VGIC_V3_CPU_LR + 15*8)]
>> +     ldr     x19, [x3, #(VGIC_V3_CPU_LR + 14*8)]
>> +     ldr     x18, [x3, #(VGIC_V3_CPU_LR + 13*8)]
>> +     ldr     x17, [x3, #(VGIC_V3_CPU_LR + 12*8)]
>> +     ldr     x16, [x3, #(VGIC_V3_CPU_LR + 11*8)]
>> +     ldr     x15, [x3, #(VGIC_V3_CPU_LR + 10*8)]
>> +     ldr     x14, [x3, #(VGIC_V3_CPU_LR + 9*8)]
>> +     ldr     x13, [x3, #(VGIC_V3_CPU_LR + 8*8)]
>> +     ldr     x12, [x3, #(VGIC_V3_CPU_LR + 7*8)]
>> +     ldr     x11, [x3, #(VGIC_V3_CPU_LR + 6*8)]
>> +     ldr     x10, [x3, #(VGIC_V3_CPU_LR + 5*8)]
>> +     ldr     x9, [x3, #(VGIC_V3_CPU_LR + 4*8)]
>> +     ldr     x8, [x3, #(VGIC_V3_CPU_LR + 3*8)]
>> +     ldr     x7, [x3, #(VGIC_V3_CPU_LR + 2*8)]
>> +     ldr     x6, [x3, #(VGIC_V3_CPU_LR + 1*8)]
>> +     ldr     x5, [x3, #VGIC_V3_CPU_LR]
>> +
>> +     adr     x24, 1f
>> +     add     x24, x24, x23, lsl #2
>> +     br      x24
>> +
>> +1:
>> +     msr     ICH_LR15_EL2, x20
>> +     msr     ICH_LR14_EL2, x19
>> +     msr     ICH_LR13_EL2, x18
>> +     msr     ICH_LR12_EL2, x17
>> +     msr     ICH_LR11_EL2, x16
>> +     msr     ICH_LR10_EL2, x15
>> +     msr     ICH_LR9_EL2,  x14
>> +     msr     ICH_LR8_EL2,  x13
>> +     msr     ICH_LR7_EL2,  x12
>> +     msr     ICH_LR6_EL2,  x11
>> +     msr     ICH_LR5_EL2,  x10
>> +     msr     ICH_LR4_EL2,   x9
>> +     msr     ICH_LR3_EL2,   x8
>> +     msr     ICH_LR2_EL2,   x7
>> +     msr     ICH_LR1_EL2,   x6
>> +     msr     ICH_LR0_EL2,   x5
>> +
>> +     // Ensure that the above will be visible via the memory-mapped
>> +     // view of the CPU interface (GICV).
>> +     isb
>> +     dsb     sy
>> +
>> +     // Prevent the guest from touching the GIC system registers
>> +     mrs     x5, ICC_SRE_EL2
>> +     and     x5, x5, #~(1 << 3)
>
> ditto on the define
>
>> +     msr     ICC_SRE_EL2, x5
>
> I trust Will reviewed all the barriers etc., but you really don't
> need an ISB or anything here?

No, we should be fine at that stage. The final ERET into the guest
ensure architectural execution of this instruction.

>> +.endm
>> +
>> +ENTRY(__save_vgic_v3_state)
>> +     save_vgic_v3_state
>> +     ret
>> +ENDPROC(__save_vgic_v3_state)
>> +
>> +ENTRY(__restore_vgic_v3_state)
>> +     restore_vgic_v3_state
>> +     ret
>> +ENDPROC(__restore_vgic_v3_state)
>> +
>> +ENTRY(__vgic_v3_get_ich_vtr_el2)
>> +     mrs     x0, ICH_VTR_EL2
>> +     ret
>> +ENDPROC(__vgic_v3_get_ich_vtr_el2)
>> +
>> +     .popsection
>> --
>> 1.8.3.4
>>
>
> Reviewed-by: Christoffer Dall <[email-masked]>
>

-- 
Jazz is not dead. It just smells funny.