From: Mark Rutland <mark.rutland@arm•com>
To: linux-arm-kernel@lists•infradead.org, kvmarm@lists•linux.dev
Cc: broonie@kernel•org, catalin.marinas@arm•com, james.morse@arm•com,
mark.rutland@arm•com, maz@kernel•org, oupton@kernel•org,
tabba@google•com, vladimir.murzin@arm•com, will@kernel•org
Subject: [PATCH v2 16/18] arm64: fpsimd: Move sve_flush_live() inline
Date: Thu, 28 May 2026 17:54:44 +0100 [thread overview]
Message-ID: <20260528165446.701944-17-mark.rutland@arm.com> (raw)
In-Reply-To: <20260528165446.701944-1-mark.rutland@arm.com>
Currently sve_flush_live() is written in out-of-line assembly. It would
be nice if we could move it inline such that control flow can be written
more clearly in C, and to permit the removal of otherwise unused
assembly macros.
The 'flush_ffr' argument is redundant as sve_flush_live() is always
called from non-streaming mode, and all callers pass 'true'. Remove the
argument and make it a requirement that the function is called from
non-streaming mode.
The 'vq_minus_1' argument is unnecessary, as sve_flush_live() can read
the live VL directly using the RDVL instruction (wrapped by the
sve_get_vl() helper function).
Move the function to C, with the simplifications above.
Signed-off-by: Mark Rutland <mark.rutland@arm•com>
Reviewed-by: Mark Brown <broonie@kernel•org>
Reviewed-by: Vladimir Murzin <vladimir.murzin@arm•com>
Cc: Catalin Marinas <catalin.marinas@arm•com>
Cc: Fuad Tabba <tabba@google•com>
Cc: James Morse <james.morse@arm•com>
Cc: Marc Zyngier <maz@kernel•org>
Cc: Oliver Upton <oupton@kernel•org>
Cc: Will Deacon <will@kernel•org>
---
arch/arm64/include/asm/fpsimd.h | 26 ++++++++++++++++++++++-
arch/arm64/include/asm/fpsimdmacros.h | 30 ---------------------------
arch/arm64/kernel/entry-common.c | 8 ++-----
arch/arm64/kernel/entry-fpsimd.S | 22 --------------------
arch/arm64/kernel/fpsimd.c | 2 +-
5 files changed, 28 insertions(+), 60 deletions(-)
diff --git a/arch/arm64/include/asm/fpsimd.h b/arch/arm64/include/asm/fpsimd.h
index d005324bbcf3e..550987b36206a 100644
--- a/arch/arm64/include/asm/fpsimd.h
+++ b/arch/arm64/include/asm/fpsimd.h
@@ -332,7 +332,31 @@ static inline void sve_load_state(const struct sve_state *state, bool ffr)
__sve_load_p(state, vl, ffr);
}
-extern void sve_flush_live(bool flush_ffr, unsigned long vq_minus_1);
+
+/*
+ * Zero all SVE registers except for the first 128 bits of each vector.
+ *
+ * The caller must ensure that the VL has been configured and the CPU must be
+ * in non-streaming mode.
+ */
+static inline void sve_flush_live(void)
+{
+ unsigned long vl = sve_get_vl();
+
+ if (vl > sizeof(__uint128_t)) {
+ asm volatile(
+ __FPSIMD_PREAMBLE
+ FOR_EACH_Z_REG("n", "mov v\\n\\().16b, v\\n\\().16b")
+ );
+ }
+
+ asm volatile(
+ __SVE_PREAMBLE
+ FOR_EACH_P_REG("n", "pfalse p\\n\\().b")
+ " wrffr p0.b\n"
+ );
+}
+
extern void sme_save_state(struct sme_state *state, int zt);
extern void sme_load_state(const struct sme_state *state, int zt);
diff --git a/arch/arm64/include/asm/fpsimdmacros.h b/arch/arm64/include/asm/fpsimdmacros.h
index 5f03fe51d0bff..9e352b5c6b764 100644
--- a/arch/arm64/include/asm/fpsimdmacros.h
+++ b/arch/arm64/include/asm/fpsimdmacros.h
@@ -40,20 +40,6 @@
.endif
.endm
-/* Deprecated macros for SVE instructions */
-
-/* WRFFR P\np.B */
-.macro _sve_wrffr np
- .arch_extension sve
- wrffr p\np\().b
-.endm
-
-/* PFALSE P\np.B */
-.macro _sve_pfalse np
- .arch_extension sve
- pfalse p\np\().b
-.endm
-
/* Deprecated macros for SME instructions */
/* RDSVL X\nx, #\imm */
@@ -131,22 +117,6 @@
.purgem _for__body
.endm
-/* Preserve the first 128-bits of Znz and zero the rest. */
-.macro _sve_flush_z nz
- _sve_check_zreg \nz
- mov v\nz\().16b, v\nz\().16b
-.endm
-
-.macro sve_flush_z
- _for n, 0, 31, _sve_flush_z \n
-.endm
-.macro sve_flush_p
- _for n, 0, 15, _sve_pfalse \n
-.endm
-.macro sve_flush_ffr
- _sve_wrffr 0
-.endm
-
.macro sme_save_za nxbase, xvl, nw
mov w\nw, #0
diff --git a/arch/arm64/kernel/entry-common.c b/arch/arm64/kernel/entry-common.c
index cb54335465f66..2352297330e12 100644
--- a/arch/arm64/kernel/entry-common.c
+++ b/arch/arm64/kernel/entry-common.c
@@ -237,12 +237,8 @@ static inline void fpsimd_syscall_enter(void)
if (!system_supports_sve())
return;
- if (test_thread_flag(TIF_SVE)) {
- unsigned int sve_vq_minus_one;
-
- sve_vq_minus_one = sve_vq_from_vl(task_get_sve_vl(current)) - 1;
- sve_flush_live(true, sve_vq_minus_one);
- }
+ if (test_thread_flag(TIF_SVE))
+ sve_flush_live();
/*
* Any live non-FPSIMD SVE state has been zeroed. Allow
diff --git a/arch/arm64/kernel/entry-fpsimd.S b/arch/arm64/kernel/entry-fpsimd.S
index 0575d90e6dffb..bff941eea9566 100644
--- a/arch/arm64/kernel/entry-fpsimd.S
+++ b/arch/arm64/kernel/entry-fpsimd.S
@@ -11,28 +11,6 @@
#include <asm/assembler.h>
#include <asm/fpsimdmacros.h>
-#ifdef CONFIG_ARM64_SVE
-
-/*
- * Zero all SVE registers but the first 128-bits of each vector
- *
- * VQ must already be configured by caller, any further updates of VQ
- * will need to ensure that the register state remains valid.
- *
- * x0 = include FFR?
- * x1 = VQ - 1
- */
-SYM_FUNC_START(sve_flush_live)
- cbz x1, 1f // A VQ-1 of 0 is 128 bits so no extra Z state
- sve_flush_z
-1: sve_flush_p
- tbz x0, #0, 2f
- sve_flush_ffr
-2: ret
-SYM_FUNC_END(sve_flush_live)
-
-#endif /* CONFIG_ARM64_SVE */
-
#ifdef CONFIG_ARM64_SME
/*
diff --git a/arch/arm64/kernel/fpsimd.c b/arch/arm64/kernel/fpsimd.c
index f9b3eeacf130d..42177b439b3c7 100644
--- a/arch/arm64/kernel/fpsimd.c
+++ b/arch/arm64/kernel/fpsimd.c
@@ -1338,7 +1338,7 @@ void do_sve_acc(unsigned long esr, struct pt_regs *regs)
if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) {
unsigned long vq = sve_vq_from_vl(task_get_sve_vl(current));
sysreg_clear_set_s(SYS_ZCR_EL1, ZCR_ELx_LEN, vq - 1);
- sve_flush_live(true, vq - 1);
+ sve_flush_live();
fpsimd_bind_task_to_cpu();
} else {
fpsimd_to_sve(current);
--
2.30.2
next prev parent reply other threads:[~2026-05-28 16:56 UTC|newest]
Thread overview: 25+ messages / expand[flat|nested] mbox.gz Atom feed top
2026-05-28 16:54 [PATCH v2 00/18] arm64+KVM: FPSIMD/SVE/SME cleanups Mark Rutland
2026-05-28 16:54 ` [PATCH v2 01/18] KVM: arm64: Don't include <asm/fpsimdmacros.h> Mark Rutland
2026-05-28 16:54 ` [PATCH v2 02/18] KVM: arm64: Don't override FFR save/restore argument Mark Rutland
2026-05-28 16:54 ` [PATCH v2 03/18] KVM: arm64: pkvm: Save host FPMR in host cpu context Mark Rutland
2026-05-28 16:54 ` [PATCH v2 04/18] KVM: arm64: pkvm: Remove struct cpu_sve_state Mark Rutland
2026-05-28 16:54 ` [PATCH v2 05/18] arm64: fpsimd: Fold sve_init_regs() into do_sve_acc() Mark Rutland
2026-05-28 16:54 ` [PATCH v2 06/18] arm64: fpsimd: Remove sve_set_vq() and sme_set_vq() Mark Rutland
2026-05-28 16:54 ` [PATCH v2 07/18] arm64: fpsimd: Use assembler for SVE instructions Mark Rutland
2026-05-29 8:41 ` Vladimir Murzin
2026-05-28 16:54 ` [PATCH v2 08/18] arm64: fpsimd: Use assembler for baseline SME instructions Mark Rutland
2026-05-28 16:54 ` [PATCH v2 09/18] arm64: fpsimd: Move sve_get_vl() and sme_get_vl() inline Mark Rutland
2026-05-28 16:54 ` [PATCH v2 10/18] arm64: sysreg: Add FPCR and FPSR Mark Rutland
2026-05-28 17:11 ` Mark Brown
2026-05-29 8:42 ` Vladimir Murzin
2026-05-28 16:54 ` [PATCH v2 11/18] arm64: fpsimd: Split FPSR/FPCR from SVE save/restore Mark Rutland
2026-05-28 17:17 ` Mark Brown
2026-05-28 16:54 ` [PATCH v2 12/18] arm64: fpsimd: Move fpsimd save/restore inline Mark Rutland
2026-05-29 8:43 ` Vladimir Murzin
2026-05-28 16:54 ` [PATCH v2 13/18] arm64: fpsimd: Use opaque type for SVE state Mark Rutland
2026-05-28 16:54 ` [PATCH v2 14/18] arm64: fpsimd: Use opaque type for SME state Mark Rutland
2026-05-28 16:54 ` [PATCH v2 15/18] arm64: fpsimd: Move SVE save/restore inline Mark Rutland
2026-05-28 16:54 ` Mark Rutland [this message]
2026-05-28 16:54 ` [PATCH v2 17/18] arm64: fpsimd: Move SME " Mark Rutland
2026-05-29 8:51 ` Vladimir Murzin
2026-05-28 16:54 ` [PATCH v2 18/18] arm64: fpsimd: Remove <asm/fpsimdmacros.h> Mark Rutland
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=20260528165446.701944-17-mark.rutland@arm.com \
--to=mark.rutland@arm$(echo .)com \
--cc=broonie@kernel$(echo .)org \
--cc=catalin.marinas@arm$(echo .)com \
--cc=james.morse@arm$(echo .)com \
--cc=kvmarm@lists$(echo .)linux.dev \
--cc=linux-arm-kernel@lists$(echo .)infradead.org \
--cc=maz@kernel$(echo .)org \
--cc=oupton@kernel$(echo .)org \
--cc=tabba@google$(echo .)com \
--cc=vladimir.murzin@arm$(echo .)com \
--cc=will@kernel$(echo .)org \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox