176 lines
4.8 KiB
ArmAsm
176 lines
4.8 KiB
ArmAsm
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
|
|
// This patch implements the support routines for the SME ABI,
|
|
// described here:
|
|
// https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#sme-support-routines
|
|
|
|
#include "../assembly.h"
|
|
|
|
|
|
#if !defined(__APPLE__)
|
|
#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
|
|
#define TPIDR2_SYMBOL_OFFSET :lo12:SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)
|
|
#else
|
|
// MachO requires @page/@pageoff directives because the global is defined
|
|
// in a different file. Otherwise this file may fail to build.
|
|
#define TPIDR2_SYMBOL SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@page
|
|
#define TPIDR2_SYMBOL_OFFSET SYMBOL_NAME(__aarch64_has_sme_and_tpidr2_el0)@pageoff
|
|
#endif
|
|
|
|
.arch armv9-a+sme
|
|
|
|
// Utility function which calls a system's abort() routine. Because the function
|
|
// is streaming-compatible it should disable streaming-SVE mode before calling
|
|
// abort(). Note that there is no need to preserve any state before the call,
|
|
// because the function does not return.
|
|
DEFINE_COMPILERRT_PRIVATE_FUNCTION(do_abort)
|
|
.cfi_startproc
|
|
.variant_pcs SYMBOL_NAME(do_abort)
|
|
stp x29, x30, [sp, #-32]!
|
|
cntd x0
|
|
// Store VG to a stack location that we describe with .cfi_offset
|
|
str x0, [sp, #16]
|
|
.cfi_def_cfa_offset 32
|
|
.cfi_offset w30, -24
|
|
.cfi_offset w29, -32
|
|
.cfi_offset 46, -16
|
|
bl __arm_sme_state
|
|
tbz x0, #0, 2f
|
|
1:
|
|
smstop sm
|
|
2:
|
|
// We can't make this into a tail-call because the unwinder would
|
|
// need to restore the value of VG.
|
|
bl SYMBOL_NAME(abort)
|
|
.cfi_endproc
|
|
END_COMPILERRT_FUNCTION(do_abort)
|
|
|
|
// __arm_sme_state fills the result registers based on a local
|
|
// that is set as part of the compiler-rt startup code.
|
|
// __aarch64_has_sme_and_tpidr2_el0
|
|
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_sme_state)
|
|
.variant_pcs __arm_sme_state
|
|
mov x0, xzr
|
|
mov x1, xzr
|
|
|
|
adrp x16, TPIDR2_SYMBOL
|
|
ldrb w16, [x16, TPIDR2_SYMBOL_OFFSET]
|
|
cbz w16, 1f
|
|
0:
|
|
orr x0, x0, #0xC000000000000000
|
|
mrs x16, SVCR
|
|
bfxil x0, x16, #0, #2
|
|
mrs x1, TPIDR2_EL0
|
|
1:
|
|
ret
|
|
END_COMPILERRT_OUTLINE_FUNCTION(__arm_sme_state)
|
|
|
|
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_restore)
|
|
.variant_pcs __arm_tpidr2_restore
|
|
// If TPIDR2_EL0 is nonnull, the subroutine aborts in some platform-specific
|
|
// manner.
|
|
mrs x14, TPIDR2_EL0
|
|
cbnz x14, 2f
|
|
|
|
// If any of the reserved bytes in the first 16 bytes of BLK are nonzero,
|
|
// the subroutine [..] aborts in some platform-defined manner.
|
|
ldrh w14, [x0, #10]
|
|
cbnz w14, 2f
|
|
ldr w14, [x0, #12]
|
|
cbnz w14, 2f
|
|
|
|
// If BLK.za_save_buffer is NULL, the subroutine does nothing.
|
|
ldr x16, [x0]
|
|
cbz x16, 1f
|
|
|
|
// If BLK.num_za_save_slices is zero, the subroutine does nothing.
|
|
ldrh w14, [x0, #8]
|
|
cbz x14, 1f
|
|
|
|
mov x15, xzr
|
|
0:
|
|
ldr za[w15,0], [x16]
|
|
addsvl x16, x16, #1
|
|
add x15, x15, #1
|
|
cmp x14, x15
|
|
b.ne 0b
|
|
1:
|
|
ret
|
|
2:
|
|
b SYMBOL_NAME(do_abort)
|
|
END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_restore)
|
|
|
|
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_tpidr2_save)
|
|
.variant_pcs __arm_tpidr2_restore
|
|
// If the current thread does not have access to TPIDR2_EL0, the subroutine
|
|
// does nothing.
|
|
adrp x14, TPIDR2_SYMBOL
|
|
ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
|
|
cbz w14, 1f
|
|
|
|
// If TPIDR2_EL0 is null, the subroutine does nothing.
|
|
mrs x16, TPIDR2_EL0
|
|
cbz x16, 1f
|
|
|
|
// If any of the reserved bytes in the first 16 bytes of the TPIDR2 block are
|
|
// nonzero, the subroutine [..] aborts in some platform-defined manner.
|
|
ldrh w14, [x16, #10]
|
|
cbnz w14, 2f
|
|
ldr w14, [x16, #12]
|
|
cbnz w14, 2f
|
|
|
|
// If num_za_save_slices is zero, the subroutine does nothing.
|
|
ldrh w14, [x16, #8]
|
|
cbz x14, 1f
|
|
|
|
// If za_save_buffer is NULL, the subroutine does nothing.
|
|
ldr x16, [x16]
|
|
cbz x16, 1f
|
|
|
|
mov x15, xzr
|
|
0:
|
|
str za[w15,0], [x16]
|
|
addsvl x16, x16, #1
|
|
add x15, x15, #1
|
|
cmp x14, x15
|
|
b.ne 0b
|
|
1:
|
|
ret
|
|
2:
|
|
b SYMBOL_NAME(do_abort)
|
|
END_COMPILERRT_OUTLINE_FUNCTION(__arm_tpidr2_save)
|
|
|
|
DEFINE_COMPILERRT_OUTLINE_FUNCTION_UNMANGLED(__arm_za_disable)
|
|
.variant_pcs __arm_tpidr2_restore
|
|
// If the current thread does not have access to SME, the subroutine does
|
|
// nothing.
|
|
adrp x14, TPIDR2_SYMBOL
|
|
ldrb w14, [x14, TPIDR2_SYMBOL_OFFSET]
|
|
cbz w14, 0f
|
|
|
|
// Otherwise, the subroutine behaves as if it did the following:
|
|
// * Call __arm_tpidr2_save.
|
|
stp x29, x30, [sp, #-16]!
|
|
.cfi_def_cfa_offset 16
|
|
mov x29, sp
|
|
.cfi_def_cfa w29, 16
|
|
.cfi_offset w30, -8
|
|
.cfi_offset w29, -16
|
|
bl __arm_tpidr2_save
|
|
|
|
// * Set TPIDR2_EL0 to null.
|
|
msr TPIDR2_EL0, xzr
|
|
|
|
// * Set PSTATE.ZA to 0.
|
|
smstop za
|
|
|
|
.cfi_def_cfa wsp, 16
|
|
ldp x29, x30, [sp], #16
|
|
.cfi_def_cfa_offset 0
|
|
.cfi_restore w30
|
|
.cfi_restore w29
|
|
0:
|
|
ret
|
|
END_COMPILERRT_OUTLINE_FUNCTION(__arm_za_disable)
|