; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --filter-out "(?!^\s*lda.*\bsp\b)^\s*.*\bsp\b" --filter "^\s*(ld|st[^r]|swp|cas|bl|add|and|eor|orn|orr|sub|mvn|sxt|cmp|ccmp|csel|dmb)"
; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo -global-isel=true -global-isel-abort=2 -O0 | FileCheck %s --check-prefixes=CHECK,GISEL
; RUN: llc %s -o - -verify-machineinstrs -mtriple=aarch64 -mattr=+v8.4a -mattr=+rcpc-immo -global-isel=false -O1 | FileCheck %s --check-prefixes=CHECK,SDAG

define i8 @load_atomic_i8_aligned_unordered(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_unordered:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep unordered, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_aligned_unordered_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_unordered_const:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep unordered, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_aligned_monotonic(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_monotonic:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep monotonic, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_aligned_monotonic_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_monotonic_const:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep monotonic, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_aligned_acquire(ptr %ptr) {
; GISEL-LABEL: load_atomic_i8_aligned_acquire:
; GISEL:    add x8, x0, #4
; GISEL:    ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_aligned_acquire:
; SDAG:    ldapurb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep acquire, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_aligned_acquire_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i8_aligned_acquire_const:
; GISEL:    add x8, x0, #4
; GISEL:    ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_aligned_acquire_const:
; SDAG:    ldapurb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep acquire, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_seq_cst:
; CHECK:    add x8, x0, #4
; CHECK:    ldarb w0, [x8]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep seq_cst, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_aligned_seq_cst_const:
; CHECK:    add x8, x0, #4
; CHECK:    ldarb w0, [x8]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep seq_cst, align 1
    ret i8 %r
}

define i16 @load_atomic_i16_aligned_unordered(ptr %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_unordered:
; CHECK:    ldrh w0, [x0, #8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep unordered, align 2
    ret i16 %r
}

define i16 @load_atomic_i16_aligned_unordered_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_unordered_const:
; CHECK:    ldrh w0, [x0, #8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep unordered, align 2
    ret i16 %r
}

define i16 @load_atomic_i16_aligned_monotonic(ptr %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_monotonic:
; CHECK:    ldrh w0, [x0, #8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep monotonic, align 2
    ret i16 %r
}

define i16 @load_atomic_i16_aligned_monotonic_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_monotonic_const:
; CHECK:    ldrh w0, [x0, #8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep monotonic, align 2
    ret i16 %r
}

define i16 @load_atomic_i16_aligned_acquire(ptr %ptr) {
; GISEL-LABEL: load_atomic_i16_aligned_acquire:
; GISEL:    add x8, x0, #8
; GISEL:    ldaprh w0, [x8]
;
; SDAG-LABEL: load_atomic_i16_aligned_acquire:
; SDAG:    ldapurh w0, [x0, #8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep acquire, align 2
    ret i16 %r
}

define i16 @load_atomic_i16_aligned_acquire_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i16_aligned_acquire_const:
; GISEL:    add x8, x0, #8
; GISEL:    ldaprh w0, [x8]
;
; SDAG-LABEL: load_atomic_i16_aligned_acquire_const:
; SDAG:    ldapurh w0, [x0, #8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep acquire, align 2
    ret i16 %r
}

define i16 @load_atomic_i16_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_seq_cst:
; CHECK:    add x8, x0, #8
; CHECK:    ldarh w0, [x8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep seq_cst, align 2
    ret i16 %r
}

define i16 @load_atomic_i16_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i16_aligned_seq_cst_const:
; CHECK:    add x8, x0, #8
; CHECK:    ldarh w0, [x8]
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep seq_cst, align 2
    ret i16 %r
}

define i32 @load_atomic_i32_aligned_unordered(ptr %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_unordered:
; CHECK:    ldr w0, [x0, #16]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep unordered, align 4
    ret i32 %r
}

define i32 @load_atomic_i32_aligned_unordered_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_unordered_const:
; CHECK:    ldr w0, [x0, #16]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep unordered, align 4
    ret i32 %r
}

define i32 @load_atomic_i32_aligned_monotonic(ptr %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_monotonic:
; CHECK:    ldr w0, [x0, #16]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep monotonic, align 4
    ret i32 %r
}

define i32 @load_atomic_i32_aligned_monotonic_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_monotonic_const:
; CHECK:    ldr w0, [x0, #16]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep monotonic, align 4
    ret i32 %r
}

define i32 @load_atomic_i32_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_acquire:
; CHECK:    ldapur w0, [x0, #16]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep acquire, align 4
    ret i32 %r
}

define i32 @load_atomic_i32_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_acquire_const:
; CHECK:    ldapur w0, [x0, #16]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep acquire, align 4
    ret i32 %r
}

define i32 @load_atomic_i32_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_seq_cst:
; CHECK:    add x8, x0, #16
; CHECK:    ldar w0, [x8]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep seq_cst, align 4
    ret i32 %r
}

define i32 @load_atomic_i32_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i32_aligned_seq_cst_const:
; CHECK:    add x8, x0, #16
; CHECK:    ldar w0, [x8]
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep seq_cst, align 4
    ret i32 %r
}

define i64 @load_atomic_i64_aligned_unordered(ptr %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_unordered:
; CHECK:    ldr x0, [x0, #32]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep unordered, align 8
    ret i64 %r
}

define i64 @load_atomic_i64_aligned_unordered_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_unordered_const:
; CHECK:    ldr x0, [x0, #32]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep unordered, align 8
    ret i64 %r
}

define i64 @load_atomic_i64_aligned_monotonic(ptr %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_monotonic:
; CHECK:    ldr x0, [x0, #32]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep monotonic, align 8
    ret i64 %r
}

define i64 @load_atomic_i64_aligned_monotonic_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_monotonic_const:
; CHECK:    ldr x0, [x0, #32]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep monotonic, align 8
    ret i64 %r
}

define i64 @load_atomic_i64_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_acquire:
; CHECK:    ldapur x0, [x0, #32]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep acquire, align 8
    ret i64 %r
}

define i64 @load_atomic_i64_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_acquire_const:
; CHECK:    ldapur x0, [x0, #32]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep acquire, align 8
    ret i64 %r
}

define i64 @load_atomic_i64_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_seq_cst:
; CHECK:    add x8, x0, #32
; CHECK:    ldar x0, [x8]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep seq_cst, align 8
    ret i64 %r
}

define i64 @load_atomic_i64_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i64_aligned_seq_cst_const:
; CHECK:    add x8, x0, #32
; CHECK:    ldar x0, [x8]
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep seq_cst, align 8
    ret i64 %r
}

define i128 @load_atomic_i128_aligned_unordered(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_unordered:
; CHECK:    ldp x0, x1, [x0, #64]
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep unordered, align 16
    ret i128 %r
}

define i128 @load_atomic_i128_aligned_unordered_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_unordered_const:
; CHECK:    ldp x0, x1, [x0, #64]
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep unordered, align 16
    ret i128 %r
}

define i128 @load_atomic_i128_aligned_monotonic(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_monotonic:
; CHECK:    ldp x0, x1, [x0, #64]
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep monotonic, align 16
    ret i128 %r
}

define i128 @load_atomic_i128_aligned_monotonic_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_monotonic_const:
; CHECK:    ldp x0, x1, [x0, #64]
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep monotonic, align 16
    ret i128 %r
}

define i128 @load_atomic_i128_aligned_acquire(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_acquire:
; CHECK:    ldp x0, x1, [x0, #64]
; CHECK:    dmb ishld
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep acquire, align 16
    ret i128 %r
}

define i128 @load_atomic_i128_aligned_acquire_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_acquire_const:
; CHECK:    ldp x0, x1, [x0, #64]
; CHECK:    dmb ishld
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep acquire, align 16
    ret i128 %r
}

define i128 @load_atomic_i128_aligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst:
; CHECK:    ldp x0, x1, [x0, #64]
; CHECK:    dmb ish
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep seq_cst, align 16
    ret i128 %r
}

define i128 @load_atomic_i128_aligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i128_aligned_seq_cst_const:
; CHECK:    ldp x0, x1, [x0, #64]
; CHECK:    dmb ish
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep seq_cst, align 16
    ret i128 %r
}

define i8 @load_atomic_i8_unaligned_unordered(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_unordered:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep unordered, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_unaligned_unordered_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_unordered_const:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep unordered, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_unaligned_monotonic(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_monotonic:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep monotonic, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_unaligned_monotonic_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_monotonic_const:
; CHECK:    ldrb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep monotonic, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_unaligned_acquire(ptr %ptr) {
; GISEL-LABEL: load_atomic_i8_unaligned_acquire:
; GISEL:    add x8, x0, #4
; GISEL:    ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_unaligned_acquire:
; SDAG:    ldapurb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep acquire, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_unaligned_acquire_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i8_unaligned_acquire_const:
; GISEL:    add x8, x0, #4
; GISEL:    ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_unaligned_acquire_const:
; SDAG:    ldapurb w0, [x0, #4]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep acquire, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_unaligned_seq_cst(ptr %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_seq_cst:
; CHECK:    add x8, x0, #4
; CHECK:    ldarb w0, [x8]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep seq_cst, align 1
    ret i8 %r
}

define i8 @load_atomic_i8_unaligned_seq_cst_const(ptr readonly %ptr) {
; CHECK-LABEL: load_atomic_i8_unaligned_seq_cst_const:
; CHECK:    add x8, x0, #4
; CHECK:    ldarb w0, [x8]
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i8, ptr %gep seq_cst, align 1
    ret i8 %r
}

define i16 @load_atomic_i16_unaligned_unordered(ptr %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_unordered:
; GISEL:    add x1, x8, #4
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_unordered:
; SDAG:    add x1, x0, #4
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep unordered, align 1
    ret i16 %r
}

define i16 @load_atomic_i16_unaligned_unordered_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_unordered_const:
; GISEL:    add x1, x8, #4
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_unordered_const:
; SDAG:    add x1, x0, #4
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i8, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep unordered, align 1
    ret i16 %r
}

define i16 @load_atomic_i16_unaligned_monotonic(ptr %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_monotonic:
; GISEL:    add x1, x8, #8
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_monotonic:
; SDAG:    add x1, x0, #8
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep monotonic, align 1
    ret i16 %r
}

define i16 @load_atomic_i16_unaligned_monotonic_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_monotonic_const:
; GISEL:    add x1, x8, #8
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_monotonic_const:
; SDAG:    add x1, x0, #8
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep monotonic, align 1
    ret i16 %r
}

define i16 @load_atomic_i16_unaligned_acquire(ptr %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_acquire:
; GISEL:    add x1, x8, #8
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_acquire:
; SDAG:    add x1, x0, #8
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep acquire, align 1
    ret i16 %r
}

define i16 @load_atomic_i16_unaligned_acquire_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_acquire_const:
; GISEL:    add x1, x8, #8
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_acquire_const:
; SDAG:    add x1, x0, #8
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep acquire, align 1
    ret i16 %r
}

define i16 @load_atomic_i16_unaligned_seq_cst(ptr %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_seq_cst:
; GISEL:    add x1, x8, #8
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_seq_cst:
; SDAG:    add x1, x0, #8
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep seq_cst, align 1
    ret i16 %r
}

define i16 @load_atomic_i16_unaligned_seq_cst_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i16_unaligned_seq_cst_const:
; GISEL:    add x1, x8, #8
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i16_unaligned_seq_cst_const:
; SDAG:    add x1, x0, #8
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i16, ptr %ptr, i32 4
    %r = load atomic i16, ptr %gep seq_cst, align 1
    ret i16 %r
}

define i32 @load_atomic_i32_unaligned_unordered(ptr %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_unordered:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_unordered:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep unordered, align 1
    ret i32 %r
}

define i32 @load_atomic_i32_unaligned_unordered_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_unordered_const:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_unordered_const:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep unordered, align 1
    ret i32 %r
}

define i32 @load_atomic_i32_unaligned_monotonic(ptr %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_monotonic:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_monotonic:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep monotonic, align 1
    ret i32 %r
}

define i32 @load_atomic_i32_unaligned_monotonic_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_monotonic_const:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_monotonic_const:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep monotonic, align 1
    ret i32 %r
}

define i32 @load_atomic_i32_unaligned_acquire(ptr %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_acquire:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_acquire:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep acquire, align 1
    ret i32 %r
}

define i32 @load_atomic_i32_unaligned_acquire_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_acquire_const:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_acquire_const:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep acquire, align 1
    ret i32 %r
}

define i32 @load_atomic_i32_unaligned_seq_cst(ptr %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_seq_cst:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_seq_cst:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep seq_cst, align 1
    ret i32 %r
}

define i32 @load_atomic_i32_unaligned_seq_cst_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i32_unaligned_seq_cst_const:
; GISEL:    add x1, x8, #16
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i32_unaligned_seq_cst_const:
; SDAG:    add x1, x0, #16
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i32, ptr %ptr, i32 4
    %r = load atomic i32, ptr %gep seq_cst, align 1
    ret i32 %r
}

define i64 @load_atomic_i64_unaligned_unordered(ptr %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_unordered:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_unordered:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep unordered, align 1
    ret i64 %r
}

define i64 @load_atomic_i64_unaligned_unordered_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_unordered_const:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_unordered_const:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep unordered, align 1
    ret i64 %r
}

define i64 @load_atomic_i64_unaligned_monotonic(ptr %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_monotonic:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_monotonic:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep monotonic, align 1
    ret i64 %r
}

define i64 @load_atomic_i64_unaligned_monotonic_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_monotonic_const:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_monotonic_const:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep monotonic, align 1
    ret i64 %r
}

define i64 @load_atomic_i64_unaligned_acquire(ptr %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_acquire:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_acquire:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep acquire, align 1
    ret i64 %r
}

define i64 @load_atomic_i64_unaligned_acquire_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_acquire_const:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_acquire_const:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep acquire, align 1
    ret i64 %r
}

define i64 @load_atomic_i64_unaligned_seq_cst(ptr %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_seq_cst:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_seq_cst:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep seq_cst, align 1
    ret i64 %r
}

define i64 @load_atomic_i64_unaligned_seq_cst_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i64_unaligned_seq_cst_const:
; GISEL:    add x1, x8, #32
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i64_unaligned_seq_cst_const:
; SDAG:    add x1, x0, #32
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i64, ptr %ptr, i32 4
    %r = load atomic i64, ptr %gep seq_cst, align 1
    ret i64 %r
}

define i128 @load_atomic_i128_unaligned_unordered(ptr %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_unordered:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_unordered:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep unordered, align 1
    ret i128 %r
}

define i128 @load_atomic_i128_unaligned_unordered_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_unordered_const:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_unordered_const:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep unordered, align 1
    ret i128 %r
}

define i128 @load_atomic_i128_unaligned_monotonic(ptr %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_monotonic:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_monotonic:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep monotonic, align 1
    ret i128 %r
}

define i128 @load_atomic_i128_unaligned_monotonic_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_monotonic_const:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_monotonic_const:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep monotonic, align 1
    ret i128 %r
}

define i128 @load_atomic_i128_unaligned_acquire(ptr %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_acquire:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_acquire:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep acquire, align 1
    ret i128 %r
}

define i128 @load_atomic_i128_unaligned_acquire_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_acquire_const:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_acquire_const:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep acquire, align 1
    ret i128 %r
}

define i128 @load_atomic_i128_unaligned_seq_cst(ptr %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_seq_cst:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_seq_cst:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep seq_cst, align 1
    ret i128 %r
}

define i128 @load_atomic_i128_unaligned_seq_cst_const(ptr readonly %ptr) {
; GISEL-LABEL: load_atomic_i128_unaligned_seq_cst_const:
; GISEL:    add x1, x8, #64
; GISEL:    bl __atomic_load
;
; SDAG-LABEL: load_atomic_i128_unaligned_seq_cst_const:
; SDAG:    add x1, x0, #64
; SDAG:    bl __atomic_load
    %gep = getelementptr inbounds i128, ptr %ptr, i32 4
    %r = load atomic i128, ptr %gep seq_cst, align 1
    ret i128 %r
}

define i8 @load_atomic_i8_from_gep() {
; GISEL-LABEL: load_atomic_i8_from_gep:
; GISEL:    bl init
; GISEL:    add x8, x8, #1
; GISEL:    ldaprb w0, [x8]
;
; SDAG-LABEL: load_atomic_i8_from_gep:
; SDAG:    bl init
; SDAG:    ldapurb w0, [sp, #13]
  %a = alloca [3 x i8]
  call void @init(ptr %a)
  %arrayidx  = getelementptr [3 x i8], ptr %a, i64 0, i64 1
  %l = load atomic i8, ptr %arrayidx acquire, align 8
  ret i8 %l
}

define i16 @load_atomic_i16_from_gep() {
; GISEL-LABEL: load_atomic_i16_from_gep:
; GISEL:    bl init
; GISEL:    add x8, x8, #2
; GISEL:    ldaprh w0, [x8]
;
; SDAG-LABEL: load_atomic_i16_from_gep:
; SDAG:    bl init
; SDAG:    ldapurh w0, [sp, #10]
  %a = alloca [3 x i16]
  call void @init(ptr %a)
  %arrayidx  = getelementptr [3 x i16], ptr %a, i64 0, i64 1
  %l = load atomic i16, ptr %arrayidx acquire, align 8
  ret i16 %l
}

define i32 @load_atomic_i32_from_gep() {
; GISEL-LABEL: load_atomic_i32_from_gep:
; GISEL:    bl init
; GISEL:    ldapur w0, [x8, #4]
;
; SDAG-LABEL: load_atomic_i32_from_gep:
; SDAG:    bl init
; SDAG:    ldapur w0, [sp, #8]
  %a = alloca [3 x i32]
  call void @init(ptr %a)
  %arrayidx  = getelementptr [3 x i32], ptr %a, i64 0, i64 1
  %l = load atomic i32, ptr %arrayidx acquire, align 8
  ret i32 %l
}

define i64 @load_atomic_i64_from_gep() {
; GISEL-LABEL: load_atomic_i64_from_gep:
; GISEL:    bl init
; GISEL:    ldapur x0, [x8, #8]
;
; SDAG-LABEL: load_atomic_i64_from_gep:
; SDAG:    bl init
; SDAG:    ldapur x0, [sp, #16]
  %a = alloca [3 x i64]
  call void @init(ptr %a)
  %arrayidx  = getelementptr [3 x i64], ptr %a, i64 0, i64 1
  %l = load atomic i64, ptr %arrayidx acquire, align 8
  ret i64 %l
}

define i128 @load_atomic_i128_from_gep() {
; GISEL-LABEL: load_atomic_i128_from_gep:
; GISEL:    bl init
; GISEL:    ldp x0, x1, [x8, #16]
; GISEL:    dmb ishld
;
; SDAG-LABEL: load_atomic_i128_from_gep:
; SDAG:    bl init
; SDAG:    dmb ishld
  %a = alloca [3 x i128]
  call void @init(ptr %a)
  %arrayidx  = getelementptr [3 x i128], ptr %a, i64 0, i64 1
  %l = load atomic i128, ptr %arrayidx acquire, align 16
  ret i128 %l
}

declare void @init(ptr)