; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
;
; Test that atomic ops in IR marked with !pcsections end up in a PC section in
; asm. All functions start with a non-atomic access, continue with an atomic
; access, and end with another non-atomic access; this is to test that the
; !pcsections propagation doesn't accidentally touch adjacent instructions.
;
; RUN: llc -O0 -mattr=cx16 < %s | FileCheck %s --check-prefixes=O0
; RUN: llc -O1 -mattr=cx16 < %s | FileCheck %s --check-prefixes=O1
; RUN: llc -O2 -mattr=cx16 < %s | FileCheck %s --check-prefixes=O2
; RUN: llc -O3 -mattr=cx16 < %s | FileCheck %s --check-prefixes=O3

target triple = "x86_64-unknown-linux-gnu"

@foo = dso_local global i64 0, align 8

define void @mixed_atomic_non_atomic(ptr %a) {
; O0-LABEL: mixed_atomic_non_atomic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    incl %eax
; O0-NEXT:    movl %eax, (%rdi)
; O0-NEXT:  .Lpcsection0:
; O0-NEXT:    movl $1, (%rdi)
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    addl $-1, %eax
; O0-NEXT:    movl %eax, (%rdi)
; O0-NEXT:    retq
;
; O1-LABEL: mixed_atomic_non_atomic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    incl (%rdi)
; O1-NEXT:  .Lpcsection0:
; O1-NEXT:    movl $1, (%rdi)
; O1-NEXT:    decl (%rdi)
; O1-NEXT:    retq
;
; O2-LABEL: mixed_atomic_non_atomic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    incl (%rdi)
; O2-NEXT:  .Lpcsection0:
; O2-NEXT:    movl $1, (%rdi)
; O2-NEXT:    decl (%rdi)
; O2-NEXT:    retq
;
; O3-LABEL: mixed_atomic_non_atomic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    incl (%rdi)
; O3-NEXT:  .Lpcsection0:
; O3-NEXT:    movl $1, (%rdi)
; O3-NEXT:    decl (%rdi)
; O3-NEXT:    retq
entry:
  ; Accesses the same location atomically and non-atomically.
  %0 = load volatile i32, ptr %a, align 4
  %inc = add nsw i32 %0, 1
  store volatile i32 %inc, ptr %a, align 4
  store atomic volatile i32 1, ptr %a monotonic, align 4, !pcsections !0
  %1 = load volatile i32, ptr %a, align 4
  %dec = add nsw i32 %1, -1
  store volatile i32 %dec, ptr %a, align 4
  ret void
}

define i64 @mixed_complex_atomic_non_atomic(ptr %a, ptr %b) {
; O0-LABEL: mixed_complex_atomic_non_atomic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection1:
; O0-NEXT:    lock xaddq %rcx, (%rdi)
; O0-NEXT:    movq (%rsi), %rax
; O0-NEXT:    movq %rax, %rdx
; O0-NEXT:    addq $1, %rdx
; O0-NEXT:    movq %rdx, (%rsi)
; O0-NEXT:    addq %rcx, %rax
; O0-NEXT:    retq
;
; O1-LABEL: mixed_complex_atomic_non_atomic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movl $1, %eax
; O1-NEXT:  .Lpcsection1:
; O1-NEXT:    lock xaddq %rax, (%rdi)
; O1-NEXT:    movq (%rsi), %rcx
; O1-NEXT:    leaq 1(%rcx), %rdx
; O1-NEXT:    movq %rdx, (%rsi)
; O1-NEXT:    addq %rcx, %rax
; O1-NEXT:    retq
;
; O2-LABEL: mixed_complex_atomic_non_atomic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movl $1, %eax
; O2-NEXT:  .Lpcsection1:
; O2-NEXT:    lock xaddq %rax, (%rdi)
; O2-NEXT:    movq (%rsi), %rcx
; O2-NEXT:    leaq 1(%rcx), %rdx
; O2-NEXT:    movq %rdx, (%rsi)
; O2-NEXT:    addq %rcx, %rax
; O2-NEXT:    retq
;
; O3-LABEL: mixed_complex_atomic_non_atomic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movl $1, %eax
; O3-NEXT:  .Lpcsection1:
; O3-NEXT:    lock xaddq %rax, (%rdi)
; O3-NEXT:    movq (%rsi), %rcx
; O3-NEXT:    leaq 1(%rcx), %rdx
; O3-NEXT:    movq %rdx, (%rsi)
; O3-NEXT:    addq %rcx, %rax
; O3-NEXT:    retq
entry:
  %0 = atomicrmw add ptr %a, i64 1 monotonic, align 8, !pcsections !0
  %1 = load i64, ptr %b, align 8
  %inc = add nsw i64 %1, 1
  store i64 %inc, ptr %b, align 8
  %add = add nsw i64 %1, %0
  ret i64 %add
}

define i8 @atomic8_load_unordered(ptr %a) {
; O0-LABEL: atomic8_load_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection2:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_load_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection2:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_load_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection2:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_load_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection2:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i8, ptr %a unordered, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i8 %x
}

define i8 @atomic8_load_monotonic(ptr %a) {
; O0-LABEL: atomic8_load_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection3:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_load_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection3:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_load_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection3:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_load_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection3:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i8, ptr %a monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i8 %x
}

define i8 @atomic8_load_acquire(ptr %a) {
; O0-LABEL: atomic8_load_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection4:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_load_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection4:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_load_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection4:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_load_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection4:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i8, ptr %a acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i8 %x
}

define i8 @atomic8_load_seq_cst(ptr %a) {
; O0-LABEL: atomic8_load_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection5:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_load_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection5:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_load_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection5:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_load_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection5:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i8, ptr %a seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i8 %x
}

define void @atomic8_store_unordered(ptr %a) {
; O0-LABEL: atomic8_store_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection6:
; O0-NEXT:    movb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_store_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection6:
; O1-NEXT:    movb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_store_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection6:
; O2-NEXT:    movb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_store_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection6:
; O3-NEXT:    movb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i8 42, ptr %a unordered, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_store_monotonic(ptr %a) {
; O0-LABEL: atomic8_store_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection7:
; O0-NEXT:    movb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_store_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection7:
; O1-NEXT:    movb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_store_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection7:
; O2-NEXT:    movb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_store_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection7:
; O3-NEXT:    movb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i8 42, ptr %a monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_store_release(ptr %a) {
; O0-LABEL: atomic8_store_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection8:
; O0-NEXT:    movb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_store_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection8:
; O1-NEXT:    movb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_store_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection8:
; O2-NEXT:    movb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_store_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection8:
; O3-NEXT:    movb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i8 42, ptr %a release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_store_seq_cst(ptr %a) {
; O0-LABEL: atomic8_store_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:  .Lpcsection9:
; O0-NEXT:    xchgb %al, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_store_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection9:
; O1-NEXT:    xchgb %al, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_store_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection9:
; O2-NEXT:    xchgb %al, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_store_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection9:
; O3-NEXT:    xchgb %al, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i8 42, ptr %a seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xchg_monotonic(ptr %a) {
; O0-LABEL: atomic8_xchg_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:  .Lpcsection10:
; O0-NEXT:    xchgb %al, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xchg_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection10:
; O1-NEXT:    xchgb %al, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xchg_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection10:
; O2-NEXT:    xchgb %al, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xchg_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection10:
; O3-NEXT:    xchgb %al, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i8 42 monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_add_monotonic(ptr %a) {
; O0-LABEL: atomic8_add_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection11:
; O0-NEXT:    lock addb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_add_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection11:
; O1-NEXT:    lock addb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_add_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection11:
; O2-NEXT:    lock addb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_add_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection11:
; O3-NEXT:    lock addb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i8 42 monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_sub_monotonic(ptr %a) {
; O0-LABEL: atomic8_sub_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection12:
; O0-NEXT:    lock subb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_sub_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection12:
; O1-NEXT:    lock subb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_sub_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection12:
; O2-NEXT:    lock subb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_sub_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection12:
; O3-NEXT:    lock subb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i8 42 monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_and_monotonic(ptr %a) {
; O0-LABEL: atomic8_and_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection13:
; O0-NEXT:    lock andb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_and_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection13:
; O1-NEXT:    lock andb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_and_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection13:
; O2-NEXT:    lock andb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_and_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection13:
; O3-NEXT:    lock andb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i8 42 monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_or_monotonic(ptr %a) {
; O0-LABEL: atomic8_or_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection14:
; O0-NEXT:    lock orb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_or_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection14:
; O1-NEXT:    lock orb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_or_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection14:
; O2-NEXT:    lock orb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_or_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection14:
; O3-NEXT:    lock orb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i8 42 monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xor_monotonic(ptr %a) {
; O0-LABEL: atomic8_xor_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection15:
; O0-NEXT:    lock xorb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xor_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection15:
; O1-NEXT:    lock xorb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xor_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection15:
; O2-NEXT:    lock xorb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xor_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection15:
; O3-NEXT:    lock xorb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i8 42 monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_nand_monotonic(ptr %a) {
; O0-LABEL: atomic8_nand_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection16:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .LBB16_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movb %al, %dl
; O0-NEXT:  .Lpcsection17:
; O0-NEXT:    notb %dl
; O0-NEXT:  .Lpcsection18:
; O0-NEXT:    orb $-43, %dl
; O0-NEXT:  .Lpcsection19:
; O0-NEXT:    lock cmpxchgb %dl, (%rcx)
; O0-NEXT:  .Lpcsection20:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection21:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .Lpcsection22:
; O0-NEXT:    jne .LBB16_2
; O0-NEXT:    jmp .LBB16_1
; O0-NEXT:  .LBB16_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_nand_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection16:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB16_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection17:
; O1-NEXT:    notb %cl
; O1-NEXT:  .Lpcsection18:
; O1-NEXT:    orb $-43, %cl
; O1-NEXT:  .Lpcsection19:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection20:
; O1-NEXT:    jne .LBB16_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_nand_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection16:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB16_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection17:
; O2-NEXT:    notb %cl
; O2-NEXT:  .Lpcsection18:
; O2-NEXT:    orb $-43, %cl
; O2-NEXT:  .Lpcsection19:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection20:
; O2-NEXT:    jne .LBB16_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_nand_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection16:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB16_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection17:
; O3-NEXT:    notb %cl
; O3-NEXT:  .Lpcsection18:
; O3-NEXT:    orb $-43, %cl
; O3-NEXT:  .Lpcsection19:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection20:
; O3-NEXT:    jne .LBB16_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i8 42 monotonic, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xchg_acquire(ptr %a) {
; O0-LABEL: atomic8_xchg_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:  .Lpcsection23:
; O0-NEXT:    xchgb %al, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xchg_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection21:
; O1-NEXT:    xchgb %al, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xchg_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection21:
; O2-NEXT:    xchgb %al, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xchg_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection21:
; O3-NEXT:    xchgb %al, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i8 42 acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_add_acquire(ptr %a) {
; O0-LABEL: atomic8_add_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection24:
; O0-NEXT:    lock addb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_add_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection22:
; O1-NEXT:    lock addb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_add_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection22:
; O2-NEXT:    lock addb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_add_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection22:
; O3-NEXT:    lock addb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i8 42 acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_sub_acquire(ptr %a) {
; O0-LABEL: atomic8_sub_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection25:
; O0-NEXT:    lock subb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_sub_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection23:
; O1-NEXT:    lock subb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_sub_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection23:
; O2-NEXT:    lock subb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_sub_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection23:
; O3-NEXT:    lock subb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i8 42 acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_and_acquire(ptr %a) {
; O0-LABEL: atomic8_and_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection26:
; O0-NEXT:    lock andb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_and_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection24:
; O1-NEXT:    lock andb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_and_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection24:
; O2-NEXT:    lock andb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_and_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection24:
; O3-NEXT:    lock andb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i8 42 acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_or_acquire(ptr %a) {
; O0-LABEL: atomic8_or_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection27:
; O0-NEXT:    lock orb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_or_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection25:
; O1-NEXT:    lock orb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_or_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection25:
; O2-NEXT:    lock orb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_or_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection25:
; O3-NEXT:    lock orb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i8 42 acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xor_acquire(ptr %a) {
; O0-LABEL: atomic8_xor_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection28:
; O0-NEXT:    lock xorb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xor_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection26:
; O1-NEXT:    lock xorb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xor_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection26:
; O2-NEXT:    lock xorb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xor_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection26:
; O3-NEXT:    lock xorb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i8 42 acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_nand_acquire(ptr %a) {
; O0-LABEL: atomic8_nand_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection29:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .LBB23_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movb %al, %dl
; O0-NEXT:  .Lpcsection30:
; O0-NEXT:    notb %dl
; O0-NEXT:  .Lpcsection31:
; O0-NEXT:    orb $-43, %dl
; O0-NEXT:  .Lpcsection32:
; O0-NEXT:    lock cmpxchgb %dl, (%rcx)
; O0-NEXT:  .Lpcsection33:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection34:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .Lpcsection35:
; O0-NEXT:    jne .LBB23_2
; O0-NEXT:    jmp .LBB23_1
; O0-NEXT:  .LBB23_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_nand_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection27:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB23_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection28:
; O1-NEXT:    notb %cl
; O1-NEXT:  .Lpcsection29:
; O1-NEXT:    orb $-43, %cl
; O1-NEXT:  .Lpcsection30:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection31:
; O1-NEXT:    jne .LBB23_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_nand_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection27:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB23_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection28:
; O2-NEXT:    notb %cl
; O2-NEXT:  .Lpcsection29:
; O2-NEXT:    orb $-43, %cl
; O2-NEXT:  .Lpcsection30:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection31:
; O2-NEXT:    jne .LBB23_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_nand_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection27:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB23_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection28:
; O3-NEXT:    notb %cl
; O3-NEXT:  .Lpcsection29:
; O3-NEXT:    orb $-43, %cl
; O3-NEXT:  .Lpcsection30:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection31:
; O3-NEXT:    jne .LBB23_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i8 42 acquire, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xchg_release(ptr %a) {
; O0-LABEL: atomic8_xchg_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:  .Lpcsection36:
; O0-NEXT:    xchgb %al, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xchg_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection32:
; O1-NEXT:    xchgb %al, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xchg_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection32:
; O2-NEXT:    xchgb %al, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xchg_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection32:
; O3-NEXT:    xchgb %al, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i8 42 release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_add_release(ptr %a) {
; O0-LABEL: atomic8_add_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection37:
; O0-NEXT:    lock addb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_add_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection33:
; O1-NEXT:    lock addb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_add_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection33:
; O2-NEXT:    lock addb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_add_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection33:
; O3-NEXT:    lock addb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i8 42 release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_sub_release(ptr %a) {
; O0-LABEL: atomic8_sub_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection38:
; O0-NEXT:    lock subb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_sub_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection34:
; O1-NEXT:    lock subb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_sub_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection34:
; O2-NEXT:    lock subb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_sub_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection34:
; O3-NEXT:    lock subb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i8 42 release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_and_release(ptr %a) {
; O0-LABEL: atomic8_and_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection39:
; O0-NEXT:    lock andb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_and_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection35:
; O1-NEXT:    lock andb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_and_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection35:
; O2-NEXT:    lock andb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_and_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection35:
; O3-NEXT:    lock andb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i8 42 release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_or_release(ptr %a) {
; O0-LABEL: atomic8_or_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection40:
; O0-NEXT:    lock orb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_or_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection36:
; O1-NEXT:    lock orb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_or_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection36:
; O2-NEXT:    lock orb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_or_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection36:
; O3-NEXT:    lock orb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i8 42 release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xor_release(ptr %a) {
; O0-LABEL: atomic8_xor_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection41:
; O0-NEXT:    lock xorb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xor_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection37:
; O1-NEXT:    lock xorb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xor_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection37:
; O2-NEXT:    lock xorb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xor_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection37:
; O3-NEXT:    lock xorb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i8 42 release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_nand_release(ptr %a) {
; O0-LABEL: atomic8_nand_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection42:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .LBB30_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movb %al, %dl
; O0-NEXT:  .Lpcsection43:
; O0-NEXT:    notb %dl
; O0-NEXT:  .Lpcsection44:
; O0-NEXT:    orb $-43, %dl
; O0-NEXT:  .Lpcsection45:
; O0-NEXT:    lock cmpxchgb %dl, (%rcx)
; O0-NEXT:  .Lpcsection46:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection47:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .Lpcsection48:
; O0-NEXT:    jne .LBB30_2
; O0-NEXT:    jmp .LBB30_1
; O0-NEXT:  .LBB30_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_nand_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection38:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB30_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection39:
; O1-NEXT:    notb %cl
; O1-NEXT:  .Lpcsection40:
; O1-NEXT:    orb $-43, %cl
; O1-NEXT:  .Lpcsection41:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection42:
; O1-NEXT:    jne .LBB30_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_nand_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection38:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB30_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection39:
; O2-NEXT:    notb %cl
; O2-NEXT:  .Lpcsection40:
; O2-NEXT:    orb $-43, %cl
; O2-NEXT:  .Lpcsection41:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection42:
; O2-NEXT:    jne .LBB30_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_nand_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection38:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB30_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection39:
; O3-NEXT:    notb %cl
; O3-NEXT:  .Lpcsection40:
; O3-NEXT:    orb $-43, %cl
; O3-NEXT:  .Lpcsection41:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection42:
; O3-NEXT:    jne .LBB30_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i8 42 release, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xchg_acq_rel(ptr %a) {
; O0-LABEL: atomic8_xchg_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:  .Lpcsection49:
; O0-NEXT:    xchgb %al, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xchg_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection43:
; O1-NEXT:    xchgb %al, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xchg_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection43:
; O2-NEXT:    xchgb %al, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xchg_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection43:
; O3-NEXT:    xchgb %al, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i8 42 acq_rel, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_add_acq_rel(ptr %a) {
; O0-LABEL: atomic8_add_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection50:
; O0-NEXT:    lock addb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_add_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection44:
; O1-NEXT:    lock addb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_add_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection44:
; O2-NEXT:    lock addb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_add_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection44:
; O3-NEXT:    lock addb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i8 42 acq_rel, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_sub_acq_rel(ptr %a) {
; O0-LABEL: atomic8_sub_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection51:
; O0-NEXT:    lock subb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_sub_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection45:
; O1-NEXT:    lock subb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_sub_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection45:
; O2-NEXT:    lock subb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_sub_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection45:
; O3-NEXT:    lock subb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i8 42 acq_rel, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_and_acq_rel(ptr %a) {
; O0-LABEL: atomic8_and_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection52:
; O0-NEXT:    lock andb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_and_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection46:
; O1-NEXT:    lock andb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_and_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection46:
; O2-NEXT:    lock andb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_and_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection46:
; O3-NEXT:    lock andb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i8 42 acq_rel, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_or_acq_rel(ptr %a) {
; O0-LABEL: atomic8_or_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection53:
; O0-NEXT:    lock orb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_or_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection47:
; O1-NEXT:    lock orb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_or_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection47:
; O2-NEXT:    lock orb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_or_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection47:
; O3-NEXT:    lock orb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i8 42 acq_rel, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xor_acq_rel(ptr %a) {
; O0-LABEL: atomic8_xor_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection54:
; O0-NEXT:    lock xorb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xor_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection48:
; O1-NEXT:    lock xorb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xor_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection48:
; O2-NEXT:    lock xorb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xor_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection48:
; O3-NEXT:    lock xorb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i8 42 acq_rel, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_nand_acq_rel(ptr %a) {
; O0-LABEL: atomic8_nand_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection55:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .LBB37_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movb %al, %dl
; O0-NEXT:  .Lpcsection56:
; O0-NEXT:    notb %dl
; O0-NEXT:  .Lpcsection57:
; O0-NEXT:    orb $-43, %dl
; O0-NEXT:  .Lpcsection58:
; O0-NEXT:    lock cmpxchgb %dl, (%rcx)
; O0-NEXT:  .Lpcsection59:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection60:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .Lpcsection61:
; O0-NEXT:    jne .LBB37_2
; O0-NEXT:    jmp .LBB37_1
; O0-NEXT:  .LBB37_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_nand_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection49:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB37_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection50:
; O1-NEXT:    notb %cl
; O1-NEXT:  .Lpcsection51:
; O1-NEXT:    orb $-43, %cl
; O1-NEXT:  .Lpcsection52:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection53:
; O1-NEXT:    jne .LBB37_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_nand_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection49:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB37_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection50:
; O2-NEXT:    notb %cl
; O2-NEXT:  .Lpcsection51:
; O2-NEXT:    orb $-43, %cl
; O2-NEXT:  .Lpcsection52:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection53:
; O2-NEXT:    jne .LBB37_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_nand_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection49:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB37_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection50:
; O3-NEXT:    notb %cl
; O3-NEXT:  .Lpcsection51:
; O3-NEXT:    orb $-43, %cl
; O3-NEXT:  .Lpcsection52:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection53:
; O3-NEXT:    jne .LBB37_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i8 42 acq_rel, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xchg_seq_cst(ptr %a) {
; O0-LABEL: atomic8_xchg_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:  .Lpcsection62:
; O0-NEXT:    xchgb %al, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xchg_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection54:
; O1-NEXT:    xchgb %al, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xchg_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection54:
; O2-NEXT:    xchgb %al, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xchg_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection54:
; O3-NEXT:    xchgb %al, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i8 42 seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_add_seq_cst(ptr %a) {
; O0-LABEL: atomic8_add_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection63:
; O0-NEXT:    lock addb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_add_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection55:
; O1-NEXT:    lock addb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_add_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection55:
; O2-NEXT:    lock addb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_add_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection55:
; O3-NEXT:    lock addb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i8 42 seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_sub_seq_cst(ptr %a) {
; O0-LABEL: atomic8_sub_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection64:
; O0-NEXT:    lock subb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_sub_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection56:
; O1-NEXT:    lock subb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_sub_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection56:
; O2-NEXT:    lock subb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_sub_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection56:
; O3-NEXT:    lock subb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i8 42 seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_and_seq_cst(ptr %a) {
; O0-LABEL: atomic8_and_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection65:
; O0-NEXT:    lock andb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_and_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection57:
; O1-NEXT:    lock andb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_and_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection57:
; O2-NEXT:    lock andb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_and_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection57:
; O3-NEXT:    lock andb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i8 42 seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_or_seq_cst(ptr %a) {
; O0-LABEL: atomic8_or_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection66:
; O0-NEXT:    lock orb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_or_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection58:
; O1-NEXT:    lock orb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_or_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection58:
; O2-NEXT:    lock orb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_or_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection58:
; O3-NEXT:    lock orb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i8 42 seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_xor_seq_cst(ptr %a) {
; O0-LABEL: atomic8_xor_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection67:
; O0-NEXT:    lock xorb $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_xor_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection59:
; O1-NEXT:    lock xorb $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_xor_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection59:
; O2-NEXT:    lock xorb $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_xor_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection59:
; O3-NEXT:    lock xorb $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i8 42 seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_nand_seq_cst(ptr %a) {
; O0-LABEL: atomic8_nand_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection68:
; O0-NEXT:    movb (%rdi), %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .LBB44_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movb %al, %dl
; O0-NEXT:  .Lpcsection69:
; O0-NEXT:    notb %dl
; O0-NEXT:  .Lpcsection70:
; O0-NEXT:    orb $-43, %dl
; O0-NEXT:  .Lpcsection71:
; O0-NEXT:    lock cmpxchgb %dl, (%rcx)
; O0-NEXT:  .Lpcsection72:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection73:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:  .Lpcsection74:
; O0-NEXT:    jne .LBB44_2
; O0-NEXT:    jmp .LBB44_1
; O0-NEXT:  .LBB44_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_nand_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection60:
; O1-NEXT:    movzbl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB44_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection61:
; O1-NEXT:    notb %cl
; O1-NEXT:  .Lpcsection62:
; O1-NEXT:    orb $-43, %cl
; O1-NEXT:  .Lpcsection63:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection64:
; O1-NEXT:    jne .LBB44_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_nand_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection60:
; O2-NEXT:    movzbl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB44_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection61:
; O2-NEXT:    notb %cl
; O2-NEXT:  .Lpcsection62:
; O2-NEXT:    orb $-43, %cl
; O2-NEXT:  .Lpcsection63:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection64:
; O2-NEXT:    jne .LBB44_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_nand_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection60:
; O3-NEXT:    movzbl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB44_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection61:
; O3-NEXT:    notb %cl
; O3-NEXT:  .Lpcsection62:
; O3-NEXT:    orb $-43, %cl
; O3-NEXT:  .Lpcsection63:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection64:
; O3-NEXT:    jne .LBB44_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i8 42 seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_cas_monotonic(ptr %a) {
; O0-LABEL: atomic8_cas_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:    movb $1, %cl
; O0-NEXT:  .Lpcsection75:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection76:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection77:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection78:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection79:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_cas_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $1, %cl
; O1-NEXT:  .Lpcsection65:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection66:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection67:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection68:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection69:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection70:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_cas_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $1, %cl
; O2-NEXT:  .Lpcsection65:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection66:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection67:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection68:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection69:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection70:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_cas_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $1, %cl
; O3-NEXT:  .Lpcsection65:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection66:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection67:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection68:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection69:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection70:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i8 42, i8 1 monotonic monotonic, align 1, !pcsections !0
  %y = cmpxchg ptr %a, i8 42, i8 1 monotonic acquire, align 1, !pcsections !0
  %z = cmpxchg ptr %a, i8 42, i8 1 monotonic seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_cas_acquire(ptr %a) {
; O0-LABEL: atomic8_cas_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:    movb $1, %cl
; O0-NEXT:  .Lpcsection80:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection81:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection82:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection83:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection84:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_cas_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $1, %cl
; O1-NEXT:  .Lpcsection71:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection72:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection73:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection74:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection75:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection76:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_cas_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $1, %cl
; O2-NEXT:  .Lpcsection71:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection72:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection73:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection74:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection75:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection76:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_cas_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $1, %cl
; O3-NEXT:  .Lpcsection71:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection72:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection73:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection74:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection75:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection76:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i8 42, i8 1 acquire monotonic, align 1, !pcsections !0
  %y = cmpxchg ptr %a, i8 42, i8 1 acquire acquire, align 1, !pcsections !0
  %z = cmpxchg ptr %a, i8 42, i8 1 acquire seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_cas_release(ptr %a) {
; O0-LABEL: atomic8_cas_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:    movb $1, %cl
; O0-NEXT:  .Lpcsection85:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection86:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection87:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection88:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection89:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_cas_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $1, %cl
; O1-NEXT:  .Lpcsection77:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection78:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection79:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection80:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection81:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection82:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_cas_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $1, %cl
; O2-NEXT:  .Lpcsection77:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection78:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection79:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection80:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection81:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection82:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_cas_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $1, %cl
; O3-NEXT:  .Lpcsection77:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection78:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection79:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection80:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection81:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection82:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i8 42, i8 1 release monotonic, align 1, !pcsections !0
  %y = cmpxchg ptr %a, i8 42, i8 1 release acquire, align 1, !pcsections !0
  %z = cmpxchg ptr %a, i8 42, i8 1 release seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_cas_acq_rel(ptr %a) {
; O0-LABEL: atomic8_cas_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:    movb $1, %cl
; O0-NEXT:  .Lpcsection90:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection91:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection92:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection93:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection94:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_cas_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $1, %cl
; O1-NEXT:  .Lpcsection83:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection84:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection85:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection86:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection87:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection88:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_cas_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $1, %cl
; O2-NEXT:  .Lpcsection83:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection84:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection85:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection86:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection87:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection88:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_cas_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $1, %cl
; O3-NEXT:  .Lpcsection83:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection84:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection85:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection86:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection87:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection88:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i8 42, i8 1 acq_rel monotonic, align 1, !pcsections !0
  %y = cmpxchg ptr %a, i8 42, i8 1 acq_rel acquire, align 1, !pcsections !0
  %z = cmpxchg ptr %a, i8 42, i8 1 acq_rel seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic8_cas_seq_cst(ptr %a) {
; O0-LABEL: atomic8_cas_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movb $42, %al
; O0-NEXT:    movb %al, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Spill
; O0-NEXT:    movb $1, %cl
; O0-NEXT:  .Lpcsection95:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection96:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection97:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:  .Lpcsection98:
; O0-NEXT:    # kill: def $dl killed $al
; O0-NEXT:    movb {{[-0-9]+}}(%r{{[sb]}}p), %al # 1-byte Reload
; O0-NEXT:  .Lpcsection99:
; O0-NEXT:    lock cmpxchgb %cl, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic8_cas_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movb $1, %cl
; O1-NEXT:  .Lpcsection89:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection90:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection91:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection92:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:  .Lpcsection93:
; O1-NEXT:    movb $42, %al
; O1-NEXT:  .Lpcsection94:
; O1-NEXT:    lock cmpxchgb %cl, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic8_cas_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movb $1, %cl
; O2-NEXT:  .Lpcsection89:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection90:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection91:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection92:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:  .Lpcsection93:
; O2-NEXT:    movb $42, %al
; O2-NEXT:  .Lpcsection94:
; O2-NEXT:    lock cmpxchgb %cl, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic8_cas_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movb $1, %cl
; O3-NEXT:  .Lpcsection89:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection90:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection91:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection92:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:  .Lpcsection93:
; O3-NEXT:    movb $42, %al
; O3-NEXT:  .Lpcsection94:
; O3-NEXT:    lock cmpxchgb %cl, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i8 42, i8 1 seq_cst monotonic, align 1, !pcsections !0
  %y = cmpxchg ptr %a, i8 42, i8 1 seq_cst acquire, align 1, !pcsections !0
  %z = cmpxchg ptr %a, i8 42, i8 1 seq_cst seq_cst, align 1, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define i16 @atomic16_load_unordered(ptr %a) {
; O0-LABEL: atomic16_load_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection100:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_load_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection95:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_load_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection95:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_load_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection95:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i16, ptr %a unordered, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i16 %x
}

define i16 @atomic16_load_monotonic(ptr %a) {
; O0-LABEL: atomic16_load_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection101:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_load_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection96:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_load_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection96:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_load_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection96:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i16, ptr %a monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i16 %x
}

define i16 @atomic16_load_acquire(ptr %a) {
; O0-LABEL: atomic16_load_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection102:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_load_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection97:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_load_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection97:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_load_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection97:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i16, ptr %a acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i16 %x
}

define i16 @atomic16_load_seq_cst(ptr %a) {
; O0-LABEL: atomic16_load_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection103:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_load_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection98:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_load_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection98:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_load_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection98:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i16, ptr %a seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i16 %x
}

define void @atomic16_store_unordered(ptr %a) {
; O0-LABEL: atomic16_store_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection104:
; O0-NEXT:    movw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_store_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection99:
; O1-NEXT:    movw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_store_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection99:
; O2-NEXT:    movw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_store_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection99:
; O3-NEXT:    movw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i16 42, ptr %a unordered, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_store_monotonic(ptr %a) {
; O0-LABEL: atomic16_store_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection105:
; O0-NEXT:    movw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_store_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection100:
; O1-NEXT:    movw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_store_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection100:
; O2-NEXT:    movw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_store_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection100:
; O3-NEXT:    movw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i16 42, ptr %a monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_store_release(ptr %a) {
; O0-LABEL: atomic16_store_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection106:
; O0-NEXT:    movw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_store_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection101:
; O1-NEXT:    movw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_store_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection101:
; O2-NEXT:    movw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_store_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection101:
; O3-NEXT:    movw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i16 42, ptr %a release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_store_seq_cst(ptr %a) {
; O0-LABEL: atomic16_store_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:  .Lpcsection107:
; O0-NEXT:    xchgw %ax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_store_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection102:
; O1-NEXT:    xchgw %ax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_store_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection102:
; O2-NEXT:    xchgw %ax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_store_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection102:
; O3-NEXT:    xchgw %ax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i16 42, ptr %a seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xchg_monotonic(ptr %a) {
; O0-LABEL: atomic16_xchg_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:  .Lpcsection108:
; O0-NEXT:    xchgw %ax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xchg_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection103:
; O1-NEXT:    xchgw %ax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xchg_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection103:
; O2-NEXT:    xchgw %ax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xchg_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection103:
; O3-NEXT:    xchgw %ax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i16 42 monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_add_monotonic(ptr %a) {
; O0-LABEL: atomic16_add_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection109:
; O0-NEXT:    lock addw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_add_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection104:
; O1-NEXT:    lock addw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_add_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection104:
; O2-NEXT:    lock addw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_add_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection104:
; O3-NEXT:    lock addw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i16 42 monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_sub_monotonic(ptr %a) {
; O0-LABEL: atomic16_sub_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection110:
; O0-NEXT:    lock subw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_sub_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection105:
; O1-NEXT:    lock subw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_sub_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection105:
; O2-NEXT:    lock subw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_sub_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection105:
; O3-NEXT:    lock subw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i16 42 monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_and_monotonic(ptr %a) {
; O0-LABEL: atomic16_and_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection111:
; O0-NEXT:    lock andw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_and_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection106:
; O1-NEXT:    lock andw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_and_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection106:
; O2-NEXT:    lock andw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_and_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection106:
; O3-NEXT:    lock andw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i16 42 monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_or_monotonic(ptr %a) {
; O0-LABEL: atomic16_or_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection112:
; O0-NEXT:    lock orw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_or_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection107:
; O1-NEXT:    lock orw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_or_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection107:
; O2-NEXT:    lock orw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_or_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection107:
; O3-NEXT:    lock orw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i16 42 monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xor_monotonic(ptr %a) {
; O0-LABEL: atomic16_xor_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection113:
; O0-NEXT:    lock xorw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xor_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection108:
; O1-NEXT:    lock xorw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xor_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection108:
; O2-NEXT:    lock xorw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xor_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection108:
; O3-NEXT:    lock xorw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i16 42 monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_nand_monotonic(ptr %a) {
; O0-LABEL: atomic16_nand_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection114:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .LBB64_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:  .Lpcsection115:
; O0-NEXT:    # implicit-def: $edx
; O0-NEXT:    movw %ax, %dx
; O0-NEXT:  .Lpcsection116:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection117:
; O0-NEXT:    orl $65493, %edx # imm = 0xFFD5
; O0-NEXT:  .Lpcsection118:
; O0-NEXT:    # kill: def $dx killed $dx killed $edx
; O0-NEXT:  .Lpcsection119:
; O0-NEXT:    lock cmpxchgw %dx, (%rcx)
; O0-NEXT:  .Lpcsection120:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection121:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .Lpcsection122:
; O0-NEXT:    jne .LBB64_2
; O0-NEXT:    jmp .LBB64_1
; O0-NEXT:  .LBB64_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_nand_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection109:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB64_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection110:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection111:
; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O1-NEXT:  .Lpcsection112:
; O1-NEXT:    # kill: def $ax killed $ax killed $eax
; O1-NEXT:  .Lpcsection113:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:  .Lpcsection114:
; O1-NEXT:    # kill: def $ax killed $ax def $eax
; O1-NEXT:  .Lpcsection115:
; O1-NEXT:    jne .LBB64_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_nand_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection109:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB64_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection110:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection111:
; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O2-NEXT:  .Lpcsection112:
; O2-NEXT:    # kill: def $ax killed $ax killed $eax
; O2-NEXT:  .Lpcsection113:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:  .Lpcsection114:
; O2-NEXT:    # kill: def $ax killed $ax def $eax
; O2-NEXT:  .Lpcsection115:
; O2-NEXT:    jne .LBB64_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_nand_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection109:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB64_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection110:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection111:
; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O3-NEXT:  .Lpcsection112:
; O3-NEXT:    # kill: def $ax killed $ax killed $eax
; O3-NEXT:  .Lpcsection113:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:  .Lpcsection114:
; O3-NEXT:    # kill: def $ax killed $ax def $eax
; O3-NEXT:  .Lpcsection115:
; O3-NEXT:    jne .LBB64_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i16 42 monotonic, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xchg_acquire(ptr %a) {
; O0-LABEL: atomic16_xchg_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:  .Lpcsection123:
; O0-NEXT:    xchgw %ax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xchg_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection116:
; O1-NEXT:    xchgw %ax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xchg_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection116:
; O2-NEXT:    xchgw %ax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xchg_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection116:
; O3-NEXT:    xchgw %ax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i16 42 acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_add_acquire(ptr %a) {
; O0-LABEL: atomic16_add_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection124:
; O0-NEXT:    lock addw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_add_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection117:
; O1-NEXT:    lock addw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_add_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection117:
; O2-NEXT:    lock addw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_add_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection117:
; O3-NEXT:    lock addw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i16 42 acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_sub_acquire(ptr %a) {
; O0-LABEL: atomic16_sub_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection125:
; O0-NEXT:    lock subw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_sub_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection118:
; O1-NEXT:    lock subw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_sub_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection118:
; O2-NEXT:    lock subw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_sub_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection118:
; O3-NEXT:    lock subw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i16 42 acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_and_acquire(ptr %a) {
; O0-LABEL: atomic16_and_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection126:
; O0-NEXT:    lock andw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_and_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection119:
; O1-NEXT:    lock andw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_and_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection119:
; O2-NEXT:    lock andw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_and_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection119:
; O3-NEXT:    lock andw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i16 42 acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_or_acquire(ptr %a) {
; O0-LABEL: atomic16_or_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection127:
; O0-NEXT:    lock orw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_or_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection120:
; O1-NEXT:    lock orw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_or_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection120:
; O2-NEXT:    lock orw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_or_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection120:
; O3-NEXT:    lock orw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i16 42 acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xor_acquire(ptr %a) {
; O0-LABEL: atomic16_xor_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection128:
; O0-NEXT:    lock xorw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xor_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection121:
; O1-NEXT:    lock xorw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xor_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection121:
; O2-NEXT:    lock xorw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xor_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection121:
; O3-NEXT:    lock xorw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i16 42 acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_nand_acquire(ptr %a) {
; O0-LABEL: atomic16_nand_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection129:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .LBB71_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:  .Lpcsection130:
; O0-NEXT:    # implicit-def: $edx
; O0-NEXT:    movw %ax, %dx
; O0-NEXT:  .Lpcsection131:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection132:
; O0-NEXT:    orl $65493, %edx # imm = 0xFFD5
; O0-NEXT:  .Lpcsection133:
; O0-NEXT:    # kill: def $dx killed $dx killed $edx
; O0-NEXT:  .Lpcsection134:
; O0-NEXT:    lock cmpxchgw %dx, (%rcx)
; O0-NEXT:  .Lpcsection135:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection136:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .Lpcsection137:
; O0-NEXT:    jne .LBB71_2
; O0-NEXT:    jmp .LBB71_1
; O0-NEXT:  .LBB71_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_nand_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection122:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB71_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection123:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection124:
; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O1-NEXT:  .Lpcsection125:
; O1-NEXT:    # kill: def $ax killed $ax killed $eax
; O1-NEXT:  .Lpcsection126:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:  .Lpcsection127:
; O1-NEXT:    # kill: def $ax killed $ax def $eax
; O1-NEXT:  .Lpcsection128:
; O1-NEXT:    jne .LBB71_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_nand_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection122:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB71_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection123:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection124:
; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O2-NEXT:  .Lpcsection125:
; O2-NEXT:    # kill: def $ax killed $ax killed $eax
; O2-NEXT:  .Lpcsection126:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:  .Lpcsection127:
; O2-NEXT:    # kill: def $ax killed $ax def $eax
; O2-NEXT:  .Lpcsection128:
; O2-NEXT:    jne .LBB71_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_nand_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection122:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB71_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection123:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection124:
; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O3-NEXT:  .Lpcsection125:
; O3-NEXT:    # kill: def $ax killed $ax killed $eax
; O3-NEXT:  .Lpcsection126:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:  .Lpcsection127:
; O3-NEXT:    # kill: def $ax killed $ax def $eax
; O3-NEXT:  .Lpcsection128:
; O3-NEXT:    jne .LBB71_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i16 42 acquire, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xchg_release(ptr %a) {
; O0-LABEL: atomic16_xchg_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:  .Lpcsection138:
; O0-NEXT:    xchgw %ax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xchg_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection129:
; O1-NEXT:    xchgw %ax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xchg_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection129:
; O2-NEXT:    xchgw %ax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xchg_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection129:
; O3-NEXT:    xchgw %ax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i16 42 release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_add_release(ptr %a) {
; O0-LABEL: atomic16_add_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection139:
; O0-NEXT:    lock addw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_add_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection130:
; O1-NEXT:    lock addw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_add_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection130:
; O2-NEXT:    lock addw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_add_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection130:
; O3-NEXT:    lock addw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i16 42 release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_sub_release(ptr %a) {
; O0-LABEL: atomic16_sub_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection140:
; O0-NEXT:    lock subw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_sub_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection131:
; O1-NEXT:    lock subw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_sub_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection131:
; O2-NEXT:    lock subw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_sub_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection131:
; O3-NEXT:    lock subw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i16 42 release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_and_release(ptr %a) {
; O0-LABEL: atomic16_and_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection141:
; O0-NEXT:    lock andw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_and_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection132:
; O1-NEXT:    lock andw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_and_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection132:
; O2-NEXT:    lock andw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_and_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection132:
; O3-NEXT:    lock andw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i16 42 release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_or_release(ptr %a) {
; O0-LABEL: atomic16_or_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection142:
; O0-NEXT:    lock orw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_or_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection133:
; O1-NEXT:    lock orw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_or_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection133:
; O2-NEXT:    lock orw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_or_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection133:
; O3-NEXT:    lock orw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i16 42 release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xor_release(ptr %a) {
; O0-LABEL: atomic16_xor_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection143:
; O0-NEXT:    lock xorw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xor_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection134:
; O1-NEXT:    lock xorw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xor_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection134:
; O2-NEXT:    lock xorw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xor_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection134:
; O3-NEXT:    lock xorw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i16 42 release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_nand_release(ptr %a) {
; O0-LABEL: atomic16_nand_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection144:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .LBB78_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:  .Lpcsection145:
; O0-NEXT:    # implicit-def: $edx
; O0-NEXT:    movw %ax, %dx
; O0-NEXT:  .Lpcsection146:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection147:
; O0-NEXT:    orl $65493, %edx # imm = 0xFFD5
; O0-NEXT:  .Lpcsection148:
; O0-NEXT:    # kill: def $dx killed $dx killed $edx
; O0-NEXT:  .Lpcsection149:
; O0-NEXT:    lock cmpxchgw %dx, (%rcx)
; O0-NEXT:  .Lpcsection150:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection151:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .Lpcsection152:
; O0-NEXT:    jne .LBB78_2
; O0-NEXT:    jmp .LBB78_1
; O0-NEXT:  .LBB78_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_nand_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection135:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB78_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection136:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection137:
; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O1-NEXT:  .Lpcsection138:
; O1-NEXT:    # kill: def $ax killed $ax killed $eax
; O1-NEXT:  .Lpcsection139:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:  .Lpcsection140:
; O1-NEXT:    # kill: def $ax killed $ax def $eax
; O1-NEXT:  .Lpcsection141:
; O1-NEXT:    jne .LBB78_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_nand_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection135:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB78_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection136:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection137:
; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O2-NEXT:  .Lpcsection138:
; O2-NEXT:    # kill: def $ax killed $ax killed $eax
; O2-NEXT:  .Lpcsection139:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:  .Lpcsection140:
; O2-NEXT:    # kill: def $ax killed $ax def $eax
; O2-NEXT:  .Lpcsection141:
; O2-NEXT:    jne .LBB78_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_nand_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection135:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB78_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection136:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection137:
; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O3-NEXT:  .Lpcsection138:
; O3-NEXT:    # kill: def $ax killed $ax killed $eax
; O3-NEXT:  .Lpcsection139:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:  .Lpcsection140:
; O3-NEXT:    # kill: def $ax killed $ax def $eax
; O3-NEXT:  .Lpcsection141:
; O3-NEXT:    jne .LBB78_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i16 42 release, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xchg_acq_rel(ptr %a) {
; O0-LABEL: atomic16_xchg_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:  .Lpcsection153:
; O0-NEXT:    xchgw %ax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xchg_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection142:
; O1-NEXT:    xchgw %ax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xchg_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection142:
; O2-NEXT:    xchgw %ax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xchg_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection142:
; O3-NEXT:    xchgw %ax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i16 42 acq_rel, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_add_acq_rel(ptr %a) {
; O0-LABEL: atomic16_add_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection154:
; O0-NEXT:    lock addw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_add_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection143:
; O1-NEXT:    lock addw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_add_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection143:
; O2-NEXT:    lock addw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_add_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection143:
; O3-NEXT:    lock addw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i16 42 acq_rel, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_sub_acq_rel(ptr %a) {
; O0-LABEL: atomic16_sub_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection155:
; O0-NEXT:    lock subw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_sub_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection144:
; O1-NEXT:    lock subw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_sub_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection144:
; O2-NEXT:    lock subw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_sub_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection144:
; O3-NEXT:    lock subw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i16 42 acq_rel, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_and_acq_rel(ptr %a) {
; O0-LABEL: atomic16_and_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection156:
; O0-NEXT:    lock andw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_and_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection145:
; O1-NEXT:    lock andw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_and_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection145:
; O2-NEXT:    lock andw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_and_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection145:
; O3-NEXT:    lock andw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i16 42 acq_rel, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_or_acq_rel(ptr %a) {
; O0-LABEL: atomic16_or_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection157:
; O0-NEXT:    lock orw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_or_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection146:
; O1-NEXT:    lock orw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_or_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection146:
; O2-NEXT:    lock orw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_or_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection146:
; O3-NEXT:    lock orw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i16 42 acq_rel, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xor_acq_rel(ptr %a) {
; O0-LABEL: atomic16_xor_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection158:
; O0-NEXT:    lock xorw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xor_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection147:
; O1-NEXT:    lock xorw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xor_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection147:
; O2-NEXT:    lock xorw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xor_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection147:
; O3-NEXT:    lock xorw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i16 42 acq_rel, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_nand_acq_rel(ptr %a) {
; O0-LABEL: atomic16_nand_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection159:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .LBB85_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:  .Lpcsection160:
; O0-NEXT:    # implicit-def: $edx
; O0-NEXT:    movw %ax, %dx
; O0-NEXT:  .Lpcsection161:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection162:
; O0-NEXT:    orl $65493, %edx # imm = 0xFFD5
; O0-NEXT:  .Lpcsection163:
; O0-NEXT:    # kill: def $dx killed $dx killed $edx
; O0-NEXT:  .Lpcsection164:
; O0-NEXT:    lock cmpxchgw %dx, (%rcx)
; O0-NEXT:  .Lpcsection165:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection166:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .Lpcsection167:
; O0-NEXT:    jne .LBB85_2
; O0-NEXT:    jmp .LBB85_1
; O0-NEXT:  .LBB85_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_nand_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection148:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB85_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection149:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection150:
; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O1-NEXT:  .Lpcsection151:
; O1-NEXT:    # kill: def $ax killed $ax killed $eax
; O1-NEXT:  .Lpcsection152:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:  .Lpcsection153:
; O1-NEXT:    # kill: def $ax killed $ax def $eax
; O1-NEXT:  .Lpcsection154:
; O1-NEXT:    jne .LBB85_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_nand_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection148:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB85_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection149:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection150:
; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O2-NEXT:  .Lpcsection151:
; O2-NEXT:    # kill: def $ax killed $ax killed $eax
; O2-NEXT:  .Lpcsection152:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:  .Lpcsection153:
; O2-NEXT:    # kill: def $ax killed $ax def $eax
; O2-NEXT:  .Lpcsection154:
; O2-NEXT:    jne .LBB85_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_nand_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection148:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB85_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection149:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection150:
; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O3-NEXT:  .Lpcsection151:
; O3-NEXT:    # kill: def $ax killed $ax killed $eax
; O3-NEXT:  .Lpcsection152:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:  .Lpcsection153:
; O3-NEXT:    # kill: def $ax killed $ax def $eax
; O3-NEXT:  .Lpcsection154:
; O3-NEXT:    jne .LBB85_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i16 42 acq_rel, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xchg_seq_cst(ptr %a) {
; O0-LABEL: atomic16_xchg_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:  .Lpcsection168:
; O0-NEXT:    xchgw %ax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xchg_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection155:
; O1-NEXT:    xchgw %ax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xchg_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection155:
; O2-NEXT:    xchgw %ax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xchg_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection155:
; O3-NEXT:    xchgw %ax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i16 42 seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_add_seq_cst(ptr %a) {
; O0-LABEL: atomic16_add_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection169:
; O0-NEXT:    lock addw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_add_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection156:
; O1-NEXT:    lock addw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_add_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection156:
; O2-NEXT:    lock addw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_add_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection156:
; O3-NEXT:    lock addw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i16 42 seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_sub_seq_cst(ptr %a) {
; O0-LABEL: atomic16_sub_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection170:
; O0-NEXT:    lock subw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_sub_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection157:
; O1-NEXT:    lock subw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_sub_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection157:
; O2-NEXT:    lock subw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_sub_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection157:
; O3-NEXT:    lock subw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i16 42 seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_and_seq_cst(ptr %a) {
; O0-LABEL: atomic16_and_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection171:
; O0-NEXT:    lock andw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_and_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection158:
; O1-NEXT:    lock andw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_and_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection158:
; O2-NEXT:    lock andw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_and_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection158:
; O3-NEXT:    lock andw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i16 42 seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_or_seq_cst(ptr %a) {
; O0-LABEL: atomic16_or_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection172:
; O0-NEXT:    lock orw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_or_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection159:
; O1-NEXT:    lock orw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_or_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection159:
; O2-NEXT:    lock orw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_or_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection159:
; O3-NEXT:    lock orw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i16 42 seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_xor_seq_cst(ptr %a) {
; O0-LABEL: atomic16_xor_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection173:
; O0-NEXT:    lock xorw $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_xor_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection160:
; O1-NEXT:    lock xorw $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_xor_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection160:
; O2-NEXT:    lock xorw $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_xor_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection160:
; O3-NEXT:    lock xorw $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i16 42 seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_nand_seq_cst(ptr %a) {
; O0-LABEL: atomic16_nand_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection174:
; O0-NEXT:    movw (%rdi), %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .LBB92_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:  .Lpcsection175:
; O0-NEXT:    # implicit-def: $edx
; O0-NEXT:    movw %ax, %dx
; O0-NEXT:  .Lpcsection176:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection177:
; O0-NEXT:    orl $65493, %edx # imm = 0xFFD5
; O0-NEXT:  .Lpcsection178:
; O0-NEXT:    # kill: def $dx killed $dx killed $edx
; O0-NEXT:  .Lpcsection179:
; O0-NEXT:    lock cmpxchgw %dx, (%rcx)
; O0-NEXT:  .Lpcsection180:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection181:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:  .Lpcsection182:
; O0-NEXT:    jne .LBB92_2
; O0-NEXT:    jmp .LBB92_1
; O0-NEXT:  .LBB92_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_nand_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection161:
; O1-NEXT:    movzwl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB92_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection162:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection163:
; O1-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O1-NEXT:  .Lpcsection164:
; O1-NEXT:    # kill: def $ax killed $ax killed $eax
; O1-NEXT:  .Lpcsection165:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:  .Lpcsection166:
; O1-NEXT:    # kill: def $ax killed $ax def $eax
; O1-NEXT:  .Lpcsection167:
; O1-NEXT:    jne .LBB92_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_nand_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection161:
; O2-NEXT:    movzwl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB92_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection162:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection163:
; O2-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O2-NEXT:  .Lpcsection164:
; O2-NEXT:    # kill: def $ax killed $ax killed $eax
; O2-NEXT:  .Lpcsection165:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:  .Lpcsection166:
; O2-NEXT:    # kill: def $ax killed $ax def $eax
; O2-NEXT:  .Lpcsection167:
; O2-NEXT:    jne .LBB92_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_nand_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection161:
; O3-NEXT:    movzwl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB92_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection162:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection163:
; O3-NEXT:    orl $65493, %ecx # imm = 0xFFD5
; O3-NEXT:  .Lpcsection164:
; O3-NEXT:    # kill: def $ax killed $ax killed $eax
; O3-NEXT:  .Lpcsection165:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:  .Lpcsection166:
; O3-NEXT:    # kill: def $ax killed $ax def $eax
; O3-NEXT:  .Lpcsection167:
; O3-NEXT:    jne .LBB92_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i16 42 seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_cas_monotonic(ptr %a) {
; O0-LABEL: atomic16_cas_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:    movw $1, %cx
; O0-NEXT:  .Lpcsection183:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection184:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection185:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection186:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection187:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_cas_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $1, %cx
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection168:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection169:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection170:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_cas_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $1, %cx
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection168:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection169:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection170:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_cas_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $1, %cx
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection168:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection169:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection170:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i16 42, i16 1 monotonic monotonic, align 2, !pcsections !0
  %y = cmpxchg ptr %a, i16 42, i16 1 monotonic acquire, align 2, !pcsections !0
  %z = cmpxchg ptr %a, i16 42, i16 1 monotonic seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_cas_acquire(ptr %a) {
; O0-LABEL: atomic16_cas_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:    movw $1, %cx
; O0-NEXT:  .Lpcsection188:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection189:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection190:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection191:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection192:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_cas_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $1, %cx
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection171:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection172:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection173:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_cas_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $1, %cx
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection171:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection172:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection173:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_cas_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $1, %cx
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection171:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection172:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection173:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i16 42, i16 1 acquire monotonic, align 2, !pcsections !0
  %y = cmpxchg ptr %a, i16 42, i16 1 acquire acquire, align 2, !pcsections !0
  %z = cmpxchg ptr %a, i16 42, i16 1 acquire seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_cas_release(ptr %a) {
; O0-LABEL: atomic16_cas_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:    movw $1, %cx
; O0-NEXT:  .Lpcsection193:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection194:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection195:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection196:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection197:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_cas_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $1, %cx
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection174:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection175:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection176:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_cas_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $1, %cx
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection174:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection175:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection176:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_cas_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $1, %cx
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection174:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection175:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection176:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i16 42, i16 1 release monotonic, align 2, !pcsections !0
  %y = cmpxchg ptr %a, i16 42, i16 1 release acquire, align 2, !pcsections !0
  %z = cmpxchg ptr %a, i16 42, i16 1 release seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_cas_acq_rel(ptr %a) {
; O0-LABEL: atomic16_cas_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:    movw $1, %cx
; O0-NEXT:  .Lpcsection198:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection199:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection200:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection201:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection202:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_cas_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $1, %cx
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection177:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection178:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection179:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_cas_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $1, %cx
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection177:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection178:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection179:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_cas_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $1, %cx
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection177:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection178:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection179:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i16 42, i16 1 acq_rel monotonic, align 2, !pcsections !0
  %y = cmpxchg ptr %a, i16 42, i16 1 acq_rel acquire, align 2, !pcsections !0
  %z = cmpxchg ptr %a, i16 42, i16 1 acq_rel seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic16_cas_seq_cst(ptr %a) {
; O0-LABEL: atomic16_cas_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movw $42, %ax
; O0-NEXT:    movw %ax, {{[-0-9]+}}(%r{{[sb]}}p) # 2-byte Spill
; O0-NEXT:    movw $1, %cx
; O0-NEXT:  .Lpcsection203:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection204:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection205:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:  .Lpcsection206:
; O0-NEXT:    # kill: def $dx killed $ax
; O0-NEXT:    movw {{[-0-9]+}}(%r{{[sb]}}p), %ax # 2-byte Reload
; O0-NEXT:  .Lpcsection207:
; O0-NEXT:    lock cmpxchgw %cx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic16_cas_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movw $1, %cx
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection180:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection181:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movw $42, %ax
; O1-NEXT:  .Lpcsection182:
; O1-NEXT:    lock cmpxchgw %cx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic16_cas_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movw $1, %cx
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection180:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection181:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movw $42, %ax
; O2-NEXT:  .Lpcsection182:
; O2-NEXT:    lock cmpxchgw %cx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic16_cas_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movw $1, %cx
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection180:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection181:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movw $42, %ax
; O3-NEXT:  .Lpcsection182:
; O3-NEXT:    lock cmpxchgw %cx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i16 42, i16 1 seq_cst monotonic, align 2, !pcsections !0
  %y = cmpxchg ptr %a, i16 42, i16 1 seq_cst acquire, align 2, !pcsections !0
  %z = cmpxchg ptr %a, i16 42, i16 1 seq_cst seq_cst, align 2, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define i32 @atomic32_load_unordered(ptr %a) {
; O0-LABEL: atomic32_load_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection208:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_load_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection183:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_load_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection183:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_load_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection183:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i32, ptr %a unordered, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i32 %x
}

define i32 @atomic32_load_monotonic(ptr %a) {
; O0-LABEL: atomic32_load_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection209:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_load_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection184:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_load_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection184:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_load_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection184:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i32, ptr %a monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i32 %x
}

define i32 @atomic32_load_acquire(ptr %a) {
; O0-LABEL: atomic32_load_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection210:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_load_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection185:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_load_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection185:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_load_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection185:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i32, ptr %a acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i32 %x
}

define i32 @atomic32_load_seq_cst(ptr %a) {
; O0-LABEL: atomic32_load_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection211:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_load_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection186:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_load_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection186:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_load_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection186:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i32, ptr %a seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i32 %x
}

define void @atomic32_store_unordered(ptr %a) {
; O0-LABEL: atomic32_store_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection212:
; O0-NEXT:    movl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_store_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection187:
; O1-NEXT:    movl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_store_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection187:
; O2-NEXT:    movl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_store_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection187:
; O3-NEXT:    movl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i32 42, ptr %a unordered, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_store_monotonic(ptr %a) {
; O0-LABEL: atomic32_store_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection213:
; O0-NEXT:    movl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_store_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection188:
; O1-NEXT:    movl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_store_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection188:
; O2-NEXT:    movl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_store_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection188:
; O3-NEXT:    movl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i32 42, ptr %a monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_store_release(ptr %a) {
; O0-LABEL: atomic32_store_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection214:
; O0-NEXT:    movl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_store_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection189:
; O1-NEXT:    movl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_store_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection189:
; O2-NEXT:    movl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_store_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection189:
; O3-NEXT:    movl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i32 42, ptr %a release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_store_seq_cst(ptr %a) {
; O0-LABEL: atomic32_store_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection215:
; O0-NEXT:    xchgl %eax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_store_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection190:
; O1-NEXT:    xchgl %eax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_store_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection190:
; O2-NEXT:    xchgl %eax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_store_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection190:
; O3-NEXT:    xchgl %eax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i32 42, ptr %a seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xchg_monotonic(ptr %a) {
; O0-LABEL: atomic32_xchg_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection216:
; O0-NEXT:    xchgl %eax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xchg_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection191:
; O1-NEXT:    xchgl %eax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xchg_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection191:
; O2-NEXT:    xchgl %eax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xchg_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection191:
; O3-NEXT:    xchgl %eax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i32 42 monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_add_monotonic(ptr %a) {
; O0-LABEL: atomic32_add_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection217:
; O0-NEXT:    lock addl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_add_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection192:
; O1-NEXT:    lock addl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_add_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection192:
; O2-NEXT:    lock addl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_add_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection192:
; O3-NEXT:    lock addl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i32 42 monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_sub_monotonic(ptr %a) {
; O0-LABEL: atomic32_sub_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection218:
; O0-NEXT:    lock subl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_sub_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection193:
; O1-NEXT:    lock subl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_sub_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection193:
; O2-NEXT:    lock subl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_sub_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection193:
; O3-NEXT:    lock subl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i32 42 monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_and_monotonic(ptr %a) {
; O0-LABEL: atomic32_and_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection219:
; O0-NEXT:    lock andl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_and_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection194:
; O1-NEXT:    lock andl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_and_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection194:
; O2-NEXT:    lock andl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_and_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection194:
; O3-NEXT:    lock andl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i32 42 monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_or_monotonic(ptr %a) {
; O0-LABEL: atomic32_or_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection220:
; O0-NEXT:    lock orl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_or_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection195:
; O1-NEXT:    lock orl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_or_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection195:
; O2-NEXT:    lock orl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_or_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection195:
; O3-NEXT:    lock orl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i32 42 monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xor_monotonic(ptr %a) {
; O0-LABEL: atomic32_xor_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection221:
; O0-NEXT:    lock xorl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xor_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection196:
; O1-NEXT:    lock xorl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xor_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection196:
; O2-NEXT:    lock xorl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xor_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection196:
; O3-NEXT:    lock xorl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i32 42 monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_nand_monotonic(ptr %a) {
; O0-LABEL: atomic32_nand_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection222:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .LBB112_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %edx
; O0-NEXT:  .Lpcsection223:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection224:
; O0-NEXT:    orl $-43, %edx
; O0-NEXT:  .Lpcsection225:
; O0-NEXT:    lock cmpxchgl %edx, (%rcx)
; O0-NEXT:  .Lpcsection226:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection227:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .Lpcsection228:
; O0-NEXT:    jne .LBB112_2
; O0-NEXT:    jmp .LBB112_1
; O0-NEXT:  .LBB112_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_nand_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection197:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB112_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection198:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection199:
; O1-NEXT:    orl $-43, %ecx
; O1-NEXT:  .Lpcsection200:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection201:
; O1-NEXT:    jne .LBB112_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_nand_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection197:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB112_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection198:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection199:
; O2-NEXT:    orl $-43, %ecx
; O2-NEXT:  .Lpcsection200:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection201:
; O2-NEXT:    jne .LBB112_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_nand_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection197:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB112_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection198:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection199:
; O3-NEXT:    orl $-43, %ecx
; O3-NEXT:  .Lpcsection200:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection201:
; O3-NEXT:    jne .LBB112_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i32 42 monotonic, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xchg_acquire(ptr %a) {
; O0-LABEL: atomic32_xchg_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection229:
; O0-NEXT:    xchgl %eax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xchg_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection202:
; O1-NEXT:    xchgl %eax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xchg_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection202:
; O2-NEXT:    xchgl %eax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xchg_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection202:
; O3-NEXT:    xchgl %eax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i32 42 acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_add_acquire(ptr %a) {
; O0-LABEL: atomic32_add_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection230:
; O0-NEXT:    lock addl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_add_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection203:
; O1-NEXT:    lock addl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_add_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection203:
; O2-NEXT:    lock addl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_add_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection203:
; O3-NEXT:    lock addl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i32 42 acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_sub_acquire(ptr %a) {
; O0-LABEL: atomic32_sub_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection231:
; O0-NEXT:    lock subl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_sub_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection204:
; O1-NEXT:    lock subl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_sub_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection204:
; O2-NEXT:    lock subl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_sub_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection204:
; O3-NEXT:    lock subl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i32 42 acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_and_acquire(ptr %a) {
; O0-LABEL: atomic32_and_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection232:
; O0-NEXT:    lock andl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_and_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection205:
; O1-NEXT:    lock andl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_and_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection205:
; O2-NEXT:    lock andl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_and_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection205:
; O3-NEXT:    lock andl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i32 42 acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_or_acquire(ptr %a) {
; O0-LABEL: atomic32_or_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection233:
; O0-NEXT:    lock orl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_or_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection206:
; O1-NEXT:    lock orl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_or_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection206:
; O2-NEXT:    lock orl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_or_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection206:
; O3-NEXT:    lock orl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i32 42 acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xor_acquire(ptr %a) {
; O0-LABEL: atomic32_xor_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection234:
; O0-NEXT:    lock xorl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xor_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection207:
; O1-NEXT:    lock xorl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xor_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection207:
; O2-NEXT:    lock xorl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xor_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection207:
; O3-NEXT:    lock xorl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i32 42 acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_nand_acquire(ptr %a) {
; O0-LABEL: atomic32_nand_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection235:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .LBB119_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %edx
; O0-NEXT:  .Lpcsection236:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection237:
; O0-NEXT:    orl $-43, %edx
; O0-NEXT:  .Lpcsection238:
; O0-NEXT:    lock cmpxchgl %edx, (%rcx)
; O0-NEXT:  .Lpcsection239:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection240:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .Lpcsection241:
; O0-NEXT:    jne .LBB119_2
; O0-NEXT:    jmp .LBB119_1
; O0-NEXT:  .LBB119_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_nand_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection208:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB119_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection209:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection210:
; O1-NEXT:    orl $-43, %ecx
; O1-NEXT:  .Lpcsection211:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection212:
; O1-NEXT:    jne .LBB119_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_nand_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection208:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB119_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection209:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection210:
; O2-NEXT:    orl $-43, %ecx
; O2-NEXT:  .Lpcsection211:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection212:
; O2-NEXT:    jne .LBB119_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_nand_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection208:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB119_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection209:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection210:
; O3-NEXT:    orl $-43, %ecx
; O3-NEXT:  .Lpcsection211:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection212:
; O3-NEXT:    jne .LBB119_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i32 42 acquire, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xchg_release(ptr %a) {
; O0-LABEL: atomic32_xchg_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection242:
; O0-NEXT:    xchgl %eax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xchg_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection213:
; O1-NEXT:    xchgl %eax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xchg_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection213:
; O2-NEXT:    xchgl %eax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xchg_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection213:
; O3-NEXT:    xchgl %eax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i32 42 release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_add_release(ptr %a) {
; O0-LABEL: atomic32_add_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection243:
; O0-NEXT:    lock addl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_add_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection214:
; O1-NEXT:    lock addl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_add_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection214:
; O2-NEXT:    lock addl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_add_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection214:
; O3-NEXT:    lock addl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i32 42 release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_sub_release(ptr %a) {
; O0-LABEL: atomic32_sub_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection244:
; O0-NEXT:    lock subl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_sub_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection215:
; O1-NEXT:    lock subl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_sub_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection215:
; O2-NEXT:    lock subl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_sub_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection215:
; O3-NEXT:    lock subl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i32 42 release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_and_release(ptr %a) {
; O0-LABEL: atomic32_and_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection245:
; O0-NEXT:    lock andl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_and_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection216:
; O1-NEXT:    lock andl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_and_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection216:
; O2-NEXT:    lock andl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_and_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection216:
; O3-NEXT:    lock andl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i32 42 release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_or_release(ptr %a) {
; O0-LABEL: atomic32_or_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection246:
; O0-NEXT:    lock orl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_or_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection217:
; O1-NEXT:    lock orl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_or_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection217:
; O2-NEXT:    lock orl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_or_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection217:
; O3-NEXT:    lock orl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i32 42 release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xor_release(ptr %a) {
; O0-LABEL: atomic32_xor_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection247:
; O0-NEXT:    lock xorl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xor_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection218:
; O1-NEXT:    lock xorl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xor_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection218:
; O2-NEXT:    lock xorl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xor_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection218:
; O3-NEXT:    lock xorl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i32 42 release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_nand_release(ptr %a) {
; O0-LABEL: atomic32_nand_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection248:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .LBB126_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %edx
; O0-NEXT:  .Lpcsection249:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection250:
; O0-NEXT:    orl $-43, %edx
; O0-NEXT:  .Lpcsection251:
; O0-NEXT:    lock cmpxchgl %edx, (%rcx)
; O0-NEXT:  .Lpcsection252:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection253:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .Lpcsection254:
; O0-NEXT:    jne .LBB126_2
; O0-NEXT:    jmp .LBB126_1
; O0-NEXT:  .LBB126_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_nand_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection219:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB126_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection220:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection221:
; O1-NEXT:    orl $-43, %ecx
; O1-NEXT:  .Lpcsection222:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection223:
; O1-NEXT:    jne .LBB126_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_nand_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection219:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB126_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection220:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection221:
; O2-NEXT:    orl $-43, %ecx
; O2-NEXT:  .Lpcsection222:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection223:
; O2-NEXT:    jne .LBB126_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_nand_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection219:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB126_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection220:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection221:
; O3-NEXT:    orl $-43, %ecx
; O3-NEXT:  .Lpcsection222:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection223:
; O3-NEXT:    jne .LBB126_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i32 42 release, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xchg_acq_rel(ptr %a) {
; O0-LABEL: atomic32_xchg_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection255:
; O0-NEXT:    xchgl %eax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xchg_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection224:
; O1-NEXT:    xchgl %eax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xchg_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection224:
; O2-NEXT:    xchgl %eax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xchg_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection224:
; O3-NEXT:    xchgl %eax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i32 42 acq_rel, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_add_acq_rel(ptr %a) {
; O0-LABEL: atomic32_add_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection256:
; O0-NEXT:    lock addl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_add_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection225:
; O1-NEXT:    lock addl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_add_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection225:
; O2-NEXT:    lock addl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_add_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection225:
; O3-NEXT:    lock addl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i32 42 acq_rel, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_sub_acq_rel(ptr %a) {
; O0-LABEL: atomic32_sub_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection257:
; O0-NEXT:    lock subl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_sub_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection226:
; O1-NEXT:    lock subl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_sub_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection226:
; O2-NEXT:    lock subl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_sub_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection226:
; O3-NEXT:    lock subl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i32 42 acq_rel, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_and_acq_rel(ptr %a) {
; O0-LABEL: atomic32_and_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection258:
; O0-NEXT:    lock andl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_and_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection227:
; O1-NEXT:    lock andl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_and_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection227:
; O2-NEXT:    lock andl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_and_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection227:
; O3-NEXT:    lock andl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i32 42 acq_rel, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_or_acq_rel(ptr %a) {
; O0-LABEL: atomic32_or_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection259:
; O0-NEXT:    lock orl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_or_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection228:
; O1-NEXT:    lock orl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_or_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection228:
; O2-NEXT:    lock orl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_or_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection228:
; O3-NEXT:    lock orl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i32 42 acq_rel, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xor_acq_rel(ptr %a) {
; O0-LABEL: atomic32_xor_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection260:
; O0-NEXT:    lock xorl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xor_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection229:
; O1-NEXT:    lock xorl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xor_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection229:
; O2-NEXT:    lock xorl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xor_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection229:
; O3-NEXT:    lock xorl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i32 42 acq_rel, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_nand_acq_rel(ptr %a) {
; O0-LABEL: atomic32_nand_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection261:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .LBB133_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %edx
; O0-NEXT:  .Lpcsection262:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection263:
; O0-NEXT:    orl $-43, %edx
; O0-NEXT:  .Lpcsection264:
; O0-NEXT:    lock cmpxchgl %edx, (%rcx)
; O0-NEXT:  .Lpcsection265:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection266:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .Lpcsection267:
; O0-NEXT:    jne .LBB133_2
; O0-NEXT:    jmp .LBB133_1
; O0-NEXT:  .LBB133_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_nand_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection230:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB133_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection231:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection232:
; O1-NEXT:    orl $-43, %ecx
; O1-NEXT:  .Lpcsection233:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection234:
; O1-NEXT:    jne .LBB133_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_nand_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection230:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB133_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection231:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection232:
; O2-NEXT:    orl $-43, %ecx
; O2-NEXT:  .Lpcsection233:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection234:
; O2-NEXT:    jne .LBB133_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_nand_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection230:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB133_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection231:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection232:
; O3-NEXT:    orl $-43, %ecx
; O3-NEXT:  .Lpcsection233:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection234:
; O3-NEXT:    jne .LBB133_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i32 42 acq_rel, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xchg_seq_cst(ptr %a) {
; O0-LABEL: atomic32_xchg_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection268:
; O0-NEXT:    xchgl %eax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xchg_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection235:
; O1-NEXT:    xchgl %eax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xchg_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection235:
; O2-NEXT:    xchgl %eax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xchg_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection235:
; O3-NEXT:    xchgl %eax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i32 42 seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_add_seq_cst(ptr %a) {
; O0-LABEL: atomic32_add_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection269:
; O0-NEXT:    lock addl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_add_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection236:
; O1-NEXT:    lock addl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_add_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection236:
; O2-NEXT:    lock addl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_add_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection236:
; O3-NEXT:    lock addl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i32 42 seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_sub_seq_cst(ptr %a) {
; O0-LABEL: atomic32_sub_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection270:
; O0-NEXT:    lock subl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_sub_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection237:
; O1-NEXT:    lock subl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_sub_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection237:
; O2-NEXT:    lock subl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_sub_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection237:
; O3-NEXT:    lock subl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i32 42 seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_and_seq_cst(ptr %a) {
; O0-LABEL: atomic32_and_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection271:
; O0-NEXT:    lock andl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_and_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection238:
; O1-NEXT:    lock andl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_and_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection238:
; O2-NEXT:    lock andl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_and_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection238:
; O3-NEXT:    lock andl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i32 42 seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_or_seq_cst(ptr %a) {
; O0-LABEL: atomic32_or_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection272:
; O0-NEXT:    lock orl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_or_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection239:
; O1-NEXT:    lock orl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_or_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection239:
; O2-NEXT:    lock orl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_or_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection239:
; O3-NEXT:    lock orl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i32 42 seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_xor_seq_cst(ptr %a) {
; O0-LABEL: atomic32_xor_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection273:
; O0-NEXT:    lock xorl $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_xor_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection240:
; O1-NEXT:    lock xorl $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_xor_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection240:
; O2-NEXT:    lock xorl $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_xor_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection240:
; O3-NEXT:    lock xorl $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i32 42 seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_nand_seq_cst(ptr %a) {
; O0-LABEL: atomic32_nand_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection274:
; O0-NEXT:    movl (%rdi), %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .LBB140_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %edx
; O0-NEXT:  .Lpcsection275:
; O0-NEXT:    notl %edx
; O0-NEXT:  .Lpcsection276:
; O0-NEXT:    orl $-43, %edx
; O0-NEXT:  .Lpcsection277:
; O0-NEXT:    lock cmpxchgl %edx, (%rcx)
; O0-NEXT:  .Lpcsection278:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection279:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:  .Lpcsection280:
; O0-NEXT:    jne .LBB140_2
; O0-NEXT:    jmp .LBB140_1
; O0-NEXT:  .LBB140_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_nand_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection241:
; O1-NEXT:    movl (%rdi), %eax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB140_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection242:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection243:
; O1-NEXT:    orl $-43, %ecx
; O1-NEXT:  .Lpcsection244:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection245:
; O1-NEXT:    jne .LBB140_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_nand_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection241:
; O2-NEXT:    movl (%rdi), %eax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB140_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection242:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection243:
; O2-NEXT:    orl $-43, %ecx
; O2-NEXT:  .Lpcsection244:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection245:
; O2-NEXT:    jne .LBB140_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_nand_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection241:
; O3-NEXT:    movl (%rdi), %eax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB140_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection242:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection243:
; O3-NEXT:    orl $-43, %ecx
; O3-NEXT:  .Lpcsection244:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection245:
; O3-NEXT:    jne .LBB140_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i32 42 seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_cas_monotonic(ptr %a) {
; O0-LABEL: atomic32_cas_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection281:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection282:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection283:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection284:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection285:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_cas_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection246:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection247:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection248:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection249:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection250:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection251:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_cas_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection246:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection247:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection248:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection249:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection250:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection251:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_cas_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection246:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection247:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection248:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection249:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection250:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection251:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i32 42, i32 1 monotonic monotonic, align 4, !pcsections !0
  %y = cmpxchg ptr %a, i32 42, i32 1 monotonic acquire, align 4, !pcsections !0
  %z = cmpxchg ptr %a, i32 42, i32 1 monotonic seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_cas_acquire(ptr %a) {
; O0-LABEL: atomic32_cas_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection286:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection287:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection288:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection289:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection290:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_cas_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection252:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection253:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection254:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection255:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection256:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection257:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_cas_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection252:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection253:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection254:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection255:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection256:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection257:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_cas_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection252:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection253:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection254:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection255:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection256:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection257:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i32 42, i32 1 acquire monotonic, align 4, !pcsections !0
  %y = cmpxchg ptr %a, i32 42, i32 1 acquire acquire, align 4, !pcsections !0
  %z = cmpxchg ptr %a, i32 42, i32 1 acquire seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_cas_release(ptr %a) {
; O0-LABEL: atomic32_cas_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection291:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection292:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection293:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection294:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection295:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_cas_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection258:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection259:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection260:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection261:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection262:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection263:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_cas_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection258:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection259:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection260:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection261:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection262:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection263:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_cas_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection258:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection259:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection260:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection261:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection262:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection263:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i32 42, i32 1 release monotonic, align 4, !pcsections !0
  %y = cmpxchg ptr %a, i32 42, i32 1 release acquire, align 4, !pcsections !0
  %z = cmpxchg ptr %a, i32 42, i32 1 release seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_cas_acq_rel(ptr %a) {
; O0-LABEL: atomic32_cas_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection296:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection297:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection298:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection299:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection300:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_cas_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection264:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection265:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection266:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection267:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection268:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection269:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_cas_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection264:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection265:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection266:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection267:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection268:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection269:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_cas_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection264:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection265:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection266:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection267:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection268:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection269:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i32 42, i32 1 acq_rel monotonic, align 4, !pcsections !0
  %y = cmpxchg ptr %a, i32 42, i32 1 acq_rel acquire, align 4, !pcsections !0
  %z = cmpxchg ptr %a, i32 42, i32 1 acq_rel seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic32_cas_seq_cst(ptr %a) {
; O0-LABEL: atomic32_cas_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection301:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection302:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection303:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:  .Lpcsection304:
; O0-NEXT:    # kill: def $edx killed $eax
; O0-NEXT:    movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
; O0-NEXT:  .Lpcsection305:
; O0-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic32_cas_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection270:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection271:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection272:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection273:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:  .Lpcsection274:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection275:
; O1-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic32_cas_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection270:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection271:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection272:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection273:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:  .Lpcsection274:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection275:
; O2-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic32_cas_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection270:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection271:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection272:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection273:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:  .Lpcsection274:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection275:
; O3-NEXT:    lock cmpxchgl %ecx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i32 42, i32 1 seq_cst monotonic, align 4, !pcsections !0
  %y = cmpxchg ptr %a, i32 42, i32 1 seq_cst acquire, align 4, !pcsections !0
  %z = cmpxchg ptr %a, i32 42, i32 1 seq_cst seq_cst, align 4, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define i64 @atomic64_load_unordered(ptr %a) {
; O0-LABEL: atomic64_load_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection306:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_load_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection276:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_load_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection276:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_load_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection276:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i64, ptr %a unordered, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i64 %x
}

define i64 @atomic64_load_monotonic(ptr %a) {
; O0-LABEL: atomic64_load_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection307:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_load_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection277:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_load_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection277:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_load_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection277:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i64, ptr %a monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i64 %x
}

define i64 @atomic64_load_acquire(ptr %a) {
; O0-LABEL: atomic64_load_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection308:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_load_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection278:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_load_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection278:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_load_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection278:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i64, ptr %a acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i64 %x
}

define i64 @atomic64_load_seq_cst(ptr %a) {
; O0-LABEL: atomic64_load_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection309:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_load_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection279:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_load_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection279:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_load_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection279:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i64, ptr %a seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i64 %x
}

define ptr @atomic64_load_seq_cst_ptr_ty(ptr %a) {
; O0-LABEL: atomic64_load_seq_cst_ptr_ty:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection310:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_load_seq_cst_ptr_ty:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection280:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_load_seq_cst_ptr_ty:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection280:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_load_seq_cst_ptr_ty:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection280:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic ptr, ptr %a seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret ptr %x
}

define void @atomic64_store_unordered(ptr %a) {
; O0-LABEL: atomic64_store_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection311:
; O0-NEXT:    movq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_store_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection281:
; O1-NEXT:    movq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_store_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection281:
; O2-NEXT:    movq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_store_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection281:
; O3-NEXT:    movq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i64 42, ptr %a unordered, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_store_monotonic(ptr %a) {
; O0-LABEL: atomic64_store_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection312:
; O0-NEXT:    movq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_store_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection282:
; O1-NEXT:    movq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_store_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection282:
; O2-NEXT:    movq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_store_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection282:
; O3-NEXT:    movq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i64 42, ptr %a monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_store_release(ptr %a) {
; O0-LABEL: atomic64_store_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection313:
; O0-NEXT:    movq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_store_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection283:
; O1-NEXT:    movq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_store_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection283:
; O2-NEXT:    movq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_store_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection283:
; O3-NEXT:    movq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i64 42, ptr %a release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_store_seq_cst(ptr %a) {
; O0-LABEL: atomic64_store_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection314:
; O0-NEXT:    xchgq %rax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_store_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection284:
; O1-NEXT:    xchgq %rax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_store_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection284:
; O2-NEXT:    xchgq %rax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_store_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection284:
; O3-NEXT:    xchgq %rax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i64 42, ptr %a seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
; O0-LABEL: atomic64_store_seq_cst_ptr_ty:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection315:
; O0-NEXT:    xchgq %rsi, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_store_seq_cst_ptr_ty:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection285:
; O1-NEXT:    xchgq %rsi, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_store_seq_cst_ptr_ty:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection285:
; O2-NEXT:    xchgq %rsi, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_store_seq_cst_ptr_ty:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection285:
; O3-NEXT:    xchgq %rsi, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic ptr %v, ptr %a seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xchg_monotonic(ptr %a) {
; O0-LABEL: atomic64_xchg_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection316:
; O0-NEXT:    xchgq %rax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xchg_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection286:
; O1-NEXT:    xchgq %rax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xchg_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection286:
; O2-NEXT:    xchgq %rax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xchg_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection286:
; O3-NEXT:    xchgq %rax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i64 42 monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_add_monotonic(ptr %a) {
; O0-LABEL: atomic64_add_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection317:
; O0-NEXT:    lock addq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_add_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection287:
; O1-NEXT:    lock addq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_add_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection287:
; O2-NEXT:    lock addq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_add_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection287:
; O3-NEXT:    lock addq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i64 42 monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_sub_monotonic(ptr %a) {
; O0-LABEL: atomic64_sub_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection318:
; O0-NEXT:    lock subq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_sub_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection288:
; O1-NEXT:    lock subq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_sub_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection288:
; O2-NEXT:    lock subq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_sub_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection288:
; O3-NEXT:    lock subq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i64 42 monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_and_monotonic(ptr %a) {
; O0-LABEL: atomic64_and_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection319:
; O0-NEXT:    lock andq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_and_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection289:
; O1-NEXT:    lock andq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_and_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection289:
; O2-NEXT:    lock andq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_and_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection289:
; O3-NEXT:    lock andq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i64 42 monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_or_monotonic(ptr %a) {
; O0-LABEL: atomic64_or_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection320:
; O0-NEXT:    lock orq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_or_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection290:
; O1-NEXT:    lock orq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_or_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection290:
; O2-NEXT:    lock orq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_or_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection290:
; O3-NEXT:    lock orq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i64 42 monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xor_monotonic(ptr %a) {
; O0-LABEL: atomic64_xor_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection321:
; O0-NEXT:    lock xorq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xor_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection291:
; O1-NEXT:    lock xorq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xor_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection291:
; O2-NEXT:    lock xorq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xor_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection291:
; O3-NEXT:    lock xorq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i64 42 monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_nand_monotonic(ptr %a) {
; O0-LABEL: atomic64_nand_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection322:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .LBB162_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %esi
; O0-NEXT:  .Lpcsection323:
; O0-NEXT:    notl %esi
; O0-NEXT:  .Lpcsection324:
; O0-NEXT:    # implicit-def: $rdx
; O0-NEXT:    movl %esi, %edx
; O0-NEXT:  .Lpcsection325:
; O0-NEXT:    orq $-43, %rdx
; O0-NEXT:  .Lpcsection326:
; O0-NEXT:    lock cmpxchgq %rdx, (%rcx)
; O0-NEXT:  .Lpcsection327:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection328:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection329:
; O0-NEXT:    jne .LBB162_2
; O0-NEXT:    jmp .LBB162_1
; O0-NEXT:  .LBB162_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_nand_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection292:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB162_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection293:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection294:
; O1-NEXT:    orq $-43, %rcx
; O1-NEXT:  .Lpcsection295:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection296:
; O1-NEXT:    jne .LBB162_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_nand_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection292:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB162_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection293:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection294:
; O2-NEXT:    orq $-43, %rcx
; O2-NEXT:  .Lpcsection295:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection296:
; O2-NEXT:    jne .LBB162_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_nand_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection292:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB162_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection293:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection294:
; O3-NEXT:    orq $-43, %rcx
; O3-NEXT:  .Lpcsection295:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection296:
; O3-NEXT:    jne .LBB162_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i64 42 monotonic, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xchg_acquire(ptr %a) {
; O0-LABEL: atomic64_xchg_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection330:
; O0-NEXT:    xchgq %rax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xchg_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection297:
; O1-NEXT:    xchgq %rax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xchg_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection297:
; O2-NEXT:    xchgq %rax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xchg_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection297:
; O3-NEXT:    xchgq %rax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i64 42 acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_add_acquire(ptr %a) {
; O0-LABEL: atomic64_add_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection331:
; O0-NEXT:    lock addq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_add_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection298:
; O1-NEXT:    lock addq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_add_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection298:
; O2-NEXT:    lock addq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_add_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection298:
; O3-NEXT:    lock addq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i64 42 acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_sub_acquire(ptr %a) {
; O0-LABEL: atomic64_sub_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection332:
; O0-NEXT:    lock subq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_sub_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection299:
; O1-NEXT:    lock subq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_sub_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection299:
; O2-NEXT:    lock subq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_sub_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection299:
; O3-NEXT:    lock subq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i64 42 acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_and_acquire(ptr %a) {
; O0-LABEL: atomic64_and_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection333:
; O0-NEXT:    lock andq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_and_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection300:
; O1-NEXT:    lock andq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_and_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection300:
; O2-NEXT:    lock andq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_and_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection300:
; O3-NEXT:    lock andq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i64 42 acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_or_acquire(ptr %a) {
; O0-LABEL: atomic64_or_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection334:
; O0-NEXT:    lock orq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_or_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection301:
; O1-NEXT:    lock orq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_or_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection301:
; O2-NEXT:    lock orq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_or_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection301:
; O3-NEXT:    lock orq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i64 42 acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xor_acquire(ptr %a) {
; O0-LABEL: atomic64_xor_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection335:
; O0-NEXT:    lock xorq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xor_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection302:
; O1-NEXT:    lock xorq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xor_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection302:
; O2-NEXT:    lock xorq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xor_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection302:
; O3-NEXT:    lock xorq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i64 42 acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_nand_acquire(ptr %a) {
; O0-LABEL: atomic64_nand_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection336:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .LBB169_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %esi
; O0-NEXT:  .Lpcsection337:
; O0-NEXT:    notl %esi
; O0-NEXT:  .Lpcsection338:
; O0-NEXT:    # implicit-def: $rdx
; O0-NEXT:    movl %esi, %edx
; O0-NEXT:  .Lpcsection339:
; O0-NEXT:    orq $-43, %rdx
; O0-NEXT:  .Lpcsection340:
; O0-NEXT:    lock cmpxchgq %rdx, (%rcx)
; O0-NEXT:  .Lpcsection341:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection342:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection343:
; O0-NEXT:    jne .LBB169_2
; O0-NEXT:    jmp .LBB169_1
; O0-NEXT:  .LBB169_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_nand_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection303:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB169_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection304:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection305:
; O1-NEXT:    orq $-43, %rcx
; O1-NEXT:  .Lpcsection306:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection307:
; O1-NEXT:    jne .LBB169_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_nand_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection303:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB169_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection304:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection305:
; O2-NEXT:    orq $-43, %rcx
; O2-NEXT:  .Lpcsection306:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection307:
; O2-NEXT:    jne .LBB169_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_nand_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection303:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB169_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection304:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection305:
; O3-NEXT:    orq $-43, %rcx
; O3-NEXT:  .Lpcsection306:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection307:
; O3-NEXT:    jne .LBB169_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i64 42 acquire, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xchg_release(ptr %a) {
; O0-LABEL: atomic64_xchg_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection344:
; O0-NEXT:    xchgq %rax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xchg_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection308:
; O1-NEXT:    xchgq %rax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xchg_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection308:
; O2-NEXT:    xchgq %rax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xchg_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection308:
; O3-NEXT:    xchgq %rax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i64 42 release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_add_release(ptr %a) {
; O0-LABEL: atomic64_add_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection345:
; O0-NEXT:    lock addq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_add_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection309:
; O1-NEXT:    lock addq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_add_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection309:
; O2-NEXT:    lock addq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_add_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection309:
; O3-NEXT:    lock addq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i64 42 release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_sub_release(ptr %a) {
; O0-LABEL: atomic64_sub_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection346:
; O0-NEXT:    lock subq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_sub_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection310:
; O1-NEXT:    lock subq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_sub_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection310:
; O2-NEXT:    lock subq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_sub_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection310:
; O3-NEXT:    lock subq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i64 42 release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_and_release(ptr %a) {
; O0-LABEL: atomic64_and_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection347:
; O0-NEXT:    lock andq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_and_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection311:
; O1-NEXT:    lock andq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_and_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection311:
; O2-NEXT:    lock andq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_and_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection311:
; O3-NEXT:    lock andq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i64 42 release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_or_release(ptr %a) {
; O0-LABEL: atomic64_or_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection348:
; O0-NEXT:    lock orq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_or_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection312:
; O1-NEXT:    lock orq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_or_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection312:
; O2-NEXT:    lock orq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_or_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection312:
; O3-NEXT:    lock orq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i64 42 release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xor_release(ptr %a) {
; O0-LABEL: atomic64_xor_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection349:
; O0-NEXT:    lock xorq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xor_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection313:
; O1-NEXT:    lock xorq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xor_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection313:
; O2-NEXT:    lock xorq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xor_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection313:
; O3-NEXT:    lock xorq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i64 42 release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_nand_release(ptr %a) {
; O0-LABEL: atomic64_nand_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection350:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .LBB176_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %esi
; O0-NEXT:  .Lpcsection351:
; O0-NEXT:    notl %esi
; O0-NEXT:  .Lpcsection352:
; O0-NEXT:    # implicit-def: $rdx
; O0-NEXT:    movl %esi, %edx
; O0-NEXT:  .Lpcsection353:
; O0-NEXT:    orq $-43, %rdx
; O0-NEXT:  .Lpcsection354:
; O0-NEXT:    lock cmpxchgq %rdx, (%rcx)
; O0-NEXT:  .Lpcsection355:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection356:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection357:
; O0-NEXT:    jne .LBB176_2
; O0-NEXT:    jmp .LBB176_1
; O0-NEXT:  .LBB176_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_nand_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection314:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB176_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection315:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection316:
; O1-NEXT:    orq $-43, %rcx
; O1-NEXT:  .Lpcsection317:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection318:
; O1-NEXT:    jne .LBB176_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_nand_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection314:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB176_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection315:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection316:
; O2-NEXT:    orq $-43, %rcx
; O2-NEXT:  .Lpcsection317:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection318:
; O2-NEXT:    jne .LBB176_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_nand_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection314:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB176_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection315:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection316:
; O3-NEXT:    orq $-43, %rcx
; O3-NEXT:  .Lpcsection317:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection318:
; O3-NEXT:    jne .LBB176_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i64 42 release, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xchg_acq_rel(ptr %a) {
; O0-LABEL: atomic64_xchg_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection358:
; O0-NEXT:    xchgq %rax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xchg_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection319:
; O1-NEXT:    xchgq %rax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xchg_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection319:
; O2-NEXT:    xchgq %rax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xchg_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection319:
; O3-NEXT:    xchgq %rax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i64 42 acq_rel, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_add_acq_rel(ptr %a) {
; O0-LABEL: atomic64_add_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection359:
; O0-NEXT:    lock addq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_add_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection320:
; O1-NEXT:    lock addq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_add_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection320:
; O2-NEXT:    lock addq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_add_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection320:
; O3-NEXT:    lock addq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i64 42 acq_rel, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_sub_acq_rel(ptr %a) {
; O0-LABEL: atomic64_sub_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection360:
; O0-NEXT:    lock subq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_sub_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection321:
; O1-NEXT:    lock subq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_sub_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection321:
; O2-NEXT:    lock subq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_sub_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection321:
; O3-NEXT:    lock subq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i64 42 acq_rel, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_and_acq_rel(ptr %a) {
; O0-LABEL: atomic64_and_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection361:
; O0-NEXT:    lock andq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_and_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection322:
; O1-NEXT:    lock andq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_and_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection322:
; O2-NEXT:    lock andq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_and_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection322:
; O3-NEXT:    lock andq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i64 42 acq_rel, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_or_acq_rel(ptr %a) {
; O0-LABEL: atomic64_or_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection362:
; O0-NEXT:    lock orq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_or_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection323:
; O1-NEXT:    lock orq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_or_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection323:
; O2-NEXT:    lock orq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_or_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection323:
; O3-NEXT:    lock orq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i64 42 acq_rel, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xor_acq_rel(ptr %a) {
; O0-LABEL: atomic64_xor_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection363:
; O0-NEXT:    lock xorq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xor_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection324:
; O1-NEXT:    lock xorq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xor_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection324:
; O2-NEXT:    lock xorq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xor_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection324:
; O3-NEXT:    lock xorq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i64 42 acq_rel, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_nand_acq_rel(ptr %a) {
; O0-LABEL: atomic64_nand_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection364:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .LBB183_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %esi
; O0-NEXT:  .Lpcsection365:
; O0-NEXT:    notl %esi
; O0-NEXT:  .Lpcsection366:
; O0-NEXT:    # implicit-def: $rdx
; O0-NEXT:    movl %esi, %edx
; O0-NEXT:  .Lpcsection367:
; O0-NEXT:    orq $-43, %rdx
; O0-NEXT:  .Lpcsection368:
; O0-NEXT:    lock cmpxchgq %rdx, (%rcx)
; O0-NEXT:  .Lpcsection369:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection370:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection371:
; O0-NEXT:    jne .LBB183_2
; O0-NEXT:    jmp .LBB183_1
; O0-NEXT:  .LBB183_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_nand_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection325:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB183_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection326:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection327:
; O1-NEXT:    orq $-43, %rcx
; O1-NEXT:  .Lpcsection328:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection329:
; O1-NEXT:    jne .LBB183_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_nand_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection325:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB183_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection326:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection327:
; O2-NEXT:    orq $-43, %rcx
; O2-NEXT:  .Lpcsection328:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection329:
; O2-NEXT:    jne .LBB183_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_nand_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection325:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB183_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection326:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection327:
; O3-NEXT:    orq $-43, %rcx
; O3-NEXT:  .Lpcsection328:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection329:
; O3-NEXT:    jne .LBB183_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i64 42 acq_rel, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xchg_seq_cst(ptr %a) {
; O0-LABEL: atomic64_xchg_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:  .Lpcsection372:
; O0-NEXT:    xchgq %rax, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xchg_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection330:
; O1-NEXT:    xchgq %rax, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xchg_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection330:
; O2-NEXT:    xchgq %rax, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xchg_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection330:
; O3-NEXT:    xchgq %rax, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i64 42 seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_add_seq_cst(ptr %a) {
; O0-LABEL: atomic64_add_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection373:
; O0-NEXT:    lock addq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_add_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection331:
; O1-NEXT:    lock addq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_add_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection331:
; O2-NEXT:    lock addq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_add_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection331:
; O3-NEXT:    lock addq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i64 42 seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_sub_seq_cst(ptr %a) {
; O0-LABEL: atomic64_sub_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection374:
; O0-NEXT:    lock subq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_sub_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection332:
; O1-NEXT:    lock subq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_sub_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection332:
; O2-NEXT:    lock subq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_sub_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection332:
; O3-NEXT:    lock subq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i64 42 seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_and_seq_cst(ptr %a) {
; O0-LABEL: atomic64_and_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection375:
; O0-NEXT:    lock andq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_and_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection333:
; O1-NEXT:    lock andq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_and_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection333:
; O2-NEXT:    lock andq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_and_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection333:
; O3-NEXT:    lock andq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i64 42 seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_or_seq_cst(ptr %a) {
; O0-LABEL: atomic64_or_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection376:
; O0-NEXT:    lock orq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_or_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection334:
; O1-NEXT:    lock orq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_or_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection334:
; O2-NEXT:    lock orq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_or_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection334:
; O3-NEXT:    lock orq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i64 42 seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_xor_seq_cst(ptr %a) {
; O0-LABEL: atomic64_xor_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection377:
; O0-NEXT:    lock xorq $42, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_xor_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection335:
; O1-NEXT:    lock xorq $42, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_xor_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection335:
; O2-NEXT:    lock xorq $42, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_xor_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection335:
; O3-NEXT:    lock xorq $42, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i64 42 seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_nand_seq_cst(ptr %a) {
; O0-LABEL: atomic64_nand_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo, %rax
; O0-NEXT:  .Lpcsection378:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .LBB190_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movl %eax, %esi
; O0-NEXT:  .Lpcsection379:
; O0-NEXT:    notl %esi
; O0-NEXT:  .Lpcsection380:
; O0-NEXT:    # implicit-def: $rdx
; O0-NEXT:    movl %esi, %edx
; O0-NEXT:  .Lpcsection381:
; O0-NEXT:    orq $-43, %rdx
; O0-NEXT:  .Lpcsection382:
; O0-NEXT:    lock cmpxchgq %rdx, (%rcx)
; O0-NEXT:  .Lpcsection383:
; O0-NEXT:    sete %cl
; O0-NEXT:  .Lpcsection384:
; O0-NEXT:    testb $1, %cl
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection385:
; O0-NEXT:    jne .LBB190_2
; O0-NEXT:    jmp .LBB190_1
; O0-NEXT:  .LBB190_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_nand_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection336:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB190_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ecx
; O1-NEXT:  .Lpcsection337:
; O1-NEXT:    notl %ecx
; O1-NEXT:  .Lpcsection338:
; O1-NEXT:    orq $-43, %rcx
; O1-NEXT:  .Lpcsection339:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection340:
; O1-NEXT:    jne .LBB190_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_nand_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection336:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB190_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ecx
; O2-NEXT:  .Lpcsection337:
; O2-NEXT:    notl %ecx
; O2-NEXT:  .Lpcsection338:
; O2-NEXT:    orq $-43, %rcx
; O2-NEXT:  .Lpcsection339:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection340:
; O2-NEXT:    jne .LBB190_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_nand_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection336:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB190_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ecx
; O3-NEXT:  .Lpcsection337:
; O3-NEXT:    notl %ecx
; O3-NEXT:  .Lpcsection338:
; O3-NEXT:    orq $-43, %rcx
; O3-NEXT:  .Lpcsection339:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection340:
; O3-NEXT:    jne .LBB190_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i64 42 seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_cas_monotonic(ptr %a) {
; O0-LABEL: atomic64_cas_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection386:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection387:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection388:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection389:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection390:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_cas_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection341:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection342:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection343:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection344:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection345:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection346:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_cas_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection341:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection342:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection343:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection344:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection345:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection346:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_cas_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection341:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection342:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection343:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection344:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection345:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection346:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i64 42, i64 1 monotonic monotonic, align 8, !pcsections !0
  %y = cmpxchg ptr %a, i64 42, i64 1 monotonic acquire, align 8, !pcsections !0
  %z = cmpxchg ptr %a, i64 42, i64 1 monotonic seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_cas_acquire(ptr %a) {
; O0-LABEL: atomic64_cas_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection391:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection392:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection393:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection394:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection395:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_cas_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection347:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection348:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection349:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection350:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection351:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection352:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_cas_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection347:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection348:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection349:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection350:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection351:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection352:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_cas_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection347:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection348:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection349:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection350:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection351:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection352:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i64 42, i64 1 acquire monotonic, align 8, !pcsections !0
  %y = cmpxchg ptr %a, i64 42, i64 1 acquire acquire, align 8, !pcsections !0
  %z = cmpxchg ptr %a, i64 42, i64 1 acquire seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_cas_release(ptr %a) {
; O0-LABEL: atomic64_cas_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection396:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection397:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection398:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection399:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection400:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_cas_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection353:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection354:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection355:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection356:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection357:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection358:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_cas_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection353:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection354:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection355:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection356:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection357:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection358:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_cas_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection353:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection354:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection355:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection356:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection357:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection358:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i64 42, i64 1 release monotonic, align 8, !pcsections !0
  %y = cmpxchg ptr %a, i64 42, i64 1 release acquire, align 8, !pcsections !0
  %z = cmpxchg ptr %a, i64 42, i64 1 release seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_cas_acq_rel(ptr %a) {
; O0-LABEL: atomic64_cas_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection401:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection402:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection403:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection404:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection405:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_cas_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection359:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection360:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection361:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection362:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection363:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection364:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_cas_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection359:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection360:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection361:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection362:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection363:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection364:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_cas_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection359:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection360:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection361:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection362:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection363:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection364:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i64 42, i64 1 acq_rel monotonic, align 8, !pcsections !0
  %y = cmpxchg ptr %a, i64 42, i64 1 acq_rel acquire, align 8, !pcsections !0
  %z = cmpxchg ptr %a, i64 42, i64 1 acq_rel seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic64_cas_seq_cst(ptr %a) {
; O0-LABEL: atomic64_cas_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movl $1, %ecx
; O0-NEXT:  .Lpcsection406:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection407:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection408:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:  .Lpcsection409:
; O0-NEXT:    # kill: def $rdx killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:  .Lpcsection410:
; O0-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O0-NEXT:    movq $3, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_cas_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:    movl $1, %ecx
; O1-NEXT:  .Lpcsection365:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection366:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection367:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection368:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:  .Lpcsection369:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection370:
; O1-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O1-NEXT:    movq $3, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_cas_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:    movl $1, %ecx
; O2-NEXT:  .Lpcsection365:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection366:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection367:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection368:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:  .Lpcsection369:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection370:
; O2-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O2-NEXT:    movq $3, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_cas_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:    movl $1, %ecx
; O3-NEXT:  .Lpcsection365:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection366:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection367:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection368:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:  .Lpcsection369:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection370:
; O3-NEXT:    lock cmpxchgq %rcx, (%rdi)
; O3-NEXT:    movq $3, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i64 42, i64 1 seq_cst monotonic, align 8, !pcsections !0
  %y = cmpxchg ptr %a, i64 42, i64 1 seq_cst acquire, align 8, !pcsections !0
  %z = cmpxchg ptr %a, i64 42, i64 1 seq_cst seq_cst, align 8, !pcsections !0
  store volatile i64 3, ptr @foo, align 8
  ret void
}

define void @atomic64_cas_seq_cst_ptr_ty(ptr %a, ptr %v1, ptr %v2) {
; O0-LABEL: atomic64_cas_seq_cst_ptr_ty:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq %rsi, %rax
; O0-NEXT:    movq foo(%rip), %rcx
; O0-NEXT:  .Lpcsection411:
; O0-NEXT:    lock cmpxchgq %rdx, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic64_cas_seq_cst_ptr_ty:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq %rsi, %rax
; O1-NEXT:    movq foo(%rip), %rcx
; O1-NEXT:  .Lpcsection371:
; O1-NEXT:    lock cmpxchgq %rdx, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic64_cas_seq_cst_ptr_ty:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq %rsi, %rax
; O2-NEXT:    movq foo(%rip), %rcx
; O2-NEXT:  .Lpcsection371:
; O2-NEXT:    lock cmpxchgq %rdx, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic64_cas_seq_cst_ptr_ty:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq %rsi, %rax
; O3-NEXT:    movq foo(%rip), %rcx
; O3-NEXT:  .Lpcsection371:
; O3-NEXT:    lock cmpxchgq %rdx, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, ptr %v1, ptr %v2 seq_cst seq_cst, align 8, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define i64 @atomic_use_cond(ptr %a) {
; O0-LABEL: atomic_use_cond:
; O0:       # %bb.0: # %entry
; O0-NEXT:  .Lpcsection412:
; O0-NEXT:    lock decq (%rdi)
; O0-NEXT:  .Lpcsection413:
; O0-NEXT:    sete %al
; O0-NEXT:    testb $1, %al
; O0-NEXT:    je .LBB197_2
; O0-NEXT:  # %bb.1: # %then
; O0-NEXT:    movl $1, %eax
; O0-NEXT:    retq
; O0-NEXT:  .LBB197_2: # %else
; O0-NEXT:    movl $2, %eax
; O0-NEXT:    retq
;
; O1-LABEL: atomic_use_cond:
; O1:       # %bb.0: # %entry
; O1-NEXT:  .Lpcsection372:
; O1-NEXT:    lock decq (%rdi)
; O1-NEXT:    jne .LBB197_2
; O1-NEXT:  # %bb.1: # %then
; O1-NEXT:    movl $1, %eax
; O1-NEXT:    retq
; O1-NEXT:  .LBB197_2: # %else
; O1-NEXT:    movl $2, %eax
; O1-NEXT:    retq
;
; O2-LABEL: atomic_use_cond:
; O2:       # %bb.0: # %entry
; O2-NEXT:  .Lpcsection372:
; O2-NEXT:    lock decq (%rdi)
; O2-NEXT:    jne .LBB197_2
; O2-NEXT:  # %bb.1: # %then
; O2-NEXT:    movl $1, %eax
; O2-NEXT:    retq
; O2-NEXT:  .LBB197_2: # %else
; O2-NEXT:    movl $2, %eax
; O2-NEXT:    retq
;
; O3-LABEL: atomic_use_cond:
; O3:       # %bb.0: # %entry
; O3-NEXT:  .Lpcsection372:
; O3-NEXT:    lock decq (%rdi)
; O3-NEXT:    jne .LBB197_2
; O3-NEXT:  # %bb.1: # %then
; O3-NEXT:    movl $1, %eax
; O3-NEXT:    retq
; O3-NEXT:  .LBB197_2: # %else
; O3-NEXT:    movl $2, %eax
; O3-NEXT:    retq
entry:
  %x = atomicrmw sub ptr %a, i64 1 seq_cst, align 8, !pcsections !0
  %y = icmp eq i64 %x, 1
  br i1 %y, label %then, label %else

then:
  ret i64 1

else:
  ret i64 2
}

define i128 @atomic128_load_unordered(ptr %a) {
; O0-LABEL: atomic128_load_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection414:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ebx
; O0-NEXT:    movq %rbx, %rax
; O0-NEXT:    movq %rbx, %rdx
; O0-NEXT:    movq %rbx, %rcx
; O0-NEXT:  .Lpcsection415:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo(%rip)
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_load_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection373:
; O1-NEXT:    xorl %eax, %eax
; O1-NEXT:  .Lpcsection374:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection375:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection376:
; O1-NEXT:    xorl %ebx, %ebx
; O1-NEXT:  .Lpcsection377:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_load_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection373:
; O2-NEXT:    xorl %eax, %eax
; O2-NEXT:  .Lpcsection374:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection375:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection376:
; O2-NEXT:    xorl %ebx, %ebx
; O2-NEXT:  .Lpcsection377:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_load_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection373:
; O3-NEXT:    xorl %eax, %eax
; O3-NEXT:  .Lpcsection374:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection375:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection376:
; O3-NEXT:    xorl %ebx, %ebx
; O3-NEXT:  .Lpcsection377:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i128, ptr %a unordered, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i128 %x
}

define i128 @atomic128_load_monotonic(ptr %a) {
; O0-LABEL: atomic128_load_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection416:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ebx
; O0-NEXT:    movq %rbx, %rax
; O0-NEXT:    movq %rbx, %rdx
; O0-NEXT:    movq %rbx, %rcx
; O0-NEXT:  .Lpcsection417:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo(%rip)
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_load_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection378:
; O1-NEXT:    xorl %eax, %eax
; O1-NEXT:  .Lpcsection379:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection380:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection381:
; O1-NEXT:    xorl %ebx, %ebx
; O1-NEXT:  .Lpcsection382:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_load_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection378:
; O2-NEXT:    xorl %eax, %eax
; O2-NEXT:  .Lpcsection379:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection380:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection381:
; O2-NEXT:    xorl %ebx, %ebx
; O2-NEXT:  .Lpcsection382:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_load_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection378:
; O3-NEXT:    xorl %eax, %eax
; O3-NEXT:  .Lpcsection379:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection380:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection381:
; O3-NEXT:    xorl %ebx, %ebx
; O3-NEXT:  .Lpcsection382:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i128, ptr %a monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i128 %x
}

define i128 @atomic128_load_acquire(ptr %a) {
; O0-LABEL: atomic128_load_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection418:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ebx
; O0-NEXT:    movq %rbx, %rax
; O0-NEXT:    movq %rbx, %rdx
; O0-NEXT:    movq %rbx, %rcx
; O0-NEXT:  .Lpcsection419:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo(%rip)
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_load_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection383:
; O1-NEXT:    xorl %eax, %eax
; O1-NEXT:  .Lpcsection384:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection385:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection386:
; O1-NEXT:    xorl %ebx, %ebx
; O1-NEXT:  .Lpcsection387:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_load_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection383:
; O2-NEXT:    xorl %eax, %eax
; O2-NEXT:  .Lpcsection384:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection385:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection386:
; O2-NEXT:    xorl %ebx, %ebx
; O2-NEXT:  .Lpcsection387:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_load_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection383:
; O3-NEXT:    xorl %eax, %eax
; O3-NEXT:  .Lpcsection384:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection385:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection386:
; O3-NEXT:    xorl %ebx, %ebx
; O3-NEXT:  .Lpcsection387:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i128, ptr %a acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i128 %x
}

define i128 @atomic128_load_seq_cst(ptr %a) {
; O0-LABEL: atomic128_load_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection420:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ebx
; O0-NEXT:    movq %rbx, %rax
; O0-NEXT:    movq %rbx, %rdx
; O0-NEXT:    movq %rbx, %rcx
; O0-NEXT:  .Lpcsection421:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo(%rip)
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_load_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection388:
; O1-NEXT:    xorl %eax, %eax
; O1-NEXT:  .Lpcsection389:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection390:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection391:
; O1-NEXT:    xorl %ebx, %ebx
; O1-NEXT:  .Lpcsection392:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_load_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection388:
; O2-NEXT:    xorl %eax, %eax
; O2-NEXT:  .Lpcsection389:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection390:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection391:
; O2-NEXT:    xorl %ebx, %ebx
; O2-NEXT:  .Lpcsection392:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_load_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection388:
; O3-NEXT:    xorl %eax, %eax
; O3-NEXT:  .Lpcsection389:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection390:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection391:
; O3-NEXT:    xorl %ebx, %ebx
; O3-NEXT:  .Lpcsection392:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic i128, ptr %a seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret i128 %x
}

define ptr @atomic128_load_seq_cst_ptr_ty(ptr %a) {
; O0-LABEL: atomic128_load_seq_cst_ptr_ty:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection422:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_load_seq_cst_ptr_ty:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection393:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_load_seq_cst_ptr_ty:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection393:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_load_seq_cst_ptr_ty:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection393:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = load atomic ptr, ptr %a seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret ptr %x
}

define void @atomic128_store_unordered(ptr %a) {
; O0-LABEL: atomic128_store_unordered:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection423:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection424:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection425:
; O0-NEXT:    jmp .LBB203_1
; O0-NEXT:  .LBB203_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection426:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection427:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection428:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection429:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection430:
; O0-NEXT:    jne .LBB203_1
; O0-NEXT:    jmp .LBB203_2
; O0-NEXT:  .LBB203_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_store_unordered:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection394:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection395:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection396:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB203_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection397:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection398:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection399:
; O1-NEXT:    jne .LBB203_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_store_unordered:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection394:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection395:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection396:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB203_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection397:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection398:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection399:
; O2-NEXT:    jne .LBB203_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_store_unordered:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection394:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection395:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection396:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB203_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection397:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection398:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection399:
; O3-NEXT:    jne .LBB203_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i128 42, ptr %a unordered, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_store_monotonic(ptr %a) {
; O0-LABEL: atomic128_store_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection431:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection432:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection433:
; O0-NEXT:    jmp .LBB204_1
; O0-NEXT:  .LBB204_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection434:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection435:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection436:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection437:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection438:
; O0-NEXT:    jne .LBB204_1
; O0-NEXT:    jmp .LBB204_2
; O0-NEXT:  .LBB204_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_store_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection400:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection401:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection402:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB204_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection403:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection404:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection405:
; O1-NEXT:    jne .LBB204_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_store_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection400:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection401:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection402:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB204_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection403:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection404:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection405:
; O2-NEXT:    jne .LBB204_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_store_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection400:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection401:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection402:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB204_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection403:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection404:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection405:
; O3-NEXT:    jne .LBB204_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i128 42, ptr %a monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_store_release(ptr %a) {
; O0-LABEL: atomic128_store_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection439:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection440:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection441:
; O0-NEXT:    jmp .LBB205_1
; O0-NEXT:  .LBB205_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection442:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection443:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection444:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection445:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection446:
; O0-NEXT:    jne .LBB205_1
; O0-NEXT:    jmp .LBB205_2
; O0-NEXT:  .LBB205_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_store_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection406:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection407:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection408:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB205_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection409:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection410:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection411:
; O1-NEXT:    jne .LBB205_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_store_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection406:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection407:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection408:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB205_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection409:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection410:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection411:
; O2-NEXT:    jne .LBB205_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_store_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection406:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection407:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection408:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB205_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection409:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection410:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection411:
; O3-NEXT:    jne .LBB205_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i128 42, ptr %a release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_store_seq_cst(ptr %a) {
; O0-LABEL: atomic128_store_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection447:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection448:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection449:
; O0-NEXT:    jmp .LBB206_1
; O0-NEXT:  .LBB206_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection450:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection451:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection452:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection453:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection454:
; O0-NEXT:    jne .LBB206_1
; O0-NEXT:    jmp .LBB206_2
; O0-NEXT:  .LBB206_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_store_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection412:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection413:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection414:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB206_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection415:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection416:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection417:
; O1-NEXT:    jne .LBB206_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_store_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection412:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection413:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection414:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB206_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection415:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection416:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection417:
; O2-NEXT:    jne .LBB206_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_store_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection412:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection413:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection414:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB206_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection415:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection416:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection417:
; O3-NEXT:    jne .LBB206_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic i128 42, ptr %a seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_store_seq_cst_ptr_ty(ptr %a, ptr %v) {
; O0-LABEL: atomic128_store_seq_cst_ptr_ty:
; O0:       # %bb.0: # %entry
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection455:
; O0-NEXT:    xchgq %rsi, (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_store_seq_cst_ptr_ty:
; O1:       # %bb.0: # %entry
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection418:
; O1-NEXT:    xchgq %rsi, (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_store_seq_cst_ptr_ty:
; O2:       # %bb.0: # %entry
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection418:
; O2-NEXT:    xchgq %rsi, (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_store_seq_cst_ptr_ty:
; O3:       # %bb.0: # %entry
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection418:
; O3-NEXT:    xchgq %rsi, (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  store atomic ptr %v, ptr %a seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xchg_monotonic(ptr %a) {
; O0-LABEL: atomic128_xchg_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection456:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection457:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection458:
; O0-NEXT:    jmp .LBB208_1
; O0-NEXT:  .LBB208_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection459:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection460:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection461:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection462:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection463:
; O0-NEXT:    jne .LBB208_1
; O0-NEXT:    jmp .LBB208_2
; O0-NEXT:  .LBB208_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xchg_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection419:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection420:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection421:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB208_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection422:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection423:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection424:
; O1-NEXT:    jne .LBB208_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xchg_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection419:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection420:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection421:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB208_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection422:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection423:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection424:
; O2-NEXT:    jne .LBB208_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xchg_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection419:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection420:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection421:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB208_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection422:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection423:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection424:
; O3-NEXT:    jne .LBB208_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i128 42 monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_add_monotonic(ptr %a) {
; O0-LABEL: atomic128_add_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection464:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection465:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection466:
; O0-NEXT:    jmp .LBB209_1
; O0-NEXT:  .LBB209_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection467:
; O0-NEXT:    addq $42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection468:
; O0-NEXT:    adcq $0, %rcx
; O0-NEXT:  .Lpcsection469:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection470:
; O0-NEXT:    jne .LBB209_1
; O0-NEXT:    jmp .LBB209_2
; O0-NEXT:  .LBB209_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_add_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection425:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection426:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB209_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection427:
; O1-NEXT:    addq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection428:
; O1-NEXT:    adcq $0, %rcx
; O1-NEXT:  .Lpcsection429:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection430:
; O1-NEXT:    jne .LBB209_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_add_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection425:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection426:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB209_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection427:
; O2-NEXT:    addq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection428:
; O2-NEXT:    adcq $0, %rcx
; O2-NEXT:  .Lpcsection429:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection430:
; O2-NEXT:    jne .LBB209_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_add_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection425:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection426:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB209_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection427:
; O3-NEXT:    addq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection428:
; O3-NEXT:    adcq $0, %rcx
; O3-NEXT:  .Lpcsection429:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection430:
; O3-NEXT:    jne .LBB209_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i128 42 monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_sub_monotonic(ptr %a) {
; O0-LABEL: atomic128_sub_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection471:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection472:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection473:
; O0-NEXT:    jmp .LBB210_1
; O0-NEXT:  .LBB210_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection474:
; O0-NEXT:    addq $-42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection475:
; O0-NEXT:    adcq $-1, %rcx
; O0-NEXT:  .Lpcsection476:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection477:
; O0-NEXT:    jne .LBB210_1
; O0-NEXT:    jmp .LBB210_2
; O0-NEXT:  .LBB210_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_sub_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection431:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection432:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB210_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection433:
; O1-NEXT:    addq $-42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection434:
; O1-NEXT:    adcq $-1, %rcx
; O1-NEXT:  .Lpcsection435:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection436:
; O1-NEXT:    jne .LBB210_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_sub_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection431:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection432:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB210_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection433:
; O2-NEXT:    addq $-42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection434:
; O2-NEXT:    adcq $-1, %rcx
; O2-NEXT:  .Lpcsection435:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection436:
; O2-NEXT:    jne .LBB210_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_sub_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection431:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection432:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB210_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection433:
; O3-NEXT:    addq $-42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection434:
; O3-NEXT:    adcq $-1, %rcx
; O3-NEXT:  .Lpcsection435:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection436:
; O3-NEXT:    jne .LBB210_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i128 42 monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_and_monotonic(ptr %a) {
; O0-LABEL: atomic128_and_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection478:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection479:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection480:
; O0-NEXT:    jmp .LBB211_1
; O0-NEXT:  .LBB211_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection481:
; O0-NEXT:    andl $42, %ecx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection482:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection483:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection484:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection485:
; O0-NEXT:    jne .LBB211_1
; O0-NEXT:    jmp .LBB211_2
; O0-NEXT:  .LBB211_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_and_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection437:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection438:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB211_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection439:
; O1-NEXT:    andl $42, %ebx
; O1-NEXT:  .Lpcsection440:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection441:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection442:
; O1-NEXT:    jne .LBB211_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_and_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection437:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection438:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB211_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection439:
; O2-NEXT:    andl $42, %ebx
; O2-NEXT:  .Lpcsection440:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection441:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection442:
; O2-NEXT:    jne .LBB211_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_and_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection437:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection438:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB211_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection439:
; O3-NEXT:    andl $42, %ebx
; O3-NEXT:  .Lpcsection440:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection441:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection442:
; O3-NEXT:    jne .LBB211_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i128 42 monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_or_monotonic(ptr %a) {
; O0-LABEL: atomic128_or_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection486:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection487:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection488:
; O0-NEXT:    jmp .LBB212_1
; O0-NEXT:  .LBB212_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection489:
; O0-NEXT:    orq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection490:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection491:
; O0-NEXT:    jne .LBB212_1
; O0-NEXT:    jmp .LBB212_2
; O0-NEXT:  .LBB212_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_or_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection443:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection444:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB212_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection445:
; O1-NEXT:    orq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection446:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection447:
; O1-NEXT:    jne .LBB212_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_or_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection443:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection444:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB212_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection445:
; O2-NEXT:    orq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection446:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection447:
; O2-NEXT:    jne .LBB212_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_or_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection443:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection444:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB212_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection445:
; O3-NEXT:    orq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection446:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection447:
; O3-NEXT:    jne .LBB212_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i128 42 monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xor_monotonic(ptr %a) {
; O0-LABEL: atomic128_xor_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection492:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection493:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection494:
; O0-NEXT:    jmp .LBB213_1
; O0-NEXT:  .LBB213_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection495:
; O0-NEXT:    xorq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection496:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection497:
; O0-NEXT:    jne .LBB213_1
; O0-NEXT:    jmp .LBB213_2
; O0-NEXT:  .LBB213_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xor_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection448:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection449:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB213_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection450:
; O1-NEXT:    xorq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection451:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection452:
; O1-NEXT:    jne .LBB213_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xor_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection448:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection449:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB213_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection450:
; O2-NEXT:    xorq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection451:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection452:
; O2-NEXT:    jne .LBB213_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xor_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection448:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection449:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB213_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection450:
; O3-NEXT:    xorq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection451:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection452:
; O3-NEXT:    jne .LBB213_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i128 42 monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_nand_monotonic(ptr %a) {
; O0-LABEL: atomic128_nand_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection498:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection499:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection500:
; O0-NEXT:    jmp .LBB214_1
; O0-NEXT:  .LBB214_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection501:
; O0-NEXT:    notl %ecx
; O0-NEXT:  .Lpcsection502:
; O0-NEXT:    # implicit-def: $rbx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection503:
; O0-NEXT:    orq $-43, %rbx
; O0-NEXT:  .Lpcsection504:
; O0-NEXT:    movq $-1, %rcx
; O0-NEXT:  .Lpcsection505:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection506:
; O0-NEXT:    jne .LBB214_1
; O0-NEXT:    jmp .LBB214_2
; O0-NEXT:  .LBB214_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_nand_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection453:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection454:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection455:
; O1-NEXT:    movq $-1, %rcx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB214_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection456:
; O1-NEXT:    notl %ebx
; O1-NEXT:  .Lpcsection457:
; O1-NEXT:    orq $-43, %rbx
; O1-NEXT:  .Lpcsection458:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection459:
; O1-NEXT:    jne .LBB214_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_nand_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection453:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection454:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection455:
; O2-NEXT:    movq $-1, %rcx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB214_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection456:
; O2-NEXT:    notl %ebx
; O2-NEXT:  .Lpcsection457:
; O2-NEXT:    orq $-43, %rbx
; O2-NEXT:  .Lpcsection458:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection459:
; O2-NEXT:    jne .LBB214_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_nand_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection453:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection454:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection455:
; O3-NEXT:    movq $-1, %rcx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB214_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection456:
; O3-NEXT:    notl %ebx
; O3-NEXT:  .Lpcsection457:
; O3-NEXT:    orq $-43, %rbx
; O3-NEXT:  .Lpcsection458:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection459:
; O3-NEXT:    jne .LBB214_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i128 42 monotonic, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xchg_acquire(ptr %a) {
; O0-LABEL: atomic128_xchg_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection507:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection508:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection509:
; O0-NEXT:    jmp .LBB215_1
; O0-NEXT:  .LBB215_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection510:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection511:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection512:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection513:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection514:
; O0-NEXT:    jne .LBB215_1
; O0-NEXT:    jmp .LBB215_2
; O0-NEXT:  .LBB215_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xchg_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection460:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection461:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection462:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB215_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection463:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection464:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection465:
; O1-NEXT:    jne .LBB215_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xchg_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection460:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection461:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection462:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB215_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection463:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection464:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection465:
; O2-NEXT:    jne .LBB215_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xchg_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection460:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection461:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection462:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB215_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection463:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection464:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection465:
; O3-NEXT:    jne .LBB215_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i128 42 acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_add_acquire(ptr %a) {
; O0-LABEL: atomic128_add_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection515:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection516:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection517:
; O0-NEXT:    jmp .LBB216_1
; O0-NEXT:  .LBB216_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection518:
; O0-NEXT:    addq $42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection519:
; O0-NEXT:    adcq $0, %rcx
; O0-NEXT:  .Lpcsection520:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection521:
; O0-NEXT:    jne .LBB216_1
; O0-NEXT:    jmp .LBB216_2
; O0-NEXT:  .LBB216_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_add_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection466:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection467:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB216_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection468:
; O1-NEXT:    addq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection469:
; O1-NEXT:    adcq $0, %rcx
; O1-NEXT:  .Lpcsection470:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection471:
; O1-NEXT:    jne .LBB216_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_add_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection466:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection467:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB216_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection468:
; O2-NEXT:    addq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection469:
; O2-NEXT:    adcq $0, %rcx
; O2-NEXT:  .Lpcsection470:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection471:
; O2-NEXT:    jne .LBB216_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_add_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection466:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection467:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB216_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection468:
; O3-NEXT:    addq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection469:
; O3-NEXT:    adcq $0, %rcx
; O3-NEXT:  .Lpcsection470:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection471:
; O3-NEXT:    jne .LBB216_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i128 42 acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_sub_acquire(ptr %a) {
; O0-LABEL: atomic128_sub_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection522:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection523:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection524:
; O0-NEXT:    jmp .LBB217_1
; O0-NEXT:  .LBB217_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection525:
; O0-NEXT:    addq $-42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection526:
; O0-NEXT:    adcq $-1, %rcx
; O0-NEXT:  .Lpcsection527:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection528:
; O0-NEXT:    jne .LBB217_1
; O0-NEXT:    jmp .LBB217_2
; O0-NEXT:  .LBB217_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_sub_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection472:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection473:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB217_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection474:
; O1-NEXT:    addq $-42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection475:
; O1-NEXT:    adcq $-1, %rcx
; O1-NEXT:  .Lpcsection476:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection477:
; O1-NEXT:    jne .LBB217_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_sub_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection472:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection473:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB217_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection474:
; O2-NEXT:    addq $-42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection475:
; O2-NEXT:    adcq $-1, %rcx
; O2-NEXT:  .Lpcsection476:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection477:
; O2-NEXT:    jne .LBB217_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_sub_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection472:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection473:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB217_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection474:
; O3-NEXT:    addq $-42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection475:
; O3-NEXT:    adcq $-1, %rcx
; O3-NEXT:  .Lpcsection476:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection477:
; O3-NEXT:    jne .LBB217_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i128 42 acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_and_acquire(ptr %a) {
; O0-LABEL: atomic128_and_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection529:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection530:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection531:
; O0-NEXT:    jmp .LBB218_1
; O0-NEXT:  .LBB218_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection532:
; O0-NEXT:    andl $42, %ecx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection533:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection534:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection535:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection536:
; O0-NEXT:    jne .LBB218_1
; O0-NEXT:    jmp .LBB218_2
; O0-NEXT:  .LBB218_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_and_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection478:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection479:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB218_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection480:
; O1-NEXT:    andl $42, %ebx
; O1-NEXT:  .Lpcsection481:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection482:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection483:
; O1-NEXT:    jne .LBB218_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_and_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection478:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection479:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB218_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection480:
; O2-NEXT:    andl $42, %ebx
; O2-NEXT:  .Lpcsection481:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection482:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection483:
; O2-NEXT:    jne .LBB218_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_and_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection478:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection479:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB218_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection480:
; O3-NEXT:    andl $42, %ebx
; O3-NEXT:  .Lpcsection481:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection482:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection483:
; O3-NEXT:    jne .LBB218_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i128 42 acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_or_acquire(ptr %a) {
; O0-LABEL: atomic128_or_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection537:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection538:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection539:
; O0-NEXT:    jmp .LBB219_1
; O0-NEXT:  .LBB219_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection540:
; O0-NEXT:    orq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection541:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection542:
; O0-NEXT:    jne .LBB219_1
; O0-NEXT:    jmp .LBB219_2
; O0-NEXT:  .LBB219_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_or_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection484:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection485:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB219_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection486:
; O1-NEXT:    orq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection487:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection488:
; O1-NEXT:    jne .LBB219_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_or_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection484:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection485:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB219_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection486:
; O2-NEXT:    orq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection487:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection488:
; O2-NEXT:    jne .LBB219_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_or_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection484:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection485:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB219_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection486:
; O3-NEXT:    orq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection487:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection488:
; O3-NEXT:    jne .LBB219_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i128 42 acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xor_acquire(ptr %a) {
; O0-LABEL: atomic128_xor_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection543:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection544:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection545:
; O0-NEXT:    jmp .LBB220_1
; O0-NEXT:  .LBB220_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection546:
; O0-NEXT:    xorq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection547:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection548:
; O0-NEXT:    jne .LBB220_1
; O0-NEXT:    jmp .LBB220_2
; O0-NEXT:  .LBB220_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xor_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection489:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection490:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB220_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection491:
; O1-NEXT:    xorq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection492:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection493:
; O1-NEXT:    jne .LBB220_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xor_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection489:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection490:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB220_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection491:
; O2-NEXT:    xorq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection492:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection493:
; O2-NEXT:    jne .LBB220_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xor_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection489:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection490:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB220_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection491:
; O3-NEXT:    xorq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection492:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection493:
; O3-NEXT:    jne .LBB220_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i128 42 acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_nand_acquire(ptr %a) {
; O0-LABEL: atomic128_nand_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection549:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection550:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection551:
; O0-NEXT:    jmp .LBB221_1
; O0-NEXT:  .LBB221_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection552:
; O0-NEXT:    notl %ecx
; O0-NEXT:  .Lpcsection553:
; O0-NEXT:    # implicit-def: $rbx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection554:
; O0-NEXT:    orq $-43, %rbx
; O0-NEXT:  .Lpcsection555:
; O0-NEXT:    movq $-1, %rcx
; O0-NEXT:  .Lpcsection556:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection557:
; O0-NEXT:    jne .LBB221_1
; O0-NEXT:    jmp .LBB221_2
; O0-NEXT:  .LBB221_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_nand_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection494:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection495:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection496:
; O1-NEXT:    movq $-1, %rcx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB221_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection497:
; O1-NEXT:    notl %ebx
; O1-NEXT:  .Lpcsection498:
; O1-NEXT:    orq $-43, %rbx
; O1-NEXT:  .Lpcsection499:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection500:
; O1-NEXT:    jne .LBB221_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_nand_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection494:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection495:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection496:
; O2-NEXT:    movq $-1, %rcx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB221_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection497:
; O2-NEXT:    notl %ebx
; O2-NEXT:  .Lpcsection498:
; O2-NEXT:    orq $-43, %rbx
; O2-NEXT:  .Lpcsection499:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection500:
; O2-NEXT:    jne .LBB221_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_nand_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection494:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection495:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection496:
; O3-NEXT:    movq $-1, %rcx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB221_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection497:
; O3-NEXT:    notl %ebx
; O3-NEXT:  .Lpcsection498:
; O3-NEXT:    orq $-43, %rbx
; O3-NEXT:  .Lpcsection499:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection500:
; O3-NEXT:    jne .LBB221_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i128 42 acquire, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xchg_release(ptr %a) {
; O0-LABEL: atomic128_xchg_release:
; O0:       # %bb.0:
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection558:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection559:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection560:
; O0-NEXT:    jmp .LBB222_1
; O0-NEXT:  .LBB222_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection561:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection562:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection563:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection564:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection565:
; O0-NEXT:    jne .LBB222_1
; O0-NEXT:    jmp .LBB222_2
; O0-NEXT:  .LBB222_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xchg_release:
; O1:       # %bb.0:
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection501:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection502:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection503:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB222_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection504:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection505:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection506:
; O1-NEXT:    jne .LBB222_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xchg_release:
; O2:       # %bb.0:
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection501:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection502:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection503:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB222_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection504:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection505:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection506:
; O2-NEXT:    jne .LBB222_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xchg_release:
; O3:       # %bb.0:
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection501:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection502:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection503:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB222_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection504:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection505:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection506:
; O3-NEXT:    jne .LBB222_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i128 42 release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_add_release(ptr %a) {
; O0-LABEL: atomic128_add_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection566:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection567:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection568:
; O0-NEXT:    jmp .LBB223_1
; O0-NEXT:  .LBB223_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection569:
; O0-NEXT:    addq $42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection570:
; O0-NEXT:    adcq $0, %rcx
; O0-NEXT:  .Lpcsection571:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection572:
; O0-NEXT:    jne .LBB223_1
; O0-NEXT:    jmp .LBB223_2
; O0-NEXT:  .LBB223_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_add_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection507:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection508:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB223_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection509:
; O1-NEXT:    addq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection510:
; O1-NEXT:    adcq $0, %rcx
; O1-NEXT:  .Lpcsection511:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection512:
; O1-NEXT:    jne .LBB223_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_add_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection507:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection508:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB223_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection509:
; O2-NEXT:    addq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection510:
; O2-NEXT:    adcq $0, %rcx
; O2-NEXT:  .Lpcsection511:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection512:
; O2-NEXT:    jne .LBB223_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_add_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection507:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection508:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB223_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection509:
; O3-NEXT:    addq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection510:
; O3-NEXT:    adcq $0, %rcx
; O3-NEXT:  .Lpcsection511:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection512:
; O3-NEXT:    jne .LBB223_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i128 42 release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_sub_release(ptr %a) {
; O0-LABEL: atomic128_sub_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection573:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection574:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection575:
; O0-NEXT:    jmp .LBB224_1
; O0-NEXT:  .LBB224_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection576:
; O0-NEXT:    addq $-42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection577:
; O0-NEXT:    adcq $-1, %rcx
; O0-NEXT:  .Lpcsection578:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection579:
; O0-NEXT:    jne .LBB224_1
; O0-NEXT:    jmp .LBB224_2
; O0-NEXT:  .LBB224_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_sub_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection513:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection514:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB224_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection515:
; O1-NEXT:    addq $-42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection516:
; O1-NEXT:    adcq $-1, %rcx
; O1-NEXT:  .Lpcsection517:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection518:
; O1-NEXT:    jne .LBB224_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_sub_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection513:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection514:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB224_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection515:
; O2-NEXT:    addq $-42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection516:
; O2-NEXT:    adcq $-1, %rcx
; O2-NEXT:  .Lpcsection517:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection518:
; O2-NEXT:    jne .LBB224_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_sub_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection513:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection514:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB224_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection515:
; O3-NEXT:    addq $-42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection516:
; O3-NEXT:    adcq $-1, %rcx
; O3-NEXT:  .Lpcsection517:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection518:
; O3-NEXT:    jne .LBB224_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i128 42 release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_and_release(ptr %a) {
; O0-LABEL: atomic128_and_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection580:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection581:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection582:
; O0-NEXT:    jmp .LBB225_1
; O0-NEXT:  .LBB225_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection583:
; O0-NEXT:    andl $42, %ecx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection584:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection585:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection586:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection587:
; O0-NEXT:    jne .LBB225_1
; O0-NEXT:    jmp .LBB225_2
; O0-NEXT:  .LBB225_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_and_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection519:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection520:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB225_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection521:
; O1-NEXT:    andl $42, %ebx
; O1-NEXT:  .Lpcsection522:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection523:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection524:
; O1-NEXT:    jne .LBB225_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_and_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection519:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection520:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB225_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection521:
; O2-NEXT:    andl $42, %ebx
; O2-NEXT:  .Lpcsection522:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection523:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection524:
; O2-NEXT:    jne .LBB225_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_and_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection519:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection520:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB225_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection521:
; O3-NEXT:    andl $42, %ebx
; O3-NEXT:  .Lpcsection522:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection523:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection524:
; O3-NEXT:    jne .LBB225_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i128 42 release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_or_release(ptr %a) {
; O0-LABEL: atomic128_or_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection588:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection589:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection590:
; O0-NEXT:    jmp .LBB226_1
; O0-NEXT:  .LBB226_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection591:
; O0-NEXT:    orq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection592:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection593:
; O0-NEXT:    jne .LBB226_1
; O0-NEXT:    jmp .LBB226_2
; O0-NEXT:  .LBB226_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_or_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection525:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection526:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB226_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection527:
; O1-NEXT:    orq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection528:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection529:
; O1-NEXT:    jne .LBB226_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_or_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection525:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection526:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB226_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection527:
; O2-NEXT:    orq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection528:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection529:
; O2-NEXT:    jne .LBB226_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_or_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection525:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection526:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB226_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection527:
; O3-NEXT:    orq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection528:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection529:
; O3-NEXT:    jne .LBB226_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i128 42 release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xor_release(ptr %a) {
; O0-LABEL: atomic128_xor_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection594:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection595:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection596:
; O0-NEXT:    jmp .LBB227_1
; O0-NEXT:  .LBB227_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection597:
; O0-NEXT:    xorq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection598:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection599:
; O0-NEXT:    jne .LBB227_1
; O0-NEXT:    jmp .LBB227_2
; O0-NEXT:  .LBB227_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xor_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection530:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection531:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB227_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection532:
; O1-NEXT:    xorq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection533:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection534:
; O1-NEXT:    jne .LBB227_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xor_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection530:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection531:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB227_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection532:
; O2-NEXT:    xorq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection533:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection534:
; O2-NEXT:    jne .LBB227_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xor_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection530:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection531:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB227_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection532:
; O3-NEXT:    xorq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection533:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection534:
; O3-NEXT:    jne .LBB227_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i128 42 release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_nand_release(ptr %a) {
; O0-LABEL: atomic128_nand_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection600:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection601:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection602:
; O0-NEXT:    jmp .LBB228_1
; O0-NEXT:  .LBB228_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection603:
; O0-NEXT:    notl %ecx
; O0-NEXT:  .Lpcsection604:
; O0-NEXT:    # implicit-def: $rbx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection605:
; O0-NEXT:    orq $-43, %rbx
; O0-NEXT:  .Lpcsection606:
; O0-NEXT:    movq $-1, %rcx
; O0-NEXT:  .Lpcsection607:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection608:
; O0-NEXT:    jne .LBB228_1
; O0-NEXT:    jmp .LBB228_2
; O0-NEXT:  .LBB228_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_nand_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection535:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection536:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection537:
; O1-NEXT:    movq $-1, %rcx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB228_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection538:
; O1-NEXT:    notl %ebx
; O1-NEXT:  .Lpcsection539:
; O1-NEXT:    orq $-43, %rbx
; O1-NEXT:  .Lpcsection540:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection541:
; O1-NEXT:    jne .LBB228_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_nand_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection535:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection536:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection537:
; O2-NEXT:    movq $-1, %rcx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB228_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection538:
; O2-NEXT:    notl %ebx
; O2-NEXT:  .Lpcsection539:
; O2-NEXT:    orq $-43, %rbx
; O2-NEXT:  .Lpcsection540:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection541:
; O2-NEXT:    jne .LBB228_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_nand_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection535:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection536:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection537:
; O3-NEXT:    movq $-1, %rcx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB228_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection538:
; O3-NEXT:    notl %ebx
; O3-NEXT:  .Lpcsection539:
; O3-NEXT:    orq $-43, %rbx
; O3-NEXT:  .Lpcsection540:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection541:
; O3-NEXT:    jne .LBB228_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i128 42 release, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xchg_acq_rel(ptr %a) {
; O0-LABEL: atomic128_xchg_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection609:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection610:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection611:
; O0-NEXT:    jmp .LBB229_1
; O0-NEXT:  .LBB229_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection612:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection613:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection614:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection615:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection616:
; O0-NEXT:    jne .LBB229_1
; O0-NEXT:    jmp .LBB229_2
; O0-NEXT:  .LBB229_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xchg_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection542:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection543:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection544:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB229_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection545:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection546:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection547:
; O1-NEXT:    jne .LBB229_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xchg_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection542:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection543:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection544:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB229_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection545:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection546:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection547:
; O2-NEXT:    jne .LBB229_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xchg_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection542:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection543:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection544:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB229_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection545:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection546:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection547:
; O3-NEXT:    jne .LBB229_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i128 42 acq_rel, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_add_acq_rel(ptr %a) {
; O0-LABEL: atomic128_add_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection617:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection618:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection619:
; O0-NEXT:    jmp .LBB230_1
; O0-NEXT:  .LBB230_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection620:
; O0-NEXT:    addq $42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection621:
; O0-NEXT:    adcq $0, %rcx
; O0-NEXT:  .Lpcsection622:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection623:
; O0-NEXT:    jne .LBB230_1
; O0-NEXT:    jmp .LBB230_2
; O0-NEXT:  .LBB230_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_add_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection548:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection549:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB230_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection550:
; O1-NEXT:    addq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection551:
; O1-NEXT:    adcq $0, %rcx
; O1-NEXT:  .Lpcsection552:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection553:
; O1-NEXT:    jne .LBB230_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_add_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection548:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection549:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB230_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection550:
; O2-NEXT:    addq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection551:
; O2-NEXT:    adcq $0, %rcx
; O2-NEXT:  .Lpcsection552:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection553:
; O2-NEXT:    jne .LBB230_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_add_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection548:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection549:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB230_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection550:
; O3-NEXT:    addq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection551:
; O3-NEXT:    adcq $0, %rcx
; O3-NEXT:  .Lpcsection552:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection553:
; O3-NEXT:    jne .LBB230_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i128 42 acq_rel, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_sub_acq_rel(ptr %a) {
; O0-LABEL: atomic128_sub_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection624:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection625:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection626:
; O0-NEXT:    jmp .LBB231_1
; O0-NEXT:  .LBB231_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection627:
; O0-NEXT:    addq $-42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection628:
; O0-NEXT:    adcq $-1, %rcx
; O0-NEXT:  .Lpcsection629:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection630:
; O0-NEXT:    jne .LBB231_1
; O0-NEXT:    jmp .LBB231_2
; O0-NEXT:  .LBB231_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_sub_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection554:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection555:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB231_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection556:
; O1-NEXT:    addq $-42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection557:
; O1-NEXT:    adcq $-1, %rcx
; O1-NEXT:  .Lpcsection558:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection559:
; O1-NEXT:    jne .LBB231_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_sub_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection554:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection555:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB231_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection556:
; O2-NEXT:    addq $-42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection557:
; O2-NEXT:    adcq $-1, %rcx
; O2-NEXT:  .Lpcsection558:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection559:
; O2-NEXT:    jne .LBB231_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_sub_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection554:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection555:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB231_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection556:
; O3-NEXT:    addq $-42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection557:
; O3-NEXT:    adcq $-1, %rcx
; O3-NEXT:  .Lpcsection558:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection559:
; O3-NEXT:    jne .LBB231_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i128 42 acq_rel, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_and_acq_rel(ptr %a) {
; O0-LABEL: atomic128_and_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection631:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection632:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection633:
; O0-NEXT:    jmp .LBB232_1
; O0-NEXT:  .LBB232_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection634:
; O0-NEXT:    andl $42, %ecx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection635:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection636:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection637:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection638:
; O0-NEXT:    jne .LBB232_1
; O0-NEXT:    jmp .LBB232_2
; O0-NEXT:  .LBB232_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_and_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection560:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection561:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB232_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection562:
; O1-NEXT:    andl $42, %ebx
; O1-NEXT:  .Lpcsection563:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection564:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection565:
; O1-NEXT:    jne .LBB232_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_and_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection560:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection561:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB232_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection562:
; O2-NEXT:    andl $42, %ebx
; O2-NEXT:  .Lpcsection563:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection564:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection565:
; O2-NEXT:    jne .LBB232_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_and_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection560:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection561:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB232_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection562:
; O3-NEXT:    andl $42, %ebx
; O3-NEXT:  .Lpcsection563:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection564:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection565:
; O3-NEXT:    jne .LBB232_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i128 42 acq_rel, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_or_acq_rel(ptr %a) {
; O0-LABEL: atomic128_or_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection639:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection640:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection641:
; O0-NEXT:    jmp .LBB233_1
; O0-NEXT:  .LBB233_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection642:
; O0-NEXT:    orq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection643:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection644:
; O0-NEXT:    jne .LBB233_1
; O0-NEXT:    jmp .LBB233_2
; O0-NEXT:  .LBB233_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_or_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection566:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection567:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB233_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection568:
; O1-NEXT:    orq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection569:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection570:
; O1-NEXT:    jne .LBB233_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_or_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection566:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection567:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB233_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection568:
; O2-NEXT:    orq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection569:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection570:
; O2-NEXT:    jne .LBB233_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_or_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection566:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection567:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB233_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection568:
; O3-NEXT:    orq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection569:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection570:
; O3-NEXT:    jne .LBB233_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i128 42 acq_rel, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xor_acq_rel(ptr %a) {
; O0-LABEL: atomic128_xor_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection645:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection646:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection647:
; O0-NEXT:    jmp .LBB234_1
; O0-NEXT:  .LBB234_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection648:
; O0-NEXT:    xorq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection649:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection650:
; O0-NEXT:    jne .LBB234_1
; O0-NEXT:    jmp .LBB234_2
; O0-NEXT:  .LBB234_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xor_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection571:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection572:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB234_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection573:
; O1-NEXT:    xorq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection574:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection575:
; O1-NEXT:    jne .LBB234_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xor_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection571:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection572:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB234_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection573:
; O2-NEXT:    xorq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection574:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection575:
; O2-NEXT:    jne .LBB234_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xor_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection571:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection572:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB234_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection573:
; O3-NEXT:    xorq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection574:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection575:
; O3-NEXT:    jne .LBB234_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i128 42 acq_rel, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_nand_acq_rel(ptr %a) {
; O0-LABEL: atomic128_nand_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection651:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection652:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection653:
; O0-NEXT:    jmp .LBB235_1
; O0-NEXT:  .LBB235_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection654:
; O0-NEXT:    notl %ecx
; O0-NEXT:  .Lpcsection655:
; O0-NEXT:    # implicit-def: $rbx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection656:
; O0-NEXT:    orq $-43, %rbx
; O0-NEXT:  .Lpcsection657:
; O0-NEXT:    movq $-1, %rcx
; O0-NEXT:  .Lpcsection658:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection659:
; O0-NEXT:    jne .LBB235_1
; O0-NEXT:    jmp .LBB235_2
; O0-NEXT:  .LBB235_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_nand_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection576:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection577:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection578:
; O1-NEXT:    movq $-1, %rcx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB235_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection579:
; O1-NEXT:    notl %ebx
; O1-NEXT:  .Lpcsection580:
; O1-NEXT:    orq $-43, %rbx
; O1-NEXT:  .Lpcsection581:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection582:
; O1-NEXT:    jne .LBB235_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_nand_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection576:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection577:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection578:
; O2-NEXT:    movq $-1, %rcx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB235_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection579:
; O2-NEXT:    notl %ebx
; O2-NEXT:  .Lpcsection580:
; O2-NEXT:    orq $-43, %rbx
; O2-NEXT:  .Lpcsection581:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection582:
; O2-NEXT:    jne .LBB235_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_nand_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection576:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection577:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection578:
; O3-NEXT:    movq $-1, %rcx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB235_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection579:
; O3-NEXT:    notl %ebx
; O3-NEXT:  .Lpcsection580:
; O3-NEXT:    orq $-43, %rbx
; O3-NEXT:  .Lpcsection581:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection582:
; O3-NEXT:    jne .LBB235_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i128 42 acq_rel, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xchg_seq_cst(ptr %a) {
; O0-LABEL: atomic128_xchg_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection660:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection661:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection662:
; O0-NEXT:    jmp .LBB236_1
; O0-NEXT:  .LBB236_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:  .Lpcsection663:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection664:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection665:
; O0-NEXT:    movl $42, %ebx
; O0-NEXT:  .Lpcsection666:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection667:
; O0-NEXT:    jne .LBB236_1
; O0-NEXT:    jmp .LBB236_2
; O0-NEXT:  .LBB236_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xchg_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection583:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection584:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection585:
; O1-NEXT:    movl $42, %ebx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB236_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:  .Lpcsection586:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection587:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection588:
; O1-NEXT:    jne .LBB236_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xchg_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection583:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection584:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection585:
; O2-NEXT:    movl $42, %ebx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB236_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:  .Lpcsection586:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection587:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection588:
; O2-NEXT:    jne .LBB236_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xchg_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection583:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection584:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection585:
; O3-NEXT:    movl $42, %ebx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB236_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:  .Lpcsection586:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection587:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection588:
; O3-NEXT:    jne .LBB236_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xchg ptr %a, i128 42 seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_add_seq_cst(ptr %a) {
; O0-LABEL: atomic128_add_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection668:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection669:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection670:
; O0-NEXT:    jmp .LBB237_1
; O0-NEXT:  .LBB237_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection671:
; O0-NEXT:    addq $42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection672:
; O0-NEXT:    adcq $0, %rcx
; O0-NEXT:  .Lpcsection673:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection674:
; O0-NEXT:    jne .LBB237_1
; O0-NEXT:    jmp .LBB237_2
; O0-NEXT:  .LBB237_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_add_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection589:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection590:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB237_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection591:
; O1-NEXT:    addq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection592:
; O1-NEXT:    adcq $0, %rcx
; O1-NEXT:  .Lpcsection593:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection594:
; O1-NEXT:    jne .LBB237_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_add_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection589:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection590:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB237_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection591:
; O2-NEXT:    addq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection592:
; O2-NEXT:    adcq $0, %rcx
; O2-NEXT:  .Lpcsection593:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection594:
; O2-NEXT:    jne .LBB237_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_add_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection589:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection590:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB237_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection591:
; O3-NEXT:    addq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection592:
; O3-NEXT:    adcq $0, %rcx
; O3-NEXT:  .Lpcsection593:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection594:
; O3-NEXT:    jne .LBB237_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw add ptr %a, i128 42 seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_sub_seq_cst(ptr %a) {
; O0-LABEL: atomic128_sub_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection675:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection676:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection677:
; O0-NEXT:    jmp .LBB238_1
; O0-NEXT:  .LBB238_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection678:
; O0-NEXT:    addq $-42, %rbx
; O0-NEXT:    movq %rdx, %rcx
; O0-NEXT:  .Lpcsection679:
; O0-NEXT:    adcq $-1, %rcx
; O0-NEXT:  .Lpcsection680:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection681:
; O0-NEXT:    jne .LBB238_1
; O0-NEXT:    jmp .LBB238_2
; O0-NEXT:  .LBB238_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_sub_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection595:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection596:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB238_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection597:
; O1-NEXT:    addq $-42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection598:
; O1-NEXT:    adcq $-1, %rcx
; O1-NEXT:  .Lpcsection599:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection600:
; O1-NEXT:    jne .LBB238_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_sub_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection595:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection596:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB238_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection597:
; O2-NEXT:    addq $-42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection598:
; O2-NEXT:    adcq $-1, %rcx
; O2-NEXT:  .Lpcsection599:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection600:
; O2-NEXT:    jne .LBB238_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_sub_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection595:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection596:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB238_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection597:
; O3-NEXT:    addq $-42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection598:
; O3-NEXT:    adcq $-1, %rcx
; O3-NEXT:  .Lpcsection599:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection600:
; O3-NEXT:    jne .LBB238_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw sub ptr %a, i128 42 seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_and_seq_cst(ptr %a) {
; O0-LABEL: atomic128_and_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection682:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection683:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection684:
; O0-NEXT:    jmp .LBB239_1
; O0-NEXT:  .LBB239_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection685:
; O0-NEXT:    andl $42, %ecx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection686:
; O0-NEXT:    xorl %ecx, %ecx
; O0-NEXT:  .Lpcsection687:
; O0-NEXT:    # kill: def $rcx killed $ecx
; O0-NEXT:  .Lpcsection688:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection689:
; O0-NEXT:    jne .LBB239_1
; O0-NEXT:    jmp .LBB239_2
; O0-NEXT:  .LBB239_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_and_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection601:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection602:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB239_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection603:
; O1-NEXT:    andl $42, %ebx
; O1-NEXT:  .Lpcsection604:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection605:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection606:
; O1-NEXT:    jne .LBB239_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_and_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection601:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection602:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB239_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection603:
; O2-NEXT:    andl $42, %ebx
; O2-NEXT:  .Lpcsection604:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection605:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection606:
; O2-NEXT:    jne .LBB239_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_and_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection601:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection602:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB239_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection603:
; O3-NEXT:    andl $42, %ebx
; O3-NEXT:  .Lpcsection604:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection605:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection606:
; O3-NEXT:    jne .LBB239_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw and ptr %a, i128 42 seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_or_seq_cst(ptr %a) {
; O0-LABEL: atomic128_or_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection690:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection691:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection692:
; O0-NEXT:    jmp .LBB240_1
; O0-NEXT:  .LBB240_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection693:
; O0-NEXT:    orq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection694:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection695:
; O0-NEXT:    jne .LBB240_1
; O0-NEXT:    jmp .LBB240_2
; O0-NEXT:  .LBB240_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_or_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection607:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection608:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB240_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection609:
; O1-NEXT:    orq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection610:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection611:
; O1-NEXT:    jne .LBB240_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_or_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection607:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection608:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB240_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection609:
; O2-NEXT:    orq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection610:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection611:
; O2-NEXT:    jne .LBB240_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_or_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection607:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection608:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB240_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection609:
; O3-NEXT:    orq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection610:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection611:
; O3-NEXT:    jne .LBB240_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw or ptr %a, i128 42 seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_xor_seq_cst(ptr %a) {
; O0-LABEL: atomic128_xor_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection696:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection697:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection698:
; O0-NEXT:    jmp .LBB241_1
; O0-NEXT:  .LBB241_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movq %rax, %rbx
; O0-NEXT:  .Lpcsection699:
; O0-NEXT:    xorq $42, %rbx
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection700:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection701:
; O0-NEXT:    jne .LBB241_1
; O0-NEXT:    jmp .LBB241_2
; O0-NEXT:  .LBB241_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_xor_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection612:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection613:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB241_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movq %rax, %rbx
; O1-NEXT:  .Lpcsection614:
; O1-NEXT:    xorq $42, %rbx
; O1-NEXT:    movq %rdx, %rcx
; O1-NEXT:  .Lpcsection615:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection616:
; O1-NEXT:    jne .LBB241_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_xor_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection612:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection613:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB241_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movq %rax, %rbx
; O2-NEXT:  .Lpcsection614:
; O2-NEXT:    xorq $42, %rbx
; O2-NEXT:    movq %rdx, %rcx
; O2-NEXT:  .Lpcsection615:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection616:
; O2-NEXT:    jne .LBB241_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_xor_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection612:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection613:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB241_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movq %rax, %rbx
; O3-NEXT:  .Lpcsection614:
; O3-NEXT:    xorq $42, %rbx
; O3-NEXT:    movq %rdx, %rcx
; O3-NEXT:  .Lpcsection615:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection616:
; O3-NEXT:    jne .LBB241_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw xor ptr %a, i128 42 seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_nand_seq_cst(ptr %a) {
; O0-LABEL: atomic128_nand_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq %rdi, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection702:
; O0-NEXT:    movq (%rdi), %rax
; O0-NEXT:  .Lpcsection703:
; O0-NEXT:    movq 8(%rdi), %rdx
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection704:
; O0-NEXT:    jmp .LBB242_1
; O0-NEXT:  .LBB242_1: # %atomicrmw.start
; O0-NEXT:    # =>This Inner Loop Header: Depth=1
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rdx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:  .Lpcsection705:
; O0-NEXT:    notl %ecx
; O0-NEXT:  .Lpcsection706:
; O0-NEXT:    # implicit-def: $rbx
; O0-NEXT:    movl %ecx, %ebx
; O0-NEXT:  .Lpcsection707:
; O0-NEXT:    orq $-43, %rbx
; O0-NEXT:  .Lpcsection708:
; O0-NEXT:    movq $-1, %rcx
; O0-NEXT:  .Lpcsection709:
; O0-NEXT:    lock cmpxchg16b (%rsi)
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection710:
; O0-NEXT:    jne .LBB242_1
; O0-NEXT:    jmp .LBB242_2
; O0-NEXT:  .LBB242_2: # %atomicrmw.end
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_nand_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection617:
; O1-NEXT:    movq (%rdi), %rax
; O1-NEXT:  .Lpcsection618:
; O1-NEXT:    movq 8(%rdi), %rdx
; O1-NEXT:  .Lpcsection619:
; O1-NEXT:    movq $-1, %rcx
; O1-NEXT:    .p2align 4, 0x90
; O1-NEXT:  .LBB242_1: # %atomicrmw.start
; O1-NEXT:    # =>This Inner Loop Header: Depth=1
; O1-NEXT:    movl %eax, %ebx
; O1-NEXT:  .Lpcsection620:
; O1-NEXT:    notl %ebx
; O1-NEXT:  .Lpcsection621:
; O1-NEXT:    orq $-43, %rbx
; O1-NEXT:  .Lpcsection622:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection623:
; O1-NEXT:    jne .LBB242_1
; O1-NEXT:  # %bb.2: # %atomicrmw.end
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_nand_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection617:
; O2-NEXT:    movq (%rdi), %rax
; O2-NEXT:  .Lpcsection618:
; O2-NEXT:    movq 8(%rdi), %rdx
; O2-NEXT:  .Lpcsection619:
; O2-NEXT:    movq $-1, %rcx
; O2-NEXT:    .p2align 4, 0x90
; O2-NEXT:  .LBB242_1: # %atomicrmw.start
; O2-NEXT:    # =>This Inner Loop Header: Depth=1
; O2-NEXT:    movl %eax, %ebx
; O2-NEXT:  .Lpcsection620:
; O2-NEXT:    notl %ebx
; O2-NEXT:  .Lpcsection621:
; O2-NEXT:    orq $-43, %rbx
; O2-NEXT:  .Lpcsection622:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection623:
; O2-NEXT:    jne .LBB242_1
; O2-NEXT:  # %bb.2: # %atomicrmw.end
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_nand_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection617:
; O3-NEXT:    movq (%rdi), %rax
; O3-NEXT:  .Lpcsection618:
; O3-NEXT:    movq 8(%rdi), %rdx
; O3-NEXT:  .Lpcsection619:
; O3-NEXT:    movq $-1, %rcx
; O3-NEXT:    .p2align 4, 0x90
; O3-NEXT:  .LBB242_1: # %atomicrmw.start
; O3-NEXT:    # =>This Inner Loop Header: Depth=1
; O3-NEXT:    movl %eax, %ebx
; O3-NEXT:  .Lpcsection620:
; O3-NEXT:    notl %ebx
; O3-NEXT:  .Lpcsection621:
; O3-NEXT:    orq $-43, %rbx
; O3-NEXT:  .Lpcsection622:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection623:
; O3-NEXT:    jne .LBB242_1
; O3-NEXT:  # %bb.2: # %atomicrmw.end
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = atomicrmw nand ptr %a, i128 42 seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_cas_monotonic(ptr %a) {
; O0-LABEL: atomic128_cas_monotonic:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection711:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection712:
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection713:
; O0-NEXT:    movl $1, %ebx
; O0-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection714:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection715:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection716:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection717:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection718:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_cas_monotonic:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection624:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection625:
; O1-NEXT:    movl $1, %ebx
; O1-NEXT:  .Lpcsection626:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection627:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection628:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection629:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection630:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection631:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection632:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection633:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection634:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection635:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection636:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_cas_monotonic:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection624:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection625:
; O2-NEXT:    movl $1, %ebx
; O2-NEXT:  .Lpcsection626:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection627:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection628:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection629:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection630:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection631:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection632:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection633:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection634:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection635:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection636:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_cas_monotonic:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection624:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection625:
; O3-NEXT:    movl $1, %ebx
; O3-NEXT:  .Lpcsection626:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection627:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection628:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection629:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection630:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection631:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection632:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection633:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection634:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection635:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection636:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i128 42, i128 1 monotonic monotonic, align 16, !pcsections !0
  %y = cmpxchg ptr %a, i128 42, i128 1 monotonic acquire, align 16, !pcsections !0
  %z = cmpxchg ptr %a, i128 42, i128 1 monotonic seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_cas_acquire(ptr %a) {
; O0-LABEL: atomic128_cas_acquire:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection719:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection720:
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection721:
; O0-NEXT:    movl $1, %ebx
; O0-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection722:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection723:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection724:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection725:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection726:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_cas_acquire:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection637:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection638:
; O1-NEXT:    movl $1, %ebx
; O1-NEXT:  .Lpcsection639:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection640:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection641:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection642:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection643:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection644:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection645:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection646:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection647:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection648:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection649:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_cas_acquire:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection637:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection638:
; O2-NEXT:    movl $1, %ebx
; O2-NEXT:  .Lpcsection639:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection640:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection641:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection642:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection643:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection644:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection645:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection646:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection647:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection648:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection649:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_cas_acquire:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection637:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection638:
; O3-NEXT:    movl $1, %ebx
; O3-NEXT:  .Lpcsection639:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection640:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection641:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection642:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection643:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection644:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection645:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection646:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection647:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection648:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection649:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i128 42, i128 1 acquire monotonic, align 16, !pcsections !0
  %y = cmpxchg ptr %a, i128 42, i128 1 acquire acquire, align 16, !pcsections !0
  %z = cmpxchg ptr %a, i128 42, i128 1 acquire seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_cas_release(ptr %a) {
; O0-LABEL: atomic128_cas_release:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection727:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection728:
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection729:
; O0-NEXT:    movl $1, %ebx
; O0-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection730:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection731:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection732:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection733:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection734:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_cas_release:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection650:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection651:
; O1-NEXT:    movl $1, %ebx
; O1-NEXT:  .Lpcsection652:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection653:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection654:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection655:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection656:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection657:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection658:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection659:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection660:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection661:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection662:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_cas_release:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection650:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection651:
; O2-NEXT:    movl $1, %ebx
; O2-NEXT:  .Lpcsection652:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection653:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection654:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection655:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection656:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection657:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection658:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection659:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection660:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection661:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection662:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_cas_release:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection650:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection651:
; O3-NEXT:    movl $1, %ebx
; O3-NEXT:  .Lpcsection652:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection653:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection654:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection655:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection656:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection657:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection658:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection659:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection660:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection661:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection662:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i128 42, i128 1 release monotonic, align 16, !pcsections !0
  %y = cmpxchg ptr %a, i128 42, i128 1 release acquire, align 16, !pcsections !0
  %z = cmpxchg ptr %a, i128 42, i128 1 release seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_cas_acq_rel(ptr %a) {
; O0-LABEL: atomic128_cas_acq_rel:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection735:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection736:
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection737:
; O0-NEXT:    movl $1, %ebx
; O0-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection738:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection739:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection740:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection741:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection742:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $1, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_cas_acq_rel:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection663:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection664:
; O1-NEXT:    movl $1, %ebx
; O1-NEXT:  .Lpcsection665:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection666:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection667:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection668:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection669:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection670:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection671:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection672:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection673:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection674:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection675:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $1, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_cas_acq_rel:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection663:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection664:
; O2-NEXT:    movl $1, %ebx
; O2-NEXT:  .Lpcsection665:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection666:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection667:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection668:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection669:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection670:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection671:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection672:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection673:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection674:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection675:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $1, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_cas_acq_rel:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection663:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection664:
; O3-NEXT:    movl $1, %ebx
; O3-NEXT:  .Lpcsection665:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection666:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection667:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection668:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection669:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection670:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection671:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection672:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection673:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection674:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection675:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $1, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i128 42, i128 1 acq_rel monotonic, align 16, !pcsections !0
  %y = cmpxchg ptr %a, i128 42, i128 1 acq_rel acquire, align 16, !pcsections !0
  %z = cmpxchg ptr %a, i128 42, i128 1 acq_rel seq_cst, align 16, !pcsections !0
  store volatile i64 1, ptr @foo, align 8
  ret void
}

define void @atomic128_cas_seq_cst(ptr %a) {
; O0-LABEL: atomic128_cas_seq_cst:
; O0:       # %bb.0: # %entry
; O0-NEXT:    pushq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 16
; O0-NEXT:    .cfi_offset %rbx, -16
; O0-NEXT:    movq foo(%rip), %rax
; O0-NEXT:  .Lpcsection743:
; O0-NEXT:    xorl %eax, %eax
; O0-NEXT:    movl %eax, %ecx
; O0-NEXT:    movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection744:
; O0-NEXT:    movl $42, %eax
; O0-NEXT:    movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:  .Lpcsection745:
; O0-NEXT:    movl $1, %ebx
; O0-NEXT:    movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection746:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection747:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection748:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Reload
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Reload
; O0-NEXT:  .Lpcsection749:
; O0-NEXT:    # kill: def $rsi killed $rax
; O0-NEXT:    movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; O0-NEXT:    movq %rcx, %rdx
; O0-NEXT:  .Lpcsection750:
; O0-NEXT:    lock cmpxchg16b (%rdi)
; O0-NEXT:    movq $3, foo
; O0-NEXT:    popq %rbx
; O0-NEXT:    .cfi_def_cfa_offset 8
; O0-NEXT:    retq
;
; O1-LABEL: atomic128_cas_seq_cst:
; O1:       # %bb.0: # %entry
; O1-NEXT:    pushq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 16
; O1-NEXT:    .cfi_offset %rbx, -16
; O1-NEXT:    movq foo(%rip), %rax
; O1-NEXT:  .Lpcsection676:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection677:
; O1-NEXT:    movl $1, %ebx
; O1-NEXT:  .Lpcsection678:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection679:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection680:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection681:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection682:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection683:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection684:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:  .Lpcsection685:
; O1-NEXT:    movl $42, %eax
; O1-NEXT:  .Lpcsection686:
; O1-NEXT:    xorl %edx, %edx
; O1-NEXT:  .Lpcsection687:
; O1-NEXT:    xorl %ecx, %ecx
; O1-NEXT:  .Lpcsection688:
; O1-NEXT:    lock cmpxchg16b (%rdi)
; O1-NEXT:    movq $3, foo(%rip)
; O1-NEXT:    popq %rbx
; O1-NEXT:    .cfi_def_cfa_offset 8
; O1-NEXT:    retq
;
; O2-LABEL: atomic128_cas_seq_cst:
; O2:       # %bb.0: # %entry
; O2-NEXT:    pushq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 16
; O2-NEXT:    .cfi_offset %rbx, -16
; O2-NEXT:    movq foo(%rip), %rax
; O2-NEXT:  .Lpcsection676:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection677:
; O2-NEXT:    movl $1, %ebx
; O2-NEXT:  .Lpcsection678:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection679:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection680:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection681:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection682:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection683:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection684:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:  .Lpcsection685:
; O2-NEXT:    movl $42, %eax
; O2-NEXT:  .Lpcsection686:
; O2-NEXT:    xorl %edx, %edx
; O2-NEXT:  .Lpcsection687:
; O2-NEXT:    xorl %ecx, %ecx
; O2-NEXT:  .Lpcsection688:
; O2-NEXT:    lock cmpxchg16b (%rdi)
; O2-NEXT:    movq $3, foo(%rip)
; O2-NEXT:    popq %rbx
; O2-NEXT:    .cfi_def_cfa_offset 8
; O2-NEXT:    retq
;
; O3-LABEL: atomic128_cas_seq_cst:
; O3:       # %bb.0: # %entry
; O3-NEXT:    pushq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 16
; O3-NEXT:    .cfi_offset %rbx, -16
; O3-NEXT:    movq foo(%rip), %rax
; O3-NEXT:  .Lpcsection676:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection677:
; O3-NEXT:    movl $1, %ebx
; O3-NEXT:  .Lpcsection678:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection679:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection680:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection681:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection682:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection683:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection684:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:  .Lpcsection685:
; O3-NEXT:    movl $42, %eax
; O3-NEXT:  .Lpcsection686:
; O3-NEXT:    xorl %edx, %edx
; O3-NEXT:  .Lpcsection687:
; O3-NEXT:    xorl %ecx, %ecx
; O3-NEXT:  .Lpcsection688:
; O3-NEXT:    lock cmpxchg16b (%rdi)
; O3-NEXT:    movq $3, foo(%rip)
; O3-NEXT:    popq %rbx
; O3-NEXT:    .cfi_def_cfa_offset 8
; O3-NEXT:    retq
entry:
  load volatile i64, ptr @foo, align 8
  %x = cmpxchg ptr %a, i128 42, i128 1 seq_cst monotonic, align 16, !pcsections !0
  %y = cmpxchg ptr %a, i128 42, i128 1 seq_cst acquire, align 16, !pcsections !0
  %z = cmpxchg ptr %a, i128 42, i128 1 seq_cst seq_cst, align 16, !pcsections !0
  store volatile i64 3, ptr @foo, align 8
  ret void
}

!0 = !{!"somesection"}