PublicArchive/rust
2
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2026-05-06 08:36:52 -04:00
Code Issues Packages Projects Releases Wiki Activity

cg_llvm: sve_tuple_{create,get,set} intrinsics

Browse Source 
Clang changed to representing tuples of scalable vectors as
structs rather than as wide vectors (that is, scalable vector types
where the `N` part of the `<vscale x N x ty>` type was multiplied by
the number of vectors). rustc mirrored this in the initial implementation
of scalable vectors.

Earlier versions of our patches used the wide vector representation and
our intrinsic patches used the legacy
`llvm.aarch64.sve.tuple.{create,get,set}{2,3,4}` intrinsics for creating
these tuples/getting/setting the vectors, which were only supported
due to LLVM's `AutoUpgrade` pass converting these intrinsics into
`llvm.vector.insert`. `AutoUpgrade` only supports these legacy intrinsics
with the wide vector representation.

With the current struct representation, Clang has special handling in
codegen for generating `insertvalue`/`extractvalue` instructions for
these operations, which must be replicated by rustc's codegen for our
intrinsics to use. This patch implements new intrinsics in
`core::intrinsics::scalable` (mirroring the structure of
`core::intrinsics::simd`) which rustc lowers to the appropriate
`insertvalue`/`extractvalue` instructions.
This commit is contained in:
David Wood
2026-02-19 11:03:14 +00:00
parent a2f7f3c1eb
commit 4fbcb031de
8 changed files with 298 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files
compiler/rustc_codegen_llvm/src/intrinsic.rs
+111 -1
View File
@@ -3,7 +3,8 @@ use std::ffi::c_uint;
use std::{assert_matches, ptr};
use rustc_abi::{
Align, BackendRepr, ExternAbi, Float, HasDataLayout, Primitive, Size, WrappingRange,
Align, BackendRepr, ExternAbi, Float, HasDataLayout, NumScalableVectors, Primitive, Size,
WrappingRange,
};
use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh, wants_wasm_eh};
use rustc_codegen_ssa::common::{IntPredicate, TypeKind};
@@ -605,6 +606,115 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
self.pointercast(val, self.type_ptr())
}
sym::sve_tuple_create2 => {
assert_matches!(
self.layout_of(fn_args.type_at(0)).backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1),
..
}
);
let tuple_ty = self.layout_of(fn_args.type_at(1));
assert_matches!(
tuple_ty.backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2),
..
}
);
let ret = self.const_poison(self.backend_type(tuple_ty));
let ret = self.insert_value(ret, args[0].immediate(), 0);
self.insert_value(ret, args[1].immediate(), 1)
}
sym::sve_tuple_create3 => {
assert_matches!(
self.layout_of(fn_args.type_at(0)).backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1),
..
}
);
let tuple_ty = self.layout_of(fn_args.type_at(1));
assert_matches!(
tuple_ty.backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(3),
..
}
);
let ret = self.const_poison(self.backend_type(tuple_ty));
let ret = self.insert_value(ret, args[0].immediate(), 0);
let ret = self.insert_value(ret, args[1].immediate(), 1);
self.insert_value(ret, args[2].immediate(), 2)
}
sym::sve_tuple_create4 => {
assert_matches!(
self.layout_of(fn_args.type_at(0)).backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1),
..
}
);
let tuple_ty = self.layout_of(fn_args.type_at(1));
assert_matches!(
tuple_ty.backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(4),
..
}
);
let ret = self.const_poison(self.backend_type(tuple_ty));
let ret = self.insert_value(ret, args[0].immediate(), 0);
let ret = self.insert_value(ret, args[1].immediate(), 1);
let ret = self.insert_value(ret, args[2].immediate(), 2);
self.insert_value(ret, args[3].immediate(), 3)
}
sym::sve_tuple_get => {
assert_matches!(
self.layout_of(fn_args.type_at(0)).backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
..
}
);
assert_matches!(
self.layout_of(fn_args.type_at(1)).backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1),
..
}
);
self.extract_value(
args[0].immediate(),
fn_args.const_at(2).to_leaf().to_i32() as u64,
)
}
sym::sve_tuple_set => {
assert_matches!(
self.layout_of(fn_args.type_at(0)).backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(2 | 3 | 4 | 5 | 6 | 7 | 8),
..
}
);
assert_matches!(
self.layout_of(fn_args.type_at(1)).backend_repr,
BackendRepr::SimdScalableVector {
number_of_vectors: NumScalableVectors(1),
..
}
);
self.insert_value(
args[0].immediate(),
args[1].immediate(),
fn_args.const_at(2).to_leaf().to_i32() as u64,
)
}
_ if name.as_str().starts_with("simd_") => {
// Unpack non-power-of-2 #[repr(packed, simd)] arguments.
// This gives them the expected layout of a regular #[repr(simd)] vector.
compiler/rustc_hir_analysis/src/check/intrinsic.rs
+6
View File
@@ -783,6 +783,12 @@ pub(crate) fn check_intrinsic_type(
sym::simd_shuffle => (3, 0, vec![param(0), param(0), param(1)], param(2)),
sym::simd_shuffle_const_generic => (2, 1, vec![param(0), param(0)], param(1)),
sym::sve_tuple_create2 => (2, 0, vec![param(0), param(0)], param(1)),
sym::sve_tuple_create3 => (2, 0, vec![param(0), param(0), param(0)], param(1)),
sym::sve_tuple_create4 => (2, 0, vec![param(0), param(0), param(0), param(0)], param(1)),
sym::sve_tuple_get => (2, 1, vec![param(0)], param(1)),
sym::sve_tuple_set => (2, 1, vec![param(0), param(1)], param(0)),
sym::atomic_cxchg | sym::atomic_cxchgweak => (
1,
2,
compiler/rustc_span/src/symbol.rs
+5
View File
@@ -1979,6 +1979,11 @@ symbols! {
suggestion,
super_let,
supertrait_item_shadowing,
sve_tuple_create2,
sve_tuple_create3,
sve_tuple_create4,
sve_tuple_get,
sve_tuple_set,
sym,
sync,
synthetic,
library/core/src/intrinsics/simd.rs → library/core/src/intrinsics/simd/mod.rs
+2
View File
@@ -2,6 +2,8 @@
//!
//! In this module, a "vector" is any `repr(simd)` type.
pub mod scalable;
use crate::marker::ConstParamTy;
/// Inserts an element into a vector, returning the updated vector.
library/core/src/intrinsics/simd/scalable.rs
+71
View File
@@ -0,0 +1,71 @@
//! Scalable vector compiler intrinsics.
//!
//! In this module, a "vector" is any `#[rustc_scalable_vector]`-annotated type.
/// Create a tuple of two vectors.
///
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
/// type `SVec`.
///
/// Corresponds to Clang's `__builtin_sve_svcreate2*` builtins.
#[cfg(target_arch = "aarch64")]
#[rustc_nounwind]
#[rustc_intrinsic]
pub unsafe fn sve_tuple_create2<SVec, SVecTup>(x0: SVec, x1: SVec) -> SVecTup;
/// Create a tuple of three vectors.
///
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
/// type `SVec`.
///
/// Corresponds to Clang's `__builtin_sve_svcreate3*` builtins.
#[cfg(target_arch = "aarch64")]
#[rustc_intrinsic]
#[rustc_nounwind]
pub unsafe fn sve_tuple_create3<SVec, SVecTup>(x0: SVec, x1: SVec, x2: SVec) -> SVecTup;
/// Create a tuple of four vectors.
///
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
/// type `SVec`.
///
/// Corresponds to Clang's `__builtin_sve_svcreate4*` builtins.
#[cfg(target_arch = "aarch64")]
#[rustc_intrinsic]
#[rustc_nounwind]
pub unsafe fn sve_tuple_create4<SVec, SVecTup>(x0: SVec, x1: SVec, x2: SVec, x3: SVec) -> SVecTup;
/// Get one vector from a tuple of vectors.
///
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
/// type `SVec`.
///
/// Corresponds to Clang's `__builtin_sve_svget*` builtins.
///
/// # Safety
///
/// `IDX` must be in-bounds of the tuple.
#[cfg(target_arch = "aarch64")]
#[rustc_intrinsic]
#[rustc_nounwind]
pub unsafe fn sve_tuple_get<SVecTup, SVec, const IDX: i32>(tuple: SVecTup) -> SVec;
/// Change one vector in a tuple of vectors.
///
/// `SVecTup` must be a scalable vector tuple (`#[rustc_scalable_vector]`) and `SVec` must be a
/// scalable vector (`#[rustc_scalable_vector(N)]`). `SVecTup` must be a tuple of vectors of
/// type `SVec`.
///
/// Corresponds to Clang's `__builtin_sve_svset*` builtins.
///
/// # Safety
///
/// `IDX` must be in-bounds of the tuple.
#[cfg(target_arch = "aarch64")]
#[rustc_intrinsic]
#[rustc_nounwind]
pub unsafe fn sve_tuple_set<SVecTup, SVec, const IDX: i32>(tuple: SVecTup, x: SVec) -> SVecTup;
tests/codegen-llvm/scalable-vectors/tuple-intrinsics.rs
+100
View File
@@ -0,0 +1,100 @@
//@ build-pass
//@ only-aarch64
#![crate_type = "lib"]
#![allow(incomplete_features, internal_features)]
#![feature(abi_unadjusted, core_intrinsics, link_llvm_intrinsics, rustc_attrs)]
// Tests that tuples of scalable vectors are passed as immediates and that the intrinsics for
// creating/getting/setting tuples of scalable vectors generate the correct assembly
#[derive(Copy, Clone)]
#[rustc_scalable_vector(4)]
#[allow(non_camel_case_types)]
pub struct svfloat32_t(f32);
#[derive(Copy, Clone)]
#[rustc_scalable_vector]
#[allow(non_camel_case_types)]
pub struct svfloat32x2_t(svfloat32_t, svfloat32_t);
#[derive(Copy, Clone)]
#[rustc_scalable_vector]
#[allow(non_camel_case_types)]
pub struct svfloat32x3_t(svfloat32_t, svfloat32_t, svfloat32_t);
#[derive(Copy, Clone)]
#[rustc_scalable_vector]
#[allow(non_camel_case_types)]
pub struct svfloat32x4_t(svfloat32_t, svfloat32_t, svfloat32_t, svfloat32_t);
#[inline(never)]
#[target_feature(enable = "sve")]
pub fn svdup_n_f32(op: f32) -> svfloat32_t {
extern "C" {
#[cfg_attr(target_arch = "aarch64", link_name = "llvm.aarch64.sve.dup.x.nxv4f32")]
fn _svdup_n_f32(op: f32) -> svfloat32_t;
}
unsafe { _svdup_n_f32(op) }
}
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float> } @svcreate2_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1)
#[no_mangle]
#[target_feature(enable = "sve")]
pub fn svcreate2_f32(x0: svfloat32_t, x1: svfloat32_t) -> svfloat32x2_t {
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
// CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
unsafe { std::intrinsics::simd::scalable::sve_tuple_create2(x0, x1) }
}
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @svcreate3_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1, <vscale x 4 x float> %x2)
#[no_mangle]
#[target_feature(enable = "sve")]
pub fn svcreate3_f32(x0: svfloat32_t, x1: svfloat32_t, x2: svfloat32_t) -> svfloat32x3_t {
// CHECK-LABEL: @_RNvCsk3YxfLN8zWY_6tuples13svcreate3_f32
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
// CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
// CHECK-NEXT: %3 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %2, <vscale x 4 x float> %x2, 2
unsafe { std::intrinsics::simd::scalable::sve_tuple_create3(x0, x1, x2) }
}
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } @svcreate4_f32(<vscale x 4 x float> %x0, <vscale x 4 x float> %x1, <vscale x 4 x float> %x2, <vscale x 4 x float> %x3)
#[no_mangle]
#[target_feature(enable = "sve")]
pub fn svcreate4_f32(
x0: svfloat32_t,
x1: svfloat32_t,
x2: svfloat32_t,
x3: svfloat32_t,
) -> svfloat32x4_t {
// CHECK-LABEL: @_RNvCsk3YxfLN8zWY_6tuples13svcreate4_f32
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } poison, <vscale x 4 x float> %x0, 0
// CHECK-NEXT: %2 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %1, <vscale x 4 x float> %x1, 1
// CHECK-NEXT: %3 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %2, <vscale x 4 x float> %x2, 2
// CHECK-NEXT: %4 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float>, <vscale x 4 x float> } %3, <vscale x 4 x float> %x3, 3
unsafe { std::intrinsics::simd::scalable::sve_tuple_create4(x0, x1, x2, x3) }
}
// CHECK: define <vscale x 4 x float> @svget2_f32({ <vscale x 4 x float>, <vscale x 4 x float> } %tup)
#[no_mangle]
#[target_feature(enable = "sve")]
pub fn svget2_f32<const IDX: i32>(tup: svfloat32x2_t) -> svfloat32_t {
// CHECK: %1 = extractvalue { <vscale x 4 x float>, <vscale x 4 x float> } %tup, 0
unsafe { std::intrinsics::simd::scalable::sve_tuple_get::<_, _, { IDX }>(tup) }
}
// CHECK: define { <vscale x 4 x float>, <vscale x 4 x float> } @svset2_f32({ <vscale x 4 x float>, <vscale x 4 x float> } %tup, <vscale x 4 x float> %x)
#[no_mangle]
#[target_feature(enable = "sve")]
pub fn svset2_f32<const IDX: i32>(tup: svfloat32x2_t, x: svfloat32_t) -> svfloat32x2_t {
// CHECK: %1 = insertvalue { <vscale x 4 x float>, <vscale x 4 x float> } %tup, <vscale x 4 x float> %x, 0
unsafe { std::intrinsics::simd::scalable::sve_tuple_set::<_, _, { IDX }>(tup, x) }
}
// This function exists only so there are calls to the generic functions
#[target_feature(enable = "sve")]
pub fn test() {
let x = svdup_n_f32(2f32);
let tup = svcreate2_f32(x, x);
let x = svget2_f32::<0>(tup);
let tup = svset2_f32::<0>(tup, x);
}
tests/ui/simd/masked-load-store-check-fail.stderr
+2 -2
View File
@@ -21,7 +21,7 @@ LL | | Simd::<u8, 4>([9; 4]),
LL | | );
| |_________^
note: function defined here
--> $SRC_DIR/core/src/intrinsics/simd.rs:LL:COL
--> $SRC_DIR/core/src/intrinsics/simd/mod.rs:LL:COL
error[E0308]: mismatched types
--> $DIR/masked-load-store-check-fail.rs:25:13
@@ -46,7 +46,7 @@ LL | | default,
LL | | );
| |_________^
note: function defined here
--> $SRC_DIR/core/src/intrinsics/simd.rs:LL:COL
--> $SRC_DIR/core/src/intrinsics/simd/mod.rs:LL:COL
error: aborting due to 2 previous errors
triagebot.toml
+1 -1
View File
@@ -1077,7 +1077,7 @@ cc = ["@Amanieu", "@folkertdev", "@sayantn"]
message = "Some changes occurred in `std_detect`"
cc = ["@Amanieu", "@folkertdev", "@sayantn"]
[mentions."library/core/src/intrinsics/simd.rs"]
[mentions."library/core/src/intrinsics/simd/mod.rs"]
message = """
Some changes occurred to the platform-builtins intrinsics. Make sure the
LLVM backend as well as portable-simd gets adapted for the changes.