Rollup merge of rust-lang#131664 - taiki-e:s390x-asm-vreg-inout, r=Am…

…anieu

Support input/output in vector registers of s390x inline assembly (under asm_experimental_reg feature)

This extends currently clobber-only vector registers (`vreg`) support to allow passing `#[repr(simd)]` types, floats (f32/f64/f128), and integers (i32/i64/i128) as input/output.

This is unstable and gated under new `#![feature(asm_experimental_reg)]` (tracking issue: rust-lang#133416). If the feature is not enabled, only clober is supported as before.

| Architecture | Register class | Target feature | Allowed types |
| ------------ | -------------- | -------------- | -------------- |
| s390x | `vreg` | `vector` | `i32`, `f32`, `i64`, `f64`, `i128`, `f128`, `i8x16`, `i16x8`, `i32x4`, `i64x2`, `f32x4`, `f64x2` |

This matches the list of types that are supported by the vector registers in LLVM:
https://github.com/llvm/llvm-project/blob/llvmorg-19.1.0/llvm/lib/Target/SystemZ/SystemZRegisterInfo.td#L301-L313

In addition to `core::simd` types and floats listed above, custom `#[repr(simd)]` types of the same size and type are also allowed. All allowed types other than i32/f32/i64/f64/i128, and relevant target features are currently unstable.

Currently there is no SIMD type for s390x in `core::arch`, but this is tracked in rust-lang#130869.

cc rust-lang#130869 about vector facility support in s390x
cc rust-lang#125398 & rust-lang#116909 about f128 support in asm

`@rustbot` label +O-SystemZ +A-inline-assembly

compiler/rustc_ast_lowering/messages.ftl

@@ -152,6 +152,8 @@ ast_lowering_register2 = register `{$reg2_name}`
 
 ast_lowering_register_class_only_clobber =
     register class `{$reg_class_name}` can only be used as a clobber, not as an input or output
+ast_lowering_register_class_only_clobber_stable =
+    register class `{$reg_class_name}` can only be used as a clobber in stable
 
 ast_lowering_register_conflict =
     register `{$reg1_name}` conflicts with register `{$reg2_name}`

compiler/rustc_ast_lowering/src/asm.rs

@@ -17,7 +17,8 @@ use super::errors::{
     InlineAsmUnsupportedTarget, InvalidAbiClobberAbi, InvalidAsmTemplateModifierConst,
     InvalidAsmTemplateModifierLabel, InvalidAsmTemplateModifierRegClass,
     InvalidAsmTemplateModifierRegClassSub, InvalidAsmTemplateModifierSym, InvalidRegister,
-    InvalidRegisterClass, RegisterClassOnlyClobber, RegisterConflict,
+    InvalidRegisterClass, RegisterClassOnlyClobber, RegisterClassOnlyClobberStable,
+    RegisterConflict,
 };
 use crate::{
     AllowReturnTypeNotation, ImplTraitContext, ImplTraitPosition, ParamMode,
@@ -61,6 +62,7 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 .emit();
             }
         }
+        let allow_experimental_reg = self.tcx.features().asm_experimental_reg();
         if asm.options.contains(InlineAsmOptions::ATT_SYNTAX)
             && !matches!(asm_arch, Some(asm::InlineAsmArch::X86 | asm::InlineAsmArch::X86_64))
             && !self.tcx.sess.opts.actually_rustdoc
@@ -324,11 +326,29 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 // means that we disallow passing a value in/out of the asm and
                 // require that the operand name an explicit register, not a
                 // register class.
-                if reg_class.is_clobber_only(asm_arch.unwrap()) && !op.is_clobber() {
-                    self.dcx().emit_err(RegisterClassOnlyClobber {
-                        op_span: op_sp,
-                        reg_class_name: reg_class.name(),
-                    });
+                if reg_class.is_clobber_only(asm_arch.unwrap(), allow_experimental_reg)
+                    && !op.is_clobber()
+                {
+                    if allow_experimental_reg || reg_class.is_clobber_only(asm_arch.unwrap(), true)
+                    {
+                        // always clobber-only
+                        self.dcx().emit_err(RegisterClassOnlyClobber {
+                            op_span: op_sp,
+                            reg_class_name: reg_class.name(),
+                        });
+                    } else {
+                        // clobber-only in stable
+                        self.tcx
+                            .sess
+                            .create_feature_err(
+                                RegisterClassOnlyClobberStable {
+                                    op_span: op_sp,
+                                    reg_class_name: reg_class.name(),
+                                },
+                                sym::asm_experimental_reg,
+                            )
+                            .emit();
+                    }
                     continue;
                 }
 

compiler/rustc_ast_lowering/src/errors.rs

@@ -279,6 +279,14 @@ pub(crate) struct RegisterClassOnlyClobber {
     pub reg_class_name: Symbol,
 }
 
+#[derive(Diagnostic)]
+#[diag(ast_lowering_register_class_only_clobber_stable)]
+pub(crate) struct RegisterClassOnlyClobberStable {
+    #[primary_span]
+    pub op_span: Span,
+    pub reg_class_name: Symbol,
+}
+
 #[derive(Diagnostic)]
 #[diag(ast_lowering_register_conflict)]
 pub(crate) struct RegisterConflict<'a> {

compiler/rustc_codegen_cranelift/src/inline_asm.rs

@@ -462,8 +462,12 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
         let mut slots_output = vec![None; self.operands.len()];
 
         let new_slot_fn = |slot_size: &mut Size, reg_class: InlineAsmRegClass| {
-            let reg_size =
-                reg_class.supported_types(self.arch).iter().map(|(ty, _)| ty.size()).max().unwrap();
+            let reg_size = reg_class
+                .supported_types(self.arch, true)
+                .iter()
+                .map(|(ty, _)| ty.size())
+                .max()
+                .unwrap();
             let align = rustc_abi::Align::from_bytes(reg_size.bytes()).unwrap();
             let offset = slot_size.align_to(align);
             *slot_size = offset + reg_size;

compiler/rustc_codegen_gcc/src/asm.rs

@@ -186,7 +186,7 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
                             // `clobber_abi` can add lots of clobbers that are not supported by the target,
                             // such as AVX-512 registers, so we just ignore unsupported registers
                             let is_target_supported =
-                                reg.reg_class().supported_types(asm_arch).iter().any(
+                                reg.reg_class().supported_types(asm_arch, true).iter().any(
                                     |&(_, feature)| {
                                         if let Some(feature) = feature {
                                             self.tcx
@@ -683,9 +683,8 @@ fn reg_to_gcc(reg: InlineAsmRegOrRegClass) -> ConstraintOrRegister {
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r",
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg_addr) => "a",
             InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f",
-            InlineAsmRegClass::S390x(
-                S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg,
-            ) => {
+            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::vreg) => "v",
+            InlineAsmRegClass::S390x(S390xInlineAsmRegClass::areg) => {
                 unreachable!("clobber-only")
             }
             InlineAsmRegClass::Sparc(SparcInlineAsmRegClass::reg) => "r",
@@ -766,7 +765,8 @@ fn dummy_output_type<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, reg: InlineAsmRegCl
             S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr,
         ) => cx.type_i32(),
         InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
-        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
+        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
+        InlineAsmRegClass::S390x(S390xInlineAsmRegClass::areg) => {
             unreachable!("clobber-only")
         }
         InlineAsmRegClass::Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),

compiler/rustc_codegen_llvm/src/asm.rs

@@ -45,7 +45,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             match *op {
                 InlineAsmOperandRef::Out { reg, late, place } => {
                     let is_target_supported = |reg_class: InlineAsmRegClass| {
-                        for &(_, feature) in reg_class.supported_types(asm_arch) {
+                        for &(_, feature) in reg_class.supported_types(asm_arch, true) {
                             if let Some(feature) = feature {
                                 if self
                                     .tcx
@@ -85,7 +85,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                         }
                         continue;
                     } else if !is_target_supported(reg.reg_class())
-                        || reg.reg_class().is_clobber_only(asm_arch)
+                        || reg.reg_class().is_clobber_only(asm_arch, true)
                     {
                         // We turn discarded outputs into clobber constraints
                         // if the target feature needed by the register class is
@@ -686,7 +686,8 @@ fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) ->
             S390x(S390xInlineAsmRegClass::reg) => "r",
             S390x(S390xInlineAsmRegClass::reg_addr) => "a",
             S390x(S390xInlineAsmRegClass::freg) => "f",
-            S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
+            S390x(S390xInlineAsmRegClass::vreg) => "v",
+            S390x(S390xInlineAsmRegClass::areg) => {
                 unreachable!("clobber-only")
             }
             Sparc(SparcInlineAsmRegClass::reg) => "r",
@@ -852,7 +853,8 @@ fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'
         Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(),
         S390x(S390xInlineAsmRegClass::reg | S390xInlineAsmRegClass::reg_addr) => cx.type_i32(),
         S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(),
-        S390x(S390xInlineAsmRegClass::vreg | S390xInlineAsmRegClass::areg) => {
+        S390x(S390xInlineAsmRegClass::vreg) => cx.type_vector(cx.type_i64(), 2),
+        S390x(S390xInlineAsmRegClass::areg) => {
             unreachable!("clobber-only")
         }
         Sparc(SparcInlineAsmRegClass::reg) => cx.type_i32(),

compiler/rustc_feature/src/unstable.rs

@@ -376,6 +376,8 @@ declare_features! (
     (unstable, arbitrary_self_types_pointers, "1.83.0", Some(44874)),
     /// Enables experimental inline assembly support for additional architectures.
     (unstable, asm_experimental_arch, "1.58.0", Some(93335)),
+    /// Enables experimental register support in inline assembly.
+    (unstable, asm_experimental_reg, "CURRENT_RUSTC_VERSION", Some(133416)),
     /// Allows using `label` operands in inline assembly.
     (unstable, asm_goto, "1.78.0", Some(119364)),
     /// Allows using `label` operands in inline assembly together with output operands.

compiler/rustc_hir_analysis/messages.ftl

@@ -431,6 +431,9 @@ hir_analysis_recursive_generic_parameter = {$param_def_kind} `{$param_name}` is
 hir_analysis_redundant_lifetime_args = unnecessary lifetime parameter `{$victim}`
     .note = you can use the `{$candidate}` lifetime directly, in place of `{$victim}`
 
+hir_analysis_register_type_unstable =
+    type `{$ty}` cannot be used with this register class in stable
+
 hir_analysis_requires_note = the `{$trait_name}` impl for `{$ty}` requires that `{$error_predicate}`
 
 hir_analysis_return_type_notation_equality_bound =

compiler/rustc_hir_analysis/src/check/intrinsicck.rs

@@ -7,12 +7,14 @@ use rustc_hir::{self as hir, LangItem};
 use rustc_middle::bug;
 use rustc_middle::ty::{self, FloatTy, IntTy, Ty, TyCtxt, TypeVisitableExt, UintTy};
 use rustc_session::lint;
-use rustc_span::Symbol;
 use rustc_span::def_id::LocalDefId;
+use rustc_span::{Symbol, sym};
 use rustc_target::asm::{
     InlineAsmReg, InlineAsmRegClass, InlineAsmRegOrRegClass, InlineAsmType, ModifierInfo,
 };
 
+use crate::errors::RegisterTypeUnstable;
+
 pub struct InlineAsmCtxt<'a, 'tcx> {
     tcx: TyCtxt<'tcx>,
     typing_env: ty::TypingEnv<'tcx>,
@@ -218,17 +220,29 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
         // Check the type against the list of types supported by the selected
         // register class.
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
+        let allow_experimental_reg = self.tcx.features().asm_experimental_reg();
         let reg_class = reg.reg_class();
-        let supported_tys = reg_class.supported_types(asm_arch);
+        let supported_tys = reg_class.supported_types(asm_arch, allow_experimental_reg);
         let Some((_, feature)) = supported_tys.iter().find(|&&(t, _)| t == asm_ty) else {
-            let msg = format!("type `{ty}` cannot be used with this register class");
-            let mut err = self.tcx.dcx().struct_span_err(expr.span, msg);
-            let supported_tys: Vec<_> = supported_tys.iter().map(|(t, _)| t.to_string()).collect();
-            err.note(format!(
-                "register class `{}` supports these types: {}",
-                reg_class.name(),
-                supported_tys.join(", "),
-            ));
+            let mut err = if !allow_experimental_reg
+                && reg_class.supported_types(asm_arch, true).iter().any(|&(t, _)| t == asm_ty)
+            {
+                self.tcx.sess.create_feature_err(
+                    RegisterTypeUnstable { span: expr.span, ty },
+                    sym::asm_experimental_reg,
+                )
+            } else {
+                let msg = format!("type `{ty}` cannot be used with this register class");
+                let mut err = self.tcx.dcx().struct_span_err(expr.span, msg);
+                let supported_tys: Vec<_> =
+                    supported_tys.iter().map(|(t, _)| t.to_string()).collect();
+                err.note(format!(
+                    "register class `{}` supports these types: {}",
+                    reg_class.name(),
+                    supported_tys.join(", "),
+                ));
+                err
+            };
             if let Some(suggest) = reg_class.suggest_class(asm_arch, asm_ty) {
                 err.help(format!("consider using the `{}` register class instead", suggest.name()));
             }
@@ -313,6 +327,7 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
             self.tcx.dcx().delayed_bug("target architecture does not support asm");
             return;
         };
+        let allow_experimental_reg = self.tcx.features().asm_experimental_reg();
         for (idx, (op, op_sp)) in asm.operands.iter().enumerate() {
             // Validate register classes against currently enabled target
             // features. We check that at least one type is available for
@@ -352,7 +367,8 @@ impl<'a, 'tcx> InlineAsmCtxt<'a, 'tcx> {
                     if let InlineAsmRegClass::Err = reg_class {
                         continue;
                     }
-                    for &(_, feature) in reg_class.supported_types(asm_arch) {
+                    for &(_, feature) in reg_class.supported_types(asm_arch, allow_experimental_reg)
+                    {
                         match feature {
                             Some(feature) => {
                                 if target_features.contains(&feature) {

compiler/rustc_hir_analysis/src/errors.rs

@@ -1708,3 +1708,11 @@ pub(crate) struct CmseEntryGeneric {
     #[primary_span]
     pub span: Span,
 }
+
+#[derive(Diagnostic)]
+#[diag(hir_analysis_register_type_unstable)]
+pub(crate) struct RegisterTypeUnstable<'a> {
+    #[primary_span]
+    pub span: Span,
+    pub ty: Ty<'a>,
+}

compiler/rustc_span/src/symbol.rs

@@ -416,6 +416,7 @@ symbols! {
         asm,
         asm_const,
         asm_experimental_arch,
+        asm_experimental_reg,
         asm_goto,
         asm_goto_with_outputs,
         asm_sym,
@@ -2140,6 +2141,7 @@ symbols! {
         vec_pop,
         vec_with_capacity,
         vecdeque_iter,
+        vector,
         version,
         vfp2,
         vis,

compiler/rustc_target/src/asm/mod.rs

@@ -604,9 +604,13 @@ impl InlineAsmRegClass {
 
     /// Returns a list of supported types for this register class, each with an
     /// options target feature required to use this type.
+    ///
+    /// At the codegen stage, it is fine to always pass true for `allow_experimental_reg`,
+    /// since all the stability checking will have been done in prior stages.
     pub fn supported_types(
         self,
         arch: InlineAsmArch,
+        allow_experimental_reg: bool,
     ) -> &'static [(InlineAsmType, Option<Symbol>)] {
         match self {
             Self::X86(r) => r.supported_types(arch),
@@ -618,7 +622,7 @@ impl InlineAsmRegClass {
             Self::Hexagon(r) => r.supported_types(arch),
             Self::LoongArch(r) => r.supported_types(arch),
             Self::Mips(r) => r.supported_types(arch),
-            Self::S390x(r) => r.supported_types(arch),
+            Self::S390x(r) => r.supported_types(arch, allow_experimental_reg),
             Self::Sparc(r) => r.supported_types(arch),
             Self::SpirV(r) => r.supported_types(arch),
             Self::Wasm(r) => r.supported_types(arch),
@@ -696,8 +700,11 @@ impl InlineAsmRegClass {
 
     /// Returns whether registers in this class can only be used as clobbers
     /// and not as inputs/outputs.
-    pub fn is_clobber_only(self, arch: InlineAsmArch) -> bool {
-        self.supported_types(arch).is_empty()
+    ///
+    /// At the codegen stage, it is fine to always pass true for `allow_experimental_reg`,
+    /// since all the stability checking will have been done in prior stages.
+    pub fn is_clobber_only(self, arch: InlineAsmArch, allow_experimental_reg: bool) -> bool {
+        self.supported_types(arch, allow_experimental_reg).is_empty()
     }
 }
 

compiler/rustc_target/src/asm/s390x.rs

@@ -38,11 +38,22 @@ impl S390xInlineAsmRegClass {
     pub fn supported_types(
         self,
         _arch: InlineAsmArch,
+        allow_experimental_reg: bool,
     ) -> &'static [(InlineAsmType, Option<Symbol>)] {
         match self {
             Self::reg | Self::reg_addr => types! { _: I8, I16, I32, I64; },
             Self::freg => types! { _: F32, F64; },
-            Self::vreg => &[],
+            Self::vreg => {
+                if allow_experimental_reg {
+                    // non-clobber-only vector register support is unstable.
+                    types! {
+                        vector: I32, F32, I64, F64, I128, F128,
+                            VecI8(16), VecI16(8), VecI32(4), VecI64(2), VecF32(4), VecF64(2);
+                    }
+                } else {
+                    &[]
+                }
+            }
             Self::areg => &[],
         }
     }

src/doc/unstable-book/src/language-features/asm-experimental-reg.md

@@ -0,0 +1,40 @@
+# `asm_experimental_arch`
+
+The tracking issue for this feature is: [#133416]
+
+[#133416]: https://github.com/rust-lang/rust/issues/133416
+
+------------------------
+
+This tracks support for additional registers in architectures where inline assembly is already stable.
+
+## Register classes
+
+| Architecture | Register class | Registers | LLVM constraint code |
+| ------------ | -------------- | --------- | -------------------- |
+| s390x | `vreg` | `v[0-31]` | `v` |
+
+> **Notes**:
+> - s390x `vreg` is clobber-only in stable.
+
+## Register class supported types
+
+| Architecture | Register class | Target feature | Allowed types |
+| ------------ | -------------- | -------------- | ------------- |
+| s390x | `vreg` | `vector` | `i32`, `f32`, `i64`, `f64`, `i128`, `f128`, `i8x16`, `i16x8`, `i32x4`, `i64x2`, `f32x4`, `f64x2` |
+
+## Register aliases
+
+| Architecture | Base register | Aliases |
+| ------------ | ------------- | ------- |
+
+## Unsupported registers
+
+| Architecture | Unsupported register | Reason |
+| ------------ | -------------------- | ------ |
+
+## Template modifiers
+
+| Architecture | Register class | Modifier | Example output | LLVM modifier |
+| ------------ | -------------- | -------- | -------------- | ------------- |
+| s390x | `vreg` | None | `%v0` | None |