diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs index 09cb74d9dcb00..5e9594dd06bb7 100644 --- a/compiler/rustc_codegen_llvm/src/builder.rs +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -507,7 +507,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' }); - let res = self.call_intrinsic(&name, &[self.type_ix(width)], &[lhs, rhs]); + let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]); (self.extract_value(res, 0), self.extract_value(res, 1)) } @@ -1038,7 +1038,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { let size = ty.primitive_size(self.tcx); let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" }; - Some(self.call_intrinsic(&name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])) + Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs])) } /* Miscellaneous instructions */ @@ -1393,7 +1393,8 @@ impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { // Forward to the `get_static` method of `CodegenCx` let global = self.cx().get_static(def_id); if self.cx().tcx.is_thread_local_static(def_id) { - let pointer = self.call_intrinsic("llvm.threadlocal.address", &[], &[global]); + let pointer = + self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]); // Cast to default address space if globals are in a different addrspace self.pointercast(pointer, self.type_ptr()) } else { @@ -1590,15 +1591,15 @@ impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { pub(crate) fn call_intrinsic( &mut self, - base_name: &str, + base_name: impl Into>, type_params: &[&'ll Type], args: &[&'ll Value], ) -> &'ll Value { - let (ty, f) = self.cx.get_intrinsic(base_name, type_params); + let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params); self.call(ty, None, None, f, args, None, None) } - fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { + fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) { let size = size.bytes(); if size == 0 { return; @@ -1608,7 +1609,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { return; } - self.call_intrinsic(intrinsic, &[self.type_ptr()], &[self.cx.const_u64(size), ptr]); + self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]); } } impl<'a, 'll, CX: Borrow>> GenericBuilder<'a, 'll, CX> { diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs index f36f3b2b16b83..bff95ea46fa7a 100644 --- a/compiler/rustc_codegen_llvm/src/context.rs +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -1,4 +1,4 @@ -use std::borrow::Borrow; +use std::borrow::{Borrow, Cow}; use std::cell::{Cell, RefCell}; use std::ffi::{CStr, c_char, c_uint}; use std::marker::PhantomData; @@ -138,7 +138,7 @@ pub(crate) struct FullCx<'ll, 'tcx> { pub rust_try_fn: Cell>, intrinsics: - RefCell), (&'ll Type, &'ll Value)>>, + RefCell, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>, /// A counter that is used for generating local symbol names local_gen_sym_counter: Cell, @@ -845,45 +845,16 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> { impl<'ll> CodegenCx<'ll, '_> { pub(crate) fn get_intrinsic( &self, - base_name: &str, + base_name: Cow<'static, str>, type_params: &[&'ll Type], ) -> (&'ll Type, &'ll Value) { - if let Some(v) = - self.intrinsics.borrow().get(&(base_name, SmallVec::from_slice(type_params))) - { - return *v; - } - - self.declare_intrinsic(base_name, type_params) - } - - fn insert_intrinsic( - &self, - base_name: &'static str, - type_params: &[&'ll Type], - args: Option<&[&'ll llvm::Type]>, - ret: &'ll llvm::Type, - ) -> (&'ll llvm::Type, &'ll llvm::Value) { - let fn_ty = if let Some(args) = args { - self.type_func(args, ret) - } else { - self.type_variadic_func(&[], ret) - }; - - let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes()) - .expect("Unknown LLVM intrinsic `{base_name}`"); - - let full_name = if intrinsic.is_overloaded() { - &intrinsic.overloaded_name(self.llmod, type_params) - } else { - base_name - }; - - let f = self.declare_cfn(full_name, llvm::UnnamedAddr::No, fn_ty); - self.intrinsics + *self + .intrinsics .borrow_mut() - .insert((base_name, SmallVec::from_slice(type_params)), (fn_ty, f)); - (fn_ty, f) + .entry((base_name, SmallVec::from_slice(type_params))) + .or_insert_with_key(|(base_name, type_params)| { + self.declare_intrinsic(base_name, type_params) + }) } fn declare_intrinsic( @@ -891,155 +862,22 @@ impl<'ll> CodegenCx<'ll, '_> { base_name: &str, type_params: &[&'ll Type], ) -> (&'ll Type, &'ll Value) { - macro_rules! param { - ($index:literal) => { - type_params[$index] - }; - ($other:expr) => { - $other - }; - } - macro_rules! ifn { - ($name:expr, fn(...) -> $ret:expr) => ( - if base_name == $name { - return self.insert_intrinsic($name, type_params, None, param!($ret)); - } - ); - ($name:expr, fn($($arg:expr),*) -> $ret:expr) => ( - if base_name == $name { - return self.insert_intrinsic($name, type_params, Some(&[$(param!($arg)),*]), param!($ret)); - } - ); - } - macro_rules! mk_struct { - ($($field_ty:expr),*) => (self.type_struct( &[$(param!($field_ty)),*], false)) - } - - let same_width_vector = |index, element_ty| { - self.type_vector(element_ty, self.vector_length(type_params[index]) as u64) - }; - - let ptr = self.type_ptr(); - let void = self.type_void(); - let i1 = self.type_i1(); - let t_i32 = self.type_i32(); - let t_i64 = self.type_i64(); - let t_isize = self.type_isize(); - let t_metadata = self.type_metadata(); - let t_token = self.type_token(); - - ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr); - ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32); - - ifn!("llvm.wasm.trunc.unsigned", fn(1) -> 0); - ifn!("llvm.wasm.trunc.signed", fn(1) -> 0); - ifn!("llvm.fptosi.sat", fn(1) -> 0); - ifn!("llvm.fptoui.sat", fn(1) -> 0); - - ifn!("llvm.trap", fn() -> void); - ifn!("llvm.debugtrap", fn() -> void); - ifn!("llvm.frameaddress", fn(t_i32) -> ptr); - - ifn!("llvm.powi", fn(0, 1) -> 0); - ifn!("llvm.pow", fn(0, 0) -> 0); - ifn!("llvm.sqrt", fn(0) -> 0); - ifn!("llvm.sin", fn(0) -> 0); - ifn!("llvm.cos", fn(0) -> 0); - ifn!("llvm.exp", fn(0) -> 0); - ifn!("llvm.exp2", fn(0) -> 0); - ifn!("llvm.log", fn(0) -> 0); - ifn!("llvm.log10", fn(0) -> 0); - ifn!("llvm.log2", fn(0) -> 0); - ifn!("llvm.fma", fn(0, 0, 0) -> 0); - ifn!("llvm.fmuladd", fn(0, 0, 0) -> 0); - ifn!("llvm.fabs", fn(0) -> 0); - ifn!("llvm.minnum", fn(0, 0) -> 0); - ifn!("llvm.minimum", fn(0, 0) -> 0); - ifn!("llvm.maxnum", fn(0, 0) -> 0); - ifn!("llvm.maximum", fn(0, 0) -> 0); - ifn!("llvm.floor", fn(0) -> 0); - ifn!("llvm.ceil", fn(0) -> 0); - ifn!("llvm.trunc", fn(0) -> 0); - ifn!("llvm.copysign", fn(0, 0) -> 0); - ifn!("llvm.round", fn(0) -> 0); - ifn!("llvm.rint", fn(0) -> 0); - ifn!("llvm.nearbyint", fn(0) -> 0); - - ifn!("llvm.ctpop", fn(0) -> 0); - ifn!("llvm.ctlz", fn(0, i1) -> 0); - ifn!("llvm.cttz", fn(0, i1) -> 0); - ifn!("llvm.bswap", fn(0) -> 0); - ifn!("llvm.bitreverse", fn(0) -> 0); - ifn!("llvm.fshl", fn(0, 0, 0) -> 0); - ifn!("llvm.fshr", fn(0, 0, 0) -> 0); - - ifn!("llvm.sadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); - ifn!("llvm.uadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); - ifn!("llvm.ssub.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); - ifn!("llvm.usub.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); - ifn!("llvm.smul.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); - ifn!("llvm.umul.with.overflow", fn(0, 0) -> mk_struct! {0, i1}); - - ifn!("llvm.sadd.sat", fn(0, 0) -> 0); - ifn!("llvm.uadd.sat", fn(0, 0) -> 0); - ifn!("llvm.ssub.sat", fn(0, 0) -> 0); - ifn!("llvm.usub.sat", fn(0, 0) -> 0); - - ifn!("llvm.scmp", fn(1, 1) -> 0); - ifn!("llvm.ucmp", fn(1, 1) -> 0); - - ifn!("llvm.lifetime.start", fn(t_i64, 0) -> void); - ifn!("llvm.lifetime.end", fn(t_i64, 0) -> void); - - ifn!("llvm.is.constant", fn(0) -> i1); - ifn!("llvm.expect", fn(0, 0) -> 0); - - ifn!("llvm.eh.typeid.for", fn(0) -> t_i32); - ifn!("llvm.localescape", fn(...) -> void); - ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr); - - ifn!("llvm.assume", fn(i1) -> void); - ifn!("llvm.prefetch", fn(0, t_i32, t_i32, t_i32) -> void); - // This isn't an "LLVM intrinsic", but LLVM's optimization passes // recognize it like one (including turning it into `bcmp` sometimes) // and we use it to implement intrinsics like `raw_eq` and `compare_bytes` if base_name == "memcmp" { - let fn_ty = self.type_func(&[ptr, ptr, t_isize], self.type_int()); + let fn_ty = self + .type_func(&[self.type_ptr(), self.type_ptr(), self.type_isize()], self.type_int()); let f = self.declare_cfn("memcmp", llvm::UnnamedAddr::No, fn_ty); - self.intrinsics.borrow_mut().insert(("memcmp", SmallVec::new()), (fn_ty, f)); return (fn_ty, f); } - // variadic intrinsics - ifn!("llvm.va_start", fn(0) -> void); - ifn!("llvm.va_end", fn(0) -> void); - ifn!("llvm.va_copy", fn(0, 0) -> void); - - if self.sess().instrument_coverage() { - ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void); - ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void); - ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void); - } - - ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1); - ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1}); - - if self.sess().opts.debuginfo != DebugInfo::None { - ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata, t_metadata) -> void); - ifn!("llvm.dbg.value", fn(t_metadata, t_metadata, t_metadata) -> void); - } - - ifn!("llvm.ptrmask", fn(0, 1) -> 0); - ifn!("llvm.threadlocal.address", fn(ptr) -> ptr); - - ifn!("llvm.masked.load", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0); - ifn!("llvm.masked.store", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void); - ifn!("llvm.masked.gather", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0); - ifn!("llvm.masked.scatter", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void); + let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes()) + .unwrap_or_else(|| bug!("Unknown intrinsic: `{base_name}`")); + let f = intrinsic.get_declaration(self.llmod, &type_params); - bug!("Unknown intrinsic: `{base_name}`") + (self.get_type_of_global(f), f) } pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs index 497c31706ec8b..f7f062849a8b5 100644 --- a/compiler/rustc_codegen_llvm/src/intrinsic.rs +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -154,8 +154,6 @@ fn call_simple_intrinsic<'ll, 'tcx>( sym::roundf64 => ("llvm.round", &[bx.type_f64()]), sym::roundf128 => ("llvm.round", &[bx.type_f128()]), - sym::ptr_mask => ("llvm.ptrmask", &[bx.type_ptr(), bx.type_isize()]), - _ => return None, }; Some(bx.call_intrinsic( @@ -181,6 +179,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { let simple = call_simple_intrinsic(self, name, args); let llval = match name { _ if simple.is_some() => simple.unwrap(), + sym::ptr_mask => { + let ptr = args[0].immediate(); + self.call_intrinsic( + "llvm.ptrmask", + &[self.val_ty(ptr), self.type_isize()], + &[ptr, args[1].immediate()], + ) + } sym::is_val_statically_known => { if let OperandValue::Immediate(imm) = args[0].val { self.call_intrinsic( @@ -232,11 +238,14 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { return Ok(()); } sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[], &[]), - sym::va_copy => self.call_intrinsic( - "llvm.va_copy", - &[self.type_ptr()], - &[args[0].immediate(), args[1].immediate()], - ), + sym::va_copy => { + let dest = args[0].immediate(); + self.call_intrinsic( + "llvm.va_copy", + &[self.val_ty(dest)], + &[dest, args[1].immediate()], + ) + } sym::va_arg => { match result.layout.backend_repr { BackendRepr::Scalar(scalar) => { @@ -309,15 +318,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::prefetch_write_instruction => (1, 0), _ => bug!(), }; + let ptr = args[0].immediate(); self.call_intrinsic( "llvm.prefetch", - &[self.type_ptr()], - &[ - args[0].immediate(), - self.const_i32(rw), - args[1].immediate(), - self.const_i32(cache_type), - ], + &[self.val_ty(ptr)], + &[ptr, self.const_i32(rw), args[1].immediate(), self.const_i32(cache_type)], ) } sym::carrying_mul_add => { @@ -378,7 +383,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { sym::ctlz | sym::cttz => { let y = self.const_bool(false); let ret = self.call_intrinsic( - &format!("llvm.{name}"), + format!("llvm.{name}"), &[llty], &[args[0].immediate(), y], ); @@ -423,7 +428,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { // always uses `u32`. let raw_shift = self.intcast(raw_shift, self.val_ty(val), false); - self.call_intrinsic(&llvm_name, &[llty], &[val, val, raw_shift]) + self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift]) } sym::saturating_add | sym::saturating_sub => { let is_add = name == sym::saturating_add; @@ -434,7 +439,7 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { if signed { 's' } else { 'u' }, if is_add { "add" } else { "sub" }, ); - self.call_intrinsic(&llvm_name, &[llty], &[lhs, rhs]) + self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs]) } _ => bug!(), } @@ -637,11 +642,11 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { } fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { - self.call_intrinsic("llvm.va_start", &[self.type_ptr()], &[va_list]) + self.call_intrinsic("llvm.va_start", &[self.val_ty(va_list)], &[va_list]) } fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { - self.call_intrinsic("llvm.va_end", &[self.type_ptr()], &[va_list]) + self.call_intrinsic("llvm.va_end", &[self.val_ty(va_list)], &[va_list]) } } @@ -1018,7 +1023,7 @@ fn codegen_emcc_try<'ll, 'tcx>( let selector = bx.extract_value(vals, 1); // Check if the typeid we got is the one for a Rust panic. - let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.type_ptr()], &[tydesc]); + let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[bx.val_ty(tydesc)], &[tydesc]); let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid); let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool()); @@ -2393,7 +2398,7 @@ fn generic_simd_intrinsic<'ll, 'tcx>( ); let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); - return Ok(bx.call_intrinsic(&llvm_intrinsic, &[vec_ty], &[lhs, rhs])); + return Ok(bx.call_intrinsic(llvm_intrinsic, &[vec_ty], &[lhs, rhs])); } span_bug!(span, "unknown SIMD intrinsic"); diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs index 59c61db5fcdcb..91ada856d5977 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -1078,8 +1078,6 @@ unsafe extern "C" { // Operations on other types pub(crate) fn LLVMVoidTypeInContext(C: &Context) -> &Type; - pub(crate) fn LLVMTokenTypeInContext(C: &Context) -> &Type; - pub(crate) fn LLVMMetadataTypeInContext(C: &Context) -> &Type; // Operations on all values pub(crate) fn LLVMTypeOf(Val: &Value) -> &Type; @@ -1198,14 +1196,12 @@ unsafe extern "C" { // Operations about llvm intrinsics pub(crate) fn LLVMLookupIntrinsicID(Name: *const c_char, NameLen: size_t) -> c_uint; - pub(crate) fn LLVMIntrinsicIsOverloaded(ID: NonZero) -> Bool; - pub(crate) fn LLVMIntrinsicCopyOverloadedName2<'a>( + pub(crate) fn LLVMGetIntrinsicDeclaration<'a>( Mod: &'a Module, ID: NonZero, ParamTypes: *const &'a Type, ParamCount: size_t, - NameLength: *mut size_t, - ) -> *mut c_char; + ) -> &'a Value; // Operations on parameters pub(crate) fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs index bc3538c768dee..661174a80dfbd 100644 --- a/compiler/rustc_codegen_llvm/src/llvm/mod.rs +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -2,9 +2,9 @@ use std::ffi::{CStr, CString}; use std::num::NonZero; +use std::ptr; use std::str::FromStr; use std::string::FromUtf8Error; -use std::{ptr, slice}; use libc::c_uint; use rustc_abi::{Align, Size, WrappingRange}; @@ -339,34 +339,14 @@ impl Intrinsic { NonZero::new(id).map(|id| Self { id }) } - pub(crate) fn is_overloaded(self) -> bool { - unsafe { LLVMIntrinsicIsOverloaded(self.id) == True } - } - - pub(crate) fn overloaded_name<'ll>( + pub(crate) fn get_declaration<'ll>( self, llmod: &'ll Module, type_params: &[&'ll Type], - ) -> String { - let mut len = 0; - let ptr = unsafe { - LLVMIntrinsicCopyOverloadedName2( - llmod, - self.id, - type_params.as_ptr(), - type_params.len(), - &mut len, - ) - }; - - let slice = unsafe { slice::from_raw_parts_mut(ptr.cast(), len) }; - let copied = str::from_utf8(slice).expect("Non-UTF8 intrinsic name").to_string(); - + ) -> &'ll Value { unsafe { - libc::free(ptr.cast()); + LLVMGetIntrinsicDeclaration(llmod, self.id, type_params.as_ptr(), type_params.len()) } - - copied } } diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs index 2364ad0c9d237..453eca2bbe173 100644 --- a/compiler/rustc_codegen_llvm/src/type_.rs +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -58,13 +58,6 @@ impl<'ll, CX: Borrow>> GenericCx<'ll, CX> { pub(crate) fn type_void(&self) -> &'ll Type { unsafe { llvm::LLVMVoidTypeInContext(self.llcx()) } } - pub(crate) fn type_token(&self) -> &'ll Type { - unsafe { llvm::LLVMTokenTypeInContext(self.llcx()) } - } - - pub(crate) fn type_metadata(&self) -> &'ll Type { - unsafe { llvm::LLVMMetadataTypeInContext(self.llcx()) } - } ///x Creates an integer type with the given number of bits, e.g., i24 pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type {