Skip to content

Use LLVMIntrinsicGetDeclaration to completely remove the hardcoded intrinsics list #142521

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 16, 2025
+55 −242
Merged

Use LLVMIntrinsicGetDeclaration to completely remove the hardcoded intrinsics list #142521

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9415f3d
Use `LLVMIntrinsicGetDeclaration` to completely remove the hardcoded …
sayantn Jun 14, 2025
a9500d6
Correctly account for different address spaces in LLVM intrinsic invo…
sayantn Jun 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions compiler/rustc_codegen_llvm/src/builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {

let name = format!("llvm.{}{oop_str}.with.overflow", if signed { 's' } else { 'u' });

let res = self.call_intrinsic(&name, &[self.type_ix(width)], &[lhs, rhs]);
let res = self.call_intrinsic(name, &[self.type_ix(width)], &[lhs, rhs]);
(self.extract_value(res, 0), self.extract_value(res, 1))
}

Expand Down Expand Up @@ -1038,7 +1038,7 @@ impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> {
let size = ty.primitive_size(self.tcx);
let name = if ty.is_signed() { "llvm.scmp" } else { "llvm.ucmp" };

Some(self.call_intrinsic(&name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
Some(self.call_intrinsic(name, &[self.type_i8(), self.type_ix(size.bits())], &[lhs, rhs]))
}

/* Miscellaneous instructions */
Expand Down Expand Up @@ -1393,7 +1393,8 @@ impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> {
// Forward to the `get_static` method of `CodegenCx`
let global = self.cx().get_static(def_id);
if self.cx().tcx.is_thread_local_static(def_id) {
let pointer = self.call_intrinsic("llvm.threadlocal.address", &[], &[global]);
let pointer =
self.call_intrinsic("llvm.threadlocal.address", &[self.val_ty(global)], &[global]);
// Cast to default address space if globals are in a different addrspace
self.pointercast(pointer, self.type_ptr())
} else {
Expand Down Expand Up @@ -1590,15 +1591,15 @@ impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
pub(crate) fn call_intrinsic(
&mut self,
base_name: &str,
base_name: impl Into<Cow<'static, str>>,
type_params: &[&'ll Type],
args: &[&'ll Value],
) -> &'ll Value {
let (ty, f) = self.cx.get_intrinsic(base_name, type_params);
let (ty, f) = self.cx.get_intrinsic(base_name.into(), type_params);
self.call(ty, None, None, f, args, None, None)
}

fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) {
fn call_lifetime_intrinsic(&mut self, intrinsic: &'static str, ptr: &'ll Value, size: Size) {
let size = size.bytes();
if size == 0 {
return;
Expand All @@ -1608,7 +1609,7 @@ impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> {
return;
}

self.call_intrinsic(intrinsic, &[self.type_ptr()], &[self.cx.const_u64(size), ptr]);
self.call_intrinsic(intrinsic, &[self.val_ty(ptr)], &[self.cx.const_u64(size), ptr]);
}
}
impl<'a, 'll, CX: Borrow<SCx<'ll>>> GenericBuilder<'a, 'll, CX> {
Expand Down
192 changes: 15 additions & 177 deletions compiler/rustc_codegen_llvm/src/context.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
use std::borrow::Borrow;
use std::borrow::{Borrow, Cow};
use std::cell::{Cell, RefCell};
use std::ffi::{CStr, c_char, c_uint};
use std::marker::PhantomData;
Expand Down Expand Up @@ -138,7 +138,7 @@ pub(crate) struct FullCx<'ll, 'tcx> {
pub rust_try_fn: Cell<Option<(&'ll Type, &'ll Value)>>,

intrinsics:
RefCell<FxHashMap<(&'static str, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,
RefCell<FxHashMap<(Cow<'static, str>, SmallVec<[&'ll Type; 2]>), (&'ll Type, &'ll Value)>>,

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Did this become a Cow because previously everything was indirected via the &'static strs in declare_intrinsic, and now no longer is?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I just wanted to reduce some allocations (because most intrinsic names are string literals, but some are generated by format!). String would work as well. Should I change it?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be fine.

/// A counter that is used for generating local symbol names
local_gen_sym_counter: Cell<usize>,
Expand Down Expand Up @@ -845,201 +845,39 @@ impl<'ll, 'tcx> MiscCodegenMethods<'tcx> for CodegenCx<'ll, 'tcx> {
impl<'ll> CodegenCx<'ll, '_> {
pub(crate) fn get_intrinsic(
&self,
base_name: &str,
base_name: Cow<'static, str>,
type_params: &[&'ll Type],
) -> (&'ll Type, &'ll Value) {
if let Some(v) =
self.intrinsics.borrow().get(&(base_name, SmallVec::from_slice(type_params)))
{
return *v;
}

self.declare_intrinsic(base_name, type_params)
}

fn insert_intrinsic(
&self,
base_name: &'static str,
type_params: &[&'ll Type],
args: Option<&[&'ll llvm::Type]>,
ret: &'ll llvm::Type,
) -> (&'ll llvm::Type, &'ll llvm::Value) {
let fn_ty = if let Some(args) = args {
self.type_func(args, ret)
} else {
self.type_variadic_func(&[], ret)
};

let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
.expect("Unknown LLVM intrinsic `{base_name}`");

let full_name = if intrinsic.is_overloaded() {
&intrinsic.overloaded_name(self.llmod, type_params)
} else {
base_name
};

let f = self.declare_cfn(full_name, llvm::UnnamedAddr::No, fn_ty);
self.intrinsics
*self
.intrinsics
.borrow_mut()
.insert((base_name, SmallVec::from_slice(type_params)), (fn_ty, f));
(fn_ty, f)
.entry((base_name, SmallVec::from_slice(type_params)))
.or_insert_with_key(|(base_name, type_params)| {
self.declare_intrinsic(base_name, type_params)
})
}

fn declare_intrinsic(
&self,
base_name: &str,
type_params: &[&'ll Type],
) -> (&'ll Type, &'ll Value) {
macro_rules! param {
($index:literal) => {
type_params[$index]
};
($other:expr) => {
$other
};
}
macro_rules! ifn {
($name:expr, fn(...) -> $ret:expr) => (
if base_name == $name {
return self.insert_intrinsic($name, type_params, None, param!($ret));
}
);
($name:expr, fn($($arg:expr),*) -> $ret:expr) => (
if base_name == $name {
return self.insert_intrinsic($name, type_params, Some(&[$(param!($arg)),*]), param!($ret));
}
);
}
macro_rules! mk_struct {
($($field_ty:expr),*) => (self.type_struct( &[$(param!($field_ty)),*], false))
}

let same_width_vector = |index, element_ty| {
self.type_vector(element_ty, self.vector_length(type_params[index]) as u64)
};

let ptr = self.type_ptr();
let void = self.type_void();
let i1 = self.type_i1();
let t_i32 = self.type_i32();
let t_i64 = self.type_i64();
let t_isize = self.type_isize();
let t_metadata = self.type_metadata();
let t_token = self.type_token();

ifn!("llvm.wasm.get.exception", fn(t_token) -> ptr);
ifn!("llvm.wasm.get.ehselector", fn(t_token) -> t_i32);

ifn!("llvm.wasm.trunc.unsigned", fn(1) -> 0);
ifn!("llvm.wasm.trunc.signed", fn(1) -> 0);
ifn!("llvm.fptosi.sat", fn(1) -> 0);
ifn!("llvm.fptoui.sat", fn(1) -> 0);

ifn!("llvm.trap", fn() -> void);
ifn!("llvm.debugtrap", fn() -> void);
ifn!("llvm.frameaddress", fn(t_i32) -> ptr);

ifn!("llvm.powi", fn(0, 1) -> 0);
ifn!("llvm.pow", fn(0, 0) -> 0);
ifn!("llvm.sqrt", fn(0) -> 0);
ifn!("llvm.sin", fn(0) -> 0);
ifn!("llvm.cos", fn(0) -> 0);
ifn!("llvm.exp", fn(0) -> 0);
ifn!("llvm.exp2", fn(0) -> 0);
ifn!("llvm.log", fn(0) -> 0);
ifn!("llvm.log10", fn(0) -> 0);
ifn!("llvm.log2", fn(0) -> 0);
ifn!("llvm.fma", fn(0, 0, 0) -> 0);
ifn!("llvm.fmuladd", fn(0, 0, 0) -> 0);
ifn!("llvm.fabs", fn(0) -> 0);
ifn!("llvm.minnum", fn(0, 0) -> 0);
ifn!("llvm.minimum", fn(0, 0) -> 0);
ifn!("llvm.maxnum", fn(0, 0) -> 0);
ifn!("llvm.maximum", fn(0, 0) -> 0);
ifn!("llvm.floor", fn(0) -> 0);
ifn!("llvm.ceil", fn(0) -> 0);
ifn!("llvm.trunc", fn(0) -> 0);
ifn!("llvm.copysign", fn(0, 0) -> 0);
ifn!("llvm.round", fn(0) -> 0);
ifn!("llvm.rint", fn(0) -> 0);
ifn!("llvm.nearbyint", fn(0) -> 0);

ifn!("llvm.ctpop", fn(0) -> 0);
ifn!("llvm.ctlz", fn(0, i1) -> 0);
ifn!("llvm.cttz", fn(0, i1) -> 0);
ifn!("llvm.bswap", fn(0) -> 0);
ifn!("llvm.bitreverse", fn(0) -> 0);
ifn!("llvm.fshl", fn(0, 0, 0) -> 0);
ifn!("llvm.fshr", fn(0, 0, 0) -> 0);

ifn!("llvm.sadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.uadd.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.ssub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.usub.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.smul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});
ifn!("llvm.umul.with.overflow", fn(0, 0) -> mk_struct! {0, i1});

ifn!("llvm.sadd.sat", fn(0, 0) -> 0);
ifn!("llvm.uadd.sat", fn(0, 0) -> 0);
ifn!("llvm.ssub.sat", fn(0, 0) -> 0);
ifn!("llvm.usub.sat", fn(0, 0) -> 0);

ifn!("llvm.scmp", fn(1, 1) -> 0);
ifn!("llvm.ucmp", fn(1, 1) -> 0);

ifn!("llvm.lifetime.start", fn(t_i64, 0) -> void);
ifn!("llvm.lifetime.end", fn(t_i64, 0) -> void);

ifn!("llvm.is.constant", fn(0) -> i1);
ifn!("llvm.expect", fn(0, 0) -> 0);

ifn!("llvm.eh.typeid.for", fn(0) -> t_i32);
ifn!("llvm.localescape", fn(...) -> void);
ifn!("llvm.localrecover", fn(ptr, ptr, t_i32) -> ptr);

ifn!("llvm.assume", fn(i1) -> void);
ifn!("llvm.prefetch", fn(0, t_i32, t_i32, t_i32) -> void);

// This isn't an "LLVM intrinsic", but LLVM's optimization passes
// recognize it like one (including turning it into `bcmp` sometimes)
// and we use it to implement intrinsics like `raw_eq` and `compare_bytes`
if base_name == "memcmp" {
let fn_ty = self.type_func(&[ptr, ptr, t_isize], self.type_int());
let fn_ty = self
.type_func(&[self.type_ptr(), self.type_ptr(), self.type_isize()], self.type_int());
let f = self.declare_cfn("memcmp", llvm::UnnamedAddr::No, fn_ty);
self.intrinsics.borrow_mut().insert(("memcmp", SmallVec::new()), (fn_ty, f));

return (fn_ty, f);
}

// variadic intrinsics
ifn!("llvm.va_start", fn(0) -> void);
ifn!("llvm.va_end", fn(0) -> void);
ifn!("llvm.va_copy", fn(0, 0) -> void);

if self.sess().instrument_coverage() {
ifn!("llvm.instrprof.increment", fn(ptr, t_i64, t_i32, t_i32) -> void);
ifn!("llvm.instrprof.mcdc.parameters", fn(ptr, t_i64, t_i32) -> void);
ifn!("llvm.instrprof.mcdc.tvbitmap.update", fn(ptr, t_i64, t_i32, ptr) -> void);
}

ifn!("llvm.type.test", fn(ptr, t_metadata) -> i1);
ifn!("llvm.type.checked.load", fn(ptr, t_i32, t_metadata) -> mk_struct! {ptr, i1});

if self.sess().opts.debuginfo != DebugInfo::None {
ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata, t_metadata) -> void);
ifn!("llvm.dbg.value", fn(t_metadata, t_metadata, t_metadata) -> void);
}

ifn!("llvm.ptrmask", fn(0, 1) -> 0);
ifn!("llvm.threadlocal.address", fn(ptr) -> ptr);

ifn!("llvm.masked.load", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
ifn!("llvm.masked.store", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
ifn!("llvm.masked.gather", fn(1, t_i32, same_width_vector(0, i1), 0) -> 0);
ifn!("llvm.masked.scatter", fn(0, 1, t_i32, same_width_vector(0, i1)) -> void);
let intrinsic = llvm::Intrinsic::lookup(base_name.as_bytes())
.unwrap_or_else(|| bug!("Unknown intrinsic: `{base_name}`"));
let f = intrinsic.get_declaration(self.llmod, &type_params);

bug!("Unknown intrinsic: `{base_name}`")
(self.get_type_of_global(f), f)
}

pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value {
Expand Down
Loading
Loading