diff --git a/src/common.rs b/src/common.rs index a503c1b3451..21d92c6cc29 100644 --- a/src/common.rs +++ b/src/common.rs @@ -126,68 +126,86 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { } } +/// The element type and element count of the array used to represent a run of `len` constant bytes. +/// +/// Larger integers are used where possible: this reduces the number of rvalues, which is a +/// significant memory saving on constant-heavy crates. +fn byte_run_shape<'gcc>(cx: &CodegenCx<'gcc, '_>, len: usize) -> (Type<'gcc>, u64) { + match len % 8 { + 0 => (cx.context.new_type::(), len as u64 / 8), + 4 => (cx.context.new_type::(), len as u64 / 4), + _ => (cx.context.new_type::(), len as u64), + } +} + +/// The type [`bytes_in_context`] gives a run of `len` constant bytes. +/// +/// Exposed separately so that the type of a constant allocation can be computed before any of its +/// rvalues exist; see [`crate::consts::const_alloc_type`]. +/// +/// The result is cached because `gcc_jit_context_new_array_type` mints a fresh type every call. +/// Two equal-but-distinct array types would key [`CodegenCx::type_struct`] differently and so +/// produce two distinct anonymous structs, and libgccjit compares struct types by identity. +pub fn bytes_type_in_context<'gcc>(cx: &CodegenCx<'gcc, '_>, len: usize) -> Type<'gcc> { + let (element_type, count) = byte_run_shape(cx, len); + if let Some(&typ) = cx.byte_array_types.borrow().get(&(element_type, count)) { + return typ; + } + let typ = new_array_type(cx.context, None, element_type, count); + cx.byte_array_types.borrow_mut().insert((element_type, count), typ); + typ +} + +// FIXME(FractalFir): Consider using `global_set_initializer` instead. Before this is done, we need to confirm that +// `global_set_initializer` is more memory efficient than the current solution. +// `global_set_initializer` calls `global_set_initializer_rvalue` under the hood - does it generate an array of rvalues, +// or is it using a more efficient representation? pub fn bytes_in_context<'gcc, 'tcx>(cx: &CodegenCx<'gcc, 'tcx>, bytes: &[u8]) -> RValue<'gcc> { - // Instead of always using an array of bytes, use an array of larger integers of target endianness - // if possible. This reduces the amount of `rvalues` we use, which reduces memory usage significantly. - // - // FIXME(FractalFir): Consider using `global_set_initializer` instead. Before this is done, we need to confirm that - // `global_set_initializer` is more memory efficient than the current solution. - // `global_set_initializer` calls `global_set_initializer_rvalue` under the hood - does it generate an array of rvalues, - // or is it using a more efficient representation? - match bytes.len() % 8 { + let typ = bytes_type_in_context(cx, bytes.len()); + let (element_type, _) = byte_run_shape(cx, bytes.len()); + let context = &cx.context; + // Since we are representing arbitrary byte runs as integers, we need to follow the target + // endianness. + let endian = cx.sess().target.options.endian; + let elements: Vec<_> = match bytes.len() % 8 { 0 => { - let context = &cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64 / 8); let (arrays, remainder) = bytes.as_chunks::<8>(); debug_assert!(remainder.is_empty()); - let elements: Vec<_> = arrays + arrays .iter() .map(|&arr| { context.new_rvalue_from_long( - byte_type, - // Since we are representing arbitrary byte runs as integers, we need to follow the target - // endianness. - match cx.sess().target.options.endian { + element_type, + match endian { rustc_abi::Endian::Little => u64::from_le_bytes(arr) as i64, rustc_abi::Endian::Big => u64::from_be_bytes(arr) as i64, }, ) }) - .collect(); - context.new_array_constructor(None, typ, &elements) + .collect() } 4 => { - let context = &cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64 / 4); let (arrays, remainder) = bytes.as_chunks::<4>(); debug_assert!(remainder.is_empty()); - let elements: Vec<_> = arrays + arrays .iter() .map(|&arr| { context.new_rvalue_from_int( - byte_type, - match cx.sess().target.options.endian { + element_type, + match endian { rustc_abi::Endian::Little => u32::from_le_bytes(arr) as i32, rustc_abi::Endian::Big => u32::from_be_bytes(arr) as i32, }, ) }) - .collect(); - context.new_array_constructor(None, typ, &elements) - } - _ => { - let context = cx.context; - let byte_type = context.new_type::(); - let typ = new_array_type(context, None, byte_type, bytes.len() as u64); - let elements: Vec<_> = bytes - .iter() - .map(|&byte| context.new_rvalue_from_int(byte_type, byte as i32)) - .collect(); - context.new_array_constructor(None, typ, &elements) + .collect() } - } + _ => bytes + .iter() + .map(|&byte| context.new_rvalue_from_int(element_type, byte as i32)) + .collect(), + }; + context.new_array_constructor(None, typ, &elements) } pub fn type_is_pointer(typ: Type<'_>) -> bool { diff --git a/src/consts.rs b/src/consts.rs index 5ebdf91fe20..b1e06f88a23 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -1,6 +1,8 @@ +use std::ops::Range; + #[cfg(feature = "master")] -use gccjit::{FnAttribute, ToRValue, VarAttribute, Visibility}; -use gccjit::{Function, GlobalKind, LValue, RValue, Type}; +use gccjit::{FnAttribute, VarAttribute, Visibility}; +use gccjit::{Function, GlobalKind, LValue, RValue, ToRValue, Type}; use rustc_abi::{self as abi, Align, HasDataLayout, Primitive, Size, WrappingRange}; use rustc_codegen_ssa::traits::{ BaseTypeCodegenMethods, ConstCodegenMethods, StaticCodegenMethods, @@ -11,15 +13,18 @@ use rustc_hir::def_id::LOCAL_CRATE; use rustc_log::tracing::trace; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::interpret::{ - self, ConstAllocation, ErrorHandled, Scalar as InterpScalar, read_target_uint, + self, ConstAllocation, CtfeProvenance, ErrorHandled, Scalar as InterpScalar, read_target_uint, }; +use rustc_middle::mono::MonoItem; use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Instance}; use rustc_middle::{bug, span_bug}; use rustc_span::def_id::DefId; use crate::base; +use crate::common::bytes_type_in_context; use crate::context::CodegenCx; +use crate::type_::struct_attributes; use crate::type_of::LayoutGccExt; pub(crate) fn const_alloc_to_gcc<'gcc, 'tcx>( @@ -99,10 +104,11 @@ impl<'gcc, 'tcx> StaticCodegenMethods for CodegenCx<'gcc, 'tcx> { let is_thread_local = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); let global = self.get_static_inner(def_id, val_llty); - #[cfg(feature = "master")] - if global.to_rvalue().get_type() != val_llty { - global.to_rvalue().set_type(val_llty); - } + debug_assert_eq!( + global.to_rvalue().get_type(), + val_llty, + "`predefine_static` declared this global with a type its initializer does not have" + ); // NOTE: Alignment from attributes has already been applied to the allocation. set_global_alignment(self, global, alloc.align); @@ -260,15 +266,14 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { return global; } - // FIXME: Once we stop removing globals in `codegen_static`, we can uncomment this code. - // let defined_in_current_codegen_unit = - // self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); - // assert!( - // !defined_in_current_codegen_unit, - // "consts::get_static() should always hit the cache for \ - // statics defined in the same CGU, but did not for `{:?}`", - // def_id - // ); + let defined_in_current_codegen_unit = + self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); + assert!( + !defined_in_current_codegen_unit, + "consts::get_static() should always hit the cache for \ + statics defined in the same CGU, but did not for `{:?}`", + def_id + ); let sym = self.tcx.symbol_name(instance).name; let fn_attrs = self.tcx.codegen_fn_attrs(def_id); @@ -332,71 +337,118 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { global } } -/// Converts a given const alloc to a gcc Rvalue, without any caching or deduplication. -/// YOU SHOULD NOT call this function directly - that may break the semantics of Rust. -/// Use `const_data_from_alloc` instead. -pub(crate) fn const_alloc_to_gcc_uncached<'gcc>( - cx: &CodegenCx<'gcc, '_>, - alloc: ConstAllocation<'_>, -) -> RValue<'gcc> { - let alloc = alloc.inner(); - let mut llvals = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); - let dl = cx.data_layout(); - let pointer_size = dl.pointer_size().bytes() as usize; +/// One field of the packed struct that a constant allocation is lowered to. +enum AllocField { + /// A run of bytes carrying no provenance. + Bytes { range: Range }, + /// A pointer with provenance, occupying one target pointer worth of bytes. + Pointer { offset: usize, prov: CtfeProvenance }, +} + +/// The field-by-field shape of `alloc`. +/// +/// [`const_alloc_to_gcc_uncached`] and [`const_alloc_type`] have to agree exactly on this, down to +/// the empty trailing run an allocation ending on a pointer produces, so both derive the shape here +/// instead of each walking the allocation on its own. +fn alloc_fields(cx: &CodegenCx<'_, '_>, alloc: &interpret::Allocation) -> Vec { + let pointer_size = cx.data_layout().pointer_size().bytes() as usize; + let mut fields = Vec::with_capacity(alloc.provenance().ptrs().len() + 1); let mut next_offset = 0; for &(offset, prov) in alloc.provenance().ptrs().iter() { - let alloc_id = prov.alloc_id(); let offset = offset.bytes(); assert_eq!(offset as usize as u64, offset); let offset = offset as usize; if offset > next_offset { - // This `inspect` is okay since we have checked that it is not within a pointer with provenance, it - // is within the bounds of the allocation, and it doesn't affect interpreter execution - // (we inspect the result after interpreter execution). Any undef byte is replaced with - // some arbitrary byte value. - // - // FIXME: relay undef bytes to codegen as undef const bytes - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(next_offset..offset); - llvals.push(cx.const_bytes(bytes)); + fields.push(AllocField::Bytes { range: next_offset..offset }); } - let ptr_offset = read_target_uint( - dl.endian, - // This `inspect` is okay since it is within the bounds of the allocation, it doesn't - // affect interpreter execution (we inspect the result after interpreter execution), - // and we properly interpret the provenance as a relocation pointer offset. - alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), - ) - .expect("const_alloc_to_gcc_uncached: could not read relocation pointer") - as u64; - - let address_space = cx.tcx.global_alloc(alloc_id).address_space(cx); - - llvals.push(cx.scalar_to_backend( - InterpScalar::from_pointer( - interpret::Pointer::new(prov, Size::from_bytes(ptr_offset)), - &cx.tcx, - ), - abi::Scalar::Initialized { - value: Primitive::Pointer(address_space), - valid_range: WrappingRange::full(dl.pointer_size()), - }, - cx.type_i8p_ext(address_space), - )); + fields.push(AllocField::Pointer { offset, prov }); next_offset = offset + pointer_size; } if alloc.len() >= next_offset { - let range = next_offset..alloc.len(); - // This `inspect` is okay since we have check that it is after all provenance, it is - // within the bounds of the allocation, and it doesn't affect interpreter execution (we - // inspect the result after interpreter execution). Any undef byte is replaced with some - // arbitrary byte value. - // - // FIXME: relay undef bytes to codegen as undef const bytes - let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); - llvals.push(cx.const_bytes(bytes)); + fields.push(AllocField::Bytes { range: next_offset..alloc.len() }); } + fields +} + +/// The type [`const_alloc_to_gcc`] gives `alloc`, computed without building any rvalue. +/// +/// This lets `predefine_static` declare a static's global with the type its initializer will have, +/// so that the two never disagree. It must not reach for the rvalue of anything it points at: +/// during the predefine pass the pointee may not be declared yet, and `alloc_to_backend` would +/// declare it with the wrong type behind our back. +pub(crate) fn const_alloc_type<'gcc>( + cx: &CodegenCx<'gcc, '_>, + alloc: ConstAllocation<'_>, +) -> Type<'gcc> { + let fields: Vec<_> = alloc_fields(cx, alloc.inner()) + .into_iter() + .map(|field| match field { + AllocField::Bytes { range } => bytes_type_in_context(cx, range.len()), + AllocField::Pointer { prov, .. } => { + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); + cx.type_i8p_ext(address_space) + } + }) + .collect(); + cx.type_struct(&fields, &struct_attributes(true, None)) +} + +/// Converts a given const alloc to a gcc Rvalue, without any caching or deduplication. +/// YOU SHOULD NOT call this function directly - that may break the semantics of Rust. +/// Use `const_data_from_alloc` instead. +pub(crate) fn const_alloc_to_gcc_uncached<'gcc>( + cx: &CodegenCx<'gcc, '_>, + alloc: ConstAllocation<'_>, +) -> RValue<'gcc> { + let alloc = alloc.inner(); + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size(); + + let llvals: Vec<_> = alloc_fields(cx, alloc) + .into_iter() + .map(|field| match field { + AllocField::Bytes { range } => { + // This `inspect` is okay since we have checked that it is not within a pointer with + // provenance, it is within the bounds of the allocation, and it doesn't affect + // interpreter execution (we inspect the result after interpreter execution). Any + // undef byte is replaced with some arbitrary byte value. + // + // FIXME: relay undef bytes to codegen as undef const bytes + cx.const_bytes(alloc.inspect_with_uninit_and_ptr_outside_interpreter(range)) + } + AllocField::Pointer { offset, prov } => { + let ptr_offset = read_target_uint( + dl.endian, + // This `inspect` is okay since it is within the bounds of the allocation, it + // doesn't affect interpreter execution (we inspect the result after interpreter + // execution), and we properly interpret the provenance as a relocation pointer + // offset. + alloc.inspect_with_uninit_and_ptr_outside_interpreter( + offset..(offset + pointer_size.bytes() as usize), + ), + ) + .expect("const_alloc_to_gcc_uncached: could not read relocation pointer") + as u64; + + let address_space = cx.tcx.global_alloc(prov.alloc_id()).address_space(cx); + + cx.scalar_to_backend( + InterpScalar::from_pointer( + interpret::Pointer::new(prov, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + abi::Scalar::Initialized { + value: Primitive::Pointer(address_space), + valid_range: WrappingRange::full(pointer_size), + }, + cx.type_i8p_ext(address_space), + ) + } + }) + .collect(); + // FIXME(bjorn3) avoid wrapping in a struct when there is only a single element. cx.const_struct(&llvals, true) } diff --git a/src/context.rs b/src/context.rs index ebbdbb72516..cd835d23e6e 100644 --- a/src/context.rs +++ b/src/context.rs @@ -97,6 +97,12 @@ pub struct CodegenCx<'gcc, 'tcx> { /// Cache of the anonymous struct types. pub struct_types: RefCell, Type<'gcc>>>, + /// Cache of the array types used for runs of constant bytes, keyed by element type and count. + /// + /// libgccjit mints a fresh type on every `new_array_type`, and struct types are keyed on their + /// field types, so without this two equal byte runs would yield two distinct anonymous structs. + pub byte_array_types: RefCell, u64), Type<'gcc>>>, + /// Cache instances of monomorphic and polymorphic items pub instances: RefCell, LValue<'gcc>>>, /// Cache function instances of monomorphic and polymorphic items @@ -314,6 +320,7 @@ impl<'gcc, 'tcx> CodegenCx<'gcc, 'tcx> { types: Default::default(), tcx, struct_types: Default::default(), + byte_array_types: Default::default(), local_gen_sym_counter: Cell::new(0), global_gen_sym_counter: Cell::new(0), eh_personality: Cell::new(None), diff --git a/src/mono_item.rs b/src/mono_item.rs index 7513978b122..144bdee65e5 100644 --- a/src/mono_item.rs +++ b/src/mono_item.rs @@ -11,6 +11,7 @@ use rustc_middle::mono::Visibility; use rustc_middle::ty::layout::{FnAbiOf, HasTypingEnv, LayoutOf}; use rustc_middle::ty::{self, Instance, TypeVisitableExt}; +use crate::consts::const_alloc_type; use crate::context::CodegenCx; use crate::type_of::LayoutGccExt; use crate::{attributes, base}; @@ -26,12 +27,24 @@ impl<'gcc, 'tcx> PreDefineCodegenMethods<'tcx> for CodegenCx<'gcc, 'tcx> { ) { let attrs = self.tcx.codegen_fn_attrs(def_id); let instance = Instance::mono(self.tcx, def_id); - let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; - // Nested statics do not have a type, so pick a dummy type and let `codegen_static` figure out - // the gcc type from the actual evaluated initializer. - let ty = - if nested { self.tcx.types.unit } else { instance.ty(self.tcx, self.typing_env()) }; - let gcc_type = self.layout_of(ty).gcc_type(self); + // Declare the global with the type its initializer will have, so that `codegen_static` + // never has to retype it afterwards. The initializer is lowered as a packed struct of byte + // runs and relocations, which almost never matches the layout type. + let gcc_type = match self.tcx.eval_static_initializer(def_id) { + Ok(alloc) => const_alloc_type(self, alloc), + // The initializer failed to evaluate; `codegen_static` bails out on it too, so this + // type is never used to hold one. + Err(_) => { + let DefKind::Static { nested, .. } = self.tcx.def_kind(def_id) else { bug!() }; + // Nested statics do not have a type, so pick a dummy one. + let ty = if nested { + self.tcx.types.unit + } else { + instance.ty(self.tcx, self.typing_env()) + }; + self.layout_of(ty).gcc_type(self) + } + }; let is_tls = attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL); diff --git a/tests/run/static_alloc_shapes.rs b/tests/run/static_alloc_shapes.rs new file mode 100644 index 00000000000..39a4d07bdf6 --- /dev/null +++ b/tests/run/static_alloc_shapes.rs @@ -0,0 +1,50 @@ +// Compiler: +// +// Run-time: +// status: 0 +// stdout: 8 +// 12 +// 5 +// 7 +// 7 +// 9 + +#![feature(no_core)] +#![no_std] +#![no_core] +#![no_main] + +extern crate mini_core; +use mini_core::*; + +// One byte run of each length class that maps to a distinct array element type. +static mut BYTES8: [u8; 8] = [1, 2, 3, 4, 5, 6, 7, 8]; +static mut BYTES12: [u8; 12] = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]; +static mut BYTES5: [u8; 5] = [1, 2, 3, 4, 5]; + +static mut VALUE: isize = 7; +static mut OTHER: isize = 9; + +// An allocation that is exactly one relocation, so it ends on a pointer with no trailing bytes. +static mut PTR: &isize = unsafe { &VALUE }; + +struct TwoRefs { + first: &'static isize, + second: &'static isize, +} + +// Two adjacent relocations, with no byte run between them. +static mut TWO_REFS: TwoRefs = TwoRefs { first: unsafe { &VALUE }, second: unsafe { &OTHER } }; + +#[no_mangle] +extern "C" fn main(_argc: isize, _argv: *const *const u8) -> i32 { + unsafe { + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES8[7] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES12[11] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, BYTES5[4] as isize); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *PTR); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *TWO_REFS.first); + libc::printf(b"%ld\n\0" as *const u8 as *const i8, *TWO_REFS.second); + } + 0 +}