diff --git a/compiler/rustc_borrowck/src/nll.rs b/compiler/rustc_borrowck/src/nll.rs
index f4a1edb0b675d..5a1358b9a311e 100644
--- a/compiler/rustc_borrowck/src/nll.rs
+++ b/compiler/rustc_borrowck/src/nll.rs
@@ -144,6 +144,8 @@ pub(crate) fn compute_regions<'tcx>(
         &lowered_constraints,
     );
 
+    let num_points = location_map.num_points();
+
     // If requested for `-Zpolonius=next`, compute loan liveness information.
     // This is done prior to `RegionInferenceContext::new`, because we may add
     // additional liveness constraints.
@@ -155,6 +157,7 @@ pub(crate) fn compute_regions<'tcx>(
             &universal_region_relations.universal_regions,
             body,
             borrow_set,
+            num_points,
         );
     }
 
diff --git a/compiler/rustc_borrowck/src/polonius/mod.rs b/compiler/rustc_borrowck/src/polonius/mod.rs
index cbac05d2eff67..b19f2cd719f29 100644
--- a/compiler/rustc_borrowck/src/polonius/mod.rs
+++ b/compiler/rustc_borrowck/src/polonius/mod.rs
@@ -41,7 +41,7 @@ mod liveness_constraints;
 use std::collections::BTreeMap;
 
 use rustc_data_structures::fx::FxHashSet;
-use rustc_index::bit_set::SparseBitMatrix;
+use rustc_index::bit_set::DenseBitSet;
 use rustc_middle::mir::{Body, Local};
 use rustc_middle::ty::RegionVid;
 use rustc_mir_dataflow::points::PointIndex;
@@ -55,7 +55,28 @@ use crate::dataflow::BorrowIndex;
 use crate::region_infer::values::LivenessValues;
 use crate::universal_regions::UniversalRegions;
 
-pub(crate) type LiveLoans = SparseBitMatrix<PointIndex, BorrowIndex>;
+#[derive(Clone)]
+pub(crate) struct LiveLoans {
+    num_points: usize,
+    // This matrix always has more rows (PointIndex) than columns (BorrowIndex),
+    // and the borrow dimension is usually very low (single digit in 90% of cases in our benchmark suite),
+    // so we store it packed in a single bitset. Rows are points, columns are borrows.
+    flat_matrix: DenseBitSet<usize>,
+}
+
+impl LiveLoans {
+    pub(crate) fn new(num_points: usize, num_borrows: usize) -> Self {
+        Self { num_points, flat_matrix: DenseBitSet::new_empty(num_points * num_borrows) }
+    }
+    pub(crate) fn insert(&mut self, row: PointIndex, col: BorrowIndex) {
+        let bit_index = row.index() + self.num_points * col.index();
+        self.flat_matrix.insert(bit_index);
+    }
+    pub(crate) fn contains(&self, row: PointIndex, col: BorrowIndex) -> bool {
+        let bit_index = row.index() + self.num_points * col.index();
+        self.flat_matrix.contains(bit_index)
+    }
+}
 
 /// This struct holds the necessary
 ///  - liveness data, created during MIR typeck, and which will be used to lazily compute the
@@ -109,6 +130,7 @@ impl PoloniusContext {
         universal_regions: &UniversalRegions<'tcx>,
         body: &Body<'tcx>,
         borrow_set: &BorrowSet<'tcx>,
+        num_points: usize,
     ) {
         // We don't need to prepare the graph (index NLL constraints, etc.) if we have no loans to
         // trace throughout localized constraints.
@@ -118,7 +140,7 @@ impl PoloniusContext {
             // step in the chain (the NLL loan scope and active loans computations).
             let graph = LocalizedConstraintGraph::new(liveness, outlives_constraints);
 
-            let mut live_loans = LiveLoans::new(borrow_set.len());
+            let mut live_loans = LiveLoans::new(num_points, borrow_set.len());
             let mut visitor = LoanLivenessVisitor { liveness, live_loans: &mut live_loans };
             graph.traverse(
                 body,