diff --git a/src/minimalloc.cc b/src/minimalloc.cc index fef45fd..89f1b50 100644 --- a/src/minimalloc.cc +++ b/src/minimalloc.cc @@ -74,6 +74,9 @@ Area Buffer::area() const { std::optional Buffer::effective_size(const Buffer& x) const { if (lifespan.upper() <= x.lifespan.lower()) return std::nullopt; if (x.lifespan.upper() <= lifespan.lower()) return std::nullopt; + // Fast path: with no gaps on either side, both buffers are fully active + // throughout the overlap, so the answer is always our own size. + if (gaps.empty() && x.gaps.empty()) return size; const Window window = {0, size}; const Window x_window = {0, x.size}; std::vector points = {{0, lifespan.lower(), kLeft, window}, diff --git a/src/solver.cc b/src/solver.cc index 5976f6f..e6d113e 100644 --- a/src/solver.cc +++ b/src/solver.cc @@ -20,6 +20,7 @@ limitations under the License. #include #include +#include #include #include #include @@ -80,7 +81,8 @@ class SolverImpl { const Problem& problem, const SweepResult& sweep_result, int64_t* backtracks, std::atomic& cancelled) : params_(params), start_time_(start_time), problem_(problem), sweep_result_(sweep_result), - backtracks_(*backtracks), cancelled_(cancelled) {} + backtracks_(*backtracks), cancelled_(cancelled), + has_timeout_(params.timeout != absl::InfiniteDuration()) {} absl::StatusOr Solve() { if (problem_.buffers.empty()) return solution_; @@ -89,6 +91,7 @@ class SolverImpl { solution_.offsets.resize(num_buffers, kNoOffset); min_offsets_.resize(num_buffers); section_data_.resize(sweep_result_.sections.size()); + section_stamp_.assign(sweep_result_.sections.size(), 0); for (BufferIdx buffer_idx = 0; buffer_idx < num_buffers; ++buffer_idx) { const BufferData& buffer_data = sweep_result_.buffer_data[buffer_idx]; for (const SectionSpan& section_span : buffer_data.section_spans) { @@ -216,10 +219,8 @@ class SolverImpl { } // Updates section data given that 'buffer_idx' is the next item to be placed. - std::vector UpdateSectionData( - const absl::flat_hash_set& affected_sections, - BufferIdx buffer_idx) { - std::vector section_changes; + void UpdateSectionData(BufferIdx buffer_idx) { + std::vector& section_changes = section_trail_; const Offset offset = assignment_.offsets[buffer_idx]; // For any section this buffer resides in, bump up the floor & drop the sum. const BufferData& buffer_data = sweep_result_.buffer_data[buffer_idx]; @@ -236,7 +237,7 @@ class SolverImpl { } } // The floor of any section cannot be lower than its lowest minimum offset. - for (const SectionIdx s_idx : affected_sections) { + for (const SectionIdx s_idx : affected_sections_) { Offset min_offset = std::numeric_limits::max(); for (const BufferIdx other_idx : sweep_result_.sections[s_idx]) { if (assignment_.offsets[other_idx] == kNoOffset) { @@ -252,15 +253,14 @@ class SolverImpl { section_data_[s_idx].floor = min_offset; } } - return section_changes; } // Restores the section data by reversing any recorded changes. - void RestoreSectionData( - const std::vector& section_changes, - BufferIdx buffer_idx) { - for (auto c = section_changes.rbegin(); c != section_changes.rend(); ++c) { - section_data_[c->section_idx].floor = c->floor; + void RestoreSectionData(size_t mark, BufferIdx buffer_idx) { + while (section_trail_.size() > mark) { + const SectionChange& c = section_trail_.back(); + section_data_[c.section_idx].floor = c.floor; + section_trail_.pop_back(); } // For any section this buffer resides in, increase the sum. const BufferData& buffer_data = sweep_result_.buffer_data[buffer_idx]; @@ -275,12 +275,10 @@ class SolverImpl { } // Updates min offset data, given that 'buffer_idx' is the next to be placed. - std::optional> UpdateMinOffsets( - BufferIdx buffer_idx, - absl::flat_hash_set& affected_sections, - bool& fixed_offset_failure) { + // Returns 'true' if this buffer is "hatless" (nothing unallocated overhead). + bool UpdateMinOffsets(BufferIdx buffer_idx, bool& fixed_offset_failure) { bool hatless = true; - std::vector offset_changes; + std::vector& offset_changes = offset_trail_; const Offset offset = assignment_.offsets[buffer_idx]; // For any overlap this buffer participates in, bump up its minimum offset. const std::vector& buffer_data = sweep_result_.buffer_data; @@ -306,18 +304,22 @@ class SolverImpl { const SectionRange& section_range = section_span.section_range; for (SectionIdx s_idx = section_range.lower(); s_idx < section_range.upper(); ++s_idx) { - affected_sections.insert(s_idx); + // Generation-stamped dedup: O(1), no hashing, no allocation. + if (section_stamp_[s_idx] == stamp_) continue; + section_stamp_[s_idx] = stamp_; + affected_sections_.push_back(s_idx); } } } - if (hatless) return std::nullopt; - return offset_changes; + return hatless; } // Restores the minimum offsets by reversing any recorded changes. - void RestoreMinOffsets(const std::vector& offset_changes) { - for (auto c = offset_changes.rbegin(); c != offset_changes.rend(); ++c) { - min_offsets_[c->buffer_idx] = c->min_offset; + void RestoreMinOffsets(size_t mark) { + while (offset_trail_.size() > mark) { + const OffsetChange& c = offset_trail_.back(); + min_offsets_[c.buffer_idx] = c.min_offset; + offset_trail_.pop_back(); } } @@ -338,10 +340,15 @@ class SolverImpl { // Orders unallocated buffers by their minimum possible offset values, using // buffer areas as a tie-breaker. - std::vector ComputeOrdering( + // Also returns the minimum height of any unallocated buffer (no other buffer + // should be assigned an offset at or above this value). The minimum is + // order-independent, so it can be folded into this pass, saving a second + // traversal plus a double indirection per element. + void ComputeOrdering( const std::vector& preordering, - const std::vector& orig_ordering) { - std::vector ordering; + const std::vector& orig_ordering, + std::vector& ordering) { + ordering.clear(); for (const auto [offset, preorder_idx] : orig_ordering) { const BufferIdx buffer_idx = preordering[preorder_idx].buffer_idx; // If this buffer has already been assigned, keep looking. @@ -351,7 +358,6 @@ class SolverImpl { {.offset = new_offset, .preorder_idx = preorder_idx}); } if (params_.dynamic_ordering) absl::c_sort(ordering, kDynamicComparator); - return ordering; } // Determines the minimum height of any unallocated buffer ... no other buffer @@ -379,19 +385,33 @@ class SolverImpl { const Offset min_offset, const PreorderIdx min_preorder_idx) { DLOG(INFO) << __func__ << " (Start) " << partition; + const int64_t depth = depth_level_++; + struct PopDepth { + int64_t& d; + ~PopDepth() { --d; } + } pop_depth{depth_level_}; if (nodes_remaining_ <= 0) { DLOG(INFO) << __func__ << " (End) " << partition << ", status " << absl::StatusCodeToString(absl::StatusCode::kAborted); return absl::StatusCode::kAborted; } --nodes_remaining_; - if (absl::Now() - start_time_ > params_.timeout || cancelled_) { + // Only touch the clock when a finite timeout was actually requested, and + // then only once every 1024 nodes. absl::Now() costs ~27ns here, which is + // comparable to the useful work at a shallow node. + if (cancelled_.load(std::memory_order_relaxed) || + (has_timeout_ && (nodes_remaining_ & 0x3FF) == 0 && + absl::Now() - start_time_ > params_.timeout)) { DLOG(INFO) << __func__ << " (End) " << partition << ", status " << absl::StatusCodeToString(absl::StatusCode::kDeadlineExceeded); return absl::StatusCode::kDeadlineExceeded; } - const std::vector ordering = - ComputeOrdering(preordering, orig_ordering); + // 'ordering' must outlive the recursive calls below (it is handed down as + // 'orig_ordering'), so keep one reusable vector per recursion depth. A + // deque is used because its references stay valid as it grows. + while (ordering_pool_.size() <= (size_t)depth) ordering_pool_.emplace_back(); + std::vector& ordering = ordering_pool_[depth]; + ComputeOrdering(preordering, orig_ordering, ordering); if (ordering.empty()) { // Store offsets for all the buffers that participate in this partition. for (const BufferIdx buffer_idx : partition.buffer_idxs) { @@ -417,12 +437,13 @@ class SolverImpl { if (offset > *buffer.offset) continue; } assignment_.offsets[buffer_idx] = offset; - absl::flat_hash_set affected_sections; + const size_t offset_mark = offset_trail_.size(); + const size_t section_mark = section_trail_.size(); + affected_sections_.clear(); + ++stamp_; bool fixed_offset_failure = false; - auto offset_changes = UpdateMinOffsets(buffer_idx, affected_sections, - fixed_offset_failure); - std::vector section_changes = - UpdateSectionData(affected_sections, buffer_idx); + const bool hatless = UpdateMinOffsets(buffer_idx, fixed_offset_failure); + UpdateSectionData(buffer_idx); absl::StatusCode status_code = absl::StatusCode::kNotFound; if (!fixed_offset_failure && Check(partition, offset)) { DLOG(INFO) << "DFS (Enter) Depth: " << std::setw(5) << depth_++ @@ -436,8 +457,8 @@ class SolverImpl { DLOG(INFO) << "DFS (Leave) Depth: " << std::setw(5) << --depth_ << ", BufferIdx: " << std::setw(5) << buffer_idx; } - RestoreSectionData(section_changes, buffer_idx); - if (offset_changes) RestoreMinOffsets(*offset_changes); + RestoreSectionData(section_mark, buffer_idx); + RestoreMinOffsets(offset_mark); assignment_.offsets[buffer_idx] = kNoOffset; // Mark it unallocated. // If a feasible solution *or* timeout, abort search. if (status_code != absl::StatusCode::kNotFound) { @@ -445,7 +466,7 @@ class SolverImpl { << absl::StatusCodeToString(status_code); return status_code; } - if (!offset_changes && params_.hatless_pruning) break; + if (hatless && params_.hatless_pruning) break; } ++backtracks_; DLOG(INFO) << __func__ << " (End) " << partition << ", status " @@ -534,12 +555,22 @@ class SolverImpl { const SweepResult& sweep_result_; int64_t& backtracks_; std::atomic& cancelled_; + const bool has_timeout_; Solution assignment_; Solution solution_; std::vector min_offsets_; std::vector section_data_; std::vector cuts_; + // Persistent undo trails: callers save size() on entry and unwind on exit, + // which removes two heap allocations per search node. + std::vector section_trail_; + std::vector offset_trail_; + std::vector affected_sections_; + std::vector section_stamp_; + uint64_t stamp_ = 0; + std::deque> ordering_pool_; + int64_t depth_level_ = 0; int64_t nodes_remaining_ = std::numeric_limits::max(); // debug diff --git a/src/sweeper.cc b/src/sweeper.cc index 941aac4..0de09d1 100644 --- a/src/sweeper.cc +++ b/src/sweeper.cc @@ -93,6 +93,15 @@ std::vector CreatePoints(const Problem& problem) { const Buffer& buffer = problem.buffers[buffer_idx]; const Lifespan& lifespan = buffer.lifespan; const Window window = {0, buffer.size}; + // Fast path: a gapless buffer contributes exactly its two endpoints, so + // skip the deque + two hash sets entirely. + if (buffer.gaps.empty() && lifespan.lower() != lifespan.upper()) { + all_points.push_back( + {buffer_idx, lifespan.lower(), kLeft, window, /*endpoint=*/true}); + all_points.push_back( + {buffer_idx, lifespan.upper(), kRight, window, /*endpoint=*/true}); + continue; + } std::deque points; absl::flat_hash_set leftTimes, rightTimes; // Insert left & right endpoints for all *windowed* gaps. @@ -148,7 +157,8 @@ SweepResult Sweep(const Problem& problem) { SweepResult result; const auto num_buffers = problem.buffers.size(); const std::vector points = CreatePoints(problem); - Section actives, alive; + ActiveSet actives, alive; + Section section_scratch; TimeValue last_section_time = -1; SectionIdx last_section_idx = 0; // Create a reverse index (from buffers to sections) for quick lookup. @@ -162,7 +172,9 @@ SweepResult Sweep(const Problem& problem) { // Create a new cross section of buffers if one doesn't yet exist. if (last_section_time < point.time_value) { last_section_time = point.time_value; - result.sections.push_back(actives); + section_scratch.assign(actives.begin(), actives.end()); + std::sort(section_scratch.begin(), section_scratch.end()); + result.sections.push_back(section_scratch); } // If it's a right endpoint, remove it from the set of active buffers. actives.erase(buffer_idx); @@ -189,13 +201,13 @@ SweepResult Sweep(const Problem& problem) { const Buffer& alive = problem.buffers[alive_idx]; auto alive_effective_size = alive.effective_size(buffer); if (alive_effective_size) { - result.buffer_data[alive_idx].overlaps.insert( + result.buffer_data[alive_idx].overlaps.push_back( {buffer_idx, *alive_effective_size}); } auto effective_size = buffer.effective_size(alive); if (effective_size) { - result.buffer_data[buffer_idx].overlaps.insert({alive_idx, - *effective_size}); + result.buffer_data[buffer_idx].overlaps.push_back( + {alive_idx, *effective_size}); } } } @@ -205,6 +217,11 @@ SweepResult Sweep(const Problem& problem) { buffer_idx_to_section_start[buffer_idx] = result.sections.size(); } } + // Each ordered pair is visited exactly once, so no dedup is needed -- just + // restore the sorted order that the btree_set used to provide. + for (BufferData& buffer_data : result.buffer_data) { + std::sort(buffer_data.overlaps.begin(), buffer_data.overlaps.end()); + } return result; } diff --git a/src/sweeper.h b/src/sweeper.h index 3ca4c82..2cf83e4 100644 --- a/src/sweeper.h +++ b/src/sweeper.h @@ -72,7 +72,11 @@ struct SectionSpan { // sections: | sec0 | sec1 | sec2 | sec3 | // |======|======|======|======|======|======|======|======|======| -using Section = absl::flat_hash_set; +// The solver only ever *iterates* a section, never queries it, so a sorted +// vector beats a hash set on locality. Sweep() still uses a hash set for the +// mutable active/alive sets and converts on the way out. +using Section = std::vector; +using ActiveSet = absl::flat_hash_set; // Partitions store various preprocessed attributes for a subset of a Problem's // buffers. Partitions are mutually exclusive -- that is, any buffer belongs to @@ -133,7 +137,7 @@ struct BufferData { std::vector section_spans; // Contains a set of buffers that overlap at some point in time with this one. - absl::btree_set overlaps; + std::vector overlaps; // Sorted; built once, read hot. bool operator==(const BufferData& x) const; }; diff --git a/tests/sweeper_test.cc b/tests/sweeper_test.cc index 94d7d68..e0fc089 100644 --- a/tests/sweeper_test.cc +++ b/tests/sweeper_test.cc @@ -306,7 +306,7 @@ TEST(SweeperTest, SuperLongBufferPreventsPartitioning) { EXPECT_EQ( Sweep(problem), (SweepResult{ - .sections = {{0, 3}, {1, 3, 2}, {3, 2}}, + .sections = {{0, 3}, {1, 2, 3}, {2, 3}}, .partitions = { {.buffer_idxs = {0, 3, 1, 2}, .section_range = {0, 3}}, }, @@ -325,7 +325,7 @@ TEST(SweeperTest, SuperLongBufferPreventsPartitioning) { TEST(CalculateCutsTest, SuperLongBufferPreventsPartitioning) { const SweepResult sweep_result = { - .sections = {{0, 3}, {1, 3, 2}, {3, 2}}, + .sections = {{0, 3}, {1, 2, 3}, {2, 3}}, .buffer_data = { {.section_spans = {{.section_range = {0, 1}, .window = {0, 2}}}, .overlaps = {{3, 2}}}, @@ -388,7 +388,7 @@ TEST(SweeperTest, BuffersOutOfOrder) { EXPECT_EQ( Sweep(problem), (SweepResult{ - .sections = {{2}, {1, 0}}, + .sections = {{2}, {0, 1}}, .partitions = { {.buffer_idxs = {2}, .section_range = {0, 1}}, {.buffer_idxs = {1, 0}, .section_range = {1, 2}}, @@ -405,7 +405,7 @@ TEST(SweeperTest, BuffersOutOfOrder) { TEST(CalculateCutsTest, BuffersOutOfOrder) { const SweepResult sweep_result = { - .sections = {{2}, {1, 0}}, + .sections = {{2}, {0, 1}}, .buffer_data = { {.section_spans = {{.section_range = {1, 2}, .window = {0, 1}}}, .overlaps = {{1, 1}}},