Skip to content

Add an optional hybrid RW mode for LSMT writable layers. #443

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ZarrianShi wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Add an optional hybrid RW mode for LSMT writable layers. #443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4196142
LSMT: Add hybrid writable layer type
ZarrianShi Aug 14, 2026
5da458c
LSMT: Reuse writable layer data in hybrid mode
ZarrianShi Aug 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
128 changes: 101 additions & 27 deletions src/overlaybd/lsmt/file.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ struct HeaderTrailer {
static const uint32_t FLAG_SHIFT_TYPE = 1; // 1:data file, 0:index file
static const uint32_t FLAG_SHIFT_SEALED = 2; // 1:YES, 0:NO
static const uint32_t FLAG_SPARSE_RW = 4; // 1:sparse file 0:normal file
static const uint32_t FLAG_HYBRID_RW = 5; // 1:hybrid rw 0:append rw

uint32_t get_flag_bit(uint32_t shift) const {
return flags & (1 << shift);
Expand Down Expand Up @@ -112,6 +113,9 @@ struct HeaderTrailer {
bool is_sparse_rw() const {
return get_flag_bit(FLAG_SPARSE_RW);
}
bool is_hybrid_rw() const {
return get_flag_bit(FLAG_HYBRID_RW);
}

void set_header() {
set_flag_bit(FLAG_SHIFT_HEADER);
Expand Down Expand Up @@ -142,6 +146,12 @@ struct HeaderTrailer {
void clr_sparse_rw() {
clr_flag_bit(FLAG_SPARSE_RW);
}
void set_hybrid_rw() {
set_flag_bit(FLAG_HYBRID_RW);
}
void clr_hybrid_rw() {
clr_flag_bit(FLAG_HYBRID_RW);
}

int set_tag(char *buf, size_t n) {
if (n > TAG_SIZE) {
Expand Down Expand Up @@ -203,10 +213,14 @@ static int write_header_trailer(IFile *file, bool is_header, bool is_sealed, boo
pht->set_data_file();
else
pht->set_index_file();
if (args.sparse_rw)
if (args.rw_type == RWType::Sparse)
pht->set_sparse_rw();
else
pht->clr_sparse_rw();
if (args.rw_type == RWType::Hybrid)
pht->set_hybrid_rw();
else
pht->clr_hybrid_rw();

pht->index_offset = index_offset;
pht->index_size = index_size;
Expand Down Expand Up @@ -345,7 +359,8 @@ static int load_layer_info(IFile **src_files, size_t n, LayerInfo &layer, bool o
}
HeaderTrailer *pht = (HeaderTrailer *)buf_top;
layer.virtual_size = pht->virtual_size;
layer.sparse_rw = pht->is_sparse_rw();
layer.rw_type = pht->is_sparse_rw() ? RWType::Sparse

@BigVan BigVan Aug 16, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

use a method like pht->filetype() ?

: (pht->is_hybrid_rw() ? RWType::Hybrid : RWType::Append);
Comment on lines 360 to +363
if (n != 1) {
ALIGNED_MEM(buf_bottom, HeaderTrailer::SPACE, ALIGNMENT4K);
//
Expand Down Expand Up @@ -384,7 +399,7 @@ static int compact(const CompactOptions &opt, atomic_uint64_t &compacted_idx_siz
LayerInfo layer;
if (load_layer_info(src_files, opt.n, layer) != 0)
return -1;
layer.sparse_rw = false;
layer.rw_type = RWType::Append;
layer.user_tag = commit_args->user_tag;
layer.uuid.clear();
if (UUID::String::is_valid((commit_args->uuid).c_str())) {
Expand Down Expand Up @@ -724,6 +739,7 @@ class LSMTFile : public LSMTReadOnlyFile {
uint64_t m_data_offset = HeaderTrailer::SPACE / ALIGNMENT;

uint8_t m_rw_tag = 0;
bool m_hybrid_rw = false;

Mutex m_rw_mtx;
IFile *m_findex = nullptr;
Expand All @@ -750,6 +766,11 @@ class LSMTFile : public LSMTReadOnlyFile {
return nullptr;
}

IComboIndex *rw_index() const {
assert(m_index != nullptr);
return static_cast<IComboIndex *>(m_index);
}

virtual int vioctl(int request, va_list args) override {
if (request == GetType) {
return LSMTReadOnlyFile::vioctl(request, args);
Expand Down Expand Up @@ -847,26 +868,66 @@ class LSMTFile : public LSMTReadOnlyFile {
offset += MAX_IO_SIZE;
}
// wait unlock
off_t moffset = -1;
{
Lock lock(m_rw_mtx);
moffset = append(m_files[m_rw_tag], buf, count);
if (moffset == 0)
return -1;
m_vsize = max(m_vsize, count + offset);
if (m_vsize < count + offset) {
LOG_INFO("resize m_visze: `->`", m_vsize, count + offset);
}
SegmentMapping m{
(uint64_t)offset / (uint64_t)ALIGNMENT,
(uint32_t)count / (uint32_t)ALIGNMENT,
(uint64_t)moffset / (uint64_t)ALIGNMENT,
auto offset_in_blocks = (uint64_t)offset / ALIGNMENT;
auto count_in_blocks = (uint32_t)count / ALIGNMENT;
auto end_in_blocks = offset_in_blocks + count_in_blocks;
unique_ptr<IWritableLayerCursor> writable_layer_cursor;
if (m_hybrid_rw) {
writable_layer_cursor = rw_index()->writable_layer_cursor(
{offset_in_blocks, count_in_blocks});
}

auto update_mapping = [&](SegmentMapping m, bool append_data) -> int {
auto data_offset = (m.offset - offset_in_blocks) * ALIGNMENT;
auto data_length = (size_t)m.length * ALIGNMENT;
if (!append_data) {
if (m_files[m_rw_tag]->pwrite((const char *)buf + data_offset, data_length,
m.moffset * ALIGNMENT) != (ssize_t)data_length) {
LOG_ERRNO_RETURN(0, -1, "rewrite data failed");
}
return 0;
}

auto moffset = append(m_files[m_rw_tag], (const char *)buf + data_offset,
data_length);
if (moffset == 0)
return -1;
m.moffset = (uint64_t)moffset / ALIGNMENT;
m_data_offset = max(m_data_offset, m.mend());
m.tag = m_rw_tag;
rw_index()->insert(m);
append_index(m);
return 0;
};
m.tag = m_rw_tag;
assert(m.length > (uint32_t)0);
m_data_offset = m.mend();
static_cast<IMemoryIndex0 *>(m_index)->insert(m);
append_index(m);

uint64_t cursor = offset_in_blocks;
SegmentMapping m;
while (writable_layer_cursor && writable_layer_cursor->next(m)) {
if (cursor < m.offset) {
SegmentMapping appended(cursor, (uint32_t)(m.offset - cursor), 0);
if (update_mapping(appended, true) != 0)
return -1;
}
if (m.zeroed) {
SegmentMapping appended(m.offset, m.length, 0);
if (update_mapping(appended, true) != 0)
return -1;
} else if (update_mapping(m, false) != 0) {
return -1;
}
cursor = m.end();
}
if (cursor < end_in_blocks) {
SegmentMapping appended(cursor, (uint32_t)(end_in_blocks - cursor), 0);
if (update_mapping(appended, true) != 0)
return -1;
}
}

return bytes;
Expand Down Expand Up @@ -1065,6 +1126,7 @@ class LSMTFile : public LSMTReadOnlyFile {
}
LOG_DEBUG("m_files.size(): `, rw_tag: `", m_files.size(), m_rw_tag);
m_findex = u->m_findex;
m_hybrid_rw = u->m_hybrid_rw;
m_vsize = u->m_vsize;
((IComboIndex *)m_index)->commit_index0();

Expand Down Expand Up @@ -1465,6 +1527,9 @@ IFileRO *open_file_ro(IFile *file, bool ownership) {
IFileRW *open_file_rw(IFile *fdata, IFile *findex, bool ownership) {
ALIGNED_MEM(buf, HeaderTrailer::SPACE, ALIGNMENT4K);
auto pht = verify_ht(fdata, buf);
if (pht && pht->is_sparse_rw() && pht->is_hybrid_rw()) {
LOG_ERROR_RETURN(EINVAL, nullptr, "invalid writable layer type flags");
}
if ((pht == nullptr) || ((pht->is_sparse_rw() == false) && (!findex))) {
LOG_ERRNO_RETURN(0, nullptr, "invalid file ptr, fdata: ` findex: `", fdata, findex);
}
Expand Down Expand Up @@ -1501,39 +1566,46 @@ IFileRW *open_file_rw(IFile *fdata, IFile *findex, bool ownership) {
}
LSMTFile *rst = nullptr;
if (pht->is_sparse_rw() == false) {
LOG_INFO("create LSMTFile object (append-only)");
LOG_INFO("create LSMTFile object (hybrid_rw: `)", pht->is_hybrid_rw());
rst = new LSMTFile;
} else {
LOG_INFO("create LSMTSparseFile object");
rst = new LSMTSparseFile;
}
rst->m_index = pi;
rst->m_hybrid_rw = pht->is_hybrid_rw();
rst->m_findex = findex;
rst->m_files.push_back(fdata);
rst->m_vsize = pht->virtual_size;
rst->m_file_ownership = ownership;
UUID raw;
raw.parse(pht->uuid);
rst->m_uuid.push_back(raw);
LOG_INFO("Layer Info: { UUID:` , Parent_UUID: `, SparseRW: `, Virtual size: `, Version: `.` }",
pht->uuid, pht->parent_uuid, pht->is_sparse_rw(), rst->m_vsize, pht->version,
pht->sub_version);
LOG_INFO("Layer Info: { UUID:` , Parent_UUID: `, SparseRW: `, HybridRW: `, Virtual size: `, Version: `.` }",
pht->uuid, pht->parent_uuid, pht->is_sparse_rw(), pht->is_hybrid_rw(), rst->m_vsize,
pht->version, pht->sub_version);
return rst;
}

IFileRW *create_file_rw(const LayerInfo &args, bool ownership) {
auto fdata = args.fdata;
auto findex = args.findex;
if ((args.sparse_rw == false) && (!fdata || !findex)) {
if (args.rw_type != RWType::Append && args.rw_type != RWType::Hybrid &&
args.rw_type != RWType::Sparse) {
LOG_ERROR_RETURN(EINVAL, nullptr, "invalid writable layer type: `",
static_cast<uint8_t>(args.rw_type));
}
if ((args.rw_type != RWType::Sparse) && (!fdata || !findex)) {
LOG_ERROR_RETURN(0, nullptr, "invalid file ptr, fdata: `, findex: `", fdata, findex);
}
LSMTFile *rst = nullptr;
if (args.sparse_rw == false) {
if (args.rw_type != RWType::Sparse) {
rst = new LSMTFile;
} else {
rst = new LSMTSparseFile;
}
rst->m_index = create_memory_index0((const SegmentMapping *)nullptr, 0, 0, 0);
rst->m_hybrid_rw = args.rw_type == RWType::Hybrid;
rst->m_findex = findex;
rst->m_files.push_back(fdata);
LOG_DEBUG("unparse uuid");
Expand All @@ -1544,14 +1616,15 @@ IFileRW *create_file_rw(const LayerInfo &args, bool ownership) {
rst->m_vsize = args.virtual_size;
rst->m_file_ownership = ownership;
write_header_trailer(fdata, true, false, true, 0, 0, args);
if (!args.sparse_rw) {
if (args.rw_type != RWType::Sparse) {
write_header_trailer(findex, true, false, false, HeaderTrailer::SPACE, 0, args);
}
HeaderTrailer tmp;
// args.parent_uuid.to_string(parent_uuid, UUID::String::LEN);
LOG_INFO("Layer Info: { UUID:`, Parent_UUID: `, Sparse: ` Virtual size: `, Version: `.` }", raw,
args.parent_uuid, args.sparse_rw, rst->m_vsize, tmp.version, tmp.sub_version);
if (args.sparse_rw) {
LOG_INFO("Layer Info: { UUID:`, Parent_UUID: `, RWType: ` Virtual size: `, Version: `.` }", raw,
args.parent_uuid, static_cast<uint8_t>(args.rw_type), rst->m_vsize, tmp.version,
tmp.sub_version);
if (args.rw_type == RWType::Sparse) {
fdata->ftruncate(args.virtual_size + HeaderTrailer::SPACE);
}
return rst;
Expand All @@ -1561,7 +1634,7 @@ IFileRW *create_warpfile(WarpFileArgs &args, bool ownership) {
auto rst = new LSMTWarpFile();
rst->m_findex = args.findex;
LayerInfo info;
info.sparse_rw = false;
info.rw_type = RWType::Append;
info.virtual_size = args.virtual_size;
info.parent_uuid.parse(args.parent_uuid);
info.uuid.parse(args.uuid);
Expand Down Expand Up @@ -1882,6 +1955,7 @@ IFileRW *stack_files(IFileRW *upper_layer, IFileRO *lower_layers, bool ownership
} else {
rst = new LSMTSparseFile;
}
rst->m_hybrid_rw = pht->is_hybrid_rw();
// TODO: also for LSMTWarpFile
if (u->m_vsize == 0) {
if (u->update_vsize(l->m_vsize) < 0) {
Expand Down
8 changes: 7 additions & 1 deletion src/overlaybd/lsmt/file.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,12 @@ class IFileRW : public IFileRO {
virtual int restack(IFileRW *upper) = 0;
};

enum class RWType : uint8_t {
Append,
Hybrid,
Sparse,
};

// create a new writable LSMT file constitued by a data file and an index file,
// optionally obtaining the ownerships of the underlying files,
// thus they will be destructed automatically.
Expand All @@ -123,7 +129,7 @@ struct LayerInfo {
UUID parent_uuid;
UUID uuid;
char *user_tag = nullptr; // a user provided string of message, 256B at most
bool sparse_rw = false;
RWType rw_type = RWType::Append;
size_t len = 0; // len of user_tag; if it's 0, it will be detected with strlen()
LayerInfo(photon::fs::IFile *_fdata = nullptr, photon::fs::IFile *_findex = nullptr)
: fdata(_fdata), findex(_findex) {
Expand Down
4 changes: 2 additions & 2 deletions src/overlaybd/lsmt/format_spec.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ The format of header is described as below. All fields are little-endian.
| sealed | 2 | this file is sealed (1) or not (0) |
| gc_layer | 3 | this is a gc layer (1) or normal layer (0) |
| sparse_rw | 4 | this is a sparse rw layer |
| info_valid | 5 | information validity of the fields *after* flags (they were initially invalid (0) after creation; and readers must resort to trailer when they meet such headers) |
| hybrid_rw | 5 | this is a hybrid rw layer |
| reserved | 6~31 | reserved for future use; must be 0s |


Expand All @@ -67,4 +67,4 @@ a 128-bit struct defined as below:
## trailer
An updated edition of header, in the same format. Trailer is useful in
append-only storage during creation of the blob. Use trailer whenever
possible.
possible.
34 changes: 34 additions & 0 deletions src/overlaybd/lsmt/index.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,36 @@ static inline size_t copy_n(IT begin, IT end, uint64_t end_offset, SegmentMappin
return m;
}

class WritableLayerCursor : public IWritableLayerCursor {
public:
set<SegmentMapping>::const_iterator m_current;
set<SegmentMapping>::const_iterator m_end;
Segment m_range;

WritableLayerCursor(set<SegmentMapping>::const_iterator current,
set<SegmentMapping>::const_iterator end, Segment range)
: m_current(current), m_end(end), m_range(range) {
}

virtual bool next(SegmentMapping &m) override {
if (m_current == m_end || m_current->offset >= m_range.end())
return false;
m = *m_current++;
if (m.offset < m_range.offset)
m.forward_offset_to(m_range.offset);
if (m.end() > m_range.end())
m.backward_end_to(m_range.end());
return true;
}
};

static unique_ptr<IWritableLayerCursor> create_writable_layer_cursor(
const set<SegmentMapping> &mapping, Segment range) {
SegmentMapping key(range.offset, range.length, 0);
return unique_ptr<IWritableLayerCursor>(
new WritableLayerCursor(mapping.lower_bound(key), mapping.end(), range));
}

static bool verify_mapping_order(const SegmentMapping *pmappings, size_t n);

bool is_avx512f_supported() {
Expand Down Expand Up @@ -565,6 +595,10 @@ class Index0 : public IComboIndex {
return m;
}

virtual unique_ptr<IWritableLayerCursor> writable_layer_cursor(Segment range) const override {
return create_writable_layer_cursor(mapping, range);
}

// dump the the whole index as an array
virtual SegmentMapping *dump(size_t alignment = 0) const override {
auto size = mapping.size();
Expand Down
13 changes: 13 additions & 0 deletions src/overlaybd/lsmt/index.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ IMemoryIndex -> IMemoryIndex0 -> IComboIndex -> Index0 ( set<SegmentMap> ) -> Co
#include <inttypes.h>
#include <cstddef>
#include <assert.h>
#include <memory>
#include <sys/types.h>

namespace LSMT {
Expand Down Expand Up @@ -144,6 +145,15 @@ class IMemoryIndex0 : public IMemoryIndex {
// virtual IMemoryIndex *make_read_only_index() const = 0;
};

class IWritableLayerCursor {
public:
virtual ~IWritableLayerCursor() {
}

// Copies the next writable-layer mapping that overlaps the requested range.
virtual bool next(SegmentMapping &mapping) = 0;
};

class IComboIndex : public IMemoryIndex0 {
public:
// backing index must NOT be IMemoryIndex0!
Expand All @@ -152,6 +162,9 @@ class IComboIndex : public IMemoryIndex0 {
virtual int front_index(const IMemoryIndex0 *fi) = 0;
virtual const IMemoryIndex0 *front_index() const = 0;

// Iterates mappings that belong to the current writable layer only.
virtual std::unique_ptr<IWritableLayerCursor> writable_layer_cursor(Segment range) const = 0;

// dump index0 which needs to compact
// and then clear the original index0.
// virtual IMemoryIndex0* gc_index() = 0;
Expand Down
Loading
Loading