Skip to content

xvalue 改为 64 字节定长 head (#329) #351

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
carsontung666 wants to merge 1 commit into from
Closed

xvalue 改为 64 字节定长 head (#329) #351

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 64 additions & 12 deletions runtime-rs/src/engine.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,11 +115,38 @@ impl Engine {
return String::new();
}
let mut head = KvspaceHead::default();
kvspaceDecodeHead(out, olen, &mut head);
let kx = String::from_utf8_lossy(&head.langtype)
.trim_end_matches('\0')
.to_string();
let (bo, bl) = (head.body_offset as usize, head.body_len.max(0) as usize);
let (bo, bl, kx);
if olen >= 64 && *out.add(1) <= 4 && *out.add(3) <= 8 {
let bodylen = u64::from_le_bytes(std::slice::from_raw_parts(out.add(8), 8).try_into().unwrap());
let cap = u64::from_le_bytes(std::slice::from_raw_parts(out.add(16), 8).try_into().unwrap());
if bodylen <= cap && 64 + bodylen <= olen as u64 {
let kid = u16::from_le_bytes([*out.add(24), *out.add(25)]);
kx = match kid {
5 => "int64",
12 => "char/utf32",
13 => "char/utf8",
14 => "char/ascii",
_ => "",
}
.to_string();
bo = 64;
bl = bodylen as usize;
} else {
kvspaceDecodeHead(out, olen, &mut head);
kx = String::from_utf8_lossy(&head.langtype)
.trim_end_matches('\0')
.to_string();
bo = head.body_offset as usize;
bl = head.body_len.max(0) as usize;
}
} else {
kvspaceDecodeHead(out, olen, &mut head);
kx = String::from_utf8_lossy(&head.langtype)
.trim_end_matches('\0')
.to_string();
bo = head.body_offset as usize;
bl = head.body_len.max(0) as usize;
}
if kx.ends_with("char/utf32") {
std::slice::from_raw_parts(out.add(bo), bl)
.chunks_exact(4)
Expand Down Expand Up @@ -225,15 +252,40 @@ impl Engine {
}
unsafe {
let mut h = KvspaceHead::default();
if kvspaceDecodeHead(tlv.as_ptr(), tlv.len() as u32, &mut h) != 0 {
let (r, bo, bl);
if tlv.len() >= 64 && tlv[1] <= 4 && tlv[3] <= 8 {
let bodylen = u64::from_le_bytes(tlv[8..16].try_into().unwrap());
let cap = u64::from_le_bytes(tlv[16..24].try_into().unwrap());
if bodylen <= cap && 64 + bodylen <= tlv.len() as u64 {
r = match tlv[0] {
2 => 2,
1 => 1,
_ => 0,
};
bo = 64;
bl = bodylen as usize;
} else if kvspaceDecodeHead(tlv.as_ptr(), tlv.len() as u32, &mut h) != 0 {
return Vec::new();
} else {
r = match h.r#ref {
2 => 2,
1 => 1,
_ => 0,
};
bo = h.body_offset as usize;
bl = h.body_len.max(0) as usize;
}
} else if kvspaceDecodeHead(tlv.as_ptr(), tlv.len() as u32, &mut h) != 0 {
return Vec::new();
} else {
r = match h.r#ref {
2 => 2,
1 => 1,
_ => 0,
};
bo = h.body_offset as usize;
bl = h.body_len.max(0) as usize;
}
let r = match h.r#ref {
2 => 2,
1 => 1,
_ => 0,
};
let (bo, bl) = (h.body_offset as usize, h.body_len.max(0) as usize);
if bo + bl > tlv.len() {
return Vec::new();
}
Expand Down
4 changes: 4 additions & 0 deletions runtime/src/runtime_internal.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,7 @@ extern int kvspaceNewFloat64(double v, uint8_t **out, uint32_t *out_len);
/* ── 数值上限 ──────────────────────────────────────────────────────── */

#define MAX_PARAMS 128
#define KVLANG_XVALUE_HEADLEN 64
#define MAX_STACK_DEPTH 256
#define X_MAX_NDIM 8

Expand Down Expand Up @@ -354,6 +355,9 @@ void kvlangXvalueNewTlv(kvlangXvalue_t *v, const char *kind, const uint8_t *raw,
void kvlangXvalueNewTlvDims(kvlangXvalue_t *v, const char *kind,
const uint8_t *raw, uint32_t raw_len,
const int32_t *dims, int32_t ndim);
int kvlangXvalueEncodeBox(const char *kind, const uint8_t *raw, uint32_t raw_len,
const int32_t *dims, int32_t ndim, uint8_t **out,
uint32_t *out_len);

void kvlangFormatFloat(char *out, size_t cap, double v);

Expand Down
152 changes: 152 additions & 0 deletions runtime/src/xvalue.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,150 @@ static uint64_t rd64(const uint8_t *p) {
return (uint64_t)rd32(p) | ((uint64_t)rd32(p + 4) << 32);
}

static void wr16(uint8_t *d, uint16_t v) { d[0] = (uint8_t)v; d[1] = (uint8_t)(v >> 8); }
static void wr32(uint8_t *d, uint32_t v) {
d[0] = (uint8_t)v; d[1] = (uint8_t)(v >> 8); d[2] = (uint8_t)(v >> 16); d[3] = (uint8_t)(v >> 24);
}
static void wr64(uint8_t *d, uint64_t v) { wr32(d, (uint32_t)v); wr32(d + 4, (uint32_t)(v >> 32)); }

#define H64_STORETYPE 1
#define H64_NDIM 3
#define H64_VID 4
#define H64_BODYLEN 8
#define H64_BODYCAP 16
#define H64_KIND 24
#define H64_DIMS 28
#define H64_ST_NONE 0
#define H64_ST_ATOM 1
#define H64_ST_ARRAY 2
#define H64_ST_INDEX 3
#define H64_ST_EXT 4

static const char *const h64_kind_names[] = {
"", KVSPACE_KIND_BOOL,
KVSPACE_KIND_INT8, KVSPACE_KIND_INT16, KVSPACE_KIND_INT32, KVSPACE_KIND_INT64,
KVSPACE_KIND_UINT8, KVSPACE_KIND_UINT16, KVSPACE_KIND_UINT32, KVSPACE_KIND_UINT64,
KVSPACE_KIND_FLOAT32, KVSPACE_KIND_FLOAT64, KVSPACE_KIND_FLOAT16,
KVSPACE_KIND_BFLOAT16, KVSPACE_KIND_FLOAT8_E4M3, KVSPACE_KIND_FLOAT8_E5M2,
KVSPACE_KIND_CHAR, KVSPACE_KIND_CHAR_UTF8, KVSPACE_KIND_CHAR_ASCII,
KVSPACE_KIND_MAP, KVSPACE_KIND_INDEX, KVSPACE_KIND_EXT_INDEX,
KVSPACE_KIND_RWIR, KVSPACE_KIND_RWFUNC, KVSPACE_KIND_DEF_RWIR,
KVSPACE_KIND_DEF_LANGTYPE, KVSPACE_KIND_SCOPE, KVSPACE_KIND_STRUCT,
KVSPACE_KIND_TIME, KVSPACE_KIND_DURATION,
};

static const char *h64_kind_name(uint16_t id) {
if (id < sizeof(h64_kind_names) / sizeof(h64_kind_names[0]))
return h64_kind_names[id];
return "*";
}

static uint16_t h64_kind_id(const char *name) {
if (!name || !name[0] || strcmp(name, KVSPACE_KIND_NONE) == 0)
return 0;
for (uint16_t i = 1; i < sizeof(h64_kind_names) / sizeof(h64_kind_names[0]); i++)
if (strcmp(name, h64_kind_names[i]) == 0)
return i;
return 255;
}

static uint8_t h64_storetype(const char *kind, int32_t ndim) {
if (!kind || !kind[0] || strcmp(kind, KVSPACE_KIND_NONE) == 0)
return H64_ST_NONE;
if (strcmp(kind, KVSPACE_KIND_EXT_INDEX) == 0)
return H64_ST_EXT;
if (strcmp(kind, KVSPACE_KIND_INDEX) == 0 || strcmp(kind, KVSPACE_KIND_RWFUNC) == 0 ||
strcmp(kind, KVSPACE_KIND_DEF_RWIR) == 0 || strcmp(kind, KVSPACE_KIND_DEF_LANGTYPE) == 0 ||
kind[0] == '/')
return H64_ST_INDEX;
if (ndim > 0)
return H64_ST_ARRAY;
return H64_ST_ATOM;
}

static int looks_head64(const uint8_t *d, uint32_t len) {
if (!d || len < KVLANG_XVALUE_HEADLEN)
return 0;
if (d[H64_STORETYPE] > H64_ST_EXT || d[H64_NDIM] > X_MAX_NDIM)
return 0;
uint64_t bl = rd64(d + H64_BODYLEN), cap = rd64(d + H64_BODYCAP);
if (bl > cap || bl > 16ull * 1024ull * 1024ull)
return 0;
if ((uint64_t)KVLANG_XVALUE_HEADLEN + bl > len)
return 0;
return 1;
}

static int decode_head64(const uint8_t *d, uint32_t len, kvspaceHead_t *h) {
memset(h, 0, sizeof(*h));
if (!looks_head64(d, len))
return -1;
int8_t ref = (int8_t)d[0];
uint8_t st = d[H64_STORETYPE];
uint8_t ndim = d[H64_NDIM];
uint16_t kid = rd16(d + H64_KIND);
const char *kn = h64_kind_name(kid);
h->headlen = KVLANG_XVALUE_HEADLEN;
h->ref = (uint8_t)(ref < 0 ? 2 : ref);
h->storetype = st;
h->ro = d[2] & 1;
h->vid = rd32(d + H64_VID);
h->body_len = (int32_t)rd64(d + H64_BODYLEN);
h->body_offset = KVLANG_XVALUE_HEADLEN;
h->ndim = ndim;
for (int i = 0; i < ndim && i < X_MAX_NDIM; i++)
h->dims[i] = (int32_t)rd32(d + H64_DIMS + i * 4);
int o = 0;
if (st == H64_ST_ARRAY && ndim > 0) {
h->langtype[o++] = '[';
for (int i = 0; i < ndim && o < 200; i++) {
if (i)
h->langtype[o++] = ',';
o += snprintf((char *)h->langtype + o, sizeof h->langtype - (size_t)o, "%u",
(unsigned)h->dims[i]);
}
h->langtype[o++] = ']';
}
size_t kl = strlen(kn);
if (o + (int)kl >= (int)sizeof h->langtype)
kl = sizeof h->langtype - (size_t)o - 1;
memcpy(h->langtype + o, kn, kl);
h->langtype[o + (int)kl] = 0;
h->langtype_len = o + (int32_t)kl;
return 0;
}

int kvlangXvalueEncodeBox(const char *kind, const uint8_t *raw, uint32_t raw_len,
const int32_t *dims, int32_t ndim, uint8_t **out, uint32_t *out_len) {
if (!out || !out_len)
return -1;
if (ndim < 0)
ndim = 0;
if (ndim > X_MAX_NDIM)
return -1;
if (!kind)
kind = "";
uint32_t total = (uint32_t)KVLANG_XVALUE_HEADLEN + raw_len;
uint8_t *buf = malloc(total ? total : 1);
if (!buf)
return -1;
memset(buf, 0, KVLANG_XVALUE_HEADLEN);
buf[0] = 0;
buf[H64_STORETYPE] = h64_storetype(kind, ndim);
buf[H64_NDIM] = (uint8_t)ndim;
wr64(buf + H64_BODYLEN, raw_len);
wr64(buf + H64_BODYCAP, raw_len);
wr16(buf + H64_KIND, h64_kind_id(kind));
for (int i = 0; i < ndim; i++)
wr32(buf + H64_DIMS + i * 4, (uint32_t)(dims ? dims[i] : 0));
if (raw_len && raw)
memcpy(buf + KVLANG_XVALUE_HEADLEN, raw, raw_len);
*out = buf;
*out_len = total;
return 0;
}


void kvlangXvalueFree(kvlangXvalue_t *v) {
if (!v->borrowed)
free(v->data);
Expand Down Expand Up @@ -82,6 +226,8 @@ static int kvlangXvalueDecodeHeadRaw(const uint8_t *d, uint32_t len,
memset(h, 0, sizeof(*h));
if (!d || len == 0)
return -1;
if (looks_head64(d, len))
return decode_head64(d, len, h);
kvspaceDecodeHead(d, len, h);
return h->langtype[0] ? 0 : -1;
}
Expand Down Expand Up @@ -119,6 +265,12 @@ static uint8_t *kvlangXvalueEncodeTlv(const char *kind, const uint8_t *raw,
int32_t ndim = al_to_dims(kind, array_len, dims);
uint8_t *tmp = NULL;
uint32_t tl = 0;
uint8_t *box = NULL;
uint32_t blen = 0;
if (kvlangXvalueEncodeBox(kind, raw, raw_len, dims, ndim, &box, &blen) == 0) {
*out_len = blen;
return box;
}
if (kvspaceTlvEncode(kind, raw, raw_len, dims, ndim, &tmp, &tl) != 0) {
*out_len = 0;
return NULL;
Expand Down
Loading