diff --git a/rayforce/README.md b/rayforce/README.md index a7f1fb096a..3dcdde7729 100644 --- a/rayforce/README.md +++ b/rayforce/README.md @@ -19,19 +19,21 @@ unset, it takes the latest GitHub release. ## Data layout -`./load` streams `hits.csv` into a splayed on-disk table under `./hits` -with `.csv.splayed` — one file per column plus the `.sym` dictionary, -parsed in parallel from an mmap of the CSV without materializing the -table in memory. The column names come from `create.rfl`; passing an -explicit name vector also tells the reader the input has no header row, -which is the shape of the published `hits.csv`. +`./load` streams `hits.parquet` into a splayed on-disk table under +`./hits` with `.parquet.splayed` — one file per column plus the `.sym` +dictionary, written row group by row group without materializing the +table in memory. The column names come from the file; `create.rfl` +gives the native type of each column in the file's order. Types follow `../clickhouse/create.sql`: `BIGINT` → `I64`, `INTEGER` → `I32`, `SMALLINT` → `I16`, `TIMESTAMP` → `TIMESTAMP`, `Date` → `DATE`. -Rayforce's CSV reader parses the `YYYY-MM-DD` and `YYYY-MM-DD HH:MM:SS` -forms in the file into its native date/timestamp types, so -`EventDate >= '2013-07-01'`, `extract(minute FROM EventTime)` and -`DATE_TRUNC('minute', EventTime)` all work on native values. +The Parquet file keeps `EventDate` as integer days since 1970-01-01 and +`EventTime`, `ClientEventTime` and `LocalEventTime` as integer Unix +seconds; their entries in the type vector, `UNIX_DATE` and +`UNIX_SECONDS`, make the reader convert them to Rayforce's native date +and timestamp types, so `EventDate >= '2013-07-01'`, +`extract(minute FROM EventTime)` and `DATE_TRUNC('minute', EventTime)` +all work on native values. ### Why every text column is SYM diff --git a/rayforce/benchmark.sh b/rayforce/benchmark.sh index ba36b0e160..47f4c87d8b 100755 --- a/rayforce/benchmark.sh +++ b/rayforce/benchmark.sh @@ -1,5 +1,5 @@ #!/bin/bash -export BENCH_DOWNLOAD_SCRIPT="download-hits-csv" +export BENCH_DOWNLOAD_SCRIPT="download-hits-parquet-single" # Rayforce runs as an IPC server here (./start), so the stop/drop_caches/start # cold cycle is meaningful and the concurrent-QPS test hits a shared process. export BENCH_RESTARTABLE=yes diff --git a/rayforce/create.rfl b/rayforce/create.rfl index 55befb5a4c..f61d471666 100644 --- a/rayforce/create.rfl +++ b/rayforce/create.rfl @@ -1,13 +1,16 @@ ; ClickBench `hits` schema for Rayforce. ; -; `load` feeds these two vectors to `.csv.splayed`, which streams -; hits.csv straight into the on-disk splayed table without a header row -; (an explicit name vector implies headerless input). +; `load` passes `hits-types` to `.parquet.splayed`, one entry per column +; of hits.parquet in file order; `hits-names` lists that order. ; ; Types follow ../clickhouse/create.sql: BIGINT -> I64, INTEGER -> I32, -; SMALLINT -> I16, TIMESTAMP -> TIMESTAMP, Date -> DATE. Every TEXT / -; VARCHAR / CHAR column is loaded as SYM (dictionary-encoded) rather -; than STR -- see README.md "Why every text column is SYM". +; SMALLINT -> I16, TIMESTAMP -> TIMESTAMP, Date -> DATE. The Parquet +; file keeps EventDate as integer days since 1970-01-01 and EventTime, +; ClientEventTime and LocalEventTime as integer Unix seconds: UNIX_DATE +; and UNIX_SECONDS make the reader convert them to native DATE and +; TIMESTAMP. Every TEXT / VARCHAR / CHAR column is loaded as SYM +; (dictionary-encoded) rather than STR -- see README.md "Why every text +; column is SYM". (set hits-names [ WatchID JavaEnable Title GoodEvent EventTime EventDate CounterID ClientIP @@ -32,10 +35,10 @@ ]) (set hits-types [ - I64 I16 SYM I16 TIMESTAMP DATE I32 I32 I32 I64 I16 I16 I16 SYM SYM I16 + I64 I16 SYM I16 UNIX_SECONDS UNIX_DATE I32 I32 I32 I64 I16 I16 I16 SYM SYM I16 I16 I32 I16 I32 I16 I16 I16 I16 I16 SYM I16 I16 I16 SYM I16 I16 I16 I16 - SYM SYM I32 I16 I16 SYM I16 I16 I16 I16 I16 TIMESTAMP I16 I16 I32 I16 SYM - I32 I16 I16 I16 I64 SYM I32 I16 I16 I16 I16 I16 SYM TIMESTAMP I16 I16 I16 + SYM SYM I32 I16 I16 SYM I16 I16 I16 I16 I16 UNIX_SECONDS I16 I16 I32 I16 SYM + I32 I16 I16 I16 I64 SYM I32 I16 I16 I16 I16 I16 SYM UNIX_SECONDS I16 I16 I16 I16 I16 I32 I32 I32 I16 SYM SYM SYM SYM I16 I32 I32 I32 I32 I32 I32 I16 SYM I64 SYM SYM I16 SYM SYM SYM SYM SYM SYM SYM SYM SYM SYM I16 I64 I64 I32 diff --git a/rayforce/install b/rayforce/install index df4e789d6d..c5b28b50cb 100755 --- a/rayforce/install +++ b/rayforce/install @@ -10,7 +10,7 @@ # (falling back to a known-good tag when the unauthenticated API call is rate # limited). # -# Last benchmarked release: v2.10.0. Bump this line to request a new run: +# Last benchmarked release: v2.11.0. Bump this line to request a new run: # the benchmark automation reruns a system when a file in its directory # changes. set -e diff --git a/rayforce/load b/rayforce/load index ebfc03216c..8d1f980c46 100755 --- a/rayforce/load +++ b/rayforce/load @@ -1,5 +1,5 @@ #!/bin/bash -# Loads hits.csv into the splayed table ./hits. +# Loads hits.parquet into the splayed table ./hits. # # The load runs in its own rayforce process, not through the server: the server # started by the driver before ./load has no table open yet and picks it up at @@ -10,6 +10,6 @@ rm -rf hits rayforce load.rfl -# The CSV is no longer needed once the columns are on disk. -rm -f hits.csv +# The Parquet file is no longer needed once the columns are on disk. +rm -f hits.parquet sync diff --git a/rayforce/load.rfl b/rayforce/load.rfl index 1003d0f102..bba9d764da 100644 --- a/rayforce/load.rfl +++ b/rayforce/load.rfl @@ -1,9 +1,9 @@ -; Stream hits.csv into a splayed on-disk table under ./hits. +; Stream hits.parquet into a splayed on-disk table under ./hits. ; -; `.csv.splayed` mmaps the CSV, parses chunks in parallel and writes one column +; `.parquet.splayed` reads the file row group by row group and writes one column ; file per field plus the `.sym` dictionary, without materialising the whole -; table in memory. The name vector implies the input has no header row, which is -; the shape of the published hits.csv. +; table in memory. The column names come from the file; `hits-types` gives the +; native type of each column in the file's order. (load "create.rfl") -(.csv.splayed hits-names hits-types "hits.csv" "hits") +(.parquet.splayed hits-types "hits.parquet" "hits") (exit 0) diff --git a/rayforce/results/20261004/c6a.2xlarge.json b/rayforce/results/20261004/c6a.2xlarge.json new file mode 100644 index 0000000000..32a0db0005 --- /dev/null +++ b/rayforce/results/20261004/c6a.2xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Rayforce", + "date": "2026-10-04", + "machine": "c6a.2xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded"], + "load_time": 1106, + "data_size": 59042332197, + "concurrent_qps": 0.21, + "concurrent_error_ratio": 0, + "result": [ + [0, 0, 0], + [0.175, 0.01, 0.01], + [0.006, 0, 0], + [2.072, 0.021, 0.02], + [2.578, 0.465, 0.485], + [1.714, 0.119, 0.118], + [0.006, 0, 0], + [0.181, 0.018, 0.018], + [5.878, 1.84, 1.84], + [6.709, 1.334, 1.27], + [3.72, 0.137, 0.131], + [4.491, 0.141, 0.141], + [0.745, 0.177, 0.178], + [3.997, 0.401, 0.413], + [1.504, 0.212, 0.207], + [3.238, 1.002, 0.967], + [7.017, 1.775, 1.362], + [8.86, 0.042, 0.042], + [8.97, 2.256, 2.202], + [0.022, 0, 0], + [0.63, 0.332, 0.334], + [2.08, 0.178, 0.175], + [4.239, 0.467, 0.474], + [0.891, 0.039, 0.037], + [0.099, 0.004, 0.005], + [0.536, 0.129, 0.124], + [0.105, 0.005, 0.005], + [2.807, 0.854, 0.845], + [3.218, 3.207, 2.841], + [0.018, 0, 0], + [4.473, 0.278, 0.278], + [7.037, 0.385, 0.364], + [39.572, 54.658, 141.651], + [1.062, 0.517, 0.528], + [1.066, 0.514, 0.517], + [1.462, 0.978, 0.976], + [0.073, 0.024, 0.024], + [0.071, 0.024, 0.024], + [0.091, 0.014, 0.013], + [0.136, 0.084, 0.064], + [0.263, 0.015, 0.015], + [0.062, 0.006, 0.006], + [0.068, 0.013, 0.013] +] + } + \ No newline at end of file diff --git a/rayforce/results/20261004/c6a.4xlarge.json b/rayforce/results/20261004/c6a.4xlarge.json new file mode 100644 index 0000000000..44f2ac68c5 --- /dev/null +++ b/rayforce/results/20261004/c6a.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Rayforce", + "date": "2026-10-04", + "machine": "c6a.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded"], + "load_time": 518, + "data_size": 59042332197, + "concurrent_qps": 0.775, + "concurrent_error_ratio": 0, + "result": [ + [0, 0, 0], + [0.569, 0.005, 0.005], + [0.007, 0, 0], + [2.07, 0.02, 0.02], + [2.396, 0.209, 0.209], + [1.774, 0.105, 0.104], + [0.008, 0, 0], + [0.181, 0.011, 0.011], + [5.568, 0.846, 0.843], + [5.931, 0.536, 0.524], + [3.67, 0.091, 0.09], + [4.445, 0.103, 0.101], + [0.846, 0.113, 0.113], + [3.869, 0.282, 0.278], + [1.436, 0.138, 0.134], + [2.566, 0.406, 0.406], + [4.408, 0.854, 0.855], + [9.243, 0.031, 0.031], + [7.923, 1.299, 1.32], + [0.022, 0, 0], + [0.567, 0.174, 0.173], + [2.082, 0.104, 0.104], + [4.136, 0.223, 0.223], + [0.875, 0.017, 0.018], + [0.084, 0.003, 0.003], + [0.538, 0.063, 0.063], + [0.07, 0.005, 0.003], + [2.457, 0.452, 0.452], + [2.62, 2.105, 2.108], + [0.019, 0, 0], + [4.442, 0.176, 0.175], + [6.81, 0.249, 0.248], + [7.031, 2.139, 1.961], + [0.89, 0.274, 0.281], + [0.889, 0.274, 0.28], + [0.995, 0.37, 0.369], + [0.068, 0.021, 0.021], + [0.069, 0.02, 0.021], + [0.092, 0.012, 0.011], + [0.125, 0.071, 0.051], + [0.192, 0.015, 0.015], + [0.061, 0.007, 0.006], + [0.06, 0.01, 0.01] +] + } + \ No newline at end of file diff --git a/rayforce/results/20261004/c6a.metal.json b/rayforce/results/20261004/c6a.metal.json new file mode 100644 index 0000000000..43bc63f644 --- /dev/null +++ b/rayforce/results/20261004/c6a.metal.json @@ -0,0 +1,60 @@ +{ + "system": "Rayforce", + "date": "2026-10-04", + "machine": "c6a.metal", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded"], + "load_time": 258, + "data_size": 59042332197, + "concurrent_qps": 7.483, + "concurrent_error_ratio": 0, + "result": [ + [0, 0, 0], + [0.094, 0.004, 0.004], + [0.006, 0, 0], + [2.083, 0.011, 0.011], + [2.234, 0.123, 0.12], + [1.68, 0.115, 0.12], + [0.006, 0, 0], + [0.148, 0.01, 0.013], + [5.09, 0.145, 0.148], + [5.819, 0.336, 0.316], + [3.729, 0.067, 0.071], + [4.465, 0.066, 0.065], + [0.683, 0.079, 0.077], + [3.798, 0.161, 0.165], + [1.431, 0.085, 0.087], + [2.441, 0.232, 0.241], + [3.887, 0.226, 0.22], + [9.294, 0.019, 0.018], + [7.07, 0.373, 0.377], + [0.036, 0.001, 0.001], + [0.577, 0.091, 0.092], + [2.098, 0.128, 0.072], + [3.992, 0.182, 0.149], + [0.579, 0.016, 0.02], + [0.064, 0.003, 0.003], + [0.552, 0.025, 0.023], + [0.066, 0.002, 0.003], + [2.651, 0.615, 0.567], + [2.222, 1.632, 1.87], + [0.032, 0, 0], + [4.486, 0.116, 0.13], + [6.814, 0.167, 0.158], + [5.873, 0.767, 0.756], + [0.879, 0.276, 0.247], + [0.912, 0.407, 0.238], + [0.848, 0.185, 0.208], + [0.136, 0.026, 0.043], + [0.146, 0.025, 0.04], + [0.165, 0.016, 0.023], + [0.193, 0.092, 0.073], + [0.211, 0.029, 0.024], + [0.11, 0.023, 0.021], + [0.12, 0.014, 0.015] +] + } + \ No newline at end of file diff --git a/rayforce/results/20261004/c6a.xlarge.json b/rayforce/results/20261004/c6a.xlarge.json new file mode 100644 index 0000000000..2158a219f5 --- /dev/null +++ b/rayforce/results/20261004/c6a.xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Rayforce", + "date": "2026-10-04", + "machine": "c6a.xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded"], + "load_time": 3351, + "data_size": 59042332197, + "concurrent_qps": 0.058, + "concurrent_error_ratio": 0.667, + "result": [ + [0, 0, 0], + [0.331, 0.02, 0.02], + [0.006, 0, 0], + [2.069, 0.022, 0.021], + [2.796, 0.678, 0.704], + [1.705, 0.111, 0.11], + [0.007, 0, 0], + [0.314, 0.033, 0.032], + [93.218, 3.544, 2.984], + [11.594, 3.723, 2.709], + [3.8, 0.21, 0.214], + [4.585, 0.234, 0.234], + [1.048, 0.309, 0.306], + [4.555, 0.904, 0.858], + [1.686, 0.368, 0.363], + [7.671, 5.523, 3.049], + [11.25, 3.118, 2.339], + [8.839, 0.08, 0.08], + [43.684, 154.708, 277.965], + [0.024, 0, 0], + [13.923, 9.518, 1.516], + [11.638, 0.733, 0.296], + [21.926, 11.274, 1.225], + [2.759, 0.067, 0.062], + [0.166, 0.008, 0.008], + [2, 0.252, 0.256], + [0.165, 0.01, 0.009], + [24.463, 7.359, 1.679], + [26.594, 38.579, 25.658], + [0.02, 0, 0], + [4.59, 0.667, 0.639], + [9.478, 6.284, 0.948], + [208.128, 254.53, 442.923], + [4.807, 2.469, 1.802], + [4.854, 12.698, 2.728], + [2.519, 1.848, 1.717], + [0.095, 0.034, 0.033], + [0.084, 0.028, 0.028], + [0.094, 0.017, 0.017], + [0.343, 0.111, 0.084], + [0.426, 0.017, 0.017], + [0.07, 0.007, 0.007], + [0.088, 0.02, 0.02] +] + } + \ No newline at end of file diff --git a/rayforce/results/20261004/c7a.metal-48xl.json b/rayforce/results/20261004/c7a.metal-48xl.json new file mode 100644 index 0000000000..c508dc6f2d --- /dev/null +++ b/rayforce/results/20261004/c7a.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Rayforce", + "date": "2026-10-04", + "machine": "c7a.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded"], + "load_time": 243, + "data_size": 59042332197, + "concurrent_qps": 10.85, + "concurrent_error_ratio": 0, + "result": [ + [0, 0, 0], + [0.131, 0.001, 0.001], + [0.007, 0, 0], + [2.085, 0.003, 0.003], + [2.167, 0.038, 0.038], + [1.674, 0.125, 0.108], + [0.007, 0, 0], + [0.132, 0.008, 0.012], + [5.079, 0.076, 0.077], + [5.721, 0.242, 0.241], + [3.686, 0.025, 0.026], + [4.454, 0.03, 0.03], + [0.626, 0.039, 0.041], + [3.687, 0.067, 0.071], + [1.378, 0.043, 0.043], + [2.231, 0.089, 0.095], + [3.772, 0.108, 0.108], + [9.215, 0.005, 0.004], + [6.899, 0.163, 0.142], + [0.037, 0.001, 0.001], + [0.576, 0.028, 0.03], + [2.109, 0.059, 0.05], + [4.121, 0.077, 0.081], + [0.868, 0.011, 0.011], + [0.104, 0.002, 0.002], + [0.552, 0.009, 0.009], + [0.085, 0.002, 0.002], + [2.532, 0.428, 0.428], + [1.624, 0.912, 0.928], + [0.033, 0, 0], + [4.401, 0.048, 0.048], + [6.702, 0.067, 0.071], + [5.477, 0.313, 0.304], + [0.757, 0.087, 0.102], + [0.76, 0.089, 0.089], + [0.69, 0.072, 0.079], + [0.107, 0.034, 0.051], + [0.106, 0.033, 0.05], + [0.116, 0.019, 0.028], + [0.15, 0.089, 0.066], + [0.31, 0.019, 0.017], + [0.09, 0.018, 0.014], + [0.078, 0.012, 0.017] +] + } + \ No newline at end of file diff --git a/rayforce/results/20261004/c8g.4xlarge.json b/rayforce/results/20261004/c8g.4xlarge.json new file mode 100644 index 0000000000..e7845b13b2 --- /dev/null +++ b/rayforce/results/20261004/c8g.4xlarge.json @@ -0,0 +1,60 @@ +{ + "system": "Rayforce", + "date": "2026-10-04", + "machine": "c8g.4xlarge", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded"], + "load_time": 459, + "data_size": 59042332197, + "concurrent_qps": 0.875, + "concurrent_error_ratio": 0, + "result": [ + [0, 0, 0], + [0.088, 0.002, 0.002], + [0.005, 0, 0], + [2.063, 0.003, 0.003], + [2.154, 0.065, 0.065], + [1.607, 0.088, 0.087], + [0.005, 0, 0], + [0.094, 0.007, 0.007], + [4.972, 0.408, 0.4], + [5.881, 0.326, 0.327], + [3.615, 0.034, 0.033], + [4.383, 0.037, 0.037], + [0.597, 0.05, 0.05], + [3.683, 0.101, 0.1], + [1.351, 0.055, 0.055], + [2.276, 0.16, 0.16], + [4.092, 0.532, 0.546], + [8.544, 0.013, 0.013], + [7.361, 0.773, 0.779], + [0.016, 0, 0], + [0.546, 0.1, 0.1], + [2.068, 0.053, 0.053], + [4.231, 0.111, 0.112], + [0.789, 0.01, 0.01], + [0.052, 0.002, 0.002], + [0.533, 0.035, 0.035], + [0.053, 0.004, 0.002], + [2.269, 0.242, 0.242], + [1.673, 1.197, 1.201], + [0.013, 0, 0], + [4.375, 0.066, 0.066], + [6.673, 0.089, 0.088], + [5.922, 1.093, 3.192], + [0.726, 0.15, 0.149], + [0.732, 0.153, 0.153], + [0.734, 0.141, 0.141], + [0.044, 0.015, 0.015], + [0.044, 0.015, 0.015], + [0.069, 0.009, 0.009], + [0.077, 0.043, 0.034], + [0.151, 0.006, 0.006], + [0.042, 0.004, 0.004], + [0.04, 0.007, 0.007] +] + } + \ No newline at end of file diff --git a/rayforce/results/20261004/c8g.metal-48xl.json b/rayforce/results/20261004/c8g.metal-48xl.json new file mode 100644 index 0000000000..993c66eabf --- /dev/null +++ b/rayforce/results/20261004/c8g.metal-48xl.json @@ -0,0 +1,60 @@ +{ + "system": "Rayforce", + "date": "2026-10-04", + "machine": "c8g.metal-48xl", + "cluster_size": 1, + "proprietary": "no", + "hardware": "cpu", + "tuned": "no", + "tags": ["C","column-oriented","embedded"], + "load_time": 244, + "data_size": 59042332197, + "concurrent_qps": 13.34, + "concurrent_error_ratio": 0, + "result": [ + [0, 0, 0], + [0.131, 0.001, 0.001], + [0.005, 0, 0], + [2.071, 0.002, 0.002], + [2.161, 0.034, 0.032], + [1.594, 0.089, 0.089], + [0.005, 0, 0], + [0.16, 0.004, 0.004], + [5.028, 0.061, 0.061], + [5.651, 0.223, 0.224], + [3.678, 0.018, 0.018], + [4.438, 0.02, 0.021], + [0.592, 0.026, 0.026], + [3.646, 0.048, 0.048], + [1.348, 0.03, 0.029], + [2.197, 0.056, 0.055], + [3.737, 0.072, 0.073], + [8.958, 0.004, 0.003], + [6.876, 0.119, 0.121], + [0.024, 0.001, 0], + [0.553, 0.022, 0.02], + [2.079, 0.027, 0.027], + [4.2, 0.061, 0.062], + [0.936, 0.008, 0.004], + [0.097, 0.002, 0.001], + [0.54, 0.007, 0.007], + [0.111, 0.004, 0.001], + [2.445, 0.368, 0.378], + [1.349, 0.683, 0.695], + [0.019, 0, 0], + [4.379, 0.033, 0.033], + [6.67, 0.05, 0.05], + [5.378, 0.244, 0.252], + [0.739, 0.11, 0.113], + [0.737, 0.109, 0.109], + [0.66, 0.046, 0.046], + [0.06, 0.017, 0.017], + [0.062, 0.017, 0.017], + [0.092, 0.011, 0.011], + [0.088, 0.042, 0.032], + [0.226, 0.012, 0.009], + [0.062, 0.011, 0.008], + [0.051, 0.008, 0.007] +] + } + \ No newline at end of file