From 59328c0eb972855fafd2ce6caa69fcacd413c406 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 14:41:14 +0100 Subject: [PATCH 1/8] rename legacy Vmify to Cff. --- Dockerfile | 2 +- README.md | 2 +- doc/SUMMARY.md | 2 +- doc/guides/presets.md | 12 +++++---- doc/guides/writing-a-custom-config.md | 7 ++++- doc/reference/steps/cff.md | 15 +++++++++++ doc/reference/steps/overview.md | 9 ++++--- doc/reference/steps/vmify.md | 16 ----------- src/presets.lua | 14 +++++----- src/prometheus/{compiler => cff}/block.lua | 0 .../{compiler => cff}/compile_core.lua | 6 ++--- .../{compiler => cff}/compile_top.lua | 0 src/prometheus/{compiler => cff}/compiler.lua | 12 ++++----- .../{compiler => cff}/constants.lua | 0 src/prometheus/{compiler => cff}/emit.lua | 2 +- .../{compiler => cff}/expressions.lua | 2 +- .../{compiler => cff}/expressions/and.lua | 0 .../{compiler => cff}/expressions/binary.lua | 0 .../{compiler => cff}/expressions/boolean.lua | 0 .../expressions/function_call.lua | 0 .../expressions/function_literal.lua | 0 .../{compiler => cff}/expressions/if_else.lua | 0 .../{compiler => cff}/expressions/index.lua | 0 .../{compiler => cff}/expressions/len.lua | 0 .../{compiler => cff}/expressions/negate.lua | 0 .../{compiler => cff}/expressions/nil.lua | 0 .../{compiler => cff}/expressions/not.lua | 0 .../{compiler => cff}/expressions/number.lua | 0 .../{compiler => cff}/expressions/or.lua | 0 .../expressions/pass_self_function_call.lua | 0 .../{compiler => cff}/expressions/string.lua | 0 .../expressions/table_constructor.lua | 0 .../{compiler => cff}/expressions/vararg.lua | 0 .../expressions/variable.lua | 0 src/prometheus/{compiler => cff}/register.lua | 2 +- .../{compiler => cff}/statements.lua | 2 +- .../statements/assignment.lua | 0 .../statements/break_statement.lua | 0 .../{compiler => cff}/statements/compound.lua | 0 .../statements/continue_statement.lua | 0 .../statements/do_statement.lua | 0 .../statements/for_in_statement.lua | 0 .../statements/for_statement.lua | 0 .../statements/function_call.lua | 0 .../statements/function_declaration.lua | 0 .../statements/if_statement.lua | 0 .../statements/local_function_declaration.lua | 0 .../statements/local_variable_declaration.lua | 0 .../statements/pass_self_function_call.lua | 0 .../statements/repeat_statement.lua | 0 .../{compiler => cff}/statements/return.lua | 0 .../statements/while_statement.lua | 0 src/prometheus/{compiler => cff}/upvalue.lua | 0 src/prometheus/steps.lua | 7 +++-- src/prometheus/steps/Cff.lua | 24 +++++++++++++++++ src/prometheus/steps/Vmify.lua | 27 ------------------- tests/harness/{vm-arity.lua => cff-arity.lua} | 4 +-- .../{vm-dispatch.lua => cff-dispatch.lua} | 6 ++--- tests/repeat-test.lua | 2 +- tests/upstream/lua51/README.md | 4 +-- web/src/vite/prometheusLuaPlugin.test.ts | 2 +- 61 files changed, 94 insertions(+), 87 deletions(-) create mode 100644 doc/reference/steps/cff.md delete mode 100644 doc/reference/steps/vmify.md rename src/prometheus/{compiler => cff}/block.lua (100%) rename src/prometheus/{compiler => cff}/compile_core.lua (91%) rename src/prometheus/{compiler => cff}/compile_top.lua (100%) rename src/prometheus/{compiler => cff}/compiler.lua (98%) rename src/prometheus/{compiler => cff}/constants.lua (100%) rename src/prometheus/{compiler => cff}/emit.lua (99%) rename src/prometheus/{compiler => cff}/expressions.lua (97%) rename src/prometheus/{compiler => cff}/expressions/and.lua (100%) rename src/prometheus/{compiler => cff}/expressions/binary.lua (100%) rename src/prometheus/{compiler => cff}/expressions/boolean.lua (100%) rename src/prometheus/{compiler => cff}/expressions/function_call.lua (100%) rename src/prometheus/{compiler => cff}/expressions/function_literal.lua (100%) rename src/prometheus/{compiler => cff}/expressions/if_else.lua (100%) rename src/prometheus/{compiler => cff}/expressions/index.lua (100%) rename src/prometheus/{compiler => cff}/expressions/len.lua (100%) rename src/prometheus/{compiler => cff}/expressions/negate.lua (100%) rename src/prometheus/{compiler => cff}/expressions/nil.lua (100%) rename src/prometheus/{compiler => cff}/expressions/not.lua (100%) rename src/prometheus/{compiler => cff}/expressions/number.lua (100%) rename src/prometheus/{compiler => cff}/expressions/or.lua (100%) rename src/prometheus/{compiler => cff}/expressions/pass_self_function_call.lua (100%) rename src/prometheus/{compiler => cff}/expressions/string.lua (100%) rename src/prometheus/{compiler => cff}/expressions/table_constructor.lua (100%) rename src/prometheus/{compiler => cff}/expressions/vararg.lua (100%) rename src/prometheus/{compiler => cff}/expressions/variable.lua (100%) rename src/prometheus/{compiler => cff}/register.lua (99%) rename src/prometheus/{compiler => cff}/statements.lua (97%) rename src/prometheus/{compiler => cff}/statements/assignment.lua (100%) rename src/prometheus/{compiler => cff}/statements/break_statement.lua (100%) rename src/prometheus/{compiler => cff}/statements/compound.lua (100%) rename src/prometheus/{compiler => cff}/statements/continue_statement.lua (100%) rename src/prometheus/{compiler => cff}/statements/do_statement.lua (100%) rename src/prometheus/{compiler => cff}/statements/for_in_statement.lua (100%) rename src/prometheus/{compiler => cff}/statements/for_statement.lua (100%) rename src/prometheus/{compiler => cff}/statements/function_call.lua (100%) rename src/prometheus/{compiler => cff}/statements/function_declaration.lua (100%) rename src/prometheus/{compiler => cff}/statements/if_statement.lua (100%) rename src/prometheus/{compiler => cff}/statements/local_function_declaration.lua (100%) rename src/prometheus/{compiler => cff}/statements/local_variable_declaration.lua (100%) rename src/prometheus/{compiler => cff}/statements/pass_self_function_call.lua (100%) rename src/prometheus/{compiler => cff}/statements/repeat_statement.lua (100%) rename src/prometheus/{compiler => cff}/statements/return.lua (100%) rename src/prometheus/{compiler => cff}/statements/while_statement.lua (100%) rename src/prometheus/{compiler => cff}/upvalue.lua (100%) create mode 100644 src/prometheus/steps/Cff.lua delete mode 100644 src/prometheus/steps/Vmify.lua rename tests/harness/{vm-arity.lua => cff-arity.lua} (97%) rename tests/harness/{vm-dispatch.lua => cff-dispatch.lua} (93%) diff --git a/Dockerfile b/Dockerfile index 5f8d4e30..08b72f52 100644 --- a/Dockerfile +++ b/Dockerfile @@ -26,7 +26,7 @@ COPY . /app RUN lua5.1 tests/harness/metadata.lua \ && timeout 30s lua5.1 tests/harness/numbers-to-expressions.lua \ - && timeout 30s lua5.1 tests/harness/vm-arity.lua \ + && timeout 30s lua5.1 tests/harness/cff-arity.lua \ && python3 tests/harness/runner.py ENTRYPOINT ["lua5.1", "docker-test-runner.lua"] diff --git a/README.md b/README.md index ba174753..93a54f41 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ official Lua 5.1 suite on Lua 5.1, inside Docker: ./scripts/run-tests.sh -c config.lua # Use a custom config ./scripts/run-tests.sh -v # Verbose output ./scripts/run-tests.sh --suite upstream --baseline-only # Verify official originals -./scripts/run-tests.sh --case lua51/closure.lua --preset Vmify --seed 4 -n 1 +./scripts/run-tests.sh --case lua51/closure.lua --preset Cff --seed 4 -n 1 ``` --- diff --git a/doc/SUMMARY.md b/doc/SUMMARY.md index 3396f43f..95f12c48 100644 --- a/doc/SUMMARY.md +++ b/doc/SUMMARY.md @@ -22,7 +22,7 @@ * [Config Object Reference](reference/config-object.md) * [Step Pipeline Overview](reference/steps/overview.md) * [Step: WrapInFunction](reference/steps/wrapinfunction.md) -* [Step: Vmify](reference/steps/vmify.md) +* [Step: Cff](reference/steps/cff.md) * [Step: SplitStrings](reference/steps/splitstrings.md) * [Step: ProxifyLocals](reference/steps/proxifylocals.md) * [Step: EncryptStrings](reference/steps/encryptstrings.md) diff --git a/doc/guides/presets.md b/doc/guides/presets.md index 61537c01..9208c8e5 100644 --- a/doc/guides/presets.md +++ b/doc/guides/presets.md @@ -4,7 +4,7 @@ Prometheus ships with these preset names in `src/presets.lua`: - `Minify` - `Weak` -- `Vmify` (test-oriented helper preset) +- `Cff` (test-oriented helper preset) - `Medium` - `Strong` @@ -24,7 +24,7 @@ prometheus-lua --preset Medium ./file.lua Steps: -1. `Vmify` +1. `Cff` 2. `ConstantArray` 3. `WrapInFunction` @@ -34,7 +34,7 @@ Steps: 1. `EncryptStrings` 2. `AntiTamper` (`UseDebug = false`) -3. `Vmify` +3. `Cff` 4. `ConstantArray` 5. `NumbersToExpressions` 6. `WrapInFunction` @@ -43,10 +43,10 @@ Steps: Steps: -1. `Vmify` +1. `Cff` 2. `EncryptStrings` 3. `AntiTamper` (`UseDebug = false`) -4. `Vmify` +4. `Cff` 5. `ConstantArray` 6. `NumbersToExpressions` 7. `WrapInFunction` @@ -56,3 +56,5 @@ Steps: - Start with `Medium`. - Move to `Strong` only after benchmarking your runtime path. - Use `Minify` when you want readability reduction only through compression/renaming style behavior. + +Preset compositions retain their current behavior in this milestone, with former `Vmify` steps renamed to `Cff`; the test preset is now named `Cff`. `Strong` therefore has two `Cff` passes for now. Its planned future sequence is one real `Vmify` pass followed by `Cff`; no real VM is available in this milestone. diff --git a/doc/guides/writing-a-custom-config.md b/doc/guides/writing-a-custom-config.md index 09a7be0d..0a9f6417 100644 --- a/doc/guides/writing-a-custom-config.md +++ b/doc/guides/writing-a-custom-config.md @@ -19,7 +19,7 @@ return { Seed = 0, Steps = { { Name = "EncryptStrings", Settings = {} }, - { Name = "Vmify", Settings = {} }, + { Name = "Cff", Settings = {} }, { Name = "WrapInFunction", Settings = { Iterations = 1 } }, } } @@ -29,6 +29,11 @@ return { `Steps` are applied in order. The same step can appear multiple times. +For existing custom configs, replace `Name = "Vmify"` with `Name = "Cff"`. +The `ControlFlowFlattening` constructor key is also accepted. `Vmify` is +reserved for a future real VM step and is not registered in this milestone. +Rename the `Vmify` preset to `Cff` in CLI commands as well. + ## Name generator values Supported string values (from `src/prometheus/namegenerators.lua`): diff --git a/doc/reference/steps/cff.md b/doc/reference/steps/cff.md new file mode 100644 index 00000000..9009cdd5 --- /dev/null +++ b/doc/reference/steps/cff.md @@ -0,0 +1,15 @@ +# Step: Cff + +Constructor key: `Cff`; `ControlFlowFlattening` is also accepted. + +`Cff` is the existing control-flow flattening step. `Vmify` is reserved for a future real VM step and is not registered in this milestone. + +## Settings + +None. + +## Example + +```lua +{ Name = "Cff", Settings = {} } +``` diff --git a/doc/reference/steps/overview.md b/doc/reference/steps/overview.md index 846133c9..386314c9 100644 --- a/doc/reference/steps/overview.md +++ b/doc/reference/steps/overview.md @@ -6,11 +6,11 @@ After step application, Prometheus performs variable renaming and then unparses ## Registered step constructor keys -Use these exact values in `Steps[i].Name`: +Use one of these constructor keys in `Steps[i].Name`: - `WrapInFunction` - `SplitStrings` -- `Vmify` +- `Cff` (also accepted as `ControlFlowFlattening`) - `ConstantArray` - `ProxifyLocals` - `AntiTamper` @@ -23,7 +23,7 @@ Use these exact values in `Steps[i].Name`: ```lua { - Name = "Vmify", + Name = "Cff", Settings = {} } ``` @@ -31,6 +31,7 @@ Use these exact values in `Steps[i].Name`: ## Notes - Some constructor keys differ from internal human-readable names. -- `Vmify` is a legacy constructor key name; behavior is closer to control-flow flattening style obfuscation than full virtualization. +- `Cff` is the existing control-flow flattening step and has no settings. `ControlFlowFlattening` is also accepted as a constructor key alias. +- `Vmify` is reserved for a future real VM step and is not registered in this milestone. - You can apply the same step more than once. - If a step returns a new AST, that AST is used for later steps. diff --git a/doc/reference/steps/vmify.md b/doc/reference/steps/vmify.md deleted file mode 100644 index 8884771b..00000000 --- a/doc/reference/steps/vmify.md +++ /dev/null @@ -1,16 +0,0 @@ -# Step: Vmify - -Constructor key: `Vmify` - -`Vmify` is a legacy step name kept for backward compatibility. -In practice, this transformation is better understood as control-flow flattening style obfuscation rather than full virtualization. - -## Settings - -None. - -## Example - -```lua -{ Name = "Vmify", Settings = {} } -``` diff --git a/src/presets.lua b/src/presets.lua index deeaf4f6..92009838 100644 --- a/src/presets.lua +++ b/src/presets.lua @@ -23,7 +23,7 @@ return { PrettyPrint = false, Seed = 0, Steps = { - { Name = "Vmify", Settings = {} }, + { Name = "Cff", Settings = {} }, { Name = "ConstantArray", Settings = { @@ -36,17 +36,17 @@ return { }, -- This is here for the tests.lua file. - -- It helps isolate any problems with the Vmify step. + -- It helps isolate any problems with the Cff step. -- It is not recommended to use this preset for obfuscation. -- Use the Weak, Medium, or Strong for obfuscation instead. - ["Vmify"] = { + ["Cff"] = { LuaVersion = "Lua51", VarNamePrefix = "", NameGenerator = "MangledShuffled", PrettyPrint = false, Seed = 0, Steps = { - { Name = "Vmify", Settings = {} }, + { Name = "Cff", Settings = {} }, }, }, @@ -65,7 +65,7 @@ return { UseDebug = false, }, }, - { Name = "Vmify", Settings = {} }, + { Name = "Cff", Settings = {} }, { Name = "ConstantArray", Settings = { @@ -89,7 +89,7 @@ return { PrettyPrint = false, Seed = 0, Steps = { - { Name = "Vmify", Settings = {} }, + { Name = "Cff", Settings = {} }, { Name = "EncryptStrings", Settings = {} }, { Name = "AntiTamper", @@ -97,7 +97,7 @@ return { UseDebug = false, }, }, - { Name = "Vmify", Settings = {} }, + { Name = "Cff", Settings = {} }, { Name = "ConstantArray", Settings = { diff --git a/src/prometheus/compiler/block.lua b/src/prometheus/cff/block.lua similarity index 100% rename from src/prometheus/compiler/block.lua rename to src/prometheus/cff/block.lua diff --git a/src/prometheus/compiler/compile_core.lua b/src/prometheus/cff/compile_core.lua similarity index 91% rename from src/prometheus/compiler/compile_core.lua rename to src/prometheus/cff/compile_core.lua index 73202504..af183011 100644 --- a/src/prometheus/compiler/compile_core.lua +++ b/src/prometheus/cff/compile_core.lua @@ -4,9 +4,9 @@ -- This Script contains the core compilation functions: compileTopNode, compileFunction, compileBlock, -- compileStatement, and compileExpression -local compileTop = require("prometheus.compiler.compile_top"); -local statementHandlers = require("prometheus.compiler.statements"); -local expressionHandlers = require("prometheus.compiler.expressions"); +local compileTop = require("prometheus.cff.compile_top"); +local statementHandlers = require("prometheus.cff.statements"); +local expressionHandlers = require("prometheus.cff.expressions"); local Ast = require("prometheus.ast"); local logger = require("logger"); diff --git a/src/prometheus/compiler/compile_top.lua b/src/prometheus/cff/compile_top.lua similarity index 100% rename from src/prometheus/compiler/compile_top.lua rename to src/prometheus/cff/compile_top.lua diff --git a/src/prometheus/compiler/compiler.lua b/src/prometheus/cff/compiler.lua similarity index 98% rename from src/prometheus/compiler/compiler.lua rename to src/prometheus/cff/compiler.lua index f02c7a56..5968dd32 100644 --- a/src/prometheus/compiler/compiler.lua +++ b/src/prometheus/cff/compiler.lua @@ -13,11 +13,11 @@ local AstKind = Ast.AstKind; local unpack = unpack or table.unpack; -local blockModule = require("prometheus.compiler.block"); -local registerModule = require("prometheus.compiler.register"); -local upvalueModule = require("prometheus.compiler.upvalue"); -local emitModule = require("prometheus.compiler.emit"); -local compileCoreModule = require("prometheus.compiler.compile_core"); +local blockModule = require("prometheus.cff.block"); +local registerModule = require("prometheus.cff.register"); +local upvalueModule = require("prometheus.cff.upvalue"); +local emitModule = require("prometheus.cff.emit"); +local compileCoreModule = require("prometheus.cff.compile_core"); local Compiler = {}; @@ -207,7 +207,7 @@ function Compiler:compile(ast) unpackScope:addReferenceToHigherScope(self.scope, self.unpackVar); -- Keep query handling shared: a per-call closure would capture the frame's - -- environment and delay its collection when another Vmify pass wraps it. + -- environment and delay its collection when another Cff pass wraps it. -- Ordinary functions receive exactly their original arguments via a tail call. local callScope = Scope:new(self.scope); local callFuncVar, callEnvVar = callScope:addVariable(), callScope:addVariable(); diff --git a/src/prometheus/compiler/constants.lua b/src/prometheus/cff/constants.lua similarity index 100% rename from src/prometheus/compiler/constants.lua rename to src/prometheus/cff/constants.lua diff --git a/src/prometheus/compiler/emit.lua b/src/prometheus/cff/emit.lua similarity index 99% rename from src/prometheus/compiler/emit.lua rename to src/prometheus/cff/emit.lua index 96af117e..b1469b1d 100644 --- a/src/prometheus/compiler/emit.lua +++ b/src/prometheus/cff/emit.lua @@ -7,7 +7,7 @@ local Ast = require("prometheus.ast"); local Scope = require("prometheus.scope"); local util = require("prometheus.util"); -local constants = require("prometheus.compiler.constants"); +local constants = require("prometheus.cff.constants"); local AstKind = Ast.AstKind; local MAX_REGS = constants.MAX_REGS; diff --git a/src/prometheus/compiler/expressions.lua b/src/prometheus/cff/expressions.lua similarity index 97% rename from src/prometheus/compiler/expressions.lua rename to src/prometheus/cff/expressions.lua index f55fb60a..82207854 100644 --- a/src/prometheus/compiler/expressions.lua +++ b/src/prometheus/cff/expressions.lua @@ -8,7 +8,7 @@ local Ast = require("prometheus.ast"); local AstKind = Ast.AstKind; local handlers = {}; -local expressions = "prometheus.compiler.expressions."; +local expressions = "prometheus.cff.expressions."; local function requireExpression(name) return require(expressions .. name); end diff --git a/src/prometheus/compiler/expressions/and.lua b/src/prometheus/cff/expressions/and.lua similarity index 100% rename from src/prometheus/compiler/expressions/and.lua rename to src/prometheus/cff/expressions/and.lua diff --git a/src/prometheus/compiler/expressions/binary.lua b/src/prometheus/cff/expressions/binary.lua similarity index 100% rename from src/prometheus/compiler/expressions/binary.lua rename to src/prometheus/cff/expressions/binary.lua diff --git a/src/prometheus/compiler/expressions/boolean.lua b/src/prometheus/cff/expressions/boolean.lua similarity index 100% rename from src/prometheus/compiler/expressions/boolean.lua rename to src/prometheus/cff/expressions/boolean.lua diff --git a/src/prometheus/compiler/expressions/function_call.lua b/src/prometheus/cff/expressions/function_call.lua similarity index 100% rename from src/prometheus/compiler/expressions/function_call.lua rename to src/prometheus/cff/expressions/function_call.lua diff --git a/src/prometheus/compiler/expressions/function_literal.lua b/src/prometheus/cff/expressions/function_literal.lua similarity index 100% rename from src/prometheus/compiler/expressions/function_literal.lua rename to src/prometheus/cff/expressions/function_literal.lua diff --git a/src/prometheus/compiler/expressions/if_else.lua b/src/prometheus/cff/expressions/if_else.lua similarity index 100% rename from src/prometheus/compiler/expressions/if_else.lua rename to src/prometheus/cff/expressions/if_else.lua diff --git a/src/prometheus/compiler/expressions/index.lua b/src/prometheus/cff/expressions/index.lua similarity index 100% rename from src/prometheus/compiler/expressions/index.lua rename to src/prometheus/cff/expressions/index.lua diff --git a/src/prometheus/compiler/expressions/len.lua b/src/prometheus/cff/expressions/len.lua similarity index 100% rename from src/prometheus/compiler/expressions/len.lua rename to src/prometheus/cff/expressions/len.lua diff --git a/src/prometheus/compiler/expressions/negate.lua b/src/prometheus/cff/expressions/negate.lua similarity index 100% rename from src/prometheus/compiler/expressions/negate.lua rename to src/prometheus/cff/expressions/negate.lua diff --git a/src/prometheus/compiler/expressions/nil.lua b/src/prometheus/cff/expressions/nil.lua similarity index 100% rename from src/prometheus/compiler/expressions/nil.lua rename to src/prometheus/cff/expressions/nil.lua diff --git a/src/prometheus/compiler/expressions/not.lua b/src/prometheus/cff/expressions/not.lua similarity index 100% rename from src/prometheus/compiler/expressions/not.lua rename to src/prometheus/cff/expressions/not.lua diff --git a/src/prometheus/compiler/expressions/number.lua b/src/prometheus/cff/expressions/number.lua similarity index 100% rename from src/prometheus/compiler/expressions/number.lua rename to src/prometheus/cff/expressions/number.lua diff --git a/src/prometheus/compiler/expressions/or.lua b/src/prometheus/cff/expressions/or.lua similarity index 100% rename from src/prometheus/compiler/expressions/or.lua rename to src/prometheus/cff/expressions/or.lua diff --git a/src/prometheus/compiler/expressions/pass_self_function_call.lua b/src/prometheus/cff/expressions/pass_self_function_call.lua similarity index 100% rename from src/prometheus/compiler/expressions/pass_self_function_call.lua rename to src/prometheus/cff/expressions/pass_self_function_call.lua diff --git a/src/prometheus/compiler/expressions/string.lua b/src/prometheus/cff/expressions/string.lua similarity index 100% rename from src/prometheus/compiler/expressions/string.lua rename to src/prometheus/cff/expressions/string.lua diff --git a/src/prometheus/compiler/expressions/table_constructor.lua b/src/prometheus/cff/expressions/table_constructor.lua similarity index 100% rename from src/prometheus/compiler/expressions/table_constructor.lua rename to src/prometheus/cff/expressions/table_constructor.lua diff --git a/src/prometheus/compiler/expressions/vararg.lua b/src/prometheus/cff/expressions/vararg.lua similarity index 100% rename from src/prometheus/compiler/expressions/vararg.lua rename to src/prometheus/cff/expressions/vararg.lua diff --git a/src/prometheus/compiler/expressions/variable.lua b/src/prometheus/cff/expressions/variable.lua similarity index 100% rename from src/prometheus/compiler/expressions/variable.lua rename to src/prometheus/cff/expressions/variable.lua diff --git a/src/prometheus/compiler/register.lua b/src/prometheus/cff/register.lua similarity index 99% rename from src/prometheus/compiler/register.lua rename to src/prometheus/cff/register.lua index ab5522b9..6baaa301 100644 --- a/src/prometheus/compiler/register.lua +++ b/src/prometheus/cff/register.lua @@ -5,7 +5,7 @@ -- This Script contains the register management for the compiler. local Ast = require("prometheus.ast"); -local constants = require("prometheus.compiler.constants"); +local constants = require("prometheus.cff.constants"); local randomStrings = require("prometheus.randomStrings"); local MAX_REGS = constants.MAX_REGS; diff --git a/src/prometheus/compiler/statements.lua b/src/prometheus/cff/statements.lua similarity index 97% rename from src/prometheus/compiler/statements.lua rename to src/prometheus/cff/statements.lua index 5b5cdd9c..6323ffad 100644 --- a/src/prometheus/compiler/statements.lua +++ b/src/prometheus/cff/statements.lua @@ -8,7 +8,7 @@ local Ast = require("prometheus.ast"); local AstKind = Ast.AstKind; local handlers = {}; -local statements = "prometheus.compiler.statements."; +local statements = "prometheus.cff.statements."; local function requireStatement(name) return require(statements .. name); end diff --git a/src/prometheus/compiler/statements/assignment.lua b/src/prometheus/cff/statements/assignment.lua similarity index 100% rename from src/prometheus/compiler/statements/assignment.lua rename to src/prometheus/cff/statements/assignment.lua diff --git a/src/prometheus/compiler/statements/break_statement.lua b/src/prometheus/cff/statements/break_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/break_statement.lua rename to src/prometheus/cff/statements/break_statement.lua diff --git a/src/prometheus/compiler/statements/compound.lua b/src/prometheus/cff/statements/compound.lua similarity index 100% rename from src/prometheus/compiler/statements/compound.lua rename to src/prometheus/cff/statements/compound.lua diff --git a/src/prometheus/compiler/statements/continue_statement.lua b/src/prometheus/cff/statements/continue_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/continue_statement.lua rename to src/prometheus/cff/statements/continue_statement.lua diff --git a/src/prometheus/compiler/statements/do_statement.lua b/src/prometheus/cff/statements/do_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/do_statement.lua rename to src/prometheus/cff/statements/do_statement.lua diff --git a/src/prometheus/compiler/statements/for_in_statement.lua b/src/prometheus/cff/statements/for_in_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/for_in_statement.lua rename to src/prometheus/cff/statements/for_in_statement.lua diff --git a/src/prometheus/compiler/statements/for_statement.lua b/src/prometheus/cff/statements/for_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/for_statement.lua rename to src/prometheus/cff/statements/for_statement.lua diff --git a/src/prometheus/compiler/statements/function_call.lua b/src/prometheus/cff/statements/function_call.lua similarity index 100% rename from src/prometheus/compiler/statements/function_call.lua rename to src/prometheus/cff/statements/function_call.lua diff --git a/src/prometheus/compiler/statements/function_declaration.lua b/src/prometheus/cff/statements/function_declaration.lua similarity index 100% rename from src/prometheus/compiler/statements/function_declaration.lua rename to src/prometheus/cff/statements/function_declaration.lua diff --git a/src/prometheus/compiler/statements/if_statement.lua b/src/prometheus/cff/statements/if_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/if_statement.lua rename to src/prometheus/cff/statements/if_statement.lua diff --git a/src/prometheus/compiler/statements/local_function_declaration.lua b/src/prometheus/cff/statements/local_function_declaration.lua similarity index 100% rename from src/prometheus/compiler/statements/local_function_declaration.lua rename to src/prometheus/cff/statements/local_function_declaration.lua diff --git a/src/prometheus/compiler/statements/local_variable_declaration.lua b/src/prometheus/cff/statements/local_variable_declaration.lua similarity index 100% rename from src/prometheus/compiler/statements/local_variable_declaration.lua rename to src/prometheus/cff/statements/local_variable_declaration.lua diff --git a/src/prometheus/compiler/statements/pass_self_function_call.lua b/src/prometheus/cff/statements/pass_self_function_call.lua similarity index 100% rename from src/prometheus/compiler/statements/pass_self_function_call.lua rename to src/prometheus/cff/statements/pass_self_function_call.lua diff --git a/src/prometheus/compiler/statements/repeat_statement.lua b/src/prometheus/cff/statements/repeat_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/repeat_statement.lua rename to src/prometheus/cff/statements/repeat_statement.lua diff --git a/src/prometheus/compiler/statements/return.lua b/src/prometheus/cff/statements/return.lua similarity index 100% rename from src/prometheus/compiler/statements/return.lua rename to src/prometheus/cff/statements/return.lua diff --git a/src/prometheus/compiler/statements/while_statement.lua b/src/prometheus/cff/statements/while_statement.lua similarity index 100% rename from src/prometheus/compiler/statements/while_statement.lua rename to src/prometheus/cff/statements/while_statement.lua diff --git a/src/prometheus/compiler/upvalue.lua b/src/prometheus/cff/upvalue.lua similarity index 100% rename from src/prometheus/compiler/upvalue.lua rename to src/prometheus/cff/upvalue.lua diff --git a/src/prometheus/steps.lua b/src/prometheus/steps.lua index 0544c8ec..d7d727fe 100644 --- a/src/prometheus/steps.lua +++ b/src/prometheus/steps.lua @@ -4,10 +4,13 @@ -- -- This Script provides a collection of obfuscation steps. +local Cff = require("prometheus.steps.Cff"); + return { WrapInFunction = require("prometheus.steps.WrapInFunction"), SplitStrings = require("prometheus.steps.SplitStrings"), - Vmify = require("prometheus.steps.Vmify"), + Cff = Cff, + ControlFlowFlattening = Cff, ConstantArray = require("prometheus.steps.ConstantArray"), ProxifyLocals = require("prometheus.steps.ProxifyLocals"), AntiTamper = require("prometheus.steps.AntiTamper"), @@ -15,4 +18,4 @@ return { NumbersToExpressions = require("prometheus.steps.NumbersToExpressions"), AddVararg = require("prometheus.steps.AddVararg"), WatermarkCheck = require("prometheus.steps.WatermarkCheck"), -} \ No newline at end of file +} diff --git a/src/prometheus/steps/Cff.lua b/src/prometheus/steps/Cff.lua new file mode 100644 index 00000000..e6c69094 --- /dev/null +++ b/src/prometheus/steps/Cff.lua @@ -0,0 +1,24 @@ +-- This Script is Part of the Prometheus Obfuscator by levno-710 +-- +-- Cff.lua +-- +-- Compile the script into Lua statements executed by a randomized block dispatcher. + +local Step = require("prometheus.step"); +local Compiler = require("prometheus.cff.compiler"); + +local Cff = Step:extend(); +Cff.Description = "Flattens control flow into Lua statements executed by a randomized block dispatcher."; +Cff.Name = "Cff"; + +Cff.SettingsDescriptor = {} + +function Cff:init(_) end + +function Cff:apply(ast) + local compiler = Compiler:new(); + + return compiler:compile(ast); +end + +return Cff; diff --git a/src/prometheus/steps/Vmify.lua b/src/prometheus/steps/Vmify.lua deleted file mode 100644 index f503de62..00000000 --- a/src/prometheus/steps/Vmify.lua +++ /dev/null @@ -1,27 +0,0 @@ --- This Script is Part of the Prometheus Obfuscator by levno-710 --- --- Vmify.lua --- --- This Script provides a Complex Obfuscation Step that will compile the entire Script to a fully custom bytecode that does not share it's instructions --- with lua, making it much harder to crack than other lua obfuscators - -local Step = require("prometheus.step"); -local Compiler = require("prometheus.compiler.compiler"); - -local Vmify = Step:extend(); -Vmify.Description = "This Step will Compile your script into a fully-custom (not a half custom like other lua obfuscators) Bytecode Format and emit a vm for executing it."; -Vmify.Name = "Vmify"; - -Vmify.SettingsDescriptor = {} - -function Vmify:init(_) end - -function Vmify:apply(ast) - -- Create Compiler - local compiler = Compiler:new(); - - -- Compile the Script into a bytecode vm - return compiler:compile(ast); -end - -return Vmify; \ No newline at end of file diff --git a/tests/harness/vm-arity.lua b/tests/harness/cff-arity.lua similarity index 97% rename from tests/harness/vm-arity.lua rename to tests/harness/cff-arity.lua index 7c0286d2..5789b4f4 100644 --- a/tests/harness/vm-arity.lua +++ b/tests/harness/cff-arity.lua @@ -3,7 +3,7 @@ Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error local function pack(...) return {n = select("#", ...), ...} end -- The outer wrapper must preserve chunk arguments and chunk return counts too. for _, seed in ipairs({1, 2, 3}) do - local config = {LuaVersion = "Lua51", Seed = seed, Steps = {{Name = "Vmify"}}} + local config = {LuaVersion = "Lua51", Seed = seed, Steps = {{Name = "Cff"}}} local generated = Prometheus.Pipeline:fromConfig(config):apply("return ...") local chunk = assert(loadstring(generated)) for _, args in ipairs({pack(), pack(nil), pack(1, nil), pack(nil, 2, nil)}) do @@ -33,4 +33,4 @@ for _, seed in ipairs({1, 2, 3}) do local actual = pack(assert(loadstring(Prometheus.Pipeline:fromConfig(config):apply(source)))()) assert(actual.n == 2 and #actual[1] == 250 and actual[2] == nil) end -print("VM chunk arity regression checks passed") +print("CFF chunk arity regression checks passed") diff --git a/tests/harness/vm-dispatch.lua b/tests/harness/cff-dispatch.lua similarity index 93% rename from tests/harness/vm-dispatch.lua rename to tests/harness/cff-dispatch.lua index 3445ce59..0da4a651 100644 --- a/tests/harness/vm-dispatch.lua +++ b/tests/harness/cff-dispatch.lua @@ -1,6 +1,6 @@ local Prometheus = require("src.prometheus") Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error -local Compiler = require("prometheus.compiler.compiler") +local Compiler = require("prometheus.cff.compiler") local createBlock = Compiler.createBlock local firstId @@ -18,7 +18,7 @@ local ok, err = pcall(function() for _, start in ipairs({0, 100}) do firstId = start for seed = 1, 10 do - local config = {LuaVersion = "Lua51", Seed = seed, Steps = {{Name = "Vmify"}}} + local config = {LuaVersion = "Lua51", Seed = seed, Steps = {{Name = "Cff"}}} local pipeline = Prometheus.Pipeline:fromConfig(config) -- Four blocks exercise the short elseif dispatcher. @@ -39,4 +39,4 @@ local ok, err = pcall(function() end) Compiler.createBlock = createBlock assert(ok, err) -print("VM dispatch boundary regression checks passed") +print("CFF dispatch boundary regression checks passed") diff --git a/tests/repeat-test.lua b/tests/repeat-test.lua index e9577469..98ea560a 100644 --- a/tests/repeat-test.lua +++ b/tests/repeat-test.lua @@ -1,6 +1,6 @@ --============================================================ -- Repeat-Until Semantics Test Suite --- Target: Vmify +-- Target: Cff -- Author: Zaenalos -- Purpose: Validate correct scope, control flow, and condition --============================================================ diff --git a/tests/upstream/lua51/README.md b/tests/upstream/lua51/README.md index fabefa42..0cb69455 100644 --- a/tests/upstream/lua51/README.md +++ b/tests/upstream/lua51/README.md @@ -83,14 +83,14 @@ From the repository root: ```sh bash scripts/run-tests.sh -b -n 10 --ci bash scripts/run-tests.sh --suite upstream --baseline-only --pass-runners -bash scripts/run-tests.sh --suite upstream --case lua51/closure.lua --preset Vmify --seed 4 -n 1 +bash scripts/run-tests.sh --suite upstream --case lua51/closure.lua --preset Cff --seed 4 -n 1 ``` The same selectors are available directly when `lua5.1`, `luau`, Python 3, `patch`, and GNU coreutils are installed: ```sh -lua5.1 docker-test-runner.lua --suite=upstream --case=lua51/closure.lua --preset=Vmify --seed=4 --iterations=1 +lua5.1 docker-test-runner.lua --suite=upstream --case=lua51/closure.lua --preset=Cff --seed=4 --iterations=1 lua5.1 tests/harness/metadata.lua python3 tests/harness/runner.py ``` diff --git a/web/src/vite/prometheusLuaPlugin.test.ts b/web/src/vite/prometheusLuaPlugin.test.ts index e09336fb..c16a15af 100644 --- a/web/src/vite/prometheusLuaPlugin.test.ts +++ b/web/src/vite/prometheusLuaPlugin.test.ts @@ -7,7 +7,7 @@ describe("luaPathToModuleName", () => { ["src/prometheus.lua", "prometheus"], ["src/presets.lua", "presets"], ["src/prometheus/pipeline.lua", "prometheus.pipeline"], - ["src/prometheus/compiler/expressions/string.lua", "prometheus.compiler.expressions.string"], + ["src/prometheus/cff/expressions/string.lua", "prometheus.cff.expressions.string"], ])("maps %s to %s", (filePath, moduleName) => { expect(luaPathToModuleName(filePath)).toBe(moduleName) }) From 3e14a310fcf05d793834b192af7ad0ccf8349266 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:13:45 +0100 Subject: [PATCH 2/8] implement register VM --- Dockerfile | 2 + README.md | 2 +- doc/SUMMARY.md | 2 + doc/advanced/register-vm.md | 56 +++++ doc/guides/presets.md | 22 +- doc/guides/writing-a-custom-config.md | 7 +- doc/reference/steps/cff.md | 6 +- doc/reference/steps/overview.md | 16 +- doc/reference/steps/vmify.md | 17 ++ src/presets.lua | 19 +- src/prometheus/steps.lua | 1 + src/prometheus/steps/Cff.lua | 5 + src/prometheus/steps/Vmify.lua | 17 ++ src/prometheus/vm/analysis.lua | 23 ++ src/prometheus/vm/compiler.lua | 167 +++++++++++++ src/prometheus/vm/emit.lua | 41 ++++ src/prometheus/vm/expressions.lua | 257 ++++++++++++++++++++ src/prometheus/vm/expressions_tables.lua | 68 ++++++ src/prometheus/vm/flatten.lua | 66 +++++ src/prometheus/vm/opcodes.lua | 13 + src/prometheus/vm/runtime.lua | 91 +++++++ src/prometheus/vm/runtime_dispatch.lua | 163 +++++++++++++ src/prometheus/vm/statements.lua | 293 +++++++++++++++++++++++ tests/harness/vm-arity.lua | 70 ++++++ tests/harness/vm-prototypes.lua | 38 +++ 25 files changed, 1437 insertions(+), 25 deletions(-) create mode 100644 doc/advanced/register-vm.md create mode 100644 doc/reference/steps/vmify.md create mode 100644 src/prometheus/steps/Vmify.lua create mode 100644 src/prometheus/vm/analysis.lua create mode 100644 src/prometheus/vm/compiler.lua create mode 100644 src/prometheus/vm/emit.lua create mode 100644 src/prometheus/vm/expressions.lua create mode 100644 src/prometheus/vm/expressions_tables.lua create mode 100644 src/prometheus/vm/flatten.lua create mode 100644 src/prometheus/vm/opcodes.lua create mode 100644 src/prometheus/vm/runtime.lua create mode 100644 src/prometheus/vm/runtime_dispatch.lua create mode 100644 src/prometheus/vm/statements.lua create mode 100644 tests/harness/vm-arity.lua create mode 100644 tests/harness/vm-prototypes.lua diff --git a/Dockerfile b/Dockerfile index 08b72f52..540b85b9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,8 @@ COPY . /app RUN lua5.1 tests/harness/metadata.lua \ && timeout 30s lua5.1 tests/harness/numbers-to-expressions.lua \ && timeout 30s lua5.1 tests/harness/cff-arity.lua \ + && timeout 30s lua5.1 tests/harness/vm-arity.lua \ + && timeout 30s lua5.1 tests/harness/vm-prototypes.lua \ && python3 tests/harness/runner.py ENTRYPOINT ["lua5.1", "docker-test-runner.lua"] diff --git a/README.md b/README.md index 93a54f41..2f03a422 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ **Prometheus** is a Lua obfuscator written in pure Lua. It applies a range of **AST-based transformations** to make source code significantly harder to read, analyze, and reverse engineer. -These include techniques such as **control-flow flattening**, **constant encryption**, and other Lua-specific obfuscation strategies. +These include **register VM virtualization**, **control-flow flattening**, **constant encryption**, and other Lua-specific obfuscation strategies. The project was inspired by the excellent [javascript-obfuscator](https://github.com/javascript-obfuscator/javascript-obfuscator). diff --git a/doc/SUMMARY.md b/doc/SUMMARY.md index 95f12c48..5d0a170f 100644 --- a/doc/SUMMARY.md +++ b/doc/SUMMARY.md @@ -23,6 +23,7 @@ * [Step Pipeline Overview](reference/steps/overview.md) * [Step: WrapInFunction](reference/steps/wrapinfunction.md) * [Step: Cff](reference/steps/cff.md) +* [Step: Vmify](reference/steps/vmify.md) * [Step: SplitStrings](reference/steps/splitstrings.md) * [Step: ProxifyLocals](reference/steps/proxifylocals.md) * [Step: EncryptStrings](reference/steps/encryptstrings.md) @@ -35,3 +36,4 @@ ## Advanced * [How the Pipeline Works](advanced/how-the-pipeline-works.md) +* [Register VM](advanced/register-vm.md) diff --git a/doc/advanced/register-vm.md b/doc/advanced/register-vm.md new file mode 100644 index 00000000..70e3df4b --- /dev/null +++ b/doc/advanced/register-vm.md @@ -0,0 +1,56 @@ +# Register VM architecture + +`Vmify` compiles the scoped Prometheus AST directly into instructions executed +by a generated Lua interpreter. It does not use `string.dump`, an external Lua +compiler, or `loadstring` to execute the original source. + +## Modules + +- `vm/analysis.lua`: lexical scope ownership and captured-local analysis. +- `vm/compiler.lua`: function prototypes, constants, registers, and captures. +- `vm/expressions.lua` and `vm/expressions_tables.lua`: expression lowering, + value lists, calls, branches, and table constructors. +- `vm/statements.lua`: statements, lexical cleanup, and loop control. +- `vm/opcodes.lua`: semantic instruction IDs. +- `vm/runtime.lua` and `vm/runtime_dispatch.lua`: runtime templates and the + generated dispatcher. +- `vm/emit.lua`: native prototype table literals and the runtime function IIFE. +- `vm/flatten.lua`: the Cff integration for generated VM interpreters. + +A prototype contains instructions, constants, child prototypes, parameter +metadata, and capture descriptors. The emitter passes the native prototype +tables as an argument to a runtime function IIFE. The generated interpreter +then uses that table data; the prototypes are not encoded as strings. + +Each invocation owns its register table and vararg tuple. Closures retain only +their referenced upvalue cells; leaving a scope detaches its cells without +destroying values shared by live closures. Value lists carry an explicit `n` +so trailing nils survive calls, varargs, table constructors, and returns. +Temporary registers are cleared after consumption, and conditional branches +consume their test register on both paths. + +## Generation and composition + +`Cff` normally flattens control flow throughout its input AST. When it receives +the AST emitted by `Vmify`, a private `vmRuntime` annotation selects a special +path: `vm/flatten.lua` flattens the interpreter function only. The large native +prototype table initialization remains outside the flattened function, keeping +it out of the Cff control structure and jump limits used by Lua and LuaJIT. + +The special path passes read-only external lexical helpers through function +arguments. This supports helpers introduced by intervening steps such as +`EncryptStrings`. If the interpreter writes to an external lexical variable, +the special path falls back to ordinary Cff compilation to preserve that +write's behavior. + +`runtime_dispatch.lua` builds a balanced numeric `if`/`else` decision tree from +the opcode IDs. The interpreter caches the current opcode and checks for an +exact ID at each leaf before running that instruction's handler. + +`Strong` applies one `Vmify` pass followed by `EncryptStrings`, `AntiTamper`, +and one `Cff` pass over the generated VM interpreter. `Weak` retains the +standalone `Cff` transformation without the VM. + +Runtime compatibility follows the documented +[limitations](../reference/limitations.md). Native debug representation and +exact resource limits are not preserved. diff --git a/doc/guides/presets.md b/doc/guides/presets.md index 9208c8e5..ff4e8804 100644 --- a/doc/guides/presets.md +++ b/doc/guides/presets.md @@ -4,11 +4,12 @@ Prometheus ships with these preset names in `src/presets.lua`: - `Minify` - `Weak` -- `Cff` (test-oriented helper preset) +- `Cff` (standalone control flow flattening) +- `Vmify` (standalone register VM) - `Medium` - `Strong` -Use with: +Use a preset with: ```bash prometheus-lua --preset Medium ./file.lua @@ -28,13 +29,21 @@ Steps: 2. `ConstantArray` 3. `WrapInFunction` +## Cff + +This test-oriented preset runs one `Cff` control flow flattening step. + +## Vmify + +This test-oriented preset compiles the input to the register VM with one `Vmify` step. + ## Medium Steps: 1. `EncryptStrings` 2. `AntiTamper` (`UseDebug = false`) -3. `Cff` +3. `Vmify` 4. `ConstantArray` 5. `NumbersToExpressions` 6. `WrapInFunction` @@ -43,7 +52,7 @@ Steps: Steps: -1. `Cff` +1. `Vmify` 2. `EncryptStrings` 3. `AntiTamper` (`UseDebug = false`) 4. `Cff` @@ -51,10 +60,11 @@ Steps: 6. `NumbersToExpressions` 7. `WrapInFunction` +In `Strong`, `Vmify` compiles the input first, then `Cff` flattens control flow in the generated VM Lua code. This preset contains one VM pass. + ## Choosing quickly - Start with `Medium`. - Move to `Strong` only after benchmarking your runtime path. - Use `Minify` when you want readability reduction only through compression/renaming style behavior. - -Preset compositions retain their current behavior in this milestone, with former `Vmify` steps renamed to `Cff`; the test preset is now named `Cff`. `Strong` therefore has two `Cff` passes for now. Its planned future sequence is one real `Vmify` pass followed by `Cff`; no real VM is available in this milestone. +- Use `Cff` and `Vmify` to exercise either step on its own in a custom pipeline. diff --git a/doc/guides/writing-a-custom-config.md b/doc/guides/writing-a-custom-config.md index 0a9f6417..b183b6cc 100644 --- a/doc/guides/writing-a-custom-config.md +++ b/doc/guides/writing-a-custom-config.md @@ -29,10 +29,9 @@ return { `Steps` are applied in order. The same step can appear multiple times. -For existing custom configs, replace `Name = "Vmify"` with `Name = "Cff"`. -The `ControlFlowFlattening` constructor key is also accepted. `Vmify` is -reserved for a future real VM step and is not registered in this milestone. -Rename the `Vmify` preset to `Cff` in CLI commands as well. +`Cff` performs control flow flattening. `ControlFlowFlattening` is also accepted as an alias for `Cff`. `Vmify` compiles the AST into register instructions and data tables, then emits Lua code with an interpreter for that data. + +If an existing config used `Name = "Vmify"` for the former control flow flattening behavior, change it to `Name = "Cff"` to keep that behavior. The same `Vmify` key now selects the register VM. The standalone `Cff` and `Vmify` presets are available with `--preset Cff` and `--preset Vmify`. ## Name generator values diff --git a/doc/reference/steps/cff.md b/doc/reference/steps/cff.md index 9009cdd5..11db4ccf 100644 --- a/doc/reference/steps/cff.md +++ b/doc/reference/steps/cff.md @@ -2,7 +2,9 @@ Constructor key: `Cff`; `ControlFlowFlattening` is also accepted. -`Cff` is the existing control-flow flattening step. `Vmify` is reserved for a future real VM step and is not registered in this milestone. +`Cff` flattens control flow into Lua statements executed by a block dispatcher. It remains available alongside the register VM step, `Vmify`. + +When `Cff` receives `Vmify` output, it recognizes the private VM runtime annotation and flattens the interpreter function only. The prototype tables remain outside the flattened function so large data initializers do not expand the control flow graph beyond Lua or LuaJIT limits. Read-only external lexical helpers are forwarded as function arguments; if the interpreter writes to an external lexical variable, `Cff` uses its ordinary compilation path instead. ## Settings @@ -13,3 +15,5 @@ None. ```lua { Name = "Cff", Settings = {} } ``` + +Use `Cff` when you want the control flow flattening behavior. Existing configs that used the old `Vmify` key for this behavior should use `Cff` now. The `Vmify` key compiles code to the register VM. diff --git a/doc/reference/steps/overview.md b/doc/reference/steps/overview.md index 386314c9..cb13b6b5 100644 --- a/doc/reference/steps/overview.md +++ b/doc/reference/steps/overview.md @@ -1,16 +1,15 @@ # Step Pipeline Overview -Prometheus applies each configured step to the AST in the order listed in `Steps`. - -After step application, Prometheus performs variable renaming and then unparses code. +Prometheus applies configured steps in the order listed in `Steps`. A step can return a new AST for the following steps to process. After the steps run, Prometheus performs variable renaming and unparses the code. ## Registered step constructor keys -Use one of these constructor keys in `Steps[i].Name`: +Use these constructor keys in `Steps[i].Name`: - `WrapInFunction` - `SplitStrings` - `Cff` (also accepted as `ControlFlowFlattening`) +- `Vmify` - `ConstantArray` - `ProxifyLocals` - `AntiTamper` @@ -23,15 +22,14 @@ Use one of these constructor keys in `Steps[i].Name`: ```lua { - Name = "Cff", + Name = "Vmify", Settings = {} } ``` ## Notes -- Some constructor keys differ from internal human-readable names. -- `Cff` is the existing control-flow flattening step and has no settings. `ControlFlowFlattening` is also accepted as a constructor key alias. -- `Vmify` is reserved for a future real VM step and is not registered in this milestone. +- `Cff` flattens control flow into Lua statements dispatched through blocks. It has no settings; `ControlFlowFlattening` is accepted as an alias. +- `Vmify` compiles the AST into register instructions and data tables, then emits Lua code with an interpreter for that data. It has no settings. +- `Vmify` and `Cff` are separate steps and may both appear in a pipeline. Later steps receive the AST returned by the previous step. - You can apply the same step more than once. -- If a step returns a new AST, that AST is used for later steps. diff --git a/doc/reference/steps/vmify.md b/doc/reference/steps/vmify.md new file mode 100644 index 00000000..6a2f3596 --- /dev/null +++ b/doc/reference/steps/vmify.md @@ -0,0 +1,17 @@ +# Step: Vmify + +Constructor key: `Vmify`. + +`Vmify` compiles the AST into per-function register instructions, constants, and prototype data, then emits Lua code with an interpreter that executes that data. This is the table-backed register VM step. It compiles the AST directly; it does not first apply `Cff`. + +## Settings + +None. + +## Example + +```lua +{ Name = "Vmify", Settings = {} } +``` + +`Vmify` is a different step from `Cff`, which performs control flow flattening. If an older config used `Vmify` to select the former control flow flattening behavior, change that step to `Cff`. The `Vmify` name now selects the register VM. diff --git a/src/presets.lua b/src/presets.lua index 92009838..3c512ed3 100644 --- a/src/presets.lua +++ b/src/presets.lua @@ -35,8 +35,7 @@ return { }, }, - -- This is here for the tests.lua file. - -- It helps isolate any problems with the Cff step. + -- Standalone control-flow flattening for tests and custom pipelines. -- It is not recommended to use this preset for obfuscation. -- Use the Weak, Medium, or Strong for obfuscation instead. ["Cff"] = { @@ -50,6 +49,18 @@ return { }, }, + -- Standalone register VM for tests and custom pipelines. + ["Vmify"] = { + LuaVersion = "Lua51", + VarNamePrefix = "", + NameGenerator = "MangledShuffled", + PrettyPrint = false, + Seed = 0, + Steps = { + { Name = "Vmify", Settings = {} }, + }, + }, + -- Medium obfuscation. Moderate obfuscation, moderate performance loss. ["Medium"] = { LuaVersion = "Lua51", @@ -65,7 +76,7 @@ return { UseDebug = false, }, }, - { Name = "Cff", Settings = {} }, + { Name = "Vmify", Settings = {} }, { Name = "ConstantArray", Settings = { @@ -89,7 +100,7 @@ return { PrettyPrint = false, Seed = 0, Steps = { - { Name = "Cff", Settings = {} }, + { Name = "Vmify", Settings = {} }, { Name = "EncryptStrings", Settings = {} }, { Name = "AntiTamper", diff --git a/src/prometheus/steps.lua b/src/prometheus/steps.lua index d7d727fe..ce45aa37 100644 --- a/src/prometheus/steps.lua +++ b/src/prometheus/steps.lua @@ -11,6 +11,7 @@ return { SplitStrings = require("prometheus.steps.SplitStrings"), Cff = Cff, ControlFlowFlattening = Cff, + Vmify = require("prometheus.steps.Vmify"), ConstantArray = require("prometheus.steps.ConstantArray"), ProxifyLocals = require("prometheus.steps.ProxifyLocals"), AntiTamper = require("prometheus.steps.AntiTamper"), diff --git a/src/prometheus/steps/Cff.lua b/src/prometheus/steps/Cff.lua index e6c69094..b9c2cf5e 100644 --- a/src/prometheus/steps/Cff.lua +++ b/src/prometheus/steps/Cff.lua @@ -6,6 +6,7 @@ local Step = require("prometheus.step"); local Compiler = require("prometheus.cff.compiler"); +local flattenVm = require("prometheus.vm.flatten"); local Cff = Step:extend(); Cff.Description = "Flattens control flow into Lua statements executed by a randomized block dispatcher."; @@ -16,6 +17,10 @@ Cff.SettingsDescriptor = {} function Cff:init(_) end function Cff:apply(ast) + if ast.vmRuntime then + local flattened = flattenVm(ast); + if flattened then return flattened; end + end local compiler = Compiler:new(); return compiler:compile(ast); diff --git a/src/prometheus/steps/Vmify.lua b/src/prometheus/steps/Vmify.lua new file mode 100644 index 00000000..05a14cd4 --- /dev/null +++ b/src/prometheus/steps/Vmify.lua @@ -0,0 +1,17 @@ +-- Compile Lua AST into register instructions and a generated Lua interpreter. +local Step = require('prometheus.step') +local Compiler = require('prometheus.vm.compiler') +local Emit = require('prometheus.vm.emit') + +local Vmify = Step:extend() +Vmify.Name = 'Vmify' +Vmify.Description = 'Compiles functions into register instructions executed by a generated Lua VM.' +Vmify.SettingsDescriptor = {} + +function Vmify:init() end + +function Vmify:apply(ast) + return Emit.ast(Compiler:compile(ast)) +end + +return Vmify diff --git a/src/prometheus/vm/analysis.lua b/src/prometheus/vm/analysis.lua new file mode 100644 index 00000000..6cc13922 --- /dev/null +++ b/src/prometheus/vm/analysis.lua @@ -0,0 +1,23 @@ +-- Identify each lexical scope's function and locals shared with nested closures. +local visitast = require('prometheus.visitast') + +return function(ast) + local analysis = {owners = {}, captured = {}} + visitast(ast, function(node, data) + if node.kind == 'Block' then + analysis.owners[node.scope] = data.functionData.scope + end + end) + visitast(ast, function(node, data) + local kind = node.kind + if kind == 'VariableExpression' or kind == 'AssignmentVariable' + or kind == 'FunctionDeclaration' then + local scope = node.scope + if not scope.isGlobal and analysis.owners[scope] ~= data.functionData.scope then + analysis.captured[scope] = analysis.captured[scope] or {} + analysis.captured[scope][node.id] = true + end + end + end) + return analysis +end diff --git a/src/prometheus/vm/compiler.lua b/src/prometheus/vm/compiler.lua new file mode 100644 index 00000000..416a6785 --- /dev/null +++ b/src/prometheus/vm/compiler.lua @@ -0,0 +1,167 @@ +-- Lower scoped AST nodes into function prototypes containing semantic instructions. +local analyze = require('prometheus.vm.analysis') +local opcodes = require('prometheus.vm.opcodes') +local Compiler = {} +Compiler.__index = Compiler + +function Compiler:new(analysis, node, parent) + local compiler = setmetatable({ + analysis = analysis, owner = node.body.scope, parent = parent, + proto = {code = {}, constants = {}, children = {}, captures = {}, + params = {}, vararg = node.kind == 'TopNode', maxRegister = 0}, + locals = {}, upvalues = {}, constantIds = {}, registers = {}, freeRegisters = {}, + scopeStack = {}, loops = {}, + }, self) + for _, arg in ipairs(node.args or {}) do + if arg.kind == 'VarargExpression' then + compiler.proto.vararg = true + else + compiler.proto.params[#compiler.proto.params + 1] = { + compiler:localRegister(arg.scope, arg.id), + compiler:isCaptured(arg.scope, arg.id), + } + end + end + return compiler +end + +function Compiler:emit(op, ...) + assert(opcodes[op], 'Unknown VM instruction: ' .. tostring(op)) + local code = self.proto.code + code[#code + 1] = {opcodes[op], ...} + return #code +end + +function Compiler:position() + return #self.proto.code + 1 +end + +function Compiler:patch(index, operand, value) + self.proto.code[index][operand + 1] = value +end + +function Compiler:alloc() + local register = table.remove(self.freeRegisters) + if not register then + register = self.proto.maxRegister + 1 + self.proto.maxRegister = register + end + assert(not self.registers[register], 'Register already allocated') + self.registers[register] = 'temporary' + return register +end + +function Compiler:free(register) + assert(self.registers[register] == 'temporary', 'Cannot release register ' .. tostring(register)) + self:emit('CLEAR', register) + self.registers[register] = nil + self.freeRegisters[#self.freeRegisters + 1] = register +end + +function Compiler:localRegister(scope, id) + assert(self.analysis.owners[scope] == self.owner, 'Local belongs to another function') + self.locals[scope] = self.locals[scope] or {} + if not self.locals[scope][id] then + local register = self:alloc() + self.registers[register] = 'local' + self.locals[scope][id] = register + end + return self.locals[scope][id] +end + +function Compiler:isCaptured(scope, id) + return not not (self.analysis.captured[scope] and self.analysis.captured[scope][id]) +end + +function Compiler:constant(value) + assert(value ~= nil, 'Nil is represented by its own instruction') + local ids = self.constantIds[type(value)] + if not ids then ids = {}; self.constantIds[type(value)] = ids end + -- NaN cannot be a table key; signed zero must not collapse into positive zero. + local key = value + if type(value) == 'number' and value == 0 then + key = 1 / value < 0 and 'negative-zero' or 'positive-zero' + end + if value == value and ids[key] then return ids[key] end + local index = #self.proto.constants + 1 + self.proto.constants[index] = value + if value == value then ids[key] = index end + return index +end + +function Compiler:load(value) + local register = self:alloc() + if value == nil then self:emit('NIL', register) + else self:emit('LOAD', register, self:constant(value)) end + return register +end + +function Compiler:capture(scope, id) + self.upvalues[scope] = self.upvalues[scope] or {} + local index = self.upvalues[scope][id] + if index then return index end + assert(self.parent, 'Unresolved root upvalue') + local description + if self.analysis.owners[scope] == self.parent.owner then + assert(self:isCaptured(scope, id), 'Captured local missing analysis') + description = {0, self.parent:localRegister(scope, id)} + else + description = {1, self.parent:capture(scope, id)} + end + index = #self.proto.captures + 1 + self.proto.captures[index] = description + self.upvalues[scope][id] = index + return index +end + +function Compiler:readVariable(scope, id) + local register = self:alloc() + if scope.isGlobal then + self:emit('GLOBAL', register, self:constant(scope:getVariableName(id))) + elseif self.analysis.owners[scope] == self.owner then + self:emit(self:isCaptured(scope, id) and 'GETCELL' or 'MOVE', + register, self:localRegister(scope, id)) + else self:emit('GETUP', register, self:capture(scope, id)) end + return register +end + +function Compiler:writeVariable(scope, id, source) + if scope.isGlobal then + self:emit('SETGLOBAL', self:constant(scope:getVariableName(id)), source) + elseif self.analysis.owners[scope] == self.owner then + self:emit(self:isCaptured(scope, id) and 'SETCELL' or 'MOVE', + self:localRegister(scope, id), source) + else self:emit('SETUP', self:capture(scope, id), source) end +end + +function Compiler:initVariable(scope, id, source) + self:emit(self:isCaptured(scope, id) and 'CELL' or 'MOVE', + self:localRegister(scope, id), source or 0) +end + +function Compiler:finish(node) + self:block(node.body) + local empty = self:alloc() + self:emit('ARGS', empty, {}, 0) + self:emit('RETURN', empty) + return self.proto +end + +function Compiler:functionLiteral(node) + local child = Compiler:new(self.analysis, node, self) + local proto = child:finish(node) + local children = self.proto.children + children[#children + 1] = proto + local register = self:alloc() + self:emit('CLOSURE', register, #children) + return register +end + +function Compiler:compile(ast) + local compiler = Compiler:new(analyze(ast), ast) + return compiler:finish(ast) +end + +require('prometheus.vm.expressions')(Compiler) +require('prometheus.vm.statements')(Compiler) +return Compiler diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua new file mode 100644 index 00000000..c3469fd1 --- /dev/null +++ b/src/prometheus/vm/emit.lua @@ -0,0 +1,41 @@ +-- Emit native table data and a generated interpreter, then parse the complete AST. +local Ast = require('prometheus.ast') +local Parser = require('prometheus.parser') +local Unparser = require('prometheus.unparser') +local Enums = require('prometheus.enums') +local Runtime = require('prometheus.vm.runtime') +local Emit = {} + +function Emit.source(proto) + local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local function literal(value) + if type(value) ~= 'table' then + return unparser:unparseExpression(Ast.ConstantNode(value), 0) + end + local parts = {} + for _, entry in ipairs(value) do parts[#parts + 1] = literal(entry) end + return '{' .. table.concat(parts, ',') .. '}' + end + local function prototype(value) + local parts = {} + for _, key in ipairs({'code', 'constants', 'captures', 'params', 'vararg', 'maxRegister'}) do + parts[#parts + 1] = key .. '=' .. literal(value[key]) + end + local children = {} + for _, child in ipairs(value.children) do children[#children + 1] = prototype(child) end + parts[#parts + 1] = 'children={' .. table.concat(children, ',') .. '}' + return '{' .. table.concat(parts, ',') .. '}' + end + local data = '{' .. prototype(proto) .. '}' + local source, count = Runtime.build():gsub('__PROTOTYPES__', function() return data end) + assert(count == 1, 'Runtime template must contain one prototype marker') + return source +end + +function Emit.ast(proto) + local ast = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(Emit.source(proto)) + ast.vmRuntime = ast.body.statements[1].args[1].base + return ast +end + +return Emit diff --git a/src/prometheus/vm/expressions.lua b/src/prometheus/vm/expressions.lua new file mode 100644 index 00000000..7f13ced7 --- /dev/null +++ b/src/prometheus/vm/expressions.lua @@ -0,0 +1,257 @@ +local Ast = require("prometheus.ast") +local Kind = Ast.AstKind +local compileTable = require("prometheus.vm.expressions_tables") + +local binary = { + [Kind.AddExpression] = "ADD", [Kind.SubExpression] = "SUB", + [Kind.MulExpression] = "MUL", [Kind.DivExpression] = "DIV", + [Kind.ModExpression] = "MOD", [Kind.PowExpression] = "POW", + [Kind.StrCatExpression] = "CONCAT", [Kind.LessThanExpression] = "LT", + [Kind.LessThanOrEqualsExpression] = "LE", [Kind.GreaterThanExpression] = "GT", + [Kind.GreaterThanOrEqualsExpression] = "GE", [Kind.EqualsExpression] = "EQ", + [Kind.NotEqualsExpression] = "NE", +} +local unary = { + [Kind.NotExpression] = "NOT", [Kind.NegateExpression] = "NEG", + [Kind.LenExpression] = "LEN", +} + +local function install(Compiler) + local function load(self, value) + return self:load(value) + end + + local function numberValue(value) + if type(value) ~= "string" then return value end + local prefix = value:sub(1, 2):lower() + if prefix == "0b" then + local result = 0 + for i = 3, #value do + local digit = value:sub(i, i) + assert(digit == "0" or digit == "1", "invalid binary number: " .. value) + result = result * 2 + (digit == "1" and 1 or 0) + end + return result + end + local number = tonumber(value) + assert(number ~= nil, "invalid numeric literal: " .. value) + return number + end + + local function packed(self, regs, tail) + local result = self:alloc() + self:emit("ARGS", result, regs, tail or 0) + return result + end + + local function expandable(node) + if not node or node.isParenthesizedExpression then return false end + return node.kind == Kind.FunctionCallExpression + or node.kind == Kind.PassSelfFunctionCallExpression + or node.kind == Kind.VarargExpression + end + + local function packList(self, expressions, prefix) + local regs = {} + for i, reg in ipairs(prefix or {}) do regs[i] = reg end + local tail = 0 + for i, node in ipairs(expressions) do + if i == #expressions and expandable(node) then + if node.kind == Kind.VarargExpression then + tail = self:alloc() + self:emit("VARARG", tail) + else + tail = self:call(node, -1)[1] + end + else + regs[#regs + 1] = self:expression(node, 1)[1] + end + end + local result = packed(self, regs, tail) + for _, reg in ipairs(regs) do self:free(reg) end + if tail ~= 0 then self:free(tail) end + return result + end + + local function scalar(self, node) + local kind = node.kind + if kind == Kind.NilExpression then + local reg = self:alloc(); self:emit("NIL", reg); return reg + elseif kind == Kind.NumberExpression then + return load(self, numberValue(node.value)) + elseif kind == Kind.BooleanExpression or kind == Kind.StringExpression then + return load(self, node.value) + elseif kind == Kind.VariableExpression then + return self:readVariable(node.scope, node.id) + elseif kind == Kind.VarargExpression then + local tuple = self:alloc(); self:emit("VARARG", tuple) + local reg = self:alloc(); self:emit("TAKE", reg, tuple, 1) + self:free(tuple); return reg + elseif kind == Kind.IndexExpression then + local base = self:expression(node.base, 1)[1] + local key = self:expression(node.index, 1)[1] + local result = self:alloc(); self:emit("GETTABLE", result, base, key) + self:free(key); self:free(base); return result + elseif binary[kind] then + local left = self:expression(node.lhs, 1)[1] + local right = self:expression(node.rhs, 1)[1] + local result = self:alloc(); self:emit(binary[kind], result, left, right) + self:free(right); self:free(left); return result + elseif unary[kind] then + local source = self:expression(node.rhs, 1)[1] + local result = self:alloc(); self:emit(unary[kind], result, source) + self:free(source); return result + elseif kind == Kind.AndExpression or kind == Kind.OrExpression then + local left = self:expression(node.lhs, 1)[1] + local result = self:alloc(); self:emit("MOVE", result, left) + local branch = self:emit(kind == Kind.AndExpression and "JFALSE" or "JTRUE", left, 0) + self:free(left) + local right = self:expression(node.rhs, 1)[1] + self:emit("MOVE", result, right); self:free(right) + local endJump = self:emit("JMP", 0) + local skipped = self:position() + self:emit("CLEAR", left) + local finish = self:position() + self:patch(branch, 2, skipped) + self:patch(endJump, 1, finish) + return result + elseif kind == Kind.IfElseExpression then + local result = self:alloc() + local endJumps = {} + local condition = self:expression(node.condition, 1)[1] + local nextBranch = self:emit("JFALSE", condition, 0); self:free(condition) + local value = self:expression(node.true_value, 1)[1] + self:emit("MOVE", result, value); self:free(value) + endJumps[#endJumps + 1] = self:emit("JMP", 0) + local falseStart = self:position() + self:patch(nextBranch, 2, falseStart) + self:emit("CLEAR", condition) + for _, clause in ipairs(node.elseifs or {}) do + condition = self:expression(clause.condition, 1)[1] + nextBranch = self:emit("JFALSE", condition, 0); self:free(condition) + value = self:expression(clause.value, 1)[1] + self:emit("MOVE", result, value); self:free(value) + endJumps[#endJumps + 1] = self:emit("JMP", 0) + falseStart = self:position() + self:patch(nextBranch, 2, falseStart) + self:emit("CLEAR", condition) + end + value = self:expression(node.false_value, 1)[1] + self:emit("MOVE", result, value); self:free(value) + local finish = self:position() + for _, jump in ipairs(endJumps) do self:patch(jump, 1, finish) end + return result + elseif kind == Kind.FunctionLiteralExpression then + return self:functionLiteral(node) + elseif kind == Kind.TableConstructorExpression then + return compileTable(self, node) + elseif kind == Kind.FunctionCallExpression or kind == Kind.PassSelfFunctionCallExpression then + return self:call(node, 1)[1] + end + error("unsupported expression kind: " .. tostring(kind)) + end + + function Compiler:expression(node, count) + count = count == nil and 1 or count + local forceScalarPack = count == -1 and node.isParenthesizedExpression + if forceScalarPack then count = 1 end + if count == 0 then + if node.kind == Kind.FunctionCallExpression or node.kind == Kind.PassSelfFunctionCallExpression then + self:call(node, 0) + else + self:free(scalar(self, node)) + end + return {} + end + if node.kind == Kind.FunctionCallExpression or node.kind == Kind.PassSelfFunctionCallExpression then + local results = self:call(node, count) + if forceScalarPack then + local tuple = packed(self, {results[1]}, 0) + self:free(results[1]) + return {tuple} + end + return results + end + if node.kind == Kind.VarargExpression and count ~= 1 then + local tuple = self:alloc(); self:emit("VARARG", tuple) + if count == -1 then return {tuple} end + local results = {} + for i = 1, count do + results[i] = self:alloc(); self:emit("TAKE", results[i], tuple, i) + end + self:free(tuple); return results + end + local value = scalar(self, node) + if forceScalarPack then + local tuple = packed(self, {value}, 0) + self:free(value) + return {tuple} + end + if count == -1 then + local tuple = packed(self, {value}, 0) + self:free(value) + return {tuple} + end + local results = {value} + while #results < count do + local nilReg = self:alloc(); self:emit("NIL", nilReg) + results[#results + 1] = nilReg + end + return results + end + + function Compiler:list(expressions, count) + expressions = expressions or {} + if count == 0 then + for _, node in ipairs(expressions) do self:expression(node, 0) end + return {} + end + local tuple = packList(self, expressions) + if count == nil then return {tuple} end + local results = {} + for i = 1, count do + results[i] = self:alloc(); self:emit("TAKE", results[i], tuple, i) + end + self:free(tuple) + return results + end + + function Compiler:call(node, count, tail) + local fn, prefix + if node.kind == Kind.PassSelfFunctionCallExpression then + local object = self:expression(node.base, 1)[1] + local key = load(self, node.passSelfFunctionName) + fn = self:alloc(); self:emit("GETTABLE", fn, object, key) + self:free(key) + prefix = {object} + else + fn = self:expression(node.base, 1)[1] + end + local args = packList(self, node.args, prefix) + if tail then + self:emit("TAILCALL", fn, args) + self:free(args); self:free(fn) + return {} + end + if count == 0 then + self:emit("CALL", 0, fn, args, 0) + self:free(args); self:free(fn); return {} + elseif count == -1 then + local result = self:alloc(); self:emit("CALL", result, fn, args, -1) + self:free(args); self:free(fn); return {result} + elseif count == 1 then + local result = self:alloc(); self:emit("CALL", result, fn, args, 1) + self:free(args); self:free(fn); return {result} + else + local tuple = self:alloc(); self:emit("CALL", tuple, fn, args, -1) + self:free(args); self:free(fn) + local results = {} + for i = 1, count do + results[i] = self:alloc(); self:emit("TAKE", results[i], tuple, i) + end + self:free(tuple); return results + end + end +end + +return install diff --git a/src/prometheus/vm/expressions_tables.lua b/src/prometheus/vm/expressions_tables.lua new file mode 100644 index 00000000..de8b3a9c --- /dev/null +++ b/src/prometheus/vm/expressions_tables.lua @@ -0,0 +1,68 @@ +local Ast = require("prometheus.ast") +local Kind = Ast.AstKind + +local function expandable(node) + return node and not node.isParenthesizedExpression and + (node.kind == Kind.FunctionCallExpression or + node.kind == Kind.PassSelfFunctionCallExpression or + node.kind == Kind.VarargExpression) +end + +local function packed(self, registers, tail) + local snapshot = {} + for i, register in ipairs(registers) do snapshot[i] = register end + local tuple = self:alloc() + self:emit("ARGS", tuple, snapshot, tail or 0) + return tuple +end + +local function flush(self, tableReg, registers, firstIndex) + if #registers == 0 then return end + local tuple = packed(self, registers, 0) + self:emit("EXPAND", tableReg, firstIndex, tuple) + self:free(tuple) + for _, register in ipairs(registers) do self:free(register) end + for i = #registers, 1, -1 do registers[i] = nil end +end + +return function(self, node) + local tableReg = self:alloc() + self:emit("TABLE", tableReg) + local arrayIndex, batchStart, batch = 1, 1, {} + for index, entry in ipairs(node.entries) do + if entry.kind == Kind.TableEntry then + if index == #node.entries and expandable(entry.value) then + local tail + if entry.value.kind == Kind.VarargExpression then + tail = self:alloc() + self:emit("VARARG", tail) + else + tail = self:call(entry.value, -1)[1] + end + local tuple = packed(self, batch, tail) + self:emit("EXPAND", tableReg, batchStart, tuple) + self:free(tuple) + for _, register in ipairs(batch) do self:free(register) end + for i = #batch, 1, -1 do batch[i] = nil end + self:free(tail) + else + local value = self:expression(entry.value, 1)[1] + batch[#batch + 1] = value + arrayIndex = arrayIndex + 1 + if #batch == 1 then batchStart = arrayIndex - 1 end + if #batch == 50 then + flush(self, tableReg, batch, batchStart) + batchStart = arrayIndex + end + end + else + local key = self:expression(entry.key, 1)[1] + local value = self:expression(entry.value, 1)[1] + self:emit("SETTABLE", tableReg, key, value) + self:free(value) + self:free(key) + end + end + flush(self, tableReg, batch, batchStart) + return tableReg +end diff --git a/src/prometheus/vm/flatten.lua b/src/prometheus/vm/flatten.lua new file mode 100644 index 00000000..c8a5b1c7 --- /dev/null +++ b/src/prometheus/vm/flatten.lua @@ -0,0 +1,66 @@ +-- Flatten a generated interpreter without virtualizing its large data initializer. +local Ast = require('prometheus.ast') +local Scope = require('prometheus.scope') +local visitast = require('prometheus.visitast') +local Parser = require('prometheus.parser') +local Unparser = require('prometheus.unparser') +local CffCompiler = require('prometheus.cff.compiler') +local Enums = require('prometheus.enums') + +local function externalVariables(ast, runtime) + local owned, variables, seen = {}, {}, {} + local wrapper = Ast.TopNode(Ast.Block({Ast.ReturnStatement({runtime})}, + Scope:new(ast.globalScope)), ast.globalScope) + visitast(wrapper, function(node) + if node.kind == 'Block' then owned[node.scope] = true end + end) + local writes = false + visitast(wrapper, function(node) + local kind = node.kind + if kind == 'VariableExpression' or kind == 'AssignmentVariable' + or kind == 'FunctionDeclaration' then + local scope = node.scope + if not scope.isGlobal and not owned[scope] then + if kind == 'AssignmentVariable' + or (kind == 'FunctionDeclaration' and #node.indices == 0) then + writes = true + end + seen[scope] = seen[scope] or {} + if not seen[scope][node.id] then + variables[#variables + 1] = scope:getVariableName(node.id) + seen[scope][node.id] = true + end + end + end + end) + -- A value prefix preserves shared tables/functions, but cannot forward writes + -- to an outer lexical local. Use the ordinary compiler in that uncommon case. + return variables, writes +end + +return function(ast) + local runtime = ast.vmRuntime + local variables, writes = externalVariables(ast, runtime) + if writes then return nil end + local parser = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local runtimeSource = unparser:unparseExpression(runtime, 0) + local prefix = '' + local arguments = '...' + if #variables > 0 then + prefix = 'local ' .. table.concat(variables, ',') .. '=...;' + arguments = 'select(' .. (#variables + 1) .. ',...)' + end + local standalone = parser:parse(prefix .. 'return (' .. runtimeSource .. ')(' .. arguments .. ')') + local flattened = unparser:unparse(CffCompiler:new():compile(standalone)) + local forwarded = #variables > 0 and table.concat(variables, ',') .. ',...' or '...' + local replacementSource = 'return function(...) return (function(...) ' + .. flattened .. ' end)(' .. forwarded .. ') end' + local replacement = parser:parse(replacementSource).body.statements[1].args[1] + visitast(ast, function(node) + if node == runtime then return replacement, true end + end) + -- Reparsing resolves injected free-variable names against their original outer + -- scopes, including helpers installed between Vmify and Cff by EncryptStrings. + return parser:parse(unparser:unparse(ast)) +end diff --git a/src/prometheus/vm/opcodes.lua b/src/prometheus/vm/opcodes.lua new file mode 100644 index 00000000..9b79116e --- /dev/null +++ b/src/prometheus/vm/opcodes.lua @@ -0,0 +1,13 @@ +-- Semantic instruction IDs for the baseline VM. Layout mutation is a later pass. +local names = { + 'LOAD', 'NIL', 'MOVE', 'CLEAR', 'CELL', 'GETCELL', 'SETCELL', + 'GETUP', 'SETUP', 'GLOBAL', 'SETGLOBAL', 'GETTABLE', 'SETTABLE', + 'TABLE', 'EXPAND', 'ARGS', 'TAKE', 'VARARG', 'CALL', 'RETURN', + 'TAILCALL', 'CLOSURE', 'JMP', 'JFALSE', 'JTRUE', + 'ADD', 'SUB', 'MUL', 'DIV', 'MOD', 'POW', 'CONCAT', + 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'NOT', 'NEG', 'LEN', + 'FORPREP', 'FORCHECK', +} +local opcodes = {} +for id, name in ipairs(names) do opcodes[name] = id end +return opcodes diff --git a/src/prometheus/vm/runtime.lua b/src/prometheus/vm/runtime.lua new file mode 100644 index 00000000..3637c66b --- /dev/null +++ b/src/prometheus/vm/runtime.lua @@ -0,0 +1,91 @@ +-- Builds the Lua 5.1 runtime used by the bytecode emitter. The emitter replaces +-- __PROTOTYPES__ with its native Lua table literal before parsing the source. +local dispatch = require("prometheus.vm.runtime_dispatch") +local opcodes = require("prometheus.vm.opcodes") + +local Runtime = {} + +-- Do not query `getfenv(wrapper)` from the wrapper body: that self-reference +-- forms a closure cycle that Cff can keep alive after the wrapper is discarded. +local template = [=[ +return (function(prototypes, ...) +local nativeGetfenv, nativeSetfenv = getfenv, setfenv +local nativeUnpack = unpack or table.unpack +local nativeTonumber, nativeError = tonumber, error +local nativeSelect, nativeIpairs = select, ipairs + +local function pack(...) + return { n = nativeSelect("#", ...), ... } +end + +local make + +local function loop(P, U, Env, R, V) + local code, K = P.code, P.constants + local pc = 1 + local I, opcode, a, b, c, d, x, y, z + while true do + I = code[pc] + pc = pc + 1 + opcode = I[1] + a, b, c, d = I[2], I[3], I[4], I[5] + __DISPATCH__ + I, opcode, a, b, c, d, x, y, z = nil, nil, nil, nil, nil, nil, nil, nil, nil + end +end + +local function start(P, U, Env, ...) + local R, incoming = {}, pack(...) + local params = P.params + for i = 1, #params do + local param = params[i] + local value = incoming[i] + if param[2] then + R[param[1]] = { value } + else + R[param[1]] = value + end + end + local V = { n = 0 } + if P.vararg then + local first = #params + 1 + if incoming.n > #params then + V.n = incoming.n - #params + end + for i = 1, V.n do + V[i] = incoming[first + i - 1] + end + end + incoming = nil + return loop(P, U, Env, R, V) +end + +make = function(P, U, inheritedEnv) + local wrapper + if nativeGetfenv and nativeSetfenv then + wrapper = function(...) + local Env = nativeGetfenv(1) + return start(P, U, Env, ...) + end + nativeSetfenv(wrapper, inheritedEnv) + else + wrapper = function(...) + return start(P, U, inheritedEnv, ...) + end + end + return wrapper +end + +local entryEnv = nativeGetfenv and nativeGetfenv(1) or _ENV or _G +return make(prototypes[1], {}, entryEnv)(...) +end)(__PROTOTYPES__, ...) +]=] + +function Runtime.build() + local source = template:gsub("__DISPATCH__", function() + return dispatch.build(opcodes) + end) + return source +end + +return Runtime diff --git a/src/prometheus/vm/runtime_dispatch.lua b/src/prometheus/vm/runtime_dispatch.lua new file mode 100644 index 00000000..e1b46daa --- /dev/null +++ b/src/prometheus/vm/runtime_dispatch.lua @@ -0,0 +1,163 @@ +-- Opcode bodies are kept separate from runtime.lua so both files stay small. +local M = {} + +local bodies = { + LOAD = [==[R[a] = K[b]]==], + NIL = [==[R[a] = nil]==], + MOVE = [==[R[a] = R[b]]==], + CLEAR = [==[R[a] = nil]==], + CELL = [==[x = R[b]; R[a] = { x }]==], + GETCELL = [==[R[a] = R[b][1]]==], + SETCELL = [==[R[a][1] = R[b]]==], + GETUP = [==[R[a] = U[b][1]]==], + SETUP = [==[U[a][1] = R[b]]==], + GLOBAL = [==[R[a] = Env[K[b]]]==], + SETGLOBAL = [==[Env[K[a]] = R[b]]==], + GETTABLE = [==[R[a] = R[b][R[c]]]==], + SETTABLE = [==[R[a][R[b]] = R[c]]==], + TABLE = [==[R[a] = {}]==], + EXPAND = [==[ + x = R[c] + for y = 1, x.n do + R[a][b + y - 1] = x[y] + end + ]==], + ARGS = [==[ + x = { n = 0 } + for _, y in nativeIpairs(I[3]) do + x.n = x.n + 1 + x[x.n] = R[y] + end + if I[4] ~= 0 then + y = R[I[4]] + for d = 1, y.n do + x.n = x.n + 1 + x[x.n] = y[d] + end + end + R[a] = x + ]==], + TAKE = [==[R[a] = R[b][c]]==], + VARARG = [==[R[a] = V]==], + CALL = [==[ + x, y = R[b], R[c] + if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then + if d == -1 then + R[a] = { n = 1, Env } + elseif d == 1 then + R[a] = Env + end + elseif d == -1 then + R[a] = pack(x(nativeUnpack(y, 1, y.n))) + elseif d == 1 then + R[a] = x(nativeUnpack(y, 1, y.n)) + else + x(nativeUnpack(y, 1, y.n)) + end + ]==], + RETURN = [==[ + x = R[a] + return nativeUnpack(x, 1, x.n) + ]==], + TAILCALL = [==[ + x, y = R[a], R[b] + if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then + return Env + end + return x(nativeUnpack(y, 1, y.n)) + ]==], + CLOSURE = [==[ + x = P.children[b] + y = {} + for d = 1, #x.captures do + local capture = x.captures[d] + if capture[1] == 0 then + y[d] = R[capture[2]] + else + y[d] = U[capture[2]] + end + end + R[a] = make(x, y, Env) + ]==], + JMP = [==[pc = a]==], + JFALSE = [==[x = R[a]; R[a] = nil; if not x then pc = b end]==], + JTRUE = [==[x = R[a]; R[a] = nil; if x then pc = b end]==], + ADD = [==[R[a] = R[b] + R[c]]==], + SUB = [==[R[a] = R[b] - R[c]]==], + MUL = [==[R[a] = R[b] * R[c]]==], + DIV = [==[R[a] = R[b] / R[c]]==], + MOD = [==[R[a] = R[b] % R[c]]==], + POW = [==[R[a] = R[b] ^ R[c]]==], + CONCAT = [==[R[a] = R[b] .. R[c]]==], + LT = [==[R[a] = R[b] < R[c]]==], + LE = [==[R[a] = R[b] <= R[c]]==], + GT = [==[R[a] = R[b] > R[c]]==], + GE = [==[R[a] = R[b] >= R[c]]==], + EQ = [==[R[a] = R[b] == R[c]]==], + NE = [==[R[a] = R[b] ~= R[c]]==], + NOT = [==[R[a] = not R[b]]==], + NEG = [==[R[a] = -R[b]]==], + LEN = [==[R[a] = #R[b]]==], + FORPREP = [==[ + x = nativeTonumber(R[a]) + if x == nil then nativeError("'for' initial value must be a number", 0) end + R[a] = x + x = nativeTonumber(R[b]) + if x == nil then nativeError("'for' limit must be a number", 0) end + R[b] = x + x = nativeTonumber(R[c]) + if x == nil then nativeError("'for' step must be a number", 0) end + R[c] = x + ]==], + FORCHECK = [==[ + if R[d] ~= R[d] then + R[a] = false + elseif R[d] > 0 then + R[a] = R[b] <= R[c] + else + R[a] = R[b] >= R[c] + end + ]==], +} + +function M.build(opcodes) + local entries = {} + for name, id in pairs(opcodes) do + local body = bodies[name] + assert(body and type(id) == "number", "Missing VM handler for opcode " .. tostring(name)) + entries[#entries + 1] = { id = id, name = name, body = body } + end + table.sort(entries, function(left, right) return left.id < right.id end) + for i = 2, #entries do + assert(entries[i - 1].id < entries[i].id, "Duplicate VM opcode ID") + end + + local function indent(source, prefix) + return prefix .. source:gsub("\n", "\n" .. prefix) + end + + local function emit(first, last, depth) + local prefix = string.rep(" ", depth) + if first == last then + local entry = entries[first] + return prefix .. "if opcode == " .. tostring(entry.id) .. " then\n" + .. indent(entry.body, prefix .. " ") .. "\n" + .. prefix .. "else\n" + .. prefix .. " nativeError('unknown VM opcode: ' .. tostring(opcode), 0)\n" + .. prefix .. "end" + end + + local middle = math.floor((first + last) / 2) + local split = entries[middle].id + return prefix .. "if opcode <= " .. tostring(split) .. " then\n" + .. emit(first, middle, depth + 1) .. "\n" + .. prefix .. "else\n" + .. emit(middle + 1, last, depth + 1) .. "\n" + .. prefix .. "end" + end + + assert(#entries > 0, "No VM opcodes registered") + return emit(1, #entries, 0) +end + +return M diff --git a/src/prometheus/vm/statements.lua b/src/prometheus/vm/statements.lua new file mode 100644 index 00000000..3ed1eaf6 --- /dev/null +++ b/src/prometheus/vm/statements.lua @@ -0,0 +1,293 @@ +local Ast = require("prometheus.ast") +local K = Ast.AstKind + +local function emitList(self, expressions, count) + return self:list(expressions or {}, count) +end + +local function release(self, regs) + for _, reg in ipairs(regs or {}) do self:free(reg) end +end + +local function emitWriteIndex(self, base, key, value) + self:emit("SETTABLE", base, key, value) +end + +return function(Compiler) + function Compiler:cleanup(scope) + if not scope or scope.isGlobal then return end + for id = 1, #scope.variables do + if scope.variables[id] then self:emit("CLEAR", self:localRegister(scope, id)) end + end + end + + function Compiler:cleanupTo(scopeDepth) + for i = #self.scopeStack, scopeDepth + 1, -1 do + self:cleanup(self.scopeStack[i]) + end + end + + function Compiler:block(block, doCleanup) + local scope = block.scope + table.insert(self.scopeStack, scope) + for _, statement in ipairs(block.statements) do self:statement(statement) end + if doCleanup ~= false then self:cleanup(scope) end + self.scopeStack[#self.scopeStack] = nil + end + + local function loopFor(self, node) + for i = #self.loops, 1, -1 do + if self.loops[i].node == node then return self.loops[i] end + end + error("loop statement is not active") + end + + local function loopRecord(self, node, continueDepth) + local record = { + node = node, + scopeDepth = #self.scopeStack, + continueDepth = continueDepth or #self.scopeStack, + breaks = {}, + continues = {}, + } + table.insert(self.loops, record) + return record + end + + local function finishLoop(self, record, breakTarget, continueTarget) + for _, jump in ipairs(record.breaks) do self:patch(jump, 1, breakTarget) end + for _, jump in ipairs(record.continues) do self:patch(jump, 1, continueTarget) end + self.loops[#self.loops] = nil + end + + local function compileIf(self, node) + local done = {} + local branches = {{condition = node.condition, body = node.body}} + for _, branch in ipairs(node.elseifs or {}) do branches[#branches + 1] = branch end + for _, branch in ipairs(branches) do + local condition = self:expression(branch.condition, 1)[1] + local skip = self:emit("JFALSE", condition, 0) + self:free(condition) + self:block(branch.body, true) + done[#done + 1] = self:emit("JMP", 0) + self:patch(skip, 2, self:position()) + end + if node.elsebody then self:block(node.elsebody, true) end + local endPos = self:position() + for _, jump in ipairs(done) do self:patch(jump, 1, endPos) end + end + + local function compileWhile(self, node) + local record = loopRecord(self, node) + local check = self:position() + local condition = self:expression(node.condition, 1)[1] + local exit = self:emit("JFALSE", condition, 0) + self:block(node.body, true) + local continueTarget = self:position() + self:emit("JMP", check) + local clearPos = self:position() + self:patch(exit, 2, clearPos) + finishLoop(self, record, clearPos, continueTarget) + self:free(condition) + end + + local function compileRepeat(self, node) + local record = loopRecord(self, node, #self.scopeStack + 1) + record.bodyScope = node.body.scope + local start = self:position() + self:block(node.body, false) + local conditionPos = self:position() + local condition = self:expression(node.condition, 1)[1] + local leave = self:emit("JTRUE", condition, 0) + self:free(condition) + self:cleanup(node.body.scope) + self:emit("JMP", start) + local cleanupExit = self:position() + self:patch(leave, 2, cleanupExit) + self:cleanup(node.body.scope) + local endPos = self:position() + finishLoop(self, record, endPos, conditionPos) + end + + local function compileNumericFor(self, node) + local values = emitList(self, {node.initialValue, node.finalValue, node.incrementBy}, 3) + local control = {self:alloc(), self:alloc(), self:alloc()} + for i = 1, 3 do self:emit("MOVE", control[i], values[i]) end + release(self, values) + self:emit("FORPREP", control[1], control[2], control[3]) + local record = loopRecord(self, node) + local check = self:position() + local ok = self:alloc() + self:emit("FORCHECK", ok, control[1], control[2], control[3]) + local exit = self:emit("JFALSE", ok, 0) + self:initVariable(node.scope, node.id, control[1]) + self:block(node.body, true) + local advance = self:position() + local nextValue = self:alloc() + self:emit("ADD", nextValue, control[1], control[3]) + self:emit("MOVE", control[1], nextValue) + self:emit("CLEAR", nextValue) + self:emit("JMP", check) + local endPos = self:position() + self:patch(exit, 2, endPos) + finishLoop(self, record, endPos, advance) + self:free(ok) + self:free(nextValue) + release(self, control) + end + + local function compileGenericFor(self, node) + local values = emitList(self, node.expressions, 3) + local iterator, state, control = values[1], values[2], values[3] + local record = loopRecord(self, node) + local check = self:position() + local args, results = self:alloc(), self:alloc() + self:emit("ARGS", args, {state, control}, 0) + self:emit("CALL", results, iterator, args, -1) + local first = self:alloc() + self:emit("TAKE", first, results, 1) + local nilreg, ended = self:alloc(), self:alloc() + local iterationTemps = {args, results, first, nilreg, ended} + self:emit("NIL", nilreg) + self:emit("EQ", ended, first, nilreg) + local exit = self:emit("JTRUE", ended, 0) + self:emit("MOVE", control, first) + for i, id in ipairs(node.ids or node.vars) do + local value = self:alloc() + self:emit("TAKE", value, results, i) + self:initVariable(node.scope, id, value) + iterationTemps[#iterationTemps + 1] = value + end + for _, reg in ipairs(iterationTemps) do self:emit("CLEAR", reg) end + self:block(node.body, true) + local continueTarget = self:position() + self:emit("JMP", check) + local endPos = self:position() + self:patch(exit, 2, endPos) + finishLoop(self, record, endPos, continueTarget) + release(self, iterationTemps) + release(self, values) + end + + local function assignmentTarget(self, target) + if target.kind == K.AssignmentVariable then return {scope = target.scope, id = target.id} end + if target.kind == K.AssignmentIndexing then + local base = self:expression(target.base, 1)[1] + local key = self:expression(target.index, 1)[1] + return {base = base, key = key} + end + error("invalid assignment target: " .. tostring(target.kind)) + end + + local function storeTarget(self, target, value) + if target.scope then self:writeVariable(target.scope, target.id, value) + else emitWriteIndex(self, target.base, target.key, value) end + end + + local function compileAssignment(self, node) + local targets = {} + for i, target in ipairs(node.lhs) do targets[i] = assignmentTarget(self, target) end + local values = emitList(self, node.rhs, #targets) + for i, target in ipairs(targets) do storeTarget(self, target, values[i]) end + release(self, values) + for _, target in ipairs(targets) do + if target.base then self:free(target.base); self:free(target.key) end + end + end + + local compoundOp = { + [K.CompoundAddStatement] = "ADD", [K.CompoundSubStatement] = "SUB", + [K.CompoundMulStatement] = "MUL", [K.CompoundDivStatement] = "DIV", + [K.CompoundModStatement] = "MOD", [K.CompoundPowStatement] = "POW", + [K.CompoundConcatStatement] = "CONCAT", + } + + local function compileCompound(self, node) + local target = assignmentTarget(self, node.lhs) + local old = self:alloc() + if target.scope then + local value = self:readVariable(target.scope, target.id) + self:emit("MOVE", old, value) + self:free(value) + else + self:emit("GETTABLE", old, target.base, target.key) + end + local rhs = self:expression(node.rhs, 1)[1] + local result = self:alloc() + self:emit(compoundOp[node.kind], result, old, rhs) + storeTarget(self, target, result) + release(self, {old, rhs, result}) + if target.base then self:free(target.base); self:free(target.key) end + end + + local function compileLocal(self, node) + local values = emitList(self, node.expressions, #node.ids) + for i, id in ipairs(node.ids) do self:initVariable(node.scope, id, values[i]) end + release(self, values) + end + + local function compileFunctionDeclaration(self, node) + local indexed = #node.indices > 0 + local base = indexed and self:readVariable(node.scope, node.id) or nil + if indexed then + for i = 1, #node.indices - 1 do + local key, nextBase = self:load(node.indices[i]), self:alloc() + self:emit("GETTABLE", nextBase, base, key) + self:free(base); self:free(key) + base = nextBase + end + end + local fn = self:functionLiteral(node) + if indexed then + local key = self:load(node.indices[#node.indices]) + self:emit("SETTABLE", base, key, fn) + self:free(key); self:free(base) + else + self:writeVariable(node.scope, node.id, fn) + end + self:free(fn) + end + + function Compiler:statement(node) + local kind = node.kind + if kind == K.NopStatement then return + elseif kind == K.DoStatement then self:block(node.body, true) + elseif kind == K.IfStatement then compileIf(self, node) + elseif kind == K.WhileStatement then compileWhile(self, node) + elseif kind == K.RepeatStatement then compileRepeat(self, node) + elseif kind == K.ForStatement then compileNumericFor(self, node) + elseif kind == K.ForInStatement then compileGenericFor(self, node) + elseif kind == K.LocalVariableDeclaration then compileLocal(self, node) + elseif kind == K.AssignmentStatement then compileAssignment(self, node) + elseif compoundOp[kind] then compileCompound(self, node) + elseif kind == K.LocalFunctionDeclaration then + local nilreg = self:load(nil) + self:initVariable(node.scope, node.id, nilreg) + self:free(nilreg) + local fn = self:functionLiteral(node) + self:writeVariable(node.scope, node.id, fn) + self:free(fn) + elseif kind == K.FunctionDeclaration then compileFunctionDeclaration(self, node) + elseif kind == K.BreakStatement or kind == K.ContinueStatement then + local record = loopFor(self, node.loop) + local depth = kind == K.ContinueStatement and record.continueDepth or record.scopeDepth + self:cleanupTo(depth) + local jump = self:emit("JMP", 0) + local list = kind == K.ContinueStatement and record.continues or record.breaks + list[#list + 1] = jump + elseif kind == K.ReturnStatement then + if #node.args == 1 and not node.args[1].isParenthesizedExpression + and (node.args[1].kind == K.FunctionCallExpression or node.args[1].kind == K.PassSelfFunctionCallExpression) then + self:call(node.args[1], -1, true) + return + end + local values = emitList(self, node.args, nil) + self:emit("RETURN", values[1]) + release(self, values) + elseif kind == K.FunctionCallStatement or kind == K.PassSelfFunctionCallStatement then + local call = {kind = kind == K.FunctionCallStatement and K.FunctionCallExpression or K.PassSelfFunctionCallExpression, + base = node.base, args = node.args, passSelfFunctionName = node.passSelfFunctionName} + self:call(call, 0, false) + else error("unsupported statement: " .. tostring(kind)) end + end +end diff --git a/tests/harness/vm-arity.lua b/tests/harness/vm-arity.lua new file mode 100644 index 00000000..7a549e84 --- /dev/null +++ b/tests/harness/vm-arity.lua @@ -0,0 +1,70 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local function pack(...) return {n = select('#', ...), ...} end +local function compile(source, steps, version, seed) + local generated = Prometheus.Pipeline:fromConfig({ + LuaVersion = version or 'Lua51', Seed = seed or 1, + Steps = steps or {{Name = 'Vmify'}}, + }):apply(source) + return assert(loadstring(generated)) +end + +for _, seed in ipairs({1, 4, 17}) do + for _, steps in ipairs({{{Name = 'Vmify'}}, {{Name = 'Vmify'}, {Name = 'Cff'}}}) do + local chunk = compile('return ...', steps, 'Lua51', seed) + local scalar = compile('return (...)', steps, 'Lua51', seed) + for _, args in ipairs({pack(), pack(nil), pack(1, nil), pack(nil, 2, nil)}) do + local result = pack(chunk(unpack(args, 1, args.n))) + assert(result.n == args.n, 'changed chunk return count') + for i = 1, args.n do assert(result[i] == args[i], 'changed chunk value') end + result = pack(scalar(unpack(args, 1, args.n))) + assert(result.n == 1 and result[1] == args[1], 'parenthesized varargs must collapse') + end + local result = pack(compile([[ + local function forward(...) return ... end + local function invoke(fn, ...) return fn(...) end + return invoke(forward, nil, 2, nil) + ]], steps, 'Lua51', seed)()) + assert(result.n == 3 and result[1] == nil and result[2] == 2 and result[3] == nil) + end +end + +-- A true tail call must remain bounded even across distinct virtual prototypes. +local result = pack(compile([[ + local even, odd + even = function(n) if n == 0 then return true, nil end; return odd(n - 1) end + odd = function(n) if n == 0 then return false, nil end; return even(n - 1) end + return even(20000) +]])()) +assert(result.n == 2 and result[1] == true and result[2] == nil) + +-- Surplus RHS expressions still run, and native list fields flush after keyed fields. +assert(compile([[ + local calls = 0 + local function side() calls = calls + 1; return 3 end + local a = 1, side(), side() + 1 + local t = {7, [1] = 99, named = true} + return calls == 2 and a == 1 and t[1] == 7 and t.named +]])()) + +-- Lower Luau control constructs to portable instructions before emitting Lua. +assert(compile([[ + local sum = 0 + for i = 1, 8 do + if i % 2 == 0 then continue end + sum += i + end + local value = if sum == 16 then 42 else 0 + return value == 42 +]], nil, 'LuaU')()) + +-- Cff must flatten the interpreter without expanding large prototype data into +-- its dispatcher: doing so can exceed native Lua and LuaJIT branch limits. +local entries = {} +for i = 1, 1000 do entries[i] = tostring(i) end +local source = 'local t={' .. table.concat(entries, ',') .. '}; return t[1],t[1000],...' +local large = compile(source, {{Name = 'Vmify'}, {Name = 'EncryptStrings'}, {Name = 'Cff'}}) +local result = pack(large(nil, 3, nil)) +assert(result.n == 5 and result[1] == 1 and result[2] == 1000) +assert(result[3] == nil and result[4] == 3 and result[5] == nil) +print('VM arity, tail calls, constructors, and Luau lowering checks passed') diff --git a/tests/harness/vm-prototypes.lua b/tests/harness/vm-prototypes.lua new file mode 100644 index 00000000..782e9913 --- /dev/null +++ b/tests/harness/vm-prototypes.lua @@ -0,0 +1,38 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local Parser = require('prometheus.parser') +local Compiler = require('prometheus.vm.compiler') +local Emit = require('prometheus.vm.emit') +local opcodes = require('prometheus.vm.opcodes') + +-- Editing instruction operands changes execution: the emitted tables are executable +-- program data, rather than metadata alongside a native copy of the source body. +local ast = Parser:new({LuaVersion = 'Lua51'}):parse('return 17') +local proto = Compiler:compile(ast) +local changed = false +for _, instruction in ipairs(proto.code) do + if instruction[1] == opcodes.LOAD and proto.constants[instruction[3]] == 17 then + proto.constants[instruction[3]] = 42 + changed = true + end +end +assert(changed) +assert(assert(loadstring(Emit.source(proto)))() == 42) +proto.code[1][1] = 1000000 +local ok, err = pcall(assert(loadstring(Emit.source(proto)))) +assert(not ok and tostring(err):find('unknown VM opcode', 1, true)) + +-- Cells capture only the requested lexical locals, including transitive captures. +local source = [[ + local shared, unused = 0, {} + return function() + return function(delta) shared = shared + delta; return shared end + end +]] +local proto = Compiler:compile(Parser:new({LuaVersion = 'Lua51'}):parse(source)) +local maker = assert(loadstring(Emit.source(proto)))() +local first, second = maker(), maker() +assert(first(2) == 2 and second(3) == 5 and first(4) == 9) +assert(#proto.children[1].captures == 1) +assert(#proto.children[1].children[1].captures == 1) +print('VM prototype execution and transitive capture checks passed') From a249a56e156db877ea2c2992ad49300ad007369c Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:23:59 +0100 Subject: [PATCH 3/8] increase test timeout --- Dockerfile | 8 ++++---- docker-test-runner.lua | 2 +- scripts/run-tests.sh | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 540b85b9..caa0b5fd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,10 +25,10 @@ WORKDIR /app COPY . /app RUN lua5.1 tests/harness/metadata.lua \ - && timeout 30s lua5.1 tests/harness/numbers-to-expressions.lua \ - && timeout 30s lua5.1 tests/harness/cff-arity.lua \ - && timeout 30s lua5.1 tests/harness/vm-arity.lua \ - && timeout 30s lua5.1 tests/harness/vm-prototypes.lua \ + && timeout 60s lua5.1 tests/harness/numbers-to-expressions.lua \ + && timeout 60s lua5.1 tests/harness/cff-arity.lua \ + && timeout 60s lua5.1 tests/harness/vm-arity.lua \ + && timeout 60s lua5.1 tests/harness/vm-prototypes.lua \ && python3 tests/harness/runner.py ENTRYPOINT ["lua5.1", "docker-test-runner.lua"] diff --git a/docker-test-runner.lua b/docker-test-runner.lua index a2ef4984..e5d4d7b3 100644 --- a/docker-test-runner.lua +++ b/docker-test-runner.lua @@ -17,7 +17,7 @@ local function integer(key, default) assert(value and value >= 1 and value == math.floor(value), "--" .. key .. " must be a positive integer") return value end -local iterations, timeout, seed = integer("iterations", 10), integer("timeout", 30), integer("seed", 1) +local iterations, timeout, seed = integer("iterations", 10), integer("timeout", 60), integer("seed", 1) assert(seed + iterations - 1 <= 2147483647, "seed range must fit a signed 32-bit integer") local suite = args.suite or "all" assert(suite == "all" or suite == "existing" or suite == "upstream", "invalid --suite") diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh index 2e4fc1b1..c0e563a8 100755 --- a/scripts/run-tests.sh +++ b/scripts/run-tests.sh @@ -33,7 +33,7 @@ Options: --preset NAME One preset --runtime NAME lua5.1 or luau (lua51 alias accepted) --seed N First obfuscation seed (default: 1) - --timeout N Seconds per subprocess (default: 30) + --timeout N Seconds per subprocess (default: 60) --baseline-only Run originals without obfuscation --artifacts DIR Host artifact directory (default: test-results) -h, --help Show this help From 1a6fba681cb9f628e5c425904a266f3634d36456 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:33:31 +0100 Subject: [PATCH 4/8] randomize VM opcode IDs and operand layouts. --- Dockerfile | 1 + doc/advanced/register-vm.md | 20 +++- doc/reference/steps/vmify.md | 6 ++ src/prometheus/vm/emit.lua | 17 ++- src/prometheus/vm/layout.lua | 35 ++++++ src/prometheus/vm/opcodes.lua | 2 +- src/prometheus/vm/runtime.lua | 17 ++- src/prometheus/vm/runtime_dispatch.lua | 6 +- tests/harness/vm-layout.lua | 141 +++++++++++++++++++++++++ 9 files changed, 228 insertions(+), 17 deletions(-) create mode 100644 src/prometheus/vm/layout.lua create mode 100644 tests/harness/vm-layout.lua diff --git a/Dockerfile b/Dockerfile index caa0b5fd..77dc084b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,6 +29,7 @@ RUN lua5.1 tests/harness/metadata.lua \ && timeout 60s lua5.1 tests/harness/cff-arity.lua \ && timeout 60s lua5.1 tests/harness/vm-arity.lua \ && timeout 60s lua5.1 tests/harness/vm-prototypes.lua \ + && timeout 60s lua5.1 tests/harness/vm-layout.lua \ && python3 tests/harness/runner.py ENTRYPOINT ["lua5.1", "docker-test-runner.lua"] diff --git a/doc/advanced/register-vm.md b/doc/advanced/register-vm.md index 70e3df4b..ca5fdeb2 100644 --- a/doc/advanced/register-vm.md +++ b/doc/advanced/register-vm.md @@ -12,9 +12,12 @@ compiler, or `loadstring` to execute the original source. value lists, calls, branches, and table constructors. - `vm/statements.lua`: statements, lexical cleanup, and loop control. - `vm/opcodes.lua`: semantic instruction IDs. +- `vm/layout.lua`: a per-emission opcode and operand layout. - `vm/runtime.lua` and `vm/runtime_dispatch.lua`: runtime templates and the - generated dispatcher. -- `vm/emit.lua`: native prototype table literals and the runtime function IIFE. + generated dispatcher, configured for the selected opcode IDs and operand + positions. +- `vm/emit.lua`: instruction encoding, native prototype table literals, and the + runtime function IIFE. - `vm/flatten.lua`: the Cff integration for generated VM interpreters. A prototype contains instructions, constants, child prototypes, parameter @@ -43,9 +46,18 @@ arguments. This supports helpers introduced by intervening steps such as the special path falls back to ordinary Cff compilation to preserve that write's behavior. +The compiler keeps canonical semantic instruction IDs. At emission, `layout.lua` +chooses a permutation of the opcode IDs and a permutation of operand slots 2 +through 5; slot 1 always contains the opcode. The emitter encodes each +instruction with that layout without changing the compiler prototypes. One +layout is shared by the root prototype and all child prototypes in the emitted +chunk. + `runtime_dispatch.lua` builds a balanced numeric `if`/`else` decision tree from -the opcode IDs. The interpreter caches the current opcode and checks for an -exact ID at each leaf before running that instruction's handler. +the selected opcode IDs. The interpreter reads each operand from its selected +slot, then checks the opcode for an exact ID at each leaf before running its +handler. For the same input, configuration, and Lua runtime, a fixed pipeline +seed reproduces the layout. Different seeds can produce different layouts. `Strong` applies one `Vmify` pass followed by `EncryptStrings`, `AntiTamper`, and one `Cff` pass over the generated VM interpreter. `Weak` retains the diff --git a/doc/reference/steps/vmify.md b/doc/reference/steps/vmify.md index 6a2f3596..6dcc98a8 100644 --- a/doc/reference/steps/vmify.md +++ b/doc/reference/steps/vmify.md @@ -4,6 +4,12 @@ Constructor key: `Vmify`. `Vmify` compiles the AST into per-function register instructions, constants, and prototype data, then emits Lua code with an interpreter that executes that data. This is the table-backed register VM step. It compiles the AST directly; it does not first apply `Cff`. +Each emitted chunk uses a generated instruction layout. Opcode IDs are +permuted, and the operand fields use a permuted selection of slots 2 through 5; +slot 1 always stores the opcode. The same layout applies to the root function +and its child prototypes. For the same input, configuration, and Lua runtime, +a fixed pipeline seed reproduces the layout. + ## Settings None. diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua index c3469fd1..61bcb385 100644 --- a/src/prometheus/vm/emit.lua +++ b/src/prometheus/vm/emit.lua @@ -4,9 +4,11 @@ local Parser = require('prometheus.parser') local Unparser = require('prometheus.unparser') local Enums = require('prometheus.enums') local Runtime = require('prometheus.vm.runtime') +local Layout = require('prometheus.vm.layout') local Emit = {} -function Emit.source(proto) +function Emit.source(proto, layout) + layout = layout or Layout.new() local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) local function literal(value) if type(value) ~= 'table' then @@ -17,8 +19,15 @@ function Emit.source(proto) return '{' .. table.concat(parts, ',') .. '}' end local function prototype(value) - local parts = {} - for _, key in ipairs({'code', 'constants', 'captures', 'params', 'vararg', 'maxRegister'}) do + local instructions = {} + for _, instruction in ipairs(value.code) do + local encoded, length = layout:encode(instruction) + local operands = {} + for slot = 1, length do operands[slot] = literal(encoded[slot]) end + instructions[#instructions + 1] = '{' .. table.concat(operands, ',') .. '}' + end + local parts = {'code={' .. table.concat(instructions, ',') .. '}'} + for _, key in ipairs({'constants', 'captures', 'params', 'vararg', 'maxRegister'}) do parts[#parts + 1] = key .. '=' .. literal(value[key]) end local children = {} @@ -27,7 +36,7 @@ function Emit.source(proto) return '{' .. table.concat(parts, ',') .. '}' end local data = '{' .. prototype(proto) .. '}' - local source, count = Runtime.build():gsub('__PROTOTYPES__', function() return data end) + local source, count = Runtime.build(layout):gsub('__PROTOTYPES__', function() return data end) assert(count == 1, 'Runtime template must contain one prototype marker') return source end diff --git a/src/prometheus/vm/layout.lua b/src/prometheus/vm/layout.lua new file mode 100644 index 00000000..c27652a9 --- /dev/null +++ b/src/prometheus/vm/layout.lua @@ -0,0 +1,35 @@ +-- Choose one instruction encoding for each generated VM. Compiler IR stays semantic. +local opcodes = require('prometheus.vm.opcodes') +local shuffle = require('prometheus.util').shuffle +local Layout = {} +Layout.__index = Layout + +function Layout.new() + local entries, ids = {}, {} + for name, id in pairs(opcodes) do + entries[#entries + 1] = {name = name, id = id} + end + table.sort(entries, function(left, right) return left.id < right.id end) + for i = 1, #entries do ids[i] = i end + shuffle(ids) + local layout = setmetatable({opcodes = {}, ids = {}, + operands = shuffle({2, 3, 4, 5})}, Layout) + for i, entry in ipairs(entries) do + layout.opcodes[entry.name] = ids[i] + layout.ids[entry.id] = ids[i] + end + return layout +end + +function Layout:encode(instruction) + local encoded = {self.ids[instruction[1]] or instruction[1]} + local length = 1 + for operand = 2, #instruction do + local slot = assert(self.operands[operand - 1], 'Missing VM operand slot') + encoded[slot] = instruction[operand] + length = math.max(length, slot) + end + return encoded, length +end + +return Layout diff --git a/src/prometheus/vm/opcodes.lua b/src/prometheus/vm/opcodes.lua index 9b79116e..0cc1f8f0 100644 --- a/src/prometheus/vm/opcodes.lua +++ b/src/prometheus/vm/opcodes.lua @@ -1,4 +1,4 @@ --- Semantic instruction IDs for the baseline VM. Layout mutation is a later pass. +-- Semantic instruction IDs for compiler IR. Emission assigns randomized IDs. local names = { 'LOAD', 'NIL', 'MOVE', 'CLEAR', 'CELL', 'GETCELL', 'SETCELL', 'GETUP', 'SETUP', 'GLOBAL', 'SETGLOBAL', 'GETTABLE', 'SETTABLE', diff --git a/src/prometheus/vm/runtime.lua b/src/prometheus/vm/runtime.lua index 3637c66b..c2674ab4 100644 --- a/src/prometheus/vm/runtime.lua +++ b/src/prometheus/vm/runtime.lua @@ -1,7 +1,7 @@ -- Builds the Lua 5.1 runtime used by the bytecode emitter. The emitter replaces -- __PROTOTYPES__ with its native Lua table literal before parsing the source. local dispatch = require("prometheus.vm.runtime_dispatch") -local opcodes = require("prometheus.vm.opcodes") +local Layout = require("prometheus.vm.layout") local Runtime = {} @@ -28,7 +28,7 @@ local function loop(P, U, Env, R, V) I = code[pc] pc = pc + 1 opcode = I[1] - a, b, c, d = I[2], I[3], I[4], I[5] + __FETCH_OPERANDS__ __DISPATCH__ I, opcode, a, b, c, d, x, y, z = nil, nil, nil, nil, nil, nil, nil, nil, nil end @@ -81,9 +81,16 @@ return make(prototypes[1], {}, entryEnv)(...) end)(__PROTOTYPES__, ...) ]=] -function Runtime.build() - local source = template:gsub("__DISPATCH__", function() - return dispatch.build(opcodes) +function Runtime.build(layout) + layout = layout or Layout.new() + local operands = layout.operands + local fetch = "a, b, c, d = I[" .. tostring(operands[1]) .. "], I[" + .. tostring(operands[2]) .. "], I[" .. tostring(operands[3]) .. "], I[" + .. tostring(operands[4]) .. "]" + local source = template:gsub("__FETCH_OPERANDS__", function() + return fetch + end):gsub("__DISPATCH__", function() + return dispatch.build(layout.opcodes) end) return source end diff --git a/src/prometheus/vm/runtime_dispatch.lua b/src/prometheus/vm/runtime_dispatch.lua index e1b46daa..5163f5c7 100644 --- a/src/prometheus/vm/runtime_dispatch.lua +++ b/src/prometheus/vm/runtime_dispatch.lua @@ -24,12 +24,12 @@ local bodies = { ]==], ARGS = [==[ x = { n = 0 } - for _, y in nativeIpairs(I[3]) do + for _, y in nativeIpairs(b) do x.n = x.n + 1 x[x.n] = R[y] end - if I[4] ~= 0 then - y = R[I[4]] + if c ~= 0 then + y = R[c] for d = 1, y.n do x.n = x.n + 1 x[x.n] = y[d] diff --git a/tests/harness/vm-layout.lua b/tests/harness/vm-layout.lua new file mode 100644 index 00000000..f7ad70c7 --- /dev/null +++ b/tests/harness/vm-layout.lua @@ -0,0 +1,141 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local Parser = require('prometheus.parser') +local Compiler = require('prometheus.vm.compiler') +local Emit = require('prometheus.vm.emit') +local Layout = require('prometheus.vm.layout') +local opcodes = require('prometheus.vm.opcodes') + +local names = {} +local namesById = {} +for name, id in pairs(opcodes) do + names[#names + 1] = name + namesById[id] = name +end +table.sort(names) + +local function signature(layout) + local parts = {} + for _, name in ipairs(names) do + parts[#parts + 1] = name .. ':' .. tostring(layout.opcodes[name]) + end + for id = 1, #names do parts[#parts + 1] = tostring(layout.ids[id]) end + for i = 1, 4 do parts[#parts + 1] = tostring(layout.operands[i]) end + return table.concat(parts, ',') +end + +local function opcodeSignature(layout) + local parts = {} + for _, name in ipairs(names) do + parts[#parts + 1] = name .. ':' .. tostring(layout.opcodes[name]) + end + return table.concat(parts, ',') +end + +local function seeded(seed) + math.randomseed(seed) + return Layout.new() +end + +local first = seeded(87123) +assert(signature(first) == signature(seeded(87123)), 'layout must repeat for the same seed') +local opcodeLayouts, operandLayouts = {}, {} +for _, seed in ipairs({1, 4, 17, 99, 87123, 234567}) do + local layout = seeded(seed) + local seen = {} + for _, name in ipairs(names) do + local id = layout.opcodes[name] + assert(type(id) == 'number' and id >= 1 and id <= #names and id % 1 == 0) + assert(not seen[id], 'opcode IDs must be a permutation') + seen[id] = true + end + for id = 1, #names do + assert(layout.ids[id] == layout.opcodes[namesById[id]], + 'canonical opcode lookup must match the name lookup') + end + opcodeLayouts[opcodeSignature(layout)] = true + local slots = {} + for i = 1, 4 do slots[layout.operands[i]] = true end + assert(slots[2] and slots[3] and slots[4] and slots[5]) + operandLayouts[table.concat(layout.operands, ',')] = true +end +local distinctOpcodes, distinctOperands = 0, 0 +for _ in pairs(opcodeLayouts) do distinctOpcodes = distinctOpcodes + 1 end +for _ in pairs(operandLayouts) do distinctOperands = distinctOperands + 1 end +assert(distinctOpcodes > 1, 'different seeds must change opcode assignments') +assert(distinctOperands > 1, 'different seeds must change operand layouts') + +local source = [[ + local function pack(...) return {n = select('#', ...), ...} end + local function make(start) + local total = start + local unused = {} + return function(step, ...) + total = total + step + return pack(total, ...) + end + end + local closure = make(10) + local object = {value = 1} + function object:bump(step, ...) self.value = self.value + step; return pack(self.value, ...) end + local values = {} + for i = 1, 4 do values[i] = i * i end + local left = closure(2, nil, 7, nil) + local right = object:bump(3, 'method', nil) + local keyed = {left[1], right[1], nil, label = 'ok', [8] = values[4]} + local rootArgs = pack(...) + return left, right, values, keyed, rootArgs, object.value +]] +local proto = Compiler:compile(Parser:new({LuaVersion = 'Lua51'}):parse(source)) + +local function clone(value) + if type(value) ~= 'table' then return value end + local result = {} + for key, item in pairs(value) do result[clone(key)] = clone(item) end + return result +end +local function equal(left, right) + if type(left) ~= type(right) then return false end + if type(left) ~= 'table' then return left == right end + for key, value in pairs(left) do if not equal(value, right[key]) then return false end end + for key in pairs(right) do if left[key] == nil and right[key] ~= nil then return false end end + return true +end +local original = clone(proto) + +-- Run all 24 physical operand orders, with the opcode field left in slot one. +local function permutations(values, index, output) + index = index or 1 + output = output or {} + if index > #values then + output[#output + 1] = {unpack(values)} + return + end + for i = index, #values do + values[index], values[i] = values[i], values[index] + permutations(values, index + 1, output) + values[index], values[i] = values[i], values[index] + end + return output +end +local allOrders = permutations({2, 3, 4, 5}) +assert(#allOrders == 24) +for index, order in ipairs(allOrders) do + local layout = seeded(3000 + index) + layout.operands = order + local run = assert(loadstring(Emit.source(proto, layout))) + local left, right, values, keyed, rootArgs, objectValue = run('root', nil, 99, nil) + assert(left.n == 4 and left[1] == 12 and left[2] == nil and left[3] == 7 and left[4] == nil) + assert(right.n == 3 and right[1] == 4 and right[2] == 'method' and right[3] == nil) + assert(values[1] == 1 and values[2] == 4 and values[3] == 9 and values[4] == 16) + assert(keyed[1] == 12 and keyed[2] == 4 and keyed[3] == nil) + assert(keyed.label == 'ok' and keyed[8] == 16) + assert(rootArgs.n == 4 and rootArgs[1] == 'root' and rootArgs[2] == nil) + assert(rootArgs[3] == 99 and rootArgs[4] == nil and objectValue == 4) + assert(equal(proto, original), 'emission must not rewrite canonical compiler IR') +end + +-- The ordinary build path also chooses a profile while retaining canonical IR. +assert(loadstring(Emit.source(proto)))('build') +assert(equal(proto, original), 'default emission must not rewrite canonical compiler IR') +print('VM opcode and operand layout checks passed') From 66cdc92ccc0418f9b0482a1f298a3b010005ba17 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 15:49:09 +0100 Subject: [PATCH 5/8] update release metadata --- benchmarks/README.md | 38 +++++++++ benchmarks/cases/closures.lua | 26 ++++++ benchmarks/cases/numeric.lua | 16 ++++ benchmarks/cases/strings.lua | 14 ++++ benchmarks/cases/tables.lua | 27 +++++++ benchmarks/generate.lua | 18 +++++ benchmarks/measure.lua | 36 +++++++++ benchmarks/run.py | 145 ++++++++++++++++++++++++++++++++++ benchmarks/v0.3-results.md | 89 +++++++++++++++++++++ src/config.lua | 4 +- 10 files changed, 411 insertions(+), 2 deletions(-) create mode 100644 benchmarks/README.md create mode 100644 benchmarks/cases/closures.lua create mode 100644 benchmarks/cases/numeric.lua create mode 100644 benchmarks/cases/strings.lua create mode 100644 benchmarks/cases/tables.lua create mode 100644 benchmarks/generate.lua create mode 100644 benchmarks/measure.lua create mode 100644 benchmarks/run.py create mode 100644 benchmarks/v0.3-results.md diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..1debfe60 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,38 @@ +# Preset benchmarks + +Recorded v0.3 measurements: [v0.3-results.md](v0.3-results.md). + +Run from the repository root with Python 3, Lua 5.1, and LuaJIT installed: + +```sh +python3 benchmarks/run.py +``` + +The runner compares original source, Minify, Cff, Vmify, Medium, and Strong. +Production presets retain AntiTamper. Seeds default to 1, 4, and 17, with three +timed batches per generated chunk. Every execution validates a deterministic +checksum against the original source. + +The workloads exercise arithmetic and branches, record sorting and aggregation, +string formatting and parsing, and closures with captured state and method +calls. The first chunk argument scales their work; the runner uses scale 1. + +Execution measures CPU time for repeated complete chunk invocations after +warmup, excluding parsing. Initialization and allocations in the generated code +are included. Garbage collection runs before each batch and may run during it. +Generation and parsing time are reported separately. Peak memory is not measured. + +Reports, raw JSON, and generated sources default to +`test-results/benchmark-v0.3/`. To choose another directory: + +```sh +python3 benchmarks/run.py --output /tmp/prometheus-benchmark +``` + +Other options include `--seeds 1 4 17`, `--samples 3`, `--target 0.1` (target +CPU seconds per batch), `--timeout 60` (seconds per process), and +`--runtimes lua5.1 luajit`. The LuaJIT run leaves JIT enabled. Tiny native JIT +times can produce large slowdown ratios; compare absolute times as well. + +Use `--resume` with the same output directory and options to reuse completed +measurements after an interrupted run. diff --git a/benchmarks/cases/closures.lua b/benchmarks/cases/closures.lua new file mode 100644 index 00000000..43ef4aaa --- /dev/null +++ b/benchmarks/cases/closures.lua @@ -0,0 +1,26 @@ +local scale = ... or 1 + +local function make_processor(seed) + local total = seed + local function add(_, ...) + for index = 1, select("#", ...) do + total = total + select(index, ...) + end + end + return { + add = add, + value = function() + return total + end, + } +end + +local checksum = 0 +for i = 1, 10000 * scale do + local seed = i % 31 + local processor = make_processor(seed) + processor:add(i, seed * 2, i % 7) + checksum = checksum + processor:value() +end + +return checksum diff --git a/benchmarks/cases/numeric.lua b/benchmarks/cases/numeric.lua new file mode 100644 index 00000000..a427f030 --- /dev/null +++ b/benchmarks/cases/numeric.lua @@ -0,0 +1,16 @@ +local scale = ... or 1 +local limit = 180000 * scale +local checksum = 0 + +for i = 1, limit do + local value = (i * 37 + 11) % 997 + if value % 5 == 0 then + checksum = checksum + value * 3 + elseif value % 2 == 0 then + checksum = checksum + value + else + checksum = checksum - value + end +end + +return checksum diff --git a/benchmarks/cases/strings.lua b/benchmarks/cases/strings.lua new file mode 100644 index 00000000..bdab5b3e --- /dev/null +++ b/benchmarks/cases/strings.lua @@ -0,0 +1,14 @@ +local scale = ... or 1 +local format = string.format +local match = string.match +local checksum = 0 + +for i = 1, 10000 * scale do + local category = (i * 17) % 23 + local quantity = (i * 31) % 1000 + local record = format("item:%02d|%04d|zone-%d", category, quantity, i % 9) + local parsed_category, parsed_quantity, zone = match(record, "^item:(%d+)|(%d+)|zone%-(%d+)$") + checksum = checksum + tonumber(parsed_category) * 101 + tonumber(parsed_quantity) + tonumber(zone) +end + +return checksum diff --git a/benchmarks/cases/tables.lua b/benchmarks/cases/tables.lua new file mode 100644 index 00000000..c9c5649f --- /dev/null +++ b/benchmarks/cases/tables.lua @@ -0,0 +1,27 @@ +local scale = ... or 1 +local rounds = 100 * scale +local checksum = 0 + +for round = 1, rounds do + local records = {} + for i = 1, 100 do + local key = (i * 73 + round * 19) % 211 + records[i] = { key = key, value = key * 3 + i } + end + + table.sort(records, function(left, right) + if left.key == right.key then + return left.value < right.value + end + return left.key < right.key + end) + + local subtotal = 0 + for i = 1, #records do + local record = records[i] + subtotal = subtotal + record.key * i + record.value + end + checksum = checksum + subtotal +end + +return checksum diff --git a/benchmarks/generate.lua b/benchmarks/generate.lua new file mode 100644 index 00000000..8f6efc0b --- /dev/null +++ b/benchmarks/generate.lua @@ -0,0 +1,18 @@ +-- Generate the actual preset, including AntiTamper, and report compiler CPU time. +local Prometheus = require('src.prometheus') +local presets = require('src.presets') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local preset, seed, input, output = unpack(arg) +local file = assert(io.open(input, 'rb')) +local source = file:read('*a') +file:close() +local config = {} +for key, value in pairs(assert(presets[preset])) do config[key] = value end +config.Seed = assert(tonumber(seed)) +local start = os.clock() +local generated = Prometheus.Pipeline:fromConfig(config):apply(source, input) +local elapsed = os.clock() - start +file = assert(io.open(output, 'wb')) +file:write(generated) +file:close() +print(string.format('%.9f', elapsed)) diff --git a/benchmarks/measure.lua b/benchmarks/measure.lua new file mode 100644 index 00000000..ff116339 --- /dev/null +++ b/benchmarks/measure.lua @@ -0,0 +1,36 @@ +-- Parse once; warm the workload, then measure CPU time per complete invocation. +local input, samples, target = arg[1], tonumber(arg[2]), tonumber(arg[3]) +io.stdout:setvbuf('no') +collectgarbage('collect') +local start = os.clock() +local run = assert(loadfile(input)) +local loadTime = os.clock() - start +local function checksum() + local value = run(1) + assert(type(value) == 'number' or type(value) == 'string', 'scalar checksum required') + return tostring(value) +end +start = os.clock() +local expected = checksum() +-- Warm up JIT traces and interpreter paths before estimating the batch size. +local elapsed = os.clock() - start +local warmups = 1 +while elapsed < target and warmups < 1000 do + start = os.clock() + assert(checksum() == expected, 'unstable workload result') + elapsed = elapsed + os.clock() - start + warmups = warmups + 1 +end +local repeats = math.max(1, math.min(10000, + math.ceil(target / math.max(elapsed / warmups, 0.000001)))) +print('checksum\t' .. expected) +print('load_seconds\t' .. string.format('%.9f', loadTime)) +print('repeats\t' .. repeats) +for sample = 1, samples do + collectgarbage('collect') + start = os.clock() + for _ = 1, repeats do + assert(checksum() == expected, 'unstable workload result') + end + print('seconds\t' .. string.format('%.9f', (os.clock() - start) / repeats)) +end diff --git a/benchmarks/run.py b/benchmarks/run.py new file mode 100644 index 00000000..e639c713 --- /dev/null +++ b/benchmarks/run.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Compare real presets on deterministic workloads; retain inputs and raw metrics.""" +import argparse +import json +import platform +from pathlib import Path +import statistics +import subprocess + +ROOT = Path(__file__).resolve().parents[1] +PRESETS = ('Minify', 'Cff', 'Vmify', 'Medium', 'Strong') + + +def cpu_model(): + cpuinfo = Path('/proc/cpuinfo') + if cpuinfo.exists(): + for line in cpuinfo.read_text().splitlines(): + if line.startswith('model name'): + return line.split(':', 1)[1].strip() + return platform.processor() or 'unknown' + + +def command(args, timeout): + result = subprocess.run(args, cwd=ROOT, text=True, capture_output=True, timeout=timeout) + if result.returncode: + raise RuntimeError(f'{args}:\n{result.stdout}\n{result.stderr}') + return result.stdout.strip() + + +def measure(runtime, path, samples, target, timeout): + fields = {} + for line in command([runtime, 'benchmarks/measure.lua', str(path), + str(samples), str(target)], timeout).splitlines(): + key, value = line.split('\t', 1) + fields.setdefault(key, []).append(value) + return dict(checksum=fields['checksum'][0], + load_seconds=float(fields['load_seconds'][0]), + repeats=int(fields['repeats'][0]), + seconds=[float(value) for value in fields['seconds']]) + + +def report(data, output): + lines = ['# Prometheus v0.3 benchmark', '', + f"Host: {data['host']}; CPU: {data.get('cpu', cpu_model())}. Runtimes: " + + '; '.join(data['versions'].values()) + '.', '', + 'Four deterministic workloads compare original source with the actual presets, ' + 'including AntiTamper. All variants must match the original checksum.', '', + f"Seeds: {data['seeds']}; {data['samples']} timed batches per seed. " + 'Each workload is parsed once and warmed up; batches adapt to approximately ' + f"{data['target_seconds']} seconds. Execution includes initialization of the emitted " + 'chunk, workload work, allocations, and checksum validation, but excludes parsing. ' + 'Garbage collection runs before each batch; collection during the batch is included.', '', + 'Execution figures are median CPU time per full invocation across batches and seeds. ' + 'Generation and parsing figures are medians across seeds. Output size is a median ' + 'across seeds. Execution ranges show the lowest and highest per-seed medians. ' + 'Measurements are local, with JIT enabled for LuaJIT; very small native ' + 'JIT times can produce large slowdown ratios. These workloads do not establish a ' + 'general performance guarantee or measure peak memory.', ''] + for runtime in data['versions']: + lines.extend([f'## {runtime}', '', + '| Workload | Preset | Execution ms (median; seed range) | Slowdown | Output KiB | Generate ms | Parse ms |', + '| --- | --- | ---: | ---: | ---: | ---: | ---: |']) + for case in data['cases']: + rows = [row for row in data['results'] if row['runtime'] == runtime and row['case'] == case] + baseline = statistics.median(next(row for row in rows if row['preset'] == 'Original')['seconds']) + for preset in ('Original',) + PRESETS: + selected = [row for row in rows if row['preset'] == preset] + seconds = statistics.median([t for row in selected for t in row['seconds']]) + seed_times = [statistics.median(row['seconds']) * 1000 for row in selected] + execution = f'{seconds * 1000:.3f}' + if len(selected) > 1: + execution += f' ({min(seed_times):.3f}–{max(seed_times):.3f})' + size = statistics.median(row['bytes'] for row in selected) / 1024 + generation = statistics.median(row['generation_seconds'] for row in selected) * 1000 + load = statistics.median(row['load_seconds'] for row in selected) * 1000 + lines.append(f'| {case} | {preset} | {execution} | {seconds / baseline:.1f}x | ' + f'{size:.1f} | {generation:.2f} | {load:.2f} |') + lines.append('') + (output / 'report.md').write_text('\n'.join(lines)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, default=ROOT / 'test-results/benchmark-v0.3') + parser.add_argument('--seeds', type=int, nargs='+', default=[1, 4, 17]) + parser.add_argument('--samples', type=int, default=3) + parser.add_argument('--target', type=float, default=0.1) + parser.add_argument('--timeout', type=int, default=60) + parser.add_argument('--runtimes', nargs='+', default=['lua5.1', 'luajit']) + parser.add_argument('--resume', action='store_true', help='reuse completed rows in the output directory') + args = parser.parse_args() + if args.samples < 1 or args.target <= 0 or args.timeout < 1: + parser.error('samples, target, and timeout must be positive') + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + cases = sorted((ROOT / 'benchmarks/cases').glob('*.lua')) + assert cases, 'No benchmark cases found' + data = dict(host=platform.platform(), cpu=cpu_model(), seeds=args.seeds, samples=args.samples, + target_seconds=args.target, cases=[case.stem for case in cases], + versions={}, results=[]) + if args.resume: + previous = json.loads((output / 'results.json').read_text()) + for key in ('seeds', 'samples', 'target_seconds', 'cases'): + assert previous[key] == data[key], f'Resume configuration mismatch: {key}' + data['results'] = previous['results'] + for runtime in args.runtimes: + version = subprocess.run([runtime, '-v'], text=True, capture_output=True, check=True) + data['versions'][runtime] = (version.stdout + version.stderr).strip() + for case in cases: + generated = [('Original', 0, case, 0.0)] + for preset in PRESETS: + for seed in args.seeds: + path = output / f'{case.stem}-{preset}-{seed}.lua' + completed = next((row for row in data['results'] if row['case'] == case.stem + and row['preset'] == preset and row['seed'] == seed), None) + if completed and path.exists(): + elapsed = completed['generation_seconds'] + else: + elapsed = float(command(['lua5.1', 'benchmarks/generate.lua', preset, + str(seed), str(case), str(path)], args.timeout)) + generated.append((preset, seed, path, elapsed)) + for runtime in args.runtimes: + expected = None + for preset, seed, path, elapsed in generated: + completed = next((row for row in data['results'] if row['case'] == case.stem + and row['runtime'] == runtime and row['preset'] == preset + and row['seed'] == seed), None) + row = completed or measure(runtime, path, args.samples, args.target, args.timeout) + if expected is None: + expected = row['checksum'] + assert row['checksum'] == expected, f'Checksum mismatch: {runtime} {path}' + if completed: + continue + row.update(case=case.stem, runtime=runtime, preset=preset, seed=seed, + bytes=path.stat().st_size, generation_seconds=elapsed) + data['results'].append(row) + print(f'{runtime} {case.stem} {preset} seed={seed}: ' + f'{statistics.median(row["seconds"]) * 1000:.3f}ms', flush=True) + (output / 'results.json').write_text(json.dumps(data, indent=2) + '\n') + report(data, output) + print(f'Report: {output / "report.md"}') + + +if __name__ == '__main__': + main() diff --git a/benchmarks/v0.3-results.md b/benchmarks/v0.3-results.md new file mode 100644 index 00000000..3c30d6e4 --- /dev/null +++ b/benchmarks/v0.3-results.md @@ -0,0 +1,89 @@ +# Prometheus v0.3 benchmark + +Host: Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.44; CPU: 13th Gen Intel(R) Core(TM) i7-13850HX. Runtimes: Lua 5.1.5 Copyright (C) 1994-2012 Lua.org, PUC-Rio; LuaJIT 2.1.1792588164 -- Copyright (C) 2005-2026 Mike Pall. https://luajit.org/. + +Four deterministic workloads compare original source with the actual presets, including AntiTamper. All variants must match the original checksum. + +Seeds: [1, 4, 17]; 3 timed batches per seed. Each workload is parsed once and warmed up; batches adapt to approximately 0.1 seconds. Execution includes initialization of the emitted chunk, workload work, allocations, and checksum validation, but excludes parsing. Garbage collection runs before each batch; collection during the batch is included. + +Execution figures are median CPU time per full invocation across batches and seeds. Generation and parsing figures are medians across seeds. Output size is a median across seeds. Execution ranges show the lowest and highest per-seed medians. Measurements are local, with JIT enabled for LuaJIT; very small native JIT times can produce large slowdown ratios. These workloads do not establish a general performance guarantee or measure peak memory. + +## lua5.1 + +| Workload | Preset | Execution ms (median; seed range) | Slowdown | Output KiB | Generate ms | Parse ms | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| closures | Original | 4.160 | 1.0x | 0.5 | 0.00 | 0.03 | +| closures | Minify | 4.172 (3.984–4.276) | 1.0x | 0.3 | 0.88 | 0.03 | +| closures | Cff | 56.355 (55.326–60.841) | 13.5x | 3.0 | 4.38 | 0.19 | +| closures | Vmify | 367.655 (333.079–802.280) | 88.4x | 8.6 | 51.74 | 0.55 | +| closures | Medium | 624.017 (558.313–643.200) | 150.0x | 126.9 | 307.56 | 2.26 | +| closures | Strong | 3469.907 (3428.758–3753.370) | 834.1x | 85.5 | 744.30 | 2.19 | +| numeric | Original | 7.122 | 1.0x | 0.3 | 0.00 | 0.02 | +| numeric | Minify | 6.636 (6.604–7.202) | 0.9x | 0.2 | 0.97 | 0.02 | +| numeric | Cff | 42.093 (38.432–47.419) | 5.9x | 2.0 | 3.08 | 0.09 | +| numeric | Vmify | 1225.860 (1140.024–1256.133) | 172.1x | 7.5 | 37.72 | 0.31 | +| numeric | Medium | 1289.751 (1249.770–1340.387) | 181.1x | 116.7 | 285.25 | 2.50 | +| numeric | Strong | 11620.304 (11356.909–12034.328) | 1631.6x | 79.1 | 783.23 | 2.23 | +| strings | Original | 7.859 | 1.0x | 0.5 | 0.00 | 0.02 | +| strings | Minify | 7.688 (7.386–8.054) | 1.0x | 0.3 | 0.91 | 0.03 | +| strings | Cff | 12.442 (12.295–12.836) | 1.6x | 2.2 | 3.02 | 0.10 | +| strings | Vmify | 174.803 (164.895–212.237) | 22.2x | 8.2 | 42.41 | 0.29 | +| strings | Medium | 335.975 (307.087–444.885) | 42.7x | 116.7 | 297.74 | 1.91 | +| strings | Strong | 1615.032 (1583.821–1757.958) | 205.5x | 82.7 | 760.12 | 2.15 | +| tables | Original | 7.455 | 1.0x | 0.6 | 0.00 | 0.03 | +| tables | Minify | 6.387 (6.369–6.420) | 0.9x | 0.3 | 0.98 | 0.03 | +| tables | Cff | 53.220 (53.106–53.451) | 7.1x | 3.5 | 5.68 | 0.20 | +| tables | Vmify | 432.169 (417.021–444.719) | 58.0x | 9.4 | 48.11 | 0.28 | +| tables | Medium | 2166.725 (2006.319–2297.363) | 290.7x | 134.6 | 340.94 | 2.83 | +| tables | Strong | 4813.635 (4780.064–5371.881) | 645.7x | 89.2 | 861.25 | 2.27 | + +## luajit + +| Workload | Preset | Execution ms (median; seed range) | Slowdown | Output KiB | Generate ms | Parse ms | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| closures | Original | 1.949 | 1.0x | 0.5 | 0.00 | 0.03 | +| closures | Minify | 1.883 (1.786–2.360) | 1.0x | 0.3 | 0.88 | 0.04 | +| closures | Cff | 29.981 (29.700–31.358) | 15.4x | 3.0 | 4.38 | 0.16 | +| closures | Vmify | 106.835 (100.749–107.317) | 54.8x | 8.6 | 51.74 | 0.28 | +| closures | Medium | 190.523 (170.302–194.515) | 97.7x | 126.9 | 307.56 | 1.88 | +| closures | Strong | 1876.224 (997.935–2183.891) | 962.5x | 85.5 | 744.30 | 1.18 | +| numeric | Original | 1.023 | 1.0x | 0.3 | 0.00 | 0.02 | +| numeric | Minify | 0.900 (0.829–0.929) | 0.9x | 0.2 | 0.97 | 0.02 | +| numeric | Cff | 2.234 (1.907–2.475) | 2.2x | 2.0 | 3.08 | 0.08 | +| numeric | Vmify | 161.181 (140.687–163.614) | 157.6x | 7.5 | 37.72 | 0.21 | +| numeric | Medium | 214.538 (213.448–223.471) | 209.8x | 116.7 | 285.25 | 1.75 | +| numeric | Strong | 4668.719 (4596.866–5062.741) | 4564.8x | 79.1 | 783.23 | 1.76 | +| strings | Original | 3.660 | 1.0x | 0.5 | 0.00 | 0.06 | +| strings | Minify | 3.370 (3.239–3.586) | 0.9x | 0.3 | 0.91 | 0.02 | +| strings | Cff | 6.155 (5.645–6.319) | 1.7x | 2.2 | 3.02 | 0.08 | +| strings | Vmify | 57.848 (56.755–62.352) | 15.8x | 8.2 | 42.41 | 0.27 | +| strings | Medium | 96.007 (94.472–101.245) | 26.2x | 116.7 | 297.74 | 1.38 | +| strings | Strong | 749.924 (719.720–787.316) | 204.9x | 82.7 | 760.12 | 1.64 | +| tables | Original | 3.353 | 1.0x | 0.6 | 0.00 | 0.02 | +| tables | Minify | 3.497 (3.129–3.554) | 1.0x | 0.3 | 0.98 | 0.03 | +| tables | Cff | 17.727 (17.614–17.744) | 5.3x | 3.5 | 5.68 | 0.11 | +| tables | Vmify | 86.852 (79.791–98.998) | 25.9x | 9.4 | 48.11 | 0.30 | +| tables | Medium | 660.594 (638.101–702.064) | 197.0x | 134.6 | 340.94 | 2.06 | +| tables | Strong | 1792.047 (1663.428–1998.492) | 534.5x | 89.2 | 861.25 | 1.77 | + +## Observations + +All 120 generated variants and 8 originals returned matching checksums on Lua +5.1 and LuaJIT. Each row contributes three timed batches. Production Medium and +Strong retained AntiTamper. One initial benchmark process exceeded 60 seconds +while repeating slow warmups; the runner was adjusted to warm up slow workloads +once before its three measured batches, then resumed the saved measurements. +The completed measurements stayed within the per-process limit. + +Vmify is substantially slower than native Lua for these workloads, particularly +for numeric loops and closures. Strong adds a large further cost when Cff +flattens the interpreter. LuaJIT improves absolute execution times but does not +remove the VM overhead. These results warrant performance work before describing +v0.3 as highly performant; Strong is particularly costly on hot paths. + +The per-seed ranges also show that performance varies with the pipeline seed. +Use both absolute times and ranges when choosing a preset. Generation was run +under Lua 5.1 for every variant; execution was measured separately under both +runtimes. Raw JSON and generated sources from this run are in +`/tmp/prometheus-v0.3-benchmark/`. To reproduce, run the command in +[README.md](README.md). diff --git a/src/config.lua b/src/config.lua index dd6f8efb..9ab55f31 100644 --- a/src/config.lua +++ b/src/config.lua @@ -6,7 +6,7 @@ local NAME = "Prometheus"; local REVISION = "Alpha"; -local VERSION = "v0.2"; +local VERSION = "v0.3"; local BY = "levno-710"; for _, currArg in pairs(arg) do @@ -32,4 +32,4 @@ return { SPACE = " "; -- The Whitespace to be used by the unparser TAB = "\t"; -- The Tab Whitespace to be used by the unparser for pretty printing -} \ No newline at end of file +} From 458f89746562595e5d9a9db1c8589ddc06967f60 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:14:37 +0100 Subject: [PATCH 6/8] optimize runtime speed --- Dockerfile | 1 + benchmarks/README.md | 3 + benchmarks/run.py | 8 +- benchmarks/v0.3-medium-native-cff.md | 67 ++++++ benchmarks/v0.3-results.md | 3 + doc/advanced/register-vm.md | 52 +++- doc/guides/presets.md | 26 +- doc/reference/steps/cff.md | 14 +- doc/reference/steps/encryptstrings.md | 18 +- src/presets.lua | 7 +- src/prometheus/cff/compile_core.lua | 3 + src/prometheus/cff/compile_top.lua | 1 + src/prometheus/cff/compiler.lua | 1 + src/prometheus/cff/native.lua | 151 ++++++++++++ src/prometheus/steps/Cff.lua | 13 +- src/prometheus/steps/EncryptStrings.lua | 6 +- src/prometheus/vm/emit.lua | 1 + src/prometheus/vm/strings.lua | 38 +++ tests/harness/cff-native.lua | 300 ++++++++++++++++++++++++ 19 files changed, 686 insertions(+), 27 deletions(-) create mode 100644 benchmarks/v0.3-medium-native-cff.md create mode 100644 src/prometheus/cff/native.lua create mode 100644 src/prometheus/vm/strings.lua create mode 100644 tests/harness/cff-native.lua diff --git a/Dockerfile b/Dockerfile index 77dc084b..d812a19f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,6 +27,7 @@ COPY . /app RUN lua5.1 tests/harness/metadata.lua \ && timeout 60s lua5.1 tests/harness/numbers-to-expressions.lua \ && timeout 60s lua5.1 tests/harness/cff-arity.lua \ + && timeout 60s lua5.1 tests/harness/cff-native.lua \ && timeout 60s lua5.1 tests/harness/vm-arity.lua \ && timeout 60s lua5.1 tests/harness/vm-prototypes.lua \ && timeout 60s lua5.1 tests/harness/vm-layout.lua \ diff --git a/benchmarks/README.md b/benchmarks/README.md index 1debfe60..f15235e6 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,6 +1,7 @@ # Preset benchmarks Recorded v0.3 measurements: [v0.3-results.md](v0.3-results.md). +Updated Medium measurements: [v0.3-medium-native-cff.md](v0.3-medium-native-cff.md). Run from the repository root with Python 3, Lua 5.1, and LuaJIT installed: @@ -34,5 +35,7 @@ CPU seconds per batch), `--timeout 60` (seconds per process), and `--runtimes lua5.1 luajit`. The LuaJIT run leaves JIT enabled. Tiny native JIT times can produce large slowdown ratios; compare absolute times as well. +Use `--presets Medium` to benchmark a subset of presets against original source. + Use `--resume` with the same output directory and options to reuse completed measurements after an interrupted run. diff --git a/benchmarks/run.py b/benchmarks/run.py index e639c713..2164b686 100644 --- a/benchmarks/run.py +++ b/benchmarks/run.py @@ -63,7 +63,7 @@ def report(data, output): for case in data['cases']: rows = [row for row in data['results'] if row['runtime'] == runtime and row['case'] == case] baseline = statistics.median(next(row for row in rows if row['preset'] == 'Original')['seconds']) - for preset in ('Original',) + PRESETS: + for preset in ('Original',) + tuple(data.get('presets', PRESETS)): selected = [row for row in rows if row['preset'] == preset] seconds = statistics.median([t for row in selected for t in row['seconds']]) seed_times = [statistics.median(row['seconds']) * 1000 for row in selected] @@ -87,6 +87,7 @@ def main(): parser.add_argument('--target', type=float, default=0.1) parser.add_argument('--timeout', type=int, default=60) parser.add_argument('--runtimes', nargs='+', default=['lua5.1', 'luajit']) + parser.add_argument('--presets', nargs='+', choices=PRESETS, default=list(PRESETS)) parser.add_argument('--resume', action='store_true', help='reuse completed rows in the output directory') args = parser.parse_args() if args.samples < 1 or args.target <= 0 or args.timeout < 1: @@ -97,18 +98,19 @@ def main(): assert cases, 'No benchmark cases found' data = dict(host=platform.platform(), cpu=cpu_model(), seeds=args.seeds, samples=args.samples, target_seconds=args.target, cases=[case.stem for case in cases], - versions={}, results=[]) + presets=args.presets, versions={}, results=[]) if args.resume: previous = json.loads((output / 'results.json').read_text()) for key in ('seeds', 'samples', 'target_seconds', 'cases'): assert previous[key] == data[key], f'Resume configuration mismatch: {key}' + assert previous.get('presets', list(PRESETS)) == args.presets, 'Resume preset mismatch' data['results'] = previous['results'] for runtime in args.runtimes: version = subprocess.run([runtime, '-v'], text=True, capture_output=True, check=True) data['versions'][runtime] = (version.stdout + version.stderr).strip() for case in cases: generated = [('Original', 0, case, 0.0)] - for preset in PRESETS: + for preset in args.presets: for seed in args.seeds: path = output / f'{case.stem}-{preset}-{seed}.lua' completed = next((row for row in data['results'] if row['case'] == case.stem diff --git a/benchmarks/v0.3-medium-native-cff.md b/benchmarks/v0.3-medium-native-cff.md new file mode 100644 index 00000000..f6128b82 --- /dev/null +++ b/benchmarks/v0.3-medium-native-cff.md @@ -0,0 +1,67 @@ +# Medium with native VM and Cff-protected helpers + +Host: Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.44; CPU: 13th Gen Intel(R) Core(TM) i7-13850HX. Runtimes: Lua 5.1.5 Copyright (C) 1994-2012 Lua.org, PUC-Rio; LuaJIT 2.1.1792588164 -- Copyright (C) 2005-2026 Mike Pall. https://luajit.org/. + +Four deterministic workloads compare original source with the actual presets, including AntiTamper. All variants must match the original checksum. + +Seeds: [1, 4, 17]; 3 timed batches per seed. Each workload is parsed once and warmed up; batches adapt to approximately 0.1 seconds. Execution includes initialization of the emitted chunk, workload work, allocations, and checksum validation, but excludes parsing. Garbage collection runs before each batch; collection during the batch is included. + +Execution figures are median CPU time per full invocation across batches and seeds. Generation and parsing figures are medians across seeds. Output size is a median across seeds. Execution ranges show the lowest and highest per-seed medians. Measurements are local, with JIT enabled for LuaJIT; very small native JIT times can produce large slowdown ratios. These workloads do not establish a general performance guarantee or measure peak memory. + +## lua5.1 + +| Workload | Preset | Execution ms (median; seed range) | Slowdown | Output KiB | Generate ms | Parse ms | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| closures | Original | 4.597 | 1.0x | 0.5 | 0.00 | 0.04 | +| closures | Medium | 415.504 (354.859–424.158) | 90.4x | 69.4 | 212.82 | 2.00 | +| numeric | Original | 6.637 | 1.0x | 0.3 | 0.00 | 0.02 | +| numeric | Medium | 1167.980 (1137.079–1191.847) | 176.0x | 61.8 | 186.06 | 1.88 | +| strings | Original | 8.361 | 1.0x | 0.5 | 0.00 | 0.02 | +| strings | Medium | 190.476 (178.980–191.010) | 22.8x | 65.1 | 198.95 | 1.33 | +| tables | Original | 5.742 | 1.0x | 0.6 | 0.00 | 0.03 | +| tables | Medium | 398.476 (386.410–410.053) | 69.4x | 71.5 | 237.49 | 2.05 | + +## luajit + +| Workload | Preset | Execution ms (median; seed range) | Slowdown | Output KiB | Generate ms | Parse ms | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| closures | Original | 1.882 | 1.0x | 0.5 | 0.00 | 0.03 | +| closures | Medium | 120.180 (113.183–121.412) | 63.8x | 69.4 | 212.82 | 1.68 | +| numeric | Original | 0.833 | 1.0x | 0.3 | 0.00 | 0.02 | +| numeric | Medium | 151.332 (151.263–154.918) | 181.6x | 61.8 | 186.06 | 1.03 | +| strings | Original | 3.540 | 1.0x | 0.5 | 0.00 | 0.03 | +| strings | Medium | 52.989 (48.474–54.711) | 15.0x | 65.1 | 198.95 | 1.29 | +| tables | Original | 3.467 | 1.0x | 0.6 | 0.00 | 0.05 | +| tables | Medium | 84.554 (81.895–94.960) | 24.4x | 71.5 | 237.49 | 1.42 | + +## Comparison with the previous Medium + +The previous figures come from the earlier local [v0.3 benchmark](v0.3-results.md). +These are separate runs on the same host; small changes can include timing +variation. Both runs used seeds 1, 4, and 17, three timed batches, and active +AntiTamper. The final pipeline preserves the VM and prototype initializers, +flattens surrounding helpers, and hoists encrypted interpreter strings into +native locals after startup decoding. Number rewriting follows Cff. + +| Runtime | Workload | Previous Medium ms | Current Medium ms | Speedup | +| --- | --- | ---: | ---: | ---: | +| lua5.1 | closures | 624.017 | 415.504 | 1.50x | +| lua5.1 | numeric | 1289.751 | 1167.980 | 1.10x | +| lua5.1 | strings | 335.975 | 190.476 | 1.76x | +| lua5.1 | tables | 2166.725 | 398.476 | 5.44x | +| luajit | closures | 190.523 | 120.180 | 1.59x | +| luajit | numeric | 214.538 | 151.332 | 1.42x | +| luajit | strings | 96.007 | 52.989 | 1.81x | +| luajit | tables | 660.594 | 84.554 | 7.81x | + +All 24 generated variants and 8 originals matched checksums, with 96 timed +batches. The core VM remains substantially slower than native Lua. This change +reduces the additional overhead of Medium while retaining Cff protection for +its decoder and surrounding helpers. + +Raw JSON and generated sources: `/tmp/prometheus-medium-native-cff-final-benchmark/`. +Reproduce with: + +```sh +python3 benchmarks/run.py --presets Medium +``` diff --git a/benchmarks/v0.3-results.md b/benchmarks/v0.3-results.md index 3c30d6e4..3d130d45 100644 --- a/benchmarks/v0.3-results.md +++ b/benchmarks/v0.3-results.md @@ -1,5 +1,8 @@ # Prometheus v0.3 benchmark +These measurements precede Medium's native VM and Cff-protected helper changes. +See the [updated Medium report](v0.3-medium-native-cff.md) for the current preset. + Host: Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.44; CPU: 13th Gen Intel(R) Core(TM) i7-13850HX. Runtimes: Lua 5.1.5 Copyright (C) 1994-2012 Lua.org, PUC-Rio; LuaJIT 2.1.1792588164 -- Copyright (C) 2005-2026 Mike Pall. https://luajit.org/. Four deterministic workloads compare original source with the actual presets, including AntiTamper. All variants must match the original checksum. diff --git a/doc/advanced/register-vm.md b/doc/advanced/register-vm.md index ca5fdeb2..1f363a5a 100644 --- a/doc/advanced/register-vm.md +++ b/doc/advanced/register-vm.md @@ -18,6 +18,8 @@ compiler, or `loadstring` to execute the original source. positions. - `vm/emit.lua`: instruction encoding, native prototype table literals, and the runtime function IIFE. +- `vm/strings.lua`: optional hoisting of generated interpreter string literals + into an outer initializer. - `vm/flatten.lua`: the Cff integration for generated VM interpreters. A prototype contains instructions, constants, child prototypes, parameter @@ -36,15 +38,29 @@ consume their test register on both paths. `Cff` normally flattens control flow throughout its input AST. When it receives the AST emitted by `Vmify`, a private `vmRuntime` annotation selects a special -path: `vm/flatten.lua` flattens the interpreter function only. The large native -prototype table initialization remains outside the flattened function, keeping -it out of the Cff control structure and jump limits used by Lua and LuaJIT. +path: by default, `vm/flatten.lua` flattens the interpreter function only. The +large native prototype table initialization remains outside the flattened +function, keeping it out of the Cff control structure and jump limits used by +Lua and LuaJIT. -The special path passes read-only external lexical helpers through function -arguments. This supports helpers introduced by intervening steps such as -`EncryptStrings`. If the interpreter writes to an external lexical variable, -the special path falls back to ordinary Cff compilation to preserve that -write's behavior. +Set `Cff`'s `ExcludeVm` setting to `true` to leave both the generated +interpreter and the native prototype table literal out of flattening. Cff then +flattens the surrounding code, including helpers introduced by intervening +steps such as string decoders or tamper checks. The interpreter remains native +Lua, so its lexical captures continue to share variables with surrounding +closures according to ordinary Lua upvalue semantics. + +`cff/native.lua` emits the excluded expressions directly and bridges their +external locals through Cff's shared upvalue cells. Garbage-collected capture +proxies keep those cells alive for native closures; captures are not value +snapshots. The `vmData` annotation identifies the entire prototype initializer, +including its child prototypes. + +With the default `ExcludeVm = false` behavior, the special path passes +read-only external lexical helpers through function arguments. This supports +helpers introduced by intervening steps such as `EncryptStrings`. If the +interpreter writes to an external lexical variable, the special path falls +back to ordinary Cff compilation to preserve that write's behavior. The compiler keeps canonical semantic instruction IDs. At emission, `layout.lua` chooses a permutation of the opcode IDs and a permutation of operand slots 2 @@ -59,9 +75,25 @@ slot, then checks the opcode for an exact ID at each leaf before running its handler. For the same input, configuration, and Lua runtime, a fixed pipeline seed reproduces the layout. Different seeds can produce different layouts. +In Medium, `EncryptStrings` uses `HoistVmStrings = true` after `Vmify`. It moves +the generated interpreter's literal strings into an outer initializer, where +normal string encryption and later `ConstantArray` processing apply. The +initializer decodes them once at startup and binds them to native lexical +locals used by the interpreter, avoiding decoder or wrapper calls on hot +paths. Strings in `vmData` prototype tables remain encrypted by the normal +`EncryptStrings` behavior. `vm/strings.lua` implements this transformation. + +Medium then applies `AntiTamper`, `ConstantArray`, and `WrapInFunction`, +followed by `Cff` with `ExcludeVm = true`. The interpreter and prototype table +initializers stay native while Cff flattens the surrounding code and generated +helpers. `NumbersToExpressions` runs after Cff: it adds no helper functions, +and this ordering prevents Cff from virtualizing arithmetic constant +expressions. + `Strong` applies one `Vmify` pass followed by `EncryptStrings`, `AntiTamper`, -and one `Cff` pass over the generated VM interpreter. `Weak` retains the -standalone `Cff` transformation without the VM. +and one default `Cff` pass over the generated VM interpreter. It retains the +default `HoistVmStrings = false` and `ExcludeVm = false` settings. `Weak` +retains the standalone `Cff` transformation without the VM. Runtime compatibility follows the documented [limitations](../reference/limitations.md). Native debug representation and diff --git a/doc/guides/presets.md b/doc/guides/presets.md index ff4e8804..aa85ab78 100644 --- a/doc/guides/presets.md +++ b/doc/guides/presets.md @@ -41,12 +41,26 @@ This test-oriented preset compiles the input to the register VM with one `Vmify` Steps: -1. `EncryptStrings` -2. `AntiTamper` (`UseDebug = false`) -3. `Vmify` +1. `Vmify` +2. `EncryptStrings` (`HoistVmStrings = true`) +3. `AntiTamper` (`UseDebug = false`) 4. `ConstantArray` -5. `NumbersToExpressions` -6. `WrapInFunction` +5. `WrapInFunction` +6. `Cff` (`ExcludeVm = true`) +7. `NumbersToExpressions` + +`HoistVmStrings` moves interpreter string literals into an outer initializer; +that initializer follows the regular encryption and constant array flow, +decodes the strings once at startup, then binds them to native lexical locals +inside the interpreter. This avoids decoder and wrapper calls on the +interpreter's hot paths. Strings in `vmData` remain encrypted normally. + +`Cff` runs after the helper-producing steps, so it flattens their surrounding +code, including string decoder, tamper check, constant array, and wrapper +code. The generated VM interpreter and its native prototype table +initializers remain outside Cff flattening. `NumbersToExpressions` runs after +Cff because it adds no helper functions and keeps arithmetic constant +expressions from being virtualized by Cff. ## Strong @@ -60,7 +74,7 @@ Steps: 6. `NumbersToExpressions` 7. `WrapInFunction` -In `Strong`, `Vmify` compiles the input first, then `Cff` flattens control flow in the generated VM Lua code. This preset contains one VM pass. +In `Strong`, `Vmify` compiles the input first, then `Cff` flattens control flow in the generated VM Lua code. It keeps the defaults `HoistVmStrings = false` and `ExcludeVm = false`; the preset is unchanged and contains one VM pass. ## Choosing quickly diff --git a/doc/reference/steps/cff.md b/doc/reference/steps/cff.md index 11db4ccf..5ce7632f 100644 --- a/doc/reference/steps/cff.md +++ b/doc/reference/steps/cff.md @@ -4,11 +4,15 @@ Constructor key: `Cff`; `ControlFlowFlattening` is also accepted. `Cff` flattens control flow into Lua statements executed by a block dispatcher. It remains available alongside the register VM step, `Vmify`. -When `Cff` receives `Vmify` output, it recognizes the private VM runtime annotation and flattens the interpreter function only. The prototype tables remain outside the flattened function so large data initializers do not expand the control flow graph beyond Lua or LuaJIT limits. Read-only external lexical helpers are forwarded as function arguments; if the interpreter writes to an external lexical variable, `Cff` uses its ordinary compilation path instead. +When `Cff` receives `Vmify` output, it recognizes the private VM runtime annotation. By default, it flattens the interpreter function and leaves the prototype tables outside it, so large data initializers do not expand the control flow graph beyond Lua or LuaJIT limits. Read-only external lexical helpers are forwarded as function arguments; if the interpreter writes to an external lexical variable, `Cff` uses its ordinary compilation path instead. + +Set `ExcludeVm` to `true` to leave the generated interpreter and native prototype table literal out of Cff flattening. Cff then flattens the surrounding code, including helpers such as decoders and tamper checks, and the outer string initializer emitted by `EncryptStrings` when `HoistVmStrings` is enabled. The interpreter keeps its native Lua lexical captures, preserving shared variable behavior between the interpreter and surrounding closures. ## Settings -None. +| Setting | Type | Default | Description | +| --- | --- | --- | --- | +| `ExcludeVm` | boolean | `false` | When true, leave the generated VM interpreter and prototype table literal out of Cff flattening. | ## Example @@ -16,4 +20,10 @@ None. { Name = "Cff", Settings = {} } ``` +To flatten code surrounding a generated VM while leaving the VM itself native: + +```lua +{ Name = "Cff", Settings = { ExcludeVm = true } } +``` + Use `Cff` when you want the control flow flattening behavior. Existing configs that used the old `Vmify` key for this behavior should use `Cff` now. The `Vmify` key compiles code to the register VM. diff --git a/doc/reference/steps/encryptstrings.md b/doc/reference/steps/encryptstrings.md index 313971e8..2c0717ab 100644 --- a/doc/reference/steps/encryptstrings.md +++ b/doc/reference/steps/encryptstrings.md @@ -6,10 +6,26 @@ Encrypts string literals and injects runtime decryption logic. ## Settings -None. +| Setting | Type | Default | Description | +| --- | --- | --- | --- | +| `HoistVmStrings` | boolean | `false` | With `Vmify` output, hoist string literals from the generated interpreter into an outer initializer. | + +When enabled after `Vmify`, interpreter strings are encrypted by the normal +`EncryptStrings` flow and can be processed by later steps such as +`ConstantArray` and `Cff`. The initializer decodes them once at startup and +binds them to native lexical locals used by the interpreter, avoiding decoder +or wrapper calls on interpreter hot paths. Strings in the VM's `vmData` +prototype tables remain encrypted by the normal `EncryptStrings` behavior. +Without `Vmify` output, this setting has no effect. ## Example ```lua { Name = "EncryptStrings", Settings = {} } ``` + +To hoist interpreter strings in a pipeline after `Vmify`: + +```lua +{ Name = "EncryptStrings", Settings = { HoistVmStrings = true } } +``` diff --git a/src/presets.lua b/src/presets.lua index 3c512ed3..d5a29e5c 100644 --- a/src/presets.lua +++ b/src/presets.lua @@ -69,14 +69,14 @@ return { PrettyPrint = false, Seed = 0, Steps = { - { Name = "EncryptStrings", Settings = {} }, + { Name = "Vmify", Settings = {} }, + { Name = "EncryptStrings", Settings = { HoistVmStrings = true } }, { Name = "AntiTamper", Settings = { UseDebug = false, }, }, - { Name = "Vmify", Settings = {} }, { Name = "ConstantArray", Settings = { @@ -87,8 +87,9 @@ return { LocalWrapperThreshold = 0, }, }, - { Name = "NumbersToExpressions", Settings = {} }, { Name = "WrapInFunction", Settings = {} }, + { Name = "Cff", Settings = { ExcludeVm = true } }, + { Name = "NumbersToExpressions", Settings = {} }, }, }, diff --git a/src/prometheus/cff/compile_core.lua b/src/prometheus/cff/compile_core.lua index af183011..ef779ab7 100644 --- a/src/prometheus/cff/compile_core.lua +++ b/src/prometheus/cff/compile_core.lua @@ -23,6 +23,9 @@ return function(Compiler) end function Compiler:compileExpression(expression, funcDepth, numReturns) + if self.nativeExpressions and self.nativeExpressions[expression] then + return self:compileNativeExpression(expression, funcDepth, numReturns); + end local handler = expressionHandlers[expression.kind]; if handler then local kind = expression.kind; diff --git a/src/prometheus/cff/compile_top.lua b/src/prometheus/cff/compile_top.lua index db99125c..b3a77c8b 100644 --- a/src/prometheus/cff/compile_top.lua +++ b/src/prometheus/cff/compile_top.lua @@ -13,6 +13,7 @@ local AstKind = Ast.AstKind; return function(Compiler) function Compiler:compileTopNode(node) + if self.nativeExpressions then self:prepareNativeExpressions(node); end local startBlock = self:createBlock(); local scope = startBlock.scope; self.startBlockId = startBlock.id; diff --git a/src/prometheus/cff/compiler.lua b/src/prometheus/cff/compiler.lua index 5968dd32..305799bf 100644 --- a/src/prometheus/cff/compiler.lua +++ b/src/prometheus/cff/compiler.lua @@ -65,6 +65,7 @@ registerModule(Compiler); upvalueModule(Compiler); emitModule(Compiler); compileCoreModule(Compiler); +require("prometheus.cff.native")(Compiler); function Compiler:pushRegisterUsageInfo() table.insert(self.registerUsageStack, { diff --git a/src/prometheus/cff/native.lua b/src/prometheus/cff/native.lua new file mode 100644 index 00000000..098422d0 --- /dev/null +++ b/src/prometheus/cff/native.lua @@ -0,0 +1,151 @@ +-- Keep selected expressions native, bridging captures through Cff's shared cells. +local Ast = require('prometheus.ast') +local Parser = require('prometheus.parser') +local Unparser = require('prometheus.unparser') +local Kind = Ast.AstKind + +local function walk(expression, ast, previsit, postvisit) + return require('prometheus.visitast')(Ast.TopNode(Ast.Block({ + Ast.ReturnStatement({expression}), + }, ast.body.scope), ast.globalScope), previsit, postvisit) +end + +local function captures(expression, ast) + local owned, variables, seen = {}, {}, {} + walk(expression, ast, function(node) + if node.kind == Kind.Block and node.scope ~= ast.body.scope then + owned[node.scope] = true + end + end) + walk(expression, ast, function(node) + if (node.kind == Kind.VariableExpression or node.kind == Kind.AssignmentVariable + or node.kind == Kind.FunctionDeclaration) and not node.scope.isGlobal + and not owned[node.scope] then + seen[node.scope] = seen[node.scope] or {} + if not seen[node.scope][node.id] then + variables[#variables + 1] = {scope = node.scope, id = node.id} + seen[node.scope][node.id] = #variables + end + end + end) + return variables, seen, owned +end + +return function(Compiler) + function Compiler:prepareNativeExpressions(ast) + self.nativeAst, self.nativeResults = ast, {} + for expression in pairs(self.nativeExpressions) do + local variables = captures(expression, ast) + for _, variable in ipairs(variables) do + self:makeUpvalue(variable.scope, variable.id) + end + end + end + + function Compiler:compileNativeExpression(expression, funcDepth, numReturns) + local ast, scope = self.nativeAst, self.activeBlock.scope + local variables, lookup, owned = captures(expression, ast) + local entries, reads = {}, {} + for _, variable in ipairs(variables) do + local depth = self.scopeFunctionDepths[variable.scope] + local id + if depth == funcDepth then + local register = self:getVarRegister(variable.scope, variable.id, funcDepth) + id = self:register(scope, register) + reads[#reads + 1] = register + else + local index = self:getUpvalueId(variable.scope, variable.id) + scope:addReferenceToHigherScope(self.containerFuncScope, self.currentUpvaluesVar) + id = Ast.IndexExpression(Ast.VariableExpression(self.containerFuncScope, + self.currentUpvaluesVar), Ast.NumberExpression(index)) + end + entries[#entries + 1] = Ast.TableEntry(id) + end + + -- Fresh marker names avoid shadowing any names inside the native expression. + local used = {} + walk(expression, ast, function(node) + if node.scope then + for _, name in pairs(node.scope.variables) do used[name] = true end + end + end) + local names = {} + for _, label in ipairs({'proxy', 'env', 'values', 'setenv', 'result'}) do + local name = '__prometheus_native_' .. label + while used[name] do name = name .. '_' end + used[name], names[label] = true, name + end + local function marker(label) + local _, id = ast.globalScope:resolve(names[label]) + return Ast.VariableExpression(ast.globalScope, id) + end + local function access(node, assignment) + local base, index + if node.scope.isGlobal then + local name = node.scope:getVariableName(node.id) + if name == '_ENV' and not assignment then return marker('env') end + base, index = marker('env'), Ast.StringExpression(name) + else + local slot = lookup[node.scope] and lookup[node.scope][node.id] + if not slot then return end + base = marker('values') + index = Ast.IndexExpression(marker('proxy'), Ast.NumberExpression(slot)) + end + if assignment then return Ast.AssignmentIndexing(base, index) end + return Ast.IndexExpression(base, index) + end + walk(expression, ast, nil, function(node) + if node.kind == Kind.VariableExpression then return access(node, false) end + if node.kind == Kind.AssignmentVariable then return access(node, true) end + if node.kind == Kind.FunctionDeclaration and not owned[node.scope] then + local target = access(node, false) + if target then + for _, key in ipairs(node.indices) do + target = Ast.IndexExpression(target, Ast.StringExpression(key)) + end + return Ast.AssignmentStatement({Ast.AssignmentIndexing(target.base, target.index)}, + {Ast.FunctionLiteralExpression(node.args, node.body)}) + end + end + end) + + local source = Unparser:new({LuaVersion = 'Lua51'}):unparseExpression(expression, 0) + local finish = '' + if expression.kind == Kind.FunctionLiteralExpression then + finish = 'if ' .. names.setenv .. ' then ' .. names.setenv .. '(' .. names.result + .. ',' .. names.env .. ') end;' + end + local parsed = Parser:new({LuaVersion = 'Lua51'}):parse('return function(' + .. names.proxy .. ',' .. names.env .. ',' .. names.values .. ') local ' + .. names.result .. '=' .. source .. ';' .. finish .. 'return ' .. names.result .. ' end') + local factory = parsed.body.statements[1].args[1] + self.nativeResults[expression] = factory.body.statements[1].expressions[1] + factory.body.scope:setParent(scope) + require('prometheus.visitast')(parsed, nil, function(node, data) + if node.kind == Kind.VariableExpression and node.scope.isGlobal then + assert(node.scope:getVariableName(node.id) == names.setenv, 'Unexpected native global') + data.scope:removeReferenceToHigherScope(node.scope, node.id) + data.scope:addReferenceToHigherScope(self.scope, self.setfenvVar) + return Ast.VariableExpression(self.scope, self.setfenvVar) + end + end) + local proxy = Ast.NilExpression() + if #entries > 0 then + scope:addReferenceToHigherScope(self.scope, self.upvaluesProxyFunctionVar) + proxy = Ast.FunctionCallExpression(Ast.VariableExpression(self.scope, + self.upvaluesProxyFunctionVar), {Ast.TableConstructorExpression(entries)}) + end + scope:addReferenceToHigherScope(self.scope, self.upvaluesTable) + local value = Ast.FunctionCallExpression(factory, {proxy, self:env(scope), + Ast.VariableExpression(self.scope, self.upvaluesTable)}) + local result = self:allocRegister(false) + self:addStatement(self:setRegister(scope, result, value), {result}, reads, true) + local registers = {result} + for i = 2, numReturns do + registers[i] = self:allocRegister(false) + self:addStatement(self:setRegister(scope, registers[i], Ast.NilExpression()), + {registers[i]}, {}, false) + end + return registers + end +end diff --git a/src/prometheus/steps/Cff.lua b/src/prometheus/steps/Cff.lua index b9c2cf5e..bc197012 100644 --- a/src/prometheus/steps/Cff.lua +++ b/src/prometheus/steps/Cff.lua @@ -12,11 +12,22 @@ local Cff = Step:extend(); Cff.Description = "Flattens control flow into Lua statements executed by a randomized block dispatcher."; Cff.Name = "Cff"; -Cff.SettingsDescriptor = {} +Cff.SettingsDescriptor = { + ExcludeVm = {type = "boolean", default = false}, +} function Cff:init(_) end function Cff:apply(ast) + if self.ExcludeVm and ast.vmRuntime then + local compiler = Compiler:new(); + compiler.nativeExpressions = {[ast.vmRuntime] = true}; + if ast.vmData then compiler.nativeExpressions[ast.vmData] = true; end + local flattened = compiler:compile(ast); + flattened.vmRuntime = compiler.nativeResults[ast.vmRuntime]; + flattened.vmData = compiler.nativeResults[ast.vmData]; + return flattened; + end if ast.vmRuntime then local flattened = flattenVm(ast); if flattened then return flattened; end diff --git a/src/prometheus/steps/EncryptStrings.lua b/src/prometheus/steps/EncryptStrings.lua index a7e031f7..7a40c21f 100644 --- a/src/prometheus/steps/EncryptStrings.lua +++ b/src/prometheus/steps/EncryptStrings.lua @@ -10,13 +10,16 @@ local Parser = require("prometheus.parser") local Enums = require("prometheus.enums") local visitast = require("prometheus.visitast"); local util = require("prometheus.util") +local hoistVmStrings = require("prometheus.vm.strings") local AstKind = Ast.AstKind; local EncryptStrings = Step:extend() EncryptStrings.Description = "This Step will encrypt strings within your Program." EncryptStrings.Name = "Encrypt Strings" -EncryptStrings.SettingsDescriptor = {} +EncryptStrings.SettingsDescriptor = { + HoistVmStrings = {type = "boolean", default = false}, +} function EncryptStrings:init(_) end @@ -187,6 +190,7 @@ end]] end function EncryptStrings:apply(ast, _) + if self.HoistVmStrings then hoistVmStrings(ast); end local Encryptor = self:CreateEncryptionService(); local code = Encryptor.genCode(); diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua index 61bcb385..619fcc40 100644 --- a/src/prometheus/vm/emit.lua +++ b/src/prometheus/vm/emit.lua @@ -44,6 +44,7 @@ end function Emit.ast(proto) local ast = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(Emit.source(proto)) ast.vmRuntime = ast.body.statements[1].args[1].base + ast.vmData = ast.body.statements[1].args[1].args[1] return ast end diff --git a/src/prometheus/vm/strings.lua b/src/prometheus/vm/strings.lua new file mode 100644 index 00000000..edeab7c1 --- /dev/null +++ b/src/prometheus/vm/strings.lua @@ -0,0 +1,38 @@ +-- Cache interpreter literals at startup so protected decoders stay off its hot path. +local Ast = require('prometheus.ast') +local visitast = require('prometheus.visitast') +local Kind = Ast.AstKind + +return function(ast) + local runtime, scope = ast.vmRuntime, ast.body.scope + if not runtime then return end + local cacheScope = runtime.body.scope + local data, cache + local entries, ids, variables, values = {}, {}, {}, {} + local wrapper = Ast.TopNode(Ast.Block({Ast.ReturnStatement({runtime})}, scope), ast.globalScope) + visitast(wrapper, nil, function(node, context) + if node.kind == Kind.StringExpression then + if not data then data, cache = scope:addVariable(), cacheScope:addVariable() end + local id = ids[node.value] + if not id then + id = #entries + 1 + ids[node.value] = id + entries[id] = Ast.TableEntry(Ast.StringExpression(node.value)) + variables[id] = cacheScope:addVariable() + values[id] = Ast.IndexExpression(Ast.VariableExpression(cacheScope, cache), + Ast.NumberExpression(id)) + cacheScope:addReferenceToHigherScope(cacheScope, cache) + end + context.scope:addReferenceToHigherScope(cacheScope, variables[id]) + return Ast.VariableExpression(cacheScope, variables[id]) + end + end) + if not data then return end + cacheScope:addReferenceToHigherScope(scope, data) + table.insert(runtime.body.statements, 1, + Ast.LocalVariableDeclaration(cacheScope, {cache}, {Ast.VariableExpression(scope, data)})) + table.insert(runtime.body.statements, 2, + Ast.LocalVariableDeclaration(cacheScope, variables, values)) + table.insert(ast.body.statements, 1, + Ast.LocalVariableDeclaration(scope, {data}, {Ast.TableConstructorExpression(entries)})) +end diff --git a/tests/harness/cff-native.lua b/tests/harness/cff-native.lua new file mode 100644 index 00000000..5a9c2eb2 --- /dev/null +++ b/tests/harness/cff-native.lua @@ -0,0 +1,300 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error + +local Parser = require('prometheus.parser') +local Unparser = require('prometheus.unparser') +local Ast = require('prometheus.ast') +local visitast = require('prometheus.visitast') +local Steps = require('prometheus.steps') +local Presets = require('src.presets') +local CffCompiler = require('prometheus.cff.compiler') + +local function pack(...) return {n = select('#', ...), ...} end +local function compile(source, steps, seed) + local generated = Prometheus.Pipeline:fromConfig({ + LuaVersion = 'Lua51', Seed = seed or 1, Steps = steps, + }):apply(source) + return assert(loadstring(generated)), generated +end + +local function countKind(root, kind) + local count = 0 + visitast(root, function(node) if node.kind == kind then count = count + 1 end end) + return count +end +local function controlFlowCounts(root) + return { + whileCount = countKind(root, Ast.AstKind.WhileStatement), + ifCount = countKind(root, Ast.AstKind.IfStatement), + functionCount = countKind(root, Ast.AstKind.FunctionLiteralExpression), + } +end +local function visitExpression(expression, parentAst, visitor) + local wrapper = Ast.TopNode(Ast.Block({Ast.ReturnStatement({expression}),}, parentAst.body.scope), + parentAst.globalScope) + visitast(wrapper, visitor) +end +local function applyCff(step, ast, pipeline) + local originalNew, compiler = CffCompiler.new + CffCompiler.new = function(class, ...) + local instance = originalNew(class, ...) + local originalCompile = instance.compile + instance.compile = function(self, input) + local output = originalCompile(self, input) + compiler = self + return output + end + return instance + end + local ok, result = pcall(function() return step:apply(ast, pipeline) end) + CffCompiler.new = originalNew + if not ok then error(result) end + return result, compiler +end + +-- Cff keeps a VM runtime function and its prototype table native, while compiling +-- the surrounding helper code into the normal randomized dispatcher. +local mediumSteps = {} +for i, step in ipairs(Presets.Medium.Steps) do + mediumSteps[i] = {Name = step.Name, Settings = step.Settings} +end +local mediumSource = [[ + local function branch(value) + if value < 0 then return 'negative' end + if value == 0 then return 'zero' end + return 'positive' + end + return branch(...), nil, ... +]] +for _, seed in ipairs({1, 4, 17}) do + local chunk = compile(mediumSource, mediumSteps, seed) + local result = pack(chunk(0, nil, 9, nil)) + assert(result.n == 6 and result[1] == 'zero' and result[2] == nil + and result[3] == 0 and result[4] == nil and result[5] == 9 and result[6] == nil, + 'Medium with active AntiTamper changed result arity or values at seed ' .. seed) +end + +-- Exercise ExcludeVm directly with an annotated native function that mutates +-- both captured locals and globals, and calls a recursive helper after the +-- original chunk has returned. +local closureAst = Parser:new({LuaVersion = 'Lua51'}):parse([[ + local outsideCounter = 0 + local sharedClosure = {value = 0} + local nativeFunction, recursiveHelper + nativeFunction = function(delta) + outsideCounter = outsideCounter + delta + sharedClosure.value = sharedClosure.value + 1 + cffNativeGlobal = (cffNativeGlobal or 0) + 1 + if delta > 0 then return recursiveHelper(delta - 1) end + return nil, outsideCounter, nil, sharedClosure.value + end + recursiveHelper = function(delta) + if delta > 0 then return nativeFunction(delta - 1) end + return nil, outsideCounter, nil, sharedClosure.value + end + return nativeFunction +]]) +local nativeFunction, nativeTable +visitast(closureAst, function(node) + if node.kind == Ast.AstKind.FunctionLiteralExpression and not nativeFunction then + nativeFunction = node + elseif node.kind == Ast.AstKind.TableConstructorExpression and not nativeTable then + nativeTable = node + end +end) +assert(nativeFunction and nativeTable) +closureAst.vmRuntime, closureAst.vmData = nativeFunction, nativeTable +local previousGlobal = cffNativeGlobal +cffNativeGlobal = nil +local cffAst, closureCompiler = applyCff(Steps.Cff:new({ExcludeVm = true}), closureAst) +assert(cffAst.vmRuntime and cffAst.vmData, 'Cff dropped VM annotations') +local runtimeFound, dataFound, dispatcherFound = false, false, false +visitast(cffAst, function(node) + if node == cffAst.vmRuntime then runtimeFound = true end + if node == cffAst.vmData then dataFound = true end + if node.kind == Ast.AstKind.WhileStatement then dispatcherFound = true end +end) +assert(runtimeFound and dataFound, 'Cff annotations do not point into the compiled AST') +assert(dispatcherFound, 'Cff did not emit a surrounding dispatcher') +local nativeExpressionCount = 0 +for _ in pairs(closureCompiler.nativeExpressions) do nativeExpressionCount = nativeExpressionCount + 1 end +assert(nativeExpressionCount == 2 and #closureCompiler.blocks > nativeExpressionCount, + 'Cff did not preserve only the annotated expressions while compiling the surrounding helper: ' + .. nativeExpressionCount .. ' native expressions, ' .. #closureCompiler.blocks .. ' blocks') + +-- The default setting retains the older behavior: flattening the VM runtime +-- changes its control-flow tree instead of preserving its annotated subtree. +local defaultVmAst = Parser:new({LuaVersion = 'Lua51'}):parse(mediumSource) +defaultVmAst = Steps.Vmify:new():apply(defaultVmAst) +local defaultRuntimeCounts = controlFlowCounts(defaultVmAst.vmRuntime) +local defaultCffAst = applyCff(Steps.Cff:new({}), defaultVmAst) +local defaultFlattenedCounts = controlFlowCounts(defaultCffAst) +local changedRuntimeTree = false +for key, value in pairs(defaultRuntimeCounts) do + if defaultFlattenedCounts[key] ~= value then changedRuntimeTree = true end +end +assert(changedRuntimeTree, 'default Cff unexpectedly preserved the VM runtime structure') + +local closureChunk = assert(loadstring(Unparser:new({LuaVersion = 'Lua51'}):unparse(cffAst))) +local returned = closureChunk() +collectgarbage('collect') +collectgarbage('collect') +local result = pack(returned(3)) +assert(result.n == 4 and result[1] == nil and result[2] == 4 + and result[3] == nil and result[4] == 2, + 'native closure lost captured writes, recursive helper, or trailing nils') +result = pack(returned(0)) +assert(result.n == 4 and result[1] == nil and result[2] == 4 + and result[3] == nil and result[4] == 3, + 'native closure did not share mutable captured cells after returning') +assert(cffNativeGlobal == 3, 'native global writes were not preserved') +cffNativeGlobal = previousGlobal + +-- Large VM prototype data must remain a separate native initializer while the +-- EncryptStrings decoder and other surrounding helpers pass through Cff. +local fields = {} +for i = 1, 1000 do fields[i] = tostring(i) end +local largeSource = 'local values={' .. table.concat(fields, ',') .. '}; return values[1],values[1000],...' +local vmSteps = { + {Name = 'Vmify'}, + {Name = 'EncryptStrings'}, + {Name = 'Cff', Settings = {ExcludeVm = true}}, +} +local large = compile(largeSource, vmSteps, 17) +result = pack(large(nil, 7, nil)) +assert(result.n == 5 and result[1] == 1 and result[2] == 1000 + and result[3] == nil and result[4] == 7 and result[5] == nil, + 'large native prototype initializer changed values or vararg arity') + +-- Inspect the full Medium AST before unparse: annotations still identify native +-- VM expressions, while the anti-tamper/encryption helper code is in a Cff +-- dispatcher generated around them. +local structurePipeline = Prometheus.Pipeline:fromConfig({ + LuaVersion = 'Lua51', Seed = 4, Steps = mediumSteps, +}) +local structureAst = structurePipeline.parser:parse(mediumSource) +local cffAstAfterStep, beforeRuntimeCounts, mediumCompiler +for _, step in ipairs(structurePipeline.steps) do + if step.Name == 'Cff' then + beforeRuntimeCounts = controlFlowCounts(structureAst.vmRuntime) + cffAstAfterStep, mediumCompiler = applyCff(step, structureAst, structurePipeline) + structureAst = cffAstAfterStep + else + structureAst = step:apply(structureAst, structurePipeline) or structureAst + end +end +assert(cffAstAfterStep.vmRuntime and cffAstAfterStep.vmData, + 'full Medium pipeline lost native VM runtime/data annotations') +assert(countKind(cffAstAfterStep, Ast.AstKind.WhileStatement) > 0 + and countKind(cffAstAfterStep, Ast.AstKind.IfStatement) > 0, + 'full Medium helpers were not emitted through the Cff dispatcher') +local mediumNativeCount = 0 +for _ in pairs(mediumCompiler.nativeExpressions) do mediumNativeCount = mediumNativeCount + 1 end +assert(mediumNativeCount == 2 and #mediumCompiler.blocks > mediumNativeCount, + 'Medium helpers were not Cff compiled around the two protected VM expressions') +-- The annotation points at the native runtime expression preserved inside its +-- generated bridge. Compare that subtree directly with the input to ExcludeVm. +local afterRuntimeCounts = controlFlowCounts(cffAstAfterStep.vmRuntime) +for key, value in pairs(beforeRuntimeCounts) do + assert(afterRuntimeCounts[key] == value, + 'ExcludeVm changed VM runtime ' .. key .. ' from ' .. value .. ' to ' .. afterRuntimeCounts[key]) +end + +-- Hoisting moves every runtime literal through a cache populated at startup; +-- user constants remain encrypted in prototype data instead of the hot VM loop. +local plaintext = 'prometheus_hoist_unique_plaintext_8a1f' +local payloadSource = 'local payload={message=' .. string.format('%q', plaintext) + .. ',again=' .. string.format('%q', plaintext) .. ',nested={"hoist_second_value"}}; ' + .. 'return payload.message,payload.again,payload.nested[1],...' +local hoistedAst = Parser:new({LuaVersion = 'Lua51'}):parse(payloadSource) +hoistedAst = Steps.Vmify:new():apply(hoistedAst) +local runtimeCountsBeforeHoist = controlFlowCounts(hoistedAst.vmRuntime) +local runtimeStringSet, runtimeStringCount = {}, 0 +visitExpression(hoistedAst.vmRuntime, hoistedAst, function(node) + if node.kind == Ast.AstKind.StringExpression and not runtimeStringSet[node.value] then + runtimeStringSet[node.value], runtimeStringCount = true, runtimeStringCount + 1 + end +end) +local hoistEncrypt = Steps.EncryptStrings:new({HoistVmStrings = true}) +math.randomseed(31) +hoistedAst = hoistEncrypt:apply(hoistedAst) +assert(countKind(hoistedAst.vmRuntime, Ast.AstKind.StringExpression) == 0, + 'Hoisted VM runtime still contains string literals on its execution path') +local runtimeCache = hoistedAst.vmRuntime.body.statements +assert(runtimeCache[1].kind == Ast.AstKind.LocalVariableDeclaration + and runtimeCache[2].kind == Ast.AstKind.LocalVariableDeclaration + and #runtimeCache[1].ids > 0 and #runtimeCache[1].expressions > 0 + and #runtimeCache[2].ids > 0 and #runtimeCache[2].expressions > 0, + 'Hoisted runtime cache locals were not initialized') +local afterHoistCounts = controlFlowCounts(hoistedAst.vmRuntime) +for key, value in pairs(runtimeCountsBeforeHoist) do + assert(afterHoistCounts[key] == value, + 'String hoisting changed VM runtime ' .. key .. ' count') +end +local plaintextFound, encryptedDataStrings = false, 0 +visitast(hoistedAst, function(node) + if node.kind == Ast.AstKind.StringExpression and node.value == plaintext then + plaintextFound = true + end +end) +visitExpression(hoistedAst.vmData, hoistedAst, function(node) + if node.kind == Ast.AstKind.StringExpression then encryptedDataStrings = encryptedDataStrings + 1 end +end) +assert(not plaintextFound and encryptedDataStrings > 0, + 'VM prototype data retained plaintext or lost encrypted string constants') +local cacheInitializer +for _, statement in ipairs(hoistedAst.body.statements) do + if statement.kind == Ast.AstKind.LocalVariableDeclaration + and statement.expressions and statement.expressions[1] + and statement.expressions[1].kind == Ast.AstKind.TableConstructorExpression then + cacheInitializer = statement.expressions[1] + end +end +local encryptedCacheStrings = 0 +if cacheInitializer then + visitExpression(cacheInitializer, hoistedAst, function(node) + if node.kind == Ast.AstKind.StringExpression then encryptedCacheStrings = encryptedCacheStrings + 1 end + end) +end +assert(cacheInitializer and encryptedCacheStrings == runtimeStringCount, + 'Distinct runtime strings were not hoisted into an encrypted outer initializer') + +-- A second hoisting pass sees no runtime string literals. It adds no empty +-- cache declaration and the two decoder layers still execute correctly. +local repeatedAst = Parser:new({LuaVersion = 'Lua51'}):parse('return 42,nil,...') +repeatedAst = Steps.Vmify:new():apply(repeatedAst) +math.randomseed(47) +repeatedAst = hoistEncrypt:apply(repeatedAst) +local repeatedRuntime = repeatedAst.vmRuntime +local firstCache, secondCache = repeatedRuntime.body.statements[1], repeatedRuntime.body.statements[2] +local runtimeBodyLength = #repeatedRuntime.body.statements +repeatedAst = hoistEncrypt:apply(repeatedAst) +assert(repeatedAst.vmRuntime == repeatedRuntime + and repeatedRuntime.body.statements[1] == firstCache + and repeatedRuntime.body.statements[2] == secondCache + and #repeatedRuntime.body.statements == runtimeBodyLength, + 'Repeated string encryption added empty runtime cache declarations') +visitast(repeatedRuntime, function(node) + if node.kind == Ast.AstKind.LocalVariableDeclaration then + assert(#node.ids > 0, 'Repeated hoisting emitted an empty local declaration') + end +end) +local repeatedChunk = compile('return 42,nil,...', { + {Name = 'Vmify'}, + {Name = 'EncryptStrings', Settings = {HoistVmStrings = true}}, + {Name = 'EncryptStrings', Settings = {HoistVmStrings = true}}, +}, 47) +result = pack(repeatedChunk(7, nil)) +assert(result.n == 4 and result[1] == 42 and result[2] == nil + and result[3] == 7 and result[4] == nil, + 'Repeated EncryptStrings hoisting changed simple results or return arity') + +-- Exercise the final Medium ordering, including Cff exclusion before numeric +-- rewriting, with a user string stored in the VM prototype table. +local payloadChunk = compile(payloadSource, mediumSteps, 17) +result = pack(payloadChunk('tail', nil)) +assert(result.n == 5 and result[1] == plaintext and result[2] == plaintext + and result[3] == 'hoist_second_value' and result[4] == 'tail' and result[5] == nil, + 'Final Medium pipeline changed hoisted payload strings or result arity') + +print('Cff native VM exclusion and captured closure checks passed') From dd67a86d7840a6d19655ddb8d1330a865c24a038 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 16:49:19 +0100 Subject: [PATCH 7/8] further optimize runtime speed --- Dockerfile | 1 + benchmarks/README.md | 1 + benchmarks/v0.3-medium-native-cff.md | 3 + benchmarks/v0.3-vm-fastpaths.md | 92 +++++++++ doc/advanced/register-vm.md | 61 ++++-- src/prometheus/vm/calls.lua | 64 +++++++ src/prometheus/vm/compiler.lua | 7 +- src/prometheus/vm/expressions.lua | 60 ++---- src/prometheus/vm/opcodes.lua | 3 + src/prometheus/vm/optimize.lua | 255 +++++++++++++++++++++++++ src/prometheus/vm/runtime.lua | 52 +++-- src/prometheus/vm/runtime_calls.lua | 93 +++++++++ src/prometheus/vm/runtime_dispatch.lua | 26 +-- src/prometheus/vm/statements.lua | 16 +- tests/harness/vm-fastpaths.lua | 245 ++++++++++++++++++++++++ tests/harness/vm-layout.lua | 11 ++ 16 files changed, 880 insertions(+), 110 deletions(-) create mode 100644 benchmarks/v0.3-vm-fastpaths.md create mode 100644 src/prometheus/vm/calls.lua create mode 100644 src/prometheus/vm/optimize.lua create mode 100644 src/prometheus/vm/runtime_calls.lua create mode 100644 tests/harness/vm-fastpaths.lua diff --git a/Dockerfile b/Dockerfile index d812a19f..c0057c9b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,6 +30,7 @@ RUN lua5.1 tests/harness/metadata.lua \ && timeout 60s lua5.1 tests/harness/cff-native.lua \ && timeout 60s lua5.1 tests/harness/vm-arity.lua \ && timeout 60s lua5.1 tests/harness/vm-prototypes.lua \ + && timeout 60s lua5.1 tests/harness/vm-fastpaths.lua \ && timeout 60s lua5.1 tests/harness/vm-layout.lua \ && python3 tests/harness/runner.py diff --git a/benchmarks/README.md b/benchmarks/README.md index f15235e6..5bf2fc29 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,6 +2,7 @@ Recorded v0.3 measurements: [v0.3-results.md](v0.3-results.md). Updated Medium measurements: [v0.3-medium-native-cff.md](v0.3-medium-native-cff.md). +Latest VM optimization measurements: [v0.3-vm-fastpaths.md](v0.3-vm-fastpaths.md). Run from the repository root with Python 3, Lua 5.1, and LuaJIT installed: diff --git a/benchmarks/v0.3-medium-native-cff.md b/benchmarks/v0.3-medium-native-cff.md index f6128b82..9680c24c 100644 --- a/benchmarks/v0.3-medium-native-cff.md +++ b/benchmarks/v0.3-medium-native-cff.md @@ -1,5 +1,8 @@ # Medium with native VM and Cff-protected helpers +These measurements precede the scalar assignment, cleanup, and fixed-arity VM +optimizations. See the [latest comparison](v0.3-vm-fastpaths.md). + Host: Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.44; CPU: 13th Gen Intel(R) Core(TM) i7-13850HX. Runtimes: Lua 5.1.5 Copyright (C) 1994-2012 Lua.org, PUC-Rio; LuaJIT 2.1.1792588164 -- Copyright (C) 2005-2026 Mike Pall. https://luajit.org/. Four deterministic workloads compare original source with the actual presets, including AntiTamper. All variants must match the original checksum. diff --git a/benchmarks/v0.3-vm-fastpaths.md b/benchmarks/v0.3-vm-fastpaths.md new file mode 100644 index 00000000..350f091d --- /dev/null +++ b/benchmarks/v0.3-vm-fastpaths.md @@ -0,0 +1,92 @@ +# VM scalar, cleanup, and call optimizations + +Host: Linux-6.18.33.2-microsoft-standard-WSL2-x86_64-with-glibc2.44; CPU: 13th Gen Intel(R) Core(TM) i7-13850HX. Runtimes: Lua 5.1.5 Copyright (C) 1994-2012 Lua.org, PUC-Rio; LuaJIT 2.1.1792588164 -- Copyright (C) 2005-2026 Mike Pall. https://luajit.org/. + +Four deterministic workloads compare original source with the actual presets, including AntiTamper. All variants must match the original checksum. + +Seeds: [1, 4, 17]; 3 timed batches per seed. Each workload is parsed once and warmed up; batches adapt to approximately 0.1 seconds. Execution includes initialization of the emitted chunk, workload work, allocations, and checksum validation, but excludes parsing. Garbage collection runs before each batch; collection during the batch is included. + +Execution figures are median CPU time per full invocation across batches and seeds. Generation and parsing figures are medians across seeds. Output size is a median across seeds. Execution ranges show the lowest and highest per-seed medians. Measurements are local, with JIT enabled for LuaJIT; very small native JIT times can produce large slowdown ratios. These workloads do not establish a general performance guarantee or measure peak memory. + +## lua5.1 + +| Workload | Preset | Execution ms (median; seed range) | Slowdown | Output KiB | Generate ms | Parse ms | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| closures | Original | 5.126 | 1.0x | 0.5 | 0.00 | 0.05 | +| closures | Medium | 283.864 (275.221–307.719) | 55.4x | 68.6 | 296.32 | 1.93 | +| numeric | Original | 8.091 | 1.0x | 0.3 | 0.00 | 0.05 | +| numeric | Medium | 933.078 (892.049–1080.061) | 115.3x | 62.8 | 302.57 | 1.99 | +| strings | Original | 9.934 | 1.0x | 0.5 | 0.00 | 0.03 | +| strings | Medium | 131.487 (126.395–132.072) | 13.2x | 64.5 | 279.18 | 1.78 | +| tables | Original | 7.037 | 1.0x | 0.6 | 0.00 | 0.06 | +| tables | Medium | 336.071 (324.357–338.127) | 47.8x | 68.5 | 298.42 | 1.86 | + +## luajit + +| Workload | Preset | Execution ms (median; seed range) | Slowdown | Output KiB | Generate ms | Parse ms | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| closures | Original | 2.414 | 1.0x | 0.5 | 0.00 | 0.03 | +| closures | Medium | 72.100 (70.802–73.010) | 29.9x | 68.6 | 296.32 | 1.56 | +| numeric | Original | 1.073 | 1.0x | 0.3 | 0.00 | 0.02 | +| numeric | Medium | 113.393 (109.472–115.918) | 105.7x | 62.8 | 302.57 | 1.64 | +| strings | Original | 4.174 | 1.0x | 0.5 | 0.00 | 0.02 | +| strings | Medium | 27.878 (26.267–28.088) | 6.7x | 64.5 | 279.18 | 1.32 | +| tables | Original | 3.964 | 1.0x | 0.6 | 0.00 | 0.03 | +| tables | Medium | 58.239 (57.085–68.127) | 14.7x | 68.5 | 298.42 | 1.66 | + +## Before and after + +The baseline is commit `458f897`, exported to an isolated source snapshot. +Both versions were benchmarked sequentially on the same host after correctness +checks finished. Both retained the Medium preset, including AntiTamper and +Cff-protected helpers. Each uses seeds 1, 4, and 17 with three timed batches. +Measurements are separate runs; small differences can include timing variation. +The original-source timing in each raw report helps show that variation. + +The compiler now lowers fixed-length assignment values directly to registers; +all right-hand expressions still run before stores. It removes safe temporary +copies, retargets scalar producers, deduplicates clears, batches adjacent clears, +and remaps jumps within basic blocks. Calls with zero through three closed +arguments and closed returns of up to three values use native argument and +return lists. Varargs, expanding final arguments, larger argument lists, and +multi-result call capture retain counted tuple paths. Small non-vararg callees +also avoid constructing an incoming argument tuple. + +Parallel assignment stores now execute right to left, matching native Lua, +including duplicate destinations and table metamethod side effects. + +| Runtime | Workload | Before ms | After ms | Speedup | Before KiB | After KiB | +| --- | --- | ---: | ---: | ---: | ---: | ---: | +| lua5.1 | closures | 401.388 | 283.864 | 1.41x | 69.4 | 68.6 | +| lua5.1 | numeric | 1411.700 | 933.078 | 1.51x | 61.8 | 62.8 | +| lua5.1 | strings | 206.824 | 131.487 | 1.57x | 65.1 | 64.5 | +| lua5.1 | tables | 455.907 | 336.071 | 1.36x | 71.5 | 68.5 | +| luajit | closures | 130.682 | 72.100 | 1.81x | 69.4 | 68.6 | +| luajit | numeric | 162.915 | 113.393 | 1.44x | 61.8 | 62.8 | +| luajit | strings | 58.727 | 27.878 | 2.11x | 65.1 | 64.5 | +| luajit | tables | 89.141 | 58.239 | 1.53x | 71.5 | 68.5 | + +All benchmark checksums matched original execution. The two runs comprise 48 +transformed preset/runtime combinations and 16 original executions, with 192 +timed batches. The semantic suite passed 1,260 transformed runs across three +seeds and 70 baselines, with zero failures or timeouts. Targeted arity, capture, +cleanup, tail-call, and all-24-operand-layout checks passed on Lua 5.1 and LuaJIT; +browser tests and the production build passed. + +The extra handlers enlarge the interpreter, while smaller instruction tables +offset that growth. Total output is slightly smaller in three workloads and +about 1 KiB larger in the numeric workload. Execution is faster in these +workloads, but the VM still has substantial overhead relative to native Lua. +This change does not alter Medium's protection strategy, opcode +randomization, operand layouts, or balanced dispatcher. + +Raw reports, JSON, and generated sources: + +- Before: `/tmp/prometheus-vm-fastpaths-before-benchmark/` +- After: `/tmp/prometheus-vm-fastpaths-after-benchmark/` + +Reproduce the current version with: + +```sh +python3 benchmarks/run.py --presets Medium +``` diff --git a/doc/advanced/register-vm.md b/doc/advanced/register-vm.md index 1f363a5a..a9a50a1b 100644 --- a/doc/advanced/register-vm.md +++ b/doc/advanced/register-vm.md @@ -10,12 +10,18 @@ compiler, or `loadstring` to execute the original source. - `vm/compiler.lua`: function prototypes, constants, registers, and captures. - `vm/expressions.lua` and `vm/expressions_tables.lua`: expression lowering, value lists, calls, branches, and table constructors. +- `vm/calls.lua`: direct-register lowering for fixed-arity calls and returns, + with packed value-list lowering for general cases. - `vm/statements.lua`: statements, lexical cleanup, and loop control. +- `vm/optimize.lua`: recursive, conservative basic-block optimizations and + cleanup batching before emission. - `vm/opcodes.lua`: semantic instruction IDs. - `vm/layout.lua`: a per-emission opcode and operand layout. - `vm/runtime.lua` and `vm/runtime_dispatch.lua`: runtime templates and the generated dispatcher, configured for the selected opcode IDs and operand positions. +- `vm/runtime_calls.lua`: fixed-arity and packed call, return, and register + cleanup handlers used by the generated runtime. - `vm/emit.lua`: instruction encoding, native prototype table literals, and the runtime function IIFE. - `vm/strings.lua`: optional hoisting of generated interpreter string literals @@ -27,12 +33,33 @@ metadata, and capture descriptors. The emitter passes the native prototype tables as an argument to a runtime function IIFE. The generated interpreter then uses that table data; the prototypes are not encoded as strings. -Each invocation owns its register table and vararg tuple. Closures retain only +Each invocation owns its register table; vararg functions also carry a counted +vararg tuple. Closures retain only their referenced upvalue cells; leaving a scope detaches its cells without destroying values shared by live closures. Value lists carry an explicit `n` so trailing nils survive calls, varargs, table constructors, and returns. -Temporary registers are cleared after consumption, and conditional branches -consume their test register on both paths. +The compiler keeps ordinary scalar values in individual registers. Small calls +and returns pass those registers directly to native Lua instead of first making +a value-list table. Temporary registers are cleared after consumption, and +conditional branches consume their test register on both paths. The optimizer +can combine adjacent register clears into one `CLEARS` instruction; it emits +that batch at the same point in the instruction stream so register lifetimes +remain intact. + +Calls with zero to three scalar arguments use `CALL0` through `CALL3`; tail +calls use `TAILCALL0` through `TAILCALL3`. Their handlers invoke the callee +directly with that exact argument count. Fixed returns use `RETURN0` through +`RETURN3`, which return exactly that many values, including nil values in any +position. Calls with four or more arguments, or with an expandable final +argument such as a call or vararg expression, use the packed path. That path +retains an explicit value count so expanded results and trailing nils are +preserved. Multi-result calls likewise use a counted value list when the +caller needs an open result list or more than one fixed result. + +For non-vararg prototypes with at most three parameters, runtime startup reads +the incoming values with `select` and avoids constructing an incoming argument +pack. Vararg functions and functions with more parameters retain the packed +initialization path, which also builds the vararg tuple when needed. ## Generation and composition @@ -62,18 +89,22 @@ helpers introduced by intervening steps such as `EncryptStrings`. If the interpreter writes to an external lexical variable, the special path falls back to ordinary Cff compilation to preserve that write's behavior. -The compiler keeps canonical semantic instruction IDs. At emission, `layout.lua` -chooses a permutation of the opcode IDs and a permutation of operand slots 2 -through 5; slot 1 always contains the opcode. The emitter encodes each -instruction with that layout without changing the compiler prototypes. One -layout is shared by the root prototype and all child prototypes in the emitted -chunk. - -`runtime_dispatch.lua` builds a balanced numeric `if`/`else` decision tree from -the selected opcode IDs. The interpreter reads each operand from its selected -slot, then checks the opcode for an exact ID at each leaf before running its -handler. For the same input, configuration, and Lua runtime, a fixed pipeline -seed reproduces the layout. Different seeds can produce different layouts. +The compiler keeps canonical semantic instruction IDs. The additional fixed +arity and cleanup instructions use the same ID scheme. At emission, +`layout.lua` chooses a permutation of the opcode IDs and a permutation of +operand slots 2 through 5; slot 1 always contains the opcode. The emitter +encodes each instruction with that layout without changing the compiler +prototypes. One layout is shared by the root prototype and all child +prototypes in the emitted chunk. + +Before emission, `optimize.lua` visits the root prototype and its children and +applies conservative basic-block-local rewrites. At emission, +`runtime_dispatch.lua` still builds a balanced numeric `if`/`else` decision +tree from the selected opcode IDs. The interpreter reads each operand from its +selected slot, then checks the opcode for an exact ID at each leaf before +running its handler. For the same input, configuration, and Lua runtime, a +fixed pipeline seed reproduces the layout. Different seeds can produce +different layouts. In Medium, `EncryptStrings` uses `HoistVmStrings = true` after `Vmify`. It moves the generated interpreter's literal strings into an outer initializer, where diff --git a/src/prometheus/vm/calls.lua b/src/prometheus/vm/calls.lua new file mode 100644 index 00000000..ac66dfcb --- /dev/null +++ b/src/prometheus/vm/calls.lua @@ -0,0 +1,64 @@ +-- Common calls and returns pass fixed value lists directly through native Lua. +local Kind = require('prometheus.ast').AstKind +local unpackValues = unpack or table.unpack + +return function(Compiler, packList, expandable) + function Compiler:call(node, count, tail) + local fn, prefix + if node.kind == Kind.PassSelfFunctionCallExpression then + local object = self:expression(node.base, 1)[1] + local key = self:load(node.passSelfFunctionName) + fn = self:alloc(); self:emit('GETTABLE', fn, object, key) + self:free(key) + prefix = {object} + else + fn = self:expression(node.base, 1)[1] + end + + local arity = #node.args + (prefix and 1 or 0) + local fixed = arity <= 3 and not expandable(node.args[#node.args]) + local args, registers + if fixed then + registers = prefix or {} + for _, argument in ipairs(node.args) do + registers[#registers + 1] = self:expression(argument, 1)[1] + end + args = registers + else + args = packList(self, node.args, prefix) + end + local function releaseArguments() + if fixed then + for _, register in ipairs(registers) do self:free(register) end + else + self:free(args) + end + self:free(fn) + end + if tail then + self:emit(fixed and ('TAILCALL' .. arity) or 'TAILCALL', fn, args) + releaseArguments() + return {} + end + local mode = count == 0 and 0 or (count == 1 and 1 or -1) + local result = count ~= 0 and self:alloc() or 0 + self:emit(fixed and ('CALL' .. arity) or 'CALL', result, fn, args, mode) + releaseArguments() + if count == 0 then return {} end + if count == -1 or count == 1 then return {result} end + local results = {} + for i = 1, count do + results[i] = self:alloc(); self:emit('TAKE', results[i], result, i) + end + self:free(result) + return results + end + + function Compiler:fixedReturn(expressions) + if #expressions > 3 or expandable(expressions[#expressions]) then return false end + local registers = self:list(expressions, #expressions) + self:emit('RETURN' .. #registers, unpackValues(registers)) + for _, register in ipairs(registers) do self:free(register) end + return true + end +end diff --git a/src/prometheus/vm/compiler.lua b/src/prometheus/vm/compiler.lua index 416a6785..e0249387 100644 --- a/src/prometheus/vm/compiler.lua +++ b/src/prometheus/vm/compiler.lua @@ -1,6 +1,7 @@ -- Lower scoped AST nodes into function prototypes containing semantic instructions. local analyze = require('prometheus.vm.analysis') local opcodes = require('prometheus.vm.opcodes') +local optimize = require('prometheus.vm.optimize') local Compiler = {} Compiler.__index = Compiler @@ -141,9 +142,7 @@ end function Compiler:finish(node) self:block(node.body) - local empty = self:alloc() - self:emit('ARGS', empty, {}, 0) - self:emit('RETURN', empty) + self:emit('RETURN0') return self.proto end @@ -159,7 +158,7 @@ end function Compiler:compile(ast) local compiler = Compiler:new(analyze(ast), ast) - return compiler:finish(ast) + return optimize(compiler:finish(ast)) end require('prometheus.vm.expressions')(Compiler) diff --git a/src/prometheus/vm/expressions.lua b/src/prometheus/vm/expressions.lua index 7f13ced7..efba259a 100644 --- a/src/prometheus/vm/expressions.lua +++ b/src/prometheus/vm/expressions.lua @@ -206,52 +206,28 @@ local function install(Compiler) for _, node in ipairs(expressions) do self:expression(node, 0) end return {} end - local tuple = packList(self, expressions) - if count == nil then return {tuple} end - local results = {} - for i = 1, count do - results[i] = self:alloc(); self:emit("TAKE", results[i], tuple, i) - end - self:free(tuple) - return results - end - - function Compiler:call(node, count, tail) - local fn, prefix - if node.kind == Kind.PassSelfFunctionCallExpression then - local object = self:expression(node.base, 1)[1] - local key = load(self, node.passSelfFunctionName) - fn = self:alloc(); self:emit("GETTABLE", fn, object, key) - self:free(key) - prefix = {object} - else - fn = self:expression(node.base, 1)[1] - end - local args = packList(self, node.args, prefix) - if tail then - self:emit("TAILCALL", fn, args) - self:free(args); self:free(fn) - return {} - end - if count == 0 then - self:emit("CALL", 0, fn, args, 0) - self:free(args); self:free(fn); return {} - elseif count == -1 then - local result = self:alloc(); self:emit("CALL", result, fn, args, -1) - self:free(args); self:free(fn); return {result} - elseif count == 1 then - local result = self:alloc(); self:emit("CALL", result, fn, args, 1) - self:free(args); self:free(fn); return {result} - else - local tuple = self:alloc(); self:emit("CALL", tuple, fn, args, -1) - self:free(args); self:free(fn) + if count ~= nil then local results = {} - for i = 1, count do - results[i] = self:alloc(); self:emit("TAKE", results[i], tuple, i) + for i, node in ipairs(expressions) do + local remaining = count - #results + if remaining <= 0 then + self:expression(node, 0) + elseif i == #expressions and expandable(node) then + for _, register in ipairs(self:expression(node, remaining)) do + results[#results + 1] = register + end + else + results[#results + 1] = self:expression(node, 1)[1] + end end - self:free(tuple); return results + while #results < count do results[#results + 1] = self:load(nil) end + return results end + local tuple = packList(self, expressions) + return {tuple} end + + require('prometheus.vm.calls')(Compiler, packList, expandable) end return install diff --git a/src/prometheus/vm/opcodes.lua b/src/prometheus/vm/opcodes.lua index 0cc1f8f0..33d37a3c 100644 --- a/src/prometheus/vm/opcodes.lua +++ b/src/prometheus/vm/opcodes.lua @@ -7,6 +7,9 @@ local names = { 'ADD', 'SUB', 'MUL', 'DIV', 'MOD', 'POW', 'CONCAT', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'NOT', 'NEG', 'LEN', 'FORPREP', 'FORCHECK', + 'CLEARS', 'CALL0', 'CALL1', 'CALL2', 'CALL3', + 'TAILCALL0', 'TAILCALL1', 'TAILCALL2', 'TAILCALL3', + 'RETURN0', 'RETURN1', 'RETURN2', 'RETURN3', } local opcodes = {} for id, name in ipairs(names) do opcodes[name] = id end diff --git a/src/prometheus/vm/optimize.lua b/src/prometheus/vm/optimize.lua new file mode 100644 index 00000000..e1772bdb --- /dev/null +++ b/src/prometheus/vm/optimize.lua @@ -0,0 +1,255 @@ +-- Conservative, basic-block-local peephole passes for VM prototypes. +local opcodes = require('prometheus.vm.opcodes') +local names = {} +for name, id in pairs(opcodes) do names[id] = name end + +local function name(ins) return names[ins[1]] end +local function copy(ins) + local result = {} + for i = 1, #ins do result[i] = ins[i] end + return result +end +local function isUnary(op) return op == 'NOT' or op == 'NEG' or op == 'LEN' end +local function isBinary(op) + return op == 'ADD' or op == 'SUB' or op == 'MUL' or op == 'DIV' + or op == 'MOD' or op == 'POW' or op == 'CONCAT' or op == 'LT' + or op == 'LE' or op == 'GT' or op == 'GE' or op == 'EQ' or op == 'NE' +end +local function producer(ins) + local op = name(ins) + if op == 'LOAD' or op == 'NIL' or op == 'GLOBAL' or op == 'GETUP' + or op == 'GETCELL' or op == 'GETTABLE' or op == 'NOT' or op == 'NEG' + or op == 'LEN' or isBinary(op) then return ins[2] end + if op == 'CALL' or op == 'CALL0' or op == 'CALL1' or op == 'CALL2' or op == 'CALL3' then + if ins[5] == 1 then return ins[2] end + end +end +local function terminal(op) + return op == 'JMP' or op == 'RETURN' or op == 'TAILCALL' + or op == 'RETURN0' or op == 'RETURN1' or op == 'RETURN2' or op == 'RETURN3' + or op == 'TAILCALL0' or op == 'TAILCALL1' or op == 'TAILCALL2' or op == 'TAILCALL3' +end + +-- Track only values proven nil. An unknown writer invalidates the proof. +local function updateNil(state, ins) + local op = name(ins) + if op == 'CLEAR' then state[ins[2]] = true; return end + if op == 'CLEARS' then + for _, reg in ipairs(ins[2] or {}) do state[reg] = true end + return + end + if op == 'NIL' then state[ins[2]] = true; return end + if op == 'MOVE' then + state[ins[2]] = state[ins[3]] and true or nil + return + end + if op == 'JTRUE' or op == 'JFALSE' then state[ins[2]] = true; return end + if op == 'FORPREP' then state[ins[2]], state[ins[3]], state[ins[4]] = nil, nil, nil; return end + if op == 'FORCHECK' then state[ins[2]] = nil; return end + if op == 'EXPAND' then return end -- Writes table fields, not VM registers. + local writes = op == 'LOAD' or op == 'NIL' or op == 'MOVE' or op == 'CELL' + or op == 'GETCELL' or op == 'GETUP' or op == 'GLOBAL' or op == 'GETTABLE' + or op == 'TABLE' or op == 'ARGS' or op == 'TAKE' or op == 'VARARG' + or op == 'CLOSURE' or isUnary(op) or isBinary(op) + if op == 'CALL' or op == 'CALL0' or op == 'CALL1' or op == 'CALL2' or op == 'CALL3' then + if ins[5] == 0 then return end + state[ins[2]] = nil; return + end + if writes then state[ins[2]] = nil; return end + if op == 'JMP' or op == 'SETCELL' or op == 'SETUP' or op == 'SETGLOBAL' + or op == 'SETTABLE' or op == 'TAILCALL' or op == 'TAILCALL0' + or op == 'TAILCALL1' or op == 'TAILCALL2' or op == 'TAILCALL3' + or op == 'RETURN' or op == 'RETURN0' or op == 'RETURN1' + or op == 'RETURN2' or op == 'RETURN3' then return end + for reg in pairs(state) do state[reg] = nil end +end + +local function initialNil(proto, first) + local state = {} + if first == 1 then + -- A back edge to the first block can carry values from an earlier pass. + for _, instruction in ipairs(proto.code) do + local op = name(instruction) + if (op == 'JMP' and instruction[2] == 1) + or ((op == 'JTRUE' or op == 'JFALSE') and instruction[3] == 1) then + return state + end + end + for reg = 1, proto.maxRegister or 0 do state[reg] = true end + for _, param in ipairs(proto.params or {}) do state[param[1]] = nil end + end + return state +end +local function rewriteBlock(block, proto) + local result, state, i = {}, initialNil(proto, block[1].pc), 1 + local function emit(item) + result[#result + 1] = item + updateNil(state, item.ins) + end + while i <= #block do + local item, ins = block[i], block[i].ins + local dst = producer(ins) + if dst and state[dst] then + local moveAt = i + 1 + while moveAt <= #block and name(block[moveAt].ins) == 'CLEAR' do + moveAt = moveAt + 1 + end + if moveAt <= #block and name(block[moveAt].ins) == 'MOVE' + and block[moveAt].ins[3] == dst then + local target, ok = block[moveAt].ins[2], true + if target == dst then ok = false end + for j = i + 1, moveAt - 1 do + local reg = block[j].ins[2] + if reg == dst or reg == target then ok = false end + end + local last, found = moveAt + 1, false + while ok and last <= #block and name(block[last].ins) == 'CLEAR' do + local reg = block[last].ins[2] + if reg == target then ok = false end + if reg == dst then found = true; break end + last = last + 1 + end + if ok and found then + local changed = copy(ins); changed[2] = target + emit({pc = item.pc, ins = changed}) + for j = i + 1, moveAt - 1 do emit(block[j]) end + for j = moveAt + 1, last do + if block[j].ins[2] ~= dst then emit(block[j]) end + end + i = last + 1 + else + emit(item); i = i + 1 + end + else + emit(item); i = i + 1 + end + elseif name(ins) == 'MOVE' then + local temp, source = ins[2], ins[3] + local consumer = i + 1 + if temp ~= source and state[temp] then + while consumer <= #block do + local pure = name(block[consumer].ins) + if pure ~= 'LOAD' and pure ~= 'NIL' then break end + local written = block[consumer].ins[2] + if written == temp or written == source then break end + consumer = consumer + 1 + end + if consumer <= #block then + local use, useOp = block[consumer].ins, name(block[consumer].ins) + local used = isUnary(useOp) and use[3] == temp + or isBinary(useOp) and (use[3] == temp or use[4] == temp) + if used and use[2] ~= temp then + local last, found = consumer + 1, false + while last <= #block and name(block[last].ins) == 'CLEAR' do + if block[last].ins[2] == temp then found = true; break end + last = last + 1 + end + if found then + -- The removed MOVE's destination is known nil and stays nil. + for j = i + 1, consumer - 1 do emit(block[j]) end + local changed = copy(use) + if changed[3] == temp then changed[3] = source end + if changed[4] == temp then changed[4] = source end + emit({pc = block[consumer].pc, ins = changed}) + for j = consumer + 1, last do + if block[j].ins[2] ~= temp then emit(block[j]) end + end + i = last + 1 + else + emit(item); i = i + 1 + end + else + emit(item); i = i + 1 + end + else + emit(item); i = i + 1 + end + else + emit(item); i = i + 1 + end + else + emit(item); i = i + 1 + end + end + return result +end + +local function compactBlock(block, proto, output) + local state = initialNil(proto, block[1].pc) + local pending = {} + local function flush() + if #pending > 0 then + if #pending > 1 and opcodes.CLEARS then + local regs, pcs = {}, {} + for _, item in ipairs(pending) do regs[#regs + 1] = item.ins[2]; pcs[#pcs + 1] = item.pc end + output[#output + 1] = {ins = {opcodes.CLEARS, regs}, pcs = pcs} + else + for _, item in ipairs(pending) do output[#output + 1] = {ins = item.ins, pcs = {item.pc}} end + end + pending = {} + end + end + for _, item in ipairs(block) do + local ins = item.ins + if name(ins) == 'CLEAR' then + local reg = ins[2] + if not state[reg] then pending[#pending + 1] = item; state[reg] = true end + else + flush() + updateNil(state, ins) + output[#output + 1] = {ins = ins, pcs = {item.pc}} + end + end + flush() +end + +local function optimizeCode(proto) + local code, count = proto.code, #proto.code + if count == 0 then return end + local leaders = {[1] = true, [count + 1] = true} + for pc, ins in ipairs(code) do + local op = name(ins) + if op == 'JMP' then + leaders[pc + 1] = true; leaders[ins[2]] = true + elseif op == 'JTRUE' or op == 'JFALSE' then + leaders[pc + 1] = true; leaders[ins[3]] = true + elseif terminal(op) then leaders[pc + 1] = true end + end + local starts = {} + for pc = 1, count + 1 do if leaders[pc] then starts[#starts + 1] = pc end end + local output, oldToNew = {}, {} + for b = 1, #starts - 1 do + local first, last = starts[b], starts[b + 1] - 1 + local block = {} + for pc = first, last do block[#block + 1] = {pc = pc, ins = code[pc]} end + block = rewriteBlock(block, proto) + local before = #output + compactBlock(block, proto, output) + -- Map retained and batched original instructions to their emitted PC. + for index = before + 1, #output do + local entry = output[index] + for _, pc in ipairs(entry.pcs) do oldToNew[pc] = index end + end + local nextPc = #output + 1 + for pc = last, first, -1 do + if oldToNew[pc] then nextPc = oldToNew[pc] else oldToNew[pc] = nextPc end + end + end + oldToNew[count + 1] = #output + 1 + for _, entry in ipairs(output) do + local ins, op = entry.ins, name(entry.ins) + if op == 'JMP' then ins[2] = oldToNew[ins[2]] + elseif op == 'JTRUE' or op == 'JFALSE' then ins[3] = oldToNew[ins[3]] end + end + local compact = {} + for _, entry in ipairs(output) do compact[#compact + 1] = entry.ins end + proto.code = compact +end + +local function optimize(proto) + for _, child in ipairs(proto.children or {}) do optimize(child) end + optimizeCode(proto) + return proto +end + +return optimize diff --git a/src/prometheus/vm/runtime.lua b/src/prometheus/vm/runtime.lua index c2674ab4..5b989bc3 100644 --- a/src/prometheus/vm/runtime.lua +++ b/src/prometheus/vm/runtime.lua @@ -35,28 +35,46 @@ local function loop(P, U, Env, R, V) end local function start(P, U, Env, ...) - local R, incoming = {}, pack(...) + local R local params = P.params - for i = 1, #params do - local param = params[i] - local value = incoming[i] - if param[2] then - R[param[1]] = { value } - else - R[param[1]] = value + local V + if not P.vararg and #params <= 3 then + R = {} + local argc = nativeSelect("#", ...) + for i = 1, #params do + local param = params[i] + local value + if i <= argc then value = nativeSelect(i, ...) end + if param[2] then + R[param[1]] = { value } + else + R[param[1]] = value + end end - end - local V = { n = 0 } - if P.vararg then - local first = #params + 1 - if incoming.n > #params then - V.n = incoming.n - #params + else + local incoming = pack(...) + R = {} + for i = 1, #params do + local param = params[i] + local value = incoming[i] + if param[2] then + R[param[1]] = { value } + else + R[param[1]] = value + end end - for i = 1, V.n do - V[i] = incoming[first + i - 1] + V = { n = 0 } + if P.vararg then + local first = #params + 1 + if incoming.n > #params then + V.n = incoming.n - #params + end + for i = 1, V.n do + V[i] = incoming[first + i - 1] + end end + incoming = nil end - incoming = nil return loop(P, U, Env, R, V) end diff --git a/src/prometheus/vm/runtime_calls.lua b/src/prometheus/vm/runtime_calls.lua new file mode 100644 index 00000000..563d8195 --- /dev/null +++ b/src/prometheus/vm/runtime_calls.lua @@ -0,0 +1,93 @@ +-- Call, return, and register clearing handlers. Fixed arities keep argument +-- passing in native Lua calls so the VM does not build a temporary arg pack. +local M = {} + +local bodies = { + CALL = [==[ + x, y = R[b], R[c] + if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then + if d == -1 then + R[a] = { n = 1, Env } + elseif d == 1 then + R[a] = Env + end + elseif d == -1 then + R[a] = pack(x(nativeUnpack(y, 1, y.n))) + elseif d == 1 then + R[a] = x(nativeUnpack(y, 1, y.n)) + else + x(nativeUnpack(y, 1, y.n)) + end + ]==], + TAILCALL = [==[ + x, y = R[a], R[b] + if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then + return Env + end + return x(nativeUnpack(y, 1, y.n)) + ]==], + CLEARS = [==[ + for x = 1, #a do + R[a[x]] = nil + end + ]==], + RETURN0 = [==[return]==], + RETURN1 = [==[return R[a]]==], + RETURN2 = [==[return R[a], R[b]]==], + RETURN3 = [==[return R[a], R[b], R[c]]==], +} + +local function callBody(arity) + local args = {} + for i = 1, arity do + args[i] = "R[c[" .. tostring(i) .. "]]" + end + local invocation = "x(" .. table.concat(args, ", ") .. ")" + local intercept = arity == 0 + and "nativeGetfenv and x == nativeGetfenv" + or "nativeGetfenv and x == nativeGetfenv and (R[c[1]] == nil or R[c[1]] == 1)" + return "x = R[b]\n" + .. "if " .. intercept .. " then\n" + .. " if d == -1 then\n" + .. " R[a] = { n = 1, Env }\n" + .. " elseif d == 1 then\n" + .. " R[a] = Env\n" + .. " end\n" + .. "elseif d == -1 then\n" + .. " R[a] = pack(" .. invocation .. ")\n" + .. "elseif d == 1 then\n" + .. " R[a] = " .. invocation .. "\n" + .. "else\n" + .. " " .. invocation .. "\n" + .. "end" +end + +local function tailCallBody(arity) + local args = {} + for i = 1, arity do + args[i] = "R[b[" .. tostring(i) .. "]]" + end + local invocation = "x(" .. table.concat(args, ", ") .. ")" + local intercept = arity == 0 + and "nativeGetfenv and x == nativeGetfenv" + or "nativeGetfenv and x == nativeGetfenv and (R[b[1]] == nil or R[b[1]] == 1)" + return "x = R[a]\n" + .. "if " .. intercept .. " then\n" + .. " return Env\n" + .. "end\n" + .. "return " .. invocation +end + +for arity = 0, 3 do + bodies["CALL" .. tostring(arity)] = callBody(arity) + bodies["TAILCALL" .. tostring(arity)] = tailCallBody(arity) +end + +function M.merge(target) + for name, body in pairs(bodies) do + target[name] = body + end + return target +end + +return M diff --git a/src/prometheus/vm/runtime_dispatch.lua b/src/prometheus/vm/runtime_dispatch.lua index 5163f5c7..653a1ca0 100644 --- a/src/prometheus/vm/runtime_dispatch.lua +++ b/src/prometheus/vm/runtime_dispatch.lua @@ -1,5 +1,6 @@ -- Opcode bodies are kept separate from runtime.lua so both files stay small. local M = {} +local callBodies = require("prometheus.vm.runtime_calls") local bodies = { LOAD = [==[R[a] = K[b]]==], @@ -39,33 +40,10 @@ local bodies = { ]==], TAKE = [==[R[a] = R[b][c]]==], VARARG = [==[R[a] = V]==], - CALL = [==[ - x, y = R[b], R[c] - if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then - if d == -1 then - R[a] = { n = 1, Env } - elseif d == 1 then - R[a] = Env - end - elseif d == -1 then - R[a] = pack(x(nativeUnpack(y, 1, y.n))) - elseif d == 1 then - R[a] = x(nativeUnpack(y, 1, y.n)) - else - x(nativeUnpack(y, 1, y.n)) - end - ]==], RETURN = [==[ x = R[a] return nativeUnpack(x, 1, x.n) ]==], - TAILCALL = [==[ - x, y = R[a], R[b] - if nativeGetfenv and x == nativeGetfenv and (y.n == 0 or y[1] == nil or y[1] == 1) then - return Env - end - return x(nativeUnpack(y, 1, y.n)) - ]==], CLOSURE = [==[ x = P.children[b] y = {} @@ -120,6 +98,8 @@ local bodies = { ]==], } +callBodies.merge(bodies) + function M.build(opcodes) local entries = {} for name, id in pairs(opcodes) do diff --git a/src/prometheus/vm/statements.lua b/src/prometheus/vm/statements.lua index 3ed1eaf6..80bc4e04 100644 --- a/src/prometheus/vm/statements.lua +++ b/src/prometheus/vm/statements.lua @@ -110,10 +110,7 @@ return function(Compiler) end local function compileNumericFor(self, node) - local values = emitList(self, {node.initialValue, node.finalValue, node.incrementBy}, 3) - local control = {self:alloc(), self:alloc(), self:alloc()} - for i = 1, 3 do self:emit("MOVE", control[i], values[i]) end - release(self, values) + local control = emitList(self, {node.initialValue, node.finalValue, node.incrementBy}, 3) self:emit("FORPREP", control[1], control[2], control[3]) local record = loopRecord(self, node) local check = self:position() @@ -141,13 +138,12 @@ return function(Compiler) local iterator, state, control = values[1], values[2], values[3] local record = loopRecord(self, node) local check = self:position() - local args, results = self:alloc(), self:alloc() - self:emit("ARGS", args, {state, control}, 0) - self:emit("CALL", results, iterator, args, -1) + local results = self:alloc() + self:emit("CALL2", results, iterator, {state, control}, -1) local first = self:alloc() self:emit("TAKE", first, results, 1) local nilreg, ended = self:alloc(), self:alloc() - local iterationTemps = {args, results, first, nilreg, ended} + local iterationTemps = {results, first, nilreg, ended} self:emit("NIL", nilreg) self:emit("EQ", ended, first, nilreg) local exit = self:emit("JTRUE", ended, 0) @@ -188,7 +184,8 @@ return function(Compiler) local targets = {} for i, target in ipairs(node.lhs) do targets[i] = assignmentTarget(self, target) end local values = emitList(self, node.rhs, #targets) - for i, target in ipairs(targets) do storeTarget(self, target, values[i]) end + -- Lua stores a parallel assignment from right to left after evaluating RHSs. + for i = #targets, 1, -1 do storeTarget(self, targets[i], values[i]) end release(self, values) for _, target in ipairs(targets) do if target.base then self:free(target.base); self:free(target.key) end @@ -281,6 +278,7 @@ return function(Compiler) self:call(node.args[1], -1, true) return end + if self:fixedReturn(node.args) then return end local values = emitList(self, node.args, nil) self:emit("RETURN", values[1]) release(self, values) diff --git a/tests/harness/vm-fastpaths.lua b/tests/harness/vm-fastpaths.lua new file mode 100644 index 00000000..a4353707 --- /dev/null +++ b/tests/harness/vm-fastpaths.lua @@ -0,0 +1,245 @@ +local Prometheus = require('src.prometheus') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +local Parser = require('prometheus.parser') +local Compiler = require('prometheus.vm.compiler') +local opcodes = require('prometheus.vm.opcodes') +local Presets = require('src.presets') + +local function pack(...) return {n = select('#', ...), ...} end +local function difference(left, right, path, seen) + if type(left) ~= type(right) then return path .. ': type mismatch' end + if type(left) ~= 'table' then + if left ~= right then return path .. ': ' .. tostring(left) .. ' ~= ' .. tostring(right) end + return nil + end + seen = seen or {} + seen[left] = seen[left] or {} + if seen[left][right] then return nil end + seen[left][right] = true + for key, value in pairs(left) do + local issue = difference(value, right[key], path .. '[' .. tostring(key) .. ']', seen) + if issue then return issue end + end + for key, value in pairs(right) do + if left[key] == nil and value ~= nil then + return path .. '[' .. tostring(key) .. ']: extra value ' .. tostring(value) + end + end + return nil +end + +local mediumSteps = {} +for i, step in ipairs(Presets.Medium.Steps) do + mediumSteps[i] = {Name = step.Name, Settings = step.Settings} +end + +local function transformed(source, steps, seed, ...) + local output = Prometheus.Pipeline:fromConfig({ + LuaVersion = 'Lua51', Seed = seed, Steps = steps, + }):apply(source) + return assert(loadstring(output))(...) +end + +local calls = [[ + local function pack(...) return {n = select('#', ...), ...} end + local function multi() return 'first', nil, 'third' end + local function gather(...) return pack(...) end + local captured = 10 + local function make(step) + return function(a, b, c) + captured = captured + step + return captured, a, b, c + end + end + local closure = make(2) + local zero = function() return multi() end + local one = function(a) return a, 'extra' end + local two = function(a, b) return a, b end + local three = function(a, b, c) return a, b, c end + local function oneValue() return 42 end + local object = {base = 7} + function object:method(a, b) return self.base, a, b end + local effects = 0 + local function discarded(...) effects = effects + select('#', ...) end + discarded('x', nil, 3) + local scalar = (multi()) + local oneResult = pack(zero()) + local twoResults = pack(two(1)) + local threeResults = pack(three(nil, 2, nil, 'surplus')) + local method = pack(object:method(nil, 9)) + local expanded = gather(1, 2, gather(3, nil, 5)) + local varargs = gather(...) + local fourPlus = gather(1, 2, 3, 4, 5, nil) + local missing = pack(one()) + local surplus = pack(one(8, 9, 10)) + local padA, padB, padC = oneValue() + local capturedResult = pack(closure(nil, 4)) + local env = getfenv() + local envs = env == getfenv(nil) and env == getfenv(1) + and env == getfenv(0) and env == getfenv(function() end) + return scalar, oneResult, twoResults, threeResults, method, expanded, + varargs, fourPlus, missing, surplus, pack(padA, padB, padC), + capturedResult, effects, envs +]] + +local assignments = [[ + local function pack(...) return {n = select('#', ...), ...} end + local trace = {} + local function mark(value) trace[#trace + 1] = value end + local target = setmetatable({}, { + __newindex = function(t, k, v) mark('set:' .. k .. ':' .. tostring(v)); rawset(t, k, v) end, + __index = function(t, k) mark('get:' .. k); return rawget(t, k) end, + }) + local function base() mark('base'); return target end + local function key() mark('key'); return 'a' end + local i = 1 + local function rhs() mark('rhs'); return 11, 22 end + base()[key()], base()['b'] = rhs() + target[i], i = 33, 2 + local first, second = 4, 5 + first, second = second, first + local duplicate = 0 + duplicate, duplicate = 1, 2 + local alias = setmetatable({}, {__newindex = function(t, k, v) + mark('alias:' .. v); rawset(t, k, v) + end}) + alias.x, alias.x = 1, 2 + local ticks = 0 + local function side() ticks = ticks + 1; return ticks end + local scalar, padded = (side()), side() + local surplus = 0 + local function ignored() surplus = surplus + 1; return 99 end + local a = 6, ignored(), ignored() + + local total = 0 + for n = 1, 8 do + if n % 2 == 0 then + total = total + n + else + total = total + 1 + end + end + local r = 0 + repeat r = r + 1 until r == 3 + local g = 0 + for _, value in ipairs({2, 3, 4}) do g = g + value end + return pack(table.concat(trace, '|'), target.a, target.b, target[1], i, + first, second, duplicate, alias.x, ticks, scalar, padded, surplus, a, total, r, g) +]] + +local controlFlow = [[ + local sum = 0 + local i = 0 + while i < 5 do i = i + 1; if i ~= 3 then sum = sum + i end end + local j = 0 + repeat j = j + 1; sum = sum + 1 until j == 2 + for n = 1, 5 do if n > 3 then break end; sum = sum + n end + local k = 0 + for _, value in ipairs({1, 2, 3, 4}) do k = k + value end + local branch = 'no' + if sum > 0 then branch = 'yes' end + return sum, k, branch +]] + +local function compare(source, args) + local expected = pack(assert(loadstring(source))(unpack(args, 1, args.n))) + for _, seed in ipairs({1, 4, 17}) do + local actual = pack(transformed(source, {{Name = 'Vmify'}}, seed, + unpack(args, 1, args.n))) + local issue = difference(expected, actual, '$') + assert(not issue, 'Vmify semantic mismatch at seed ' .. seed .. ': ' .. tostring(issue)) + actual = pack(transformed(source, mediumSteps, seed, unpack(args, 1, args.n))) + issue = difference(expected, actual, '$') + assert(not issue, 'Medium semantic mismatch at seed ' .. seed .. ': ' .. tostring(issue)) + end +end + +for _, args in ipairs({pack(), pack(nil), pack(1, nil), pack(nil, 2, nil), + pack(1, nil, 3, 4, nil)}) do + compare(calls, args) +end +compare(assignments, pack()) +compare(controlFlow, pack()) + +-- Make sure representative fixed-arity code actually selects every fast path. +local fixed = [[ + local function r0() end + local function r1() return 1 end + local function r2() return 1, nil end + local function r3() return 1, nil, 3 end + local function c1(a) return a end + local function c2(a, b) return a, b end + local function c3(a, b, c) return a, b, c end + r0(); c1(1); c2(1, 2); c3(1, 2, 3) + return 1, 2, 3 +]] +local proto = Compiler:compile(Parser:new({LuaVersion = 'Lua51'}):parse(fixed)) +local counts = {} +local namesById = {} +for name, id in pairs(opcodes) do namesById[id] = name end +local function inspect(value) + for _, instruction in ipairs(value.code) do + local name = namesById[instruction[1]] + counts[name] = (counts[name] or 0) + 1 + end + for _, child in ipairs(value.children) do inspect(child) end +end +inspect(proto) +for arity = 0, 3 do + assert((counts['CALL' .. arity] or 0) > 0, 'missing CALL' .. arity .. ' fast path') + assert((counts['RETURN' .. arity] or 0) > 0, 'missing RETURN' .. arity .. ' fast path') +end +assert(not counts.ARGS and not counts.TAKE, 'fixed scalar fixture should not pack result lists') +local assignmentProto = Compiler:compile(Parser:new({LuaVersion = 'Lua51'}):parse(assignments)) +counts = {} +inspect(assignmentProto) +assert((counts.CLEARS or 0) > 0, 'temporary cleanup should batch into CLEARS') + +-- A deep tail-call chain also checks that the fixed calls remain tail calls. +local tail = [[ + local even, odd + even = function(n) if n == 0 then return true, nil end; return odd(n - 1) end + odd = function(n) if n == 0 then return false, nil end; return even(n - 1) end + return even(20000) +]] +local result = pack(transformed(tail, {{Name = 'Vmify'}}, 17)) +assert(result.n == 2 and result[1] == true and result[2] == nil) +local tailProto = Compiler:compile(Parser:new({LuaVersion = 'Lua51'}):parse(tail)) +counts = {} +inspect(tailProto) +assert((counts.TAILCALL1 or 0) > 0, 'fixed tail recursion should use TAILCALL1') + +-- Luau continue is lowered before optimization and must retain its loop target. +local continueSource = [[ + local sum = 0 + for i = 1, 8 do + if i % 2 == 0 then continue end + sum += i + end + return sum +]] +local continued = Prometheus.Pipeline:fromConfig({LuaVersion = 'LuaU', Seed = 4, + Steps = {{Name = 'Vmify'}}}):apply(continueSource) +assert(assert(loadstring(continued))() == 16) + +-- Informational microbenchmark, with a correctness check and no timing threshold. +local benchmark = [[ + local function add(a, b) return a + b end + local total = 0 + for i = 1, 3000 do total = total + add(i, 1) end + return total +]] +local nativeChunk = assert(loadstring(benchmark)) +local output = Prometheus.Pipeline:fromConfig({LuaVersion = 'Lua51', Seed = 17, + Steps = {{Name = 'Vmify'}}}):apply(benchmark) +local vmChunk = assert(loadstring(output)) +local start = os.clock() +local native = nativeChunk() +local nativeSeconds = os.clock() - start +start = os.clock() +local virtual = vmChunk() +local vmSeconds = os.clock() - start +assert(native == virtual) +print(string.format('VM fast-path benchmark: native %.4fs, Vmify %.4fs (3000 fixed calls)', + nativeSeconds, vmSeconds)) +print('VM fast-path behavioral and IR checks passed') diff --git a/tests/harness/vm-layout.lua b/tests/harness/vm-layout.lua index f7ad70c7..4423b1d2 100644 --- a/tests/harness/vm-layout.lua +++ b/tests/harness/vm-layout.lua @@ -67,6 +67,17 @@ assert(distinctOperands > 1, 'different seeds must change operand layouts') local source = [[ local function pack(...) return {n = select('#', ...), ...} end + local function fixed(a, b, c) return a, b, c end + local function pair(a, b) return a, b end + local function empty() end + empty() + local zero, one = pack(fixed()), pack(fixed(nil)) + local two, three = pack(fixed(2, nil)), pack(fixed(3, nil, 5)) + local first, second = pair(8, nil) + assert(zero.n == 3 and one.n == 3 and zero[1] == nil and one[1] == nil) + assert(two.n == 3 and two[1] == 2 and two[2] == nil and two[3] == nil) + assert(three.n == 3 and three[1] == 3 and three[2] == nil and three[3] == 5) + assert(first == 8 and second == nil) local function make(start) local total = start local unused = {} From 3a6175a54f512c82e342a7ac3106eb3ebd38b825 Mon Sep 17 00:00:00 2001 From: Elias Oelschner <62939318+levno-710@users.noreply.github.com> Date: Tue, 27 Oct 2026 17:05:41 +0100 Subject: [PATCH 8/8] fix cli warnings caused by vm reparsing --- src/prometheus/cff/native.lua | 2 +- src/prometheus/parser.lua | 4 +- src/prometheus/tokenizer.lua | 4 +- src/prometheus/vm/flatten.lua | 2 +- tests/harness/internal-identifiers.lua | 56 ++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 4 deletions(-) create mode 100644 tests/harness/internal-identifiers.lua diff --git a/src/prometheus/cff/native.lua b/src/prometheus/cff/native.lua index 098422d0..cab95386 100644 --- a/src/prometheus/cff/native.lua +++ b/src/prometheus/cff/native.lua @@ -115,7 +115,7 @@ return function(Compiler) finish = 'if ' .. names.setenv .. ' then ' .. names.setenv .. '(' .. names.result .. ',' .. names.env .. ') end;' end - local parsed = Parser:new({LuaVersion = 'Lua51'}):parse('return function(' + local parsed = Parser:new({LuaVersion = 'Lua51', AllowInternalIdentifiers = true}):parse('return function(' .. names.proxy .. ',' .. names.env .. ',' .. names.values .. ') local ' .. names.result .. '=' .. source .. ';' .. finish .. 'return ' .. names.result .. ' end') local factory = parsed.body.statements[1].args[1] diff --git a/src/prometheus/parser.lua b/src/prometheus/parser.lua index 57ce4dca..f863e9e4 100644 --- a/src/prometheus/parser.lua +++ b/src/prometheus/parser.lua @@ -9,6 +9,7 @@ -- -- Settings Object: -- luaVersion : The LuaVersion of the Script - Currently Supported : Lua51 and LuaU +-- AllowInternalIdentifiers : Skip reserved-prefix warnings for generated code (default false) -- local Tokenizer = require("prometheus.tokenizer"); @@ -63,7 +64,8 @@ function Parser:new(settings) local parser = { luaVersion = luaVersion, tokenizer = Tokenizer:new({ - luaVersion = luaVersion + luaVersion = luaVersion, + AllowInternalIdentifiers = settings and settings.AllowInternalIdentifiers == true, }), tokens = {}; length = 0; diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index 19295605..d0ef1f35 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -114,6 +114,7 @@ function Tokenizer:new(settings) length = 0, source = "", -- Source to Tokenize luaVersion = luaVersion, -- LuaVersion to be used while Tokenizing + allowInternalIdentifiers = settings and settings.AllowInternalIdentifiers == true, conventions = conventions; NumberChars = conventions.NumberChars, @@ -358,7 +359,8 @@ function Tokenizer:ident() local tk = token(self, startPos, Tokenizer.TokenKind.Ident, source); - if(string.sub(source, 1, string.len(config.IdentPrefix)) == config.IdentPrefix) then + if(not self.allowInternalIdentifiers + and string.sub(source, 1, string.len(config.IdentPrefix)) == config.IdentPrefix) then logger:warn(generateWarning(tk, string.format("identifiers should not start with \"%s\" as this may break the program", config.IdentPrefix))); end diff --git a/src/prometheus/vm/flatten.lua b/src/prometheus/vm/flatten.lua index c8a5b1c7..d559b106 100644 --- a/src/prometheus/vm/flatten.lua +++ b/src/prometheus/vm/flatten.lua @@ -42,7 +42,7 @@ return function(ast) local runtime = ast.vmRuntime local variables, writes = externalVariables(ast, runtime) if writes then return nil end - local parser = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local parser = Parser:new({LuaVersion = Enums.LuaVersion.Lua51, AllowInternalIdentifiers = true}) local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) local runtimeSource = unparser:unparseExpression(runtime, 0) local prefix = '' diff --git a/tests/harness/internal-identifiers.lua b/tests/harness/internal-identifiers.lua new file mode 100644 index 00000000..95ca6003 --- /dev/null +++ b/tests/harness/internal-identifiers.lua @@ -0,0 +1,56 @@ +local Prometheus = require('src.prometheus') +local Parser = require('prometheus.parser') +local logger = Prometheus.Logger + +local originalWarn, originalLevel = logger.warnCallback, logger.logLevel +local warnings = {} +logger.logLevel = logger.LogLevel.Warn +logger.warnCallback = function(message) warnings[#warnings + 1] = message end + +local function prefixWarnings() + local count = 0 + for _, message in ipairs(warnings) do + if message:find('identifiers should not start', 1, true) then count = count + 1 end + end + return count +end + +local ok, err = pcall(function() + local source = 'local __prometheus_example = 1; return __prometheus_example' + local parser = Parser:new({LuaVersion = 'Lua51'}) + parser:parse(source) + assert(prefixWarnings() == 2, 'User identifiers must still warn by default') + + warnings = {} + Parser:new({LuaVersion = 'Lua51', AllowInternalIdentifiers = false}):parse(source) + assert(prefixWarnings() == 2, 'Explicit false must preserve reserved-prefix warnings') + + warnings = {} + local internal = Parser:new({LuaVersion = 'Lua51', AllowInternalIdentifiers = true}) + internal:parse(source) + assert(#warnings == 0, 'Internal identifiers must not warn') + internal:parse('local __prometheus_example = 1, 2') + assert(prefixWarnings() == 0 and #warnings == 1 + and warnings[1]:find('assigning 2 values to 1 variable', 1, true), + 'Internal parsing must preserve unrelated warnings') + + warnings = {} + parser:parse(source) + assert(prefixWarnings() == 2, 'Internal parsing must not change other parser instances') + + for _, exclude in ipairs({true, false}) do + warnings = {} + local output = Prometheus.Pipeline:fromConfig({ + LuaVersion = 'Lua51', Seed = 1, + Steps = {{Name = 'Vmify'}, {Name = 'Cff', Settings = {ExcludeVm = exclude}}}, + }):apply('local value = ...; return value + 1, nil') + assert(prefixWarnings() == 0, 'Cff generated-code parsing emitted reserved-prefix warnings') + local chunk = assert(loadstring(output)) + assert(select('#', chunk(41)) == 2 and chunk(41) == 42, + 'Cff generated-code parsing changed runtime results') + end +end) + +logger.warnCallback, logger.logLevel = originalWarn, originalLevel +assert(ok, err) +print('Internal identifier diagnostics passed')