From 2fc7eeffe95f76fb1a167ab10150f662132ffc1f Mon Sep 17 00:00:00 2001 From: tlredz Date: Wed, 28 Oct 2026 20:20:09 -0300 Subject: [PATCH 1/3] add luau-support (AI help) --- .dockerignore | 3 + .gitignore | 3 + Dockerfile | 8 +- README.md | 6 +- doc/guides/cli-usage.md | 7 +- doc/reference/cli-options.md | 15 +- doc/reference/limitations.md | 31 ++ src/cli.lua | 165 ++++++---- src/prometheus/cff/expressions/if_else.lua | 3 + .../cff/statements/for_in_statement.lua | 2 +- src/prometheus/enums.lua | 2 +- src/prometheus/luau_types.lua | 305 ++++++++++++++++++ src/prometheus/parser.lua | 203 ++++++++++-- src/prometheus/pipeline.lua | 15 +- src/prometheus/tokenizer.lua | 102 +++++- src/prometheus/unparser.lua | 2 +- src/prometheus/visitast.lua | 2 +- tests/harness/cli.py | 111 +++++++ tests/harness/luau-parser.lua | 62 ++++ tests/harness/luau-syntax.lua | 21 ++ tests/harness/luau-syntax.py | 154 +++++++++ tests/harness/luau.py | 52 +++ tests/luau/conditionals.luau | 18 ++ tests/luau/const.luau | 39 +++ tests/luau/interpolation.luau | 55 ++++ .../luau/invalid/const-function-reassign.luau | 2 + tests/luau/invalid/const-missing-value.luau | 1 + tests/luau/invalid/const-reassign.luau | 4 + .../invalid/interpolation-double-brace.luau | 1 + tests/luau/invalid/interpolation-empty.luau | 1 + .../luau/invalid/interpolation-unclosed.luau | 1 + tests/luau/invalid/type-operator-chain.luau | 1 + tests/luau/invalid/type-pack-context.luau | 1 + tests/luau/invalid/typeof-expression.luau | 1 + tests/luau/iteration.luau | 48 +++ tests/luau/test-config.lua | 9 + tests/luau/types.luau | 37 +++ tests/luau/types/a.luau | 1 + tests/luau/types/a.obfuscated.luau | 1 + .../luau/unsupported/protected-metatable.luau | 5 + 40 files changed, 1401 insertions(+), 99 deletions(-) create mode 100644 src/prometheus/luau_types.lua create mode 100644 tests/harness/cli.py create mode 100644 tests/harness/luau-parser.lua create mode 100644 tests/harness/luau-syntax.lua create mode 100644 tests/harness/luau-syntax.py create mode 100644 tests/harness/luau.py create mode 100644 tests/luau/conditionals.luau create mode 100644 tests/luau/const.luau create mode 100644 tests/luau/interpolation.luau create mode 100644 tests/luau/invalid/const-function-reassign.luau create mode 100644 tests/luau/invalid/const-missing-value.luau create mode 100644 tests/luau/invalid/const-reassign.luau create mode 100644 tests/luau/invalid/interpolation-double-brace.luau create mode 100644 tests/luau/invalid/interpolation-empty.luau create mode 100644 tests/luau/invalid/interpolation-unclosed.luau create mode 100644 tests/luau/invalid/type-operator-chain.luau create mode 100644 tests/luau/invalid/type-pack-context.luau create mode 100644 tests/luau/invalid/typeof-expression.luau create mode 100644 tests/luau/iteration.luau create mode 100644 tests/luau/test-config.lua create mode 100644 tests/luau/types.luau create mode 100644 tests/luau/types/a.luau create mode 100644 tests/luau/types/a.obfuscated.luau create mode 100644 tests/luau/unsupported/protected-metatable.luau diff --git a/.dockerignore b/.dockerignore index 238021da..9869235b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,6 @@ Dockerfile test-results __pycache__ + +# Local reference implementation +.reference-obfuscator/ diff --git a/.gitignore b/.gitignore index f26c1274..17a8eb3b 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,6 @@ config.lua # Lua test logs and reproduction artifacts /test-results/ __pycache__/ + +# Local reference implementation, not part of Prometheus +/.reference-obfuscator/ diff --git a/Dockerfile b/Dockerfile index c0057c9b..173531b3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,8 +14,8 @@ RUN --mount=type=secret,id=ca-bundle \ if [ -f /run/secrets/ca-bundle ]; then export CURL_CA_BUNDLE=/run/secrets/ca-bundle; fi; \ curl -fsSLo /tmp/luau-ubuntu.zip \ https://github.com/luau-lang/luau/releases/latest/download/luau-ubuntu.zip \ - && unzip -j /tmp/luau-ubuntu.zip luau -d /usr/local/bin \ - && chmod +x /usr/local/bin/luau \ + && unzip -j /tmp/luau-ubuntu.zip luau luau-compile -d /usr/local/bin \ + && chmod +x /usr/local/bin/luau /usr/local/bin/luau-compile \ && rm /tmp/luau-ubuntu.zip # Verify installations @@ -25,6 +25,10 @@ WORKDIR /app COPY . /app RUN lua5.1 tests/harness/metadata.lua \ + && timeout 60s lua5.1 tests/harness/luau-parser.lua \ + && python3 tests/harness/luau-syntax.py \ + && python3 tests/harness/luau.py \ + && python3 tests/harness/cli.py \ && timeout 60s lua5.1 tests/harness/numbers-to-expressions.lua \ && timeout 60s lua5.1 tests/harness/cff-arity.lua \ && timeout 60s lua5.1 tests/harness/cff-native.lua \ diff --git a/README.md b/README.md index 2f03a422..848c68b4 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,11 @@ The project was inspired by the excellent [javascript-obfuscator](https://github Currently, Prometheus supports: - **Lua 5.1** -- **LuaU** *(basic support is available, but still not fully finished)* +- **Luau** *(partial support, including type erasure, interpolation, `const`, and generalized iteration; see [limitations](doc/reference/limitations.md#luau))* + +To process Luau, pass `--luau` (or `--LuaU`) to the CLI or set `LuaVersion = "LuaU"` in +configuration. Type annotations, assertions, and type declarations are removed; +output still targets the Luau runtime. See [Luau limitations](doc/reference/limitations.md#luau). Lua 5.1's legacy implicit `arg` table in vararg functions is unsupported. Use `...` and `select('#', ...)` instead; see [Limitations](doc/reference/limitations.md). diff --git a/doc/guides/cli-usage.md b/doc/guides/cli-usage.md index aeddc021..6c7339ec 100644 --- a/doc/guides/cli-usage.md +++ b/doc/guides/cli-usage.md @@ -18,6 +18,7 @@ prometheus-lua --preset Medium ./input.lua If `--out` is not provided: - `input.lua` -> `input.obfuscated.lua` +- `input.luau` -> `input.obfuscated.luau` - `input` -> `input.obfuscated.lua` ## Common workflows @@ -37,7 +38,7 @@ prometheus-lua --config ./prometheus.config.lua ./src/main.lua Force Lua target: ```bash -prometheus-lua --preset Medium --LuaU ./src/main.lua +prometheus-lua --preset medium --luau ./src/main.luau ``` Enable pretty output: @@ -48,6 +49,8 @@ prometheus-lua --preset Minify --pretty ./src/main.lua ## Notes -- Unknown `--...` options are ignored with a warning. +- Command, option, and preset names are case-insensitive; paths preserve their case. +- Options with values accept both `--preset medium` and `--preset=medium`. +- Unknown options and missing option values are errors. - If no config/preset is passed, Prometheus falls back to `Minify`. - `update` command uses the official installer script from GitHub. diff --git a/doc/reference/cli-options.md b/doc/reference/cli-options.md index c95d485c..433060d5 100644 --- a/doc/reference/cli-options.md +++ b/doc/reference/cli-options.md @@ -15,14 +15,19 @@ | `--preset` | `--p` | `` | Use built-in preset by name | | `--config` | `--c` | `` | Load config Lua file | | `--out` | `--o` | `` | Output file path | -| `--Lua51` | - | none | Force Lua 5.1 conventions | -| `--LuaU` | - | none | Force LuaU conventions | +| `--lua51` | - | none | Force Lua 5.1 conventions | +| `--luau` | - | none | Force Luau conventions | | `--pretty` | - | none | Pretty-print generated output | | `--nocolors` | - | none | Disable ANSI colors | | `--saveerrors` | - | none | Write parser/runtime errors to `.error.txt` | ## Parsing behavior -- First non-option argument is treated as input file. -- A second non-option argument raises an error. -- Unknown options are ignored with a warning. +- Command names, option names, and built-in preset names are case-insensitive. Existing spellings + such as `--LuaU` and `--Lua51` continue to work. +- File paths and configuration values retain their original case. +- Options that take a value accept either `--option value` or `--option=value`. +- The first non-option argument is the input file; a second one is an error. +- Use `--` before an input filename beginning with `-`. For an option value + beginning with `-`, use `--option=value` or a path prefixed with `./`. +- Unknown options, missing values, and values supplied to flags are errors. diff --git a/doc/reference/limitations.md b/doc/reference/limitations.md index 4848f63b..3161f960 100644 --- a/doc/reference/limitations.md +++ b/doc/reference/limitations.md @@ -12,6 +12,37 @@ obfuscation: Source code constructed or loaded at runtime is not automatically obfuscated. +## Luau + +Use `--LuaU` or `LuaVersion = "LuaU"` for Luau input. Type declarations, +annotations, assertions (`::`), and type function declarations are removed; +Prometheus does not type-check them, evaluate `typeof` expressions in type +positions, or retain exported types for consumers. The type grammar is validated +before erasure, including type packs, generic parameter defaults, and union and +intersection grouping. Unknown type names remain syntactically valid; use Luau's +type checker to resolve names and verify type compatibility. Run the output on Luau. + +Conditional expressions, including `elseif` branches, interpolated strings, +contextual [`const` declarations](https://rfcs.luau.org/const-keyword.html), and +generalized `for` iteration are supported. `const` bindings are checked for +reassignment and lowered to locals; tables +referenced by a `const` remain mutable. Interpolation uses Luau's `%*` string +formatting and therefore requires a Luau runtime for the transformed code. + +Minify leaves generalized iteration to Luau. The `Cff` and `Vmify` compilers normalize tables, `__iter`, callable objects, and +ordinary iterator triples. This normalization uses the runtime's `type`, +`getmetatable`, `rawget`, `next`, and `error` globals; they must retain their +standard behavior. Generalized iteration over objects with protected +metatables is not supported by these compilers: Luau hides the real metatable +from ordinary Lua code. Use an explicit iterator function, for example +`for k, v in pairs(t) do`, when appropriate. A non-table protection value +is rejected; a table used as `__metatable` cannot be distinguished from a +real metatable and may cause different behavior. + +Luau's floor division (`//` and `//=`) and the use of +`continue` as an identifier are not supported. Type function bodies must use +syntax that the parser supports even though those declarations are erased. + ## Replacing a running function's environment Prometheus does not support replacing a function's environment while that diff --git a/src/cli.lua b/src/cli.lua index 0e8282ed..5bf9a24c 100644 --- a/src/cli.lua +++ b/src/cli.lua @@ -41,14 +41,17 @@ local function print_help() print(" --version, -v Print CLI version") print("") print("Obfuscation options:") - print(" --preset, --p Use preset (e.g. Minify, Medium, High)") + print(" --preset, --p Use preset (e.g. Minify, Medium, Strong)") print(" --config, --c Use custom config Lua file") print(" --out, --o Set output path") - print(" --Lua51 Force Lua 5.1 target") - print(" --LuaU Force LuaU target") + print(" --lua51 Force Lua 5.1 target") + print(" --luau Force Luau target") print(" --pretty Pretty print output") print(" --nocolors Disable colored logs") print(" --saveerrors Persist parser errors to file") + print("") + print("Command, option, and preset names are case-insensitive; paths keep their original case.") + print("Options with a value also accept --option=value. Use -- before an input starting with '-'.") end local function run_update() @@ -71,17 +74,17 @@ local function run_update() print("Update completed") end -if arg[1] == "update" then +local firstArg = arg[1] and arg[1]:lower() +if firstArg == "update" then run_update() os.exit(0) end - -if arg[1] == "--version" or arg[1] == "-v" then +if firstArg == "--version" or firstArg == "-v" then print(get_version()) os.exit(0) end -if arg[1] == "--help" or arg[1] == "-h" or arg[1] == "help" then +if firstArg == "--help" or firstArg == "-h" or firstArg == "help" then print_help() os.exit(0) end @@ -148,80 +151,127 @@ local function load_chunk(content, chunkName, environment) return load(content, chunkName, "t", environment) end +local function output_stem(path) + local base, extension = path:match("^(.*)(%.[^/\\%.]+)$") + if extension and (extension:lower() == ".lua" or extension:lower() == ".luau") then + return base, extension + end + return path, ".lua" +end + +local function find_preset(name) + local exact = Prometheus.Presets[name] + if exact then return exact end + for presetName, preset in pairs(Prometheus.Presets) do + if presetName:lower() == name:lower() then return preset end + end +end + local function run_cli() - -- CLI local config, sourceFile, outFile, luaVersion, prettyPrint - + local optionsEnabled = true Prometheus.colors.enabled = true - -- Parse Arguments local i = 1 while i <= #arg do local curr = arg[i] - if curr:sub(1, 2) == "--" then - if curr == "--preset" or curr == "--p" then - if config then - Prometheus.Logger:warn("The config was set multiple times") + if optionsEnabled and curr == "--" then + optionsEnabled = false + elseif optionsEnabled and #curr > 1 and curr:sub(1, 1) == "-" then + local option, inlineValue = curr:match("^(%-%-[^=]+)=(.*)$") + option = (option or curr):lower() + + local function value() + if inlineValue ~= nil then + if inlineValue == "" then + Prometheus.Logger:error(string.format('The option "%s" requires a value', curr)) + end + return inlineValue + end + local nextArg = arg[i + 1] + if not nextArg or nextArg == "" or nextArg:sub(1, 1) == "-" then + Prometheus.Logger:error(string.format('The option "%s" requires a value', curr)) end - i = i + 1 - local preset = Prometheus.Presets[arg[i]] + return nextArg + end + + local takesValue = option == "--preset" or option == "--p" or option == "--config" + or option == "--c" or option == "--out" or option == "--o" + local isFlag = option == "--help" or option == "-h" or option == "--version" or option == "-v" + or option == "--nocolors" or option == "--lua51" or option == "--luau" + or option == "--pretty" or option == "--saveerrors" + if inlineValue ~= nil and not takesValue and isFlag then + Prometheus.Logger:error(string.format('The option "%s" does not accept a value', curr)) + end + + if option == "--help" or option == "-h" then + print_help() + return + elseif option == "--version" or option == "-v" then + print(get_version()) + return + elseif option == "--preset" or option == "--p" then + local name = value() + local preset = find_preset(name) if not preset then - Prometheus.Logger:error(string.format('A Preset with the name "%s" was not found!', tostring(arg[i]))) + Prometheus.Logger:error(string.format('A preset with the name "%s" was not found!', name)) end - - config = preset - elseif curr == "--config" or curr == "--c" then - i = i + 1 - local filename = tostring(arg[i]) + if config then Prometheus.Logger:warn("The config was set multiple times") end + config = {} + for key, entry in pairs(preset) do config[key] = entry end + elseif option == "--config" or option == "--c" then + local filename = value() if not file_exists(filename) then Prometheus.Logger:error(string.format('The config file "%s" was not found!', filename)) end - local content = table.concat(lines_from(filename), "\n") - -- Load Config from File local func, err = load_chunk(content, "@" .. filename, {}) if not func then Prometheus.Logger:error(string.format('Failed to parse config file "%s": %s', filename, tostring(err))) end + if config then Prometheus.Logger:warn("The config was set multiple times") end config = func() - elseif curr == "--out" or curr == "--o" then - i = i + 1 - if outFile then - Prometheus.Logger:warn("The output file was specified multiple times!") + if type(config) ~= "table" then + Prometheus.Logger:error(string.format('The config file "%s" must return a table', filename)) end - outFile = arg[i] - elseif curr == "--nocolors" then + elseif option == "--out" or option == "--o" then + local filename = value() + if outFile then Prometheus.Logger:warn("The output file was specified multiple times!") end + outFile = filename + elseif option == "--nocolors" then Prometheus.colors.enabled = false - elseif curr == "--Lua51" then + elseif option == "--lua51" then luaVersion = "Lua51" - elseif curr == "--LuaU" then + elseif option == "--luau" then luaVersion = "LuaU" - elseif curr == "--pretty" then + elseif option == "--pretty" then prettyPrint = true - elseif curr == "--saveerrors" then - -- Override error callback + elseif option == "--saveerrors" then Prometheus.Logger.errorCallback = function(...) - local args = { ... } - local message = table.concat(args, " ") + local message = table.concat({ ... }, " ") io.stderr:write(Prometheus.colors(Prometheus.Config.NameUpper .. ": " .. message, "red") .. "\n") - - local fileName = sourceFile:sub(-4) == ".lua" and sourceFile:sub(0, -5) .. ".error.txt" - or sourceFile .. ".error.txt" - local handle = io.open(fileName, "w") - handle:write(message) - handle:close() - + if sourceFile then + local stem = output_stem(sourceFile) + local filename = stem .. ".error.txt" + local handle, writeError = io.open(filename, "w") + if handle then + handle:write(message) + handle:close() + else + io.stderr:write(string.format('Cannot write error file "%s": %s\n', filename, tostring(writeError))) + end + end os.exit(1) end else - Prometheus.Logger:warn(string.format('The option "%s" is not valid and therefore ignored', curr)) + Prometheus.Logger:error(string.format('Unknown option "%s"', curr)) end else if sourceFile then - Prometheus.Logger:error(string.format('Unexpected argument "%s"', arg[i])) + Prometheus.Logger:error(string.format('Unexpected argument "%s"', curr)) end - sourceFile = tostring(arg[i]) + sourceFile = curr end i = i + 1 end @@ -229,26 +279,21 @@ local function run_cli() if not sourceFile then Prometheus.Logger:error("No input file was specified!") end - if not config then Prometheus.Logger:warn("No config was specified, falling back to Minify preset") - config = Prometheus.Presets.Minify + config = {} + for key, entry in pairs(Prometheus.Presets.Minify) do config[key] = entry end end - -- Add Option to override Lua Version config.LuaVersion = luaVersion or config.LuaVersion config.PrettyPrint = prettyPrint ~= nil and prettyPrint or config.PrettyPrint if not file_exists(sourceFile) then - Prometheus.Logger:error(string.format('The File "%s" was not found!', sourceFile)) + Prometheus.Logger:error(string.format('The file "%s" was not found!', sourceFile)) end - if not outFile then - if sourceFile:sub(-4) == ".lua" then - outFile = sourceFile:sub(0, -5) .. ".obfuscated.lua" - else - outFile = sourceFile .. ".obfuscated.lua" - end + local stem, extension = output_stem(sourceFile) + outFile = stem .. ".obfuscated" .. extension end local source = table.concat(lines_from(sourceFile), "\n") @@ -256,8 +301,10 @@ local function run_cli() local out = pipeline:apply(source, sourceFile) Prometheus.Logger:info(string.format('Writing output to "%s"', outFile)) - -- Write Output - local handle = io.open(outFile, "w") + local handle, writeError = io.open(outFile, "w") + if not handle then + Prometheus.Logger:error(string.format('Cannot write output file "%s": %s', outFile, tostring(writeError))) + end handle:write(out) handle:close() end diff --git a/src/prometheus/cff/expressions/if_else.lua b/src/prometheus/cff/expressions/if_else.lua index 317b0c68..72892249 100644 --- a/src/prometheus/cff/expressions/if_else.lua +++ b/src/prometheus/cff/expressions/if_else.lua @@ -41,6 +41,7 @@ return function(self, expression, funcDepth, numReturns) local trueReg = self:compileExpression(expression.true_value, funcDepth, 1)[1]; self:addStatement(self:copyRegisters(scope, {resReg}, {trueReg}), {resReg}, {trueReg}, false); + self:freeRegister(trueReg, false); self:addStatement(self:setRegister(scope, self.POS_REGISTER, Ast.NumberExpression(finalBlock.id)), {self.POS_REGISTER}, {}, false); for _, elif in ipairs(expression.elseifs) do @@ -57,6 +58,7 @@ return function(self, expression, funcDepth, numReturns) elifScope = elifBlock.scope; local valueReg = self:compileExpression(elif.value, funcDepth, 1)[1]; self:addStatement(self:copyRegisters(elifScope, {resReg}, {valueReg}), {resReg}, {valueReg}, false); + self:freeRegister(valueReg, false); self:addStatement(self:setRegister(elifScope, self.POS_REGISTER, Ast.NumberExpression(finalBlock.id)), {self.POS_REGISTER}, {}, false); end @@ -64,6 +66,7 @@ return function(self, expression, funcDepth, numReturns) scope = self.activeBlock.scope; local falseReg = self:compileExpression(expression.false_value, funcDepth, 1)[1]; self:addStatement(self:copyRegisters(scope, {resReg}, {falseReg}), {resReg}, {falseReg}, false); + self:freeRegister(falseReg, false); self:addStatement(self:setRegister(scope, self.POS_REGISTER, Ast.NumberExpression(finalBlock.id)), {self.POS_REGISTER}, {}, false); self.registers[self.POS_REGISTER] = posState; diff --git a/src/prometheus/cff/statements/for_in_statement.lua b/src/prometheus/cff/statements/for_in_statement.lua index 3a489321..149b0e87 100644 --- a/src/prometheus/cff/statements/for_in_statement.lua +++ b/src/prometheus/cff/statements/for_in_statement.lua @@ -74,7 +74,7 @@ return function(self, statement, funcDepth) self:addStatement(Ast.AssignmentStatement({ self:posAssignment(scope) }, { - Ast.OrExpression(Ast.AndExpression(self:register(scope, exprregs[3]), Ast.NumberExpression(bodyBlock.id)), Ast.NumberExpression(finalBlock.id)) + Ast.OrExpression(Ast.AndExpression(Ast.NotEqualsExpression(self:register(scope, exprregs[3]), Ast.NilExpression()), Ast.NumberExpression(bodyBlock.id)), Ast.NumberExpression(finalBlock.id)) }), {self.POS_REGISTER}, {exprregs[3]}, false); self:setActiveBlock(bodyBlock); diff --git a/src/prometheus/enums.lua b/src/prometheus/enums.lua index e8880be2..62a79122 100644 --- a/src/prometheus/enums.lua +++ b/src/prometheus/enums.lua @@ -65,7 +65,7 @@ Enums.Conventions = { "repeat", "return", "then", "true", "until", "while" }, - SymbolChars = chararray("+-*/%^#=~<>(){}[];:,."), + SymbolChars = chararray("+-*/%^#=~<>(){}[];:,.?|&"), MaxSymbolLength = 3, Symbols = { "+", "-", "*", "/", "%", "^", "#", diff --git a/src/prometheus/luau_types.lua b/src/prometheus/luau_types.lua new file mode 100644 index 00000000..3571244a --- /dev/null +++ b/src/prometheus/luau_types.lua @@ -0,0 +1,305 @@ +-- Validate Luau's type grammar before erasing compile-time syntax. +-- Types, packs, generic declarations, and generic arguments have distinct rules. +-- Reference: luau-lang/luau Ast/src/Parser.cpp (parseType and related routines). +local Scope = require('prometheus.scope') +local logger = require('logger') + +return function(Parser, token, is, consume, expect, peek, get) + local S, I, K = token.Symbol, token.Ident, token.Keyword + local TYPE = {kind = 'type'} + local parseType, parseSimpleType, parseTypeSuffix, parseReturnType + local parseGenerics, parseTypeArguments + + local function fail(self, message) + local tk = peek(self) + local diagnostic = string.format('Parsing Error at Position %s:%s, %s', + tostring(tk.line or '?'), tostring(tk.linePos or '?'), message) + if self.disableLog then error(diagnostic, 0) end + logger:error(diagnostic) + end + + local function startsPack(self) + return is(self, S, '...') or (is(self, I) and is(self, S, '...', 1)) + end + + local function typeFollows(self) + return is(self, S, '?') or is(self, S, '|') or is(self, S, '&') + end + + local function parsePack(self) + if consume(self, S, '...') then + parseType(self) + else + expect(self, I) + expect(self, S, '...') + end + return {kind = 'pack'} + end + + -- A pack is a tail: it cannot be followed by another element in a type list. + local function parseTypeList(self) + local result = {kind = 'pack', count = 0, tail = false, named = false} + if is(self, S, ')') then return result end + repeat + if startsPack(self) then + parsePack(self) + result.tail = true + return result + end + if is(self, I) and is(self, S, ':', 1) then + get(self) + get(self) + result.named = true + end + parseType(self) + result.count = result.count + 1 + until not consume(self, S, ',') + return result + end + + local function parseFunctionTail(self) + expect(self, S, '->') + parseReturnType(self) + return TYPE + end + + local function parseParenthesized(self, allowPack) + local generic = is(self, S, '<') + if generic then parseGenerics(self, false) end + expect(self, S, '(') + local list = parseTypeList(self) + expect(self, S, ')') + if generic or list.named or is(self, S, '->') or is(self, S, ':') then + return parseFunctionTail(self) + end + if allowPack then return list end + if list.count == 1 and not list.tail then return TYPE end + fail(self, "Expected '->' after function parameter types; a type pack is not a single type") + end + + parseReturnType = function(self) + if startsPack(self) then return parsePack(self) end + if not consume(self, S, '(') then + parseType(self) + return {kind = 'pack', count = 1} + end + local list = parseTypeList(self) + expect(self, S, ')') + if list.named or is(self, S, '->') or is(self, S, ':') then + parseFunctionTail(self) + return {kind = 'pack', count = 1} + end + -- A single parenthesized return type can have union/optional suffixes. + if list.count == 1 then parseTypeSuffix(self, TYPE) end + return list + end + + local function parseTable(self) + expect(self, S, '{') + local properties, indexer = 0, false + while not consume(self, S, '}') do + if (is(self, I, 'read') or is(self, I, 'write')) and not is(self, S, ':', 1) then + get(self) + end + if consume(self, S, '[') then + if is(self, token.String) and is(self, S, ']', 1) then + local name = get(self) + if name.value:find('\0', 1, true) then fail(self, 'A table type property cannot contain a null byte') end + properties = properties + 1 + else + if indexer then fail(self, 'A table type cannot contain more than one indexer') end + parseType(self) + indexer = true + end + expect(self, S, ']') + expect(self, S, ':') + parseType(self) + elseif properties == 0 and not indexer and not (is(self, I) and is(self, S, ':', 1)) then + -- Array shorthand is exactly {Type}, not a list of table fields. + parseType(self) + expect(self, S, '}') + return TYPE + else + expect(self, I) + expect(self, S, ':') + parseType(self) + properties = properties + 1 + end + if not consume(self, S, ',') and not consume(self, S, ';') then + expect(self, S, '}') + return TYPE + end + end + return TYPE + end + + parseSimpleType = function(self, allowPack) + if is(self, S, '(') or is(self, S, '<') then + return parseParenthesized(self, allowPack) + elseif is(self, S, '{') then + return parseTable(self) + elseif is(self, token.String) or is(self, K, 'nil') or is(self, K, 'true') or is(self, K, 'false') then + get(self) + return TYPE + elseif is(self, I) then + local name = get(self) + if consume(self, S, '.') then + expect(self, I) + elseif name.value == 'typeof' then + expect(self, S, '(') + -- Parse the real expression grammar in a detached scope. It is never + -- evaluated or added to the runtime AST and its reference graph. + self:expression(Scope:new(Scope:newGlobal())) + expect(self, S, ')') + return TYPE + end + if is(self, S, '...') then fail(self, 'A type pack is not allowed in this type position') end + if is(self, S, '<') then parseTypeArguments(self) end + return TYPE + end + fail(self, 'Expected a type, got ' .. tostring(peek(self).source)) + end + + parseTypeSuffix = function(self, initial) + local union, intersection = false, false + local hasType = initial ~= nil + while true do + if consume(self, S, '|') then + parseSimpleType(self, false) + union, hasType = true, true + elseif consume(self, S, '&') then + parseSimpleType(self, false) + intersection, hasType = true, true + elseif consume(self, S, '?') then + if not hasType then fail(self, "Expected a type before '?'") end + union = true + elseif is(self, S, '...') then + fail(self, "Unexpected '...' after a type") + else + break + end + if union and intersection then + fail(self, 'Mixing union and intersection types requires parentheses') + end + end + return TYPE + end + + parseType = function(self) + local initial + if not is(self, S, '|') and not is(self, S, '&') then initial = parseSimpleType(self, false) end + return parseTypeSuffix(self, initial) + end + + local function parseTypeOrPack(self) + if startsPack(self) then return parsePack(self) end + if is(self, S, '(') then + local result = parseSimpleType(self, true) + if result.kind == 'pack' then + if result.count == 1 and not result.tail and typeFollows(self) then + return parseTypeSuffix(self, TYPE) + end + return result + end + return parseTypeSuffix(self, result) + end + return parseType(self) + end + + parseTypeArguments = function(self) + expect(self, S, '<') + if consume(self, S, '>') then return end + repeat + parseTypeOrPack(self) + until not consume(self, S, ',') + expect(self, S, '>') + end + + parseGenerics = function(self, defaultsAllowed) + if not consume(self, S, '<') then return false end + local seenPack, seenDefault = false, false + repeat + expect(self, I) + local pack = consume(self, S, '...') + if seenPack and not pack then fail(self, 'Generic types must precede generic type packs') end + seenPack = seenPack or pack + if consume(self, S, '=') then + if not defaultsAllowed then fail(self, 'Generic defaults are only allowed in type aliases') end + seenDefault = true + if pack then + local result = parseTypeOrPack(self) + if result.kind ~= 'pack' then fail(self, 'Expected a type pack as the generic default') end + else + parseType(self) + end + elseif seenDefault then + fail(self, 'A generic parameter after a default must also have a default') + end + until not consume(self, S, ',') + expect(self, S, '>') + return true + end + + function Parser:parseLuauTypeAnnotation() + if self.luaVersion == 'LuaU' and consume(self, S, ':') then + parseType(self) + return true + end + return false + end + + function Parser:parseLuauReturnAnnotation() + if self.luaVersion == 'LuaU' and consume(self, S, ':') then + parseReturnType(self) + return true + end + return false + end + + function Parser:parseLuauVarargAnnotation() + if self.luaVersion == 'LuaU' and consume(self, S, ':') then + if is(self, I) and is(self, S, '...', 1) then parsePack(self) else parseType(self) end + return true + end + return false + end + + function Parser:parseLuauGenerics() + if self.luaVersion == 'LuaU' then return parseGenerics(self, false) end + return false + end + + function Parser:parseLuauCast() + if self.luaVersion == 'LuaU' and consume(self, S, '::') then + parseType(self) + return true + end + return false + end + + function Parser:parseLuauDeclaration() + if self.luaVersion ~= 'LuaU' then return false end + local offset = is(self, I, 'export') and 1 or 0 + if not is(self, I, 'type', offset) then return false end + if not is(self, I, nil, offset + 1) and not is(self, K, 'function', offset + 1) then return false end + if offset == 1 then get(self) end + get(self) + if consume(self, K, 'function') then + expect(self, I) + parseGenerics(self, false) + expect(self, S, '(') + local scope = Scope:new(Scope:newGlobal()) + self:functionArgList(scope) + expect(self, S, ')') + self:parseLuauReturnAnnotation() + self:block(nil, false, scope) + expect(self, K, 'end') + else + expect(self, I) + parseGenerics(self, true) + expect(self, S, '=') + parseType(self) + end + return true + end +end diff --git a/src/prometheus/parser.lua b/src/prometheus/parser.lua index f863e9e4..09817040 100644 --- a/src/prometheus/parser.lua +++ b/src/prometheus/parser.lua @@ -47,7 +47,8 @@ local function generateError(self, message) if(self.index > self.length) then token = self.tokens[self.length]; elseif(self.index < 1) then - return "Parsing Error at Position 0:0, " .. message; + token = self.tokens[1]; + if not token then return "Parsing Error at Position 0:0, " .. message end else token = self.tokens[self.index]; end @@ -70,6 +71,7 @@ function Parser:new(settings) tokens = {}; length = 0; index = 0; + lowerLuauIteration = settings and settings.LowerLuauIteration == true; }; setmetatable(parser, self); @@ -101,23 +103,14 @@ local function get(self) end local function is(self, kind, sourceOrN, n) - local token = peek(self, n); - - local source = nil; - if(type(sourceOrN) == "string") then - source = sourceOrN; - else + local source; + if type(sourceOrN) == "number" then n = sourceOrN; + else + source = sourceOrN; end - n = n or 0; - - if(token.kind == kind) then - if(source == nil or token.source == source) then - return true; - end - end - - return false; + local tk = peek(self, n); + return tk.kind == kind and (source == nil or tk.source == source); end local function consume(self, kind, source) @@ -142,10 +135,79 @@ local function expect(self, kind, source) end end +require("prometheus.luau_types")(Parser, TokenKind, is, consume, expect, peek, get); + +-- The compilers call iterator functions directly. Normalize Luau's additional +-- generalized iteration cases before compiling while keeping Lua 5.1 unchanged. +local iteratorHelper = [=[ +local function %s(iterator, state, control) + if type(iterator) == "function" then return iterator, state, control end + local mt = getmetatable(iterator) + if mt ~= nil and type(mt) ~= "table" then + error("Generalized iteration with a protected metatable is unsupported", 2) + end + if mt then + local iter = rawget(mt, "__iter") + if iter ~= nil then return iter(iterator) end + if rawget(mt, "__call") ~= nil then return iterator, state, control end + end + if type(iterator) == "table" then return next, iterator, nil end + return iterator, state, control +end +]=] + +local function markConst(scope, id) + scope.constBindings = scope.constBindings or {}; + scope.constBindings[id] = true; +end + +local function assertMutable(self, scope, id) + if scope.constBindings and scope.constBindings[id] then + if self.disableLog then error() end + logger:error(generateError(self, string.format("Cannot reassign constant '%s'", scope:getVariableName(id)))); + end +end + -- Parse the given code to an Abstract Syntax Tree function Parser:parse(code) + -- A failed parse must not leave input behind for the next invocation. + self.tokenizer:reset(); + self.tokens, self.index, self.length = {}, 0, 0; + self.disableLog = false; + self.luauIteratorName = nil; self.tokenizer:append(code); self.tokens = self.tokenizer:scanAll(); + + if self.luaVersion == LuaVersion.LuaU and self.lowerLuauIteration then + local hasLoop, names = false, {}; + local function collectTokens(tokens) + for _, tk in ipairs(tokens) do + if tk.kind == TokenKind.Keyword and tk.source == "for" then hasLoop = true end + if tk.kind == TokenKind.Ident then names[tk.source] = true end + if tk.kind == TokenKind.InterpolatedString then + for _, part in ipairs(tk.value) do + if part.tokens then collectTokens(part.tokens) end + end + end + end + end + collectTokens(self.tokens); + if hasLoop then + local i = 0; + repeat + i = i + 1; + self.luauIteratorName = "prometheus_luau_iterator_" .. i; + until not names[self.luauIteratorName]; + + -- Tokenize the helper separately to keep input error locations intact. + local helper = Tokenizer:new({LuaVersion = LuaVersion.LuaU}); + helper:append(string.format(iteratorHelper, self.luauIteratorName)); + local tokens = helper:scanAll(); + tokens[#tokens] = nil; + for _, tk in ipairs(self.tokens) do tokens[#tokens + 1] = tk end + self.tokens = tokens; + end + end self.length = #self.tokens; -- Create Global Variable Scope @@ -186,8 +248,8 @@ end function Parser:statement(scope, currentLoop) -- Skip all semicolons before next real statement -- NOP statements are therefore ignored - while(consume(self, TokenKind.Symbol, ";")) do - + while true do + if not consume(self, TokenKind.Symbol, ";") and not self:parseLuauDeclaration() then break end end -- Break Statement - only valid inside of Loops @@ -284,12 +346,15 @@ function Parser:statement(scope, currentLoop) local baseScope = obj.scope; local baseId = obj.id; local indices = obj.indices; + if #indices == 0 then assertMutable(self, baseScope, baseId) end local funcScope = Scope:new(scope); + self:parseLuauGenerics(); expect(self, TokenKind.Symbol, "("); local args = self:functionArgList(funcScope); expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); if(obj.passSelf) then local id = funcScope:addVariable("self", obj.token); @@ -302,19 +367,24 @@ function Parser:statement(scope, currentLoop) return Ast.FunctionDeclaration(baseScope, baseId, indices, args, body); end - -- Local Function or Variable Declaration - if(consume(self, TokenKind.Keyword, "local")) then + -- 'const' is contextual and remains a valid identifier outside declarations. + local isConst = self.luaVersion == LuaVersion.LuaU and is(self, TokenKind.Ident, "const") + and (is(self, TokenKind.Ident, nil, 1) or is(self, TokenKind.Keyword, "function", 1)); + if consume(self, TokenKind.Keyword, "local") or (isConst and consume(self, TokenKind.Ident, "const")) then -- Local Function Declaration if(consume(self, TokenKind.Keyword, "function")) then local ident = expect(self, TokenKind.Ident); local name = ident.value; local id = scope:addVariable(name, ident); + if isConst then markConst(scope, id) end local funcScope = Scope:new(scope); + self:parseLuauGenerics(); expect(self, TokenKind.Symbol, "("); local args = self:functionArgList(funcScope); expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); local body = self:block(nil, false, funcScope); expect(self, TokenKind.Keyword, "end"); @@ -324,9 +394,26 @@ function Parser:statement(scope, currentLoop) -- Local Variable Declaration local ids = self:nameList(scope); + if isConst then + for _, id in ipairs(ids) do markConst(scope, id) end + end local expressions = {}; if(consume(self, TokenKind.Symbol, "=")) then expressions = self:exprList(scope); + elseif isConst then + if self.disableLog then error() end + logger:error(generateError(self, "A const declaration requires an initializer")); + end + + if isConst and #expressions < #ids then + local last = expressions[#expressions]; + local expands = last and not last.isParenthesizedExpression + and (last.kind == AstKind.FunctionCallExpression or last.kind == AstKind.PassSelfFunctionCallExpression + or last.kind == AstKind.VarargExpression); + if not expands then + if self.disableLog then error() end + logger:error(generateError(self, "A const declaration requires an initializer for each binding")); + end end -- Variables can only be reffered to in the next statement, so the id's are enabled after the expressions have been parsed @@ -344,11 +431,17 @@ function Parser:statement(scope, currentLoop) -- For Statement if(consume(self, TokenKind.Keyword, "for")) then -- Normal for Statement - if(is(self, TokenKind.Symbol, "=", 1)) then + local savedIndex = self.index; + expect(self, TokenKind.Ident); + self:parseLuauTypeAnnotation(); + local isNumericFor = is(self, TokenKind.Symbol, "="); + self.index = savedIndex; + if(isNumericFor) then local forScope = Scope:new(scope); local ident = expect(self, TokenKind.Ident); local varId = forScope:addDisabledVariable(ident.value, ident); + self:parseLuauTypeAnnotation(); expect(self, TokenKind.Symbol, "="); local initialValue = self:expression(scope); @@ -374,6 +467,10 @@ function Parser:statement(scope, currentLoop) local ids = self:nameList(forScope); expect(self, TokenKind.Keyword, "in"); local expressions = self:exprList(scope); + if self.luauIteratorName then + local helperScope, helperId = scope:resolve(self.luauIteratorName); + expressions = {Ast.FunctionCallExpression(Ast.VariableExpression(helperScope, helperId), expressions)}; + end -- Enable Ids after Expression Parsing so that code like this works: -- local z = {10,20} -- for y,z in ipairs(z) do @@ -391,6 +488,10 @@ function Parser:statement(scope, currentLoop) local expr = self:primaryExpression(scope); -- Variable Assignment or Function Call if expr then + if expr.isInterpolatedStringExpression then + if self.disableLog then error() end + logger:error(generateError(self, "An interpolated string is not a valid statement")); + end -- Function Call Statement if(expr.kind == AstKind.FunctionCallExpression) then return Ast.FunctionCallStatement(expr.base, expr.args); @@ -408,6 +509,7 @@ function Parser:statement(scope, currentLoop) end if(expr.kind == AstKind.VariableExpression) then expr.kind = AstKind.AssignmentVariable + assertMutable(self, expr.scope, expr.id); end if(self.luaVersion == LuaVersion.LuaU) then @@ -466,6 +568,7 @@ function Parser:statement(scope, currentLoop) end if(expr.kind == AstKind.VariableExpression) then expr.kind = AstKind.AssignmentVariable + assertMutable(self, expr.scope, expr.id); end table.insert(lhs, expr); else @@ -519,11 +622,13 @@ function Parser:nameList(scope) local ident = expect(self, TokenKind.Ident); local id = scope:addDisabledVariable(ident.value, ident); + self:parseLuauTypeAnnotation(); table.insert(ids, id); while(consume(self, TokenKind.Symbol, ",")) do ident = expect(self, TokenKind.Ident); id = scope:addDisabledVariable(ident.value, ident); + self:parseLuauTypeAnnotation(); table.insert(ids, id); end @@ -717,6 +822,12 @@ end function Parser:expressionPow(scope) local lhs = self:tableOrFunctionLiteral(scope); + while self:parseLuauCast() do + if lhs.kind == AstKind.FunctionCallExpression or lhs.kind == AstKind.PassSelfFunctionCallExpression or lhs.kind == AstKind.VarargExpression then + lhs.isParenthesizedExpression = true; + end + end + if(consume(self, TokenKind.Symbol, "^")) then -- Allow unary operators on the rhs (e.g. 2 ^ #x, 2 ^ -x) while preserving right-associativity. ~ SpinnySpiwal local rhs = self:expressionUnary(scope); @@ -745,9 +856,11 @@ function Parser:expressionFunctionLiteral(parentScope) expect(self, TokenKind.Keyword, "function"); + self:parseLuauGenerics(); expect(self, TokenKind.Symbol, "("); local args = self:functionArgList(scope); expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); local body = self:block(nil, false, scope); expect(self, TokenKind.Keyword, "end"); @@ -758,6 +871,7 @@ end function Parser:functionArgList(scope) local args = {}; if(consume(self, TokenKind.Symbol, "...")) then + self:parseLuauVarargAnnotation(); table.insert(args, Ast.VarargExpression()); return args; end @@ -765,18 +879,21 @@ function Parser:functionArgList(scope) if(is(self, TokenKind.Ident)) then local ident = get(self); local name = ident.value; + self:parseLuauTypeAnnotation(); local id = scope:addVariable(name, ident); table.insert(args, Ast.VariableExpression(scope, id)); while(consume(self, TokenKind.Symbol, ",")) do if(consume(self, TokenKind.Symbol, "...")) then + self:parseLuauVarargAnnotation(); table.insert(args, Ast.VarargExpression()); return args; end - ident = get(self); + ident = expect(self, TokenKind.Ident); name = ident.value; + self:parseLuauTypeAnnotation(); id = scope:addVariable(name, ident); table.insert(args, Ast.VariableExpression(scope, id)); @@ -892,6 +1009,41 @@ function Parser:expressionIndex(scope, base) return base; end +function Parser:interpolatedString(scope, parts) + local format, expressions = {}, {}; + for _, part in ipairs(parts) do + if part.literal then + format[#format + 1] = part.literal:gsub("%%", "%%%%"); + else + local savedTokens, savedIndex, savedLength = self.tokens, self.index, self.length; + self.tokens, self.index, self.length = part.tokens, 0, #part.tokens; + local ok, expression = pcall(function() + local result = self:expression(scope); + expect(self, TokenKind.Eof); + return result; + end); + self.tokens, self.index, self.length = savedTokens, savedIndex, savedLength; + if not ok then error(expression, 0) end + if expression.kind == AstKind.FunctionCallExpression or expression.kind == AstKind.PassSelfFunctionCallExpression + or expression.kind == AstKind.VarargExpression then + expression.isParenthesizedExpression = true; + end + expressions[#expressions + 1] = expression; + format[#format + 1] = "%*"; + end + end + local template = table.concat(format); + if #expressions == 0 then + return Ast.StringExpression(template:gsub("%%%%", "%%")); + end + -- Luau's %* formatting has interpolation's conversion and evaluation order. + -- Looking up the string method avoids shadowing of the global string library. + local result = Ast.PassSelfFunctionCallExpression(Ast.StringExpression(template), "format", expressions); + result.isParenthesizedExpression = true; + result.isInterpolatedStringExpression = true; + return result; +end + function Parser:expressionLiteral(scope) -- () expression if(consume(self, TokenKind.Symbol, "(")) then @@ -903,6 +1055,11 @@ function Parser:expressionLiteral(scope) return expr; end + -- Luau interpolated string + if(is(self, TokenKind.InterpolatedString)) then + return self:interpolatedString(scope, get(self).value); + end + -- String Literal if(is(self, TokenKind.String)) then return Ast.StringExpression(get(self).value); @@ -943,7 +1100,7 @@ function Parser:expressionLiteral(scope) end -- IfElse - if(LuaVersion.LuaU) then + if(self.luaVersion == LuaVersion.LuaU) then if(consume(self, TokenKind.Keyword, "if")) then local condition = self:expression(scope); expect(self, TokenKind.Keyword, "then"); diff --git a/src/prometheus/pipeline.lua b/src/prometheus/pipeline.lua index fe21f853..499c1261 100644 --- a/src/prometheus/pipeline.lua +++ b/src/prometheus/pipeline.lua @@ -134,12 +134,14 @@ function Pipeline:setLuaVersion(luaVersion) }); self.unparser = Unparser:new({ LuaVersion = luaVersion; + PrettyPrint = self.PrettyPrint; }); + self.LuaVersion = luaVersion; self.conventions = conventions; end function Pipeline:getLuaVersion() - return self.luaVersion; + return self.LuaVersion; end function Pipeline:setNameGenerator(nameGenerator) @@ -196,7 +198,16 @@ function Pipeline:apply(code, filename) local parserStartTime = gettime(); local sourceLen = string.len(code); - local ast = self.parser:parse(code); + local parser = self.parser; + if self.LuaVersion == LuaVersion.LuaU then + for _, step in ipairs(self.steps) do + if step.Name == "Vmify" or step.Name == "Cff" then + parser = Parser:new({LuaVersion = LuaVersion.LuaU, LowerLuauIteration = true}); + break; + end + end + end + local ast = parser:parse(code); local parserTimeDiff = gettime() - parserStartTime; logger:info(string.format("Parsing Done in %.2f seconds", parserTimeDiff)); diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index f80c8543..1a6ddad7 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -19,7 +19,7 @@ local Tokenizer = {}; Tokenizer.EOF_CHAR = ""; Tokenizer.WHITESPACE_CHARS = lookupify{ - " ", "\t", "\n", "\r", + " ", "\t", "\n", "\r", "\v", "\f", } Tokenizer.ANNOTATION_CHARS = lookupify(chararray("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_")) @@ -34,6 +34,7 @@ Tokenizer.TokenKind = { Ident = "Identifier", Number = "Number", String = "String", + InterpolatedString = "InterpolatedString", } Tokenizer.EOF_TOKEN = { @@ -432,6 +433,100 @@ function Tokenizer:singleLineString() return token(self, startPos, Tokenizer.TokenKind.String, table.concat(buffer)) end +-- Scan interpolation expressions with the regular lexer, including nested +-- strings, comments, table constructors, and interpolated strings. +function Tokenizer:interpolatedString() + local startPos = self.index; + expect(self, "`"); + local parts, literal = {}, {}; + local function flush() + if #literal > 0 then + parts[#parts + 1] = {literal = table.concat(literal)}; + literal = {}; + end + end + local function escapedCharacter() + local char = get(self); + local escape = self.EscapeSequences[char]; + if type(escape) == "string" then return escape end + if self.NumericalEscapes and self.NumberCharsLookup[char] then + local digits = char; + for _ = 1, 2 do + if is(self, self.NumberCharsLookup) then digits = digits .. get(self) else break end + end + local value = tonumber(digits); + if value > 255 then logger:error(generateError(self, "Decimal escape exceeds 255")) end + return string.char(value); + end + if self.UnicodeEscapes and char == "u" then + expect(self, "{"); + local hex = ""; + while is(self, self.HexNumberCharsLookup) do hex = hex .. get(self) end + expect(self, "}"); + local value = tonumber(hex, 16); + if not value or value > 0x10ffff then + logger:error(generateError(self, "Invalid Unicode escape")); + end + return util.utf8char(value); + end + if self.HexEscapes and char == "x" then + return string.char(tonumber(expect(self, self.HexNumberCharsLookup) .. expect(self, self.HexNumberCharsLookup), 16)); + end + if self.EscapeZIgnoreNextWhitespace and char == "z" then + while is(self, Tokenizer.WHITESPACE_CHARS) do self.index = self.index + 1 end + return ""; + end + if char == "\n" or char == "\r" then + local nextChar = peek(self); + if (char == "\n" and nextChar == "\r") or (char == "\r" and nextChar == "\n") then + self.index = self.index + 1; + end + return "\n"; + end + return char; + end + + while self.index < self.length do + local char = get(self); + if char == "`" then + flush(); + return token(self, startPos, Tokenizer.TokenKind.InterpolatedString, parts); + elseif char == "\n" or char == "\r" then + logger:error(generateError(self, "Unterminated interpolated string")); + elseif char == "\\" then + literal[#literal + 1] = escapedCharacter(); + elseif char == "{" then + if is(self, "{") then + logger:error(generateError(self, "Escape a literal '{' as \\{ in an interpolated string")); + end + flush(); + local tokens, depth = {}, 1; + while depth > 0 do + local tk = self:next(); + if tk.kind == Tokenizer.TokenKind.Eof then + logger:error(generateError(self, "Expected '}' to close interpolation")); + end + if tk.kind == Tokenizer.TokenKind.Symbol then + if tk.source == "{" then depth = depth + 1 end + if tk.source == "}" then depth = depth - 1 end + end + if depth > 0 then + tokens[#tokens + 1] = tk; + else + tokens[#tokens + 1] = { + kind = Tokenizer.TokenKind.Eof, source = "", value = "", + line = tk.line, linePos = tk.linePos, + }; + end + end + parts[#parts + 1] = {tokens = tokens}; + else + literal[#literal + 1] = char; + end + end + logger:error(generateError(self, "Unterminated interpolated string")); +end + function Tokenizer:multiLineString() local startPos = self.index; if(is(self, "[")) then @@ -520,6 +615,11 @@ function Tokenizer:next() return self:ident(); end + -- Backtick strings are only available in Luau. + if(self.luaVersion == LuaVersion.LuaU and is(self, "`")) then + return self:interpolatedString(); + end + -- Singleline String Literals if(is(self, self.StringStartLookup)) then return self:singleLineString(); diff --git a/src/prometheus/unparser.lua b/src/prometheus/unparser.lua index abf06aa8..be47b590 100644 --- a/src/prometheus/unparser.lua +++ b/src/prometheus/unparser.lua @@ -699,7 +699,7 @@ function Unparser:unparseExpression(expression, tabbing) push(self:unparseExpression(expression.condition)); push(" then "); push(self:unparseExpression(expression.true_value)); - for _, elseifexp in pairs(expression.elseifs) do + for _, elseifexp in ipairs(expression.elseifs) do push(" elseif "); push(self:unparseExpression(elseifexp.condition)); push(" then "); diff --git a/src/prometheus/visitast.lua b/src/prometheus/visitast.lua index c7e963dd..9213fa21 100644 --- a/src/prometheus/visitast.lua +++ b/src/prometheus/visitast.lua @@ -239,7 +239,7 @@ function visitExpression(expression, previsit, postvisit, data) if(expression.kind == AstKind.IfElseExpression) then expression.condition = visitExpression(expression.condition, previsit, postvisit, data); expression.true_value = visitExpression(expression.true_value, previsit, postvisit, data); - for i, elif in pairs(expression.elseifs) do + for i, elif in ipairs(expression.elseifs) do elif.condition = visitExpression(elif.condition, previsit, postvisit, data); elif.value = visitExpression(elif.value, previsit, postvisit, data); end diff --git a/tests/harness/cli.py b/tests/harness/cli.py new file mode 100644 index 00000000..a71a2caa --- /dev/null +++ b/tests/harness/cli.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Exercise CLI argument handling using the real Lua interpreter.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +cli = root / 'cli.lua' + + +def run(args, cwd, success=True): + result = subprocess.run([lua, str(cli), *map(str, args)], cwd=cwd, + capture_output=True, text=True, timeout=60) + if (result.returncode == 0) != success: + raise AssertionError(f'{args!r}: exit {result.returncode}\n{result.stdout}\n{result.stderr}') + return result + + +def execute(path, cwd): + result = subprocess.run([lua, str(path)], cwd=cwd, capture_output=True, text=True, timeout=60, check=True) + assert result.stdout == '42\n', result.stdout + + +with tempfile.TemporaryDirectory(prefix='prometheus-cli-') as directory: + tmp = Path(directory) + source = tmp / 'Mixed Source.LUAU' + source.write_text('local value: number = 41\nprint(value + 1)\n') + + for index, preset in enumerate(('minify', 'WEAK', 'cFf', 'vMiFy', 'medium', 'STRONG')): + output = tmp / f'Output {index}.lua' + run(['--PRESET', preset, '--luau', '--NoCoLoRs', '--OUT', output, source], tmp) + execute(output, tmp) + + output = tmp / 'MixedCase.lua' + run(['--PrEsEt=MiNiFy', '--lUaU', '--o=' + str(output), source], tmp) + execute(output, tmp) + run(['--p', 'MINIFY', '--LuaU', '--O', output, source], tmp) + execute(output, tmp) + + config = tmp / 'My Config.LUA' + config.write_text('return {LuaVersion = "LuaU", Seed = 1, Steps = {}}\n') + run(['--CONFIG=' + str(config), '--OUT=' + str(output), source], tmp) + execute(output, tmp) + run(['--C', config, '--O', output, source], tmp) + execute(output, tmp) + + run(['--PrEsEt', 'minify', '--LUAU', source], tmp) + execute(tmp / 'Mixed Source.obfuscated.LUAU', tmp) + normal = tmp / 'Input.lua' + normal.write_text('print(42)\n') + run(['--lua51', '--p=minify', normal], tmp) + execute(tmp / 'Input.obfuscated.lua', tmp) + no_extension = tmp / 'Input' + no_extension.write_text('print(42)\n') + run(['--Lua51', '--p=minify', no_extension], tmp) + execute(tmp / 'Input.obfuscated.lua', tmp) + + (tmp / '--input.lua').write_text('print(42)\n') + run(['--preset=minify', '--OUT=--result.lua', '--', '--input.lua'], tmp) + execute(tmp / '--result.lua', tmp) + assert 'Usage:' in run(['--HELP'], tmp).stdout + assert 'Usage:' in run([normal, '--HELP'], tmp).stdout + assert run(['--VERSION'], tmp).stdout.strip() + assert run([normal, '-V'], tmp).stdout.strip() + + failures = [ + (['--luau', '--preset'], 'requires a value'), + (['--out'], 'requires a value'), + (['--CONFIG', '--luau', normal], 'requires a value'), + (['--out=', normal], 'requires a value'), + (['--out', '', normal], 'requires a value'), + (['--luau=true', normal], 'does not accept a value'), + (['--UNKNOWN', normal], 'Unknown option'), + (['--UNKNOWN=value', normal], 'Unknown option'), + (['-unknown', normal], 'Unknown option'), + (['--preset', 'NotARealPreset', normal], 'NotARealPreset'), + ([normal, source], 'Unexpected argument'), + (['--SAVEERRORS'], 'No input file was specified'), + (['--out=' + str(tmp / 'missing' / 'out.lua'), normal], 'Cannot write output file'), + ] + for args, message in failures: + result = run(args, tmp, success=False) + assert message in result.stderr, (args, result.stderr) + assert "attempt to index" not in result.stderr + + mock_update = tmp / 'mock-update.lua' + mock_update.write_text("""local cli = arg[1] +arg = {'UPDate'} +os.execute = function() return 0 end +os.exit = function(status) error('mock exit: ' .. status) end +local ok, err = pcall(dofile, cli) +assert(not ok and tostring(err):find('mock exit: 0', 1, true)) +""") + subprocess.run([lua, str(mock_update), str(cli)], cwd=tmp, capture_output=True, + text=True, timeout=60, check=True) + + malformed_type = tmp / 'MalformedType.luau' + malformed_type.write_text('local x: |& usudhd subd = 100\n') + rejected_output = tmp / 'RejectedOutput.luau' + for preset in ('minify', 'cff', 'vmify', 'weak', 'medium', 'strong'): + result = run(['--luau', '--preset', preset, '--out', rejected_output, malformed_type], tmp, success=False) + assert 'Expected a type' in result.stderr and '1:11' in result.stderr, result.stderr + assert not rejected_output.exists() + + invalid = tmp / 'Invalid.LUAU' + invalid.write_text('local value: = 1\n') + run(['--SaVeErRoRs', '--luau', '--preset=minify', invalid], tmp, success=False) + assert 'Parsing Error' in (tmp / 'Invalid.error.txt').read_text() +print('CLI case handling, paths, options, and diagnostics passed') diff --git a/tests/harness/luau-parser.lua b/tests/harness/luau-parser.lua new file mode 100644 index 00000000..7a3d0db7 --- /dev/null +++ b/tests/harness/luau-parser.lua @@ -0,0 +1,62 @@ +local Prometheus = require('src.prometheus') +local Parser = require('prometheus.parser') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error + +local function fails(source, version) + local ok = pcall(function() Parser:new({LuaVersion = version}):parse(source) end) + assert(not ok, 'expected invalid ' .. version .. ' source to fail: ' .. source) +end +fails('local x: number = 1', 'Lua51') +fails('local x = if true then 1 else 2', 'Lua51') +fails('type T =', 'LuaU') +fails('local x: |& usudhd subd = 100', 'LuaU') +fails('local x: number | string & boolean = nil', 'LuaU') +fails('type T = typeof({value = })', 'LuaU') +fails('type T = {}', 'LuaU') +fails('local function f() end', 'LuaU') +fails('local x: (number, string) = nil', 'LuaU') +fails('local x: {a: number = 1', 'LuaU') +fails('local f = function(x: ) end', 'LuaU') +fails('const x = 1', 'Lua51') +fails('local x = `text {1}`', 'Lua51') +fails('const value', 'LuaU') +fails('const x, y = 1', 'LuaU') +fails('local function f() return 1, 2 end; const x, y = (f())', 'LuaU') +fails('const x = 1; x = 2', 'LuaU') +fails('const x = 1; x += 2', 'LuaU') +fails('const x = 1; local y; y, x = 2, 3', 'LuaU') +fails('const x = 1; local function f() x = 2 end', 'LuaU') +fails('const function f() end; function f() end', 'LuaU') +fails('local x = `unclosed', 'LuaU') +fails('local x = `{}`', 'LuaU') +fails('`value {1}`', 'LuaU') +fails('local x = `{{1}}`', 'LuaU') +fails('local x = `value {1`', 'LuaU') +fails('local x = `\\u{110000}`', 'LuaU') + +local emptyOk, emptyError = pcall(function() + Parser:new({LuaVersion = 'LuaU'}):parse('local x = `empty {}`') +end) +assert(not emptyOk and tostring(emptyError):find('Position 1:', 1, true), + 'empty interpolation errors must refer to the original source') + +local ok, message = pcall(function() + Parser:new({LuaVersion = 'LuaU', LowerLuauIteration = true}):parse( + 'for i in {} do end\nlocal x: = 1') +end) +assert(not ok and tostring(message):find('Position 2:', 1, true), + 'iterator helper must not shift source error lines') + +local reusable = Parser:new({LuaVersion = 'LuaU'}) +assert(not pcall(function() reusable:parse('local x: |& Broken = 1') end)) +assert(reusable:parse('local x: number = 1').body.statements[1]) + +local pipeline = Prometheus.Pipeline:fromConfig({LuaVersion = 'Lua51', Seed = 1}) +assert(pipeline:getLuaVersion() == 'Lua51') +pipeline:setLuaVersion('LuaU') +assert(pipeline:getLuaVersion() == 'LuaU') +local result = assert(loadstring(pipeline:apply('local x: number = 3; return x')))() +assert(result == 3) +local whitespace = 'local x = `a\\z' .. string.char(11, 12) .. ' b`; return x' +assert(assert(loadstring(pipeline:apply(whitespace)))() == 'ab') +print('Luau parser and version selection checks passed') diff --git a/tests/harness/luau-syntax.lua b/tests/harness/luau-syntax.lua new file mode 100644 index 00000000..35afbefe --- /dev/null +++ b/tests/harness/luau-syntax.lua @@ -0,0 +1,21 @@ +-- Parse a corpus without executing it, then emit the type-erased AST. +local Prometheus = require('src.prometheus') +local Parser = require('prometheus.parser') +local Unparser = require('prometheus.unparser') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +Prometheus.Logger.errorCallback = function(message) error(message, 0) end +math.randomseed(1) + +for path in assert(io.open(arg[1], 'r')):lines() do + local file = assert(io.open(path, 'rb')) + local source = file:read('*a') + file:close() + local ok, message = pcall(function() + local ast = Parser:new({LuaVersion = 'LuaU'}):parse(source) + local generated = Unparser:new({LuaVersion = 'LuaU'}):unparse(ast) + local output = assert(io.open(path .. '.out.luau', 'wb')) + output:write(generated) + output:close() + end) + io.write(ok and 'accept' or 'reject', '\t', ok and '' or tostring(message):gsub('[\r\n\t]', ' '), '\n') +end diff --git a/tests/harness/luau-syntax.py b/tests/harness/luau-syntax.py new file mode 100644 index 00000000..570a3bd1 --- /dev/null +++ b/tests/harness/luau-syntax.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 +"""Compare type syntax acceptance and erasure against the official Luau compiler.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +compiler = os.environ.get('LUAU_COMPILE', 'luau-compile') +cases = [] + + +def add(label, source, expected=None): + cases.append((label, source, expected)) + + +# These are syntax checks, not type checking: unknown names are valid type references. +add('reported regression', 'local x: |& usudhd subd = 100', False) +add('unknown type', 'local x: UnknownType = nil', True) +add('adjacent statement', 'local x: UnknownType subd = 100', True) +add('grouped combinations', 'type T = (Foo | Bar) & Baz', True) +add('ungrouped combinations', 'type T = Foo | Bar & Baz', False) + +forms = [ + 'number', 'nil', 'true', 'false', '"tag"', '[=[tag]=]', 'Foo', 'Pkg.Value', 'Pkg.Sub.Value', + '| Foo', '& Foo', '|& Foo', '&| Foo', '|| Foo', '&& Foo', '|', '&', '?', + 'Foo | Bar', 'Foo & Bar', 'Foo | Bar & Baz', 'Foo & Bar | Baz', + 'Foo | & Bar', 'Foo & | Bar', 'Foo?', 'Foo??', 'Foo? & Bar', 'Foo & Bar?', + '(Foo | Bar) & Baz', '(Foo & Bar) | Baz', 'Foo & (Bar?)', '(Foo & Bar)?', + '()', '(Foo)', '((Foo))', '(Foo, Bar)', '(name: Foo)', '(Foo,)', '(Foo)?', '(Foo, Bar)?', + '...', '...Foo', 'Foo...', 'Pkg.Value...', 'Foo...?','(Foo...)', '(...Foo)', '(Foo, ...Bar)', + '() -> ()', '(Foo) -> Bar', '(name: Foo, other: Bar) -> (Foo, Bar)', + '(Foo, ...Bar) -> (Foo, ...Bar)', '(Foo..., Bar) -> Baz', '(...Foo, Bar) -> Baz', + '(Foo...) -> Bar', '(Foo,) -> Bar', '() ->', '() : Foo', '() -> () -> Foo', + '(T) -> T', '(T, P...) -> (T, P...)', '(T) -> T', + '(T...) -> U', '(T) -> T', '<>(Foo) -> Bar', + 'Container<>', 'Container', 'Container', 'Container', + 'Container<,Foo>', 'Container', 'Container<...Foo, Bar>', + 'Container<(Foo, Bar)>', 'Container<()>', 'Container<(Foo)?>', 'Container<(Foo, Bar)?>', + 'Container<(Foo) | Bar>', 'Container>>', 'Container', + '{}', '{Foo}', '{Foo,}', '{Foo, Bar}', '{{x: Foo}}', '{x: Foo}', + '{x: Foo, y: Bar;}', '{x: Foo, Bar}', '{read x: Foo, write y: Bar}', + '{read: Foo, write: Bar}', '{read Foo}', '{read}', '{readonly x: Foo}', + '{[Foo]: Bar}', '{[Foo]: Bar, [Baz]: Qux}', '{["key"]: Foo, [Bar]: Baz}', + '{["bad\\000"]: Foo}', '{...Foo}', '{x: () -> (Foo, Bar)}', + 'typeof(value)', 'typeof({value = 1})', 'typeof((function() return 1 end)())', + 'typeof()', 'typeof(+)', 'typeof(value other)', 'typeof(1, 2)', 'typeof({x = })', + 'typeof', 'typeof.Prefix', 'typeof(function(x: |& Foo) return x end)', + '`not a singleton`', '123', 'function', +] +contexts = [ + ('local annotation', 'local x: %s = nil'), + ('function parameter', 'local function f(x: %s) end'), + ('vararg annotation', 'local function f(...: %s) end'), + ('return annotation', 'local function f(): %s return nil end'), + ('type alias', 'type Alias = %s'), + ('type assertion', 'local x = nil :: %s'), + ('type argument', 'type Alias = Container<%s>'), +] +for form in forms: + for context, template in contexts: + add(f'{context}: {form}', template % form) + +# Generic declarations differ from applications and allow defaults only in aliases. +parameters = [ + 'T', '', 'T,', ',T', 'number?', 'Pkg.T', 'T | U', 'T & U', + 'T, U', 'T...', 'T, P...', 'P..., T', 'P..., Q...', 'T...,', + 'T = number', 'T = number, U', 'T = number, U = string', + 'T, U = number', 'T = number, P...', 'T = number, P... = ()', + 'P... = ()', 'P... = (Foo, Bar)', 'P... = ...Foo', 'P... = Q...', + 'P... = Foo', 'P... = (Foo)?', 'P... = (Foo, ...Bar)', + 'P... = (), Q...', 'P... = (), Q... = ()', 'T = (Foo, Bar)', +] +for params in parameters: + add(f'alias generics: {params}', f'type Alias<{params}> = number') + add(f'function generics: {params}', f'local function f<{params}>() end') + add(f'anonymous generics: {params}', f'local f = function<{params}>() end') + add(f'function type generics: {params}', f'type Alias = <{params}>() -> ()') + +# Mutate complete types in multiple contexts so malformed input cannot simply be erased. +for op in ('|', '&'): + for following in ('Foo', '?', '| Bar', '& Bar', ')', '...', ''): + add(f'operator sequence: {op} {following}', f'local x: {op} {following} = nil') + for suffix in ('', '|', '&', 'Foo', '()', ', Bar', '; Bar'): + add(f'unfinished chain: {op} {suffix}', f'type Alias = Foo {op} {suffix}') + +for expr in ('1 +', '{x = }', 'value other', 'function() local x: |& T = 1 end', 'if true then 1'): + add(f'typeof malformed expression: {expr}', f'type Alias = typeof({expr})', False) +add('typeof does not execute', 'type Alias = typeof(error("not executed"))', True) +add('valid type function signature', 'type function f(x: T, ...: P...): T return x end', True) +add('invalid type function generics', 'type function f() end', False) + + + +# Token-level edits of valid compound types exercise malformed boundaries and nesting. +seeds = [ + ['Foo', '|', 'Bar'], + ['(', 'Foo', '&', 'Bar', ')', '?'], + ['{', 'field', ':', 'Foo', ',', '[', 'Bar', ']', ':', 'Baz', '}'], + ['(', 'Foo', ',', '...', 'Bar', ')', '->', '(', 'Baz', ',', 'Qux', '...', ')'], + ['Container', '<', 'Foo', ',', '(', 'Bar', ',', 'Baz', ')', '>'], + ['<', 'T', ',', 'P', '...', '>', '(', 'T', ',', 'P', '...', ')', '->', '(', 'T', ',', 'P', '...', ')'], + ['typeof', '(', '{', 'field', '=', '1', '}', ')'], +] +mutations = set() +for seed in seeds: + for index in range(len(seed)): + mutations.add(' '.join(seed[:index] + seed[index + 1:])) + for index in range(len(seed) + 1): + for inserted in ('|', '&', '?', ',', ':', '...', 'Foo'): + mutations.add(' '.join(seed[:index] + [inserted] + seed[index:])) +for index, form in enumerate(sorted(mutations)): + add(f'mutation local {index}', 'local x: ' + form + ' = nil') + add(f'mutation alias {index}', 'type Alias = ' + form) + + +def official(path): + result = subprocess.run([compiler, '--binary', str(path)], cwd=root, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True, timeout=10) + assert result.returncode in (0, 1), (path, result.returncode, result.stderr) + return result.returncode == 0, result.stderr + + +with tempfile.TemporaryDirectory(prefix='prometheus-type-syntax-') as directory: + temporary = Path(directory) + paths = [] + for index, (_, source, _) in enumerate(cases): + path = temporary / f'{index:04d}.luau' + path.write_text(source) + paths.append(path) + manifest = temporary / 'manifest.txt' + manifest.write_text(''.join(str(path) + '\n' for path in paths)) + run = subprocess.run([lua, 'tests/harness/luau-syntax.lua', str(manifest)], cwd=root, + capture_output=True, text=True, timeout=60, check=True) + parsed = run.stdout.splitlines() + assert len(parsed) == len(cases), (len(parsed), len(cases), run.stdout[:500], run.stderr) + differences = [] + accepted = rejected = 0 + for (label, source, expected), path, line in zip(cases, paths, parsed): + native, error = official(path) + assert expected is None or native == expected, (label, source, error) + ours = line.split('\t', 1)[0] == 'accept' + accepted += native + rejected += not native + if native != ours: + differences.append(f'{label}\n {source}\n Luau: {native}; Prometheus: {line}\n {error.strip()}') + elif ours: + output_ok, output_error = official(Path(str(path) + '.out.luau')) + if not output_ok: + differences.append(f'{label}: invalid type-erased output\n {output_error}') + assert not differences, f'{len(differences)} syntax mismatches:\n' + '\n'.join(differences[:25]) + print(f'{len(cases)} syntax cases match Luau: {accepted} accepted, {rejected} rejected; emitted code also compiled') diff --git a/tests/harness/luau.py b/tests/harness/luau.py new file mode 100644 index 00000000..54183d15 --- /dev/null +++ b/tests/harness/luau.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Compare Luau fixtures with the output of each obfuscation pipeline.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +luau = os.environ.get('LUAU', 'luau') +names = ('Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong') + + +def run(args): + return subprocess.run(args, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, timeout=60, check=True).stdout + + +count = 0 +with tempfile.TemporaryDirectory(prefix='prometheus-luau-') as tmp: + output = str(Path(tmp) / 'output.luau') + for fixture in sorted((root / 'tests/luau').glob('*.luau')): + expected = run([luau, str(fixture)]) + for index, name in enumerate(names, 1): + for seed in (1, 17): + try: + run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', + str(index), str(seed), str(fixture), output]) + actual = run([luau, output]) + except subprocess.CalledProcessError as exc: + raise AssertionError(f'{fixture.name}, {name}, seed {seed}:\n{exc.stderr}\n{exc.stdout}') from exc + assert actual == expected, (f'{fixture.name}, {name}, seed {seed}: ' + f'expected {expected!r}, got {actual!r}') + count += 1 + print(f'{fixture.name}: all presets passed', flush=True) + for invalid in sorted((root / 'tests/luau/invalid').glob('*.luau')): + baseline = subprocess.run([luau, str(invalid)], cwd=root, capture_output=True, text=True, timeout=60) + transformed = subprocess.run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', + '1', '1', str(invalid), output], cwd=root, + capture_output=True, text=True, timeout=60) + assert baseline.returncode != 0 and transformed.returncode != 0, invalid.name + print('Invalid type, const, and interpolation examples were rejected', flush=True) + fixture = root / 'tests/luau/unsupported/protected-metatable.luau' + assert run([luau, str(fixture)]) == '1\n' + for index in (2, 3): + run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', + str(index), '1', str(fixture), output]) + result = subprocess.run([luau, output], cwd=root, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, text=True, timeout=60) + assert result.returncode != 0 and 'protected metatable' in result.stderr +print(f'{count} Luau transformations matched their baseline') +print('Protected metatables are rejected by Cff and Vmify') diff --git a/tests/luau/conditionals.luau b/tests/luau/conditionals.luau new file mode 100644 index 00000000..6f1de218 --- /dev/null +++ b/tests/luau/conditionals.luau @@ -0,0 +1,18 @@ +local log = '' +local function condition(label, result) + log ..= label + return result +end +local function value(label, result) + log ..= label + return result +end +local first = if condition('a', false) then value('A', 'bad') + elseif condition('b', false) then value('B', 'bad') + elseif condition('c', true) then value('C', false) + else value('D', true) +local second = if condition('d', false) then 1 elseif condition('e', true) then nil else 3 +local third = if true then (if false then 1 elseif true then 2 else 3) else 4 +local function pair() return 5, 6 end +local a, b = if true then pair() elseif false then pair() else pair() +print(log, first, second, third, a, b) diff --git a/tests/luau/const.luau b/tests/luau/const.luau new file mode 100644 index 00000000..950001ae --- /dev/null +++ b/tests/luau/const.luau @@ -0,0 +1,39 @@ +const amount: number, label = 3, 'count' +const data: {count: number} = {count = amount} +data.count += 2 +const function factorial(n: number): number + if n == 0 then return 1 end + return n * factorial(n - 1) +end +const function closure() + return function() return amount, label, data.count end +end +local get = closure() +local a, b, c = get() +do + local amount = 1 + amount = 2 + assert(amount == 2) +end +local previous = 4 +const previous = previous + 1 +const other, missing = 8, nil +local function produce() return 9 end +const expanded, absent = produce() +local const = function(x) return x + 1 end +local value = const(4) +local tableWithConst = {const = value} +tableWithConst.const += 1 +local function asParameter(const) + const += 2 + return const +end +local total = 0 +for const = 1, 3 do total += const end +for const, v in ipairs({4, 5}) do total += const + v end +do + const const = function(x) return x * 2 end + total += const(3) +end +print(a, b, c, factorial(5), previous, other, missing) +print(tableWithConst.const, asParameter(3), const(1), expanded, absent, total) diff --git a/tests/luau/interpolation.luau b/tests/luau/interpolation.luau new file mode 100644 index 00000000..9e1fadf4 --- /dev/null +++ b/tests/luau/interpolation.luau @@ -0,0 +1,55 @@ +local events = '' +local object = setmetatable({}, { + __tostring = function() + events ..= 't' + return 'object' + end, +}) +local function first() + events ..= 'a' + return object, 'ignored' +end +local function second() + events ..= 'b' + return nil, 'ignored' +end +local function nothing() end +local function fromVarargs(...) + return `vararg={...}` +end +local string = {format = function() return 'wrong format' end} +local tostring = function() return 'wrong tostring' end +local combined = `[{first()}] [{second()}] [{nothing()}]` +local nested = `outer {`inner {({x = {9}}).x[1]}`} and {'}'} and {if true then 'yes' elseif false then 'bad' else 'no'}` +local multiline = `multi {( + function() return {answer = 42} end +)().answer}` +local escaped = `literal \{x} \` \\ \x41 \065 \u{42} 100%` +local withoutExpressions = `plain %%` +local formatPercent = `%% and %s: {42}%` +local upper = (`hello {1}`):upper() +local nul = `A\000{4}B` +local surrogate = `\u{D800}` +local empty = `` +local trimmed = `trim\z + med` +local newline = `line\ +end` +local env = getfenv() +env['prometheus_luau_iterator_1'] = 'global value' +local collision = `{prometheus_luau_iterator_1}` +env['prometheus_luau_iterator_1'] = nil +local embeddedLoop = `{(function() + local total = 0 + for _, value in {2, 3} do total += value end + return total +end)()}` +local savedString, savedTostring = env.string, env.tostring +env.string = {format = function() return 'changed global' end} +env.tostring = function() return 'changed tostring' end +local hiddenGlobals = `{true}|{nil}|{5.5}` +env.string, env.tostring = savedString, savedTostring +print(combined, events) +print(nested, multiline) +print(escaped, withoutExpressions, formatPercent, #nul, nul:byte(2), empty == '', trimmed, #newline) +print(hiddenGlobals, fromVarargs(1, 2), collision, embeddedLoop, upper, #surrogate, surrogate:byte(1, 3)) diff --git a/tests/luau/invalid/const-function-reassign.luau b/tests/luau/invalid/const-function-reassign.luau new file mode 100644 index 00000000..ae7d92fb --- /dev/null +++ b/tests/luau/invalid/const-function-reassign.luau @@ -0,0 +1,2 @@ +const function f() return 1 end +function f() return 2 end diff --git a/tests/luau/invalid/const-missing-value.luau b/tests/luau/invalid/const-missing-value.luau new file mode 100644 index 00000000..47c5e596 --- /dev/null +++ b/tests/luau/invalid/const-missing-value.luau @@ -0,0 +1 @@ +const first, second = 1 diff --git a/tests/luau/invalid/const-reassign.luau b/tests/luau/invalid/const-reassign.luau new file mode 100644 index 00000000..e41d1264 --- /dev/null +++ b/tests/luau/invalid/const-reassign.luau @@ -0,0 +1,4 @@ +const x = 1 +local function mutate() + x = 2 +end diff --git a/tests/luau/invalid/interpolation-double-brace.luau b/tests/luau/invalid/interpolation-double-brace.luau new file mode 100644 index 00000000..3b415efb --- /dev/null +++ b/tests/luau/invalid/interpolation-double-brace.luau @@ -0,0 +1 @@ +local value = `before {{1}} after` diff --git a/tests/luau/invalid/interpolation-empty.luau b/tests/luau/invalid/interpolation-empty.luau new file mode 100644 index 00000000..82be4cf1 --- /dev/null +++ b/tests/luau/invalid/interpolation-empty.luau @@ -0,0 +1 @@ +local value = `before {} after` diff --git a/tests/luau/invalid/interpolation-unclosed.luau b/tests/luau/invalid/interpolation-unclosed.luau new file mode 100644 index 00000000..f1f4c7c0 --- /dev/null +++ b/tests/luau/invalid/interpolation-unclosed.luau @@ -0,0 +1 @@ +local value = `before {1` diff --git a/tests/luau/invalid/type-operator-chain.luau b/tests/luau/invalid/type-operator-chain.luau new file mode 100644 index 00000000..b7dbd200 --- /dev/null +++ b/tests/luau/invalid/type-operator-chain.luau @@ -0,0 +1 @@ +local x: |& usudhd subd = 100 diff --git a/tests/luau/invalid/type-pack-context.luau b/tests/luau/invalid/type-pack-context.luau new file mode 100644 index 00000000..6cdc855b --- /dev/null +++ b/tests/luau/invalid/type-pack-context.luau @@ -0,0 +1 @@ +local value: (number, string) = nil diff --git a/tests/luau/invalid/typeof-expression.luau b/tests/luau/invalid/typeof-expression.luau new file mode 100644 index 00000000..5b2544dd --- /dev/null +++ b/tests/luau/invalid/typeof-expression.luau @@ -0,0 +1 @@ +type Invalid = typeof({value = }) diff --git a/tests/luau/iteration.luau b/tests/luau/iteration.luau new file mode 100644 index 00000000..6a12f44a --- /dev/null +++ b/tests/luau/iteration.luau @@ -0,0 +1,48 @@ +local function prometheus_luau_iterator_1() return 2 end +local sum = prometheus_luau_iterator_1() - 2 +local source = {3, 4, 5} +local evaluated = 0 +local function tableOnce() + evaluated += 1 + return source +end +for i, value in tableOnce() do + if i == 2 then continue end + sum += i * value +end +local calls = 0 +local iterable = setmetatable({}, { + __iter = function(self) + calls += 1 + return function(state, control) + assert(state == 91) + if control == nil then return false, 11 end + if control == false then return 2, 21 end + end, 91, nil + end, +}) +for key, value in iterable do + sum += value + if key == false then sum += 1 end +end +local callable = setmetatable({}, { + __call = function(self, state, key) + if key == nil then return 1, 9 end + end, +}) +for _, value in callable do sum += value end +local function iteratorFactory() + return next, {10, 20}, nil +end +for key, value in iteratorFactory() do sum += key + value end +local protected = setmetatable({17}, {__metatable = 'locked'}) +for _, value in pairs(protected) do sum += value end +local frozen = table.freeze({1, 2}) +for _, value in frozen do sum += value end +local captured = {} +for key, value in {8, 9} do + captured[#captured + 1] = function() return key, value end +end +local k1, v1 = captured[1]() +local k2, v2 = captured[2]() +print(sum, calls, evaluated, k1, v1, k2, v2) diff --git a/tests/luau/test-config.lua b/tests/luau/test-config.lua new file mode 100644 index 00000000..97856f8a --- /dev/null +++ b/tests/luau/test-config.lua @@ -0,0 +1,9 @@ +local presets = require('src.presets') +local configs = {} +for _, name in ipairs({'Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong'}) do + local config = presets[name] + config.Name = name + config.LuaVersion = 'LuaU' + configs[#configs + 1] = config +end +return configs diff --git a/tests/luau/types.luau b/tests/luau/types.luau new file mode 100644 index 00000000..be81f011 --- /dev/null +++ b/tests/luau/types.luau @@ -0,0 +1,37 @@ +-- Type syntax is erased while runtime expressions retain their value counts. +type Name = string +export type Box = {value: T, tag: U?, [number]: boolean} +type Optional = T | nil +type Callback = (first: number, T...) -> (string, ...number) +type TableIntersection = {read value: number} & {write name: string} +type function getNumber(ty) + return types.number +end +type FunctionResult = getNumber +local evaluated = 0 +local function observe() + evaluated += 1 + return {value = 1} +end +type FromTypeof = typeof(observe()) + +local type = function(value) return value end +local export = 3 +local box: Box = {value = 7, tag = 'ok'} +local function identity(value: T, ...: string): T + return value +end +local anonymous = function(value: T): T return value end +local object = {} +function object:method(value: T): T return value end +local function values(): (number, number) return 11, 22 end +local first, second = values() :: number +local function castVarargs(...: number): number + return ... :: number +end +local fromTypeFunction: FunctionResult = 0 +local total = fromTypeFunction +for i: number = 1, 3 do total += i end +for _, value: number in ipairs({1, 2}) do total += value end +local value = if box.value == 0 then 'zero' elseif box.value == 7 then 'seven' elseif box.value == 8 then 'eight' else 'other' +print(evaluated, export, type(value), anonymous(identity(object:method(box.value))), total, first, second, castVarargs(4, 5)) diff --git a/tests/luau/types/a.luau b/tests/luau/types/a.luau new file mode 100644 index 00000000..e57a90f8 --- /dev/null +++ b/tests/luau/types/a.luau @@ -0,0 +1 @@ +local x: number = 100 \ No newline at end of file diff --git a/tests/luau/types/a.obfuscated.luau b/tests/luau/types/a.obfuscated.luau new file mode 100644 index 00000000..36586aac --- /dev/null +++ b/tests/luau/types/a.obfuscated.luau @@ -0,0 +1 @@ +local U=100 \ No newline at end of file diff --git a/tests/luau/unsupported/protected-metatable.luau b/tests/luau/unsupported/protected-metatable.luau new file mode 100644 index 00000000..b2b58f92 --- /dev/null +++ b/tests/luau/unsupported/protected-metatable.luau @@ -0,0 +1,5 @@ +local t = setmetatable({1}, { + __metatable = 'locked', + __iter = function(self) return next, self end, +}) +for _, value in t do print(value) end From 8dfc7cedf24460c9f91518566e55e9bb7c45020b Mon Sep 17 00:00:00 2001 From: tlredz Date: Wed, 28 Oct 2026 21:40:17 -0300 Subject: [PATCH 2/3] fixing things --- .gitignore | 24 +++- doc/reference/limitations.md | 15 +- prompt.txt | 17 +++ src/prometheus/ast.lua | 24 ++++ src/prometheus/cff/compiler.lua | 4 +- src/prometheus/cff/expressions.lua | 1 + src/prometheus/cff/expressions/binary.lua | 10 +- src/prometheus/cff/native.lua | 4 +- src/prometheus/cff/statements.lua | 1 + src/prometheus/cff/statements/compound.lua | 117 ++++++++++------ src/prometheus/enums.lua | 7 +- src/prometheus/luau_types.lua | 4 +- src/prometheus/parser.lua | 110 ++++++++++----- src/prometheus/steps/Cff.lua | 9 +- src/prometheus/tokenizer.lua | 58 +++++++- src/prometheus/unparser.lua | 7 +- src/prometheus/visitast.lua | 2 + src/prometheus/vm/emit.lua | 16 ++- src/prometheus/vm/expressions.lua | 8 +- src/prometheus/vm/flatten.lua | 9 +- src/prometheus/vm/opcodes.lua | 2 +- src/prometheus/vm/optimize.lua | 2 +- src/prometheus/vm/runtime.lua | 6 +- src/prometheus/vm/runtime_dispatch.lua | 12 +- src/prometheus/vm/statements.lua | 32 ++++- tests/harness/luau-parser.lua | 19 +++ tests/harness/luau-syntax.py | 115 +++++++++++++++- tests/harness/luau.py | 2 +- tests/luau/compound-assignment.luau | 47 +++++++ tests/luau/floor-division.luau | 128 ++++++++++++++++++ .../luau/invalid/numeral-missing-digits.luau | 1 + .../luau/invalid/numeral-token-boundary.luau | 1 + .../invalid/semicolon-empty-statement.luau | 1 + .../luau/invalid/vararg-nested-function.luau | 5 + tests/luau/invalid/vararg-non-variadic.luau | 3 + tests/luau/keywords.luau | 43 ++++++ tests/luau/numerals.luau | 25 ++++ tests/luau/semicolons.luau | 20 +++ tests/luau/types/a.obfuscated.luau | 1 - tests/luau/varargs.luau | 33 +++++ 40 files changed, 822 insertions(+), 123 deletions(-) create mode 100644 prompt.txt create mode 100644 tests/luau/compound-assignment.luau create mode 100644 tests/luau/floor-division.luau create mode 100644 tests/luau/invalid/numeral-missing-digits.luau create mode 100644 tests/luau/invalid/numeral-token-boundary.luau create mode 100644 tests/luau/invalid/semicolon-empty-statement.luau create mode 100644 tests/luau/invalid/vararg-nested-function.luau create mode 100644 tests/luau/invalid/vararg-non-variadic.luau create mode 100644 tests/luau/keywords.luau create mode 100644 tests/luau/numerals.luau create mode 100644 tests/luau/semicolons.luau delete mode 100644 tests/luau/types/a.obfuscated.luau create mode 100644 tests/luau/varargs.luau diff --git a/.gitignore b/.gitignore index 17a8eb3b..32261ce8 100644 --- a/.gitignore +++ b/.gitignore @@ -23,9 +23,9 @@ web/playwright-report .vite # Ignore Local Test Files -test.lua -test.obfuscated.lua -config.lua +/test.lua +/test.obfuscated.lua +/config.lua # Lua test logs and reproduction artifacts /test-results/ @@ -33,3 +33,21 @@ __pycache__/ # Local reference implementation, not part of Prometheus /.reference-obfuscator/ + +# Local runtimes, uploaded archives, and generated debugging output +/.tools/ +/scratch/ +/Prometheus*.zip +/src.zip +/lune +/lune-*.zip +/luau +/luau-compile +/luau-*.zip +*.obfuscated.lua +*.obfuscated.luau +*.out.luau +*.log +*.tmp +.DS_Store +Thumbs.db diff --git a/doc/reference/limitations.md b/doc/reference/limitations.md index 3161f960..0cb1ca5d 100644 --- a/doc/reference/limitations.md +++ b/doc/reference/limitations.md @@ -39,9 +39,18 @@ from ordinary Lua code. Use an explicit iterator function, for example is rejected; a table used as `__metatable` cannot be distinguished from a real metatable and may cause different behavior. -Luau's floor division (`//` and `//=`) and the use of -`continue` as an identifier are not supported. Type function bodies must use -syntax that the parser supports even though those declarations are erased. +Floor division (`//` and `//=`) is supported in every preset and preserves +Luau's native `__idiv` metamethod. Contextual keywords such as `continue`, +`type`, `export`, and `const` remain valid identifiers outside their statement +contexts. Decimal, hexadecimal, and binary numerals support Luau's underscore +separators; malformed number tokens are rejected. + +Luau accepts one optional semicolon after a statement, including a type +declaration. Leading or repeated empty semicolons are rejected. Varargs are +valid at chunk level and inside functions that declare `...`; an inner function +does not inherit its enclosing function's varargs. These checks also apply to +erased type syntax. Type function bodies must use syntax that the parser +supports even though those declarations are erased. ## Replacing a running function's environment diff --git a/prompt.txt b/prompt.txt new file mode 100644 index 00000000..b6e50aa8 --- /dev/null +++ b/prompt.txt @@ -0,0 +1,17 @@ +contexto: `src.zip` é o meu obfuscator, `prometheus.zip` é o prometheus que eu estou modificando para dar suporte a luau. +caso comente no codigo, comente em inglês. e adicione arquivos inuteis no .gitignore + +- hybrid keywords + - no prometheus não temos a validação de se é uma "hybrid keyword" ou uma keyword normal. incluindo o "continue", e varias outras. +- floor division + - atualmente não possui suporte a floor division. o `src.zip` possui algumas coisas sobre. +- verificar vararg (não tenho certeza sobre esse problema) + - atualmente não existe nada que diga se a vararg está em um lugar valido + - caso se a vararg esteja dentro de uma função, mas a função não possui vararg, não é avisado (não da erro na ofuscação). +- semicolon + - a forma que o ";" é "pulado" em luau é completamente diferente de lua. + - em luau, é muito mais rigido sobre a declaração de Semicolon, e atualmente não possui essa verificação. +- numerais em luau (não tenho certeza sobre isso) + - o prometheus não possui suporte para numerais de luau + +pequeno aviso: nem tudo em `src.zip` está correto, pois é uma versão mais antiga. \ No newline at end of file diff --git a/src/prometheus/ast.lua b/src/prometheus/ast.lua index ef390dd7..5da82aa5 100644 --- a/src/prometheus/ast.lua +++ b/src/prometheus/ast.lua @@ -33,6 +33,7 @@ local AstKind = { CompoundSubStatement = "CompoundSubStatement"; CompoundMulStatement = "CompoundMulStatement"; CompoundDivStatement = "CompoundDivStatement"; + CompoundFloorDivStatement = "CompoundFloorDivStatement"; CompoundModStatement = "CompoundModStatement"; CompoundPowStatement = "CompoundPowStatement"; CompoundConcatStatement = "CompoundConcatStatement"; @@ -60,6 +61,7 @@ local AstKind = { SubExpression = "SubExpression"; MulExpression = "MulExpression"; DivExpression = "DivExpression"; + FloorDivExpression = "FloorDivExpression"; ModExpression = "ModExpression"; NotExpression = "NotExpression"; LenExpression = "LenExpression"; @@ -101,6 +103,7 @@ local astKindExpressionLookup = { [AstKind.SubExpression] = 8; [AstKind.MulExpression] = 7; [AstKind.DivExpression] = 7; + [AstKind.FloorDivExpression] = 7; [AstKind.ModExpression] = 7; [AstKind.NotExpression] = 5; [AstKind.LenExpression] = 5; @@ -272,6 +275,14 @@ function Ast.CompoundDivStatement(lhs, rhs) } end +function Ast.CompoundFloorDivStatement(lhs, rhs) + return { + kind = AstKind.CompoundFloorDivStatement, + lhs = lhs, + rhs = rhs, + } +end + function Ast.CompoundPowStatement(lhs, rhs) return { kind = AstKind.CompoundPowStatement, @@ -656,6 +667,19 @@ function Ast.DivExpression(lhs, rhs, simplify) } end +function Ast.FloorDivExpression(lhs, rhs, simplify) + if simplify and lhs.isConstant and rhs.isConstant + and type(lhs.value) == "number" and type(rhs.value) == "number" and rhs.value ~= 0 then + return Ast.NumberExpression(math.floor(lhs.value / rhs.value)); + end + return { + kind = AstKind.FloorDivExpression, + lhs = lhs, + rhs = rhs, + isConstant = false, + } +end + function Ast.ModExpression(lhs, rhs, simplify) if(simplify and rhs.isConstant and lhs.isConstant) then local success, val = pcall(function() return lhs.value % rhs.value end); diff --git a/src/prometheus/cff/compiler.lua b/src/prometheus/cff/compiler.lua index 305799bf..89a01279 100644 --- a/src/prometheus/cff/compiler.lua +++ b/src/prometheus/cff/compiler.lua @@ -21,8 +21,9 @@ local compileCoreModule = require("prometheus.cff.compile_core"); local Compiler = {}; -function Compiler:new() +function Compiler:new(settings) local compiler = { + luaVersion = settings and settings.LuaVersion or "Lua51"; blocks = {}; registers = {}; activeBlock = nil; @@ -49,6 +50,7 @@ function Compiler:new() AstKind.SubExpression, AstKind.MulExpression, AstKind.DivExpression, + AstKind.FloorDivExpression, AstKind.ModExpression, AstKind.PowExpression, }; diff --git a/src/prometheus/cff/expressions.lua b/src/prometheus/cff/expressions.lua index 82207854..26498590 100644 --- a/src/prometheus/cff/expressions.lua +++ b/src/prometheus/cff/expressions.lua @@ -43,6 +43,7 @@ handlers[AstKind.AddExpression] = binaryHandler; handlers[AstKind.SubExpression] = binaryHandler; handlers[AstKind.MulExpression] = binaryHandler; handlers[AstKind.DivExpression] = binaryHandler; +handlers[AstKind.FloorDivExpression] = binaryHandler; handlers[AstKind.ModExpression] = binaryHandler; handlers[AstKind.PowExpression] = binaryHandler; diff --git a/src/prometheus/cff/expressions/binary.lua b/src/prometheus/cff/expressions/binary.lua index 380d689f..25c7eca4 100644 --- a/src/prometheus/cff/expressions/binary.lua +++ b/src/prometheus/cff/expressions/binary.lua @@ -12,8 +12,14 @@ return function(self, expression, funcDepth, numReturns) for i = 1, numReturns do regs[i] = self:allocRegister(); if i == 1 then - local lhsReg = self:compileExpression(expression.lhs, funcDepth, 1)[1]; - local rhsReg = self:compileExpression(expression.rhs, funcDepth, 1)[1]; + local lhsReg, rhsReg; + local lhs = expression.lhs; + local deferRead = expression.kind == Ast.AstKind.FloorDivExpression + and lhs.kind == Ast.AstKind.VariableExpression and not lhs.scope.isGlobal + and self.scopeFunctionDepths[lhs.scope] == funcDepth; + if not deferRead then lhsReg = self:compileExpression(lhs, funcDepth, 1)[1] end + rhsReg = self:compileExpression(expression.rhs, funcDepth, 1)[1]; + if deferRead then lhsReg = self:compileExpression(lhs, funcDepth, 1)[1] end local binaryExpr = Ast[expression.kind](self:register(scope, lhsReg), self:register(scope, rhsReg)); self:addStatement(self:setRegister(scope, regs[i], binaryExpr), {regs[i]}, {lhsReg, rhsReg}, true); diff --git a/src/prometheus/cff/native.lua b/src/prometheus/cff/native.lua index cab95386..5d799822 100644 --- a/src/prometheus/cff/native.lua +++ b/src/prometheus/cff/native.lua @@ -109,13 +109,13 @@ return function(Compiler) end end) - local source = Unparser:new({LuaVersion = 'Lua51'}):unparseExpression(expression, 0) + local source = Unparser:new({LuaVersion = self.luaVersion}):unparseExpression(expression, 0) local finish = '' if expression.kind == Kind.FunctionLiteralExpression then finish = 'if ' .. names.setenv .. ' then ' .. names.setenv .. '(' .. names.result .. ',' .. names.env .. ') end;' end - local parsed = Parser:new({LuaVersion = 'Lua51', AllowInternalIdentifiers = true}):parse('return function(' + local parsed = Parser:new({LuaVersion = self.luaVersion, AllowInternalIdentifiers = true}):parse('return function(' .. names.proxy .. ',' .. names.env .. ',' .. names.values .. ') local ' .. names.result .. '=' .. source .. ';' .. finish .. 'return ' .. names.result .. ' end') local factory = parsed.body.statements[1].args[1] diff --git a/src/prometheus/cff/statements.lua b/src/prometheus/cff/statements.lua index 6323ffad..f9d509c8 100644 --- a/src/prometheus/cff/statements.lua +++ b/src/prometheus/cff/statements.lua @@ -35,6 +35,7 @@ handlers[AstKind.CompoundAddStatement] = compoundHandler; handlers[AstKind.CompoundSubStatement] = compoundHandler; handlers[AstKind.CompoundMulStatement] = compoundHandler; handlers[AstKind.CompoundDivStatement] = compoundHandler; +handlers[AstKind.CompoundFloorDivStatement] = compoundHandler; handlers[AstKind.CompoundModStatement] = compoundHandler; handlers[AstKind.CompoundPowStatement] = compoundHandler; handlers[AstKind.CompoundConcatStatement] = compoundHandler; diff --git a/src/prometheus/cff/statements/compound.lua b/src/prometheus/cff/statements/compound.lua index 059a1305..54525a10 100644 --- a/src/prometheus/cff/statements/compound.lua +++ b/src/prometheus/cff/statements/compound.lua @@ -7,56 +7,89 @@ local Ast = require("prometheus.ast"); local AstKind = Ast.AstKind; -local compoundConstructors = { - [AstKind.CompoundAddStatement] = Ast.CompoundAddStatement, - [AstKind.CompoundSubStatement] = Ast.CompoundSubStatement, - [AstKind.CompoundMulStatement] = Ast.CompoundMulStatement, - [AstKind.CompoundDivStatement] = Ast.CompoundDivStatement, - [AstKind.CompoundModStatement] = Ast.CompoundModStatement, - [AstKind.CompoundPowStatement] = Ast.CompoundPowStatement, - [AstKind.CompoundConcatStatement] = Ast.CompoundConcatStatement, +local binaryConstructors = { + [AstKind.CompoundAddStatement] = Ast.AddExpression, + [AstKind.CompoundSubStatement] = Ast.SubExpression, + [AstKind.CompoundMulStatement] = Ast.MulExpression, + [AstKind.CompoundDivStatement] = Ast.DivExpression, + [AstKind.CompoundFloorDivStatement] = Ast.FloorDivExpression, + [AstKind.CompoundModStatement] = Ast.ModExpression, + [AstKind.CompoundPowStatement] = Ast.PowExpression, + [AstKind.CompoundConcatStatement] = Ast.StrCatExpression, }; return function(self, statement, funcDepth) local scope = self.activeBlock.scope; - local compoundConstructor = compoundConstructors[statement.kind]; - if statement.lhs.kind == AstKind.AssignmentIndexing then - local indexing = statement.lhs; - local baseReg = self:compileExpression(indexing.base, funcDepth, 1)[1]; - local indexReg = self:compileExpression(indexing.index, funcDepth, 1)[1]; - local valueReg = self:compileExpression(statement.rhs, funcDepth, 1)[1]; + local target = statement.lhs; + local baseReg, indexReg, oldReg; + local function isCurrentLocal(expression) + return expression.kind == AstKind.VariableExpression and not expression.scope.isGlobal + and self.scopeFunctionDepths[expression.scope] == funcDepth; + end + local function capture(expression) + local reg = self:compileExpression(expression, funcDepth, 1)[1]; + if self:isVarRegister(reg) then + local copy = self:allocRegister(false); + self:addStatement(self:copyRegisters(scope, {copy}, {reg}), {copy}, {reg}, false); + reg = copy; + end + return reg; + end + + local indexed = target.kind == AstKind.AssignmentIndexing; + local localTarget = not indexed and not target.scope.isGlobal + and self.scopeFunctionDepths[target.scope] == funcDepth; + local deferRead = localTarget and statement.kind ~= AstKind.CompoundConcatStatement; + if indexed then + -- Luau addresses direct locals through their current registers, including + -- changes made while evaluating the key, RHS, or an operator metamethod. + if not isCurrentLocal(target.base) then baseReg = capture(target.base) end + indexReg = capture(target.index); + if not baseReg then baseReg = capture(target.base) end + oldReg = self:allocRegister(false); + self:addStatement(self:setRegister(scope, oldReg, Ast.IndexExpression(self:register(scope, baseReg), + self:register(scope, indexReg))), {oldReg}, {baseReg, indexReg}, true); + elseif not deferRead then + oldReg = capture(Ast.VariableExpression(target.scope, target.id)); + end + -- Arithmetic reads current-function locals after RHS. Concatenation, + -- globals, upvalues, and indexed values retain their earlier value. + local valueReg = self:compileExpression(statement.rhs, funcDepth, 1)[1]; + if deferRead then oldReg = self:compileExpression(Ast.VariableExpression(target.scope, target.id), funcDepth, 1)[1] end + local resultReg = self:allocRegister(false); + local result = binaryConstructors[statement.kind](self:register(scope, oldReg), self:register(scope, valueReg)); + self:addStatement(self:setRegister(scope, resultReg, result), {resultReg}, {oldReg, valueReg}, true); - self:addStatement(compoundConstructor(Ast.AssignmentIndexing(self:register(scope, baseReg), self:register(scope, indexReg)), self:register(scope, valueReg)), {}, {baseReg, indexReg, valueReg}, true); + if indexed then + if isCurrentLocal(target.base) then + self:freeRegister(baseReg, false); + baseReg = capture(target.base); + end + if isCurrentLocal(target.index) then + self:freeRegister(indexReg, false); + indexReg = capture(target.index); + end + self:addStatement(Ast.AssignmentStatement({Ast.AssignmentIndexing(self:register(scope, baseReg), + self:register(scope, indexReg))}, {self:register(scope, resultReg)}), {}, {baseReg, indexReg, resultReg}, true); self:freeRegister(baseReg, false); self:freeRegister(indexReg, false); - self:freeRegister(valueReg, false); - else - local valueReg = self:compileExpression(statement.rhs, funcDepth, 1)[1]; - local primaryExpr = statement.lhs; - if primaryExpr.scope.isGlobal then - local tmpReg = self:allocRegister(false); - self:addStatement(self:setRegister(scope, tmpReg, Ast.StringExpression(primaryExpr.scope:getVariableName(primaryExpr.id))), {tmpReg}, {}, false); - self:addStatement(compoundConstructor(Ast.AssignmentIndexing(self:env(scope), self:register(scope, tmpReg)), - self:register(scope, valueReg)), {}, {tmpReg, valueReg}, true); - self:freeRegister(tmpReg, false); - self:freeRegister(valueReg, false); + elseif target.scope.isGlobal then + self:addStatement(Ast.AssignmentStatement({Ast.AssignmentIndexing(self:env(scope), + Ast.StringExpression(target.scope:getVariableName(target.id)))}, {self:register(scope, resultReg)}), {}, {resultReg}, true); + elseif localTarget then + local reg = self:getVarRegister(target.scope, target.id, funcDepth); + if self:isUpvalue(target.scope, target.id) then + self:addStatement(self:setUpvalueMember(scope, self:register(scope, reg), self:register(scope, resultReg)), {}, {reg, resultReg}, true); else - if self.scopeFunctionDepths[primaryExpr.scope] == funcDepth then - if self:isUpvalue(primaryExpr.scope, primaryExpr.id) then - local reg = self:getVarRegister(primaryExpr.scope, primaryExpr.id, funcDepth); - self:addStatement(self:setUpvalueMember(scope, self:register(scope, reg), self:register(scope, valueReg), compoundConstructor), {}, {reg, valueReg}, true); - else - local reg = self:getVarRegister(primaryExpr.scope, primaryExpr.id, funcDepth, valueReg); - if reg ~= valueReg then - self:addStatement(self:setRegister(scope, reg, self:register(scope, valueReg), compoundConstructor), {reg}, {valueReg}, false); - end - end - else - local upvalId = self:getUpvalueId(primaryExpr.scope, primaryExpr.id); - scope:addReferenceToHigherScope(self.containerFuncScope, self.currentUpvaluesVar); - self:addStatement(self:setUpvalueMember(scope, Ast.IndexExpression(Ast.VariableExpression(self.containerFuncScope, self.currentUpvaluesVar), Ast.NumberExpression(upvalId)), self:register(scope, valueReg), compoundConstructor), {}, {valueReg}, true); - end - self:freeRegister(valueReg, false); + self:addStatement(self:setRegister(scope, reg, self:register(scope, resultReg)), {reg}, {resultReg}, false); end + else + local upvalId = self:getUpvalueId(target.scope, target.id); + scope:addReferenceToHigherScope(self.containerFuncScope, self.currentUpvaluesVar); + self:addStatement(self:setUpvalueMember(scope, Ast.IndexExpression(Ast.VariableExpression(self.containerFuncScope, + self.currentUpvaluesVar), Ast.NumberExpression(upvalId)), self:register(scope, resultReg)), {}, {resultReg}, true); end + self:freeRegister(oldReg, false); + self:freeRegister(valueReg, false); + self:freeRegister(resultReg, false); end; diff --git a/src/prometheus/enums.lua b/src/prometheus/enums.lua index 62a79122..14571168 100644 --- a/src/prometheus/enums.lua +++ b/src/prometheus/enums.lua @@ -59,18 +59,19 @@ Enums.Conventions = { }, [Enums.LuaVersion.LuaU] = { Keywords = { - "and", "break", "do", "else", "elseif", "continue", + "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in", "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while" }, + HybridKeywords = {"continue", "type", "export", "const", "typeof", "read", "write"}, SymbolChars = chararray("+-*/%^#=~<>(){}[];:,.?|&"), MaxSymbolLength = 3, Symbols = { - "+", "-", "*", "/", "%", "^", "#", + "+", "-", "*", "/", "//", "%", "^", "#", "==", "~=", "<=", ">=", "<", ">", "=", - "+=", "-=", "/=", "%=", "^=", "..=", "*=", + "+=", "-=", "/=", "//=", "%=", "^=", "..=", "*=", "(", ")", "{", "}", "[", "]", ";", ":", ",", ".", "..", "...", "::", "->", "?", "|", "&", diff --git a/src/prometheus/luau_types.lua b/src/prometheus/luau_types.lua index 3571244a..f6bb9474 100644 --- a/src/prometheus/luau_types.lua +++ b/src/prometheus/luau_types.lua @@ -289,10 +289,10 @@ return function(Parser, token, is, consume, expect, peek, get) parseGenerics(self, false) expect(self, S, '(') local scope = Scope:new(Scope:newGlobal()) - self:functionArgList(scope) + local args = self:functionArgList(scope) expect(self, S, ')') self:parseLuauReturnAnnotation() - self:block(nil, false, scope) + self:functionBody(scope, args) expect(self, K, 'end') else expect(self, I) diff --git a/src/prometheus/parser.lua b/src/prometheus/parser.lua index 09817040..1580636a 100644 --- a/src/prometheus/parser.lua +++ b/src/prometheus/parser.lua @@ -5,8 +5,6 @@ -- This Script provides a class for parsing of lua code. -- This Parser is Capable of parsing LuaU and Lua5.1 -- --- Note that when parsing LuaU "continue" is treated as a Keyword, so no variable may be named "continue" even though this would be valid in LuaU --- -- Settings Object: -- luaVersion : The LuaVersion of the Script - Currently Supported : Lua51 and LuaU -- AllowInternalIdentifiers : Skip reserved-prefix warnings for generated code (default false) @@ -174,6 +172,7 @@ function Parser:parse(code) self.tokenizer:reset(); self.tokens, self.index, self.length = {}, 0, 0; self.disableLog = false; + self.varargAllowed = true; self.luauIteratorName = nil; self.tokenizer:append(code); self.tokens = self.tokenizer:scanAll(); @@ -234,22 +233,25 @@ function Parser:block(parentScope, currentLoop, scope) scope = scope or Scope:new(parentScope); local statements = {}; - repeat - local statement, isTerminatingStatement = self:statement(scope, currentLoop); - table.insert(statements, statement); - until isTerminatingStatement or not statement - - -- Consume Eventual Semicolon after terminating return, break or continue - consume(self, TokenKind.Symbol, ";"); + while true do + -- Erased type declarations still count as statements for semicolon syntax. + local declaration = self:parseLuauDeclaration(); + local statement, isTerminatingStatement; + if not declaration then statement, isTerminatingStatement = self:statement(scope, currentLoop) end + if not declaration and not statement then break end + if statement then table.insert(statements, statement) end + consume(self, TokenKind.Symbol, ";"); + if isTerminatingStatement then break end + end return Ast.Block(statements, scope); end function Parser:statement(scope, currentLoop) - -- Skip all semicolons before next real statement - -- NOP statements are therefore ignored - while true do - if not consume(self, TokenKind.Symbol, ";") and not self:parseLuauDeclaration() then break end + -- Preserve the legacy Lua 5.1 empty-statement behavior. Luau permits only + -- one optional semicolon after an actual statement, consumed by block(). + if self.luaVersion == LuaVersion.Lua51 then + while consume(self, TokenKind.Symbol, ";") do end end -- Break Statement - only valid inside of Loops @@ -262,16 +264,6 @@ function Parser:statement(scope, currentLoop) return Ast.BreakStatement(currentLoop, scope), true; end - -- Continue Statement - only valid inside of Loops - only valid in LuaU - if(self.luaVersion == LuaVersion.LuaU and consume(self, TokenKind.Keyword, "continue")) then - if(not currentLoop) then - if self.disableLog then error() end; - logger:error(generateError(self, "the continue Statement is only valid inside of loops")); - end - -- Return true as Second value because continue must be the last Statement in a block - return Ast.ContinueStatement(currentLoop, scope), true; - end - -- do ... end Statement if(consume(self, TokenKind.Keyword, "do")) then local body = self:block(scope, currentLoop); @@ -302,7 +294,7 @@ function Parser:statement(scope, currentLoop) -- Return Statement if(consume(self, TokenKind.Keyword, "return")) then local args = {}; - if(not is(self, TokenKind.Keyword, "end") and not is(self, TokenKind.Keyword, "elseif") and not is(self, TokenKind.Keyword, "else") and not is(self, TokenKind.Symbol, ";") and not is(self, TokenKind.Eof)) then + if(not is(self, TokenKind.Keyword, "end") and not is(self, TokenKind.Keyword, "elseif") and not is(self, TokenKind.Keyword, "else") and not is(self, TokenKind.Keyword, "until") and not is(self, TokenKind.Symbol, ";") and not is(self, TokenKind.Eof)) then args = self:exprList(scope); end -- Return true as Second value because return must be the last Statement in a block @@ -361,7 +353,7 @@ function Parser:statement(scope, currentLoop) table.insert(args, 1, Ast.VariableExpression(funcScope, id)); end - local body = self:block(nil, false, funcScope); + local body = self:functionBody(funcScope, args); expect(self, TokenKind.Keyword, "end"); return Ast.FunctionDeclaration(baseScope, baseId, indices, args, body); @@ -386,7 +378,7 @@ function Parser:statement(scope, currentLoop) expect(self, TokenKind.Symbol, ")"); self:parseLuauReturnAnnotation(); - local body = self:block(nil, false, funcScope); + local body = self:functionBody(funcScope, args); expect(self, TokenKind.Keyword, "end"); return Ast.LocalFunctionDeclaration(scope, id, args, body); @@ -504,6 +496,19 @@ function Parser:statement(scope, currentLoop) -- Variable Assignment if(expr.kind == AstKind.IndexExpression or expr.kind == AstKind.VariableExpression) then + -- A call or assignment using this name takes precedence over continue. + local nextToken = peek(self); + local assignmentFollows = nextToken.kind == TokenKind.Symbol + and (nextToken.source == "=" or nextToken.source == "," or nextToken.source:match("^[+%-%*/%%%^%.]+=$")); + if self.luaVersion == LuaVersion.LuaU and expr.kind == AstKind.VariableExpression + and not expr.isParenthesizedExpression and expr.scope:getVariableName(expr.id) == "continue" + and not assignmentFollows then + if not currentLoop then + if self.disableLog then error() end + logger:error(generateError(self, "the continue Statement is only valid inside of loops")); + end + return Ast.ContinueStatement(currentLoop, scope), true; + end if(expr.kind == AstKind.IndexExpression) then expr.kind = AstKind.AssignmentIndexing end @@ -534,6 +539,11 @@ function Parser:statement(scope, currentLoop) return Ast.CompoundDivStatement(expr, rhs); end + if(consume(self, TokenKind.Symbol, "//=")) then + local rhs = self:expression(scope); + return Ast.CompoundFloorDivStatement(expr, rhs); + end + if(consume(self, TokenKind.Symbol, "%=")) then local rhs = self:expression(scope); return Ast.CompoundModStatement(expr, rhs); @@ -592,17 +602,13 @@ function Parser:statement(scope, currentLoop) end function Parser:primaryExpression(scope) - local i = self.index; - local s = self; - self.disableLog = true; - local status, val = pcall(self.expressionFunctionCall, self, scope); - self.disableLog = false; - if(status) then - return val; - else - self.index = i; + local tk = peek(self); + if tk.kind == TokenKind.Eof or (tk.kind == TokenKind.Keyword + and (tk.source == "end" or tk.source == "else" or tk.source == "elseif" or tk.source == "until")) then return nil; end + -- Invalid calls must keep their diagnostics, including vararg and type errors. + return self:expressionFunctionCall(scope); end -- List of expressions Seperated by a comma @@ -790,6 +796,12 @@ function Parser:expressionMulDivMod(scope) found = true; end + if(consume(self, TokenKind.Symbol, "//")) then + local rhs = self:expressionUnary(scope); + curr = Ast.FloorDivExpression(curr, rhs, true); + found = true; + end + if(consume(self, TokenKind.Symbol, "%")) then local rhs = self:expressionUnary(scope); curr = Ast.ModExpression(curr, rhs, true); @@ -862,12 +874,22 @@ function Parser:expressionFunctionLiteral(parentScope) expect(self, TokenKind.Symbol, ")"); self:parseLuauReturnAnnotation(); - local body = self:block(nil, false, scope); + local body = self:functionBody(scope, args); expect(self, TokenKind.Keyword, "end"); return Ast.FunctionLiteralExpression(args, body); end +-- An inner function never inherits its enclosing function's vararg parameter. +-- Signature annotations are parsed before entering the new function context. +function Parser:functionBody(scope, args) + local savedVararg = self.varargAllowed; + self.varargAllowed = #args > 0 and args[#args].kind == AstKind.VarargExpression; + local body = self:block(nil, false, scope); + self.varargAllowed = savedVararg; + return body; +end + function Parser:functionArgList(scope) local args = {}; if(consume(self, TokenKind.Symbol, "...")) then @@ -903,6 +925,15 @@ function Parser:functionArgList(scope) return args; end +local function checkCallLine(self) + if self.luaVersion == LuaVersion.LuaU and is(self, TokenKind.Symbol, "(") + and peek(self, -1).line ~= peek(self).line then + local message = generateError(self, "Ambiguous function call; use ';' to separate statements"); + if self.disableLog then error(message, 0) end + logger:error(message); + end +end + function Parser:expressionFunctionCall(scope, base) base = base or self:expressionIndex(scope); @@ -910,6 +941,7 @@ function Parser:expressionFunctionCall(scope, base) return base; end + checkCallLine(self); -- Normal Function Call local args = {}; if(is(self, TokenKind.String)) then @@ -970,6 +1002,7 @@ function Parser:expressionIndex(scope, base) if(consume(self, TokenKind.Symbol, ":")) then local passSelfFunctionName = expect(self, TokenKind.Ident).value; local args = {}; + checkCallLine(self); if(is(self, TokenKind.String)) then args = { Ast.StringExpression(get(self).value), @@ -1087,6 +1120,11 @@ function Parser:expressionLiteral(scope) -- Vararg Literal if(consume(self, TokenKind.Symbol, "...")) then + if not self.varargAllowed then + local message = generateError(self, "Cannot use '...' outside a vararg function"); + if self.disableLog then error(message, 0) end + logger:error(message); + end return Ast.VarargExpression(); end diff --git a/src/prometheus/steps/Cff.lua b/src/prometheus/steps/Cff.lua index bc197012..29219c50 100644 --- a/src/prometheus/steps/Cff.lua +++ b/src/prometheus/steps/Cff.lua @@ -18,9 +18,10 @@ Cff.SettingsDescriptor = { function Cff:init(_) end -function Cff:apply(ast) +function Cff:apply(ast, pipeline) + local settings = {LuaVersion = pipeline and pipeline:getLuaVersion() or "Lua51"}; if self.ExcludeVm and ast.vmRuntime then - local compiler = Compiler:new(); + local compiler = Compiler:new(settings); compiler.nativeExpressions = {[ast.vmRuntime] = true}; if ast.vmData then compiler.nativeExpressions[ast.vmData] = true; end local flattened = compiler:compile(ast); @@ -29,10 +30,10 @@ function Cff:apply(ast) return flattened; end if ast.vmRuntime then - local flattened = flattenVm(ast); + local flattened = flattenVm(ast, settings.LuaVersion); if flattened then return flattened; end end - local compiler = Compiler:new(); + local compiler = Compiler:new(settings); return compiler:compile(ast); end diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index 1a6ddad7..cc89c04d 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -71,13 +71,14 @@ local function generateWarning(token, message) end function Tokenizer:getPosition(i) + if i <= 0 or self.length == 0 then return 1, 1 end local column = self.columnMap[i] if not column then --// `i` is bigger than self.length, this shouldn't happen, but it did. (Theres probably some error in the tokenizer, cant find it.) column = self.columnMap[#self.columnMap] end - return column.id, column.charMap[i] + return column.id, column.charMap[i] or column.length + 1 end --// Prepare columnMap for getPosition @@ -122,6 +123,7 @@ function Tokenizer:new(settings) NumberCharsLookup = lookupify(conventions.NumberChars), Keywords = conventions.Keywords, KeywordsLookup = lookupify(conventions.Keywords), + HybridKeywordsLookup = lookupify(conventions.HybridKeywords or {}), BinaryNumberChars = conventions.BinaryNumberChars, BinaryNumberCharsLookup = lookupify(conventions.BinaryNumberChars); BinaryNums = conventions.BinaryNums, @@ -298,8 +300,59 @@ local function int(self, chars, seperators) return table.concat(buffer); end +-- Parse radix integers without depending on the host's integer width or rounding +-- at every digit. Luau saturates overflowing radix literals at UINT64_MAX. +local function radixNumber(source, base) + local high, low = 0, 0; + for i = 1, #source do + local digit = tonumber(source:sub(i, i), base); + local nextLow = low * base + digit; + high = high * base + math.floor(nextLow / 4294967296); + low = nextLow % 4294967296; + if high >= 4294967296 then return 18446744073709551616 end + end + return high * 4294967296.0 + low; +end + +function Tokenizer:luauNumber() + local startPos = self.index; + if is(self, ".") then self.index = self.index + 1 end + -- Match Luau's entire number-like token before validating its value. In + -- particular, 1foo and 1..2 must not split into otherwise valid tokens. + repeat + self.index = self.index + 1; + until not (is(self, self.NumberCharsLookup) or is(self, ".") or is(self, "_")) + if is(self, "e") or is(self, "E") then + self.index = self.index + 1; + if is(self, "+") or is(self, "-") then self.index = self.index + 1 end + end + while is(self, self.IdentCharsLookup) do self.index = self.index + 1 end + + local source = self.source:sub(startPos + 1, self.index):gsub("_", ""); + local prefix = source:sub(1, 2):lower(); + local value; + if prefix == "0x" or prefix == "0b" then + local digits = source:sub(3); + local pattern = prefix == "0x" and "^[0-9a-fA-F]+$" or "^[01]+$"; + if digits:match(pattern) then value = radixNumber(digits, prefix == "0x" and 16 or 2) end + else + local mantissa = source; + if source:find("[eE]") then + mantissa = source:match("^(.-)[eE][+-]?%d+$"); + end + if mantissa and (mantissa:match("^%d+%.?%d*$") or mantissa:match("^%.%d+$")) then + value = tonumber(source); + end + end + if value == nil then + logger:error(generateError(self, "Malformed number '" .. self.source:sub(startPos + 1, self.index) .. "'")); + end + return token(self, startPos, Tokenizer.TokenKind.Number, value); +end + -- Lex the next token as a Number function Tokenizer:number() + if self.luaVersion == LuaVersion.LuaU then return self:luauNumber() end local startPos = self.index; local source = expect(self, setmetatable({["."] = true}, {__index = self.NumberCharsLookup})); @@ -359,6 +412,9 @@ function Tokenizer:ident() end local tk = token(self, startPos, Tokenizer.TokenKind.Ident, source); + -- Contextual keywords remain identifiers until the parser sees a declaration + -- or a bare continue statement; they are legal names everywhere else. + tk.isHybridKeyword = self.HybridKeywordsLookup[source] == true; if(not self.allowInternalIdentifiers and string.sub(source, 1, string.len(config.IdentPrefix)) == config.IdentPrefix) then diff --git a/src/prometheus/unparser.lua b/src/prometheus/unparser.lua index be47b590..0c2046b0 100644 --- a/src/prometheus/unparser.lua +++ b/src/prometheus/unparser.lua @@ -5,7 +5,7 @@ -- This Script provides a class for lua code generation from an ast -- This UnParser is Capable of generating LuaU and Lua5.1 -- --- Note that a LuaU ast can only be unparsed as LuaU if it contains any continue statements +-- Luau-only nodes require Luau output unless a compiler has lowered them. -- -- Settings Object: -- luaVersion : The LuaVersion of the Script @@ -368,6 +368,7 @@ function Unparser:unparseStatement(statement, tabbing) [AstKind.CompoundSubStatement] = "-=", [AstKind.CompoundMulStatement] = "*=", [AstKind.CompoundDivStatement] = "/=", + [AstKind.CompoundFloorDivStatement] = "//=", [AstKind.CompoundModStatement] = "%=", [AstKind.CompoundPowStatement] = "^=", [AstKind.CompoundConcatStatement] = "..=", @@ -518,12 +519,16 @@ function Unparser:unparseExpression(expression, tabbing) [AstKind.SubExpression] = "-", [AstKind.MulExpression] = "*", [AstKind.DivExpression] = "/", + [AstKind.FloorDivExpression] = "//", [AstKind.ModExpression] = "%", [AstKind.PowExpression] = "^", } op = arithmeticOps[expression.kind] if op then + if op == "//" and self.luaVersion ~= LuaVersion.LuaU then + logger:error("Floor division requires Luau output"); + end k = expression.kind local lhs = self:unparseExpression(expression.lhs, tabbing); if(Ast.astKindExpressionToNumber(expression.lhs.kind) >= Ast.astKindExpressionToNumber(k)) then diff --git a/src/prometheus/visitast.lua b/src/prometheus/visitast.lua index 9213fa21..b84efb56 100644 --- a/src/prometheus/visitast.lua +++ b/src/prometheus/visitast.lua @@ -45,6 +45,7 @@ local compundStats = lookupify{ AstKind.CompoundSubStatement, AstKind.CompoundMulStatement, AstKind.CompoundDivStatement, + AstKind.CompoundFloorDivStatement, AstKind.CompoundModStatement, AstKind.CompoundPowStatement, AstKind.CompoundConcatStatement, @@ -183,6 +184,7 @@ local binaryExpressions = lookupify{ AstKind.SubExpression, AstKind.MulExpression, AstKind.DivExpression, + AstKind.FloorDivExpression, AstKind.ModExpression, AstKind.PowExpression, } diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua index 619fcc40..f1310659 100644 --- a/src/prometheus/vm/emit.lua +++ b/src/prometheus/vm/emit.lua @@ -5,8 +5,19 @@ local Unparser = require('prometheus.unparser') local Enums = require('prometheus.enums') local Runtime = require('prometheus.vm.runtime') local Layout = require('prometheus.vm.layout') +local opcodes = require('prometheus.vm.opcodes') local Emit = {} +local function usesFloorDivision(proto) + for _, instruction in ipairs(proto.code) do + if instruction[1] == opcodes.IDIV then return true end + end + for _, child in ipairs(proto.children) do + if usesFloorDivision(child) then return true end + end + return false +end + function Emit.source(proto, layout) layout = layout or Layout.new() local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) @@ -36,13 +47,14 @@ function Emit.source(proto, layout) return '{' .. table.concat(parts, ',') .. '}' end local data = '{' .. prototype(proto) .. '}' - local source, count = Runtime.build(layout):gsub('__PROTOTYPES__', function() return data end) + local source, count = Runtime.build(layout, usesFloorDivision(proto)):gsub('__PROTOTYPES__', function() return data end) assert(count == 1, 'Runtime template must contain one prototype marker') return source end function Emit.ast(proto) - local ast = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(Emit.source(proto)) + local version = usesFloorDivision(proto) and Enums.LuaVersion.LuaU or Enums.LuaVersion.Lua51 + local ast = Parser:new({LuaVersion = version}):parse(Emit.source(proto)) ast.vmRuntime = ast.body.statements[1].args[1].base ast.vmData = ast.body.statements[1].args[1].args[1] return ast diff --git a/src/prometheus/vm/expressions.lua b/src/prometheus/vm/expressions.lua index efba259a..b59489fa 100644 --- a/src/prometheus/vm/expressions.lua +++ b/src/prometheus/vm/expressions.lua @@ -5,6 +5,7 @@ local compileTable = require("prometheus.vm.expressions_tables") local binary = { [Kind.AddExpression] = "ADD", [Kind.SubExpression] = "SUB", [Kind.MulExpression] = "MUL", [Kind.DivExpression] = "DIV", + [Kind.FloorDivExpression] = "IDIV", [Kind.ModExpression] = "MOD", [Kind.PowExpression] = "POW", [Kind.StrCatExpression] = "CONCAT", [Kind.LessThanExpression] = "LT", [Kind.LessThanOrEqualsExpression] = "LE", [Kind.GreaterThanExpression] = "GT", @@ -93,8 +94,13 @@ local function install(Compiler) local result = self:alloc(); self:emit("GETTABLE", result, base, key) self:free(key); self:free(base); return result elseif binary[kind] then - local left = self:expression(node.lhs, 1)[1] + local left + local lhs = node.lhs + local deferRead = kind == Kind.FloorDivExpression and lhs.kind == Kind.VariableExpression + and not lhs.scope.isGlobal and self.analysis.owners[lhs.scope] == self.owner + if not deferRead then left = self:expression(lhs, 1)[1] end local right = self:expression(node.rhs, 1)[1] + if deferRead then left = self:expression(lhs, 1)[1] end local result = self:alloc(); self:emit(binary[kind], result, left, right) self:free(right); self:free(left); return result elseif unary[kind] then diff --git a/src/prometheus/vm/flatten.lua b/src/prometheus/vm/flatten.lua index d559b106..57026fe9 100644 --- a/src/prometheus/vm/flatten.lua +++ b/src/prometheus/vm/flatten.lua @@ -38,12 +38,13 @@ local function externalVariables(ast, runtime) return variables, writes end -return function(ast) +return function(ast, luaVersion) + luaVersion = luaVersion or Enums.LuaVersion.Lua51 local runtime = ast.vmRuntime local variables, writes = externalVariables(ast, runtime) if writes then return nil end - local parser = Parser:new({LuaVersion = Enums.LuaVersion.Lua51, AllowInternalIdentifiers = true}) - local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local parser = Parser:new({LuaVersion = luaVersion, AllowInternalIdentifiers = true}) + local unparser = Unparser:new({LuaVersion = luaVersion}) local runtimeSource = unparser:unparseExpression(runtime, 0) local prefix = '' local arguments = '...' @@ -52,7 +53,7 @@ return function(ast) arguments = 'select(' .. (#variables + 1) .. ',...)' end local standalone = parser:parse(prefix .. 'return (' .. runtimeSource .. ')(' .. arguments .. ')') - local flattened = unparser:unparse(CffCompiler:new():compile(standalone)) + local flattened = unparser:unparse(CffCompiler:new({LuaVersion = luaVersion}):compile(standalone)) local forwarded = #variables > 0 and table.concat(variables, ',') .. ',...' or '...' local replacementSource = 'return function(...) return (function(...) ' .. flattened .. ' end)(' .. forwarded .. ') end' diff --git a/src/prometheus/vm/opcodes.lua b/src/prometheus/vm/opcodes.lua index 33d37a3c..2487b645 100644 --- a/src/prometheus/vm/opcodes.lua +++ b/src/prometheus/vm/opcodes.lua @@ -4,7 +4,7 @@ local names = { 'GETUP', 'SETUP', 'GLOBAL', 'SETGLOBAL', 'GETTABLE', 'SETTABLE', 'TABLE', 'EXPAND', 'ARGS', 'TAKE', 'VARARG', 'CALL', 'RETURN', 'TAILCALL', 'CLOSURE', 'JMP', 'JFALSE', 'JTRUE', - 'ADD', 'SUB', 'MUL', 'DIV', 'MOD', 'POW', 'CONCAT', + 'ADD', 'SUB', 'MUL', 'DIV', 'IDIV', 'MOD', 'POW', 'CONCAT', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'NOT', 'NEG', 'LEN', 'FORPREP', 'FORCHECK', 'CLEARS', 'CALL0', 'CALL1', 'CALL2', 'CALL3', diff --git a/src/prometheus/vm/optimize.lua b/src/prometheus/vm/optimize.lua index e1772bdb..b4c36fc8 100644 --- a/src/prometheus/vm/optimize.lua +++ b/src/prometheus/vm/optimize.lua @@ -12,7 +12,7 @@ end local function isUnary(op) return op == 'NOT' or op == 'NEG' or op == 'LEN' end local function isBinary(op) return op == 'ADD' or op == 'SUB' or op == 'MUL' or op == 'DIV' - or op == 'MOD' or op == 'POW' or op == 'CONCAT' or op == 'LT' + or op == 'IDIV' or op == 'MOD' or op == 'POW' or op == 'CONCAT' or op == 'LT' or op == 'LE' or op == 'GT' or op == 'GE' or op == 'EQ' or op == 'NE' end local function producer(ins) diff --git a/src/prometheus/vm/runtime.lua b/src/prometheus/vm/runtime.lua index 5b989bc3..f031ebe0 100644 --- a/src/prometheus/vm/runtime.lua +++ b/src/prometheus/vm/runtime.lua @@ -1,4 +1,4 @@ --- Builds the Lua 5.1 runtime used by the bytecode emitter. The emitter replaces +-- Builds the interpreter runtime used by the bytecode emitter. The emitter replaces -- __PROTOTYPES__ with its native Lua table literal before parsing the source. local dispatch = require("prometheus.vm.runtime_dispatch") local Layout = require("prometheus.vm.layout") @@ -99,7 +99,7 @@ return make(prototypes[1], {}, entryEnv)(...) end)(__PROTOTYPES__, ...) ]=] -function Runtime.build(layout) +function Runtime.build(layout, floorDivision) layout = layout or Layout.new() local operands = layout.operands local fetch = "a, b, c, d = I[" .. tostring(operands[1]) .. "], I[" @@ -108,7 +108,7 @@ function Runtime.build(layout) local source = template:gsub("__FETCH_OPERANDS__", function() return fetch end):gsub("__DISPATCH__", function() - return dispatch.build(layout.opcodes) + return dispatch.build(layout.opcodes, floorDivision) end) return source end diff --git a/src/prometheus/vm/runtime_dispatch.lua b/src/prometheus/vm/runtime_dispatch.lua index 653a1ca0..bec793d5 100644 --- a/src/prometheus/vm/runtime_dispatch.lua +++ b/src/prometheus/vm/runtime_dispatch.lua @@ -64,6 +64,7 @@ local bodies = { SUB = [==[R[a] = R[b] - R[c]]==], MUL = [==[R[a] = R[b] * R[c]]==], DIV = [==[R[a] = R[b] / R[c]]==], + IDIV = [==[R[a] = R[b] // R[c]]==], MOD = [==[R[a] = R[b] % R[c]]==], POW = [==[R[a] = R[b] ^ R[c]]==], CONCAT = [==[R[a] = R[b] .. R[c]]==], @@ -100,12 +101,15 @@ local bodies = { callBodies.merge(bodies) -function M.build(opcodes) +function M.build(opcodes, floorDivision) local entries = {} for name, id in pairs(opcodes) do - local body = bodies[name] - assert(body and type(id) == "number", "Missing VM handler for opcode " .. tostring(name)) - entries[#entries + 1] = { id = id, name = name, body = body } + -- Keep Lua 5.1 runtimes portable when no prototype uses floor division. + if name ~= "IDIV" or floorDivision then + local body = bodies[name] + assert(body and type(id) == "number", "Missing VM handler for opcode " .. tostring(name)) + entries[#entries + 1] = { id = id, name = name, body = body } + end end table.sort(entries, function(left, right) return left.id < right.id end) for i = 2, #entries do diff --git a/src/prometheus/vm/statements.lua b/src/prometheus/vm/statements.lua index 80bc4e04..139e2b29 100644 --- a/src/prometheus/vm/statements.lua +++ b/src/prometheus/vm/statements.lua @@ -165,11 +165,18 @@ return function(Compiler) release(self, values) end - local function assignmentTarget(self, target) + local function isCurrentLocal(self, expression) + return expression.kind == K.VariableExpression and not expression.scope.isGlobal + and self.analysis.owners[expression.scope] == self.owner + end + + local function assignmentTarget(self, target, compound) if target.kind == K.AssignmentVariable then return {scope = target.scope, id = target.id} end if target.kind == K.AssignmentIndexing then - local base = self:expression(target.base, 1)[1] + local base + if not compound or not isCurrentLocal(self, target.base) then base = self:expression(target.base, 1)[1] end local key = self:expression(target.index, 1)[1] + if not base then base = self:expression(target.base, 1)[1] end return {base = base, key = key} end error("invalid assignment target: " .. tostring(target.kind)) @@ -195,13 +202,22 @@ return function(Compiler) local compoundOp = { [K.CompoundAddStatement] = "ADD", [K.CompoundSubStatement] = "SUB", [K.CompoundMulStatement] = "MUL", [K.CompoundDivStatement] = "DIV", + [K.CompoundFloorDivStatement] = "IDIV", [K.CompoundModStatement] = "MOD", [K.CompoundPowStatement] = "POW", [K.CompoundConcatStatement] = "CONCAT", } local function compileCompound(self, node) - local target = assignmentTarget(self, node.lhs) + local target = assignmentTarget(self, node.lhs, true) local old = self:alloc() + local rhs + local localTarget = target.scope and not target.scope.isGlobal + and self.analysis.owners[target.scope] == self.owner + local deferRead = localTarget and node.kind ~= K.CompoundConcatStatement + -- Concatenation snapshots its left operand even when it is a local. + if deferRead then + rhs = self:expression(node.rhs, 1)[1] + end if target.scope then local value = self:readVariable(target.scope, target.id) self:emit("MOVE", old, value) @@ -209,9 +225,17 @@ return function(Compiler) else self:emit("GETTABLE", old, target.base, target.key) end - local rhs = self:expression(node.rhs, 1)[1] + if not deferRead then rhs = self:expression(node.rhs, 1)[1] end local result = self:alloc() self:emit(compoundOp[node.kind], result, old, rhs) + if target.base and isCurrentLocal(self, node.lhs.base) then + self:free(target.base) + target.base = self:expression(node.lhs.base, 1)[1] + end + if target.key and isCurrentLocal(self, node.lhs.index) then + self:free(target.key) + target.key = self:expression(node.lhs.index, 1)[1] + end storeTarget(self, target, result) release(self, {old, rhs, result}) if target.base then self:free(target.base); self:free(target.key) end diff --git a/tests/harness/luau-parser.lua b/tests/harness/luau-parser.lua index 7a3d0db7..77c4306b 100644 --- a/tests/harness/luau-parser.lua +++ b/tests/harness/luau-parser.lua @@ -51,6 +51,25 @@ local reusable = Parser:new({LuaVersion = 'LuaU'}) assert(not pcall(function() reusable:parse('local x: |& Broken = 1') end)) assert(reusable:parse('local x: number = 1').body.statements[1]) +for _, version in ipairs({'Lua51', 'LuaU'}) do + for _, source in ipairs({ + 'local function f() return ... end', + 'function object:f() print(...) end', + 'local function outer(...) return function() return ... end end', + 'local function f() do local args = {...} end end', + }) do fails(source, version) end + local parser = Parser:new({LuaVersion = version}) + assert(not pcall(function() parser:parse('local f = function() return ... end') end)) + assert(parser:parse('local function f() return 1 end; return ...')) + assert(parser:parse('local function f(...) return function(...) return ... end end')) +end +local validCall, varargError = pcall(function() + reusable:parse('local function f()\n print(...)\nend') +end) +assert(not validCall and tostring(varargError):find("Cannot use '...'", 1, true) + and tostring(varargError):find('Position 2:', 1, true), + 'invalid varargs inside calls must retain their original diagnostic') + local pipeline = Prometheus.Pipeline:fromConfig({LuaVersion = 'Lua51', Seed = 1}) assert(pipeline:getLuaVersion() == 'Lua51') pipeline:setLuaVersion('LuaU') diff --git a/tests/harness/luau-syntax.py b/tests/harness/luau-syntax.py index 570a3bd1..3086efbf 100644 --- a/tests/harness/luau-syntax.py +++ b/tests/harness/luau-syntax.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Compare type syntax acceptance and erasure against the official Luau compiler.""" +"""Compare Luau syntax acceptance and emitted code against the official compiler.""" import os from pathlib import Path import subprocess @@ -115,6 +115,119 @@ def add(label, source, expected=None): add(f'mutation alias {index}', 'type Alias = ' + form) +# Contextual words must remain usable in every identifier position. +for word in ('continue', 'type', 'export', 'const', 'typeof', 'read', 'write', 'declare'): + for template in ( + 'local %s = 1; return %s', + 'local function %s() end; %s()', + 'function %s() end; %s()', + 'local function f(%s) return %s end', + 'local t = {%s = 1}; return t.%s', + 'local t = {}; function t:%s() end; t:%s()', + 'for %s = 1, 2 do print(%s) end', + 'for %s in pairs({}) do print(%s) end', + 'type %s = number; local x: %s = 1', + ): + expected = not (word == 'typeof' and template.startswith('type ')) + add(f'contextual identifier {word}: {template}', template % (word, word), expected) +for source in ( + 'continue', 'continue = 1', 'continue, x = 1, 2', 'continue += 1', + 'continue //= 2', 'continue()', 'continue {}', 'continue "text"', + 'continue.field = 1', 'continue[1] = 1', 'continue:method()', + 'while true do continue end', 'while true do (continue) end', + 'while true do continue; end', 'while true do continue;; end', + 'while true do continue; print(1) end', + 'while true do continue() print(1) end', + 'local continue = 1; while true do continue end', + 'const continue = 1; while true do continue end', + 'while true do local function f() continue end end', + 'repeat continue until true', 'while true do do continue end end', + 'type = 1; export = 2; const = 3', + 'type(number); export(); const()', +): + add(f'contextual statement: {source}', source) + +# Luau accepts a single trailing semicolon, never an empty statement. +for statement in ( + 'local x = 1', 'x = 1', 'print()', 'do end', 'while true do break end', + 'for i = 1, 2 do end', 'repeat until true', 'function f() end', + 'local function f() end', 'type T = number', 'type function f() end', + 'const x = 1', 'export type T = number', +): + for before in ('', ';', ';;'): + for after in ('', ';', ';;'): + for context in ('%s', 'do %s end', 'local function f() %s end'): + source = context % (before + statement + after) + add(f'semicolons: {source}', source, not before and len(after) <= 1) +for source in ( + '', ';', ';;', 'do ; end', 'return;', 'return;;', + 'repeat return until true', 'repeat return; until true', + 'while true do break; end', 'while true do break;; end', + 'type A = number; type B = A;', 'type A = number;; type B = A', + 'return {1; 2; x = 3;}', 'type T = {x: number; y: string;}', + 'print(); (function() end)()', 'print()\n(function() end)()', + 'local x = f\n(1)', 'local x = f(\n1\n)', 'f -- comment\n(1)', + 'object:method\n()', 'f\n{}', 'f\n"text"', + 'while true do continue\n(1) end', +): + add(f'statement boundary: {source}', source) + +# Variadic state belongs to the nearest function, including erased type syntax. +for signature in ('()', '(x)', '(...)', '(x, ...)'): + for body in ( + 'return ...', 'print(...)', 'do local x = ... end', + 'local f = function() return ... end', + 'local function f(...) return ... end; return ...', + 'local function f() return 1 end; return ...', + 'local x = `value={...}`', 'type T = typeof(...)', + 'local x: typeof(...)', 'type function f() return ... end', + 'type function f(...) return ... end; return ...', + 'local function f(x: typeof(...)): typeof(...) return x end', + ): + for declaration in ('local function f%s %s end', 'function t:f%s %s end', + 'local f = function%s %s end'): + source = declaration % (signature, body) + add(f'vararg context: {source}', source) +for source in ( + 'return ...', 'return (...)', 'local x: typeof(...) = ...', + 'local function f(): typeof(...) end', + 'local function f(...): typeof(...) return ... end', + 'local function f(x: typeof(...)) end', + 'local function f() end; return ...', + 'local function f(...) return function(...) return ... end end', + 'local function f(..., x) end', 'local function f(..., ...) end', + 'local function f() type T = typeof(function(...) return ... end) end', +): + add(f'vararg boundary: {source}', source) + +# Number-like tokens must be validated as a whole, including discarded types. +numerals = [ + '0', '01', '1.', '.5', '1_000', '1__2_', '1_.2_5', '1_e+_2_', + '0xFF', '0Xff', '0_x_F_F_', '0b101', '0B101', '0_b_1_0_', + '0b__1__0_', '1e999', '1e-999', '0x', '0b', '0x_', '0b_', + '0b102', '0xFG', '0b0b1', '1e', '1e+', '1e_+2', '1..2', + '1print()', '0x1.2', '0x1p2', '0x1p-2', '0x1e+2', '1e2.3', +] +for literal in ('1', '.5', '1.', '1e2', '0b1', '0xF', '0X1e'): + for suffix in ('_', '__', 'e', 'e+', 'e-', 'e+_2', 'e_+2', '.', '..', 'a', 'p2', 'f'): + numerals.append(literal + suffix) +for prefix, digit in (('0x', 'f'), ('0b', '1')): + for length in (1, 8, 16, 17, 53, 64, 65, 128): + numerals.append(prefix + digit * length) +for literal in sorted(set(numerals)): + for template in ('return %s', 'local value = %s; print(value)', 'type T = typeof(%s)'): + add(f'numeral: {template % literal}', template % literal) + +for source in ( + 'local a, b, c = 1, 2, 3; return a // b * c % 2 / 3', + 'local a, b = 1, 2; a //= b; return a', + 'local t = {x = 1}; t.x //= 2', + 'return -2 ^ 2 // 3', 'return 2 ^ -3 // 2', + 'return 1 /// 2', 'local a = 1; a //== 2', +): + add(f'floor division: {source}', source) + + def official(path): result = subprocess.run([compiler, '--binary', str(path)], cwd=root, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, diff --git a/tests/harness/luau.py b/tests/harness/luau.py index 54183d15..c8be33c2 100644 --- a/tests/harness/luau.py +++ b/tests/harness/luau.py @@ -39,7 +39,7 @@ def run(args): '1', '1', str(invalid), output], cwd=root, capture_output=True, text=True, timeout=60) assert baseline.returncode != 0 and transformed.returncode != 0, invalid.name - print('Invalid type, const, and interpolation examples were rejected', flush=True) + print('Invalid Luau examples were rejected', flush=True) fixture = root / 'tests/luau/unsupported/protected-metatable.luau' assert run([luau, str(fixture)]) == '1\n' for index in (2, 3): diff --git a/tests/luau/compound-assignment.luau b/tests/luau/compound-assignment.luau new file mode 100644 index 00000000..addd5222 --- /dev/null +++ b/tests/luau/compound-assignment.luau @@ -0,0 +1,47 @@ +-- The shared compound handlers must preserve all existing arithmetic operators. +local value = 20 +local function change() + value = 100 + return 2 +end +value += change() +assert(value == 102) +value -= change() +assert(value == 98) +value *= change() +assert(value == 200) +value /= change() +assert(value == 50) +value %= change() +assert(value == 0) +value ^= change() +assert(value == 10000) + +local selfValue = 7 +selfValue += selfValue +assert(selfValue == 14) +selfValue /= selfValue +assert(selfValue == 1) +selfValue -= selfValue +assert(selfValue == 0) + +-- Concatenation snapshots a local before RHS, unlike arithmetic assignment. +local text = 'old' +local function suffix() + text = 'new' + return '!' +end +text ..= suffix() +assert(text == 'old!') + +local object = {value = 20} +local function changeField() + object.value = 100 + return 2 +end +object.value += changeField() +assert(object.value == 22) +object.value = 20 +object.value -= changeField() +assert(object.value == 18) +print('Compound assignment checks passed') diff --git a/tests/luau/floor-division.luau b/tests/luau/floor-division.luau new file mode 100644 index 00000000..863f7865 --- /dev/null +++ b/tests/luau/floor-division.luau @@ -0,0 +1,128 @@ +local function divide(a, b) + return a // b +end +assert(divide(7, 2) == 3 and divide(-7, 2) == -4) +assert(divide(7, -2) == -4 and divide(-7, -2) == 3) +assert(divide(7.5, 2) == 3 and divide(-7.5, 2) == -4) +local a, b, c = 29, 3, 2 +assert(a // b // c == 4) +assert(a // (b // c) == 29) +assert(a / b // c == 4 and a // b / c == 4.5) +assert(a * b // c % 5 == 3) +assert(2 + a // b * c == 20 and -2 ^ 2 // 3 == -2) +assert(a // b ^ c == 3 and (a // b) ^ c == 81) + +-- Preserve IEEE special values and the sign of zero. +local zero, infinity = 0, math.huge +assert(divide(1, zero) == infinity and divide(-1, zero) == -infinity) +local nan = divide(zero, zero) +assert(nan ~= nan) +assert(1 / divide(-zero, 2) == -infinity) +assert(1 / divide(1, -infinity) == -infinity) + +-- Floor division uses __idiv, with no fallback to __div or result rounding. +local tag = {} +local meta = { + __idiv = function(left, right) return tag, 'discarded' end, + __div = function() error('__div must not handle //') end, +} +local operand = setmetatable({}, meta) +assert(divide(operand, 2) == tag and divide(2, operand) == tag) +assert(select('#', divide(operand, 2)) == 1) +assert(divide('7', 2) == 3) +assert(not pcall(divide, 'invalid', 2)) +assert(not pcall(divide, setmetatable({}, {__div = function() return 1 end}), 2)) + +local value = 20 +local function changeValue() + value = 100 + return 2 +end +assert(value // changeValue() == 50) +value = 20 +value //= changeValue() +assert(value == 50) +value = 20 +local function divideUpvalue() + return value // changeValue() +end +assert(divideUpvalue() == 10 and value == 100) +value = 50 +do + local function update() + value //= changeValue() + end + update() + assert(value == 25) +end + +floorDivisionGlobal = 20 +local function changeGlobal() + floorDivisionGlobal = 100 + return 2 +end +floorDivisionGlobal //= changeGlobal() +assert(floorDivisionGlobal == 10) + +local object, calls = {value = 20}, '' +local function base() + calls ..= 'b' + return object +end +local function key() + calls ..= 'k' + return 'value' +end +local function rhs() + calls ..= 'r' + object.value = 100 + return 2 +end +base()[key()] //= rhs() +assert(object.value == 10 and calls == 'bkr') + +local original = {value = 20} +local replacement = {value = 100} +local holder, index = original, 'value' +holder[index] //= (function() + holder, index = replacement, 'other' + return 2 +end)() +assert(original.value == 20 and replacement.value == 100 and replacement.other == 10) + +local first, second = {value = 20}, {value = 40} +local current = first +local function switchBase() + current = second + return 'value' +end +current[switchBase()] //= 2 +assert(first.value == 20 and second.value == 20) + +local fixedHolder, fixedKey = first, 'value' +local function updateOuterTarget() + fixedHolder[fixedKey] //= (function() + fixedHolder, fixedKey = second, 'other' + return 2 + end)() +end +updateOuterTarget() +assert(first.value == 10 and second.other == nil) +local selfDivision = 7 +selfDivision //= selfDivision +assert(selfDivision == 1) + +local proxy = setmetatable({}, { + __index = function(_, key) + calls ..= 'i' + return operand + end, + __newindex = function(_, key, value) + assert(value == tag) + calls ..= 's' + end, +}) +calls = '' +proxy.value //= (function() calls ..= 'r'; return 2 end)() +assert(calls == 'irs') +print('Floor division checks passed') diff --git a/tests/luau/invalid/numeral-missing-digits.luau b/tests/luau/invalid/numeral-missing-digits.luau new file mode 100644 index 00000000..5c32089f --- /dev/null +++ b/tests/luau/invalid/numeral-missing-digits.luau @@ -0,0 +1 @@ +return 0b diff --git a/tests/luau/invalid/numeral-token-boundary.luau b/tests/luau/invalid/numeral-token-boundary.luau new file mode 100644 index 00000000..ad402a1f --- /dev/null +++ b/tests/luau/invalid/numeral-token-boundary.luau @@ -0,0 +1 @@ +local value = 1print('invalid') diff --git a/tests/luau/invalid/semicolon-empty-statement.luau b/tests/luau/invalid/semicolon-empty-statement.luau new file mode 100644 index 00000000..0ef80e4a --- /dev/null +++ b/tests/luau/invalid/semicolon-empty-statement.luau @@ -0,0 +1 @@ +local value = 1;; diff --git a/tests/luau/invalid/vararg-nested-function.luau b/tests/luau/invalid/vararg-nested-function.luau new file mode 100644 index 00000000..d53b3f3d --- /dev/null +++ b/tests/luau/invalid/vararg-nested-function.luau @@ -0,0 +1,5 @@ +local function outer(...) + return function() + return ... + end +end diff --git a/tests/luau/invalid/vararg-non-variadic.luau b/tests/luau/invalid/vararg-non-variadic.luau new file mode 100644 index 00000000..2370ab79 --- /dev/null +++ b/tests/luau/invalid/vararg-non-variadic.luau @@ -0,0 +1,3 @@ +local function fixed() + return ... +end diff --git a/tests/luau/keywords.luau b/tests/luau/keywords.luau new file mode 100644 index 00000000..cce93104 --- /dev/null +++ b/tests/luau/keywords.luau @@ -0,0 +1,43 @@ +-- Contextual keywords are ordinary names in expressions, bindings, and fields. +local continue, type, export, const = 4, 5, 6, 7 +local function names(continue, type, export, const, typeof, read, write) + return continue + type + export + const + typeof + read + write +end +assert(names(continue, type, export, const, 8, 9, 10) == 49) + +local object = {continue = 11, type = 12, export = 13, const = 14} +function object:continue(type) + return self.type + type +end +assert(object:continue(3) == 15) +function object.export(continue) + return continue * 2 +end +assert(object.export(5) == 10 and object.const == 14) + +local sum = 0 +for continue = 1, 3 do + sum += continue +end +for type, export in ipairs({4, 5}) do + sum += type + export +end +assert(sum == 18) + +local calls = 0 +do + local function continue() + calls += 1 + end + for i = 1, 4 do + continue() + if i % 2 == 0 then continue end + calls += 10 + end +end +assert(calls == 24) +type Continue = {continue: number, type: number} +export type Export = Continue +local typed: Export = {continue = 1, type = 2} +assert(typed.continue + typed.type == 3) +print('Contextual keyword checks passed') diff --git a/tests/luau/numerals.luau b/tests/luau/numerals.luau new file mode 100644 index 00000000..1983b26e --- /dev/null +++ b/tests/luau/numerals.luau @@ -0,0 +1,25 @@ +local decimal = {1_000, 1__2_, 1_.2_5, .5_0, 1_e+_2_, 2.5e-1, 01, 1.} +local expected = {1000, 12, 1.25, 0.5, 100, 0.25, 1, 1} +for i, value in ipairs(decimal) do assert(value == expected[i]) end + +local radix = {0b1010, 0B11, 0b__1__0_, 0_b_1_0_, 0xFF, 0XABCDEF, 0_x_F_F_} +local expectedRadix = {10, 3, 2, 2, 255, 11259375, 255} +for i, value in ipairs(radix) do assert(value == expectedRadix[i]) end + +-- Radix conversion rounds once and saturates at the unsigned 64-bit limit. +local boundaries = { + 0x1FFFFFFFFFFFFF, 0x20000000000001, 0x20000000000003, + 0xFFFFFFFFFFFFF800, 0xFFFFFFFFFFFFFFFF, + 0x10000000000000000, 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF, + 0b11111111111111111111111111111111111111111111111111111111111111111, +} +local expectedBoundaries = { + 9007199254740991, 9007199254740992, 9007199254740996, + 18446744073709549568, 18446744073709551616, + 18446744073709551616, 18446744073709551616, 18446744073709551616, +} +for i, value in ipairs(boundaries) do assert(value == expectedBoundaries[i]) end +local overflow, underflow = 1e999, 1e-999 +assert(overflow == math.huge and underflow == 0) +assert(1_0 .. 'x' == '10x') +print('Luau numeral checks passed') diff --git a/tests/luau/semicolons.luau b/tests/luau/semicolons.luau new file mode 100644 index 00000000..3d2419dc --- /dev/null +++ b/tests/luau/semicolons.luau @@ -0,0 +1,20 @@ +type Value = number; +export type Exported = Value; +type function Identity(value) return value end; +local value: Exported = 0; +do value += 1; end; +if value == 1 then value += 1; else error('wrong branch'); end; +while value < 10 do + value += 1; + if value == 3 then continue; end; + break; +end; +repeat value += 1; until true; +for i = 1, 2 do value += i; end; +local function stop() + repeat return; until true; +end; +stop(); +(function() value += 1; end)(); +assert(value == 9); +print('Semicolon checks passed'); diff --git a/tests/luau/types/a.obfuscated.luau b/tests/luau/types/a.obfuscated.luau deleted file mode 100644 index 36586aac..00000000 --- a/tests/luau/types/a.obfuscated.luau +++ /dev/null @@ -1 +0,0 @@ -local U=100 \ No newline at end of file diff --git a/tests/luau/varargs.luau b/tests/luau/varargs.luau new file mode 100644 index 00000000..8ac6a28b --- /dev/null +++ b/tests/luau/varargs.luau @@ -0,0 +1,33 @@ +local function pack(...) + return {n = select('#', ...), ...} +end +assert(pack(...).n == 0) +local function outer(...) + local args = pack(...) + local function fixed() return args.n end + local function inner(first, ...) + return first, fixed(), ... + end + do + assert(select('#', ...) == args.n) + end + return inner('first', ...) +end +local result = pack(outer(1, nil, 3, nil)) +assert(result.n == 6 and result[1] == 'first' and result[2] == 4) +assert(result[3] == 1 and result[4] == nil and result[5] == 3 and result[6] == nil) + +local object = {} +function object:collect(...: number) + return self, ... +end +result = pack(object:collect(1, nil, 2)) +assert(result.n == 4 and result[1] == object and result[2] == 1 and result[4] == 2) +local function interpolate(...) + return `value={(...)}` +end +assert(interpolate(7, 8) == 'value=7') +local function fixed() return 5 end +local function variadic(...) return ... end +assert(fixed() == 5 and select('#', variadic(nil, nil)) == 2) +print('Vararg scope checks passed') From abb861a9821f4227db8ba58ffe1fcfd795c09a44 Mon Sep 17 00:00:00 2001 From: tlredz Date: Wed, 28 Oct 2026 22:25:10 -0300 Subject: [PATCH 3/3] last luau fixes --- .gitignore | 2 + CONTRIBUTING.md | 18 +- Dockerfile | 7 + doc/reference/limitations.md | 37 ++- src/prometheus/ast.lua | 28 ++ src/prometheus/cff/compile_top.lua | 7 + src/prometheus/cff/compiler.lua | 3 +- src/prometheus/cff/expressions.lua | 1 + src/prometheus/cff/expressions/binary.lua | 2 +- src/prometheus/cff/expressions/index.lua | 10 +- src/prometheus/cff/expressions/number.lua | 2 +- .../cff/expressions/table_constructor.lua | 7 +- src/prometheus/cff/native.lua | 1 + src/prometheus/enums.lua | 4 +- src/prometheus/luau_attributes.lua | 78 ++++++ src/prometheus/luau_exports.lua | 48 ++++ src/prometheus/luau_iteration.lua | 87 ++++++ src/prometheus/luau_types.lua | 15 +- src/prometheus/parser.lua | 257 ++++++++++++------ src/prometheus/pipeline.lua | 2 + src/prometheus/scope.lua | 10 +- src/prometheus/steps/Vmify.lua | 4 +- src/prometheus/tokenizer.lua | 91 +++++-- src/prometheus/unparser.lua | 14 +- src/prometheus/vm/compiler.lua | 7 +- src/prometheus/vm/emit.lua | 19 +- src/prometheus/vm/expressions.lua | 11 +- src/prometheus/vm/expressions_tables.lua | 7 +- tests/harness/luau-recent.py | 54 ++++ tests/harness/luau-strings.lua | 34 +++ tests/harness/luau-syntax.py | 60 ++++ tests/harness/luau.py | 9 - tests/luau/attributes-generics.luau | 18 ++ tests/luau/evaluation-order.luau | 36 +++ tests/luau/invalid/attribute-duplicate.luau | 2 + tests/luau/invalid/const-typeof.luau | 2 + tests/luau/invalid/generic-instantiation.luau | 2 + .../invalid/parenthesized-assignment.luau | 2 + tests/luau/invalid/repeat-continue-local.luau | 4 + tests/luau/invalid/string-unicode-range.luau | 1 + tests/luau/invalid/type-cast-chain.luau | 2 + tests/luau/protected-metatable.luau | 56 ++++ tests/luau/recent/exports.luau | 9 + tests/luau/recent/integers.luau | 14 + 44 files changed, 931 insertions(+), 153 deletions(-) create mode 100644 src/prometheus/luau_attributes.lua create mode 100644 src/prometheus/luau_exports.lua create mode 100644 src/prometheus/luau_iteration.lua create mode 100644 tests/harness/luau-recent.py create mode 100644 tests/harness/luau-strings.lua create mode 100644 tests/luau/attributes-generics.luau create mode 100644 tests/luau/evaluation-order.luau create mode 100644 tests/luau/invalid/attribute-duplicate.luau create mode 100644 tests/luau/invalid/const-typeof.luau create mode 100644 tests/luau/invalid/generic-instantiation.luau create mode 100644 tests/luau/invalid/parenthesized-assignment.luau create mode 100644 tests/luau/invalid/repeat-continue-local.luau create mode 100644 tests/luau/invalid/string-unicode-range.luau create mode 100644 tests/luau/invalid/type-cast-chain.luau create mode 100644 tests/luau/protected-metatable.luau create mode 100644 tests/luau/recent/exports.luau create mode 100644 tests/luau/recent/integers.luau diff --git a/.gitignore b/.gitignore index 32261ce8..e520251e 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,8 @@ __pycache__/ /luau /luau-compile /luau-*.zip +/LuaJIT-*/ +/luajit-source.tar.gz *.obfuscated.lua *.obfuscated.luau *.out.luau diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c294b845..0beeecd7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,10 @@ Thanks for contributing to Prometheus. ## Running Tests -Tests run inside a Docker container with lua5.1 and Luau: +The obfuscator source must remain valid Lua 5.1 and run in both Lua 5.1 and +LuaJIT. `LuaVersion` selects the input/output dialect; Luau is a target runtime, +not a dependency of the obfuscator host. Tests run inside a Docker container +with Lua 5.1, LuaJIT, and the official Luau runtime: ```bash ./scripts/run-tests.sh # Run all tests (default: 10 iterations) @@ -22,6 +25,19 @@ Tests run inside a Docker container with lua5.1 and Luau: ./scripts/run-tests.sh -v # Verbose output ``` +The Docker build runs Luau syntax and execution checks with both Lua 5.1 and +LuaJIT as the obfuscator host. The same checks can be run directly: + +```bash +LUA51=luajit python3 tests/harness/luau-syntax.py +LUA51=luajit python3 tests/harness/luau.py +LUA51=luajit python3 tests/harness/luau-recent.py +luajit tests/harness/luau-strings.lua +``` + +`LUAU` and `LUAU_COMPILE` select the target runtime and compiler. The recent +syntax checks require a Luau version with value exports and integer literals. + ### Test File Metadata Test files can include metadata comments at the top of the file: diff --git a/Dockerfile b/Dockerfile index 173531b3..34fc3623 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM ubuntu:24.04 RUN apt-get update && apt-get install -y \ lua5.1 \ + luajit \ python3 \ patch \ git \ @@ -28,6 +29,12 @@ RUN lua5.1 tests/harness/metadata.lua \ && timeout 60s lua5.1 tests/harness/luau-parser.lua \ && python3 tests/harness/luau-syntax.py \ && python3 tests/harness/luau.py \ + && python3 tests/harness/luau-recent.py \ + && lua5.1 tests/harness/luau-strings.lua \ + && LUA51=luajit python3 tests/harness/luau-syntax.py \ + && LUA51=luajit python3 tests/harness/luau.py \ + && LUA51=luajit python3 tests/harness/luau-recent.py \ + && luajit tests/harness/luau-strings.lua \ && python3 tests/harness/cli.py \ && timeout 60s lua5.1 tests/harness/numbers-to-expressions.lua \ && timeout 60s lua5.1 tests/harness/cff-arity.lua \ diff --git a/doc/reference/limitations.md b/doc/reference/limitations.md index 0cb1ca5d..7ed69dbb 100644 --- a/doc/reference/limitations.md +++ b/doc/reference/limitations.md @@ -14,6 +14,10 @@ Source code constructed or loaded at runtime is not automatically obfuscated. ## Luau +The obfuscator itself is written in Lua 5.1 and supports LuaJIT. `LuaVersion` +selects the input/output language; a Luau runtime is needed to execute Luau +output, not to run the obfuscator. + Use `--LuaU` or `LuaVersion = "LuaU"` for Luau input. Type declarations, annotations, assertions (`::`), and type function declarations are removed; Prometheus does not type-check them, evaluate `typeof` expressions in type @@ -29,15 +33,13 @@ reassignment and lowered to locals; tables referenced by a `const` remain mutable. Interpolation uses Luau's `%*` string formatting and therefore requires a Luau runtime for the transformed code. -Minify leaves generalized iteration to Luau. The `Cff` and `Vmify` compilers normalize tables, `__iter`, callable objects, and -ordinary iterator triples. This normalization uses the runtime's `type`, -`getmetatable`, `rawget`, `next`, and `error` globals; they must retain their -standard behavior. Generalized iteration over objects with protected -metatables is not supported by these compilers: Luau hides the real metatable -from ordinary Lua code. Use an explicit iterator function, for example -`for k, v in pairs(t) do`, when appropriate. A non-table protection value -is rejected; a table used as `__metatable` cannot be distinguished from a -real metatable and may cause different behavior. +Generalized iteration supports tables, `__iter`, callable objects, ordinary +iterator triples, and protected metatables in every preset. `Cff` and `Vmify` +retain native iterator initialization in a small function and compile the loop +body as a callback. This preserves the caller's coroutine, break/continue, +outer returns, iteration-local captures, and trailing nil return values without +reflecting on the object's metatable. Vararg/return transport uses the standard +`select` and `unpack` globals. Floor division (`//` and `//=`) is supported in every preset and preserves Luau's native `__idiv` metamethod. Contextual keywords such as `continue`, @@ -52,6 +54,23 @@ does not inherit its enclosing function's varargs. These checks also apply to erased type syntax. Type function bodies must use syntax that the parser supports even though those declarations are erased. +Explicit generic instantiation such as `identity<>(value)` is validated +and erased. Function attributes `@native`, `@checked`, `@deprecated`, and their +`@[...]` forms are validated. Minify preserves function attributes. Virtualizing +a function changes its layout, so its original per-function compilation hints +do not directly describe the generated dispatcher. Header directives +`--!native` and `--!optimize 0/1/2` are preserved by every preset. + +Recent Luau runtimes also support value exports (`export local`, `export const`, +and `export function`) and 64-bit integer literals such as `9007199254740993i`. +Exports are lowered to a module table with live bindings and a frozen module +return, including duplicate-export and top-level-return checks. Integer +literals retain their exact digits and bit patterns without converting them +to the obfuscator host's double-precision numbers. Their available operations +follow the selected Luau runtime; no arithmetic behavior is emulated by the +obfuscator. Lune 0.11.0 does not accept these newer input forms. Integer output +therefore requires a more recent Luau runtime. + ## Replacing a running function's environment Prometheus does not support replacing a function's environment while that diff --git a/src/prometheus/ast.lua b/src/prometheus/ast.lua index 5da82aa5..8a9715ba 100644 --- a/src/prometheus/ast.lua +++ b/src/prometheus/ast.lua @@ -45,6 +45,7 @@ local AstKind = { -- Expression Nodes BooleanExpression = "BooleanExpression"; NumberExpression = "NumberExpression"; + IntegerExpression = "IntegerExpression"; StringExpression = "StringExpression"; NilExpression = "NilExpression"; VarargExpression = "VarargExpression"; @@ -87,6 +88,7 @@ local AstKind = { local astKindExpressionLookup = { [AstKind.BooleanExpression] = 0; [AstKind.NumberExpression] = 0; + [AstKind.IntegerExpression] = 0; [AstKind.StringExpression] = 0; [AstKind.NilExpression] = 0; [AstKind.VarargExpression] = 0; @@ -420,6 +422,10 @@ function Ast.LocalVariableDeclaration(scope, ids, expressions) } end +function Ast.IntegerExpression(source) + return {kind = AstKind.IntegerExpression, value = source, isConstant = false}; +end + function Ast.VarargExpression() return { kind = AstKind.VarargExpression; @@ -712,6 +718,28 @@ function Ast.NotExpression(rhs, simplify) end function Ast.NegateExpression(rhs, simplify) + if simplify and rhs.kind == AstKind.IntegerExpression then + local source = rhs.value; + if source:sub(1, 1) == "-" then return Ast.IntegerExpression(source:sub(2)) end + local prefix = source:sub(1, 2):lower(); + if prefix ~= "0x" and prefix ~= "0b" then return Ast.IntegerExpression("-" .. source) end + local digits = source:sub(3, -2):lower(); + if prefix == "0b" then + digits = string.rep("0", (4 - #digits % 4) % 4) .. digits; + local hex = {}; + for i = 1, #digits, 4 do hex[#hex + 1] = string.format("%x", tonumber(digits:sub(i, i + 3), 2)) end + digits = table.concat(hex); + end + digits = digits:gsub("^0+", ""); + digits = string.rep("0", 16 - #digits) .. digits; + local hex, carry = {}, 1; + for i = 16, 1, -1 do + local value = 15 - tonumber(digits:sub(i, i), 16) + carry; + hex[i] = string.format("%x", value % 16); + carry = math.floor(value / 16); + end + return Ast.IntegerExpression("0x" .. table.concat(hex) .. "i"); + end if(simplify and rhs.isConstant) then local success, val = pcall(function() return -rhs.value end); if success then diff --git a/src/prometheus/cff/compile_top.lua b/src/prometheus/cff/compile_top.lua index b3a77c8b..f718e4b0 100644 --- a/src/prometheus/cff/compile_top.lua +++ b/src/prometheus/cff/compile_top.lua @@ -13,6 +13,13 @@ local AstKind = Ast.AstKind; return function(Compiler) function Compiler:compileTopNode(node) + visitast(node, function(expression) + if expression.isNativeLuau then + self.nativeExpressions = self.nativeExpressions or {}; + self.nativeExpressions[expression] = true; + return nil, true; + end + end); if self.nativeExpressions then self:prepareNativeExpressions(node); end local startBlock = self:createBlock(); local scope = startBlock.scope; diff --git a/src/prometheus/cff/compiler.lua b/src/prometheus/cff/compiler.lua index 89a01279..518f7192 100644 --- a/src/prometheus/cff/compiler.lua +++ b/src/prometheus/cff/compiler.lua @@ -23,7 +23,7 @@ local Compiler = {}; function Compiler:new(settings) local compiler = { - luaVersion = settings and settings.LuaVersion or "Lua51"; + luaVersion = settings and settings.LuaVersion; blocks = {}; registers = {}; activeBlock = nil; @@ -85,6 +85,7 @@ function Compiler:popRegisterUsageInfo() end function Compiler:compile(ast) + self.luaVersion = self.luaVersion or ast.luaVersion or "Lua51"; self.blocks = {}; self.registers = {}; self.activeBlock = nil; diff --git a/src/prometheus/cff/expressions.lua b/src/prometheus/cff/expressions.lua index 26498590..a5f58f73 100644 --- a/src/prometheus/cff/expressions.lua +++ b/src/prometheus/cff/expressions.lua @@ -14,6 +14,7 @@ local function requireExpression(name) end handlers[AstKind.StringExpression] = requireExpression("string"); handlers[AstKind.NumberExpression] = requireExpression("number"); +handlers[AstKind.IntegerExpression] = requireExpression("number"); handlers[AstKind.BooleanExpression] = requireExpression("boolean"); handlers[AstKind.NilExpression] = requireExpression("nil"); handlers[AstKind.VariableExpression] = requireExpression("variable"); diff --git a/src/prometheus/cff/expressions/binary.lua b/src/prometheus/cff/expressions/binary.lua index 25c7eca4..49efd8cd 100644 --- a/src/prometheus/cff/expressions/binary.lua +++ b/src/prometheus/cff/expressions/binary.lua @@ -14,7 +14,7 @@ return function(self, expression, funcDepth, numReturns) if i == 1 then local lhsReg, rhsReg; local lhs = expression.lhs; - local deferRead = expression.kind == Ast.AstKind.FloorDivExpression + local deferRead = self.luaVersion == "LuaU" and expression.kind ~= Ast.AstKind.StrCatExpression and lhs.kind == Ast.AstKind.VariableExpression and not lhs.scope.isGlobal and self.scopeFunctionDepths[lhs.scope] == funcDepth; if not deferRead then lhsReg = self:compileExpression(lhs, funcDepth, 1)[1] end diff --git a/src/prometheus/cff/expressions/index.lua b/src/prometheus/cff/expressions/index.lua index c3574c7d..602f0611 100644 --- a/src/prometheus/cff/expressions/index.lua +++ b/src/prometheus/cff/expressions/index.lua @@ -12,11 +12,13 @@ return function(self, expression, funcDepth, numReturns) for i = 1, numReturns do regs[i] = self:allocRegister(); if i == 1 then - local baseReg = self:compileExpression(expression.base, funcDepth, 1)[1]; + local baseReg; + local base = expression.base; + local deferRead = self.luaVersion == "LuaU" and base.kind == Ast.AstKind.VariableExpression + and not base.scope.isGlobal and self.scopeFunctionDepths[base.scope] == funcDepth; + if not deferRead then baseReg = self:compileExpression(base, funcDepth, 1)[1] end local indexReg = self:compileExpression(expression.index, funcDepth, 1)[1]; - - - + if deferRead then baseReg = self:compileExpression(base, funcDepth, 1)[1] end self:addStatement(self:setRegister(scope, regs[i], Ast.IndexExpression(self:register(scope, baseReg), self:register(scope, indexReg))), {regs[i]}, {baseReg, indexReg}, true); self:freeRegister(baseReg, false); diff --git a/src/prometheus/cff/expressions/number.lua b/src/prometheus/cff/expressions/number.lua index 4fe5e3b9..0315666e 100644 --- a/src/prometheus/cff/expressions/number.lua +++ b/src/prometheus/cff/expressions/number.lua @@ -18,7 +18,7 @@ return function(self, expression, funcDepth, numReturns) self:addStatement(self:setRegister(scope, regs[i], Ast.NumberExpression(0)), {regs[i]}, {}, false); self:addStatement(self:setRegister(scope, regs[i], Ast.NegateExpression(self:register(scope, regs[i]))), {regs[i]}, {regs[i]}, false); else - self:addStatement(self:setRegister(scope, regs[i], Ast.NumberExpression(expression.value)), {regs[i]}, {}, false); + self:addStatement(self:setRegister(scope, regs[i], Ast[expression.kind](expression.value)), {regs[i]}, {}, false); end else self:addStatement(self:setRegister(scope, regs[i], Ast.NilExpression()), {regs[i]}, {}, false); diff --git a/src/prometheus/cff/expressions/table_constructor.lua b/src/prometheus/cff/expressions/table_constructor.lua index f9ce495a..c0624c53 100644 --- a/src/prometheus/cff/expressions/table_constructor.lua +++ b/src/prometheus/cff/expressions/table_constructor.lua @@ -30,8 +30,13 @@ return function(self, expression, funcDepth, numReturns) table.insert(entryRegs, reg); end else - local keyReg = self:compileExpression(entry.key, funcDepth, 1)[1]; + local keyReg; + local key = entry.key; + local deferRead = self.luaVersion == "LuaU" and key.kind == AstKind.VariableExpression + and not key.scope.isGlobal and self.scopeFunctionDepths[key.scope] == funcDepth; + if not deferRead then keyReg = self:compileExpression(key, funcDepth, 1)[1] end local valReg = self:compileExpression(entry.value, funcDepth, 1)[1]; + if deferRead then keyReg = self:compileExpression(key, funcDepth, 1)[1] end table.insert(entries, Ast.KeyedTableEntry(self:register(scope, keyReg), self:register(scope, valReg))); table.insert(entryRegs, valReg); table.insert(entryRegs, keyReg); diff --git a/src/prometheus/cff/native.lua b/src/prometheus/cff/native.lua index 5d799822..00df7075 100644 --- a/src/prometheus/cff/native.lua +++ b/src/prometheus/cff/native.lua @@ -120,6 +120,7 @@ return function(Compiler) .. names.result .. '=' .. source .. ';' .. finish .. 'return ' .. names.result .. ' end') local factory = parsed.body.statements[1].args[1] self.nativeResults[expression] = factory.body.statements[1].expressions[1] + self.nativeResults[expression].isNativeLuau = expression.isNativeLuau factory.body.scope:setParent(scope) require('prometheus.visitast')(parsed, nil, function(node, data) if node.kind == Kind.VariableExpression and node.scope.isGlobal then diff --git a/src/prometheus/enums.lua b/src/prometheus/enums.lua index 14571168..ea3ac288 100644 --- a/src/prometheus/enums.lua +++ b/src/prometheus/enums.lua @@ -66,7 +66,7 @@ Enums.Conventions = { }, HybridKeywords = {"continue", "type", "export", "const", "typeof", "read", "write"}, - SymbolChars = chararray("+-*/%^#=~<>(){}[];:,.?|&"), + SymbolChars = chararray("+-*/%^#=~<>(){}[];:,.?|&@"), MaxSymbolLength = 3, Symbols = { "+", "-", "*", "/", "//", "%", "^", "#", @@ -74,7 +74,7 @@ Enums.Conventions = { "+=", "-=", "/=", "//=", "%=", "^=", "..=", "*=", "(", ")", "{", "}", "[", "]", ";", ":", ",", ".", "..", "...", - "::", "->", "?", "|", "&", + "::", "->", "?", "|", "&", "@", }, IdentChars = chararray("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789"), diff --git a/src/prometheus/luau_attributes.lua b/src/prometheus/luau_attributes.lua new file mode 100644 index 00000000..a76b821f --- /dev/null +++ b/src/prometheus/luau_attributes.lua @@ -0,0 +1,78 @@ +-- Parse function attributes separately from runtime expressions and type erasure. +local Ast = require('prometheus.ast') +local logger = require('logger') +local K = Ast.AstKind + +return function(Parser, token, is, consume, expect, peek, get) + local function fail(self, message) + local tk = peek(self) + logger:error(string.format('Parsing Error at Position %s:%s, %s', tk.line, tk.linePos, message)) + end + local function literal(node) + if node.isParenthesizedExpression then return false end + if node.kind == K.StringExpression or node.kind == K.NumberExpression + or node.kind == K.BooleanExpression or node.kind == K.NilExpression then return true end + if node.kind ~= K.TableConstructorExpression then return false end + for _, entry in ipairs(node.entries) do + if entry.key and not literal(entry.key) then return false end + if not literal(entry.value) then return false end + end + return true + end + + function Parser:parseLuauAttributes(scope) + if self.luaVersion ~= 'LuaU' or not is(self, token.Symbol, '@') then return nil end + local attributes, seen = {}, {} + local function attribute(listed) + local name = expect(self, token.Ident).value + if name ~= 'native' and name ~= 'checked' and name ~= 'deprecated' then fail(self, 'Invalid function attribute ' .. name) end + if seen[name] then fail(self, 'Duplicate function attribute ' .. name) end + seen[name] = true + local args = {} + if listed then + local savedSimplify = self.noSimplify + self.noSimplify = true + if consume(self, token.Symbol, '(') then + if not is(self, token.Symbol, ')') then args = self:exprList(scope) end + expect(self, token.Symbol, ')') + elseif is(self, token.Symbol, '{') then + args = {self:tableConstructor(scope)} + elseif is(self, token.String) then + args = {Ast.StringExpression(get(self).value)} + end + self.noSimplify = savedSimplify + end + for _, arg in ipairs(args) do if not literal(arg) then fail(self, 'Only literals are allowed in attributes') end end + if name == 'deprecated' then + if #args > 1 then fail(self, 'Deprecated accepts at most one argument') end + local arg = args[1] + if arg and arg.kind ~= K.TableConstructorExpression then fail(self, 'Deprecated requires a table literal') end + if arg then + for _, entry in ipairs(arg.entries) do + if not entry.isRecord or not entry.key or entry.key.kind ~= K.StringExpression + or (entry.key.value ~= 'use' and entry.key.value ~= 'reason') + or entry.value.kind ~= K.StringExpression then + fail(self, 'Deprecated table accepts only string fields use and reason') + end + end + end + end + end + while is(self, token.Symbol, '@') do + local first = self.index + 1 + local at = get(self) + if peek(self).startPos ~= at.endPos then fail(self, "Expected an attribute name or '[' immediately after '@'") end + if consume(self, token.Symbol, '[') then + attribute(true) + while consume(self, token.Symbol, ',') do attribute(true) end + expect(self, token.Symbol, ']') + else + attribute(false) + end + local pieces = {} + for i = first, self.index do pieces[#pieces + 1] = self.tokens[i].source end + attributes[#attributes + 1] = table.concat(pieces, ' '):gsub('^@ ', '@') + end + return attributes + end +end diff --git a/src/prometheus/luau_exports.lua b/src/prometheus/luau_exports.lua new file mode 100644 index 00000000..0b6cdde4 --- /dev/null +++ b/src/prometheus/luau_exports.lua @@ -0,0 +1,48 @@ +-- Lower value exports to a module table while retaining live mutable bindings. +local Ast = require('prometheus.ast') +local visitast = require('prometheus.visitast') +local K = Ast.AstKind + +return function(ast, bindings) + if #bindings == 0 then return ast end + local scope, lookup = ast.body.scope, {} + local tableId = scope:addVariable() + for _, binding in ipairs(bindings) do + lookup[binding.scope] = lookup[binding.scope] or {} + lookup[binding.scope][binding.id] = binding.name + end + local function access(owner, id, useScope, assignment) + local name = lookup[owner] and lookup[owner][id] + if not name then return nil end + if useScope ~= scope then useScope:addReferenceToHigherScope(scope, tableId) end + local base = Ast.VariableExpression(scope, tableId) + return (assignment and Ast.AssignmentIndexing or Ast.IndexExpression)(base, Ast.StringExpression(name)) + end + visitast(ast, function(node, data) + if node.kind == K.VariableExpression or node.kind == K.AssignmentVariable then + return access(node.scope, node.id, data.scope, node.kind == K.AssignmentVariable) + elseif node.kind == K.LocalVariableDeclaration and node.isExported then + local targets = {} + for _, id in ipairs(node.ids) do targets[#targets + 1] = access(node.scope, id, data.scope, true) end + local values = node.expressions + if #values == 0 then values = {Ast.NilExpression()} end + return Ast.AssignmentStatement(targets, values) + elseif (node.kind == K.LocalFunctionDeclaration and node.isExported) or node.kind == K.FunctionDeclaration then + local target = access(node.scope, node.id, data.scope, true) + if not target then return nil end + for _, name in ipairs(node.indices or {}) do + target.kind = K.IndexExpression + target = Ast.AssignmentIndexing(target, Ast.StringExpression(name)) + end + local value = Ast.FunctionLiteralExpression(node.args, node.body) + value.attributes = node.attributes + return Ast.AssignmentStatement({target}, {value}) + end + end) + table.insert(ast.body.statements, 1, Ast.LocalVariableDeclaration(scope, {tableId}, {Ast.TableConstructorExpression({})})) + local tableScope, libraryId = scope:resolveGlobal('table') + local freeze = Ast.IndexExpression(Ast.VariableExpression(tableScope, libraryId), Ast.StringExpression('freeze')) + ast.body.statements[#ast.body.statements + 1] = Ast.ReturnStatement{ + Ast.FunctionCallExpression(freeze, {Ast.VariableExpression(scope, tableId)})} + return ast +end diff --git a/src/prometheus/luau_iteration.lua b/src/prometheus/luau_iteration.lua new file mode 100644 index 00000000..45a56209 --- /dev/null +++ b/src/prometheus/luau_iteration.lua @@ -0,0 +1,87 @@ +-- Keep iterator initialization native so protected __iter metatables work without +-- reflection. The callback remains in the caller's coroutine and shares captures. +local Ast = require('prometheus.ast') +local Scope = require('prometheus.scope') +local visitast = require('prometheus.visitast') +local K = Ast.AstKind + +return function(ast, Parser) + local function parsedFunction(source, parent) + local parsed = Parser:new({LuaVersion = 'LuaU', AllowInternalIdentifiers = true}):parse('return ' .. source) + local result = parsed.body.statements[1].args[1] + result.body.scope:setParent(parent) + return result + end + local function global(scope, name) + local owner, id = scope:resolveGlobal(name) + return Ast.VariableExpression(owner, id) + end + local function reference(scope, owner, id) + if scope ~= owner then scope:addReferenceToHigherScope(owner, id) end + return Ast.VariableExpression(owner, id) + end + local function pack(scope, expressions) + local fn = parsedFunction('function(...) return {n=select("#",...),...} end', scope) + return Ast.FunctionCallExpression(fn, expressions) + end + local function unpackValues(scope, owner, id) + return Ast.FunctionCallExpression(global(scope, 'unpack'), { + reference(scope, owner, id), Ast.NumberExpression(1), + Ast.IndexExpression(reference(scope, owner, id), Ast.StringExpression('n')), + }) + end + + visitast(ast, nil, function(node) + if node.kind ~= K.ForInStatement then return nil end + local scope = Scope:new(node.parentScope) + local iterator, state, control = scope:addVariable(), scope:addVariable(), scope:addVariable() + local action, values = scope:addVariable(), scope:addVariable() + local savedArgs + node.body.scope:setParent(scope) + local wrapper = Ast.TopNode(node.body, ast.globalScope) + visitast(wrapper, function(child, data) + if data.functionData.depth ~= 0 then return nil end + if child.kind == K.VarargExpression then + savedArgs = savedArgs or scope:addVariable() + local result = unpackValues(data.scope, scope, savedArgs) + result.isParenthesizedExpression = child.isParenthesizedExpression + return result, true + elseif child.kind == K.BreakStatement and child.loop == node then + return Ast.ReturnStatement{Ast.NumberExpression(1)}, true + elseif child.kind == K.ContinueStatement and child.loop == node then + return Ast.ReturnStatement{}, true + end + end, function(child, data) + if data.functionData.depth == 0 and child.kind == K.ReturnStatement and not child.iterationControl then + local result = Ast.ReturnStatement{Ast.NumberExpression(2), pack(data.scope, child.args)} + result.iterationControl = true + return result + end + end) + -- Return nodes introduced for break/continue were skipped by previsit. + local args, names = {}, {} + for index, id in ipairs(node.ids) do + args[index] = Ast.VariableExpression(node.scope, id) + names[index] = 'v' .. index + end + local callback = Ast.FunctionLiteralExpression(args, node.body) + local parameters = table.concat(names, ',') + local helper = parsedFunction('function(callback,iterator,state,control) for ' .. parameters + .. ' in iterator,state,control do local action,values=callback(' .. parameters + .. ') if action then return action,values end end end', scope) + helper.isNativeLuau = true + local statements = {} + if savedArgs then + statements[#statements + 1] = Ast.LocalVariableDeclaration(scope, {savedArgs}, {pack(scope, {Ast.VarargExpression()})}) + end + statements[#statements + 1] = Ast.LocalVariableDeclaration(scope, {iterator, state, control}, node.expressions) + statements[#statements + 1] = Ast.LocalVariableDeclaration(scope, {action, values}, { + Ast.FunctionCallExpression(helper, {callback, Ast.VariableExpression(scope, iterator), + Ast.VariableExpression(scope, state), Ast.VariableExpression(scope, control)})}) + local returnScope = Scope:new(scope) + statements[#statements + 1] = Ast.IfStatement(Ast.EqualsExpression(Ast.VariableExpression(scope, action), Ast.NumberExpression(2)), + Ast.Block({Ast.ReturnStatement{unpackValues(returnScope, scope, values)}}, returnScope), {}) + return Ast.DoStatement(Ast.Block(statements, scope)) + end) + return ast +end diff --git a/src/prometheus/luau_types.lua b/src/prometheus/luau_types.lua index f6bb9474..42b3b5bc 100644 --- a/src/prometheus/luau_types.lua +++ b/src/prometheus/luau_types.lua @@ -149,7 +149,7 @@ return function(Parser, token, is, consume, expect, peek, get) expect(self, S, '(') -- Parse the real expression grammar in a detached scope. It is never -- evaluated or added to the runtime AST and its reference graph. - self:expression(Scope:new(Scope:newGlobal())) + self:expression(Scope:new(self.lexicalScope or Scope:newGlobal())) expect(self, S, ')') return TYPE end @@ -277,6 +277,14 @@ return function(Parser, token, is, consume, expect, peek, get) return false end + function Parser:parseLuauInstantiation() + if self.luaVersion ~= 'LuaU' or not is(self, S, '<') or not is(self, S, '<', 1) then return false end + get(self) + parseTypeArguments(self) + expect(self, S, '>') + return true + end + function Parser:parseLuauDeclaration() if self.luaVersion ~= 'LuaU' then return false end local offset = is(self, I, 'export') and 1 or 0 @@ -288,12 +296,15 @@ return function(Parser, token, is, consume, expect, peek, get) expect(self, I) parseGenerics(self, false) expect(self, S, '(') - local scope = Scope:new(Scope:newGlobal()) + local scope = Scope:new(self.lexicalScope or Scope:newGlobal()) + local savedRoot = self.typeFunctionRoot + self.typeFunctionRoot = scope local args = self:functionArgList(scope) expect(self, S, ')') self:parseLuauReturnAnnotation() self:functionBody(scope, args) expect(self, K, 'end') + self.typeFunctionRoot = savedRoot else expect(self, I) parseGenerics(self, true) diff --git a/src/prometheus/parser.lua b/src/prometheus/parser.lua index 1580636a..dc2dfa7f 100644 --- a/src/prometheus/parser.lua +++ b/src/prometheus/parser.lua @@ -16,6 +16,8 @@ local util = require("prometheus.util"); local Ast = require("prometheus.ast"); local Scope = require("prometheus.scope"); local logger = require("logger"); +local lowerExports = require("prometheus.luau_exports"); +local lowerIteration = require("prometheus.luau_iteration"); local AstKind = Ast.AstKind; @@ -134,25 +136,7 @@ local function expect(self, kind, source) end require("prometheus.luau_types")(Parser, TokenKind, is, consume, expect, peek, get); - --- The compilers call iterator functions directly. Normalize Luau's additional --- generalized iteration cases before compiling while keeping Lua 5.1 unchanged. -local iteratorHelper = [=[ -local function %s(iterator, state, control) - if type(iterator) == "function" then return iterator, state, control end - local mt = getmetatable(iterator) - if mt ~= nil and type(mt) ~= "table" then - error("Generalized iteration with a protected metatable is unsupported", 2) - end - if mt then - local iter = rawget(mt, "__iter") - if iter ~= nil then return iter(iterator) end - if rawget(mt, "__call") ~= nil then return iterator, state, control end - end - if type(iterator) == "table" then return next, iterator, nil end - return iterator, state, control -end -]=] +require("prometheus.luau_attributes")(Parser, TokenKind, is, consume, expect, peek, get); local function markConst(scope, id) scope.constBindings = scope.constBindings or {}; @@ -160,61 +144,72 @@ local function markConst(scope, id) end local function assertMutable(self, scope, id) + scope.assignedBindings = scope.assignedBindings or {}; + scope.assignedBindings[id] = true; if scope.constBindings and scope.constBindings[id] then if self.disableLog then error() end logger:error(generateError(self, string.format("Cannot reassign constant '%s'", scope:getVariableName(id)))); end end +-- Match Luau's reachable-branch check for locals skipped by repeat/continue. +local function repeatContinue(block, loop) + for _, node in ipairs(block.statements) do + if node.kind == AstKind.ContinueStatement then + return node.loop == loop and node.sourceToken or nil; + elseif node.kind == AstKind.ReturnStatement or node.kind == AstKind.BreakStatement then + return nil; + elseif node.kind == AstKind.DoStatement then + local found = repeatContinue(node.body, loop); + if found then return found end + elseif node.kind == AstKind.IfStatement then + local branches = {{condition = node.condition, body = node.body}}; + for _, branch in ipairs(node.elseifs) do branches[#branches + 1] = branch end + local remaining = true; + for _, branch in ipairs(branches) do + if remaining and (not branch.condition.isConstant or branch.condition.value) then + local found = repeatContinue(branch.body, loop); + if found then return found end + if branch.condition.isConstant then remaining = false end + end + end + if remaining and node.elsebody then + local found = repeatContinue(node.elsebody, loop); + if found then return found end + end + end + end +end + -- Parse the given code to an Abstract Syntax Tree function Parser:parse(code) -- A failed parse must not leave input behind for the next invocation. self.tokenizer:reset(); self.tokens, self.index, self.length = {}, 0, 0; self.disableLog = false; + self.noSimplify = false; self.varargAllowed = true; - self.luauIteratorName = nil; + self.functionDepth = 0; + self.lexicalScope = nil; + self.typeFunctionRoot = nil; + self.moduleScope = nil; + self.exportBindings, self.exportNames = {}, {}; + self.hasModuleReturn = false; self.tokenizer:append(code); self.tokens = self.tokenizer:scanAll(); - if self.luaVersion == LuaVersion.LuaU and self.lowerLuauIteration then - local hasLoop, names = false, {}; - local function collectTokens(tokens) - for _, tk in ipairs(tokens) do - if tk.kind == TokenKind.Keyword and tk.source == "for" then hasLoop = true end - if tk.kind == TokenKind.Ident then names[tk.source] = true end - if tk.kind == TokenKind.InterpolatedString then - for _, part in ipairs(tk.value) do - if part.tokens then collectTokens(part.tokens) end - end - end - end - end - collectTokens(self.tokens); - if hasLoop then - local i = 0; - repeat - i = i + 1; - self.luauIteratorName = "prometheus_luau_iterator_" .. i; - until not names[self.luauIteratorName]; - - -- Tokenize the helper separately to keep input error locations intact. - local helper = Tokenizer:new({LuaVersion = LuaVersion.LuaU}); - helper:append(string.format(iteratorHelper, self.luauIteratorName)); - local tokens = helper:scanAll(); - tokens[#tokens] = nil; - for _, tk in ipairs(self.tokens) do tokens[#tokens + 1] = tk end - self.tokens = tokens; - end - end self.length = #self.tokens; -- Create Global Variable Scope local globalScope = Scope:newGlobal(); local ast = Ast.TopNode(self:block(globalScope, false), globalScope); + ast.luaVersion = self.luaVersion; + ast.directives = self.luaVersion == LuaVersion.LuaU and self.tokenizer.directives or nil; -- File Must be Over when Top Node is Fully Parsed expect(self, TokenKind.Eof); + if #self.exportBindings > 0 then lowerExports(ast, self.exportBindings) end + if self.luaVersion == LuaVersion.LuaU and self.lowerLuauIteration then lowerIteration(ast, Parser) end logger:debug("Cleaning up Parser for next Use ...") -- Clean Up @@ -231,6 +226,9 @@ end -- Parse a Code Block function Parser:block(parentScope, currentLoop, scope) scope = scope or Scope:new(parentScope); + if not self.moduleScope then self.moduleScope = scope end + local savedScope = self.lexicalScope; + self.lexicalScope = scope; local statements = {}; while true do @@ -244,10 +242,56 @@ function Parser:block(parentScope, currentLoop, scope) if isTerminatingStatement then break end end + self.lexicalScope = savedScope; return Ast.Block(statements, scope); end function Parser:statement(scope, currentLoop) + local attributes = self:parseLuauAttributes(scope); + if attributes then + local node, terminating = self:statement(scope, currentLoop); + if not node or (node.kind ~= AstKind.FunctionDeclaration and node.kind ~= AstKind.LocalFunctionDeclaration) then + logger:error(generateError(self, "Function attributes require a function declaration")); + end + if node.attributes then logger:error(generateError(self, "Duplicate function attribute list")) end + node.attributes = attributes; + return node, terminating; + end + if self.luaVersion == LuaVersion.LuaU and is(self, TokenKind.Ident, "export") + and (is(self, TokenKind.Keyword, "local", 1) or is(self, TokenKind.Keyword, "function", 1) or is(self, TokenKind.Ident, "const", 1)) then + get(self); + if scope ~= self.moduleScope or self.functionDepth ~= 0 then logger:error(generateError(self, "Value exports require a top-level statement")) end + if self.hasModuleReturn then logger:error(generateError(self, "Value exports conflict with a top-level return")) end + local node; + if consume(self, TokenKind.Keyword, "function") then + local ident = expect(self, TokenKind.Ident); + local id = scope:addVariable(ident.value, ident); + markConst(scope, id); + local funcScope = Scope:new(scope); + self:parseLuauGenerics(); + expect(self, TokenKind.Symbol, "("); + local args = self:functionArgList(funcScope); + expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); + local body = self:functionBody(funcScope, args); + expect(self, TokenKind.Keyword, "end"); + node = Ast.LocalFunctionDeclaration(scope, id, args, body); + else + if is(self, TokenKind.Keyword, "function", 1) or is(self, TokenKind.Symbol, "@", 1) then + logger:error(generateError(self, "Use 'export function' for an exported function")); + end + node = self:statement(scope, currentLoop); + if not node or node.kind ~= AstKind.LocalVariableDeclaration then logger:error(generateError(self, "Expected an exported binding")) end + end + node.isExported = true; + for _, id in ipairs(node.ids or {node.id}) do + local name = scope:getVariableName(id); + if self.exportNames[name] then logger:error(generateError(self, "Duplicate exported identifier '" .. name .. "'")) end + self.exportNames[name] = true; + self.exportBindings[#self.exportBindings + 1] = {scope = scope, id = id, name = name}; + end + return node; + end -- Preserve the legacy Lua 5.1 empty-statement behavior. Luau permits only -- one optional semicolon after an actual statement, consumed by block(). if self.luaVersion == LuaVersion.Lua51 then @@ -288,11 +332,30 @@ function Parser:statement(scope, currentLoop) stat.body = self:block(nil, stat, repeatScope); expect(self, TokenKind.Keyword, "until"); stat.condition = self:expression(repeatScope); + stat.continueToken = repeatContinue(stat.body, stat); + if self.luaVersion == LuaVersion.LuaU and stat.continueToken then + local visitast = require("prometheus.visitast"); + visitast(Ast.TopNode(Ast.Block({Ast.ReturnStatement{stat.condition}}, repeatScope), scope), function(node, data) + if node.kind == AstKind.VariableExpression and node.scope == repeatScope then + local declaration = repeatScope.declarationTokens and repeatScope.declarationTokens[node.id]; + local capturedConstant = data.functionData.depth > 0 and repeatScope.constantInitializers + and repeatScope.constantInitializers[node.id] and not (repeatScope.assignedBindings and repeatScope.assignedBindings[node.id]); + if declaration and declaration.startPos > stat.continueToken.startPos and not capturedConstant then + logger:error(generateError(self, "Local '" .. repeatScope:getVariableName(node.id) + .. "' in the repeat condition is skipped by continue")); + end + end + end); + end return stat; end -- Return Statement if(consume(self, TokenKind.Keyword, "return")) then + if self.luaVersion == LuaVersion.LuaU and self.functionDepth == 0 then + self.hasModuleReturn = true; + if #self.exportBindings > 0 then logger:error(generateError(self, "Value exports conflict with a top-level return")) end + end local args = {}; if(not is(self, TokenKind.Keyword, "end") and not is(self, TokenKind.Keyword, "elseif") and not is(self, TokenKind.Keyword, "else") and not is(self, TokenKind.Keyword, "until") and not is(self, TokenKind.Symbol, ";") and not is(self, TokenKind.Eof)) then args = self:exprList(scope); @@ -361,7 +424,7 @@ function Parser:statement(scope, currentLoop) -- 'const' is contextual and remains a valid identifier outside declarations. local isConst = self.luaVersion == LuaVersion.LuaU and is(self, TokenKind.Ident, "const") - and (is(self, TokenKind.Ident, nil, 1) or is(self, TokenKind.Keyword, "function", 1)); + and (is(self, TokenKind.Ident, nil, 1) or is(self, TokenKind.Keyword, "function", 1) or is(self, TokenKind.Symbol, "@", 1)); if consume(self, TokenKind.Keyword, "local") or (isConst and consume(self, TokenKind.Ident, "const")) then -- Local Function Declaration if(consume(self, TokenKind.Keyword, "function")) then @@ -409,6 +472,10 @@ function Parser:statement(scope, currentLoop) end -- Variables can only be reffered to in the next statement, so the id's are enabled after the expressions have been parsed + scope.constantInitializers = scope.constantInitializers or {}; + for i, id in ipairs(ids) do + if expressions[i] and expressions[i].isConstant then scope.constantInitializers[id] = true end + end self:enableNameList(scope, ids); if(#expressions > #ids) then @@ -459,10 +526,6 @@ function Parser:statement(scope, currentLoop) local ids = self:nameList(forScope); expect(self, TokenKind.Keyword, "in"); local expressions = self:exprList(scope); - if self.luauIteratorName then - local helperScope, helperId = scope:resolve(self.luauIteratorName); - expressions = {Ast.FunctionCallExpression(Ast.VariableExpression(helperScope, helperId), expressions)}; - end -- Enable Ids after Expression Parsing so that code like this works: -- local z = {10,20} -- for y,z in ipairs(z) do @@ -507,12 +570,16 @@ function Parser:statement(scope, currentLoop) if self.disableLog then error() end logger:error(generateError(self, "the continue Statement is only valid inside of loops")); end - return Ast.ContinueStatement(currentLoop, scope), true; + local node = Ast.ContinueStatement(currentLoop, scope); + node.sourceToken = peek(self, -1); + return node, true; end if(expr.kind == AstKind.IndexExpression) then + if expr.isParenthesizedExpression then logger:error(generateError(self, "Assigned expression must be a variable or field")) end expr.kind = AstKind.AssignmentIndexing end if(expr.kind == AstKind.VariableExpression) then + if expr.isParenthesizedExpression then logger:error(generateError(self, "Assigned expression must be a variable or field")) end expr.kind = AstKind.AssignmentVariable assertMutable(self, expr.scope, expr.id); end @@ -573,6 +640,7 @@ function Parser:statement(scope, currentLoop) end if(expr.kind == AstKind.IndexExpression or expr.kind == AstKind.VariableExpression) then + if expr.isParenthesizedExpression then logger:error(generateError(self, "Assigned expression must be a variable or field")) end if(expr.kind == AstKind.IndexExpression) then expr.kind = AstKind.AssignmentIndexing end @@ -677,7 +745,11 @@ end -- Expression function Parser:expression(scope) - return self:expressionOr(scope); + local savedScope = self.lexicalScope; + self.lexicalScope = scope; + local result = self:expressionOr(scope); + self.lexicalScope = savedScope; + return result; end function Parser:expressionOr(scope) @@ -685,7 +757,7 @@ function Parser:expressionOr(scope) if(consume(self, TokenKind.Keyword, "or")) then local rhs = self:expressionOr(scope); - return Ast.OrExpression(lhs, rhs, true); + return Ast.OrExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -696,7 +768,7 @@ function Parser:expressionAnd(scope) if(consume(self, TokenKind.Keyword, "and")) then local rhs = self:expressionAnd(scope); - return Ast.AndExpression(lhs, rhs, true); + return Ast.AndExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -708,37 +780,37 @@ function Parser:expressionComparision(scope) local found = false; if(consume(self, TokenKind.Symbol, "<")) then local rhs = self:expressionStrCat(scope); - curr = Ast.LessThanExpression(curr, rhs, true); + curr = Ast.LessThanExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, ">")) then local rhs = self:expressionStrCat(scope); - curr = Ast.GreaterThanExpression(curr, rhs, true); + curr = Ast.GreaterThanExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "<=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.LessThanOrEqualsExpression(curr, rhs, true); + curr = Ast.LessThanOrEqualsExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, ">=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.GreaterThanOrEqualsExpression(curr, rhs, true); + curr = Ast.GreaterThanOrEqualsExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "~=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.NotEqualsExpression(curr, rhs, true); + curr = Ast.NotEqualsExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "==")) then local rhs = self:expressionStrCat(scope); - curr = Ast.EqualsExpression(curr, rhs, true); + curr = Ast.EqualsExpression(curr, rhs, not self.noSimplify); found = true; end until not found; @@ -751,7 +823,7 @@ function Parser:expressionStrCat(scope) if(consume(self, TokenKind.Symbol, "..")) then local rhs = self:expressionStrCat(scope); - return Ast.StrCatExpression(lhs, rhs, true); + return Ast.StrCatExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -764,13 +836,13 @@ function Parser:expressionAddSub(scope) local found = false; if(consume(self, TokenKind.Symbol, "+")) then local rhs = self:expressionMulDivMod(scope); - curr = Ast.AddExpression(curr, rhs, true); + curr = Ast.AddExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "-")) then local rhs = self:expressionMulDivMod(scope); - curr = Ast.SubExpression(curr, rhs, true); + curr = Ast.SubExpression(curr, rhs, not self.noSimplify); found = true; end until not found; @@ -786,25 +858,25 @@ function Parser:expressionMulDivMod(scope) local found = false; if(consume(self, TokenKind.Symbol, "*")) then local rhs = self:expressionUnary(scope); - curr = Ast.MulExpression(curr, rhs, true); + curr = Ast.MulExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "/")) then local rhs = self:expressionUnary(scope); - curr = Ast.DivExpression(curr, rhs, true); + curr = Ast.DivExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "//")) then local rhs = self:expressionUnary(scope); - curr = Ast.FloorDivExpression(curr, rhs, true); + curr = Ast.FloorDivExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "%")) then local rhs = self:expressionUnary(scope); - curr = Ast.ModExpression(curr, rhs, true); + curr = Ast.ModExpression(curr, rhs, not self.noSimplify); found = true; end until not found; @@ -815,17 +887,17 @@ end function Parser:expressionUnary(scope) if(consume(self, TokenKind.Keyword, "not")) then local rhs = self:expressionUnary(scope); - return Ast.NotExpression(rhs, true); + return Ast.NotExpression(rhs, not self.noSimplify); end if(consume(self, TokenKind.Symbol, "#")) then local rhs = self:expressionUnary(scope); - return Ast.LenExpression(rhs, true); + return Ast.LenExpression(rhs, not self.noSimplify); end if(consume(self, TokenKind.Symbol, "-")) then local rhs = self:expressionUnary(scope); - return Ast.NegateExpression(rhs, true); + return Ast.NegateExpression(rhs, not self.noSimplify); end return self:expressionPow(scope); @@ -834,7 +906,7 @@ end function Parser:expressionPow(scope) local lhs = self:tableOrFunctionLiteral(scope); - while self:parseLuauCast() do + if self:parseLuauCast() then if lhs.kind == AstKind.FunctionCallExpression or lhs.kind == AstKind.PassSelfFunctionCallExpression or lhs.kind == AstKind.VarargExpression then lhs.isParenthesizedExpression = true; end @@ -843,7 +915,7 @@ function Parser:expressionPow(scope) if(consume(self, TokenKind.Symbol, "^")) then -- Allow unary operators on the rhs (e.g. 2 ^ #x, 2 ^ -x) while preserving right-associativity. ~ SpinnySpiwal local rhs = self:expressionUnary(scope); - return Ast.PowExpression(lhs, rhs, true); + return Ast.PowExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -851,6 +923,8 @@ end -- Table Literals and Function Literals cannot directly be called or indexed function Parser:tableOrFunctionLiteral(scope) + local attributes = self:parseLuauAttributes(scope); + if attributes then return self:expressionFunctionLiteral(scope, attributes) end if(is(self, TokenKind.Symbol, "{")) then return self:tableConstructor(scope); @@ -863,7 +937,7 @@ function Parser:tableOrFunctionLiteral(scope) return self:expressionFunctionCall(scope); end -function Parser:expressionFunctionLiteral(parentScope) +function Parser:expressionFunctionLiteral(parentScope, attributes) local scope = Scope:new(parentScope); expect(self, TokenKind.Keyword, "function"); @@ -877,7 +951,9 @@ function Parser:expressionFunctionLiteral(parentScope) local body = self:functionBody(scope, args); expect(self, TokenKind.Keyword, "end"); - return Ast.FunctionLiteralExpression(args, body); + local node = Ast.FunctionLiteralExpression(args, body); + node.attributes = attributes; + return node; end -- An inner function never inherits its enclosing function's vararg parameter. @@ -885,7 +961,9 @@ end function Parser:functionBody(scope, args) local savedVararg = self.varargAllowed; self.varargAllowed = #args > 0 and args[#args].kind == AstKind.VarargExpression; + self.functionDepth = self.functionDepth + 1; local body = self:block(nil, false, scope); + self.functionDepth = self.functionDepth - 1; self.varargAllowed = savedVararg; return body; end @@ -941,6 +1019,16 @@ function Parser:expressionFunctionCall(scope, base) return base; end + local instantiated = false; + while self:parseLuauInstantiation() do + instantiated = true; + if base.kind == AstKind.FunctionCallExpression or base.kind == AstKind.PassSelfFunctionCallExpression then + base.isParenthesizedExpression = true; + end + end + if instantiated and (is(self, TokenKind.Symbol, ".") or is(self, TokenKind.Symbol, "[") or is(self, TokenKind.Symbol, ":")) then + return self:expressionFunctionCall(scope, self:expressionIndex(scope, base)); + end checkCallLine(self); -- Normal Function Call local args = {}; @@ -1001,6 +1089,7 @@ function Parser:expressionIndex(scope, base) -- Function Passing self if(consume(self, TokenKind.Symbol, ":")) then local passSelfFunctionName = expect(self, TokenKind.Ident).value; + self:parseLuauInstantiation(); local args = {}; checkCallLine(self); if(is(self, TokenKind.String)) then @@ -1099,6 +1188,7 @@ function Parser:expressionLiteral(scope) end -- Number Literal + if is(self, TokenKind.Integer) then return Ast.IntegerExpression(get(self).value) end if(is(self, TokenKind.Number)) then return Ast.NumberExpression(get(self).value); end @@ -1134,6 +1224,11 @@ function Parser:expressionLiteral(scope) local name = ident.value; local scope, id = scope:resolve(name); + if self.typeFunctionRoot and not scope.isGlobal then + local current = scope; + while current and current ~= self.typeFunctionRoot do current = current.parentScope end + if not current then logger:error(generateError(self, "Type function cannot reference outer local '" .. name .. "'")) end + end return Ast.VariableExpression(scope, id); end @@ -1184,7 +1279,9 @@ function Parser:tableConstructor(scope) local key = Ast.StringExpression(get(self).value); expect(self, TokenKind.Symbol, "="); local value = self:expression(scope); - table.insert(entries, Ast.KeyedTableEntry(key, value)); + local entry = Ast.KeyedTableEntry(key, value); + entry.isRecord = true; + table.insert(entries, entry); else local value = self:expression(scope); table.insert(entries, Ast.TableEntry(value)); diff --git a/src/prometheus/pipeline.lua b/src/prometheus/pipeline.lua index 499c1261..80127494 100644 --- a/src/prometheus/pipeline.lua +++ b/src/prometheus/pipeline.lua @@ -208,6 +208,7 @@ function Pipeline:apply(code, filename) end end local ast = parser:parse(code); + local directives = ast.directives; local parserTimeDiff = gettime() - parserStartTime; logger:info(string.format("Parsing Done in %.2f seconds", parserTimeDiff)); @@ -224,6 +225,7 @@ function Pipeline:apply(code, filename) end -- Rename Variables Step + ast.directives = directives; self:renameVariables(ast); code = self:unparse(ast); diff --git a/src/prometheus/scope.lua b/src/prometheus/scope.lua index 3f01b953..de4cff7d 100644 --- a/src/prometheus/scope.lua +++ b/src/prometheus/scope.lua @@ -93,6 +93,10 @@ function Scope:addVariable(name, token) table.insert(self.variables, name); local id = #self.variables; self.variablesLookup[name] = id; + if token then + self.declarationTokens = self.declarationTokens or {}; + self.declarationTokens[id] = token; + end return id; end @@ -119,6 +123,10 @@ function Scope:addDisabledVariable(name, token) table.insert(self.variables, name); local id = #self.variables; + if token then + self.declarationTokens = self.declarationTokens or {}; + self.declarationTokens[id] = token; + end return id; end @@ -331,4 +339,4 @@ function Scope:renameVariables(settings) end end -return Scope; \ No newline at end of file +return Scope; diff --git a/src/prometheus/steps/Vmify.lua b/src/prometheus/steps/Vmify.lua index 05a14cd4..b2affb3b 100644 --- a/src/prometheus/steps/Vmify.lua +++ b/src/prometheus/steps/Vmify.lua @@ -10,8 +10,8 @@ Vmify.SettingsDescriptor = {} function Vmify:init() end -function Vmify:apply(ast) - return Emit.ast(Compiler:compile(ast)) +function Vmify:apply(ast, pipeline) + return Emit.ast(Compiler:compile(ast, pipeline and pipeline:getLuaVersion())) end return Vmify diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index cc89c04d..7a362bac 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -33,6 +33,7 @@ Tokenizer.TokenKind = { Symbol = "Symbol", Ident = "Identifier", Number = "Number", + Integer = "Integer", String = "String", InterpolatedString = "InterpolatedString", } @@ -46,6 +47,7 @@ Tokenizer.EOF_TOKEN = { } local function token(self, startPos, kind, value) + if kind ~= Tokenizer.TokenKind.Eof then self.hasTokens = true end local line, linePos = self:getPosition(self.index); local annotations = self.annotations self.annotations = {}; @@ -71,35 +73,29 @@ local function generateWarning(token, message) end function Tokenizer:getPosition(i) - if i <= 0 or self.length == 0 then return 1, 1 end - local column = self.columnMap[i] - - if not column then --// `i` is bigger than self.length, this shouldn't happen, but it did. (Theres probably some error in the tokenizer, cant find it.) - column = self.columnMap[#self.columnMap] - end - - return column.id, column.charMap[i] or column.length + 1 + i = math.max(1, i); + local starts = self.lineStarts or {1}; + local first, last = 1, #starts; + while first < last do + local middle = math.floor((first + last + 1) / 2); + if starts[middle] <= i then first = middle else last = middle - 1 end + end + return first, i - starts[first] + 1; end --// Prepare columnMap for getPosition function Tokenizer:prepareGetPosition() - local columnMap, column = {}, { charMap = {}, id = 1, length = 0 } - - for index = 1, self.length do - local character = string.sub(self.source, index, index) -- NOTE_1: this could use table.clone to reduce amount of NEWTABLE (if that causes any performance issues) - - local columnLength = column.length + 1 - column.length = columnLength - column.charMap[index] = columnLength - - if character == "\n" then - column = { charMap = {}, id = column.id + 1, length = 0 } -- NOTE_1 + local starts, index = {1}, 1; + while index <= self.length do + local char = self.source:sub(index, index); + if char == "\n" or char == "\r" then + local nextChar = self.source:sub(index + 1, index + 1); + if (char == "\n" and nextChar == "\r") or (char == "\r" and nextChar == "\n") then index = index + 1 end + starts[#starts + 1] = index + 1; end - - columnMap[index] = column + index = index + 1; end - - self.columnMap = columnMap + self.lineStarts = starts; end -- Constructor for Tokenizer @@ -163,7 +159,9 @@ function Tokenizer:reset() self.length = 0; self.source = ""; self.annotations = {}; - self.columnMap = {}; + self.lineStarts = {1}; + self.hasTokens = false; + self.directives = {}; end -- Append String to this Tokenizer @@ -241,6 +239,7 @@ end -- skip one or 0 Comments and return wether one was found function Tokenizer:skipComment() if(is(self, "-", 0) and is(self, "-", 1)) then + local commentStart = self.index; self.index = self.index + 2; if(is(self, "[")) then self.index = self.index + 1; @@ -271,7 +270,14 @@ function Tokenizer:skipComment() end -- Single Line Comment -- Get all Chars to next Newline - while(self.index < self.length and self:parseAnnotation() ~= "\n") do end + while self.index < self.length and not is(self, "\n") and not is(self, "\r") do self:parseAnnotation() end + if not self.hasTokens then + local comment = self.source:sub(commentStart + 1, self.index); + if comment:match("^%-%-!native%s*$") or comment:match("^%-%-!optimize%s+[012]%s*$") then + self.directives = self.directives or {}; + self.directives[#self.directives + 1] = comment; + end + end return true; end return false; @@ -330,6 +336,23 @@ function Tokenizer:luauNumber() local source = self.source:sub(startPos + 1, self.index):gsub("_", ""); local prefix = source:sub(1, 2):lower(); + if source:sub(-1) == "i" then + local digits = source:sub(1, -2); + -- Radix integer literals are unsigned bit patterns; decimals are signed. + local valid; + if prefix == "0x" then + digits = source:sub(3, -2); + valid = digits:match("^[0-9a-fA-F]+$") and #digits:gsub("^0+", "") <= 16; + elseif prefix == "0b" then + digits = source:sub(3, -2); + valid = digits:match("^[01]+$") and #digits:gsub("^0+", "") <= 64; + else + local significant = digits:gsub("^0+", ""); + valid = digits:match("^%d+$") and (#significant < 19 or (#significant == 19 and significant <= "9223372036854775807")); + end + if not valid then logger:error(generateError(self, "Malformed or overflowing integer '" .. source .. "'")) end + return token(self, startPos, Tokenizer.TokenKind.Integer, source); + end local value; if prefix == "0x" or prefix == "0b" then local digits = source:sub(3); @@ -433,7 +456,7 @@ function Tokenizer:singleLineString() local char = get(self); -- Single Line String may not contain Linebreaks except when they are escaped by \ - if(char == '\n') then + if(char == '\n' or char == '\r') then self.index = self.index - 1; logger:error(generateError(self, "Unterminated String")); end @@ -459,7 +482,9 @@ function Tokenizer:singleLineString() numstr = numstr .. char; end - char = string.char(tonumber(numstr)); + local value = tonumber(numstr); + if value > 255 then logger:error(generateError(self, "Decimal escape exceeds 255")) end + char = string.char(value); elseif(self.UnicodeEscapes and char == "u") then expect(self, "{"); @@ -468,7 +493,9 @@ function Tokenizer:singleLineString() num = num .. get(self); end expect(self, "}"); - char = util.utf8char(tonumber(num, 16)); + local value = tonumber(num, 16); + if not value or value > 0x10ffff then logger:error(generateError(self, "Invalid Unicode escape")) end + char = util.utf8char(value); elseif(self.HexEscapes and char == "x") then local hex = expect(self, self.HexNumberCharsLookup) .. expect(self, self.HexNumberCharsLookup); char = string.char(tonumber(hex, 16)); @@ -477,6 +504,12 @@ function Tokenizer:singleLineString() while(is(self, Tokenizer.WHITESPACE_CHARS)) do self.index = self.index + 1; end + elseif char == "\n" or char == "\r" then + local nextChar = peek(self); + if (char == "\r" and nextChar == "\n") or (self.luaVersion == LuaVersion.Lua51 and char == "\n" and nextChar == "\r") then + self.index = self.index + 1; + end + char = "\n"; end end @@ -534,7 +567,7 @@ function Tokenizer:interpolatedString() end if char == "\n" or char == "\r" then local nextChar = peek(self); - if (char == "\n" and nextChar == "\r") or (char == "\r" and nextChar == "\n") then + if char == "\r" and nextChar == "\n" then self.index = self.index + 1; end return "\n"; diff --git a/src/prometheus/unparser.lua b/src/prometheus/unparser.lua index 0c2046b0..8b8708af 100644 --- a/src/prometheus/unparser.lua +++ b/src/prometheus/unparser.lua @@ -107,7 +107,11 @@ function Unparser:unparse(ast) logger:error("Unparser:unparse expects a TopNode as first argument") end - return self:unparseBlock(ast.body); + local source = self:unparseBlock(ast.body); + if self.luaVersion == LuaVersion.LuaU and ast.directives and #ast.directives > 0 then + source = table.concat(ast.directives, "\n") .. "\n" .. source; + end + return source; end -- Helper to join parts table (optimized string building) @@ -383,7 +387,8 @@ function Unparser:unparseStatement(statement, tabbing) end end - return self:tabs(tabbing, false) .. joinParts(parts); + local attributes = statement.attributes and (table.concat(statement.attributes, " ") .. " ") or ""; + return self:tabs(tabbing, false) .. attributes .. joinParts(parts); end function Unparser:unparseExpression(expression, tabbing) @@ -407,6 +412,10 @@ function Unparser:unparseExpression(expression, tabbing) return expression.value and "true" or "false"; end + if expression.kind == AstKind.IntegerExpression then + if self.luaVersion ~= LuaVersion.LuaU then logger:error("Integer literals require a recent Luau runtime") end + return expression.value; + end if(expression.kind == AstKind.NumberExpression) then local value = expression.value; local str = tostring(value); @@ -637,6 +646,7 @@ function Unparser:unparseExpression(expression, tabbing) k = AstKind.FunctionLiteralExpression; if(expression.kind == k) then + if expression.attributes then push(table.concat(expression.attributes, " "), " ") end push("function", "("); for i, arg in ipairs(expression.args) do if i > 1 then diff --git a/src/prometheus/vm/compiler.lua b/src/prometheus/vm/compiler.lua index ec85b9fc..089cedda 100644 --- a/src/prometheus/vm/compiler.lua +++ b/src/prometheus/vm/compiler.lua @@ -5,9 +5,10 @@ local optimize = require('prometheus.vm.optimize') local Compiler = {} Compiler.__index = Compiler -function Compiler:new(analysis, node, parent) +function Compiler:new(analysis, node, parent, luaVersion) local compiler = setmetatable({ analysis = analysis, owner = node.body.scope, parent = parent, + luaVersion = parent and parent.luaVersion or luaVersion or node.luaVersion or 'Lua51', proto = {code = {}, constants = {}, children = {}, captures = {}, params = {}, vararg = node.kind == 'TopNode', maxRegister = 0}, locals = {}, upvalues = {}, constantIds = {}, registers = {}, freeRegisters = {}, @@ -161,8 +162,8 @@ function Compiler:functionLiteral(node) return register end -function Compiler:compile(ast) - local compiler = Compiler:new(analyze(ast), ast) +function Compiler:compile(ast, luaVersion) + local compiler = Compiler:new(analyze(ast), ast, nil, luaVersion) return optimize(compiler:finish(ast)) end diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua index f1310659..84b8bbff 100644 --- a/src/prometheus/vm/emit.lua +++ b/src/prometheus/vm/emit.lua @@ -8,6 +8,14 @@ local Layout = require('prometheus.vm.layout') local opcodes = require('prometheus.vm.opcodes') local Emit = {} +local function usesNativeLuau(proto) + for _, value in ipairs(proto.constants) do + if type(value) == 'table' and (value.isNativeLuau or value.kind == Ast.AstKind.IntegerExpression) then return true end + end + for _, child in ipairs(proto.children) do if usesNativeLuau(child) then return true end end + return false +end + local function usesFloorDivision(proto) for _, instruction in ipairs(proto.code) do if instruction[1] == opcodes.IDIV then return true end @@ -20,8 +28,9 @@ end function Emit.source(proto, layout) layout = layout or Layout.new() - local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.LuaU}) local function literal(value) + if type(value) == 'table' and (value.isNativeLuau or value.kind == Ast.AstKind.IntegerExpression) then return unparser:unparseExpression(value, 0) end if type(value) ~= 'table' then return unparser:unparseExpression(Ast.ConstantNode(value), 0) end @@ -53,10 +62,16 @@ function Emit.source(proto, layout) end function Emit.ast(proto) - local version = usesFloorDivision(proto) and Enums.LuaVersion.LuaU or Enums.LuaVersion.Lua51 + local version = (usesFloorDivision(proto) or usesNativeLuau(proto)) and Enums.LuaVersion.LuaU or Enums.LuaVersion.Lua51 local ast = Parser:new({LuaVersion = version}):parse(Emit.source(proto)) ast.vmRuntime = ast.body.statements[1].args[1].base ast.vmData = ast.body.statements[1].args[1].args[1] + if usesNativeLuau(proto) then + local visitast = require('prometheus.visitast') + visitast(Ast.TopNode(Ast.Block({Ast.ReturnStatement{ast.vmData}}, ast.body.scope), ast.globalScope), function(node) + if node.kind == Ast.AstKind.FunctionLiteralExpression then node.isNativeLuau = true; return nil, true end + end) + end return ast end diff --git a/src/prometheus/vm/expressions.lua b/src/prometheus/vm/expressions.lua index b59489fa..ad470668 100644 --- a/src/prometheus/vm/expressions.lua +++ b/src/prometheus/vm/expressions.lua @@ -80,6 +80,8 @@ local function install(Compiler) local reg = self:alloc(); self:emit("NIL", reg); return reg elseif kind == Kind.NumberExpression then return load(self, numberValue(node.value)) + elseif kind == Kind.IntegerExpression then + return load(self, node) elseif kind == Kind.BooleanExpression or kind == Kind.StringExpression then return load(self, node.value) elseif kind == Kind.VariableExpression then @@ -89,14 +91,18 @@ local function install(Compiler) local reg = self:alloc(); self:emit("TAKE", reg, tuple, 1) self:free(tuple); return reg elseif kind == Kind.IndexExpression then - local base = self:expression(node.base, 1)[1] + local base + local currentLocal = self.luaVersion == 'LuaU' and node.base.kind == Kind.VariableExpression + and not node.base.scope.isGlobal and self.analysis.owners[node.base.scope] == self.owner + if not currentLocal then base = self:expression(node.base, 1)[1] end local key = self:expression(node.index, 1)[1] + if currentLocal then base = self:expression(node.base, 1)[1] end local result = self:alloc(); self:emit("GETTABLE", result, base, key) self:free(key); self:free(base); return result elseif binary[kind] then local left local lhs = node.lhs - local deferRead = kind == Kind.FloorDivExpression and lhs.kind == Kind.VariableExpression + local deferRead = self.luaVersion == 'LuaU' and kind ~= Kind.StrCatExpression and lhs.kind == Kind.VariableExpression and not lhs.scope.isGlobal and self.analysis.owners[lhs.scope] == self.owner if not deferRead then left = self:expression(lhs, 1)[1] end local right = self:expression(node.rhs, 1)[1] @@ -148,6 +154,7 @@ local function install(Compiler) for _, jump in ipairs(endJumps) do self:patch(jump, 1, finish) end return result elseif kind == Kind.FunctionLiteralExpression then + if node.isNativeLuau then return load(self, node) end return self:functionLiteral(node) elseif kind == Kind.TableConstructorExpression then return compileTable(self, node) diff --git a/src/prometheus/vm/expressions_tables.lua b/src/prometheus/vm/expressions_tables.lua index de8b3a9c..429f8859 100644 --- a/src/prometheus/vm/expressions_tables.lua +++ b/src/prometheus/vm/expressions_tables.lua @@ -56,8 +56,13 @@ return function(self, node) end end else - local key = self:expression(entry.key, 1)[1] + local key + local keyNode = entry.key + local deferRead = self.luaVersion == 'LuaU' and keyNode.kind == Kind.VariableExpression + and not keyNode.scope.isGlobal and self.analysis.owners[keyNode.scope] == self.owner + if not deferRead then key = self:expression(keyNode, 1)[1] end local value = self:expression(entry.value, 1)[1] + if deferRead then key = self:expression(keyNode, 1)[1] end self:emit("SETTABLE", tableReg, key, value) self:free(value) self:free(key) diff --git a/tests/harness/luau-recent.py b/tests/harness/luau-recent.py new file mode 100644 index 00000000..4f4d27be --- /dev/null +++ b/tests/harness/luau-recent.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Validate recent Luau values using a Lua 5.1 or LuaJIT obfuscator host.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +luau = os.environ.get('LUAU', 'luau') +names = ('Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong') + + +def run(args): + result = subprocess.run(args, cwd=root, capture_output=True, text=True, timeout=60) + assert result.returncode == 0, (args, result.stdout, result.stderr) + return result.stdout + + +def execute(source, fixture, runner): + delimiter = '=' + while ']' + delimiter + ']' in source: + delimiter += '=' + quoted = '[' + delimiter + '[' + source + ']' + delimiter + ']' + checks = '' + if fixture.stem == 'exports': + checks = ''' +assert(type(result) == 'table' and table.isfrozen(result)) +assert(result.counter == 3 and result.fixed == 2 and result.optional == nil) +local first, second = result.get() +assert(first == 3 and second == 2) +assert(not pcall(result.increment)) +assert(not pcall(function() result.fixed = 9 end)) +print('Value exports, module return, and freezing checks passed') +''' + runner.write_text('local result = assert(loadstring(' + quoted + '))()\n' + checks) + return run([luau, str(runner)]) + + +count = 0 +with tempfile.TemporaryDirectory(prefix='prometheus-recent-luau-') as directory: + directory = Path(directory) + output, runner = directory / 'output.luau', directory / 'run.luau' + for fixture in sorted((root / 'tests/luau/recent').glob('*.luau')): + expected = execute(fixture.read_text(), fixture, runner) + for index, name in enumerate(names, 1): + for seed in (1, 17): + run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', + str(index), str(seed), str(fixture), str(output)]) + actual = execute(output.read_text(), fixture, runner) + assert actual == expected, (fixture.name, name, seed, expected, actual) + count += 1 + print(f'{fixture.name}: all presets passed', flush=True) +print(f'{count} recent Luau transformations matched their baseline') diff --git a/tests/harness/luau-strings.lua b/tests/harness/luau-strings.lua new file mode 100644 index 00000000..67dc3b3a --- /dev/null +++ b/tests/harness/luau-strings.lua @@ -0,0 +1,34 @@ +local Prometheus = require('src.prometheus') +local Parser = require('prometheus.parser') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +Prometheus.Logger.errorCallback = function(message) error(message, 0) end + +for _, version in ipairs({'Lua51', 'LuaU'}) do + for _, ending in ipairs({'\n', '\r', '\r\n', '\n\r'}) do + for _, steps in ipairs({{}, {{Name = 'Cff'}}, {{Name = 'Vmify'}}}) do + local pipeline = Prometheus.Pipeline:fromConfig({LuaVersion = version, Seed = 17, Steps = steps}) + local source = 'return "a\\' .. ending .. 'b"' + if version == 'LuaU' and ending == '\n\r' then + assert(not pcall(function() pipeline:apply(source) end)) + else + assert(assert(loadstring(pipeline:apply(source)))() == 'a\nb') + end + local source = '-- comment' .. ending .. 'return 42' + assert(assert(loadstring(pipeline:apply(source)))() == 42) + end + local parser = Parser:new({LuaVersion = version}) + assert(not pcall(function() parser:parse('return "a' .. ending .. 'b"') end)) + local ok, message = pcall(function() parser:parse('-- comment' .. ending .. 'local x = "\\u{110000}"') end) + assert(not ok and tostring(message):find('Position 2:', 1, true)) + assert(parser:parse('return "valid"')) + end +end +for _, name in ipairs({'Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong'}) do + local config = {} + for key, value in pairs(Prometheus.Presets[name]) do config[key] = value end + config.LuaVersion, config.Seed = 'LuaU', 17 + local output = Prometheus.Pipeline:fromConfig(config):apply('--!native\r\n--!optimize 1\nreturn 42') + local prefix = '--!native\n--!optimize 1\n' + assert(output:sub(1, #prefix) == prefix) +end +print('String normalization, diagnostics, and directive checks passed') diff --git a/tests/harness/luau-syntax.py b/tests/harness/luau-syntax.py index 3086efbf..4a9b019c 100644 --- a/tests/harness/luau-syntax.py +++ b/tests/harness/luau-syntax.py @@ -228,6 +228,66 @@ def add(label, source, expected=None): add(f'floor division: {source}', source) +# Attributes, explicit instantiations, and newer value syntax share runtime names. +for attribute in ( + '@native', '@checked', '@deprecated', '@[native]', '@[native, checked]', + '@[deprecated({use = "replacement", reason = "old"})]', + '@[native(1)]', '@[native("text", true, nil, {})]', + '@[deprecated({})]', '@[deprecated("text")]', '@[deprecated(1)]', + '@[deprecated({["use"] = "replacement"})]', '@[deprecated({bad = "x"})]', + '@[deprecated({use = 1})]', '@[native(1 + 2)]', '@[native(-1)]', + '@[native((1))]', '@[native(function() end)]', '@[native(value)]', + '@[native({field = value})]', '@[native(1i)]', '@[native(-1i)]', + '@[native, native]', '@native @native', '@[]', '@[native,]', '@unknown', '@ native', +): + for template in ('%s local function f() end', '%s function f() end', + 'local f = %s function() end', 'local %s function f() end', '%s local x = 1'): + add(f'attribute: {attribute}: {template}', template % attribute) +for args in ('number', 'number, string', '', 'number,', '...number', 'T...', + '(number, string)', 'number | string', 'typeof(1)', 'number = string', '|& T'): + for template in ('return f<<%s>>(1)', 'return object:method<<%s>>(1)', + 'return f<<%s>>', 'return f<<%s>>.field', 'return f<<%s>>[1]', + 'return f<<%s>><>(1)'): + add(f'explicit instantiation: {template % args}', template % args) + +for literal in ('0i', '01i', '1_000i', '9223372036854775807i', '9223372036854775808i', + '-9223372036854775808i', '0xffffffffffffffffi', '0x10000000000000000i', + '0b' + '1' * 64 + 'i', '0b' + '1' * 65 + 'i', '0x0001i', + '0bi', '0xi', '1.0i', '1e2i', '.1i', '1ii', '0b2i'): + add(f'integer literal: {literal}', 'return ' + literal) +for declaration in ('export local x = 1', 'export const x = 1', 'export function f() end', + 'export local x', 'export local x, y = 1, 2', + 'export local function f() end', 'export const function f() end'): + for template in ('%s', 'do %s end', 'local function f() %s end', + '%s; return 1', 'do return 1 end; %s', '@native %s'): + add(f'value exports: {template % declaration}', template % declaration) +for source in ( + 'export local x = 1; export local x = 2', + 'export const x = 1; x = 2', 'export function f() end; f = 1', + 'export local x = 1; function x() end', + 'const x = 1; type T = typeof(function() x = 2 end)', + 'local x = 1; type function f() return x end', + 'type function f(x) return function() return x end end', + 'local x = 1; return x :: number :: number', + 'local x = 1; return (x :: number) :: number', + 'local x = 1; (x) = 2', 'local t = {}; (t.x) = 2', + 'local t = {}; (t).x = 2', 'local x, y = 1, 2; x, (y) = 3, 4', +): + add(f'scope or lvalue validation: {source}', source) +for branch in ('if flag then continue end', 'if false then continue end', + 'if true then continue end', 'do continue end', + 'if false then continue elseif flag then continue end', + 'if true then print(1) else continue end'): + for condition in ('value', '(function() return value end)()', 'true'): + add(f'repeat continue: {branch}: {condition}', f'repeat {branch} local value = true until {condition}') +for ending in ('\n', '\r', '\r\n', '\n\r'): + add(f'raw newline {ending!r}', 'return "a' + ending + 'b"') + add(f'escaped newline {ending!r}', 'return "a\\' + ending + 'b"') + add(f'comment newline {ending!r}', '-- comment' + ending + 'return 1') +for escape in (r'\u{}', r'\u{110000}', r'\u{D800}', r'\u{10FFFF}', r'\255', r'\256', r'\xFG', r'\q'): + add('string escape ' + escape, 'return "' + escape + '"') + + def official(path): result = subprocess.run([compiler, '--binary', str(path)], cwd=root, stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, diff --git a/tests/harness/luau.py b/tests/harness/luau.py index c8be33c2..781248f4 100644 --- a/tests/harness/luau.py +++ b/tests/harness/luau.py @@ -40,13 +40,4 @@ def run(args): capture_output=True, text=True, timeout=60) assert baseline.returncode != 0 and transformed.returncode != 0, invalid.name print('Invalid Luau examples were rejected', flush=True) - fixture = root / 'tests/luau/unsupported/protected-metatable.luau' - assert run([luau, str(fixture)]) == '1\n' - for index in (2, 3): - run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', - str(index), '1', str(fixture), output]) - result = subprocess.run([luau, output], cwd=root, stdout=subprocess.PIPE, - stderr=subprocess.PIPE, text=True, timeout=60) - assert result.returncode != 0 and 'protected metatable' in result.stderr print(f'{count} Luau transformations matched their baseline') -print('Protected metatables are rejected by Cff and Vmify') diff --git a/tests/luau/attributes-generics.luau b/tests/luau/attributes-generics.luau new file mode 100644 index 00000000..4e7049a0 --- /dev/null +++ b/tests/luau/attributes-generics.luau @@ -0,0 +1,18 @@ +--!optimize 1 +@native +local function identity(value: T): T return value end +@checked local function add(value: number): number return value + 1 end +@[native, deprecated] +local function old(value) return value end +local anonymous = @native function(value) return value * 2 end +local object = {} +function object:identity(value: T): T return value end +assert(identity<>(7) == 7) +assert(object:identity<>('value') == 'value') +local instantiated = identity<> +assert(instantiated(9) == 9) +local function factory() return identity, 'discarded' end +assert((factory()<>)(11) == 11) +assert(select('#', factory()<>) == 1) +assert(add(4) == 5 and old(3) == 3 and anonymous(5) == 10) +print('Attribute and explicit generic checks passed') diff --git a/tests/luau/evaluation-order.luau b/tests/luau/evaluation-order.luau new file mode 100644 index 00000000..18c9630c --- /dev/null +++ b/tests/luau/evaluation-order.luau @@ -0,0 +1,36 @@ +local value = 20 +local function rhs() value = 100; return 50 end +assert(value + rhs() == 150) +value = 20; assert(value - rhs() == 50) +value = 20; assert(value * rhs() == 5000) +value = 20; assert(value / rhs() == 2) +value = 20; assert(value // rhs() == 2) +value = 20; assert(value % rhs() == 0) +value = 20; assert(value ^ rhs() == 1e100) +value = 20; assert(value .. rhs() == '2050') +value = 20; assert(not (value < rhs())) +value = 20; assert(not (value <= rhs())) +value = 20; assert(value > rhs()) +value = 20; assert(value >= rhs()) +local function equalRhs() value = 100; return 100 end +value = 20; assert(value == equalRhs()) +value = 20; assert(not (value ~= equalRhs())) + +local function upvalue() + value = 20 + return value + rhs() +end +assert(upvalue() == 70) +evaluationOrderGlobal = 20 +local function globalRhs() evaluationOrderGlobal = 100; return 50 end +assert(evaluationOrderGlobal + globalRhs() == 70) + +local first, second = {key = 1}, {key = 2} +local object = first +local function key() object = second; return 'key' end +assert(object[key()] == 2) +local field = 'before' +local function fieldValue() field = 'after'; return 3 end +local constructed = {[field] = fieldValue()} +assert(constructed.before == nil and constructed.after == 3) +print('Luau evaluation order checks passed') diff --git a/tests/luau/invalid/attribute-duplicate.luau b/tests/luau/invalid/attribute-duplicate.luau new file mode 100644 index 00000000..26094cb1 --- /dev/null +++ b/tests/luau/invalid/attribute-duplicate.luau @@ -0,0 +1,2 @@ +@native @native +local function invalid() end diff --git a/tests/luau/invalid/const-typeof.luau b/tests/luau/invalid/const-typeof.luau new file mode 100644 index 00000000..85c0184e --- /dev/null +++ b/tests/luau/invalid/const-typeof.luau @@ -0,0 +1,2 @@ +const value = 1 +type Invalid = typeof(function() value = 2 end) diff --git a/tests/luau/invalid/generic-instantiation.luau b/tests/luau/invalid/generic-instantiation.luau new file mode 100644 index 00000000..bd6ba528 --- /dev/null +++ b/tests/luau/invalid/generic-instantiation.luau @@ -0,0 +1,2 @@ +local function identity(value) return value end +return identity<>(1) diff --git a/tests/luau/invalid/parenthesized-assignment.luau b/tests/luau/invalid/parenthesized-assignment.luau new file mode 100644 index 00000000..2c4bf4c5 --- /dev/null +++ b/tests/luau/invalid/parenthesized-assignment.luau @@ -0,0 +1,2 @@ +local value = 1 +(value) = 2 diff --git a/tests/luau/invalid/repeat-continue-local.luau b/tests/luau/invalid/repeat-continue-local.luau new file mode 100644 index 00000000..ecccbfb8 --- /dev/null +++ b/tests/luau/invalid/repeat-continue-local.luau @@ -0,0 +1,4 @@ +repeat + if flag then continue end + local value = true +until value diff --git a/tests/luau/invalid/string-unicode-range.luau b/tests/luau/invalid/string-unicode-range.luau new file mode 100644 index 00000000..20296c9a --- /dev/null +++ b/tests/luau/invalid/string-unicode-range.luau @@ -0,0 +1 @@ +return "\u{110000}" diff --git a/tests/luau/invalid/type-cast-chain.luau b/tests/luau/invalid/type-cast-chain.luau new file mode 100644 index 00000000..8fa0fd64 --- /dev/null +++ b/tests/luau/invalid/type-cast-chain.luau @@ -0,0 +1,2 @@ +local value = 1 +return value :: number :: number diff --git a/tests/luau/protected-metatable.luau b/tests/luau/protected-metatable.luau new file mode 100644 index 00000000..44caa9eb --- /dev/null +++ b/tests/luau/protected-metatable.luau @@ -0,0 +1,56 @@ +local t = setmetatable({1}, { + __metatable = 'locked', + __iter = function(self) return next, self end, +}) +for _, value in t do print(value) end + +local iterator = setmetatable({}, { + __metatable = {}, + __iter = function() + local i = 0 + return function() + i += 1 + if i <= 3 then return i, i * 10, nil end + end + end, +}) +local sum = 0 +for i, value in iterator do + if i == 2 then continue end + sum += value +end +assert(sum == 40) +local function returning(...) + for i, value in iterator do + if i == 2 then return ..., value, nil end + end +end +local result = table.pack(returning(5, 6)) +assert(result.n == 3 and result[1] == 5 and result[2] == 20 and result[3] == nil) +local count = 0 +for _ in iterator do count += 1; break end +assert(count == 1) +local function nested() + for i in iterator do + for j in iterator do + if j == 2 then return i, j end + end + end +end +local a, b = nested() +assert(a == 1 and b == 2) +local caller +local co = coroutine.create(function() + caller = coroutine.running() + for i in iterator do + assert(coroutine.running() == caller) + coroutine.yield(i) + end +end) +for expected = 1, 3 do + local ok, value = coroutine.resume(co) + assert(ok and value == expected) +end +assert(coroutine.resume(co)) +assert(coroutine.status(co) == 'dead') +print('Protected iterator and callback control checks passed') diff --git a/tests/luau/recent/exports.luau b/tests/luau/recent/exports.luau new file mode 100644 index 00000000..ab720571 --- /dev/null +++ b/tests/luau/recent/exports.luau @@ -0,0 +1,9 @@ +export local counter = 1 +export const fixed = 2 +export local optional +export function get() return counter, fixed end +export function increment() counter += 1 end +counter += 2 +assert(counter == 3) +local first, second = get() +assert(first == 3 and second == 2) diff --git a/tests/luau/recent/integers.luau b/tests/luau/recent/integers.luau new file mode 100644 index 00000000..61e6a00c --- /dev/null +++ b/tests/luau/recent/integers.luau @@ -0,0 +1,14 @@ +-- Integer literals require a runtime that supports Luau's integer type. +local exact = 9007199254740993i +assert(type(exact) == 'integer' and exact == 0x20000000000001i) +assert(exact ~= 9007199254740992i) +assert(0xffffffffffffffffi == -1i) +assert(-0xffffffffffffffffi == 1i) +assert(-0x8000000000000000i == 0x8000000000000000i) +assert(0b1111i == 15i and 1_000i == 1000i) +local values = {[exact] = 'exact', [9007199254740992i] = 'other'} +assert(values[exact] == 'exact') +local function valuesFunction() return exact, nil, 9223372036854775807i end +local a, b, c = valuesFunction() +assert(a == exact and b == nil and c == 0x7fffffffffffffffi) +print('Integer literal precision and sign checks passed')