diff --git a/.dockerignore b/.dockerignore index 238021d..9869235 100644 --- a/.dockerignore +++ b/.dockerignore @@ -16,3 +16,6 @@ Dockerfile test-results __pycache__ + +# Local reference implementation +.reference-obfuscator/ diff --git a/.gitignore b/.gitignore index f26c127..e520251 100644 --- a/.gitignore +++ b/.gitignore @@ -23,10 +23,33 @@ web/playwright-report .vite # Ignore Local Test Files -test.lua -test.obfuscated.lua -config.lua +/test.lua +/test.obfuscated.lua +/config.lua # Lua test logs and reproduction artifacts /test-results/ __pycache__/ + +# Local reference implementation, not part of Prometheus +/.reference-obfuscator/ + +# Local runtimes, uploaded archives, and generated debugging output +/.tools/ +/scratch/ +/Prometheus*.zip +/src.zip +/lune +/lune-*.zip +/luau +/luau-compile +/luau-*.zip +/LuaJIT-*/ +/luajit-source.tar.gz +*.obfuscated.lua +*.obfuscated.luau +*.out.luau +*.log +*.tmp +.DS_Store +Thumbs.db diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c294b84..0beeecd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,7 +12,10 @@ Thanks for contributing to Prometheus. ## Running Tests -Tests run inside a Docker container with lua5.1 and Luau: +The obfuscator source must remain valid Lua 5.1 and run in both Lua 5.1 and +LuaJIT. `LuaVersion` selects the input/output dialect; Luau is a target runtime, +not a dependency of the obfuscator host. Tests run inside a Docker container +with Lua 5.1, LuaJIT, and the official Luau runtime: ```bash ./scripts/run-tests.sh # Run all tests (default: 10 iterations) @@ -22,6 +25,19 @@ Tests run inside a Docker container with lua5.1 and Luau: ./scripts/run-tests.sh -v # Verbose output ``` +The Docker build runs Luau syntax and execution checks with both Lua 5.1 and +LuaJIT as the obfuscator host. The same checks can be run directly: + +```bash +LUA51=luajit python3 tests/harness/luau-syntax.py +LUA51=luajit python3 tests/harness/luau.py +LUA51=luajit python3 tests/harness/luau-recent.py +luajit tests/harness/luau-strings.lua +``` + +`LUAU` and `LUAU_COMPILE` select the target runtime and compiler. The recent +syntax checks require a Luau version with value exports and integer literals. + ### Test File Metadata Test files can include metadata comments at the top of the file: diff --git a/Dockerfile b/Dockerfile index c0057c9..34fc362 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM ubuntu:24.04 RUN apt-get update && apt-get install -y \ lua5.1 \ + luajit \ python3 \ patch \ git \ @@ -14,8 +15,8 @@ RUN --mount=type=secret,id=ca-bundle \ if [ -f /run/secrets/ca-bundle ]; then export CURL_CA_BUNDLE=/run/secrets/ca-bundle; fi; \ curl -fsSLo /tmp/luau-ubuntu.zip \ https://github.com/luau-lang/luau/releases/latest/download/luau-ubuntu.zip \ - && unzip -j /tmp/luau-ubuntu.zip luau -d /usr/local/bin \ - && chmod +x /usr/local/bin/luau \ + && unzip -j /tmp/luau-ubuntu.zip luau luau-compile -d /usr/local/bin \ + && chmod +x /usr/local/bin/luau /usr/local/bin/luau-compile \ && rm /tmp/luau-ubuntu.zip # Verify installations @@ -25,6 +26,16 @@ WORKDIR /app COPY . /app RUN lua5.1 tests/harness/metadata.lua \ + && timeout 60s lua5.1 tests/harness/luau-parser.lua \ + && python3 tests/harness/luau-syntax.py \ + && python3 tests/harness/luau.py \ + && python3 tests/harness/luau-recent.py \ + && lua5.1 tests/harness/luau-strings.lua \ + && LUA51=luajit python3 tests/harness/luau-syntax.py \ + && LUA51=luajit python3 tests/harness/luau.py \ + && LUA51=luajit python3 tests/harness/luau-recent.py \ + && luajit tests/harness/luau-strings.lua \ + && python3 tests/harness/cli.py \ && timeout 60s lua5.1 tests/harness/numbers-to-expressions.lua \ && timeout 60s lua5.1 tests/harness/cff-arity.lua \ && timeout 60s lua5.1 tests/harness/cff-native.lua \ diff --git a/README.md b/README.md index 2f03a42..848c68b 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,11 @@ The project was inspired by the excellent [javascript-obfuscator](https://github Currently, Prometheus supports: - **Lua 5.1** -- **LuaU** *(basic support is available, but still not fully finished)* +- **Luau** *(partial support, including type erasure, interpolation, `const`, and generalized iteration; see [limitations](doc/reference/limitations.md#luau))* + +To process Luau, pass `--luau` (or `--LuaU`) to the CLI or set `LuaVersion = "LuaU"` in +configuration. Type annotations, assertions, and type declarations are removed; +output still targets the Luau runtime. See [Luau limitations](doc/reference/limitations.md#luau). Lua 5.1's legacy implicit `arg` table in vararg functions is unsupported. Use `...` and `select('#', ...)` instead; see [Limitations](doc/reference/limitations.md). diff --git a/doc/guides/cli-usage.md b/doc/guides/cli-usage.md index aeddc02..6c7339e 100644 --- a/doc/guides/cli-usage.md +++ b/doc/guides/cli-usage.md @@ -18,6 +18,7 @@ prometheus-lua --preset Medium ./input.lua If `--out` is not provided: - `input.lua` -> `input.obfuscated.lua` +- `input.luau` -> `input.obfuscated.luau` - `input` -> `input.obfuscated.lua` ## Common workflows @@ -37,7 +38,7 @@ prometheus-lua --config ./prometheus.config.lua ./src/main.lua Force Lua target: ```bash -prometheus-lua --preset Medium --LuaU ./src/main.lua +prometheus-lua --preset medium --luau ./src/main.luau ``` Enable pretty output: @@ -48,6 +49,8 @@ prometheus-lua --preset Minify --pretty ./src/main.lua ## Notes -- Unknown `--...` options are ignored with a warning. +- Command, option, and preset names are case-insensitive; paths preserve their case. +- Options with values accept both `--preset medium` and `--preset=medium`. +- Unknown options and missing option values are errors. - If no config/preset is passed, Prometheus falls back to `Minify`. - `update` command uses the official installer script from GitHub. diff --git a/doc/reference/cli-options.md b/doc/reference/cli-options.md index c95d485..433060d 100644 --- a/doc/reference/cli-options.md +++ b/doc/reference/cli-options.md @@ -15,14 +15,19 @@ | `--preset` | `--p` | `` | Use built-in preset by name | | `--config` | `--c` | `` | Load config Lua file | | `--out` | `--o` | `` | Output file path | -| `--Lua51` | - | none | Force Lua 5.1 conventions | -| `--LuaU` | - | none | Force LuaU conventions | +| `--lua51` | - | none | Force Lua 5.1 conventions | +| `--luau` | - | none | Force Luau conventions | | `--pretty` | - | none | Pretty-print generated output | | `--nocolors` | - | none | Disable ANSI colors | | `--saveerrors` | - | none | Write parser/runtime errors to `.error.txt` | ## Parsing behavior -- First non-option argument is treated as input file. -- A second non-option argument raises an error. -- Unknown options are ignored with a warning. +- Command names, option names, and built-in preset names are case-insensitive. Existing spellings + such as `--LuaU` and `--Lua51` continue to work. +- File paths and configuration values retain their original case. +- Options that take a value accept either `--option value` or `--option=value`. +- The first non-option argument is the input file; a second one is an error. +- Use `--` before an input filename beginning with `-`. For an option value + beginning with `-`, use `--option=value` or a path prefixed with `./`. +- Unknown options, missing values, and values supplied to flags are errors. diff --git a/doc/reference/limitations.md b/doc/reference/limitations.md index 4848f63..7ed69db 100644 --- a/doc/reference/limitations.md +++ b/doc/reference/limitations.md @@ -12,6 +12,65 @@ obfuscation: Source code constructed or loaded at runtime is not automatically obfuscated. +## Luau + +The obfuscator itself is written in Lua 5.1 and supports LuaJIT. `LuaVersion` +selects the input/output language; a Luau runtime is needed to execute Luau +output, not to run the obfuscator. + +Use `--LuaU` or `LuaVersion = "LuaU"` for Luau input. Type declarations, +annotations, assertions (`::`), and type function declarations are removed; +Prometheus does not type-check them, evaluate `typeof` expressions in type +positions, or retain exported types for consumers. The type grammar is validated +before erasure, including type packs, generic parameter defaults, and union and +intersection grouping. Unknown type names remain syntactically valid; use Luau's +type checker to resolve names and verify type compatibility. Run the output on Luau. + +Conditional expressions, including `elseif` branches, interpolated strings, +contextual [`const` declarations](https://rfcs.luau.org/const-keyword.html), and +generalized `for` iteration are supported. `const` bindings are checked for +reassignment and lowered to locals; tables +referenced by a `const` remain mutable. Interpolation uses Luau's `%*` string +formatting and therefore requires a Luau runtime for the transformed code. + +Generalized iteration supports tables, `__iter`, callable objects, ordinary +iterator triples, and protected metatables in every preset. `Cff` and `Vmify` +retain native iterator initialization in a small function and compile the loop +body as a callback. This preserves the caller's coroutine, break/continue, +outer returns, iteration-local captures, and trailing nil return values without +reflecting on the object's metatable. Vararg/return transport uses the standard +`select` and `unpack` globals. + +Floor division (`//` and `//=`) is supported in every preset and preserves +Luau's native `__idiv` metamethod. Contextual keywords such as `continue`, +`type`, `export`, and `const` remain valid identifiers outside their statement +contexts. Decimal, hexadecimal, and binary numerals support Luau's underscore +separators; malformed number tokens are rejected. + +Luau accepts one optional semicolon after a statement, including a type +declaration. Leading or repeated empty semicolons are rejected. Varargs are +valid at chunk level and inside functions that declare `...`; an inner function +does not inherit its enclosing function's varargs. These checks also apply to +erased type syntax. Type function bodies must use syntax that the parser +supports even though those declarations are erased. + +Explicit generic instantiation such as `identity<>(value)` is validated +and erased. Function attributes `@native`, `@checked`, `@deprecated`, and their +`@[...]` forms are validated. Minify preserves function attributes. Virtualizing +a function changes its layout, so its original per-function compilation hints +do not directly describe the generated dispatcher. Header directives +`--!native` and `--!optimize 0/1/2` are preserved by every preset. + +Recent Luau runtimes also support value exports (`export local`, `export const`, +and `export function`) and 64-bit integer literals such as `9007199254740993i`. +Exports are lowered to a module table with live bindings and a frozen module +return, including duplicate-export and top-level-return checks. Integer +literals retain their exact digits and bit patterns without converting them +to the obfuscator host's double-precision numbers. Their available operations +follow the selected Luau runtime; no arithmetic behavior is emulated by the +obfuscator. Lune 0.11.0 does not accept these newer input forms. Integer output +therefore requires a more recent Luau runtime. + ## Replacing a running function's environment Prometheus does not support replacing a function's environment while that diff --git a/prompt.txt b/prompt.txt new file mode 100644 index 0000000..b6e50aa --- /dev/null +++ b/prompt.txt @@ -0,0 +1,17 @@ +contexto: `src.zip` é o meu obfuscator, `prometheus.zip` é o prometheus que eu estou modificando para dar suporte a luau. +caso comente no codigo, comente em inglês. e adicione arquivos inuteis no .gitignore + +- hybrid keywords + - no prometheus não temos a validação de se é uma "hybrid keyword" ou uma keyword normal. incluindo o "continue", e varias outras. +- floor division + - atualmente não possui suporte a floor division. o `src.zip` possui algumas coisas sobre. +- verificar vararg (não tenho certeza sobre esse problema) + - atualmente não existe nada que diga se a vararg está em um lugar valido + - caso se a vararg esteja dentro de uma função, mas a função não possui vararg, não é avisado (não da erro na ofuscação). +- semicolon + - a forma que o ";" é "pulado" em luau é completamente diferente de lua. + - em luau, é muito mais rigido sobre a declaração de Semicolon, e atualmente não possui essa verificação. +- numerais em luau (não tenho certeza sobre isso) + - o prometheus não possui suporte para numerais de luau + +pequeno aviso: nem tudo em `src.zip` está correto, pois é uma versão mais antiga. \ No newline at end of file diff --git a/src/cli.lua b/src/cli.lua index 0e8282e..5bf9a24 100644 --- a/src/cli.lua +++ b/src/cli.lua @@ -41,14 +41,17 @@ local function print_help() print(" --version, -v Print CLI version") print("") print("Obfuscation options:") - print(" --preset, --p Use preset (e.g. Minify, Medium, High)") + print(" --preset, --p Use preset (e.g. Minify, Medium, Strong)") print(" --config, --c Use custom config Lua file") print(" --out, --o Set output path") - print(" --Lua51 Force Lua 5.1 target") - print(" --LuaU Force LuaU target") + print(" --lua51 Force Lua 5.1 target") + print(" --luau Force Luau target") print(" --pretty Pretty print output") print(" --nocolors Disable colored logs") print(" --saveerrors Persist parser errors to file") + print("") + print("Command, option, and preset names are case-insensitive; paths keep their original case.") + print("Options with a value also accept --option=value. Use -- before an input starting with '-'.") end local function run_update() @@ -71,17 +74,17 @@ local function run_update() print("Update completed") end -if arg[1] == "update" then +local firstArg = arg[1] and arg[1]:lower() +if firstArg == "update" then run_update() os.exit(0) end - -if arg[1] == "--version" or arg[1] == "-v" then +if firstArg == "--version" or firstArg == "-v" then print(get_version()) os.exit(0) end -if arg[1] == "--help" or arg[1] == "-h" or arg[1] == "help" then +if firstArg == "--help" or firstArg == "-h" or firstArg == "help" then print_help() os.exit(0) end @@ -148,80 +151,127 @@ local function load_chunk(content, chunkName, environment) return load(content, chunkName, "t", environment) end +local function output_stem(path) + local base, extension = path:match("^(.*)(%.[^/\\%.]+)$") + if extension and (extension:lower() == ".lua" or extension:lower() == ".luau") then + return base, extension + end + return path, ".lua" +end + +local function find_preset(name) + local exact = Prometheus.Presets[name] + if exact then return exact end + for presetName, preset in pairs(Prometheus.Presets) do + if presetName:lower() == name:lower() then return preset end + end +end + local function run_cli() - -- CLI local config, sourceFile, outFile, luaVersion, prettyPrint - + local optionsEnabled = true Prometheus.colors.enabled = true - -- Parse Arguments local i = 1 while i <= #arg do local curr = arg[i] - if curr:sub(1, 2) == "--" then - if curr == "--preset" or curr == "--p" then - if config then - Prometheus.Logger:warn("The config was set multiple times") + if optionsEnabled and curr == "--" then + optionsEnabled = false + elseif optionsEnabled and #curr > 1 and curr:sub(1, 1) == "-" then + local option, inlineValue = curr:match("^(%-%-[^=]+)=(.*)$") + option = (option or curr):lower() + + local function value() + if inlineValue ~= nil then + if inlineValue == "" then + Prometheus.Logger:error(string.format('The option "%s" requires a value', curr)) + end + return inlineValue + end + local nextArg = arg[i + 1] + if not nextArg or nextArg == "" or nextArg:sub(1, 1) == "-" then + Prometheus.Logger:error(string.format('The option "%s" requires a value', curr)) end - i = i + 1 - local preset = Prometheus.Presets[arg[i]] + return nextArg + end + + local takesValue = option == "--preset" or option == "--p" or option == "--config" + or option == "--c" or option == "--out" or option == "--o" + local isFlag = option == "--help" or option == "-h" or option == "--version" or option == "-v" + or option == "--nocolors" or option == "--lua51" or option == "--luau" + or option == "--pretty" or option == "--saveerrors" + if inlineValue ~= nil and not takesValue and isFlag then + Prometheus.Logger:error(string.format('The option "%s" does not accept a value', curr)) + end + + if option == "--help" or option == "-h" then + print_help() + return + elseif option == "--version" or option == "-v" then + print(get_version()) + return + elseif option == "--preset" or option == "--p" then + local name = value() + local preset = find_preset(name) if not preset then - Prometheus.Logger:error(string.format('A Preset with the name "%s" was not found!', tostring(arg[i]))) + Prometheus.Logger:error(string.format('A preset with the name "%s" was not found!', name)) end - - config = preset - elseif curr == "--config" or curr == "--c" then - i = i + 1 - local filename = tostring(arg[i]) + if config then Prometheus.Logger:warn("The config was set multiple times") end + config = {} + for key, entry in pairs(preset) do config[key] = entry end + elseif option == "--config" or option == "--c" then + local filename = value() if not file_exists(filename) then Prometheus.Logger:error(string.format('The config file "%s" was not found!', filename)) end - local content = table.concat(lines_from(filename), "\n") - -- Load Config from File local func, err = load_chunk(content, "@" .. filename, {}) if not func then Prometheus.Logger:error(string.format('Failed to parse config file "%s": %s', filename, tostring(err))) end + if config then Prometheus.Logger:warn("The config was set multiple times") end config = func() - elseif curr == "--out" or curr == "--o" then - i = i + 1 - if outFile then - Prometheus.Logger:warn("The output file was specified multiple times!") + if type(config) ~= "table" then + Prometheus.Logger:error(string.format('The config file "%s" must return a table', filename)) end - outFile = arg[i] - elseif curr == "--nocolors" then + elseif option == "--out" or option == "--o" then + local filename = value() + if outFile then Prometheus.Logger:warn("The output file was specified multiple times!") end + outFile = filename + elseif option == "--nocolors" then Prometheus.colors.enabled = false - elseif curr == "--Lua51" then + elseif option == "--lua51" then luaVersion = "Lua51" - elseif curr == "--LuaU" then + elseif option == "--luau" then luaVersion = "LuaU" - elseif curr == "--pretty" then + elseif option == "--pretty" then prettyPrint = true - elseif curr == "--saveerrors" then - -- Override error callback + elseif option == "--saveerrors" then Prometheus.Logger.errorCallback = function(...) - local args = { ... } - local message = table.concat(args, " ") + local message = table.concat({ ... }, " ") io.stderr:write(Prometheus.colors(Prometheus.Config.NameUpper .. ": " .. message, "red") .. "\n") - - local fileName = sourceFile:sub(-4) == ".lua" and sourceFile:sub(0, -5) .. ".error.txt" - or sourceFile .. ".error.txt" - local handle = io.open(fileName, "w") - handle:write(message) - handle:close() - + if sourceFile then + local stem = output_stem(sourceFile) + local filename = stem .. ".error.txt" + local handle, writeError = io.open(filename, "w") + if handle then + handle:write(message) + handle:close() + else + io.stderr:write(string.format('Cannot write error file "%s": %s\n', filename, tostring(writeError))) + end + end os.exit(1) end else - Prometheus.Logger:warn(string.format('The option "%s" is not valid and therefore ignored', curr)) + Prometheus.Logger:error(string.format('Unknown option "%s"', curr)) end else if sourceFile then - Prometheus.Logger:error(string.format('Unexpected argument "%s"', arg[i])) + Prometheus.Logger:error(string.format('Unexpected argument "%s"', curr)) end - sourceFile = tostring(arg[i]) + sourceFile = curr end i = i + 1 end @@ -229,26 +279,21 @@ local function run_cli() if not sourceFile then Prometheus.Logger:error("No input file was specified!") end - if not config then Prometheus.Logger:warn("No config was specified, falling back to Minify preset") - config = Prometheus.Presets.Minify + config = {} + for key, entry in pairs(Prometheus.Presets.Minify) do config[key] = entry end end - -- Add Option to override Lua Version config.LuaVersion = luaVersion or config.LuaVersion config.PrettyPrint = prettyPrint ~= nil and prettyPrint or config.PrettyPrint if not file_exists(sourceFile) then - Prometheus.Logger:error(string.format('The File "%s" was not found!', sourceFile)) + Prometheus.Logger:error(string.format('The file "%s" was not found!', sourceFile)) end - if not outFile then - if sourceFile:sub(-4) == ".lua" then - outFile = sourceFile:sub(0, -5) .. ".obfuscated.lua" - else - outFile = sourceFile .. ".obfuscated.lua" - end + local stem, extension = output_stem(sourceFile) + outFile = stem .. ".obfuscated" .. extension end local source = table.concat(lines_from(sourceFile), "\n") @@ -256,8 +301,10 @@ local function run_cli() local out = pipeline:apply(source, sourceFile) Prometheus.Logger:info(string.format('Writing output to "%s"', outFile)) - -- Write Output - local handle = io.open(outFile, "w") + local handle, writeError = io.open(outFile, "w") + if not handle then + Prometheus.Logger:error(string.format('Cannot write output file "%s": %s', outFile, tostring(writeError))) + end handle:write(out) handle:close() end diff --git a/src/prometheus/ast.lua b/src/prometheus/ast.lua index ef390dd..8a9715b 100644 --- a/src/prometheus/ast.lua +++ b/src/prometheus/ast.lua @@ -33,6 +33,7 @@ local AstKind = { CompoundSubStatement = "CompoundSubStatement"; CompoundMulStatement = "CompoundMulStatement"; CompoundDivStatement = "CompoundDivStatement"; + CompoundFloorDivStatement = "CompoundFloorDivStatement"; CompoundModStatement = "CompoundModStatement"; CompoundPowStatement = "CompoundPowStatement"; CompoundConcatStatement = "CompoundConcatStatement"; @@ -44,6 +45,7 @@ local AstKind = { -- Expression Nodes BooleanExpression = "BooleanExpression"; NumberExpression = "NumberExpression"; + IntegerExpression = "IntegerExpression"; StringExpression = "StringExpression"; NilExpression = "NilExpression"; VarargExpression = "VarargExpression"; @@ -60,6 +62,7 @@ local AstKind = { SubExpression = "SubExpression"; MulExpression = "MulExpression"; DivExpression = "DivExpression"; + FloorDivExpression = "FloorDivExpression"; ModExpression = "ModExpression"; NotExpression = "NotExpression"; LenExpression = "LenExpression"; @@ -85,6 +88,7 @@ local AstKind = { local astKindExpressionLookup = { [AstKind.BooleanExpression] = 0; [AstKind.NumberExpression] = 0; + [AstKind.IntegerExpression] = 0; [AstKind.StringExpression] = 0; [AstKind.NilExpression] = 0; [AstKind.VarargExpression] = 0; @@ -101,6 +105,7 @@ local astKindExpressionLookup = { [AstKind.SubExpression] = 8; [AstKind.MulExpression] = 7; [AstKind.DivExpression] = 7; + [AstKind.FloorDivExpression] = 7; [AstKind.ModExpression] = 7; [AstKind.NotExpression] = 5; [AstKind.LenExpression] = 5; @@ -272,6 +277,14 @@ function Ast.CompoundDivStatement(lhs, rhs) } end +function Ast.CompoundFloorDivStatement(lhs, rhs) + return { + kind = AstKind.CompoundFloorDivStatement, + lhs = lhs, + rhs = rhs, + } +end + function Ast.CompoundPowStatement(lhs, rhs) return { kind = AstKind.CompoundPowStatement, @@ -409,6 +422,10 @@ function Ast.LocalVariableDeclaration(scope, ids, expressions) } end +function Ast.IntegerExpression(source) + return {kind = AstKind.IntegerExpression, value = source, isConstant = false}; +end + function Ast.VarargExpression() return { kind = AstKind.VarargExpression; @@ -656,6 +673,19 @@ function Ast.DivExpression(lhs, rhs, simplify) } end +function Ast.FloorDivExpression(lhs, rhs, simplify) + if simplify and lhs.isConstant and rhs.isConstant + and type(lhs.value) == "number" and type(rhs.value) == "number" and rhs.value ~= 0 then + return Ast.NumberExpression(math.floor(lhs.value / rhs.value)); + end + return { + kind = AstKind.FloorDivExpression, + lhs = lhs, + rhs = rhs, + isConstant = false, + } +end + function Ast.ModExpression(lhs, rhs, simplify) if(simplify and rhs.isConstant and lhs.isConstant) then local success, val = pcall(function() return lhs.value % rhs.value end); @@ -688,6 +718,28 @@ function Ast.NotExpression(rhs, simplify) end function Ast.NegateExpression(rhs, simplify) + if simplify and rhs.kind == AstKind.IntegerExpression then + local source = rhs.value; + if source:sub(1, 1) == "-" then return Ast.IntegerExpression(source:sub(2)) end + local prefix = source:sub(1, 2):lower(); + if prefix ~= "0x" and prefix ~= "0b" then return Ast.IntegerExpression("-" .. source) end + local digits = source:sub(3, -2):lower(); + if prefix == "0b" then + digits = string.rep("0", (4 - #digits % 4) % 4) .. digits; + local hex = {}; + for i = 1, #digits, 4 do hex[#hex + 1] = string.format("%x", tonumber(digits:sub(i, i + 3), 2)) end + digits = table.concat(hex); + end + digits = digits:gsub("^0+", ""); + digits = string.rep("0", 16 - #digits) .. digits; + local hex, carry = {}, 1; + for i = 16, 1, -1 do + local value = 15 - tonumber(digits:sub(i, i), 16) + carry; + hex[i] = string.format("%x", value % 16); + carry = math.floor(value / 16); + end + return Ast.IntegerExpression("0x" .. table.concat(hex) .. "i"); + end if(simplify and rhs.isConstant) then local success, val = pcall(function() return -rhs.value end); if success then diff --git a/src/prometheus/cff/compile_top.lua b/src/prometheus/cff/compile_top.lua index b3a77c8..f718e4b 100644 --- a/src/prometheus/cff/compile_top.lua +++ b/src/prometheus/cff/compile_top.lua @@ -13,6 +13,13 @@ local AstKind = Ast.AstKind; return function(Compiler) function Compiler:compileTopNode(node) + visitast(node, function(expression) + if expression.isNativeLuau then + self.nativeExpressions = self.nativeExpressions or {}; + self.nativeExpressions[expression] = true; + return nil, true; + end + end); if self.nativeExpressions then self:prepareNativeExpressions(node); end local startBlock = self:createBlock(); local scope = startBlock.scope; diff --git a/src/prometheus/cff/compiler.lua b/src/prometheus/cff/compiler.lua index 305799b..518f719 100644 --- a/src/prometheus/cff/compiler.lua +++ b/src/prometheus/cff/compiler.lua @@ -21,8 +21,9 @@ local compileCoreModule = require("prometheus.cff.compile_core"); local Compiler = {}; -function Compiler:new() +function Compiler:new(settings) local compiler = { + luaVersion = settings and settings.LuaVersion; blocks = {}; registers = {}; activeBlock = nil; @@ -49,6 +50,7 @@ function Compiler:new() AstKind.SubExpression, AstKind.MulExpression, AstKind.DivExpression, + AstKind.FloorDivExpression, AstKind.ModExpression, AstKind.PowExpression, }; @@ -83,6 +85,7 @@ function Compiler:popRegisterUsageInfo() end function Compiler:compile(ast) + self.luaVersion = self.luaVersion or ast.luaVersion or "Lua51"; self.blocks = {}; self.registers = {}; self.activeBlock = nil; diff --git a/src/prometheus/cff/expressions.lua b/src/prometheus/cff/expressions.lua index 8220785..a5f58f7 100644 --- a/src/prometheus/cff/expressions.lua +++ b/src/prometheus/cff/expressions.lua @@ -14,6 +14,7 @@ local function requireExpression(name) end handlers[AstKind.StringExpression] = requireExpression("string"); handlers[AstKind.NumberExpression] = requireExpression("number"); +handlers[AstKind.IntegerExpression] = requireExpression("number"); handlers[AstKind.BooleanExpression] = requireExpression("boolean"); handlers[AstKind.NilExpression] = requireExpression("nil"); handlers[AstKind.VariableExpression] = requireExpression("variable"); @@ -43,6 +44,7 @@ handlers[AstKind.AddExpression] = binaryHandler; handlers[AstKind.SubExpression] = binaryHandler; handlers[AstKind.MulExpression] = binaryHandler; handlers[AstKind.DivExpression] = binaryHandler; +handlers[AstKind.FloorDivExpression] = binaryHandler; handlers[AstKind.ModExpression] = binaryHandler; handlers[AstKind.PowExpression] = binaryHandler; diff --git a/src/prometheus/cff/expressions/binary.lua b/src/prometheus/cff/expressions/binary.lua index 380d689..49efd8c 100644 --- a/src/prometheus/cff/expressions/binary.lua +++ b/src/prometheus/cff/expressions/binary.lua @@ -12,8 +12,14 @@ return function(self, expression, funcDepth, numReturns) for i = 1, numReturns do regs[i] = self:allocRegister(); if i == 1 then - local lhsReg = self:compileExpression(expression.lhs, funcDepth, 1)[1]; - local rhsReg = self:compileExpression(expression.rhs, funcDepth, 1)[1]; + local lhsReg, rhsReg; + local lhs = expression.lhs; + local deferRead = self.luaVersion == "LuaU" and expression.kind ~= Ast.AstKind.StrCatExpression + and lhs.kind == Ast.AstKind.VariableExpression and not lhs.scope.isGlobal + and self.scopeFunctionDepths[lhs.scope] == funcDepth; + if not deferRead then lhsReg = self:compileExpression(lhs, funcDepth, 1)[1] end + rhsReg = self:compileExpression(expression.rhs, funcDepth, 1)[1]; + if deferRead then lhsReg = self:compileExpression(lhs, funcDepth, 1)[1] end local binaryExpr = Ast[expression.kind](self:register(scope, lhsReg), self:register(scope, rhsReg)); self:addStatement(self:setRegister(scope, regs[i], binaryExpr), {regs[i]}, {lhsReg, rhsReg}, true); diff --git a/src/prometheus/cff/expressions/if_else.lua b/src/prometheus/cff/expressions/if_else.lua index 317b0c6..7289224 100644 --- a/src/prometheus/cff/expressions/if_else.lua +++ b/src/prometheus/cff/expressions/if_else.lua @@ -41,6 +41,7 @@ return function(self, expression, funcDepth, numReturns) local trueReg = self:compileExpression(expression.true_value, funcDepth, 1)[1]; self:addStatement(self:copyRegisters(scope, {resReg}, {trueReg}), {resReg}, {trueReg}, false); + self:freeRegister(trueReg, false); self:addStatement(self:setRegister(scope, self.POS_REGISTER, Ast.NumberExpression(finalBlock.id)), {self.POS_REGISTER}, {}, false); for _, elif in ipairs(expression.elseifs) do @@ -57,6 +58,7 @@ return function(self, expression, funcDepth, numReturns) elifScope = elifBlock.scope; local valueReg = self:compileExpression(elif.value, funcDepth, 1)[1]; self:addStatement(self:copyRegisters(elifScope, {resReg}, {valueReg}), {resReg}, {valueReg}, false); + self:freeRegister(valueReg, false); self:addStatement(self:setRegister(elifScope, self.POS_REGISTER, Ast.NumberExpression(finalBlock.id)), {self.POS_REGISTER}, {}, false); end @@ -64,6 +66,7 @@ return function(self, expression, funcDepth, numReturns) scope = self.activeBlock.scope; local falseReg = self:compileExpression(expression.false_value, funcDepth, 1)[1]; self:addStatement(self:copyRegisters(scope, {resReg}, {falseReg}), {resReg}, {falseReg}, false); + self:freeRegister(falseReg, false); self:addStatement(self:setRegister(scope, self.POS_REGISTER, Ast.NumberExpression(finalBlock.id)), {self.POS_REGISTER}, {}, false); self.registers[self.POS_REGISTER] = posState; diff --git a/src/prometheus/cff/expressions/index.lua b/src/prometheus/cff/expressions/index.lua index c3574c7..602f061 100644 --- a/src/prometheus/cff/expressions/index.lua +++ b/src/prometheus/cff/expressions/index.lua @@ -12,11 +12,13 @@ return function(self, expression, funcDepth, numReturns) for i = 1, numReturns do regs[i] = self:allocRegister(); if i == 1 then - local baseReg = self:compileExpression(expression.base, funcDepth, 1)[1]; + local baseReg; + local base = expression.base; + local deferRead = self.luaVersion == "LuaU" and base.kind == Ast.AstKind.VariableExpression + and not base.scope.isGlobal and self.scopeFunctionDepths[base.scope] == funcDepth; + if not deferRead then baseReg = self:compileExpression(base, funcDepth, 1)[1] end local indexReg = self:compileExpression(expression.index, funcDepth, 1)[1]; - - - + if deferRead then baseReg = self:compileExpression(base, funcDepth, 1)[1] end self:addStatement(self:setRegister(scope, regs[i], Ast.IndexExpression(self:register(scope, baseReg), self:register(scope, indexReg))), {regs[i]}, {baseReg, indexReg}, true); self:freeRegister(baseReg, false); diff --git a/src/prometheus/cff/expressions/number.lua b/src/prometheus/cff/expressions/number.lua index 4fe5e3b..0315666 100644 --- a/src/prometheus/cff/expressions/number.lua +++ b/src/prometheus/cff/expressions/number.lua @@ -18,7 +18,7 @@ return function(self, expression, funcDepth, numReturns) self:addStatement(self:setRegister(scope, regs[i], Ast.NumberExpression(0)), {regs[i]}, {}, false); self:addStatement(self:setRegister(scope, regs[i], Ast.NegateExpression(self:register(scope, regs[i]))), {regs[i]}, {regs[i]}, false); else - self:addStatement(self:setRegister(scope, regs[i], Ast.NumberExpression(expression.value)), {regs[i]}, {}, false); + self:addStatement(self:setRegister(scope, regs[i], Ast[expression.kind](expression.value)), {regs[i]}, {}, false); end else self:addStatement(self:setRegister(scope, regs[i], Ast.NilExpression()), {regs[i]}, {}, false); diff --git a/src/prometheus/cff/expressions/table_constructor.lua b/src/prometheus/cff/expressions/table_constructor.lua index f9ce495..c0624c5 100644 --- a/src/prometheus/cff/expressions/table_constructor.lua +++ b/src/prometheus/cff/expressions/table_constructor.lua @@ -30,8 +30,13 @@ return function(self, expression, funcDepth, numReturns) table.insert(entryRegs, reg); end else - local keyReg = self:compileExpression(entry.key, funcDepth, 1)[1]; + local keyReg; + local key = entry.key; + local deferRead = self.luaVersion == "LuaU" and key.kind == AstKind.VariableExpression + and not key.scope.isGlobal and self.scopeFunctionDepths[key.scope] == funcDepth; + if not deferRead then keyReg = self:compileExpression(key, funcDepth, 1)[1] end local valReg = self:compileExpression(entry.value, funcDepth, 1)[1]; + if deferRead then keyReg = self:compileExpression(key, funcDepth, 1)[1] end table.insert(entries, Ast.KeyedTableEntry(self:register(scope, keyReg), self:register(scope, valReg))); table.insert(entryRegs, valReg); table.insert(entryRegs, keyReg); diff --git a/src/prometheus/cff/native.lua b/src/prometheus/cff/native.lua index cab9538..00df707 100644 --- a/src/prometheus/cff/native.lua +++ b/src/prometheus/cff/native.lua @@ -109,17 +109,18 @@ return function(Compiler) end end) - local source = Unparser:new({LuaVersion = 'Lua51'}):unparseExpression(expression, 0) + local source = Unparser:new({LuaVersion = self.luaVersion}):unparseExpression(expression, 0) local finish = '' if expression.kind == Kind.FunctionLiteralExpression then finish = 'if ' .. names.setenv .. ' then ' .. names.setenv .. '(' .. names.result .. ',' .. names.env .. ') end;' end - local parsed = Parser:new({LuaVersion = 'Lua51', AllowInternalIdentifiers = true}):parse('return function(' + local parsed = Parser:new({LuaVersion = self.luaVersion, AllowInternalIdentifiers = true}):parse('return function(' .. names.proxy .. ',' .. names.env .. ',' .. names.values .. ') local ' .. names.result .. '=' .. source .. ';' .. finish .. 'return ' .. names.result .. ' end') local factory = parsed.body.statements[1].args[1] self.nativeResults[expression] = factory.body.statements[1].expressions[1] + self.nativeResults[expression].isNativeLuau = expression.isNativeLuau factory.body.scope:setParent(scope) require('prometheus.visitast')(parsed, nil, function(node, data) if node.kind == Kind.VariableExpression and node.scope.isGlobal then diff --git a/src/prometheus/cff/statements.lua b/src/prometheus/cff/statements.lua index 6323ffa..f9d509c 100644 --- a/src/prometheus/cff/statements.lua +++ b/src/prometheus/cff/statements.lua @@ -35,6 +35,7 @@ handlers[AstKind.CompoundAddStatement] = compoundHandler; handlers[AstKind.CompoundSubStatement] = compoundHandler; handlers[AstKind.CompoundMulStatement] = compoundHandler; handlers[AstKind.CompoundDivStatement] = compoundHandler; +handlers[AstKind.CompoundFloorDivStatement] = compoundHandler; handlers[AstKind.CompoundModStatement] = compoundHandler; handlers[AstKind.CompoundPowStatement] = compoundHandler; handlers[AstKind.CompoundConcatStatement] = compoundHandler; diff --git a/src/prometheus/cff/statements/compound.lua b/src/prometheus/cff/statements/compound.lua index 059a130..54525a1 100644 --- a/src/prometheus/cff/statements/compound.lua +++ b/src/prometheus/cff/statements/compound.lua @@ -7,56 +7,89 @@ local Ast = require("prometheus.ast"); local AstKind = Ast.AstKind; -local compoundConstructors = { - [AstKind.CompoundAddStatement] = Ast.CompoundAddStatement, - [AstKind.CompoundSubStatement] = Ast.CompoundSubStatement, - [AstKind.CompoundMulStatement] = Ast.CompoundMulStatement, - [AstKind.CompoundDivStatement] = Ast.CompoundDivStatement, - [AstKind.CompoundModStatement] = Ast.CompoundModStatement, - [AstKind.CompoundPowStatement] = Ast.CompoundPowStatement, - [AstKind.CompoundConcatStatement] = Ast.CompoundConcatStatement, +local binaryConstructors = { + [AstKind.CompoundAddStatement] = Ast.AddExpression, + [AstKind.CompoundSubStatement] = Ast.SubExpression, + [AstKind.CompoundMulStatement] = Ast.MulExpression, + [AstKind.CompoundDivStatement] = Ast.DivExpression, + [AstKind.CompoundFloorDivStatement] = Ast.FloorDivExpression, + [AstKind.CompoundModStatement] = Ast.ModExpression, + [AstKind.CompoundPowStatement] = Ast.PowExpression, + [AstKind.CompoundConcatStatement] = Ast.StrCatExpression, }; return function(self, statement, funcDepth) local scope = self.activeBlock.scope; - local compoundConstructor = compoundConstructors[statement.kind]; - if statement.lhs.kind == AstKind.AssignmentIndexing then - local indexing = statement.lhs; - local baseReg = self:compileExpression(indexing.base, funcDepth, 1)[1]; - local indexReg = self:compileExpression(indexing.index, funcDepth, 1)[1]; - local valueReg = self:compileExpression(statement.rhs, funcDepth, 1)[1]; + local target = statement.lhs; + local baseReg, indexReg, oldReg; + local function isCurrentLocal(expression) + return expression.kind == AstKind.VariableExpression and not expression.scope.isGlobal + and self.scopeFunctionDepths[expression.scope] == funcDepth; + end + local function capture(expression) + local reg = self:compileExpression(expression, funcDepth, 1)[1]; + if self:isVarRegister(reg) then + local copy = self:allocRegister(false); + self:addStatement(self:copyRegisters(scope, {copy}, {reg}), {copy}, {reg}, false); + reg = copy; + end + return reg; + end + + local indexed = target.kind == AstKind.AssignmentIndexing; + local localTarget = not indexed and not target.scope.isGlobal + and self.scopeFunctionDepths[target.scope] == funcDepth; + local deferRead = localTarget and statement.kind ~= AstKind.CompoundConcatStatement; + if indexed then + -- Luau addresses direct locals through their current registers, including + -- changes made while evaluating the key, RHS, or an operator metamethod. + if not isCurrentLocal(target.base) then baseReg = capture(target.base) end + indexReg = capture(target.index); + if not baseReg then baseReg = capture(target.base) end + oldReg = self:allocRegister(false); + self:addStatement(self:setRegister(scope, oldReg, Ast.IndexExpression(self:register(scope, baseReg), + self:register(scope, indexReg))), {oldReg}, {baseReg, indexReg}, true); + elseif not deferRead then + oldReg = capture(Ast.VariableExpression(target.scope, target.id)); + end + -- Arithmetic reads current-function locals after RHS. Concatenation, + -- globals, upvalues, and indexed values retain their earlier value. + local valueReg = self:compileExpression(statement.rhs, funcDepth, 1)[1]; + if deferRead then oldReg = self:compileExpression(Ast.VariableExpression(target.scope, target.id), funcDepth, 1)[1] end + local resultReg = self:allocRegister(false); + local result = binaryConstructors[statement.kind](self:register(scope, oldReg), self:register(scope, valueReg)); + self:addStatement(self:setRegister(scope, resultReg, result), {resultReg}, {oldReg, valueReg}, true); - self:addStatement(compoundConstructor(Ast.AssignmentIndexing(self:register(scope, baseReg), self:register(scope, indexReg)), self:register(scope, valueReg)), {}, {baseReg, indexReg, valueReg}, true); + if indexed then + if isCurrentLocal(target.base) then + self:freeRegister(baseReg, false); + baseReg = capture(target.base); + end + if isCurrentLocal(target.index) then + self:freeRegister(indexReg, false); + indexReg = capture(target.index); + end + self:addStatement(Ast.AssignmentStatement({Ast.AssignmentIndexing(self:register(scope, baseReg), + self:register(scope, indexReg))}, {self:register(scope, resultReg)}), {}, {baseReg, indexReg, resultReg}, true); self:freeRegister(baseReg, false); self:freeRegister(indexReg, false); - self:freeRegister(valueReg, false); - else - local valueReg = self:compileExpression(statement.rhs, funcDepth, 1)[1]; - local primaryExpr = statement.lhs; - if primaryExpr.scope.isGlobal then - local tmpReg = self:allocRegister(false); - self:addStatement(self:setRegister(scope, tmpReg, Ast.StringExpression(primaryExpr.scope:getVariableName(primaryExpr.id))), {tmpReg}, {}, false); - self:addStatement(compoundConstructor(Ast.AssignmentIndexing(self:env(scope), self:register(scope, tmpReg)), - self:register(scope, valueReg)), {}, {tmpReg, valueReg}, true); - self:freeRegister(tmpReg, false); - self:freeRegister(valueReg, false); + elseif target.scope.isGlobal then + self:addStatement(Ast.AssignmentStatement({Ast.AssignmentIndexing(self:env(scope), + Ast.StringExpression(target.scope:getVariableName(target.id)))}, {self:register(scope, resultReg)}), {}, {resultReg}, true); + elseif localTarget then + local reg = self:getVarRegister(target.scope, target.id, funcDepth); + if self:isUpvalue(target.scope, target.id) then + self:addStatement(self:setUpvalueMember(scope, self:register(scope, reg), self:register(scope, resultReg)), {}, {reg, resultReg}, true); else - if self.scopeFunctionDepths[primaryExpr.scope] == funcDepth then - if self:isUpvalue(primaryExpr.scope, primaryExpr.id) then - local reg = self:getVarRegister(primaryExpr.scope, primaryExpr.id, funcDepth); - self:addStatement(self:setUpvalueMember(scope, self:register(scope, reg), self:register(scope, valueReg), compoundConstructor), {}, {reg, valueReg}, true); - else - local reg = self:getVarRegister(primaryExpr.scope, primaryExpr.id, funcDepth, valueReg); - if reg ~= valueReg then - self:addStatement(self:setRegister(scope, reg, self:register(scope, valueReg), compoundConstructor), {reg}, {valueReg}, false); - end - end - else - local upvalId = self:getUpvalueId(primaryExpr.scope, primaryExpr.id); - scope:addReferenceToHigherScope(self.containerFuncScope, self.currentUpvaluesVar); - self:addStatement(self:setUpvalueMember(scope, Ast.IndexExpression(Ast.VariableExpression(self.containerFuncScope, self.currentUpvaluesVar), Ast.NumberExpression(upvalId)), self:register(scope, valueReg), compoundConstructor), {}, {valueReg}, true); - end - self:freeRegister(valueReg, false); + self:addStatement(self:setRegister(scope, reg, self:register(scope, resultReg)), {reg}, {resultReg}, false); end + else + local upvalId = self:getUpvalueId(target.scope, target.id); + scope:addReferenceToHigherScope(self.containerFuncScope, self.currentUpvaluesVar); + self:addStatement(self:setUpvalueMember(scope, Ast.IndexExpression(Ast.VariableExpression(self.containerFuncScope, + self.currentUpvaluesVar), Ast.NumberExpression(upvalId)), self:register(scope, resultReg)), {}, {resultReg}, true); end + self:freeRegister(oldReg, false); + self:freeRegister(valueReg, false); + self:freeRegister(resultReg, false); end; diff --git a/src/prometheus/cff/statements/for_in_statement.lua b/src/prometheus/cff/statements/for_in_statement.lua index 3a48932..149b0e8 100644 --- a/src/prometheus/cff/statements/for_in_statement.lua +++ b/src/prometheus/cff/statements/for_in_statement.lua @@ -74,7 +74,7 @@ return function(self, statement, funcDepth) self:addStatement(Ast.AssignmentStatement({ self:posAssignment(scope) }, { - Ast.OrExpression(Ast.AndExpression(self:register(scope, exprregs[3]), Ast.NumberExpression(bodyBlock.id)), Ast.NumberExpression(finalBlock.id)) + Ast.OrExpression(Ast.AndExpression(Ast.NotEqualsExpression(self:register(scope, exprregs[3]), Ast.NilExpression()), Ast.NumberExpression(bodyBlock.id)), Ast.NumberExpression(finalBlock.id)) }), {self.POS_REGISTER}, {exprregs[3]}, false); self:setActiveBlock(bodyBlock); diff --git a/src/prometheus/enums.lua b/src/prometheus/enums.lua index e8880be..ea3ac28 100644 --- a/src/prometheus/enums.lua +++ b/src/prometheus/enums.lua @@ -59,21 +59,22 @@ Enums.Conventions = { }, [Enums.LuaVersion.LuaU] = { Keywords = { - "and", "break", "do", "else", "elseif", "continue", + "and", "break", "do", "else", "elseif", "end", "false", "for", "function", "if", "in", "local", "nil", "not", "or", "repeat", "return", "then", "true", "until", "while" }, + HybridKeywords = {"continue", "type", "export", "const", "typeof", "read", "write"}, - SymbolChars = chararray("+-*/%^#=~<>(){}[];:,."), + SymbolChars = chararray("+-*/%^#=~<>(){}[];:,.?|&@"), MaxSymbolLength = 3, Symbols = { - "+", "-", "*", "/", "%", "^", "#", + "+", "-", "*", "/", "//", "%", "^", "#", "==", "~=", "<=", ">=", "<", ">", "=", - "+=", "-=", "/=", "%=", "^=", "..=", "*=", + "+=", "-=", "/=", "//=", "%=", "^=", "..=", "*=", "(", ")", "{", "}", "[", "]", ";", ":", ",", ".", "..", "...", - "::", "->", "?", "|", "&", + "::", "->", "?", "|", "&", "@", }, IdentChars = chararray("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_0123456789"), diff --git a/src/prometheus/luau_attributes.lua b/src/prometheus/luau_attributes.lua new file mode 100644 index 0000000..a76b821 --- /dev/null +++ b/src/prometheus/luau_attributes.lua @@ -0,0 +1,78 @@ +-- Parse function attributes separately from runtime expressions and type erasure. +local Ast = require('prometheus.ast') +local logger = require('logger') +local K = Ast.AstKind + +return function(Parser, token, is, consume, expect, peek, get) + local function fail(self, message) + local tk = peek(self) + logger:error(string.format('Parsing Error at Position %s:%s, %s', tk.line, tk.linePos, message)) + end + local function literal(node) + if node.isParenthesizedExpression then return false end + if node.kind == K.StringExpression or node.kind == K.NumberExpression + or node.kind == K.BooleanExpression or node.kind == K.NilExpression then return true end + if node.kind ~= K.TableConstructorExpression then return false end + for _, entry in ipairs(node.entries) do + if entry.key and not literal(entry.key) then return false end + if not literal(entry.value) then return false end + end + return true + end + + function Parser:parseLuauAttributes(scope) + if self.luaVersion ~= 'LuaU' or not is(self, token.Symbol, '@') then return nil end + local attributes, seen = {}, {} + local function attribute(listed) + local name = expect(self, token.Ident).value + if name ~= 'native' and name ~= 'checked' and name ~= 'deprecated' then fail(self, 'Invalid function attribute ' .. name) end + if seen[name] then fail(self, 'Duplicate function attribute ' .. name) end + seen[name] = true + local args = {} + if listed then + local savedSimplify = self.noSimplify + self.noSimplify = true + if consume(self, token.Symbol, '(') then + if not is(self, token.Symbol, ')') then args = self:exprList(scope) end + expect(self, token.Symbol, ')') + elseif is(self, token.Symbol, '{') then + args = {self:tableConstructor(scope)} + elseif is(self, token.String) then + args = {Ast.StringExpression(get(self).value)} + end + self.noSimplify = savedSimplify + end + for _, arg in ipairs(args) do if not literal(arg) then fail(self, 'Only literals are allowed in attributes') end end + if name == 'deprecated' then + if #args > 1 then fail(self, 'Deprecated accepts at most one argument') end + local arg = args[1] + if arg and arg.kind ~= K.TableConstructorExpression then fail(self, 'Deprecated requires a table literal') end + if arg then + for _, entry in ipairs(arg.entries) do + if not entry.isRecord or not entry.key or entry.key.kind ~= K.StringExpression + or (entry.key.value ~= 'use' and entry.key.value ~= 'reason') + or entry.value.kind ~= K.StringExpression then + fail(self, 'Deprecated table accepts only string fields use and reason') + end + end + end + end + end + while is(self, token.Symbol, '@') do + local first = self.index + 1 + local at = get(self) + if peek(self).startPos ~= at.endPos then fail(self, "Expected an attribute name or '[' immediately after '@'") end + if consume(self, token.Symbol, '[') then + attribute(true) + while consume(self, token.Symbol, ',') do attribute(true) end + expect(self, token.Symbol, ']') + else + attribute(false) + end + local pieces = {} + for i = first, self.index do pieces[#pieces + 1] = self.tokens[i].source end + attributes[#attributes + 1] = table.concat(pieces, ' '):gsub('^@ ', '@') + end + return attributes + end +end diff --git a/src/prometheus/luau_exports.lua b/src/prometheus/luau_exports.lua new file mode 100644 index 0000000..0b6cdde --- /dev/null +++ b/src/prometheus/luau_exports.lua @@ -0,0 +1,48 @@ +-- Lower value exports to a module table while retaining live mutable bindings. +local Ast = require('prometheus.ast') +local visitast = require('prometheus.visitast') +local K = Ast.AstKind + +return function(ast, bindings) + if #bindings == 0 then return ast end + local scope, lookup = ast.body.scope, {} + local tableId = scope:addVariable() + for _, binding in ipairs(bindings) do + lookup[binding.scope] = lookup[binding.scope] or {} + lookup[binding.scope][binding.id] = binding.name + end + local function access(owner, id, useScope, assignment) + local name = lookup[owner] and lookup[owner][id] + if not name then return nil end + if useScope ~= scope then useScope:addReferenceToHigherScope(scope, tableId) end + local base = Ast.VariableExpression(scope, tableId) + return (assignment and Ast.AssignmentIndexing or Ast.IndexExpression)(base, Ast.StringExpression(name)) + end + visitast(ast, function(node, data) + if node.kind == K.VariableExpression or node.kind == K.AssignmentVariable then + return access(node.scope, node.id, data.scope, node.kind == K.AssignmentVariable) + elseif node.kind == K.LocalVariableDeclaration and node.isExported then + local targets = {} + for _, id in ipairs(node.ids) do targets[#targets + 1] = access(node.scope, id, data.scope, true) end + local values = node.expressions + if #values == 0 then values = {Ast.NilExpression()} end + return Ast.AssignmentStatement(targets, values) + elseif (node.kind == K.LocalFunctionDeclaration and node.isExported) or node.kind == K.FunctionDeclaration then + local target = access(node.scope, node.id, data.scope, true) + if not target then return nil end + for _, name in ipairs(node.indices or {}) do + target.kind = K.IndexExpression + target = Ast.AssignmentIndexing(target, Ast.StringExpression(name)) + end + local value = Ast.FunctionLiteralExpression(node.args, node.body) + value.attributes = node.attributes + return Ast.AssignmentStatement({target}, {value}) + end + end) + table.insert(ast.body.statements, 1, Ast.LocalVariableDeclaration(scope, {tableId}, {Ast.TableConstructorExpression({})})) + local tableScope, libraryId = scope:resolveGlobal('table') + local freeze = Ast.IndexExpression(Ast.VariableExpression(tableScope, libraryId), Ast.StringExpression('freeze')) + ast.body.statements[#ast.body.statements + 1] = Ast.ReturnStatement{ + Ast.FunctionCallExpression(freeze, {Ast.VariableExpression(scope, tableId)})} + return ast +end diff --git a/src/prometheus/luau_iteration.lua b/src/prometheus/luau_iteration.lua new file mode 100644 index 0000000..45a5620 --- /dev/null +++ b/src/prometheus/luau_iteration.lua @@ -0,0 +1,87 @@ +-- Keep iterator initialization native so protected __iter metatables work without +-- reflection. The callback remains in the caller's coroutine and shares captures. +local Ast = require('prometheus.ast') +local Scope = require('prometheus.scope') +local visitast = require('prometheus.visitast') +local K = Ast.AstKind + +return function(ast, Parser) + local function parsedFunction(source, parent) + local parsed = Parser:new({LuaVersion = 'LuaU', AllowInternalIdentifiers = true}):parse('return ' .. source) + local result = parsed.body.statements[1].args[1] + result.body.scope:setParent(parent) + return result + end + local function global(scope, name) + local owner, id = scope:resolveGlobal(name) + return Ast.VariableExpression(owner, id) + end + local function reference(scope, owner, id) + if scope ~= owner then scope:addReferenceToHigherScope(owner, id) end + return Ast.VariableExpression(owner, id) + end + local function pack(scope, expressions) + local fn = parsedFunction('function(...) return {n=select("#",...),...} end', scope) + return Ast.FunctionCallExpression(fn, expressions) + end + local function unpackValues(scope, owner, id) + return Ast.FunctionCallExpression(global(scope, 'unpack'), { + reference(scope, owner, id), Ast.NumberExpression(1), + Ast.IndexExpression(reference(scope, owner, id), Ast.StringExpression('n')), + }) + end + + visitast(ast, nil, function(node) + if node.kind ~= K.ForInStatement then return nil end + local scope = Scope:new(node.parentScope) + local iterator, state, control = scope:addVariable(), scope:addVariable(), scope:addVariable() + local action, values = scope:addVariable(), scope:addVariable() + local savedArgs + node.body.scope:setParent(scope) + local wrapper = Ast.TopNode(node.body, ast.globalScope) + visitast(wrapper, function(child, data) + if data.functionData.depth ~= 0 then return nil end + if child.kind == K.VarargExpression then + savedArgs = savedArgs or scope:addVariable() + local result = unpackValues(data.scope, scope, savedArgs) + result.isParenthesizedExpression = child.isParenthesizedExpression + return result, true + elseif child.kind == K.BreakStatement and child.loop == node then + return Ast.ReturnStatement{Ast.NumberExpression(1)}, true + elseif child.kind == K.ContinueStatement and child.loop == node then + return Ast.ReturnStatement{}, true + end + end, function(child, data) + if data.functionData.depth == 0 and child.kind == K.ReturnStatement and not child.iterationControl then + local result = Ast.ReturnStatement{Ast.NumberExpression(2), pack(data.scope, child.args)} + result.iterationControl = true + return result + end + end) + -- Return nodes introduced for break/continue were skipped by previsit. + local args, names = {}, {} + for index, id in ipairs(node.ids) do + args[index] = Ast.VariableExpression(node.scope, id) + names[index] = 'v' .. index + end + local callback = Ast.FunctionLiteralExpression(args, node.body) + local parameters = table.concat(names, ',') + local helper = parsedFunction('function(callback,iterator,state,control) for ' .. parameters + .. ' in iterator,state,control do local action,values=callback(' .. parameters + .. ') if action then return action,values end end end', scope) + helper.isNativeLuau = true + local statements = {} + if savedArgs then + statements[#statements + 1] = Ast.LocalVariableDeclaration(scope, {savedArgs}, {pack(scope, {Ast.VarargExpression()})}) + end + statements[#statements + 1] = Ast.LocalVariableDeclaration(scope, {iterator, state, control}, node.expressions) + statements[#statements + 1] = Ast.LocalVariableDeclaration(scope, {action, values}, { + Ast.FunctionCallExpression(helper, {callback, Ast.VariableExpression(scope, iterator), + Ast.VariableExpression(scope, state), Ast.VariableExpression(scope, control)})}) + local returnScope = Scope:new(scope) + statements[#statements + 1] = Ast.IfStatement(Ast.EqualsExpression(Ast.VariableExpression(scope, action), Ast.NumberExpression(2)), + Ast.Block({Ast.ReturnStatement{unpackValues(returnScope, scope, values)}}, returnScope), {}) + return Ast.DoStatement(Ast.Block(statements, scope)) + end) + return ast +end diff --git a/src/prometheus/luau_types.lua b/src/prometheus/luau_types.lua new file mode 100644 index 0000000..42b3b5b --- /dev/null +++ b/src/prometheus/luau_types.lua @@ -0,0 +1,316 @@ +-- Validate Luau's type grammar before erasing compile-time syntax. +-- Types, packs, generic declarations, and generic arguments have distinct rules. +-- Reference: luau-lang/luau Ast/src/Parser.cpp (parseType and related routines). +local Scope = require('prometheus.scope') +local logger = require('logger') + +return function(Parser, token, is, consume, expect, peek, get) + local S, I, K = token.Symbol, token.Ident, token.Keyword + local TYPE = {kind = 'type'} + local parseType, parseSimpleType, parseTypeSuffix, parseReturnType + local parseGenerics, parseTypeArguments + + local function fail(self, message) + local tk = peek(self) + local diagnostic = string.format('Parsing Error at Position %s:%s, %s', + tostring(tk.line or '?'), tostring(tk.linePos or '?'), message) + if self.disableLog then error(diagnostic, 0) end + logger:error(diagnostic) + end + + local function startsPack(self) + return is(self, S, '...') or (is(self, I) and is(self, S, '...', 1)) + end + + local function typeFollows(self) + return is(self, S, '?') or is(self, S, '|') or is(self, S, '&') + end + + local function parsePack(self) + if consume(self, S, '...') then + parseType(self) + else + expect(self, I) + expect(self, S, '...') + end + return {kind = 'pack'} + end + + -- A pack is a tail: it cannot be followed by another element in a type list. + local function parseTypeList(self) + local result = {kind = 'pack', count = 0, tail = false, named = false} + if is(self, S, ')') then return result end + repeat + if startsPack(self) then + parsePack(self) + result.tail = true + return result + end + if is(self, I) and is(self, S, ':', 1) then + get(self) + get(self) + result.named = true + end + parseType(self) + result.count = result.count + 1 + until not consume(self, S, ',') + return result + end + + local function parseFunctionTail(self) + expect(self, S, '->') + parseReturnType(self) + return TYPE + end + + local function parseParenthesized(self, allowPack) + local generic = is(self, S, '<') + if generic then parseGenerics(self, false) end + expect(self, S, '(') + local list = parseTypeList(self) + expect(self, S, ')') + if generic or list.named or is(self, S, '->') or is(self, S, ':') then + return parseFunctionTail(self) + end + if allowPack then return list end + if list.count == 1 and not list.tail then return TYPE end + fail(self, "Expected '->' after function parameter types; a type pack is not a single type") + end + + parseReturnType = function(self) + if startsPack(self) then return parsePack(self) end + if not consume(self, S, '(') then + parseType(self) + return {kind = 'pack', count = 1} + end + local list = parseTypeList(self) + expect(self, S, ')') + if list.named or is(self, S, '->') or is(self, S, ':') then + parseFunctionTail(self) + return {kind = 'pack', count = 1} + end + -- A single parenthesized return type can have union/optional suffixes. + if list.count == 1 then parseTypeSuffix(self, TYPE) end + return list + end + + local function parseTable(self) + expect(self, S, '{') + local properties, indexer = 0, false + while not consume(self, S, '}') do + if (is(self, I, 'read') or is(self, I, 'write')) and not is(self, S, ':', 1) then + get(self) + end + if consume(self, S, '[') then + if is(self, token.String) and is(self, S, ']', 1) then + local name = get(self) + if name.value:find('\0', 1, true) then fail(self, 'A table type property cannot contain a null byte') end + properties = properties + 1 + else + if indexer then fail(self, 'A table type cannot contain more than one indexer') end + parseType(self) + indexer = true + end + expect(self, S, ']') + expect(self, S, ':') + parseType(self) + elseif properties == 0 and not indexer and not (is(self, I) and is(self, S, ':', 1)) then + -- Array shorthand is exactly {Type}, not a list of table fields. + parseType(self) + expect(self, S, '}') + return TYPE + else + expect(self, I) + expect(self, S, ':') + parseType(self) + properties = properties + 1 + end + if not consume(self, S, ',') and not consume(self, S, ';') then + expect(self, S, '}') + return TYPE + end + end + return TYPE + end + + parseSimpleType = function(self, allowPack) + if is(self, S, '(') or is(self, S, '<') then + return parseParenthesized(self, allowPack) + elseif is(self, S, '{') then + return parseTable(self) + elseif is(self, token.String) or is(self, K, 'nil') or is(self, K, 'true') or is(self, K, 'false') then + get(self) + return TYPE + elseif is(self, I) then + local name = get(self) + if consume(self, S, '.') then + expect(self, I) + elseif name.value == 'typeof' then + expect(self, S, '(') + -- Parse the real expression grammar in a detached scope. It is never + -- evaluated or added to the runtime AST and its reference graph. + self:expression(Scope:new(self.lexicalScope or Scope:newGlobal())) + expect(self, S, ')') + return TYPE + end + if is(self, S, '...') then fail(self, 'A type pack is not allowed in this type position') end + if is(self, S, '<') then parseTypeArguments(self) end + return TYPE + end + fail(self, 'Expected a type, got ' .. tostring(peek(self).source)) + end + + parseTypeSuffix = function(self, initial) + local union, intersection = false, false + local hasType = initial ~= nil + while true do + if consume(self, S, '|') then + parseSimpleType(self, false) + union, hasType = true, true + elseif consume(self, S, '&') then + parseSimpleType(self, false) + intersection, hasType = true, true + elseif consume(self, S, '?') then + if not hasType then fail(self, "Expected a type before '?'") end + union = true + elseif is(self, S, '...') then + fail(self, "Unexpected '...' after a type") + else + break + end + if union and intersection then + fail(self, 'Mixing union and intersection types requires parentheses') + end + end + return TYPE + end + + parseType = function(self) + local initial + if not is(self, S, '|') and not is(self, S, '&') then initial = parseSimpleType(self, false) end + return parseTypeSuffix(self, initial) + end + + local function parseTypeOrPack(self) + if startsPack(self) then return parsePack(self) end + if is(self, S, '(') then + local result = parseSimpleType(self, true) + if result.kind == 'pack' then + if result.count == 1 and not result.tail and typeFollows(self) then + return parseTypeSuffix(self, TYPE) + end + return result + end + return parseTypeSuffix(self, result) + end + return parseType(self) + end + + parseTypeArguments = function(self) + expect(self, S, '<') + if consume(self, S, '>') then return end + repeat + parseTypeOrPack(self) + until not consume(self, S, ',') + expect(self, S, '>') + end + + parseGenerics = function(self, defaultsAllowed) + if not consume(self, S, '<') then return false end + local seenPack, seenDefault = false, false + repeat + expect(self, I) + local pack = consume(self, S, '...') + if seenPack and not pack then fail(self, 'Generic types must precede generic type packs') end + seenPack = seenPack or pack + if consume(self, S, '=') then + if not defaultsAllowed then fail(self, 'Generic defaults are only allowed in type aliases') end + seenDefault = true + if pack then + local result = parseTypeOrPack(self) + if result.kind ~= 'pack' then fail(self, 'Expected a type pack as the generic default') end + else + parseType(self) + end + elseif seenDefault then + fail(self, 'A generic parameter after a default must also have a default') + end + until not consume(self, S, ',') + expect(self, S, '>') + return true + end + + function Parser:parseLuauTypeAnnotation() + if self.luaVersion == 'LuaU' and consume(self, S, ':') then + parseType(self) + return true + end + return false + end + + function Parser:parseLuauReturnAnnotation() + if self.luaVersion == 'LuaU' and consume(self, S, ':') then + parseReturnType(self) + return true + end + return false + end + + function Parser:parseLuauVarargAnnotation() + if self.luaVersion == 'LuaU' and consume(self, S, ':') then + if is(self, I) and is(self, S, '...', 1) then parsePack(self) else parseType(self) end + return true + end + return false + end + + function Parser:parseLuauGenerics() + if self.luaVersion == 'LuaU' then return parseGenerics(self, false) end + return false + end + + function Parser:parseLuauCast() + if self.luaVersion == 'LuaU' and consume(self, S, '::') then + parseType(self) + return true + end + return false + end + + function Parser:parseLuauInstantiation() + if self.luaVersion ~= 'LuaU' or not is(self, S, '<') or not is(self, S, '<', 1) then return false end + get(self) + parseTypeArguments(self) + expect(self, S, '>') + return true + end + + function Parser:parseLuauDeclaration() + if self.luaVersion ~= 'LuaU' then return false end + local offset = is(self, I, 'export') and 1 or 0 + if not is(self, I, 'type', offset) then return false end + if not is(self, I, nil, offset + 1) and not is(self, K, 'function', offset + 1) then return false end + if offset == 1 then get(self) end + get(self) + if consume(self, K, 'function') then + expect(self, I) + parseGenerics(self, false) + expect(self, S, '(') + local scope = Scope:new(self.lexicalScope or Scope:newGlobal()) + local savedRoot = self.typeFunctionRoot + self.typeFunctionRoot = scope + local args = self:functionArgList(scope) + expect(self, S, ')') + self:parseLuauReturnAnnotation() + self:functionBody(scope, args) + expect(self, K, 'end') + self.typeFunctionRoot = savedRoot + else + expect(self, I) + parseGenerics(self, true) + expect(self, S, '=') + parseType(self) + end + return true + end +end diff --git a/src/prometheus/parser.lua b/src/prometheus/parser.lua index f863e9e..dc2dfa7 100644 --- a/src/prometheus/parser.lua +++ b/src/prometheus/parser.lua @@ -5,8 +5,6 @@ -- This Script provides a class for parsing of lua code. -- This Parser is Capable of parsing LuaU and Lua5.1 -- --- Note that when parsing LuaU "continue" is treated as a Keyword, so no variable may be named "continue" even though this would be valid in LuaU --- -- Settings Object: -- luaVersion : The LuaVersion of the Script - Currently Supported : Lua51 and LuaU -- AllowInternalIdentifiers : Skip reserved-prefix warnings for generated code (default false) @@ -18,6 +16,8 @@ local util = require("prometheus.util"); local Ast = require("prometheus.ast"); local Scope = require("prometheus.scope"); local logger = require("logger"); +local lowerExports = require("prometheus.luau_exports"); +local lowerIteration = require("prometheus.luau_iteration"); local AstKind = Ast.AstKind; @@ -47,7 +47,8 @@ local function generateError(self, message) if(self.index > self.length) then token = self.tokens[self.length]; elseif(self.index < 1) then - return "Parsing Error at Position 0:0, " .. message; + token = self.tokens[1]; + if not token then return "Parsing Error at Position 0:0, " .. message end else token = self.tokens[self.index]; end @@ -70,6 +71,7 @@ function Parser:new(settings) tokens = {}; length = 0; index = 0; + lowerLuauIteration = settings and settings.LowerLuauIteration == true; }; setmetatable(parser, self); @@ -101,23 +103,14 @@ local function get(self) end local function is(self, kind, sourceOrN, n) - local token = peek(self, n); - - local source = nil; - if(type(sourceOrN) == "string") then - source = sourceOrN; - else + local source; + if type(sourceOrN) == "number" then n = sourceOrN; + else + source = sourceOrN; end - n = n or 0; - - if(token.kind == kind) then - if(source == nil or token.source == source) then - return true; - end - end - - return false; + local tk = peek(self, n); + return tk.kind == kind and (source == nil or tk.source == source); end local function consume(self, kind, source) @@ -142,18 +135,81 @@ local function expect(self, kind, source) end end +require("prometheus.luau_types")(Parser, TokenKind, is, consume, expect, peek, get); +require("prometheus.luau_attributes")(Parser, TokenKind, is, consume, expect, peek, get); + +local function markConst(scope, id) + scope.constBindings = scope.constBindings or {}; + scope.constBindings[id] = true; +end + +local function assertMutable(self, scope, id) + scope.assignedBindings = scope.assignedBindings or {}; + scope.assignedBindings[id] = true; + if scope.constBindings and scope.constBindings[id] then + if self.disableLog then error() end + logger:error(generateError(self, string.format("Cannot reassign constant '%s'", scope:getVariableName(id)))); + end +end + +-- Match Luau's reachable-branch check for locals skipped by repeat/continue. +local function repeatContinue(block, loop) + for _, node in ipairs(block.statements) do + if node.kind == AstKind.ContinueStatement then + return node.loop == loop and node.sourceToken or nil; + elseif node.kind == AstKind.ReturnStatement or node.kind == AstKind.BreakStatement then + return nil; + elseif node.kind == AstKind.DoStatement then + local found = repeatContinue(node.body, loop); + if found then return found end + elseif node.kind == AstKind.IfStatement then + local branches = {{condition = node.condition, body = node.body}}; + for _, branch in ipairs(node.elseifs) do branches[#branches + 1] = branch end + local remaining = true; + for _, branch in ipairs(branches) do + if remaining and (not branch.condition.isConstant or branch.condition.value) then + local found = repeatContinue(branch.body, loop); + if found then return found end + if branch.condition.isConstant then remaining = false end + end + end + if remaining and node.elsebody then + local found = repeatContinue(node.elsebody, loop); + if found then return found end + end + end + end +end + -- Parse the given code to an Abstract Syntax Tree function Parser:parse(code) + -- A failed parse must not leave input behind for the next invocation. + self.tokenizer:reset(); + self.tokens, self.index, self.length = {}, 0, 0; + self.disableLog = false; + self.noSimplify = false; + self.varargAllowed = true; + self.functionDepth = 0; + self.lexicalScope = nil; + self.typeFunctionRoot = nil; + self.moduleScope = nil; + self.exportBindings, self.exportNames = {}, {}; + self.hasModuleReturn = false; self.tokenizer:append(code); self.tokens = self.tokenizer:scanAll(); + self.length = #self.tokens; -- Create Global Variable Scope local globalScope = Scope:newGlobal(); local ast = Ast.TopNode(self:block(globalScope, false), globalScope); + ast.luaVersion = self.luaVersion; + ast.directives = self.luaVersion == LuaVersion.LuaU and self.tokenizer.directives or nil; -- File Must be Over when Top Node is Fully Parsed expect(self, TokenKind.Eof); + if #self.exportBindings > 0 then lowerExports(ast, self.exportBindings) end + if self.luaVersion == LuaVersion.LuaU and self.lowerLuauIteration then lowerIteration(ast, Parser) end logger:debug("Cleaning up Parser for next Use ...") -- Clean Up @@ -170,24 +226,76 @@ end -- Parse a Code Block function Parser:block(parentScope, currentLoop, scope) scope = scope or Scope:new(parentScope); + if not self.moduleScope then self.moduleScope = scope end + local savedScope = self.lexicalScope; + self.lexicalScope = scope; local statements = {}; - repeat - local statement, isTerminatingStatement = self:statement(scope, currentLoop); - table.insert(statements, statement); - until isTerminatingStatement or not statement - - -- Consume Eventual Semicolon after terminating return, break or continue - consume(self, TokenKind.Symbol, ";"); + while true do + -- Erased type declarations still count as statements for semicolon syntax. + local declaration = self:parseLuauDeclaration(); + local statement, isTerminatingStatement; + if not declaration then statement, isTerminatingStatement = self:statement(scope, currentLoop) end + if not declaration and not statement then break end + if statement then table.insert(statements, statement) end + consume(self, TokenKind.Symbol, ";"); + if isTerminatingStatement then break end + end + self.lexicalScope = savedScope; return Ast.Block(statements, scope); end function Parser:statement(scope, currentLoop) - -- Skip all semicolons before next real statement - -- NOP statements are therefore ignored - while(consume(self, TokenKind.Symbol, ";")) do - + local attributes = self:parseLuauAttributes(scope); + if attributes then + local node, terminating = self:statement(scope, currentLoop); + if not node or (node.kind ~= AstKind.FunctionDeclaration and node.kind ~= AstKind.LocalFunctionDeclaration) then + logger:error(generateError(self, "Function attributes require a function declaration")); + end + if node.attributes then logger:error(generateError(self, "Duplicate function attribute list")) end + node.attributes = attributes; + return node, terminating; + end + if self.luaVersion == LuaVersion.LuaU and is(self, TokenKind.Ident, "export") + and (is(self, TokenKind.Keyword, "local", 1) or is(self, TokenKind.Keyword, "function", 1) or is(self, TokenKind.Ident, "const", 1)) then + get(self); + if scope ~= self.moduleScope or self.functionDepth ~= 0 then logger:error(generateError(self, "Value exports require a top-level statement")) end + if self.hasModuleReturn then logger:error(generateError(self, "Value exports conflict with a top-level return")) end + local node; + if consume(self, TokenKind.Keyword, "function") then + local ident = expect(self, TokenKind.Ident); + local id = scope:addVariable(ident.value, ident); + markConst(scope, id); + local funcScope = Scope:new(scope); + self:parseLuauGenerics(); + expect(self, TokenKind.Symbol, "("); + local args = self:functionArgList(funcScope); + expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); + local body = self:functionBody(funcScope, args); + expect(self, TokenKind.Keyword, "end"); + node = Ast.LocalFunctionDeclaration(scope, id, args, body); + else + if is(self, TokenKind.Keyword, "function", 1) or is(self, TokenKind.Symbol, "@", 1) then + logger:error(generateError(self, "Use 'export function' for an exported function")); + end + node = self:statement(scope, currentLoop); + if not node or node.kind ~= AstKind.LocalVariableDeclaration then logger:error(generateError(self, "Expected an exported binding")) end + end + node.isExported = true; + for _, id in ipairs(node.ids or {node.id}) do + local name = scope:getVariableName(id); + if self.exportNames[name] then logger:error(generateError(self, "Duplicate exported identifier '" .. name .. "'")) end + self.exportNames[name] = true; + self.exportBindings[#self.exportBindings + 1] = {scope = scope, id = id, name = name}; + end + return node; + end + -- Preserve the legacy Lua 5.1 empty-statement behavior. Luau permits only + -- one optional semicolon after an actual statement, consumed by block(). + if self.luaVersion == LuaVersion.Lua51 then + while consume(self, TokenKind.Symbol, ";") do end end -- Break Statement - only valid inside of Loops @@ -200,16 +308,6 @@ function Parser:statement(scope, currentLoop) return Ast.BreakStatement(currentLoop, scope), true; end - -- Continue Statement - only valid inside of Loops - only valid in LuaU - if(self.luaVersion == LuaVersion.LuaU and consume(self, TokenKind.Keyword, "continue")) then - if(not currentLoop) then - if self.disableLog then error() end; - logger:error(generateError(self, "the continue Statement is only valid inside of loops")); - end - -- Return true as Second value because continue must be the last Statement in a block - return Ast.ContinueStatement(currentLoop, scope), true; - end - -- do ... end Statement if(consume(self, TokenKind.Keyword, "do")) then local body = self:block(scope, currentLoop); @@ -234,13 +332,32 @@ function Parser:statement(scope, currentLoop) stat.body = self:block(nil, stat, repeatScope); expect(self, TokenKind.Keyword, "until"); stat.condition = self:expression(repeatScope); + stat.continueToken = repeatContinue(stat.body, stat); + if self.luaVersion == LuaVersion.LuaU and stat.continueToken then + local visitast = require("prometheus.visitast"); + visitast(Ast.TopNode(Ast.Block({Ast.ReturnStatement{stat.condition}}, repeatScope), scope), function(node, data) + if node.kind == AstKind.VariableExpression and node.scope == repeatScope then + local declaration = repeatScope.declarationTokens and repeatScope.declarationTokens[node.id]; + local capturedConstant = data.functionData.depth > 0 and repeatScope.constantInitializers + and repeatScope.constantInitializers[node.id] and not (repeatScope.assignedBindings and repeatScope.assignedBindings[node.id]); + if declaration and declaration.startPos > stat.continueToken.startPos and not capturedConstant then + logger:error(generateError(self, "Local '" .. repeatScope:getVariableName(node.id) + .. "' in the repeat condition is skipped by continue")); + end + end + end); + end return stat; end -- Return Statement if(consume(self, TokenKind.Keyword, "return")) then + if self.luaVersion == LuaVersion.LuaU and self.functionDepth == 0 then + self.hasModuleReturn = true; + if #self.exportBindings > 0 then logger:error(generateError(self, "Value exports conflict with a top-level return")) end + end local args = {}; - if(not is(self, TokenKind.Keyword, "end") and not is(self, TokenKind.Keyword, "elseif") and not is(self, TokenKind.Keyword, "else") and not is(self, TokenKind.Symbol, ";") and not is(self, TokenKind.Eof)) then + if(not is(self, TokenKind.Keyword, "end") and not is(self, TokenKind.Keyword, "elseif") and not is(self, TokenKind.Keyword, "else") and not is(self, TokenKind.Keyword, "until") and not is(self, TokenKind.Symbol, ";") and not is(self, TokenKind.Eof)) then args = self:exprList(scope); end -- Return true as Second value because return must be the last Statement in a block @@ -284,39 +401,47 @@ function Parser:statement(scope, currentLoop) local baseScope = obj.scope; local baseId = obj.id; local indices = obj.indices; + if #indices == 0 then assertMutable(self, baseScope, baseId) end local funcScope = Scope:new(scope); + self:parseLuauGenerics(); expect(self, TokenKind.Symbol, "("); local args = self:functionArgList(funcScope); expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); if(obj.passSelf) then local id = funcScope:addVariable("self", obj.token); table.insert(args, 1, Ast.VariableExpression(funcScope, id)); end - local body = self:block(nil, false, funcScope); + local body = self:functionBody(funcScope, args); expect(self, TokenKind.Keyword, "end"); return Ast.FunctionDeclaration(baseScope, baseId, indices, args, body); end - -- Local Function or Variable Declaration - if(consume(self, TokenKind.Keyword, "local")) then + -- 'const' is contextual and remains a valid identifier outside declarations. + local isConst = self.luaVersion == LuaVersion.LuaU and is(self, TokenKind.Ident, "const") + and (is(self, TokenKind.Ident, nil, 1) or is(self, TokenKind.Keyword, "function", 1) or is(self, TokenKind.Symbol, "@", 1)); + if consume(self, TokenKind.Keyword, "local") or (isConst and consume(self, TokenKind.Ident, "const")) then -- Local Function Declaration if(consume(self, TokenKind.Keyword, "function")) then local ident = expect(self, TokenKind.Ident); local name = ident.value; local id = scope:addVariable(name, ident); + if isConst then markConst(scope, id) end local funcScope = Scope:new(scope); + self:parseLuauGenerics(); expect(self, TokenKind.Symbol, "("); local args = self:functionArgList(funcScope); expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); - local body = self:block(nil, false, funcScope); + local body = self:functionBody(funcScope, args); expect(self, TokenKind.Keyword, "end"); return Ast.LocalFunctionDeclaration(scope, id, args, body); @@ -324,12 +449,33 @@ function Parser:statement(scope, currentLoop) -- Local Variable Declaration local ids = self:nameList(scope); + if isConst then + for _, id in ipairs(ids) do markConst(scope, id) end + end local expressions = {}; if(consume(self, TokenKind.Symbol, "=")) then expressions = self:exprList(scope); + elseif isConst then + if self.disableLog then error() end + logger:error(generateError(self, "A const declaration requires an initializer")); + end + + if isConst and #expressions < #ids then + local last = expressions[#expressions]; + local expands = last and not last.isParenthesizedExpression + and (last.kind == AstKind.FunctionCallExpression or last.kind == AstKind.PassSelfFunctionCallExpression + or last.kind == AstKind.VarargExpression); + if not expands then + if self.disableLog then error() end + logger:error(generateError(self, "A const declaration requires an initializer for each binding")); + end end -- Variables can only be reffered to in the next statement, so the id's are enabled after the expressions have been parsed + scope.constantInitializers = scope.constantInitializers or {}; + for i, id in ipairs(ids) do + if expressions[i] and expressions[i].isConstant then scope.constantInitializers[id] = true end + end self:enableNameList(scope, ids); if(#expressions > #ids) then @@ -344,11 +490,17 @@ function Parser:statement(scope, currentLoop) -- For Statement if(consume(self, TokenKind.Keyword, "for")) then -- Normal for Statement - if(is(self, TokenKind.Symbol, "=", 1)) then + local savedIndex = self.index; + expect(self, TokenKind.Ident); + self:parseLuauTypeAnnotation(); + local isNumericFor = is(self, TokenKind.Symbol, "="); + self.index = savedIndex; + if(isNumericFor) then local forScope = Scope:new(scope); local ident = expect(self, TokenKind.Ident); local varId = forScope:addDisabledVariable(ident.value, ident); + self:parseLuauTypeAnnotation(); expect(self, TokenKind.Symbol, "="); local initialValue = self:expression(scope); @@ -391,6 +543,10 @@ function Parser:statement(scope, currentLoop) local expr = self:primaryExpression(scope); -- Variable Assignment or Function Call if expr then + if expr.isInterpolatedStringExpression then + if self.disableLog then error() end + logger:error(generateError(self, "An interpolated string is not a valid statement")); + end -- Function Call Statement if(expr.kind == AstKind.FunctionCallExpression) then return Ast.FunctionCallStatement(expr.base, expr.args); @@ -403,11 +559,29 @@ function Parser:statement(scope, currentLoop) -- Variable Assignment if(expr.kind == AstKind.IndexExpression or expr.kind == AstKind.VariableExpression) then + -- A call or assignment using this name takes precedence over continue. + local nextToken = peek(self); + local assignmentFollows = nextToken.kind == TokenKind.Symbol + and (nextToken.source == "=" or nextToken.source == "," or nextToken.source:match("^[+%-%*/%%%^%.]+=$")); + if self.luaVersion == LuaVersion.LuaU and expr.kind == AstKind.VariableExpression + and not expr.isParenthesizedExpression and expr.scope:getVariableName(expr.id) == "continue" + and not assignmentFollows then + if not currentLoop then + if self.disableLog then error() end + logger:error(generateError(self, "the continue Statement is only valid inside of loops")); + end + local node = Ast.ContinueStatement(currentLoop, scope); + node.sourceToken = peek(self, -1); + return node, true; + end if(expr.kind == AstKind.IndexExpression) then + if expr.isParenthesizedExpression then logger:error(generateError(self, "Assigned expression must be a variable or field")) end expr.kind = AstKind.AssignmentIndexing end if(expr.kind == AstKind.VariableExpression) then + if expr.isParenthesizedExpression then logger:error(generateError(self, "Assigned expression must be a variable or field")) end expr.kind = AstKind.AssignmentVariable + assertMutable(self, expr.scope, expr.id); end if(self.luaVersion == LuaVersion.LuaU) then @@ -432,6 +606,11 @@ function Parser:statement(scope, currentLoop) return Ast.CompoundDivStatement(expr, rhs); end + if(consume(self, TokenKind.Symbol, "//=")) then + local rhs = self:expression(scope); + return Ast.CompoundFloorDivStatement(expr, rhs); + end + if(consume(self, TokenKind.Symbol, "%=")) then local rhs = self:expression(scope); return Ast.CompoundModStatement(expr, rhs); @@ -461,11 +640,13 @@ function Parser:statement(scope, currentLoop) end if(expr.kind == AstKind.IndexExpression or expr.kind == AstKind.VariableExpression) then + if expr.isParenthesizedExpression then logger:error(generateError(self, "Assigned expression must be a variable or field")) end if(expr.kind == AstKind.IndexExpression) then expr.kind = AstKind.AssignmentIndexing end if(expr.kind == AstKind.VariableExpression) then expr.kind = AstKind.AssignmentVariable + assertMutable(self, expr.scope, expr.id); end table.insert(lhs, expr); else @@ -489,17 +670,13 @@ function Parser:statement(scope, currentLoop) end function Parser:primaryExpression(scope) - local i = self.index; - local s = self; - self.disableLog = true; - local status, val = pcall(self.expressionFunctionCall, self, scope); - self.disableLog = false; - if(status) then - return val; - else - self.index = i; + local tk = peek(self); + if tk.kind == TokenKind.Eof or (tk.kind == TokenKind.Keyword + and (tk.source == "end" or tk.source == "else" or tk.source == "elseif" or tk.source == "until")) then return nil; end + -- Invalid calls must keep their diagnostics, including vararg and type errors. + return self:expressionFunctionCall(scope); end -- List of expressions Seperated by a comma @@ -519,11 +696,13 @@ function Parser:nameList(scope) local ident = expect(self, TokenKind.Ident); local id = scope:addDisabledVariable(ident.value, ident); + self:parseLuauTypeAnnotation(); table.insert(ids, id); while(consume(self, TokenKind.Symbol, ",")) do ident = expect(self, TokenKind.Ident); id = scope:addDisabledVariable(ident.value, ident); + self:parseLuauTypeAnnotation(); table.insert(ids, id); end @@ -566,7 +745,11 @@ end -- Expression function Parser:expression(scope) - return self:expressionOr(scope); + local savedScope = self.lexicalScope; + self.lexicalScope = scope; + local result = self:expressionOr(scope); + self.lexicalScope = savedScope; + return result; end function Parser:expressionOr(scope) @@ -574,7 +757,7 @@ function Parser:expressionOr(scope) if(consume(self, TokenKind.Keyword, "or")) then local rhs = self:expressionOr(scope); - return Ast.OrExpression(lhs, rhs, true); + return Ast.OrExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -585,7 +768,7 @@ function Parser:expressionAnd(scope) if(consume(self, TokenKind.Keyword, "and")) then local rhs = self:expressionAnd(scope); - return Ast.AndExpression(lhs, rhs, true); + return Ast.AndExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -597,37 +780,37 @@ function Parser:expressionComparision(scope) local found = false; if(consume(self, TokenKind.Symbol, "<")) then local rhs = self:expressionStrCat(scope); - curr = Ast.LessThanExpression(curr, rhs, true); + curr = Ast.LessThanExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, ">")) then local rhs = self:expressionStrCat(scope); - curr = Ast.GreaterThanExpression(curr, rhs, true); + curr = Ast.GreaterThanExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "<=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.LessThanOrEqualsExpression(curr, rhs, true); + curr = Ast.LessThanOrEqualsExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, ">=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.GreaterThanOrEqualsExpression(curr, rhs, true); + curr = Ast.GreaterThanOrEqualsExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "~=")) then local rhs = self:expressionStrCat(scope); - curr = Ast.NotEqualsExpression(curr, rhs, true); + curr = Ast.NotEqualsExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "==")) then local rhs = self:expressionStrCat(scope); - curr = Ast.EqualsExpression(curr, rhs, true); + curr = Ast.EqualsExpression(curr, rhs, not self.noSimplify); found = true; end until not found; @@ -640,7 +823,7 @@ function Parser:expressionStrCat(scope) if(consume(self, TokenKind.Symbol, "..")) then local rhs = self:expressionStrCat(scope); - return Ast.StrCatExpression(lhs, rhs, true); + return Ast.StrCatExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -653,13 +836,13 @@ function Parser:expressionAddSub(scope) local found = false; if(consume(self, TokenKind.Symbol, "+")) then local rhs = self:expressionMulDivMod(scope); - curr = Ast.AddExpression(curr, rhs, true); + curr = Ast.AddExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "-")) then local rhs = self:expressionMulDivMod(scope); - curr = Ast.SubExpression(curr, rhs, true); + curr = Ast.SubExpression(curr, rhs, not self.noSimplify); found = true; end until not found; @@ -675,19 +858,25 @@ function Parser:expressionMulDivMod(scope) local found = false; if(consume(self, TokenKind.Symbol, "*")) then local rhs = self:expressionUnary(scope); - curr = Ast.MulExpression(curr, rhs, true); + curr = Ast.MulExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "/")) then local rhs = self:expressionUnary(scope); - curr = Ast.DivExpression(curr, rhs, true); + curr = Ast.DivExpression(curr, rhs, not self.noSimplify); + found = true; + end + + if(consume(self, TokenKind.Symbol, "//")) then + local rhs = self:expressionUnary(scope); + curr = Ast.FloorDivExpression(curr, rhs, not self.noSimplify); found = true; end if(consume(self, TokenKind.Symbol, "%")) then local rhs = self:expressionUnary(scope); - curr = Ast.ModExpression(curr, rhs, true); + curr = Ast.ModExpression(curr, rhs, not self.noSimplify); found = true; end until not found; @@ -698,17 +887,17 @@ end function Parser:expressionUnary(scope) if(consume(self, TokenKind.Keyword, "not")) then local rhs = self:expressionUnary(scope); - return Ast.NotExpression(rhs, true); + return Ast.NotExpression(rhs, not self.noSimplify); end if(consume(self, TokenKind.Symbol, "#")) then local rhs = self:expressionUnary(scope); - return Ast.LenExpression(rhs, true); + return Ast.LenExpression(rhs, not self.noSimplify); end if(consume(self, TokenKind.Symbol, "-")) then local rhs = self:expressionUnary(scope); - return Ast.NegateExpression(rhs, true); + return Ast.NegateExpression(rhs, not self.noSimplify); end return self:expressionPow(scope); @@ -717,10 +906,16 @@ end function Parser:expressionPow(scope) local lhs = self:tableOrFunctionLiteral(scope); + if self:parseLuauCast() then + if lhs.kind == AstKind.FunctionCallExpression or lhs.kind == AstKind.PassSelfFunctionCallExpression or lhs.kind == AstKind.VarargExpression then + lhs.isParenthesizedExpression = true; + end + end + if(consume(self, TokenKind.Symbol, "^")) then -- Allow unary operators on the rhs (e.g. 2 ^ #x, 2 ^ -x) while preserving right-associativity. ~ SpinnySpiwal local rhs = self:expressionUnary(scope); - return Ast.PowExpression(lhs, rhs, true); + return Ast.PowExpression(lhs, rhs, not self.noSimplify); end return lhs; @@ -728,6 +923,8 @@ end -- Table Literals and Function Literals cannot directly be called or indexed function Parser:tableOrFunctionLiteral(scope) + local attributes = self:parseLuauAttributes(scope); + if attributes then return self:expressionFunctionLiteral(scope, attributes) end if(is(self, TokenKind.Symbol, "{")) then return self:tableConstructor(scope); @@ -740,24 +937,41 @@ function Parser:tableOrFunctionLiteral(scope) return self:expressionFunctionCall(scope); end -function Parser:expressionFunctionLiteral(parentScope) +function Parser:expressionFunctionLiteral(parentScope, attributes) local scope = Scope:new(parentScope); expect(self, TokenKind.Keyword, "function"); + self:parseLuauGenerics(); expect(self, TokenKind.Symbol, "("); local args = self:functionArgList(scope); expect(self, TokenKind.Symbol, ")"); + self:parseLuauReturnAnnotation(); - local body = self:block(nil, false, scope); + local body = self:functionBody(scope, args); expect(self, TokenKind.Keyword, "end"); - return Ast.FunctionLiteralExpression(args, body); + local node = Ast.FunctionLiteralExpression(args, body); + node.attributes = attributes; + return node; +end + +-- An inner function never inherits its enclosing function's vararg parameter. +-- Signature annotations are parsed before entering the new function context. +function Parser:functionBody(scope, args) + local savedVararg = self.varargAllowed; + self.varargAllowed = #args > 0 and args[#args].kind == AstKind.VarargExpression; + self.functionDepth = self.functionDepth + 1; + local body = self:block(nil, false, scope); + self.functionDepth = self.functionDepth - 1; + self.varargAllowed = savedVararg; + return body; end function Parser:functionArgList(scope) local args = {}; if(consume(self, TokenKind.Symbol, "...")) then + self:parseLuauVarargAnnotation(); table.insert(args, Ast.VarargExpression()); return args; end @@ -765,18 +979,21 @@ function Parser:functionArgList(scope) if(is(self, TokenKind.Ident)) then local ident = get(self); local name = ident.value; + self:parseLuauTypeAnnotation(); local id = scope:addVariable(name, ident); table.insert(args, Ast.VariableExpression(scope, id)); while(consume(self, TokenKind.Symbol, ",")) do if(consume(self, TokenKind.Symbol, "...")) then + self:parseLuauVarargAnnotation(); table.insert(args, Ast.VarargExpression()); return args; end - ident = get(self); + ident = expect(self, TokenKind.Ident); name = ident.value; + self:parseLuauTypeAnnotation(); id = scope:addVariable(name, ident); table.insert(args, Ast.VariableExpression(scope, id)); @@ -786,6 +1003,15 @@ function Parser:functionArgList(scope) return args; end +local function checkCallLine(self) + if self.luaVersion == LuaVersion.LuaU and is(self, TokenKind.Symbol, "(") + and peek(self, -1).line ~= peek(self).line then + local message = generateError(self, "Ambiguous function call; use ';' to separate statements"); + if self.disableLog then error(message, 0) end + logger:error(message); + end +end + function Parser:expressionFunctionCall(scope, base) base = base or self:expressionIndex(scope); @@ -793,6 +1019,17 @@ function Parser:expressionFunctionCall(scope, base) return base; end + local instantiated = false; + while self:parseLuauInstantiation() do + instantiated = true; + if base.kind == AstKind.FunctionCallExpression or base.kind == AstKind.PassSelfFunctionCallExpression then + base.isParenthesizedExpression = true; + end + end + if instantiated and (is(self, TokenKind.Symbol, ".") or is(self, TokenKind.Symbol, "[") or is(self, TokenKind.Symbol, ":")) then + return self:expressionFunctionCall(scope, self:expressionIndex(scope, base)); + end + checkCallLine(self); -- Normal Function Call local args = {}; if(is(self, TokenKind.String)) then @@ -852,7 +1089,9 @@ function Parser:expressionIndex(scope, base) -- Function Passing self if(consume(self, TokenKind.Symbol, ":")) then local passSelfFunctionName = expect(self, TokenKind.Ident).value; + self:parseLuauInstantiation(); local args = {}; + checkCallLine(self); if(is(self, TokenKind.String)) then args = { Ast.StringExpression(get(self).value), @@ -892,6 +1131,41 @@ function Parser:expressionIndex(scope, base) return base; end +function Parser:interpolatedString(scope, parts) + local format, expressions = {}, {}; + for _, part in ipairs(parts) do + if part.literal then + format[#format + 1] = part.literal:gsub("%%", "%%%%"); + else + local savedTokens, savedIndex, savedLength = self.tokens, self.index, self.length; + self.tokens, self.index, self.length = part.tokens, 0, #part.tokens; + local ok, expression = pcall(function() + local result = self:expression(scope); + expect(self, TokenKind.Eof); + return result; + end); + self.tokens, self.index, self.length = savedTokens, savedIndex, savedLength; + if not ok then error(expression, 0) end + if expression.kind == AstKind.FunctionCallExpression or expression.kind == AstKind.PassSelfFunctionCallExpression + or expression.kind == AstKind.VarargExpression then + expression.isParenthesizedExpression = true; + end + expressions[#expressions + 1] = expression; + format[#format + 1] = "%*"; + end + end + local template = table.concat(format); + if #expressions == 0 then + return Ast.StringExpression(template:gsub("%%%%", "%%")); + end + -- Luau's %* formatting has interpolation's conversion and evaluation order. + -- Looking up the string method avoids shadowing of the global string library. + local result = Ast.PassSelfFunctionCallExpression(Ast.StringExpression(template), "format", expressions); + result.isParenthesizedExpression = true; + result.isInterpolatedStringExpression = true; + return result; +end + function Parser:expressionLiteral(scope) -- () expression if(consume(self, TokenKind.Symbol, "(")) then @@ -903,12 +1177,18 @@ function Parser:expressionLiteral(scope) return expr; end + -- Luau interpolated string + if(is(self, TokenKind.InterpolatedString)) then + return self:interpolatedString(scope, get(self).value); + end + -- String Literal if(is(self, TokenKind.String)) then return Ast.StringExpression(get(self).value); end -- Number Literal + if is(self, TokenKind.Integer) then return Ast.IntegerExpression(get(self).value) end if(is(self, TokenKind.Number)) then return Ast.NumberExpression(get(self).value); end @@ -930,6 +1210,11 @@ function Parser:expressionLiteral(scope) -- Vararg Literal if(consume(self, TokenKind.Symbol, "...")) then + if not self.varargAllowed then + local message = generateError(self, "Cannot use '...' outside a vararg function"); + if self.disableLog then error(message, 0) end + logger:error(message); + end return Ast.VarargExpression(); end @@ -939,11 +1224,16 @@ function Parser:expressionLiteral(scope) local name = ident.value; local scope, id = scope:resolve(name); + if self.typeFunctionRoot and not scope.isGlobal then + local current = scope; + while current and current ~= self.typeFunctionRoot do current = current.parentScope end + if not current then logger:error(generateError(self, "Type function cannot reference outer local '" .. name .. "'")) end + end return Ast.VariableExpression(scope, id); end -- IfElse - if(LuaVersion.LuaU) then + if(self.luaVersion == LuaVersion.LuaU) then if(consume(self, TokenKind.Keyword, "if")) then local condition = self:expression(scope); expect(self, TokenKind.Keyword, "then"); @@ -989,7 +1279,9 @@ function Parser:tableConstructor(scope) local key = Ast.StringExpression(get(self).value); expect(self, TokenKind.Symbol, "="); local value = self:expression(scope); - table.insert(entries, Ast.KeyedTableEntry(key, value)); + local entry = Ast.KeyedTableEntry(key, value); + entry.isRecord = true; + table.insert(entries, entry); else local value = self:expression(scope); table.insert(entries, Ast.TableEntry(value)); diff --git a/src/prometheus/pipeline.lua b/src/prometheus/pipeline.lua index fe21f85..8012749 100644 --- a/src/prometheus/pipeline.lua +++ b/src/prometheus/pipeline.lua @@ -134,12 +134,14 @@ function Pipeline:setLuaVersion(luaVersion) }); self.unparser = Unparser:new({ LuaVersion = luaVersion; + PrettyPrint = self.PrettyPrint; }); + self.LuaVersion = luaVersion; self.conventions = conventions; end function Pipeline:getLuaVersion() - return self.luaVersion; + return self.LuaVersion; end function Pipeline:setNameGenerator(nameGenerator) @@ -196,7 +198,17 @@ function Pipeline:apply(code, filename) local parserStartTime = gettime(); local sourceLen = string.len(code); - local ast = self.parser:parse(code); + local parser = self.parser; + if self.LuaVersion == LuaVersion.LuaU then + for _, step in ipairs(self.steps) do + if step.Name == "Vmify" or step.Name == "Cff" then + parser = Parser:new({LuaVersion = LuaVersion.LuaU, LowerLuauIteration = true}); + break; + end + end + end + local ast = parser:parse(code); + local directives = ast.directives; local parserTimeDiff = gettime() - parserStartTime; logger:info(string.format("Parsing Done in %.2f seconds", parserTimeDiff)); @@ -213,6 +225,7 @@ function Pipeline:apply(code, filename) end -- Rename Variables Step + ast.directives = directives; self:renameVariables(ast); code = self:unparse(ast); diff --git a/src/prometheus/scope.lua b/src/prometheus/scope.lua index 3f01b95..de4cff7 100644 --- a/src/prometheus/scope.lua +++ b/src/prometheus/scope.lua @@ -93,6 +93,10 @@ function Scope:addVariable(name, token) table.insert(self.variables, name); local id = #self.variables; self.variablesLookup[name] = id; + if token then + self.declarationTokens = self.declarationTokens or {}; + self.declarationTokens[id] = token; + end return id; end @@ -119,6 +123,10 @@ function Scope:addDisabledVariable(name, token) table.insert(self.variables, name); local id = #self.variables; + if token then + self.declarationTokens = self.declarationTokens or {}; + self.declarationTokens[id] = token; + end return id; end @@ -331,4 +339,4 @@ function Scope:renameVariables(settings) end end -return Scope; \ No newline at end of file +return Scope; diff --git a/src/prometheus/steps/Cff.lua b/src/prometheus/steps/Cff.lua index bc19701..29219c5 100644 --- a/src/prometheus/steps/Cff.lua +++ b/src/prometheus/steps/Cff.lua @@ -18,9 +18,10 @@ Cff.SettingsDescriptor = { function Cff:init(_) end -function Cff:apply(ast) +function Cff:apply(ast, pipeline) + local settings = {LuaVersion = pipeline and pipeline:getLuaVersion() or "Lua51"}; if self.ExcludeVm and ast.vmRuntime then - local compiler = Compiler:new(); + local compiler = Compiler:new(settings); compiler.nativeExpressions = {[ast.vmRuntime] = true}; if ast.vmData then compiler.nativeExpressions[ast.vmData] = true; end local flattened = compiler:compile(ast); @@ -29,10 +30,10 @@ function Cff:apply(ast) return flattened; end if ast.vmRuntime then - local flattened = flattenVm(ast); + local flattened = flattenVm(ast, settings.LuaVersion); if flattened then return flattened; end end - local compiler = Compiler:new(); + local compiler = Compiler:new(settings); return compiler:compile(ast); end diff --git a/src/prometheus/steps/Vmify.lua b/src/prometheus/steps/Vmify.lua index 05a14cd..b2affb3 100644 --- a/src/prometheus/steps/Vmify.lua +++ b/src/prometheus/steps/Vmify.lua @@ -10,8 +10,8 @@ Vmify.SettingsDescriptor = {} function Vmify:init() end -function Vmify:apply(ast) - return Emit.ast(Compiler:compile(ast)) +function Vmify:apply(ast, pipeline) + return Emit.ast(Compiler:compile(ast, pipeline and pipeline:getLuaVersion())) end return Vmify diff --git a/src/prometheus/tokenizer.lua b/src/prometheus/tokenizer.lua index f80c854..7a362ba 100644 --- a/src/prometheus/tokenizer.lua +++ b/src/prometheus/tokenizer.lua @@ -19,7 +19,7 @@ local Tokenizer = {}; Tokenizer.EOF_CHAR = ""; Tokenizer.WHITESPACE_CHARS = lookupify{ - " ", "\t", "\n", "\r", + " ", "\t", "\n", "\r", "\v", "\f", } Tokenizer.ANNOTATION_CHARS = lookupify(chararray("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_")) @@ -33,7 +33,9 @@ Tokenizer.TokenKind = { Symbol = "Symbol", Ident = "Identifier", Number = "Number", + Integer = "Integer", String = "String", + InterpolatedString = "InterpolatedString", } Tokenizer.EOF_TOKEN = { @@ -45,6 +47,7 @@ Tokenizer.EOF_TOKEN = { } local function token(self, startPos, kind, value) + if kind ~= Tokenizer.TokenKind.Eof then self.hasTokens = true end local line, linePos = self:getPosition(self.index); local annotations = self.annotations self.annotations = {}; @@ -70,34 +73,29 @@ local function generateWarning(token, message) end function Tokenizer:getPosition(i) - local column = self.columnMap[i] - - if not column then --// `i` is bigger than self.length, this shouldn't happen, but it did. (Theres probably some error in the tokenizer, cant find it.) - column = self.columnMap[#self.columnMap] + i = math.max(1, i); + local starts = self.lineStarts or {1}; + local first, last = 1, #starts; + while first < last do + local middle = math.floor((first + last + 1) / 2); + if starts[middle] <= i then first = middle else last = middle - 1 end end - - return column.id, column.charMap[i] + return first, i - starts[first] + 1; end --// Prepare columnMap for getPosition function Tokenizer:prepareGetPosition() - local columnMap, column = {}, { charMap = {}, id = 1, length = 0 } - - for index = 1, self.length do - local character = string.sub(self.source, index, index) -- NOTE_1: this could use table.clone to reduce amount of NEWTABLE (if that causes any performance issues) - - local columnLength = column.length + 1 - column.length = columnLength - column.charMap[index] = columnLength - - if character == "\n" then - column = { charMap = {}, id = column.id + 1, length = 0 } -- NOTE_1 + local starts, index = {1}, 1; + while index <= self.length do + local char = self.source:sub(index, index); + if char == "\n" or char == "\r" then + local nextChar = self.source:sub(index + 1, index + 1); + if (char == "\n" and nextChar == "\r") or (char == "\r" and nextChar == "\n") then index = index + 1 end + starts[#starts + 1] = index + 1; end - - columnMap[index] = column + index = index + 1; end - - self.columnMap = columnMap + self.lineStarts = starts; end -- Constructor for Tokenizer @@ -121,6 +119,7 @@ function Tokenizer:new(settings) NumberCharsLookup = lookupify(conventions.NumberChars), Keywords = conventions.Keywords, KeywordsLookup = lookupify(conventions.Keywords), + HybridKeywordsLookup = lookupify(conventions.HybridKeywords or {}), BinaryNumberChars = conventions.BinaryNumberChars, BinaryNumberCharsLookup = lookupify(conventions.BinaryNumberChars); BinaryNums = conventions.BinaryNums, @@ -160,7 +159,9 @@ function Tokenizer:reset() self.length = 0; self.source = ""; self.annotations = {}; - self.columnMap = {}; + self.lineStarts = {1}; + self.hasTokens = false; + self.directives = {}; end -- Append String to this Tokenizer @@ -238,6 +239,7 @@ end -- skip one or 0 Comments and return wether one was found function Tokenizer:skipComment() if(is(self, "-", 0) and is(self, "-", 1)) then + local commentStart = self.index; self.index = self.index + 2; if(is(self, "[")) then self.index = self.index + 1; @@ -268,7 +270,14 @@ function Tokenizer:skipComment() end -- Single Line Comment -- Get all Chars to next Newline - while(self.index < self.length and self:parseAnnotation() ~= "\n") do end + while self.index < self.length and not is(self, "\n") and not is(self, "\r") do self:parseAnnotation() end + if not self.hasTokens then + local comment = self.source:sub(commentStart + 1, self.index); + if comment:match("^%-%-!native%s*$") or comment:match("^%-%-!optimize%s+[012]%s*$") then + self.directives = self.directives or {}; + self.directives[#self.directives + 1] = comment; + end + end return true; end return false; @@ -297,8 +306,76 @@ local function int(self, chars, seperators) return table.concat(buffer); end +-- Parse radix integers without depending on the host's integer width or rounding +-- at every digit. Luau saturates overflowing radix literals at UINT64_MAX. +local function radixNumber(source, base) + local high, low = 0, 0; + for i = 1, #source do + local digit = tonumber(source:sub(i, i), base); + local nextLow = low * base + digit; + high = high * base + math.floor(nextLow / 4294967296); + low = nextLow % 4294967296; + if high >= 4294967296 then return 18446744073709551616 end + end + return high * 4294967296.0 + low; +end + +function Tokenizer:luauNumber() + local startPos = self.index; + if is(self, ".") then self.index = self.index + 1 end + -- Match Luau's entire number-like token before validating its value. In + -- particular, 1foo and 1..2 must not split into otherwise valid tokens. + repeat + self.index = self.index + 1; + until not (is(self, self.NumberCharsLookup) or is(self, ".") or is(self, "_")) + if is(self, "e") or is(self, "E") then + self.index = self.index + 1; + if is(self, "+") or is(self, "-") then self.index = self.index + 1 end + end + while is(self, self.IdentCharsLookup) do self.index = self.index + 1 end + + local source = self.source:sub(startPos + 1, self.index):gsub("_", ""); + local prefix = source:sub(1, 2):lower(); + if source:sub(-1) == "i" then + local digits = source:sub(1, -2); + -- Radix integer literals are unsigned bit patterns; decimals are signed. + local valid; + if prefix == "0x" then + digits = source:sub(3, -2); + valid = digits:match("^[0-9a-fA-F]+$") and #digits:gsub("^0+", "") <= 16; + elseif prefix == "0b" then + digits = source:sub(3, -2); + valid = digits:match("^[01]+$") and #digits:gsub("^0+", "") <= 64; + else + local significant = digits:gsub("^0+", ""); + valid = digits:match("^%d+$") and (#significant < 19 or (#significant == 19 and significant <= "9223372036854775807")); + end + if not valid then logger:error(generateError(self, "Malformed or overflowing integer '" .. source .. "'")) end + return token(self, startPos, Tokenizer.TokenKind.Integer, source); + end + local value; + if prefix == "0x" or prefix == "0b" then + local digits = source:sub(3); + local pattern = prefix == "0x" and "^[0-9a-fA-F]+$" or "^[01]+$"; + if digits:match(pattern) then value = radixNumber(digits, prefix == "0x" and 16 or 2) end + else + local mantissa = source; + if source:find("[eE]") then + mantissa = source:match("^(.-)[eE][+-]?%d+$"); + end + if mantissa and (mantissa:match("^%d+%.?%d*$") or mantissa:match("^%.%d+$")) then + value = tonumber(source); + end + end + if value == nil then + logger:error(generateError(self, "Malformed number '" .. self.source:sub(startPos + 1, self.index) .. "'")); + end + return token(self, startPos, Tokenizer.TokenKind.Number, value); +end + -- Lex the next token as a Number function Tokenizer:number() + if self.luaVersion == LuaVersion.LuaU then return self:luauNumber() end local startPos = self.index; local source = expect(self, setmetatable({["."] = true}, {__index = self.NumberCharsLookup})); @@ -358,6 +435,9 @@ function Tokenizer:ident() end local tk = token(self, startPos, Tokenizer.TokenKind.Ident, source); + -- Contextual keywords remain identifiers until the parser sees a declaration + -- or a bare continue statement; they are legal names everywhere else. + tk.isHybridKeyword = self.HybridKeywordsLookup[source] == true; if(not self.allowInternalIdentifiers and string.sub(source, 1, string.len(config.IdentPrefix)) == config.IdentPrefix) then @@ -376,7 +456,7 @@ function Tokenizer:singleLineString() local char = get(self); -- Single Line String may not contain Linebreaks except when they are escaped by \ - if(char == '\n') then + if(char == '\n' or char == '\r') then self.index = self.index - 1; logger:error(generateError(self, "Unterminated String")); end @@ -402,7 +482,9 @@ function Tokenizer:singleLineString() numstr = numstr .. char; end - char = string.char(tonumber(numstr)); + local value = tonumber(numstr); + if value > 255 then logger:error(generateError(self, "Decimal escape exceeds 255")) end + char = string.char(value); elseif(self.UnicodeEscapes and char == "u") then expect(self, "{"); @@ -411,7 +493,9 @@ function Tokenizer:singleLineString() num = num .. get(self); end expect(self, "}"); - char = util.utf8char(tonumber(num, 16)); + local value = tonumber(num, 16); + if not value or value > 0x10ffff then logger:error(generateError(self, "Invalid Unicode escape")) end + char = util.utf8char(value); elseif(self.HexEscapes and char == "x") then local hex = expect(self, self.HexNumberCharsLookup) .. expect(self, self.HexNumberCharsLookup); char = string.char(tonumber(hex, 16)); @@ -420,6 +504,12 @@ function Tokenizer:singleLineString() while(is(self, Tokenizer.WHITESPACE_CHARS)) do self.index = self.index + 1; end + elseif char == "\n" or char == "\r" then + local nextChar = peek(self); + if (char == "\r" and nextChar == "\n") or (self.luaVersion == LuaVersion.Lua51 and char == "\n" and nextChar == "\r") then + self.index = self.index + 1; + end + char = "\n"; end end @@ -432,6 +522,100 @@ function Tokenizer:singleLineString() return token(self, startPos, Tokenizer.TokenKind.String, table.concat(buffer)) end +-- Scan interpolation expressions with the regular lexer, including nested +-- strings, comments, table constructors, and interpolated strings. +function Tokenizer:interpolatedString() + local startPos = self.index; + expect(self, "`"); + local parts, literal = {}, {}; + local function flush() + if #literal > 0 then + parts[#parts + 1] = {literal = table.concat(literal)}; + literal = {}; + end + end + local function escapedCharacter() + local char = get(self); + local escape = self.EscapeSequences[char]; + if type(escape) == "string" then return escape end + if self.NumericalEscapes and self.NumberCharsLookup[char] then + local digits = char; + for _ = 1, 2 do + if is(self, self.NumberCharsLookup) then digits = digits .. get(self) else break end + end + local value = tonumber(digits); + if value > 255 then logger:error(generateError(self, "Decimal escape exceeds 255")) end + return string.char(value); + end + if self.UnicodeEscapes and char == "u" then + expect(self, "{"); + local hex = ""; + while is(self, self.HexNumberCharsLookup) do hex = hex .. get(self) end + expect(self, "}"); + local value = tonumber(hex, 16); + if not value or value > 0x10ffff then + logger:error(generateError(self, "Invalid Unicode escape")); + end + return util.utf8char(value); + end + if self.HexEscapes and char == "x" then + return string.char(tonumber(expect(self, self.HexNumberCharsLookup) .. expect(self, self.HexNumberCharsLookup), 16)); + end + if self.EscapeZIgnoreNextWhitespace and char == "z" then + while is(self, Tokenizer.WHITESPACE_CHARS) do self.index = self.index + 1 end + return ""; + end + if char == "\n" or char == "\r" then + local nextChar = peek(self); + if char == "\r" and nextChar == "\n" then + self.index = self.index + 1; + end + return "\n"; + end + return char; + end + + while self.index < self.length do + local char = get(self); + if char == "`" then + flush(); + return token(self, startPos, Tokenizer.TokenKind.InterpolatedString, parts); + elseif char == "\n" or char == "\r" then + logger:error(generateError(self, "Unterminated interpolated string")); + elseif char == "\\" then + literal[#literal + 1] = escapedCharacter(); + elseif char == "{" then + if is(self, "{") then + logger:error(generateError(self, "Escape a literal '{' as \\{ in an interpolated string")); + end + flush(); + local tokens, depth = {}, 1; + while depth > 0 do + local tk = self:next(); + if tk.kind == Tokenizer.TokenKind.Eof then + logger:error(generateError(self, "Expected '}' to close interpolation")); + end + if tk.kind == Tokenizer.TokenKind.Symbol then + if tk.source == "{" then depth = depth + 1 end + if tk.source == "}" then depth = depth - 1 end + end + if depth > 0 then + tokens[#tokens + 1] = tk; + else + tokens[#tokens + 1] = { + kind = Tokenizer.TokenKind.Eof, source = "", value = "", + line = tk.line, linePos = tk.linePos, + }; + end + end + parts[#parts + 1] = {tokens = tokens}; + else + literal[#literal + 1] = char; + end + end + logger:error(generateError(self, "Unterminated interpolated string")); +end + function Tokenizer:multiLineString() local startPos = self.index; if(is(self, "[")) then @@ -520,6 +704,11 @@ function Tokenizer:next() return self:ident(); end + -- Backtick strings are only available in Luau. + if(self.luaVersion == LuaVersion.LuaU and is(self, "`")) then + return self:interpolatedString(); + end + -- Singleline String Literals if(is(self, self.StringStartLookup)) then return self:singleLineString(); diff --git a/src/prometheus/unparser.lua b/src/prometheus/unparser.lua index abf06aa..8b8708a 100644 --- a/src/prometheus/unparser.lua +++ b/src/prometheus/unparser.lua @@ -5,7 +5,7 @@ -- This Script provides a class for lua code generation from an ast -- This UnParser is Capable of generating LuaU and Lua5.1 -- --- Note that a LuaU ast can only be unparsed as LuaU if it contains any continue statements +-- Luau-only nodes require Luau output unless a compiler has lowered them. -- -- Settings Object: -- luaVersion : The LuaVersion of the Script @@ -107,7 +107,11 @@ function Unparser:unparse(ast) logger:error("Unparser:unparse expects a TopNode as first argument") end - return self:unparseBlock(ast.body); + local source = self:unparseBlock(ast.body); + if self.luaVersion == LuaVersion.LuaU and ast.directives and #ast.directives > 0 then + source = table.concat(ast.directives, "\n") .. "\n" .. source; + end + return source; end -- Helper to join parts table (optimized string building) @@ -368,6 +372,7 @@ function Unparser:unparseStatement(statement, tabbing) [AstKind.CompoundSubStatement] = "-=", [AstKind.CompoundMulStatement] = "*=", [AstKind.CompoundDivStatement] = "/=", + [AstKind.CompoundFloorDivStatement] = "//=", [AstKind.CompoundModStatement] = "%=", [AstKind.CompoundPowStatement] = "^=", [AstKind.CompoundConcatStatement] = "..=", @@ -382,7 +387,8 @@ function Unparser:unparseStatement(statement, tabbing) end end - return self:tabs(tabbing, false) .. joinParts(parts); + local attributes = statement.attributes and (table.concat(statement.attributes, " ") .. " ") or ""; + return self:tabs(tabbing, false) .. attributes .. joinParts(parts); end function Unparser:unparseExpression(expression, tabbing) @@ -406,6 +412,10 @@ function Unparser:unparseExpression(expression, tabbing) return expression.value and "true" or "false"; end + if expression.kind == AstKind.IntegerExpression then + if self.luaVersion ~= LuaVersion.LuaU then logger:error("Integer literals require a recent Luau runtime") end + return expression.value; + end if(expression.kind == AstKind.NumberExpression) then local value = expression.value; local str = tostring(value); @@ -518,12 +528,16 @@ function Unparser:unparseExpression(expression, tabbing) [AstKind.SubExpression] = "-", [AstKind.MulExpression] = "*", [AstKind.DivExpression] = "/", + [AstKind.FloorDivExpression] = "//", [AstKind.ModExpression] = "%", [AstKind.PowExpression] = "^", } op = arithmeticOps[expression.kind] if op then + if op == "//" and self.luaVersion ~= LuaVersion.LuaU then + logger:error("Floor division requires Luau output"); + end k = expression.kind local lhs = self:unparseExpression(expression.lhs, tabbing); if(Ast.astKindExpressionToNumber(expression.lhs.kind) >= Ast.astKindExpressionToNumber(k)) then @@ -632,6 +646,7 @@ function Unparser:unparseExpression(expression, tabbing) k = AstKind.FunctionLiteralExpression; if(expression.kind == k) then + if expression.attributes then push(table.concat(expression.attributes, " "), " ") end push("function", "("); for i, arg in ipairs(expression.args) do if i > 1 then @@ -699,7 +714,7 @@ function Unparser:unparseExpression(expression, tabbing) push(self:unparseExpression(expression.condition)); push(" then "); push(self:unparseExpression(expression.true_value)); - for _, elseifexp in pairs(expression.elseifs) do + for _, elseifexp in ipairs(expression.elseifs) do push(" elseif "); push(self:unparseExpression(elseifexp.condition)); push(" then "); diff --git a/src/prometheus/visitast.lua b/src/prometheus/visitast.lua index c7e963d..b84efb5 100644 --- a/src/prometheus/visitast.lua +++ b/src/prometheus/visitast.lua @@ -45,6 +45,7 @@ local compundStats = lookupify{ AstKind.CompoundSubStatement, AstKind.CompoundMulStatement, AstKind.CompoundDivStatement, + AstKind.CompoundFloorDivStatement, AstKind.CompoundModStatement, AstKind.CompoundPowStatement, AstKind.CompoundConcatStatement, @@ -183,6 +184,7 @@ local binaryExpressions = lookupify{ AstKind.SubExpression, AstKind.MulExpression, AstKind.DivExpression, + AstKind.FloorDivExpression, AstKind.ModExpression, AstKind.PowExpression, } @@ -239,7 +241,7 @@ function visitExpression(expression, previsit, postvisit, data) if(expression.kind == AstKind.IfElseExpression) then expression.condition = visitExpression(expression.condition, previsit, postvisit, data); expression.true_value = visitExpression(expression.true_value, previsit, postvisit, data); - for i, elif in pairs(expression.elseifs) do + for i, elif in ipairs(expression.elseifs) do elif.condition = visitExpression(elif.condition, previsit, postvisit, data); elif.value = visitExpression(elif.value, previsit, postvisit, data); end diff --git a/src/prometheus/vm/compiler.lua b/src/prometheus/vm/compiler.lua index ec85b9f..089cedd 100644 --- a/src/prometheus/vm/compiler.lua +++ b/src/prometheus/vm/compiler.lua @@ -5,9 +5,10 @@ local optimize = require('prometheus.vm.optimize') local Compiler = {} Compiler.__index = Compiler -function Compiler:new(analysis, node, parent) +function Compiler:new(analysis, node, parent, luaVersion) local compiler = setmetatable({ analysis = analysis, owner = node.body.scope, parent = parent, + luaVersion = parent and parent.luaVersion or luaVersion or node.luaVersion or 'Lua51', proto = {code = {}, constants = {}, children = {}, captures = {}, params = {}, vararg = node.kind == 'TopNode', maxRegister = 0}, locals = {}, upvalues = {}, constantIds = {}, registers = {}, freeRegisters = {}, @@ -161,8 +162,8 @@ function Compiler:functionLiteral(node) return register end -function Compiler:compile(ast) - local compiler = Compiler:new(analyze(ast), ast) +function Compiler:compile(ast, luaVersion) + local compiler = Compiler:new(analyze(ast), ast, nil, luaVersion) return optimize(compiler:finish(ast)) end diff --git a/src/prometheus/vm/emit.lua b/src/prometheus/vm/emit.lua index 619fcc4..84b8bbf 100644 --- a/src/prometheus/vm/emit.lua +++ b/src/prometheus/vm/emit.lua @@ -5,12 +5,32 @@ local Unparser = require('prometheus.unparser') local Enums = require('prometheus.enums') local Runtime = require('prometheus.vm.runtime') local Layout = require('prometheus.vm.layout') +local opcodes = require('prometheus.vm.opcodes') local Emit = {} +local function usesNativeLuau(proto) + for _, value in ipairs(proto.constants) do + if type(value) == 'table' and (value.isNativeLuau or value.kind == Ast.AstKind.IntegerExpression) then return true end + end + for _, child in ipairs(proto.children) do if usesNativeLuau(child) then return true end end + return false +end + +local function usesFloorDivision(proto) + for _, instruction in ipairs(proto.code) do + if instruction[1] == opcodes.IDIV then return true end + end + for _, child in ipairs(proto.children) do + if usesFloorDivision(child) then return true end + end + return false +end + function Emit.source(proto, layout) layout = layout or Layout.new() - local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.LuaU}) local function literal(value) + if type(value) == 'table' and (value.isNativeLuau or value.kind == Ast.AstKind.IntegerExpression) then return unparser:unparseExpression(value, 0) end if type(value) ~= 'table' then return unparser:unparseExpression(Ast.ConstantNode(value), 0) end @@ -36,15 +56,22 @@ function Emit.source(proto, layout) return '{' .. table.concat(parts, ',') .. '}' end local data = '{' .. prototype(proto) .. '}' - local source, count = Runtime.build(layout):gsub('__PROTOTYPES__', function() return data end) + local source, count = Runtime.build(layout, usesFloorDivision(proto)):gsub('__PROTOTYPES__', function() return data end) assert(count == 1, 'Runtime template must contain one prototype marker') return source end function Emit.ast(proto) - local ast = Parser:new({LuaVersion = Enums.LuaVersion.Lua51}):parse(Emit.source(proto)) + local version = (usesFloorDivision(proto) or usesNativeLuau(proto)) and Enums.LuaVersion.LuaU or Enums.LuaVersion.Lua51 + local ast = Parser:new({LuaVersion = version}):parse(Emit.source(proto)) ast.vmRuntime = ast.body.statements[1].args[1].base ast.vmData = ast.body.statements[1].args[1].args[1] + if usesNativeLuau(proto) then + local visitast = require('prometheus.visitast') + visitast(Ast.TopNode(Ast.Block({Ast.ReturnStatement{ast.vmData}}, ast.body.scope), ast.globalScope), function(node) + if node.kind == Ast.AstKind.FunctionLiteralExpression then node.isNativeLuau = true; return nil, true end + end) + end return ast end diff --git a/src/prometheus/vm/expressions.lua b/src/prometheus/vm/expressions.lua index efba259..ad47066 100644 --- a/src/prometheus/vm/expressions.lua +++ b/src/prometheus/vm/expressions.lua @@ -5,6 +5,7 @@ local compileTable = require("prometheus.vm.expressions_tables") local binary = { [Kind.AddExpression] = "ADD", [Kind.SubExpression] = "SUB", [Kind.MulExpression] = "MUL", [Kind.DivExpression] = "DIV", + [Kind.FloorDivExpression] = "IDIV", [Kind.ModExpression] = "MOD", [Kind.PowExpression] = "POW", [Kind.StrCatExpression] = "CONCAT", [Kind.LessThanExpression] = "LT", [Kind.LessThanOrEqualsExpression] = "LE", [Kind.GreaterThanExpression] = "GT", @@ -79,6 +80,8 @@ local function install(Compiler) local reg = self:alloc(); self:emit("NIL", reg); return reg elseif kind == Kind.NumberExpression then return load(self, numberValue(node.value)) + elseif kind == Kind.IntegerExpression then + return load(self, node) elseif kind == Kind.BooleanExpression or kind == Kind.StringExpression then return load(self, node.value) elseif kind == Kind.VariableExpression then @@ -88,13 +91,22 @@ local function install(Compiler) local reg = self:alloc(); self:emit("TAKE", reg, tuple, 1) self:free(tuple); return reg elseif kind == Kind.IndexExpression then - local base = self:expression(node.base, 1)[1] + local base + local currentLocal = self.luaVersion == 'LuaU' and node.base.kind == Kind.VariableExpression + and not node.base.scope.isGlobal and self.analysis.owners[node.base.scope] == self.owner + if not currentLocal then base = self:expression(node.base, 1)[1] end local key = self:expression(node.index, 1)[1] + if currentLocal then base = self:expression(node.base, 1)[1] end local result = self:alloc(); self:emit("GETTABLE", result, base, key) self:free(key); self:free(base); return result elseif binary[kind] then - local left = self:expression(node.lhs, 1)[1] + local left + local lhs = node.lhs + local deferRead = self.luaVersion == 'LuaU' and kind ~= Kind.StrCatExpression and lhs.kind == Kind.VariableExpression + and not lhs.scope.isGlobal and self.analysis.owners[lhs.scope] == self.owner + if not deferRead then left = self:expression(lhs, 1)[1] end local right = self:expression(node.rhs, 1)[1] + if deferRead then left = self:expression(lhs, 1)[1] end local result = self:alloc(); self:emit(binary[kind], result, left, right) self:free(right); self:free(left); return result elseif unary[kind] then @@ -142,6 +154,7 @@ local function install(Compiler) for _, jump in ipairs(endJumps) do self:patch(jump, 1, finish) end return result elseif kind == Kind.FunctionLiteralExpression then + if node.isNativeLuau then return load(self, node) end return self:functionLiteral(node) elseif kind == Kind.TableConstructorExpression then return compileTable(self, node) diff --git a/src/prometheus/vm/expressions_tables.lua b/src/prometheus/vm/expressions_tables.lua index de8b3a9..429f885 100644 --- a/src/prometheus/vm/expressions_tables.lua +++ b/src/prometheus/vm/expressions_tables.lua @@ -56,8 +56,13 @@ return function(self, node) end end else - local key = self:expression(entry.key, 1)[1] + local key + local keyNode = entry.key + local deferRead = self.luaVersion == 'LuaU' and keyNode.kind == Kind.VariableExpression + and not keyNode.scope.isGlobal and self.analysis.owners[keyNode.scope] == self.owner + if not deferRead then key = self:expression(keyNode, 1)[1] end local value = self:expression(entry.value, 1)[1] + if deferRead then key = self:expression(keyNode, 1)[1] end self:emit("SETTABLE", tableReg, key, value) self:free(value) self:free(key) diff --git a/src/prometheus/vm/flatten.lua b/src/prometheus/vm/flatten.lua index d559b10..57026fe 100644 --- a/src/prometheus/vm/flatten.lua +++ b/src/prometheus/vm/flatten.lua @@ -38,12 +38,13 @@ local function externalVariables(ast, runtime) return variables, writes end -return function(ast) +return function(ast, luaVersion) + luaVersion = luaVersion or Enums.LuaVersion.Lua51 local runtime = ast.vmRuntime local variables, writes = externalVariables(ast, runtime) if writes then return nil end - local parser = Parser:new({LuaVersion = Enums.LuaVersion.Lua51, AllowInternalIdentifiers = true}) - local unparser = Unparser:new({LuaVersion = Enums.LuaVersion.Lua51}) + local parser = Parser:new({LuaVersion = luaVersion, AllowInternalIdentifiers = true}) + local unparser = Unparser:new({LuaVersion = luaVersion}) local runtimeSource = unparser:unparseExpression(runtime, 0) local prefix = '' local arguments = '...' @@ -52,7 +53,7 @@ return function(ast) arguments = 'select(' .. (#variables + 1) .. ',...)' end local standalone = parser:parse(prefix .. 'return (' .. runtimeSource .. ')(' .. arguments .. ')') - local flattened = unparser:unparse(CffCompiler:new():compile(standalone)) + local flattened = unparser:unparse(CffCompiler:new({LuaVersion = luaVersion}):compile(standalone)) local forwarded = #variables > 0 and table.concat(variables, ',') .. ',...' or '...' local replacementSource = 'return function(...) return (function(...) ' .. flattened .. ' end)(' .. forwarded .. ') end' diff --git a/src/prometheus/vm/opcodes.lua b/src/prometheus/vm/opcodes.lua index 33d37a3..2487b64 100644 --- a/src/prometheus/vm/opcodes.lua +++ b/src/prometheus/vm/opcodes.lua @@ -4,7 +4,7 @@ local names = { 'GETUP', 'SETUP', 'GLOBAL', 'SETGLOBAL', 'GETTABLE', 'SETTABLE', 'TABLE', 'EXPAND', 'ARGS', 'TAKE', 'VARARG', 'CALL', 'RETURN', 'TAILCALL', 'CLOSURE', 'JMP', 'JFALSE', 'JTRUE', - 'ADD', 'SUB', 'MUL', 'DIV', 'MOD', 'POW', 'CONCAT', + 'ADD', 'SUB', 'MUL', 'DIV', 'IDIV', 'MOD', 'POW', 'CONCAT', 'LT', 'LE', 'GT', 'GE', 'EQ', 'NE', 'NOT', 'NEG', 'LEN', 'FORPREP', 'FORCHECK', 'CLEARS', 'CALL0', 'CALL1', 'CALL2', 'CALL3', diff --git a/src/prometheus/vm/optimize.lua b/src/prometheus/vm/optimize.lua index e1772bd..b4c36fc 100644 --- a/src/prometheus/vm/optimize.lua +++ b/src/prometheus/vm/optimize.lua @@ -12,7 +12,7 @@ end local function isUnary(op) return op == 'NOT' or op == 'NEG' or op == 'LEN' end local function isBinary(op) return op == 'ADD' or op == 'SUB' or op == 'MUL' or op == 'DIV' - or op == 'MOD' or op == 'POW' or op == 'CONCAT' or op == 'LT' + or op == 'IDIV' or op == 'MOD' or op == 'POW' or op == 'CONCAT' or op == 'LT' or op == 'LE' or op == 'GT' or op == 'GE' or op == 'EQ' or op == 'NE' end local function producer(ins) diff --git a/src/prometheus/vm/runtime.lua b/src/prometheus/vm/runtime.lua index 5b989bc..f031ebe 100644 --- a/src/prometheus/vm/runtime.lua +++ b/src/prometheus/vm/runtime.lua @@ -1,4 +1,4 @@ --- Builds the Lua 5.1 runtime used by the bytecode emitter. The emitter replaces +-- Builds the interpreter runtime used by the bytecode emitter. The emitter replaces -- __PROTOTYPES__ with its native Lua table literal before parsing the source. local dispatch = require("prometheus.vm.runtime_dispatch") local Layout = require("prometheus.vm.layout") @@ -99,7 +99,7 @@ return make(prototypes[1], {}, entryEnv)(...) end)(__PROTOTYPES__, ...) ]=] -function Runtime.build(layout) +function Runtime.build(layout, floorDivision) layout = layout or Layout.new() local operands = layout.operands local fetch = "a, b, c, d = I[" .. tostring(operands[1]) .. "], I[" @@ -108,7 +108,7 @@ function Runtime.build(layout) local source = template:gsub("__FETCH_OPERANDS__", function() return fetch end):gsub("__DISPATCH__", function() - return dispatch.build(layout.opcodes) + return dispatch.build(layout.opcodes, floorDivision) end) return source end diff --git a/src/prometheus/vm/runtime_dispatch.lua b/src/prometheus/vm/runtime_dispatch.lua index 653a1ca..bec793d 100644 --- a/src/prometheus/vm/runtime_dispatch.lua +++ b/src/prometheus/vm/runtime_dispatch.lua @@ -64,6 +64,7 @@ local bodies = { SUB = [==[R[a] = R[b] - R[c]]==], MUL = [==[R[a] = R[b] * R[c]]==], DIV = [==[R[a] = R[b] / R[c]]==], + IDIV = [==[R[a] = R[b] // R[c]]==], MOD = [==[R[a] = R[b] % R[c]]==], POW = [==[R[a] = R[b] ^ R[c]]==], CONCAT = [==[R[a] = R[b] .. R[c]]==], @@ -100,12 +101,15 @@ local bodies = { callBodies.merge(bodies) -function M.build(opcodes) +function M.build(opcodes, floorDivision) local entries = {} for name, id in pairs(opcodes) do - local body = bodies[name] - assert(body and type(id) == "number", "Missing VM handler for opcode " .. tostring(name)) - entries[#entries + 1] = { id = id, name = name, body = body } + -- Keep Lua 5.1 runtimes portable when no prototype uses floor division. + if name ~= "IDIV" or floorDivision then + local body = bodies[name] + assert(body and type(id) == "number", "Missing VM handler for opcode " .. tostring(name)) + entries[#entries + 1] = { id = id, name = name, body = body } + end end table.sort(entries, function(left, right) return left.id < right.id end) for i = 2, #entries do diff --git a/src/prometheus/vm/statements.lua b/src/prometheus/vm/statements.lua index 80bc4e0..139e2b2 100644 --- a/src/prometheus/vm/statements.lua +++ b/src/prometheus/vm/statements.lua @@ -165,11 +165,18 @@ return function(Compiler) release(self, values) end - local function assignmentTarget(self, target) + local function isCurrentLocal(self, expression) + return expression.kind == K.VariableExpression and not expression.scope.isGlobal + and self.analysis.owners[expression.scope] == self.owner + end + + local function assignmentTarget(self, target, compound) if target.kind == K.AssignmentVariable then return {scope = target.scope, id = target.id} end if target.kind == K.AssignmentIndexing then - local base = self:expression(target.base, 1)[1] + local base + if not compound or not isCurrentLocal(self, target.base) then base = self:expression(target.base, 1)[1] end local key = self:expression(target.index, 1)[1] + if not base then base = self:expression(target.base, 1)[1] end return {base = base, key = key} end error("invalid assignment target: " .. tostring(target.kind)) @@ -195,13 +202,22 @@ return function(Compiler) local compoundOp = { [K.CompoundAddStatement] = "ADD", [K.CompoundSubStatement] = "SUB", [K.CompoundMulStatement] = "MUL", [K.CompoundDivStatement] = "DIV", + [K.CompoundFloorDivStatement] = "IDIV", [K.CompoundModStatement] = "MOD", [K.CompoundPowStatement] = "POW", [K.CompoundConcatStatement] = "CONCAT", } local function compileCompound(self, node) - local target = assignmentTarget(self, node.lhs) + local target = assignmentTarget(self, node.lhs, true) local old = self:alloc() + local rhs + local localTarget = target.scope and not target.scope.isGlobal + and self.analysis.owners[target.scope] == self.owner + local deferRead = localTarget and node.kind ~= K.CompoundConcatStatement + -- Concatenation snapshots its left operand even when it is a local. + if deferRead then + rhs = self:expression(node.rhs, 1)[1] + end if target.scope then local value = self:readVariable(target.scope, target.id) self:emit("MOVE", old, value) @@ -209,9 +225,17 @@ return function(Compiler) else self:emit("GETTABLE", old, target.base, target.key) end - local rhs = self:expression(node.rhs, 1)[1] + if not deferRead then rhs = self:expression(node.rhs, 1)[1] end local result = self:alloc() self:emit(compoundOp[node.kind], result, old, rhs) + if target.base and isCurrentLocal(self, node.lhs.base) then + self:free(target.base) + target.base = self:expression(node.lhs.base, 1)[1] + end + if target.key and isCurrentLocal(self, node.lhs.index) then + self:free(target.key) + target.key = self:expression(node.lhs.index, 1)[1] + end storeTarget(self, target, result) release(self, {old, rhs, result}) if target.base then self:free(target.base); self:free(target.key) end diff --git a/tests/harness/cli.py b/tests/harness/cli.py new file mode 100644 index 0000000..a71a2ca --- /dev/null +++ b/tests/harness/cli.py @@ -0,0 +1,111 @@ +#!/usr/bin/env python3 +"""Exercise CLI argument handling using the real Lua interpreter.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +cli = root / 'cli.lua' + + +def run(args, cwd, success=True): + result = subprocess.run([lua, str(cli), *map(str, args)], cwd=cwd, + capture_output=True, text=True, timeout=60) + if (result.returncode == 0) != success: + raise AssertionError(f'{args!r}: exit {result.returncode}\n{result.stdout}\n{result.stderr}') + return result + + +def execute(path, cwd): + result = subprocess.run([lua, str(path)], cwd=cwd, capture_output=True, text=True, timeout=60, check=True) + assert result.stdout == '42\n', result.stdout + + +with tempfile.TemporaryDirectory(prefix='prometheus-cli-') as directory: + tmp = Path(directory) + source = tmp / 'Mixed Source.LUAU' + source.write_text('local value: number = 41\nprint(value + 1)\n') + + for index, preset in enumerate(('minify', 'WEAK', 'cFf', 'vMiFy', 'medium', 'STRONG')): + output = tmp / f'Output {index}.lua' + run(['--PRESET', preset, '--luau', '--NoCoLoRs', '--OUT', output, source], tmp) + execute(output, tmp) + + output = tmp / 'MixedCase.lua' + run(['--PrEsEt=MiNiFy', '--lUaU', '--o=' + str(output), source], tmp) + execute(output, tmp) + run(['--p', 'MINIFY', '--LuaU', '--O', output, source], tmp) + execute(output, tmp) + + config = tmp / 'My Config.LUA' + config.write_text('return {LuaVersion = "LuaU", Seed = 1, Steps = {}}\n') + run(['--CONFIG=' + str(config), '--OUT=' + str(output), source], tmp) + execute(output, tmp) + run(['--C', config, '--O', output, source], tmp) + execute(output, tmp) + + run(['--PrEsEt', 'minify', '--LUAU', source], tmp) + execute(tmp / 'Mixed Source.obfuscated.LUAU', tmp) + normal = tmp / 'Input.lua' + normal.write_text('print(42)\n') + run(['--lua51', '--p=minify', normal], tmp) + execute(tmp / 'Input.obfuscated.lua', tmp) + no_extension = tmp / 'Input' + no_extension.write_text('print(42)\n') + run(['--Lua51', '--p=minify', no_extension], tmp) + execute(tmp / 'Input.obfuscated.lua', tmp) + + (tmp / '--input.lua').write_text('print(42)\n') + run(['--preset=minify', '--OUT=--result.lua', '--', '--input.lua'], tmp) + execute(tmp / '--result.lua', tmp) + assert 'Usage:' in run(['--HELP'], tmp).stdout + assert 'Usage:' in run([normal, '--HELP'], tmp).stdout + assert run(['--VERSION'], tmp).stdout.strip() + assert run([normal, '-V'], tmp).stdout.strip() + + failures = [ + (['--luau', '--preset'], 'requires a value'), + (['--out'], 'requires a value'), + (['--CONFIG', '--luau', normal], 'requires a value'), + (['--out=', normal], 'requires a value'), + (['--out', '', normal], 'requires a value'), + (['--luau=true', normal], 'does not accept a value'), + (['--UNKNOWN', normal], 'Unknown option'), + (['--UNKNOWN=value', normal], 'Unknown option'), + (['-unknown', normal], 'Unknown option'), + (['--preset', 'NotARealPreset', normal], 'NotARealPreset'), + ([normal, source], 'Unexpected argument'), + (['--SAVEERRORS'], 'No input file was specified'), + (['--out=' + str(tmp / 'missing' / 'out.lua'), normal], 'Cannot write output file'), + ] + for args, message in failures: + result = run(args, tmp, success=False) + assert message in result.stderr, (args, result.stderr) + assert "attempt to index" not in result.stderr + + mock_update = tmp / 'mock-update.lua' + mock_update.write_text("""local cli = arg[1] +arg = {'UPDate'} +os.execute = function() return 0 end +os.exit = function(status) error('mock exit: ' .. status) end +local ok, err = pcall(dofile, cli) +assert(not ok and tostring(err):find('mock exit: 0', 1, true)) +""") + subprocess.run([lua, str(mock_update), str(cli)], cwd=tmp, capture_output=True, + text=True, timeout=60, check=True) + + malformed_type = tmp / 'MalformedType.luau' + malformed_type.write_text('local x: |& usudhd subd = 100\n') + rejected_output = tmp / 'RejectedOutput.luau' + for preset in ('minify', 'cff', 'vmify', 'weak', 'medium', 'strong'): + result = run(['--luau', '--preset', preset, '--out', rejected_output, malformed_type], tmp, success=False) + assert 'Expected a type' in result.stderr and '1:11' in result.stderr, result.stderr + assert not rejected_output.exists() + + invalid = tmp / 'Invalid.LUAU' + invalid.write_text('local value: = 1\n') + run(['--SaVeErRoRs', '--luau', '--preset=minify', invalid], tmp, success=False) + assert 'Parsing Error' in (tmp / 'Invalid.error.txt').read_text() +print('CLI case handling, paths, options, and diagnostics passed') diff --git a/tests/harness/luau-parser.lua b/tests/harness/luau-parser.lua new file mode 100644 index 0000000..77c4306 --- /dev/null +++ b/tests/harness/luau-parser.lua @@ -0,0 +1,81 @@ +local Prometheus = require('src.prometheus') +local Parser = require('prometheus.parser') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error + +local function fails(source, version) + local ok = pcall(function() Parser:new({LuaVersion = version}):parse(source) end) + assert(not ok, 'expected invalid ' .. version .. ' source to fail: ' .. source) +end +fails('local x: number = 1', 'Lua51') +fails('local x = if true then 1 else 2', 'Lua51') +fails('type T =', 'LuaU') +fails('local x: |& usudhd subd = 100', 'LuaU') +fails('local x: number | string & boolean = nil', 'LuaU') +fails('type T = typeof({value = })', 'LuaU') +fails('type T = {}', 'LuaU') +fails('local function f() end', 'LuaU') +fails('local x: (number, string) = nil', 'LuaU') +fails('local x: {a: number = 1', 'LuaU') +fails('local f = function(x: ) end', 'LuaU') +fails('const x = 1', 'Lua51') +fails('local x = `text {1}`', 'Lua51') +fails('const value', 'LuaU') +fails('const x, y = 1', 'LuaU') +fails('local function f() return 1, 2 end; const x, y = (f())', 'LuaU') +fails('const x = 1; x = 2', 'LuaU') +fails('const x = 1; x += 2', 'LuaU') +fails('const x = 1; local y; y, x = 2, 3', 'LuaU') +fails('const x = 1; local function f() x = 2 end', 'LuaU') +fails('const function f() end; function f() end', 'LuaU') +fails('local x = `unclosed', 'LuaU') +fails('local x = `{}`', 'LuaU') +fails('`value {1}`', 'LuaU') +fails('local x = `{{1}}`', 'LuaU') +fails('local x = `value {1`', 'LuaU') +fails('local x = `\\u{110000}`', 'LuaU') + +local emptyOk, emptyError = pcall(function() + Parser:new({LuaVersion = 'LuaU'}):parse('local x = `empty {}`') +end) +assert(not emptyOk and tostring(emptyError):find('Position 1:', 1, true), + 'empty interpolation errors must refer to the original source') + +local ok, message = pcall(function() + Parser:new({LuaVersion = 'LuaU', LowerLuauIteration = true}):parse( + 'for i in {} do end\nlocal x: = 1') +end) +assert(not ok and tostring(message):find('Position 2:', 1, true), + 'iterator helper must not shift source error lines') + +local reusable = Parser:new({LuaVersion = 'LuaU'}) +assert(not pcall(function() reusable:parse('local x: |& Broken = 1') end)) +assert(reusable:parse('local x: number = 1').body.statements[1]) + +for _, version in ipairs({'Lua51', 'LuaU'}) do + for _, source in ipairs({ + 'local function f() return ... end', + 'function object:f() print(...) end', + 'local function outer(...) return function() return ... end end', + 'local function f() do local args = {...} end end', + }) do fails(source, version) end + local parser = Parser:new({LuaVersion = version}) + assert(not pcall(function() parser:parse('local f = function() return ... end') end)) + assert(parser:parse('local function f() return 1 end; return ...')) + assert(parser:parse('local function f(...) return function(...) return ... end end')) +end +local validCall, varargError = pcall(function() + reusable:parse('local function f()\n print(...)\nend') +end) +assert(not validCall and tostring(varargError):find("Cannot use '...'", 1, true) + and tostring(varargError):find('Position 2:', 1, true), + 'invalid varargs inside calls must retain their original diagnostic') + +local pipeline = Prometheus.Pipeline:fromConfig({LuaVersion = 'Lua51', Seed = 1}) +assert(pipeline:getLuaVersion() == 'Lua51') +pipeline:setLuaVersion('LuaU') +assert(pipeline:getLuaVersion() == 'LuaU') +local result = assert(loadstring(pipeline:apply('local x: number = 3; return x')))() +assert(result == 3) +local whitespace = 'local x = `a\\z' .. string.char(11, 12) .. ' b`; return x' +assert(assert(loadstring(pipeline:apply(whitespace)))() == 'ab') +print('Luau parser and version selection checks passed') diff --git a/tests/harness/luau-recent.py b/tests/harness/luau-recent.py new file mode 100644 index 0000000..4f4d27b --- /dev/null +++ b/tests/harness/luau-recent.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Validate recent Luau values using a Lua 5.1 or LuaJIT obfuscator host.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +luau = os.environ.get('LUAU', 'luau') +names = ('Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong') + + +def run(args): + result = subprocess.run(args, cwd=root, capture_output=True, text=True, timeout=60) + assert result.returncode == 0, (args, result.stdout, result.stderr) + return result.stdout + + +def execute(source, fixture, runner): + delimiter = '=' + while ']' + delimiter + ']' in source: + delimiter += '=' + quoted = '[' + delimiter + '[' + source + ']' + delimiter + ']' + checks = '' + if fixture.stem == 'exports': + checks = ''' +assert(type(result) == 'table' and table.isfrozen(result)) +assert(result.counter == 3 and result.fixed == 2 and result.optional == nil) +local first, second = result.get() +assert(first == 3 and second == 2) +assert(not pcall(result.increment)) +assert(not pcall(function() result.fixed = 9 end)) +print('Value exports, module return, and freezing checks passed') +''' + runner.write_text('local result = assert(loadstring(' + quoted + '))()\n' + checks) + return run([luau, str(runner)]) + + +count = 0 +with tempfile.TemporaryDirectory(prefix='prometheus-recent-luau-') as directory: + directory = Path(directory) + output, runner = directory / 'output.luau', directory / 'run.luau' + for fixture in sorted((root / 'tests/luau/recent').glob('*.luau')): + expected = execute(fixture.read_text(), fixture, runner) + for index, name in enumerate(names, 1): + for seed in (1, 17): + run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', + str(index), str(seed), str(fixture), str(output)]) + actual = execute(output.read_text(), fixture, runner) + assert actual == expected, (fixture.name, name, seed, expected, actual) + count += 1 + print(f'{fixture.name}: all presets passed', flush=True) +print(f'{count} recent Luau transformations matched their baseline') diff --git a/tests/harness/luau-strings.lua b/tests/harness/luau-strings.lua new file mode 100644 index 0000000..67dc3b3 --- /dev/null +++ b/tests/harness/luau-strings.lua @@ -0,0 +1,34 @@ +local Prometheus = require('src.prometheus') +local Parser = require('prometheus.parser') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +Prometheus.Logger.errorCallback = function(message) error(message, 0) end + +for _, version in ipairs({'Lua51', 'LuaU'}) do + for _, ending in ipairs({'\n', '\r', '\r\n', '\n\r'}) do + for _, steps in ipairs({{}, {{Name = 'Cff'}}, {{Name = 'Vmify'}}}) do + local pipeline = Prometheus.Pipeline:fromConfig({LuaVersion = version, Seed = 17, Steps = steps}) + local source = 'return "a\\' .. ending .. 'b"' + if version == 'LuaU' and ending == '\n\r' then + assert(not pcall(function() pipeline:apply(source) end)) + else + assert(assert(loadstring(pipeline:apply(source)))() == 'a\nb') + end + local source = '-- comment' .. ending .. 'return 42' + assert(assert(loadstring(pipeline:apply(source)))() == 42) + end + local parser = Parser:new({LuaVersion = version}) + assert(not pcall(function() parser:parse('return "a' .. ending .. 'b"') end)) + local ok, message = pcall(function() parser:parse('-- comment' .. ending .. 'local x = "\\u{110000}"') end) + assert(not ok and tostring(message):find('Position 2:', 1, true)) + assert(parser:parse('return "valid"')) + end +end +for _, name in ipairs({'Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong'}) do + local config = {} + for key, value in pairs(Prometheus.Presets[name]) do config[key] = value end + config.LuaVersion, config.Seed = 'LuaU', 17 + local output = Prometheus.Pipeline:fromConfig(config):apply('--!native\r\n--!optimize 1\nreturn 42') + local prefix = '--!native\n--!optimize 1\n' + assert(output:sub(1, #prefix) == prefix) +end +print('String normalization, diagnostics, and directive checks passed') diff --git a/tests/harness/luau-syntax.lua b/tests/harness/luau-syntax.lua new file mode 100644 index 0000000..35afbef --- /dev/null +++ b/tests/harness/luau-syntax.lua @@ -0,0 +1,21 @@ +-- Parse a corpus without executing it, then emit the type-erased AST. +local Prometheus = require('src.prometheus') +local Parser = require('prometheus.parser') +local Unparser = require('prometheus.unparser') +Prometheus.Logger.logLevel = Prometheus.Logger.LogLevel.Error +Prometheus.Logger.errorCallback = function(message) error(message, 0) end +math.randomseed(1) + +for path in assert(io.open(arg[1], 'r')):lines() do + local file = assert(io.open(path, 'rb')) + local source = file:read('*a') + file:close() + local ok, message = pcall(function() + local ast = Parser:new({LuaVersion = 'LuaU'}):parse(source) + local generated = Unparser:new({LuaVersion = 'LuaU'}):unparse(ast) + local output = assert(io.open(path .. '.out.luau', 'wb')) + output:write(generated) + output:close() + end) + io.write(ok and 'accept' or 'reject', '\t', ok and '' or tostring(message):gsub('[\r\n\t]', ' '), '\n') +end diff --git a/tests/harness/luau-syntax.py b/tests/harness/luau-syntax.py new file mode 100644 index 0000000..4a9b019 --- /dev/null +++ b/tests/harness/luau-syntax.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Compare Luau syntax acceptance and emitted code against the official compiler.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +compiler = os.environ.get('LUAU_COMPILE', 'luau-compile') +cases = [] + + +def add(label, source, expected=None): + cases.append((label, source, expected)) + + +# These are syntax checks, not type checking: unknown names are valid type references. +add('reported regression', 'local x: |& usudhd subd = 100', False) +add('unknown type', 'local x: UnknownType = nil', True) +add('adjacent statement', 'local x: UnknownType subd = 100', True) +add('grouped combinations', 'type T = (Foo | Bar) & Baz', True) +add('ungrouped combinations', 'type T = Foo | Bar & Baz', False) + +forms = [ + 'number', 'nil', 'true', 'false', '"tag"', '[=[tag]=]', 'Foo', 'Pkg.Value', 'Pkg.Sub.Value', + '| Foo', '& Foo', '|& Foo', '&| Foo', '|| Foo', '&& Foo', '|', '&', '?', + 'Foo | Bar', 'Foo & Bar', 'Foo | Bar & Baz', 'Foo & Bar | Baz', + 'Foo | & Bar', 'Foo & | Bar', 'Foo?', 'Foo??', 'Foo? & Bar', 'Foo & Bar?', + '(Foo | Bar) & Baz', '(Foo & Bar) | Baz', 'Foo & (Bar?)', '(Foo & Bar)?', + '()', '(Foo)', '((Foo))', '(Foo, Bar)', '(name: Foo)', '(Foo,)', '(Foo)?', '(Foo, Bar)?', + '...', '...Foo', 'Foo...', 'Pkg.Value...', 'Foo...?','(Foo...)', '(...Foo)', '(Foo, ...Bar)', + '() -> ()', '(Foo) -> Bar', '(name: Foo, other: Bar) -> (Foo, Bar)', + '(Foo, ...Bar) -> (Foo, ...Bar)', '(Foo..., Bar) -> Baz', '(...Foo, Bar) -> Baz', + '(Foo...) -> Bar', '(Foo,) -> Bar', '() ->', '() : Foo', '() -> () -> Foo', + '(T) -> T', '(T, P...) -> (T, P...)', '(T) -> T', + '(T...) -> U', '(T) -> T', '<>(Foo) -> Bar', + 'Container<>', 'Container', 'Container', 'Container', + 'Container<,Foo>', 'Container', 'Container<...Foo, Bar>', + 'Container<(Foo, Bar)>', 'Container<()>', 'Container<(Foo)?>', 'Container<(Foo, Bar)?>', + 'Container<(Foo) | Bar>', 'Container>>', 'Container', + '{}', '{Foo}', '{Foo,}', '{Foo, Bar}', '{{x: Foo}}', '{x: Foo}', + '{x: Foo, y: Bar;}', '{x: Foo, Bar}', '{read x: Foo, write y: Bar}', + '{read: Foo, write: Bar}', '{read Foo}', '{read}', '{readonly x: Foo}', + '{[Foo]: Bar}', '{[Foo]: Bar, [Baz]: Qux}', '{["key"]: Foo, [Bar]: Baz}', + '{["bad\\000"]: Foo}', '{...Foo}', '{x: () -> (Foo, Bar)}', + 'typeof(value)', 'typeof({value = 1})', 'typeof((function() return 1 end)())', + 'typeof()', 'typeof(+)', 'typeof(value other)', 'typeof(1, 2)', 'typeof({x = })', + 'typeof', 'typeof.Prefix', 'typeof(function(x: |& Foo) return x end)', + '`not a singleton`', '123', 'function', +] +contexts = [ + ('local annotation', 'local x: %s = nil'), + ('function parameter', 'local function f(x: %s) end'), + ('vararg annotation', 'local function f(...: %s) end'), + ('return annotation', 'local function f(): %s return nil end'), + ('type alias', 'type Alias = %s'), + ('type assertion', 'local x = nil :: %s'), + ('type argument', 'type Alias = Container<%s>'), +] +for form in forms: + for context, template in contexts: + add(f'{context}: {form}', template % form) + +# Generic declarations differ from applications and allow defaults only in aliases. +parameters = [ + 'T', '', 'T,', ',T', 'number?', 'Pkg.T', 'T | U', 'T & U', + 'T, U', 'T...', 'T, P...', 'P..., T', 'P..., Q...', 'T...,', + 'T = number', 'T = number, U', 'T = number, U = string', + 'T, U = number', 'T = number, P...', 'T = number, P... = ()', + 'P... = ()', 'P... = (Foo, Bar)', 'P... = ...Foo', 'P... = Q...', + 'P... = Foo', 'P... = (Foo)?', 'P... = (Foo, ...Bar)', + 'P... = (), Q...', 'P... = (), Q... = ()', 'T = (Foo, Bar)', +] +for params in parameters: + add(f'alias generics: {params}', f'type Alias<{params}> = number') + add(f'function generics: {params}', f'local function f<{params}>() end') + add(f'anonymous generics: {params}', f'local f = function<{params}>() end') + add(f'function type generics: {params}', f'type Alias = <{params}>() -> ()') + +# Mutate complete types in multiple contexts so malformed input cannot simply be erased. +for op in ('|', '&'): + for following in ('Foo', '?', '| Bar', '& Bar', ')', '...', ''): + add(f'operator sequence: {op} {following}', f'local x: {op} {following} = nil') + for suffix in ('', '|', '&', 'Foo', '()', ', Bar', '; Bar'): + add(f'unfinished chain: {op} {suffix}', f'type Alias = Foo {op} {suffix}') + +for expr in ('1 +', '{x = }', 'value other', 'function() local x: |& T = 1 end', 'if true then 1'): + add(f'typeof malformed expression: {expr}', f'type Alias = typeof({expr})', False) +add('typeof does not execute', 'type Alias = typeof(error("not executed"))', True) +add('valid type function signature', 'type function f(x: T, ...: P...): T return x end', True) +add('invalid type function generics', 'type function f() end', False) + + + +# Token-level edits of valid compound types exercise malformed boundaries and nesting. +seeds = [ + ['Foo', '|', 'Bar'], + ['(', 'Foo', '&', 'Bar', ')', '?'], + ['{', 'field', ':', 'Foo', ',', '[', 'Bar', ']', ':', 'Baz', '}'], + ['(', 'Foo', ',', '...', 'Bar', ')', '->', '(', 'Baz', ',', 'Qux', '...', ')'], + ['Container', '<', 'Foo', ',', '(', 'Bar', ',', 'Baz', ')', '>'], + ['<', 'T', ',', 'P', '...', '>', '(', 'T', ',', 'P', '...', ')', '->', '(', 'T', ',', 'P', '...', ')'], + ['typeof', '(', '{', 'field', '=', '1', '}', ')'], +] +mutations = set() +for seed in seeds: + for index in range(len(seed)): + mutations.add(' '.join(seed[:index] + seed[index + 1:])) + for index in range(len(seed) + 1): + for inserted in ('|', '&', '?', ',', ':', '...', 'Foo'): + mutations.add(' '.join(seed[:index] + [inserted] + seed[index:])) +for index, form in enumerate(sorted(mutations)): + add(f'mutation local {index}', 'local x: ' + form + ' = nil') + add(f'mutation alias {index}', 'type Alias = ' + form) + + +# Contextual words must remain usable in every identifier position. +for word in ('continue', 'type', 'export', 'const', 'typeof', 'read', 'write', 'declare'): + for template in ( + 'local %s = 1; return %s', + 'local function %s() end; %s()', + 'function %s() end; %s()', + 'local function f(%s) return %s end', + 'local t = {%s = 1}; return t.%s', + 'local t = {}; function t:%s() end; t:%s()', + 'for %s = 1, 2 do print(%s) end', + 'for %s in pairs({}) do print(%s) end', + 'type %s = number; local x: %s = 1', + ): + expected = not (word == 'typeof' and template.startswith('type ')) + add(f'contextual identifier {word}: {template}', template % (word, word), expected) +for source in ( + 'continue', 'continue = 1', 'continue, x = 1, 2', 'continue += 1', + 'continue //= 2', 'continue()', 'continue {}', 'continue "text"', + 'continue.field = 1', 'continue[1] = 1', 'continue:method()', + 'while true do continue end', 'while true do (continue) end', + 'while true do continue; end', 'while true do continue;; end', + 'while true do continue; print(1) end', + 'while true do continue() print(1) end', + 'local continue = 1; while true do continue end', + 'const continue = 1; while true do continue end', + 'while true do local function f() continue end end', + 'repeat continue until true', 'while true do do continue end end', + 'type = 1; export = 2; const = 3', + 'type(number); export(); const()', +): + add(f'contextual statement: {source}', source) + +# Luau accepts a single trailing semicolon, never an empty statement. +for statement in ( + 'local x = 1', 'x = 1', 'print()', 'do end', 'while true do break end', + 'for i = 1, 2 do end', 'repeat until true', 'function f() end', + 'local function f() end', 'type T = number', 'type function f() end', + 'const x = 1', 'export type T = number', +): + for before in ('', ';', ';;'): + for after in ('', ';', ';;'): + for context in ('%s', 'do %s end', 'local function f() %s end'): + source = context % (before + statement + after) + add(f'semicolons: {source}', source, not before and len(after) <= 1) +for source in ( + '', ';', ';;', 'do ; end', 'return;', 'return;;', + 'repeat return until true', 'repeat return; until true', + 'while true do break; end', 'while true do break;; end', + 'type A = number; type B = A;', 'type A = number;; type B = A', + 'return {1; 2; x = 3;}', 'type T = {x: number; y: string;}', + 'print(); (function() end)()', 'print()\n(function() end)()', + 'local x = f\n(1)', 'local x = f(\n1\n)', 'f -- comment\n(1)', + 'object:method\n()', 'f\n{}', 'f\n"text"', + 'while true do continue\n(1) end', +): + add(f'statement boundary: {source}', source) + +# Variadic state belongs to the nearest function, including erased type syntax. +for signature in ('()', '(x)', '(...)', '(x, ...)'): + for body in ( + 'return ...', 'print(...)', 'do local x = ... end', + 'local f = function() return ... end', + 'local function f(...) return ... end; return ...', + 'local function f() return 1 end; return ...', + 'local x = `value={...}`', 'type T = typeof(...)', + 'local x: typeof(...)', 'type function f() return ... end', + 'type function f(...) return ... end; return ...', + 'local function f(x: typeof(...)): typeof(...) return x end', + ): + for declaration in ('local function f%s %s end', 'function t:f%s %s end', + 'local f = function%s %s end'): + source = declaration % (signature, body) + add(f'vararg context: {source}', source) +for source in ( + 'return ...', 'return (...)', 'local x: typeof(...) = ...', + 'local function f(): typeof(...) end', + 'local function f(...): typeof(...) return ... end', + 'local function f(x: typeof(...)) end', + 'local function f() end; return ...', + 'local function f(...) return function(...) return ... end end', + 'local function f(..., x) end', 'local function f(..., ...) end', + 'local function f() type T = typeof(function(...) return ... end) end', +): + add(f'vararg boundary: {source}', source) + +# Number-like tokens must be validated as a whole, including discarded types. +numerals = [ + '0', '01', '1.', '.5', '1_000', '1__2_', '1_.2_5', '1_e+_2_', + '0xFF', '0Xff', '0_x_F_F_', '0b101', '0B101', '0_b_1_0_', + '0b__1__0_', '1e999', '1e-999', '0x', '0b', '0x_', '0b_', + '0b102', '0xFG', '0b0b1', '1e', '1e+', '1e_+2', '1..2', + '1print()', '0x1.2', '0x1p2', '0x1p-2', '0x1e+2', '1e2.3', +] +for literal in ('1', '.5', '1.', '1e2', '0b1', '0xF', '0X1e'): + for suffix in ('_', '__', 'e', 'e+', 'e-', 'e+_2', 'e_+2', '.', '..', 'a', 'p2', 'f'): + numerals.append(literal + suffix) +for prefix, digit in (('0x', 'f'), ('0b', '1')): + for length in (1, 8, 16, 17, 53, 64, 65, 128): + numerals.append(prefix + digit * length) +for literal in sorted(set(numerals)): + for template in ('return %s', 'local value = %s; print(value)', 'type T = typeof(%s)'): + add(f'numeral: {template % literal}', template % literal) + +for source in ( + 'local a, b, c = 1, 2, 3; return a // b * c % 2 / 3', + 'local a, b = 1, 2; a //= b; return a', + 'local t = {x = 1}; t.x //= 2', + 'return -2 ^ 2 // 3', 'return 2 ^ -3 // 2', + 'return 1 /// 2', 'local a = 1; a //== 2', +): + add(f'floor division: {source}', source) + + +# Attributes, explicit instantiations, and newer value syntax share runtime names. +for attribute in ( + '@native', '@checked', '@deprecated', '@[native]', '@[native, checked]', + '@[deprecated({use = "replacement", reason = "old"})]', + '@[native(1)]', '@[native("text", true, nil, {})]', + '@[deprecated({})]', '@[deprecated("text")]', '@[deprecated(1)]', + '@[deprecated({["use"] = "replacement"})]', '@[deprecated({bad = "x"})]', + '@[deprecated({use = 1})]', '@[native(1 + 2)]', '@[native(-1)]', + '@[native((1))]', '@[native(function() end)]', '@[native(value)]', + '@[native({field = value})]', '@[native(1i)]', '@[native(-1i)]', + '@[native, native]', '@native @native', '@[]', '@[native,]', '@unknown', '@ native', +): + for template in ('%s local function f() end', '%s function f() end', + 'local f = %s function() end', 'local %s function f() end', '%s local x = 1'): + add(f'attribute: {attribute}: {template}', template % attribute) +for args in ('number', 'number, string', '', 'number,', '...number', 'T...', + '(number, string)', 'number | string', 'typeof(1)', 'number = string', '|& T'): + for template in ('return f<<%s>>(1)', 'return object:method<<%s>>(1)', + 'return f<<%s>>', 'return f<<%s>>.field', 'return f<<%s>>[1]', + 'return f<<%s>><>(1)'): + add(f'explicit instantiation: {template % args}', template % args) + +for literal in ('0i', '01i', '1_000i', '9223372036854775807i', '9223372036854775808i', + '-9223372036854775808i', '0xffffffffffffffffi', '0x10000000000000000i', + '0b' + '1' * 64 + 'i', '0b' + '1' * 65 + 'i', '0x0001i', + '0bi', '0xi', '1.0i', '1e2i', '.1i', '1ii', '0b2i'): + add(f'integer literal: {literal}', 'return ' + literal) +for declaration in ('export local x = 1', 'export const x = 1', 'export function f() end', + 'export local x', 'export local x, y = 1, 2', + 'export local function f() end', 'export const function f() end'): + for template in ('%s', 'do %s end', 'local function f() %s end', + '%s; return 1', 'do return 1 end; %s', '@native %s'): + add(f'value exports: {template % declaration}', template % declaration) +for source in ( + 'export local x = 1; export local x = 2', + 'export const x = 1; x = 2', 'export function f() end; f = 1', + 'export local x = 1; function x() end', + 'const x = 1; type T = typeof(function() x = 2 end)', + 'local x = 1; type function f() return x end', + 'type function f(x) return function() return x end end', + 'local x = 1; return x :: number :: number', + 'local x = 1; return (x :: number) :: number', + 'local x = 1; (x) = 2', 'local t = {}; (t.x) = 2', + 'local t = {}; (t).x = 2', 'local x, y = 1, 2; x, (y) = 3, 4', +): + add(f'scope or lvalue validation: {source}', source) +for branch in ('if flag then continue end', 'if false then continue end', + 'if true then continue end', 'do continue end', + 'if false then continue elseif flag then continue end', + 'if true then print(1) else continue end'): + for condition in ('value', '(function() return value end)()', 'true'): + add(f'repeat continue: {branch}: {condition}', f'repeat {branch} local value = true until {condition}') +for ending in ('\n', '\r', '\r\n', '\n\r'): + add(f'raw newline {ending!r}', 'return "a' + ending + 'b"') + add(f'escaped newline {ending!r}', 'return "a\\' + ending + 'b"') + add(f'comment newline {ending!r}', '-- comment' + ending + 'return 1') +for escape in (r'\u{}', r'\u{110000}', r'\u{D800}', r'\u{10FFFF}', r'\255', r'\256', r'\xFG', r'\q'): + add('string escape ' + escape, 'return "' + escape + '"') + + +def official(path): + result = subprocess.run([compiler, '--binary', str(path)], cwd=root, + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True, timeout=10) + assert result.returncode in (0, 1), (path, result.returncode, result.stderr) + return result.returncode == 0, result.stderr + + +with tempfile.TemporaryDirectory(prefix='prometheus-type-syntax-') as directory: + temporary = Path(directory) + paths = [] + for index, (_, source, _) in enumerate(cases): + path = temporary / f'{index:04d}.luau' + path.write_text(source) + paths.append(path) + manifest = temporary / 'manifest.txt' + manifest.write_text(''.join(str(path) + '\n' for path in paths)) + run = subprocess.run([lua, 'tests/harness/luau-syntax.lua', str(manifest)], cwd=root, + capture_output=True, text=True, timeout=60, check=True) + parsed = run.stdout.splitlines() + assert len(parsed) == len(cases), (len(parsed), len(cases), run.stdout[:500], run.stderr) + differences = [] + accepted = rejected = 0 + for (label, source, expected), path, line in zip(cases, paths, parsed): + native, error = official(path) + assert expected is None or native == expected, (label, source, error) + ours = line.split('\t', 1)[0] == 'accept' + accepted += native + rejected += not native + if native != ours: + differences.append(f'{label}\n {source}\n Luau: {native}; Prometheus: {line}\n {error.strip()}') + elif ours: + output_ok, output_error = official(Path(str(path) + '.out.luau')) + if not output_ok: + differences.append(f'{label}: invalid type-erased output\n {output_error}') + assert not differences, f'{len(differences)} syntax mismatches:\n' + '\n'.join(differences[:25]) + print(f'{len(cases)} syntax cases match Luau: {accepted} accepted, {rejected} rejected; emitted code also compiled') diff --git a/tests/harness/luau.py b/tests/harness/luau.py new file mode 100644 index 0000000..781248f --- /dev/null +++ b/tests/harness/luau.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +"""Compare Luau fixtures with the output of each obfuscation pipeline.""" +import os +from pathlib import Path +import subprocess +import tempfile + +root = Path(__file__).resolve().parents[2] +lua = os.environ.get('LUA51', 'lua5.1') +luau = os.environ.get('LUAU', 'luau') +names = ('Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong') + + +def run(args): + return subprocess.run(args, cwd=root, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + text=True, timeout=60, check=True).stdout + + +count = 0 +with tempfile.TemporaryDirectory(prefix='prometheus-luau-') as tmp: + output = str(Path(tmp) / 'output.luau') + for fixture in sorted((root / 'tests/luau').glob('*.luau')): + expected = run([luau, str(fixture)]) + for index, name in enumerate(names, 1): + for seed in (1, 17): + try: + run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', + str(index), str(seed), str(fixture), output]) + actual = run([luau, output]) + except subprocess.CalledProcessError as exc: + raise AssertionError(f'{fixture.name}, {name}, seed {seed}:\n{exc.stderr}\n{exc.stdout}') from exc + assert actual == expected, (f'{fixture.name}, {name}, seed {seed}: ' + f'expected {expected!r}, got {actual!r}') + count += 1 + print(f'{fixture.name}: all presets passed', flush=True) + for invalid in sorted((root / 'tests/luau/invalid').glob('*.luau')): + baseline = subprocess.run([luau, str(invalid)], cwd=root, capture_output=True, text=True, timeout=60) + transformed = subprocess.run([lua, 'scripts/test/obfuscate.lua', 'tests/luau/test-config.lua', + '1', '1', str(invalid), output], cwd=root, + capture_output=True, text=True, timeout=60) + assert baseline.returncode != 0 and transformed.returncode != 0, invalid.name + print('Invalid Luau examples were rejected', flush=True) +print(f'{count} Luau transformations matched their baseline') diff --git a/tests/luau/attributes-generics.luau b/tests/luau/attributes-generics.luau new file mode 100644 index 0000000..4e7049a --- /dev/null +++ b/tests/luau/attributes-generics.luau @@ -0,0 +1,18 @@ +--!optimize 1 +@native +local function identity(value: T): T return value end +@checked local function add(value: number): number return value + 1 end +@[native, deprecated] +local function old(value) return value end +local anonymous = @native function(value) return value * 2 end +local object = {} +function object:identity(value: T): T return value end +assert(identity<>(7) == 7) +assert(object:identity<>('value') == 'value') +local instantiated = identity<> +assert(instantiated(9) == 9) +local function factory() return identity, 'discarded' end +assert((factory()<>)(11) == 11) +assert(select('#', factory()<>) == 1) +assert(add(4) == 5 and old(3) == 3 and anonymous(5) == 10) +print('Attribute and explicit generic checks passed') diff --git a/tests/luau/compound-assignment.luau b/tests/luau/compound-assignment.luau new file mode 100644 index 0000000..addd522 --- /dev/null +++ b/tests/luau/compound-assignment.luau @@ -0,0 +1,47 @@ +-- The shared compound handlers must preserve all existing arithmetic operators. +local value = 20 +local function change() + value = 100 + return 2 +end +value += change() +assert(value == 102) +value -= change() +assert(value == 98) +value *= change() +assert(value == 200) +value /= change() +assert(value == 50) +value %= change() +assert(value == 0) +value ^= change() +assert(value == 10000) + +local selfValue = 7 +selfValue += selfValue +assert(selfValue == 14) +selfValue /= selfValue +assert(selfValue == 1) +selfValue -= selfValue +assert(selfValue == 0) + +-- Concatenation snapshots a local before RHS, unlike arithmetic assignment. +local text = 'old' +local function suffix() + text = 'new' + return '!' +end +text ..= suffix() +assert(text == 'old!') + +local object = {value = 20} +local function changeField() + object.value = 100 + return 2 +end +object.value += changeField() +assert(object.value == 22) +object.value = 20 +object.value -= changeField() +assert(object.value == 18) +print('Compound assignment checks passed') diff --git a/tests/luau/conditionals.luau b/tests/luau/conditionals.luau new file mode 100644 index 0000000..6f1de21 --- /dev/null +++ b/tests/luau/conditionals.luau @@ -0,0 +1,18 @@ +local log = '' +local function condition(label, result) + log ..= label + return result +end +local function value(label, result) + log ..= label + return result +end +local first = if condition('a', false) then value('A', 'bad') + elseif condition('b', false) then value('B', 'bad') + elseif condition('c', true) then value('C', false) + else value('D', true) +local second = if condition('d', false) then 1 elseif condition('e', true) then nil else 3 +local third = if true then (if false then 1 elseif true then 2 else 3) else 4 +local function pair() return 5, 6 end +local a, b = if true then pair() elseif false then pair() else pair() +print(log, first, second, third, a, b) diff --git a/tests/luau/const.luau b/tests/luau/const.luau new file mode 100644 index 0000000..950001a --- /dev/null +++ b/tests/luau/const.luau @@ -0,0 +1,39 @@ +const amount: number, label = 3, 'count' +const data: {count: number} = {count = amount} +data.count += 2 +const function factorial(n: number): number + if n == 0 then return 1 end + return n * factorial(n - 1) +end +const function closure() + return function() return amount, label, data.count end +end +local get = closure() +local a, b, c = get() +do + local amount = 1 + amount = 2 + assert(amount == 2) +end +local previous = 4 +const previous = previous + 1 +const other, missing = 8, nil +local function produce() return 9 end +const expanded, absent = produce() +local const = function(x) return x + 1 end +local value = const(4) +local tableWithConst = {const = value} +tableWithConst.const += 1 +local function asParameter(const) + const += 2 + return const +end +local total = 0 +for const = 1, 3 do total += const end +for const, v in ipairs({4, 5}) do total += const + v end +do + const const = function(x) return x * 2 end + total += const(3) +end +print(a, b, c, factorial(5), previous, other, missing) +print(tableWithConst.const, asParameter(3), const(1), expanded, absent, total) diff --git a/tests/luau/evaluation-order.luau b/tests/luau/evaluation-order.luau new file mode 100644 index 0000000..18c9630 --- /dev/null +++ b/tests/luau/evaluation-order.luau @@ -0,0 +1,36 @@ +local value = 20 +local function rhs() value = 100; return 50 end +assert(value + rhs() == 150) +value = 20; assert(value - rhs() == 50) +value = 20; assert(value * rhs() == 5000) +value = 20; assert(value / rhs() == 2) +value = 20; assert(value // rhs() == 2) +value = 20; assert(value % rhs() == 0) +value = 20; assert(value ^ rhs() == 1e100) +value = 20; assert(value .. rhs() == '2050') +value = 20; assert(not (value < rhs())) +value = 20; assert(not (value <= rhs())) +value = 20; assert(value > rhs()) +value = 20; assert(value >= rhs()) +local function equalRhs() value = 100; return 100 end +value = 20; assert(value == equalRhs()) +value = 20; assert(not (value ~= equalRhs())) + +local function upvalue() + value = 20 + return value + rhs() +end +assert(upvalue() == 70) +evaluationOrderGlobal = 20 +local function globalRhs() evaluationOrderGlobal = 100; return 50 end +assert(evaluationOrderGlobal + globalRhs() == 70) + +local first, second = {key = 1}, {key = 2} +local object = first +local function key() object = second; return 'key' end +assert(object[key()] == 2) +local field = 'before' +local function fieldValue() field = 'after'; return 3 end +local constructed = {[field] = fieldValue()} +assert(constructed.before == nil and constructed.after == 3) +print('Luau evaluation order checks passed') diff --git a/tests/luau/floor-division.luau b/tests/luau/floor-division.luau new file mode 100644 index 0000000..863f786 --- /dev/null +++ b/tests/luau/floor-division.luau @@ -0,0 +1,128 @@ +local function divide(a, b) + return a // b +end +assert(divide(7, 2) == 3 and divide(-7, 2) == -4) +assert(divide(7, -2) == -4 and divide(-7, -2) == 3) +assert(divide(7.5, 2) == 3 and divide(-7.5, 2) == -4) +local a, b, c = 29, 3, 2 +assert(a // b // c == 4) +assert(a // (b // c) == 29) +assert(a / b // c == 4 and a // b / c == 4.5) +assert(a * b // c % 5 == 3) +assert(2 + a // b * c == 20 and -2 ^ 2 // 3 == -2) +assert(a // b ^ c == 3 and (a // b) ^ c == 81) + +-- Preserve IEEE special values and the sign of zero. +local zero, infinity = 0, math.huge +assert(divide(1, zero) == infinity and divide(-1, zero) == -infinity) +local nan = divide(zero, zero) +assert(nan ~= nan) +assert(1 / divide(-zero, 2) == -infinity) +assert(1 / divide(1, -infinity) == -infinity) + +-- Floor division uses __idiv, with no fallback to __div or result rounding. +local tag = {} +local meta = { + __idiv = function(left, right) return tag, 'discarded' end, + __div = function() error('__div must not handle //') end, +} +local operand = setmetatable({}, meta) +assert(divide(operand, 2) == tag and divide(2, operand) == tag) +assert(select('#', divide(operand, 2)) == 1) +assert(divide('7', 2) == 3) +assert(not pcall(divide, 'invalid', 2)) +assert(not pcall(divide, setmetatable({}, {__div = function() return 1 end}), 2)) + +local value = 20 +local function changeValue() + value = 100 + return 2 +end +assert(value // changeValue() == 50) +value = 20 +value //= changeValue() +assert(value == 50) +value = 20 +local function divideUpvalue() + return value // changeValue() +end +assert(divideUpvalue() == 10 and value == 100) +value = 50 +do + local function update() + value //= changeValue() + end + update() + assert(value == 25) +end + +floorDivisionGlobal = 20 +local function changeGlobal() + floorDivisionGlobal = 100 + return 2 +end +floorDivisionGlobal //= changeGlobal() +assert(floorDivisionGlobal == 10) + +local object, calls = {value = 20}, '' +local function base() + calls ..= 'b' + return object +end +local function key() + calls ..= 'k' + return 'value' +end +local function rhs() + calls ..= 'r' + object.value = 100 + return 2 +end +base()[key()] //= rhs() +assert(object.value == 10 and calls == 'bkr') + +local original = {value = 20} +local replacement = {value = 100} +local holder, index = original, 'value' +holder[index] //= (function() + holder, index = replacement, 'other' + return 2 +end)() +assert(original.value == 20 and replacement.value == 100 and replacement.other == 10) + +local first, second = {value = 20}, {value = 40} +local current = first +local function switchBase() + current = second + return 'value' +end +current[switchBase()] //= 2 +assert(first.value == 20 and second.value == 20) + +local fixedHolder, fixedKey = first, 'value' +local function updateOuterTarget() + fixedHolder[fixedKey] //= (function() + fixedHolder, fixedKey = second, 'other' + return 2 + end)() +end +updateOuterTarget() +assert(first.value == 10 and second.other == nil) +local selfDivision = 7 +selfDivision //= selfDivision +assert(selfDivision == 1) + +local proxy = setmetatable({}, { + __index = function(_, key) + calls ..= 'i' + return operand + end, + __newindex = function(_, key, value) + assert(value == tag) + calls ..= 's' + end, +}) +calls = '' +proxy.value //= (function() calls ..= 'r'; return 2 end)() +assert(calls == 'irs') +print('Floor division checks passed') diff --git a/tests/luau/interpolation.luau b/tests/luau/interpolation.luau new file mode 100644 index 0000000..9e1fadf --- /dev/null +++ b/tests/luau/interpolation.luau @@ -0,0 +1,55 @@ +local events = '' +local object = setmetatable({}, { + __tostring = function() + events ..= 't' + return 'object' + end, +}) +local function first() + events ..= 'a' + return object, 'ignored' +end +local function second() + events ..= 'b' + return nil, 'ignored' +end +local function nothing() end +local function fromVarargs(...) + return `vararg={...}` +end +local string = {format = function() return 'wrong format' end} +local tostring = function() return 'wrong tostring' end +local combined = `[{first()}] [{second()}] [{nothing()}]` +local nested = `outer {`inner {({x = {9}}).x[1]}`} and {'}'} and {if true then 'yes' elseif false then 'bad' else 'no'}` +local multiline = `multi {( + function() return {answer = 42} end +)().answer}` +local escaped = `literal \{x} \` \\ \x41 \065 \u{42} 100%` +local withoutExpressions = `plain %%` +local formatPercent = `%% and %s: {42}%` +local upper = (`hello {1}`):upper() +local nul = `A\000{4}B` +local surrogate = `\u{D800}` +local empty = `` +local trimmed = `trim\z + med` +local newline = `line\ +end` +local env = getfenv() +env['prometheus_luau_iterator_1'] = 'global value' +local collision = `{prometheus_luau_iterator_1}` +env['prometheus_luau_iterator_1'] = nil +local embeddedLoop = `{(function() + local total = 0 + for _, value in {2, 3} do total += value end + return total +end)()}` +local savedString, savedTostring = env.string, env.tostring +env.string = {format = function() return 'changed global' end} +env.tostring = function() return 'changed tostring' end +local hiddenGlobals = `{true}|{nil}|{5.5}` +env.string, env.tostring = savedString, savedTostring +print(combined, events) +print(nested, multiline) +print(escaped, withoutExpressions, formatPercent, #nul, nul:byte(2), empty == '', trimmed, #newline) +print(hiddenGlobals, fromVarargs(1, 2), collision, embeddedLoop, upper, #surrogate, surrogate:byte(1, 3)) diff --git a/tests/luau/invalid/attribute-duplicate.luau b/tests/luau/invalid/attribute-duplicate.luau new file mode 100644 index 0000000..26094cb --- /dev/null +++ b/tests/luau/invalid/attribute-duplicate.luau @@ -0,0 +1,2 @@ +@native @native +local function invalid() end diff --git a/tests/luau/invalid/const-function-reassign.luau b/tests/luau/invalid/const-function-reassign.luau new file mode 100644 index 0000000..ae7d92f --- /dev/null +++ b/tests/luau/invalid/const-function-reassign.luau @@ -0,0 +1,2 @@ +const function f() return 1 end +function f() return 2 end diff --git a/tests/luau/invalid/const-missing-value.luau b/tests/luau/invalid/const-missing-value.luau new file mode 100644 index 0000000..47c5e59 --- /dev/null +++ b/tests/luau/invalid/const-missing-value.luau @@ -0,0 +1 @@ +const first, second = 1 diff --git a/tests/luau/invalid/const-reassign.luau b/tests/luau/invalid/const-reassign.luau new file mode 100644 index 0000000..e41d126 --- /dev/null +++ b/tests/luau/invalid/const-reassign.luau @@ -0,0 +1,4 @@ +const x = 1 +local function mutate() + x = 2 +end diff --git a/tests/luau/invalid/const-typeof.luau b/tests/luau/invalid/const-typeof.luau new file mode 100644 index 0000000..85c0184 --- /dev/null +++ b/tests/luau/invalid/const-typeof.luau @@ -0,0 +1,2 @@ +const value = 1 +type Invalid = typeof(function() value = 2 end) diff --git a/tests/luau/invalid/generic-instantiation.luau b/tests/luau/invalid/generic-instantiation.luau new file mode 100644 index 0000000..bd6ba52 --- /dev/null +++ b/tests/luau/invalid/generic-instantiation.luau @@ -0,0 +1,2 @@ +local function identity(value) return value end +return identity<>(1) diff --git a/tests/luau/invalid/interpolation-double-brace.luau b/tests/luau/invalid/interpolation-double-brace.luau new file mode 100644 index 0000000..3b415ef --- /dev/null +++ b/tests/luau/invalid/interpolation-double-brace.luau @@ -0,0 +1 @@ +local value = `before {{1}} after` diff --git a/tests/luau/invalid/interpolation-empty.luau b/tests/luau/invalid/interpolation-empty.luau new file mode 100644 index 0000000..82be4cf --- /dev/null +++ b/tests/luau/invalid/interpolation-empty.luau @@ -0,0 +1 @@ +local value = `before {} after` diff --git a/tests/luau/invalid/interpolation-unclosed.luau b/tests/luau/invalid/interpolation-unclosed.luau new file mode 100644 index 0000000..f1f4c7c --- /dev/null +++ b/tests/luau/invalid/interpolation-unclosed.luau @@ -0,0 +1 @@ +local value = `before {1` diff --git a/tests/luau/invalid/numeral-missing-digits.luau b/tests/luau/invalid/numeral-missing-digits.luau new file mode 100644 index 0000000..5c32089 --- /dev/null +++ b/tests/luau/invalid/numeral-missing-digits.luau @@ -0,0 +1 @@ +return 0b diff --git a/tests/luau/invalid/numeral-token-boundary.luau b/tests/luau/invalid/numeral-token-boundary.luau new file mode 100644 index 0000000..ad402a1 --- /dev/null +++ b/tests/luau/invalid/numeral-token-boundary.luau @@ -0,0 +1 @@ +local value = 1print('invalid') diff --git a/tests/luau/invalid/parenthesized-assignment.luau b/tests/luau/invalid/parenthesized-assignment.luau new file mode 100644 index 0000000..2c4bf4c --- /dev/null +++ b/tests/luau/invalid/parenthesized-assignment.luau @@ -0,0 +1,2 @@ +local value = 1 +(value) = 2 diff --git a/tests/luau/invalid/repeat-continue-local.luau b/tests/luau/invalid/repeat-continue-local.luau new file mode 100644 index 0000000..ecccbfb --- /dev/null +++ b/tests/luau/invalid/repeat-continue-local.luau @@ -0,0 +1,4 @@ +repeat + if flag then continue end + local value = true +until value diff --git a/tests/luau/invalid/semicolon-empty-statement.luau b/tests/luau/invalid/semicolon-empty-statement.luau new file mode 100644 index 0000000..0ef80e4 --- /dev/null +++ b/tests/luau/invalid/semicolon-empty-statement.luau @@ -0,0 +1 @@ +local value = 1;; diff --git a/tests/luau/invalid/string-unicode-range.luau b/tests/luau/invalid/string-unicode-range.luau new file mode 100644 index 0000000..20296c9 --- /dev/null +++ b/tests/luau/invalid/string-unicode-range.luau @@ -0,0 +1 @@ +return "\u{110000}" diff --git a/tests/luau/invalid/type-cast-chain.luau b/tests/luau/invalid/type-cast-chain.luau new file mode 100644 index 0000000..8fa0fd6 --- /dev/null +++ b/tests/luau/invalid/type-cast-chain.luau @@ -0,0 +1,2 @@ +local value = 1 +return value :: number :: number diff --git a/tests/luau/invalid/type-operator-chain.luau b/tests/luau/invalid/type-operator-chain.luau new file mode 100644 index 0000000..b7dbd20 --- /dev/null +++ b/tests/luau/invalid/type-operator-chain.luau @@ -0,0 +1 @@ +local x: |& usudhd subd = 100 diff --git a/tests/luau/invalid/type-pack-context.luau b/tests/luau/invalid/type-pack-context.luau new file mode 100644 index 0000000..6cdc855 --- /dev/null +++ b/tests/luau/invalid/type-pack-context.luau @@ -0,0 +1 @@ +local value: (number, string) = nil diff --git a/tests/luau/invalid/typeof-expression.luau b/tests/luau/invalid/typeof-expression.luau new file mode 100644 index 0000000..5b2544d --- /dev/null +++ b/tests/luau/invalid/typeof-expression.luau @@ -0,0 +1 @@ +type Invalid = typeof({value = }) diff --git a/tests/luau/invalid/vararg-nested-function.luau b/tests/luau/invalid/vararg-nested-function.luau new file mode 100644 index 0000000..d53b3f3 --- /dev/null +++ b/tests/luau/invalid/vararg-nested-function.luau @@ -0,0 +1,5 @@ +local function outer(...) + return function() + return ... + end +end diff --git a/tests/luau/invalid/vararg-non-variadic.luau b/tests/luau/invalid/vararg-non-variadic.luau new file mode 100644 index 0000000..2370ab7 --- /dev/null +++ b/tests/luau/invalid/vararg-non-variadic.luau @@ -0,0 +1,3 @@ +local function fixed() + return ... +end diff --git a/tests/luau/iteration.luau b/tests/luau/iteration.luau new file mode 100644 index 0000000..6a12f44 --- /dev/null +++ b/tests/luau/iteration.luau @@ -0,0 +1,48 @@ +local function prometheus_luau_iterator_1() return 2 end +local sum = prometheus_luau_iterator_1() - 2 +local source = {3, 4, 5} +local evaluated = 0 +local function tableOnce() + evaluated += 1 + return source +end +for i, value in tableOnce() do + if i == 2 then continue end + sum += i * value +end +local calls = 0 +local iterable = setmetatable({}, { + __iter = function(self) + calls += 1 + return function(state, control) + assert(state == 91) + if control == nil then return false, 11 end + if control == false then return 2, 21 end + end, 91, nil + end, +}) +for key, value in iterable do + sum += value + if key == false then sum += 1 end +end +local callable = setmetatable({}, { + __call = function(self, state, key) + if key == nil then return 1, 9 end + end, +}) +for _, value in callable do sum += value end +local function iteratorFactory() + return next, {10, 20}, nil +end +for key, value in iteratorFactory() do sum += key + value end +local protected = setmetatable({17}, {__metatable = 'locked'}) +for _, value in pairs(protected) do sum += value end +local frozen = table.freeze({1, 2}) +for _, value in frozen do sum += value end +local captured = {} +for key, value in {8, 9} do + captured[#captured + 1] = function() return key, value end +end +local k1, v1 = captured[1]() +local k2, v2 = captured[2]() +print(sum, calls, evaluated, k1, v1, k2, v2) diff --git a/tests/luau/keywords.luau b/tests/luau/keywords.luau new file mode 100644 index 0000000..cce9310 --- /dev/null +++ b/tests/luau/keywords.luau @@ -0,0 +1,43 @@ +-- Contextual keywords are ordinary names in expressions, bindings, and fields. +local continue, type, export, const = 4, 5, 6, 7 +local function names(continue, type, export, const, typeof, read, write) + return continue + type + export + const + typeof + read + write +end +assert(names(continue, type, export, const, 8, 9, 10) == 49) + +local object = {continue = 11, type = 12, export = 13, const = 14} +function object:continue(type) + return self.type + type +end +assert(object:continue(3) == 15) +function object.export(continue) + return continue * 2 +end +assert(object.export(5) == 10 and object.const == 14) + +local sum = 0 +for continue = 1, 3 do + sum += continue +end +for type, export in ipairs({4, 5}) do + sum += type + export +end +assert(sum == 18) + +local calls = 0 +do + local function continue() + calls += 1 + end + for i = 1, 4 do + continue() + if i % 2 == 0 then continue end + calls += 10 + end +end +assert(calls == 24) +type Continue = {continue: number, type: number} +export type Export = Continue +local typed: Export = {continue = 1, type = 2} +assert(typed.continue + typed.type == 3) +print('Contextual keyword checks passed') diff --git a/tests/luau/numerals.luau b/tests/luau/numerals.luau new file mode 100644 index 0000000..1983b26 --- /dev/null +++ b/tests/luau/numerals.luau @@ -0,0 +1,25 @@ +local decimal = {1_000, 1__2_, 1_.2_5, .5_0, 1_e+_2_, 2.5e-1, 01, 1.} +local expected = {1000, 12, 1.25, 0.5, 100, 0.25, 1, 1} +for i, value in ipairs(decimal) do assert(value == expected[i]) end + +local radix = {0b1010, 0B11, 0b__1__0_, 0_b_1_0_, 0xFF, 0XABCDEF, 0_x_F_F_} +local expectedRadix = {10, 3, 2, 2, 255, 11259375, 255} +for i, value in ipairs(radix) do assert(value == expectedRadix[i]) end + +-- Radix conversion rounds once and saturates at the unsigned 64-bit limit. +local boundaries = { + 0x1FFFFFFFFFFFFF, 0x20000000000001, 0x20000000000003, + 0xFFFFFFFFFFFFF800, 0xFFFFFFFFFFFFFFFF, + 0x10000000000000000, 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF, + 0b11111111111111111111111111111111111111111111111111111111111111111, +} +local expectedBoundaries = { + 9007199254740991, 9007199254740992, 9007199254740996, + 18446744073709549568, 18446744073709551616, + 18446744073709551616, 18446744073709551616, 18446744073709551616, +} +for i, value in ipairs(boundaries) do assert(value == expectedBoundaries[i]) end +local overflow, underflow = 1e999, 1e-999 +assert(overflow == math.huge and underflow == 0) +assert(1_0 .. 'x' == '10x') +print('Luau numeral checks passed') diff --git a/tests/luau/protected-metatable.luau b/tests/luau/protected-metatable.luau new file mode 100644 index 0000000..44caa9e --- /dev/null +++ b/tests/luau/protected-metatable.luau @@ -0,0 +1,56 @@ +local t = setmetatable({1}, { + __metatable = 'locked', + __iter = function(self) return next, self end, +}) +for _, value in t do print(value) end + +local iterator = setmetatable({}, { + __metatable = {}, + __iter = function() + local i = 0 + return function() + i += 1 + if i <= 3 then return i, i * 10, nil end + end + end, +}) +local sum = 0 +for i, value in iterator do + if i == 2 then continue end + sum += value +end +assert(sum == 40) +local function returning(...) + for i, value in iterator do + if i == 2 then return ..., value, nil end + end +end +local result = table.pack(returning(5, 6)) +assert(result.n == 3 and result[1] == 5 and result[2] == 20 and result[3] == nil) +local count = 0 +for _ in iterator do count += 1; break end +assert(count == 1) +local function nested() + for i in iterator do + for j in iterator do + if j == 2 then return i, j end + end + end +end +local a, b = nested() +assert(a == 1 and b == 2) +local caller +local co = coroutine.create(function() + caller = coroutine.running() + for i in iterator do + assert(coroutine.running() == caller) + coroutine.yield(i) + end +end) +for expected = 1, 3 do + local ok, value = coroutine.resume(co) + assert(ok and value == expected) +end +assert(coroutine.resume(co)) +assert(coroutine.status(co) == 'dead') +print('Protected iterator and callback control checks passed') diff --git a/tests/luau/recent/exports.luau b/tests/luau/recent/exports.luau new file mode 100644 index 0000000..ab72057 --- /dev/null +++ b/tests/luau/recent/exports.luau @@ -0,0 +1,9 @@ +export local counter = 1 +export const fixed = 2 +export local optional +export function get() return counter, fixed end +export function increment() counter += 1 end +counter += 2 +assert(counter == 3) +local first, second = get() +assert(first == 3 and second == 2) diff --git a/tests/luau/recent/integers.luau b/tests/luau/recent/integers.luau new file mode 100644 index 0000000..61e6a00 --- /dev/null +++ b/tests/luau/recent/integers.luau @@ -0,0 +1,14 @@ +-- Integer literals require a runtime that supports Luau's integer type. +local exact = 9007199254740993i +assert(type(exact) == 'integer' and exact == 0x20000000000001i) +assert(exact ~= 9007199254740992i) +assert(0xffffffffffffffffi == -1i) +assert(-0xffffffffffffffffi == 1i) +assert(-0x8000000000000000i == 0x8000000000000000i) +assert(0b1111i == 15i and 1_000i == 1000i) +local values = {[exact] = 'exact', [9007199254740992i] = 'other'} +assert(values[exact] == 'exact') +local function valuesFunction() return exact, nil, 9223372036854775807i end +local a, b, c = valuesFunction() +assert(a == exact and b == nil and c == 0x7fffffffffffffffi) +print('Integer literal precision and sign checks passed') diff --git a/tests/luau/semicolons.luau b/tests/luau/semicolons.luau new file mode 100644 index 0000000..3d2419d --- /dev/null +++ b/tests/luau/semicolons.luau @@ -0,0 +1,20 @@ +type Value = number; +export type Exported = Value; +type function Identity(value) return value end; +local value: Exported = 0; +do value += 1; end; +if value == 1 then value += 1; else error('wrong branch'); end; +while value < 10 do + value += 1; + if value == 3 then continue; end; + break; +end; +repeat value += 1; until true; +for i = 1, 2 do value += i; end; +local function stop() + repeat return; until true; +end; +stop(); +(function() value += 1; end)(); +assert(value == 9); +print('Semicolon checks passed'); diff --git a/tests/luau/test-config.lua b/tests/luau/test-config.lua new file mode 100644 index 0000000..97856f8 --- /dev/null +++ b/tests/luau/test-config.lua @@ -0,0 +1,9 @@ +local presets = require('src.presets') +local configs = {} +for _, name in ipairs({'Minify', 'Cff', 'Vmify', 'Weak', 'Medium', 'Strong'}) do + local config = presets[name] + config.Name = name + config.LuaVersion = 'LuaU' + configs[#configs + 1] = config +end +return configs diff --git a/tests/luau/types.luau b/tests/luau/types.luau new file mode 100644 index 0000000..be81f01 --- /dev/null +++ b/tests/luau/types.luau @@ -0,0 +1,37 @@ +-- Type syntax is erased while runtime expressions retain their value counts. +type Name = string +export type Box = {value: T, tag: U?, [number]: boolean} +type Optional = T | nil +type Callback = (first: number, T...) -> (string, ...number) +type TableIntersection = {read value: number} & {write name: string} +type function getNumber(ty) + return types.number +end +type FunctionResult = getNumber +local evaluated = 0 +local function observe() + evaluated += 1 + return {value = 1} +end +type FromTypeof = typeof(observe()) + +local type = function(value) return value end +local export = 3 +local box: Box = {value = 7, tag = 'ok'} +local function identity(value: T, ...: string): T + return value +end +local anonymous = function(value: T): T return value end +local object = {} +function object:method(value: T): T return value end +local function values(): (number, number) return 11, 22 end +local first, second = values() :: number +local function castVarargs(...: number): number + return ... :: number +end +local fromTypeFunction: FunctionResult = 0 +local total = fromTypeFunction +for i: number = 1, 3 do total += i end +for _, value: number in ipairs({1, 2}) do total += value end +local value = if box.value == 0 then 'zero' elseif box.value == 7 then 'seven' elseif box.value == 8 then 'eight' else 'other' +print(evaluated, export, type(value), anonymous(identity(object:method(box.value))), total, first, second, castVarargs(4, 5)) diff --git a/tests/luau/types/a.luau b/tests/luau/types/a.luau new file mode 100644 index 0000000..e57a90f --- /dev/null +++ b/tests/luau/types/a.luau @@ -0,0 +1 @@ +local x: number = 100 \ No newline at end of file diff --git a/tests/luau/unsupported/protected-metatable.luau b/tests/luau/unsupported/protected-metatable.luau new file mode 100644 index 0000000..b2b58f9 --- /dev/null +++ b/tests/luau/unsupported/protected-metatable.luau @@ -0,0 +1,5 @@ +local t = setmetatable({1}, { + __metatable = 'locked', + __iter = function(self) return next, self end, +}) +for _, value in t do print(value) end diff --git a/tests/luau/varargs.luau b/tests/luau/varargs.luau new file mode 100644 index 0000000..8ac6a28 --- /dev/null +++ b/tests/luau/varargs.luau @@ -0,0 +1,33 @@ +local function pack(...) + return {n = select('#', ...), ...} +end +assert(pack(...).n == 0) +local function outer(...) + local args = pack(...) + local function fixed() return args.n end + local function inner(first, ...) + return first, fixed(), ... + end + do + assert(select('#', ...) == args.n) + end + return inner('first', ...) +end +local result = pack(outer(1, nil, 3, nil)) +assert(result.n == 6 and result[1] == 'first' and result[2] == 4) +assert(result[3] == 1 and result[4] == nil and result[5] == 3 and result[6] == nil) + +local object = {} +function object:collect(...: number) + return self, ... +end +result = pack(object:collect(1, nil, 2)) +assert(result.n == 4 and result[1] == object and result[2] == 1 and result[4] == 2) +local function interpolate(...) + return `value={(...)}` +end +assert(interpolate(7, 8) == 'value=7') +local function fixed() return 5 end +local function variadic(...) return ... end +assert(fixed() == 5 and select('#', variadic(nil, nil)) == 2) +print('Vararg scope checks passed')