diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 28c6e7b7..74593a46 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -39,12 +39,16 @@ jobs: run: npm i --ignore-scripts - name: Run benchmarks - run: npm start y 100 10 40 + run: | + npm start y 100 10 40 + npm start y 100 10 40 post - name: Compare results run: | node ./benchmark compare -t node ./benchmark compare -u + node ./benchmark compare -t -s post + node ./benchmark compare -u -s post - name: Commit and push updated results uses: github-actions-x/commit@722d56b8968bf00ced78407bbe2ead81062d8baa # v2.9 @@ -54,6 +58,6 @@ jobs: commit-message: 'chore: update benchmark results' force-add: 'true' rebase: 'true' - files: benchmark-results.json README.md + files: benchmark-results.json benchmark-results-post.json README.md name: Github Actions email: <> diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18df7530..affa5713 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: run: npm i --ignore-scripts - name: Run tests - run: npm test && npm start y 1 1 1 + run: npm test && npm start y 1 1 1 && npm start y 1 1 1 post automerge: name: Automerge Dependabot PRs diff --git a/.npmrc b/.npmrc index 1b961bfa..bf79d622 100644 --- a/.npmrc +++ b/.npmrc @@ -1,4 +1,5 @@ ignore-scripts=true min-release-age=7 +min-release-age-exclude[]=fastify package-lock=false save-exact=false diff --git a/README.md b/README.md index 21325c28..25b8a9ad 100644 --- a/README.md +++ b/README.md @@ -68,6 +68,19 @@ node ./benchmark [arguments (optional)] > You may also compare all test results, at once, in a single table; `benchmark compare -t` > You can also extend the comparison table with percentage values based on fastest result; `benchmark compare -p` + +#### Scenarios + +* `get` (default): `GET /` responding with `{ hello: 'world' }`. Servers live in [`benchmarks/`](./benchmarks). +* `post`: `POST /` with a `{"hello":"world"}` JSON body, parsed and echoed back. Servers live in [`benchmarks/post/`](./benchmarks/post). + +Pass the scenario as the fifth argument when benchmarking non-interactively, and with `-s` when comparing: + +``` +node ./benchmark bench y 100 10 40 post +node ./benchmark compare -t -s post +``` + # Benchmarks * __Machine:__ linux x64 | 4 vCPUs | 15.6GB Mem diff --git a/benchmark-bench.js b/benchmark-bench.js index d1fe9f76..9550524a 100644 --- a/benchmark-bench.js +++ b/benchmark-bench.js @@ -1,6 +1,7 @@ import inquirer from 'inquirer' import bench from './lib/bench.js' import { choices, list } from './lib/packages.js' +import { names, servers } from './lib/scenarios.js' const argv = process.argv.slice(2) run().catch(err => { @@ -10,7 +11,8 @@ run().catch(err => { async function run () { const options = await getBenchmarkOptions() - const modules = options.all ? choices : await select() + const available = servers(options.scenario) + const modules = options.all ? choices.filter(c => available.has(c)) : await select(available) return bench(options, modules) } @@ -23,6 +25,13 @@ async function getBenchmarkOptions () { message: 'Do you want to run all benchmark tests?', default: false }, + { + type: 'list', + name: 'scenario', + message: 'Which scenario do you want to run?', + choices: names, + default: 'get' + }, { type: 'input', name: 'connections', @@ -57,16 +66,17 @@ async function getBenchmarkOptions () { } function parseArgv () { - const [all, connections, pipelining, duration] = argv + const [all, connections, pipelining, duration, scenario = 'get'] = argv return { all: all === 'y', connections: +connections, pipelining: +pipelining, - duration: +duration + duration: +duration, + scenario } } -async function select () { +async function select (available) { const result = await inquirer.prompt([ { type: 'checkbox', @@ -74,9 +84,9 @@ async function select () { name: 'list', choices: [ new inquirer.Separator(' = The usual ='), - ...list(), + ...list().filter(c => available.has(c.name)), new inquirer.Separator(' = The extras = '), - ...list(true) + ...list(true).filter(c => available.has(c.name)) ], validate: function (answer) { if (answer.length < 1) { diff --git a/benchmark-compare.js b/benchmark-compare.js index 1d138cf2..7882ab85 100644 --- a/benchmark-compare.js +++ b/benchmark-compare.js @@ -5,19 +5,20 @@ import { program } from 'commander' import inquirer from 'inquirer' import Table from 'cli-table' import chalk from 'chalk' -import { join } from 'node:path' import { readdirSync, readFileSync, writeFileSync } from 'node:fs' import { info } from './lib/packages.js' import { compare } from './lib/autocannon.js' - -const resultsPath = join(process.cwd(), 'results') +import { headings, names, scenario as getScenario } from './lib/scenarios.js' program.option('-t, --table', 'print table') .option('-m --markdown', 'format table for markdown') .option('-u --update', 'update README.md') + .option('-s --scenario ', `scenario to compare (${names.join(', ')})`, 'get') .parse(process.argv) const opts = program.opts() +const scenario = getScenario(opts.scenario) +const resultsPath = scenario.resultsDirectory if (opts.markdown || opts.update) { chalk.level = 0 @@ -46,17 +47,27 @@ function formatHasRouter (hasRouter) { function updateReadme () { const machineInfo = `${platform()} ${arch()} | ${cpus().length} vCPUs | ${(totalmem() / (1024 ** 3)).toFixed(1)}GB Mem` - const benchmarkMd = `# Benchmarks + const benchmarkMd = `${scenario.heading} * __Machine:__ ${machineInfo} * __Node:__ \`${process.version}\` * __Run:__ ${new Date()} -* __Method:__ \`autocannon -c 100 -d 40 -p 10 localhost:3000\` (two rounds; one to warm-up, one to measure) +* __Method:__ \`${scenario.method}\` (two rounds; one to warm-up, one to measure) ${compareResults(true)} ` const md = readFileSync('README.md', 'utf8') - writeFileSync('README.md', md.split('# Benchmarks', 1)[0] + benchmarkMd, 'utf8') + const start = md.indexOf(scenario.heading) + if (start === -1) { + writeFileSync('README.md', `${md.trimEnd()}\n\n${benchmarkMd}`, 'utf8') + return + } + // Replace only this scenario's section, up to the next scenario heading + const end = Math.min(...headings() + .map(heading => md.indexOf(heading, start + scenario.heading.length)) + .filter(index => index !== -1), md.length) + const rest = end < md.length ? `\n${md.slice(end)}` : '' + writeFileSync('README.md', md.slice(0, start) + benchmarkMd + rest, 'utf8') } function compareResults (markdown) { @@ -132,7 +143,7 @@ function compareResults (markdown) { bold(beBold, throughput ? (throughput / 1024 / 1024).toFixed(2) : 'N/A') ]) } - writeFileSync('benchmark-results.json', JSON.stringify(outputResults), 'utf8') + writeFileSync(scenario.resultsFile, JSON.stringify(outputResults), 'utf8') return table.toString() } @@ -156,7 +167,7 @@ async function compareResultsInteractive () { }]) const [a, b] = [firstChoice.choice, secondChoice.choice] - const result = compare(a, b) + const result = compare(a, b, resultsPath) const fastest = chalk.bold.yellow(result.fastest) const fastestAverage = chalk.green(result.fastestAverage) diff --git a/benchmarks/post/express.cjs b/benchmarks/post/express.cjs new file mode 100644 index 00000000..49b3f8fc --- /dev/null +++ b/benchmarks/post/express.cjs @@ -0,0 +1,14 @@ +'use strict' + +const express = require('express') + +const app = express() + +app.disable('etag') +app.disable('x-powered-by') + +app.post('/', express.json(), function (req, res) { + res.json(req.body) +}) + +app.listen(3000) diff --git a/benchmarks/post/fastify.cjs b/benchmarks/post/fastify.cjs new file mode 100644 index 00000000..37749f9b --- /dev/null +++ b/benchmarks/post/fastify.cjs @@ -0,0 +1,24 @@ +'use strict' + +const fastify = require('fastify')() + +const schema = { + schema: { + response: { + 200: { + type: 'object', + properties: { + hello: { + type: 'string' + } + } + } + } + } +} + +fastify.post('/', schema, function (req, reply) { + reply.send(req.body) +}) + +fastify.listen({ port: 3000, host: '127.0.0.1' }) diff --git a/benchmarks/post/h3.cjs b/benchmarks/post/h3.cjs new file mode 100644 index 00000000..6e2d0075 --- /dev/null +++ b/benchmarks/post/h3.cjs @@ -0,0 +1,7 @@ +'use strict' + +const { H3, readBody, serve } = require('h3') + +const app = new H3().post('/', (event) => readBody(event)) + +serve(app, { port: process.env.PORT || 3000 }) diff --git a/benchmarks/post/hapi.cjs b/benchmarks/post/hapi.cjs new file mode 100644 index 00000000..c9c5c093 --- /dev/null +++ b/benchmarks/post/hapi.cjs @@ -0,0 +1,26 @@ +'use strict' + +const Hapi = require('@hapi/hapi') + +async function start () { + const server = Hapi.server({ port: 3000, debug: false }) + + server.route({ + method: 'POST', + path: '/', + config: { + cache: false, + response: { + ranges: false + }, + state: { parse: false } + }, + handler: function (request) { + return request.payload + } + }) + + await server.start() +} + +start() diff --git a/benchmarks/post/hono.mjs b/benchmarks/post/hono.mjs new file mode 100644 index 00000000..328573a4 --- /dev/null +++ b/benchmarks/post/hono.mjs @@ -0,0 +1,7 @@ +import { serve } from '@hono/node-server' +import { Hono } from 'hono' + +const app = new Hono() +app.post('/', async (c) => c.json(await c.req.json())) + +serve(app) diff --git a/benchmarks/post/koa.cjs b/benchmarks/post/koa.cjs new file mode 100644 index 00000000..4140d9ef --- /dev/null +++ b/benchmarks/post/koa.cjs @@ -0,0 +1,16 @@ +'use strict' + +const Koa = require('koa') +const { bodyParser } = require('@koa/bodyparser') +const app = new Koa() + +app.use(bodyParser()) +app.use(ctx => { + ctx.body = ctx.request.body +}) + +const _server = app.listen(3000) + +process.on('SIGINT', () => { + _server.close() +}) diff --git a/benchmarks/post/node-http.cjs b/benchmarks/post/node-http.cjs new file mode 100644 index 00000000..15a3d644 --- /dev/null +++ b/benchmarks/post/node-http.cjs @@ -0,0 +1,15 @@ +'use strict' + +const server = require('node:http').createServer(function (req, res) { + let body = '' + req.setEncoding('utf8') + req.on('data', function (chunk) { + body += chunk + }) + req.on('end', function () { + res.setHeader('content-type', 'application/json; charset=utf-8') + res.end(JSON.stringify(JSON.parse(body))) + }) +}) + +server.listen(3000) diff --git a/benchmarks/post/restify.cjs b/benchmarks/post/restify.cjs new file mode 100644 index 00000000..87f71c6d --- /dev/null +++ b/benchmarks/post/restify.cjs @@ -0,0 +1,12 @@ +'use strict' + +const restify = require('restify') + +const server = restify.createServer() +server.use(restify.plugins.bodyParser()) +server.post('/', function (req, res, next) { + res.send(req.body) + return next() +}) + +server.listen(3000, function () {}) diff --git a/lib/autocannon.js b/lib/autocannon.js index 11f35567..8e76c305 100644 --- a/lib/autocannon.js +++ b/lib/autocannon.js @@ -10,8 +10,6 @@ const mkdir = promisify(_mkdir) const access = promisify(_access) const require = createRequire(import.meta.url) -const resultsDirectory = join(process.cwd(), 'results') - const run = (opts = {}) => new Promise((resolve, reject) => { opts.url = 'http://127.0.0.1:3000' autocannon(opts, (err, result) => { @@ -23,11 +21,11 @@ const run = (opts = {}) => new Promise((resolve, reject) => { }) }) -const writeResult = async (handler, result) => { +const writeResult = async (resultsDirectory, handler, result) => { try { await access(resultsDirectory) } catch { - await mkdir(resultsDirectory) + await mkdir(resultsDirectory, { recursive: true }) } result.server = handler @@ -36,12 +34,12 @@ const writeResult = async (handler, result) => { return writeFile(dest, JSON.stringify(result)) } -export async function fire (opts, handler, save) { +export async function fire (opts, handler, resultsDirectory) { const result = await run(opts) - return save ? writeResult(handler, result) : null + return resultsDirectory ? writeResult(resultsDirectory, handler, result) : null } -const _compare = (a, b) => { +const _compare = (a, b, resultsDirectory) => { const resA = require(`${resultsDirectory}/${a}.json`) const resB = require(`${resultsDirectory}/${b}.json`) const comp = compare(resA, resB) diff --git a/lib/bench.js b/lib/bench.js index c3239c40..bab462ae 100644 --- a/lib/bench.js +++ b/lib/bench.js @@ -5,25 +5,25 @@ import { fork } from 'node:child_process' import ora from 'ora' import { join } from 'node:path' import { fire } from './autocannon.js' -import { fileURLToPath } from 'node:url' +import { scenario } from './scenarios.js' import assert from 'node:assert' -const __dirname = fileURLToPath(new URL('.', import.meta.url)) - const doBench = async (opts, handler) => { + const { serversDirectory, resultsDirectory, request } = scenario(opts.scenario) + const autocannonOpts = { ...opts, ...request } const spinner = ora(`Started ${handler}`).start() let forked try { - await access(join(__dirname, '..', 'benchmarks', handler + '.cjs')) - forked = fork(join(__dirname, '..', 'benchmarks', handler + '.cjs')) + await access(join(serversDirectory, handler + '.cjs')) + forked = fork(join(serversDirectory, handler + '.cjs')) } catch { - forked = fork(join(__dirname, '..', 'benchmarks', handler + '.mjs')) + forked = fork(join(serversDirectory, handler + '.mjs')) } try { spinner.color = 'magenta' spinner.text = `Warming ${handler}` - await fire(opts, handler, false) + await fire(autocannonOpts, handler) } catch (error) { return console.log(error) } finally { @@ -32,7 +32,7 @@ const doBench = async (opts, handler) => { } try { - await fire(opts, handler, true) + await fire(autocannonOpts, handler, resultsDirectory) assert.ok(forked.kill('SIGINT')) spinner.text = `Results saved for ${handler}` spinner.succeed() diff --git a/lib/scenarios.js b/lib/scenarios.js new file mode 100644 index 00000000..f2b195b6 --- /dev/null +++ b/lib/scenarios.js @@ -0,0 +1,47 @@ +import { readdirSync } from 'node:fs' +import { join, parse } from 'node:path' +import { fileURLToPath } from 'node:url' + +const root = fileURLToPath(new URL('..', import.meta.url)) + +const scenarios = { + get: { + serversDirectory: join(root, 'benchmarks'), + resultsDirectory: join(process.cwd(), 'results'), + resultsFile: 'benchmark-results.json', + heading: '# Benchmarks', + method: 'autocannon -c 100 -d 40 -p 10 localhost:3000', + request: {} + }, + post: { + serversDirectory: join(root, 'benchmarks', 'post'), + resultsDirectory: join(process.cwd(), 'results', 'post'), + resultsFile: 'benchmark-results-post.json', + heading: '# POST Benchmarks', + method: 'autocannon -c 100 -d 40 -p 10 -m POST -H content-type=application/json -b \'{"hello":"world"}\' localhost:3000', + request: { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ hello: 'world' }) + } + } +} + +export const names = Object.keys(scenarios) + +export function scenario (name = 'get') { + if (!Object.hasOwn(scenarios, name)) { + throw new Error(`Unknown scenario "${name}", expected one of: ${names.join(', ')}`) + } + return scenarios[name] +} + +export function headings () { + return Object.values(scenarios).map(({ heading }) => heading) +} + +export function servers (name) { + return new Set(readdirSync(scenario(name).serversDirectory) + .filter((file) => /\.[cm]js$/.test(file)) + .map((file) => parse(file).name)) +} diff --git a/package.json b/package.json index f1aceb77..653a08a6 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@elysia/node": "^1.4.5", "@hapi/hapi": "^21.1.0", "@hono/node-server": "^2.0.1", + "@koa/bodyparser": "^6.1.0", "@koa/router": "^15.1.1", "@tinyhttp/app": "^3.0.1", "@trpc/server": "^11.1.0",