feat: add POST benchmark scenario

.github/workflows/benchmarks.yml

@@ -39,12 +39,16 @@ jobs:
       run: npm i --ignore-scripts
 
     - name: Run benchmarks
-      run: npm start y 100 10 40
+      run: |
+        npm start y 100 10 40
+        npm start y 100 10 40 post
 
     - name: Compare results
       run: |
         node ./benchmark compare -t
         node ./benchmark compare -u
+        node ./benchmark compare -t -s post
+        node ./benchmark compare -u -s post
 
     - name: Commit and push updated results
       uses: github-actions-x/commit@722d56b8968bf00ced78407bbe2ead81062d8baa # v2.9
@@ -54,6 +58,6 @@ jobs:
         commit-message: 'chore: update benchmark results'
         force-add: 'true'
         rebase: 'true'
-        files: benchmark-results.json README.md
+        files: benchmark-results.json benchmark-results-post.json README.md
         name: Github Actions
         email: <>

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
         run: npm i --ignore-scripts
 
       - name: Run tests
-        run: npm test && npm start y 1 1 1
+        run: npm test && npm start y 1 1 1 && npm start y 1 1 1 post
 
   automerge:
     name: Automerge Dependabot PRs

.npmrc

@@ -1,4 +1,5 @@
 ignore-scripts=true
 min-release-age=7
+min-release-age-exclude[]=fastify
 package-lock=false
 save-exact=false

README.md

@@ -68,6 +68,19 @@ node ./benchmark [arguments (optional)]
 > You may also compare all test results, at once, in a single table; `benchmark compare -t`
 
 > You can also extend the comparison table with percentage values based on fastest result; `benchmark compare -p`
+
+#### Scenarios
+
+* `get` (default): `GET /` responding with `{ hello: 'world' }`. Servers live in [`benchmarks/`](./benchmarks).
+* `post`: `POST /` with a `{"hello":"world"}` JSON body, parsed and echoed back. Servers live in [`benchmarks/post/`](./benchmarks/post).
+
+Pass the scenario as the fifth argument when benchmarking non-interactively, and with `-s` when comparing:
+
+```
+node ./benchmark bench y 100 10 40 post
+node ./benchmark compare -t -s post
+```
+
 # Benchmarks
 
 * __Machine:__ linux x64 | 4 vCPUs | 15.6GB Mem

benchmark-bench.js

@@ -1,6 +1,7 @@
 import inquirer from 'inquirer'
 import bench from './lib/bench.js'
 import { choices, list } from './lib/packages.js'
+import { names, servers } from './lib/scenarios.js'
 const argv = process.argv.slice(2)
 
 run().catch(err => {
@@ -10,7 +11,8 @@ run().catch(err => {
 
 async function run () {
   const options = await getBenchmarkOptions()
-  const modules = options.all ? choices : await select()
+  const available = servers(options.scenario)
+  const modules = options.all ? choices.filter(c => available.has(c)) : await select(available)
   return bench(options, modules)
 }
 
@@ -23,6 +25,13 @@ async function getBenchmarkOptions () {
       message: 'Do you want to run all benchmark tests?',
       default: false
     },
+    {
+      type: 'list',
+      name: 'scenario',
+      message: 'Which scenario do you want to run?',
+      choices: names,
+      default: 'get'
+    },
     {
       type: 'input',
       name: 'connections',
@@ -57,26 +66,27 @@ async function getBenchmarkOptions () {
 }
 
 function parseArgv () {
-  const [all, connections, pipelining, duration] = argv
+  const [all, connections, pipelining, duration, scenario = 'get'] = argv
   return {
     all: all === 'y',
     connections: +connections,
     pipelining: +pipelining,
-    duration: +duration
+    duration: +duration,
+    scenario
   }
 }
 
-async function select () {
+async function select (available) {
   const result = await inquirer.prompt([
     {
       type: 'checkbox',
       message: 'Select packages',
       name: 'list',
       choices: [
         new inquirer.Separator(' = The usual ='),
-        ...list(),
+        ...list().filter(c => available.has(c.name)),
         new inquirer.Separator(' = The extras = '),
-        ...list(true)
+        ...list(true).filter(c => available.has(c.name))
       ],
       validate: function (answer) {
         if (answer.length < 1) {

benchmark-compare.js

@@ -5,19 +5,20 @@ import { program } from 'commander'
 import inquirer from 'inquirer'
 import Table from 'cli-table'
 import chalk from 'chalk'
-import { join } from 'node:path'
 import { readdirSync, readFileSync, writeFileSync } from 'node:fs'
 import { info } from './lib/packages.js'
 import { compare } from './lib/autocannon.js'
-
-const resultsPath = join(process.cwd(), 'results')
+import { headings, names, scenario as getScenario } from './lib/scenarios.js'
 
 program.option('-t, --table', 'print table')
   .option('-m --markdown', 'format table for markdown')
   .option('-u --update', 'update README.md')
+  .option('-s --scenario <name>', `scenario to compare (${names.join(', ')})`, 'get')
   .parse(process.argv)
 
 const opts = program.opts()
+const scenario = getScenario(opts.scenario)
+const resultsPath = scenario.resultsDirectory
 
 if (opts.markdown || opts.update) {
   chalk.level = 0
@@ -46,17 +47,27 @@ function formatHasRouter (hasRouter) {
 
 function updateReadme () {
   const machineInfo = `${platform()} ${arch()} | ${cpus().length} vCPUs | ${(totalmem() / (1024 ** 3)).toFixed(1)}GB Mem`
-  const benchmarkMd = `# Benchmarks
+  const benchmarkMd = `${scenario.heading}
 
 * __Machine:__ ${machineInfo}
 * __Node:__ \`${process.version}\`
 * __Run:__ ${new Date()}
-* __Method:__ \`autocannon -c 100 -d 40 -p 10 localhost:3000\` (two rounds; one to warm-up, one to measure)
+* __Method:__ \`${scenario.method}\` (two rounds; one to warm-up, one to measure)
 
 ${compareResults(true)}
 `
   const md = readFileSync('README.md', 'utf8')
-  writeFileSync('README.md', md.split('# Benchmarks', 1)[0] + benchmarkMd, 'utf8')
+  const start = md.indexOf(scenario.heading)
+  if (start === -1) {
+    writeFileSync('README.md', `${md.trimEnd()}\n\n${benchmarkMd}`, 'utf8')
+    return
+  }
+  // Replace only this scenario's section, up to the next scenario heading
+  const end = Math.min(...headings()
+    .map(heading => md.indexOf(heading, start + scenario.heading.length))
+    .filter(index => index !== -1), md.length)
+  const rest = end < md.length ? `\n${md.slice(end)}` : ''
+  writeFileSync('README.md', md.slice(0, start) + benchmarkMd + rest, 'utf8')
 }
 
 function compareResults (markdown) {
@@ -132,7 +143,7 @@ function compareResults (markdown) {
       bold(beBold, throughput ? (throughput / 1024 / 1024).toFixed(2) : 'N/A')
     ])
   }
-  writeFileSync('benchmark-results.json', JSON.stringify(outputResults), 'utf8')
+  writeFileSync(scenario.resultsFile, JSON.stringify(outputResults), 'utf8')
   return table.toString()
 }
 
@@ -156,7 +167,7 @@ async function compareResultsInteractive () {
   }])
 
   const [a, b] = [firstChoice.choice, secondChoice.choice]
-  const result = compare(a, b)
+  const result = compare(a, b, resultsPath)
 
   const fastest = chalk.bold.yellow(result.fastest)
   const fastestAverage = chalk.green(result.fastestAverage)

benchmarks/post/express.cjs

@@ -0,0 +1,14 @@
+'use strict'
+
+const express = require('express')
+
+const app = express()
+
+app.disable('etag')
+app.disable('x-powered-by')
+
+app.post('/', express.json(), function (req, res) {
+  res.json(req.body)
+})
+
+app.listen(3000)

benchmarks/post/fastify.cjs

@@ -0,0 +1,24 @@
+'use strict'
+
+const fastify = require('fastify')()
+
+const schema = {
+  schema: {
+    response: {
+      200: {
+        type: 'object',
+        properties: {
+          hello: {
+            type: 'string'
+          }
+        }
+      }
+    }
+  }
+}
+
+fastify.post('/', schema, function (req, reply) {
+  reply.send(req.body)
+})
+
+fastify.listen({ port: 3000, host: '127.0.0.1' })

benchmarks/post/h3.cjs

@@ -0,0 +1,7 @@
+'use strict'
+
+const { H3, readBody, serve } = require('h3')
+
+const app = new H3().post('/', (event) => readBody(event))
+
+serve(app, { port: process.env.PORT || 3000 })

benchmarks/post/hapi.cjs

@@ -0,0 +1,26 @@
+'use strict'
+
+const Hapi = require('@hapi/hapi')
+
+async function start () {
+  const server = Hapi.server({ port: 3000, debug: false })
+
+  server.route({
+    method: 'POST',
+    path: '/',
+    config: {
+      cache: false,
+      response: {
+        ranges: false
+      },
+      state: { parse: false }
+    },
+    handler: function (request) {
+      return request.payload
+    }
+  })
+
+  await server.start()
+}
+
+start()

benchmarks/post/hono.mjs

@@ -0,0 +1,7 @@
+import { serve } from '@hono/node-server'
+import { Hono } from 'hono'
+
+const app = new Hono()
+app.post('/', async (c) => c.json(await c.req.json()))
+
+serve(app)

benchmarks/post/koa.cjs

@@ -0,0 +1,16 @@
+'use strict'
+
+const Koa = require('koa')
+const { bodyParser } = require('@koa/bodyparser')
+const app = new Koa()
+
+app.use(bodyParser())
+app.use(ctx => {
+  ctx.body = ctx.request.body
+})
+
+const _server = app.listen(3000)
+
+process.on('SIGINT', () => {
+  _server.close()
+})

benchmarks/post/node-http.cjs

@@ -0,0 +1,15 @@
+'use strict'
+
+const server = require('node:http').createServer(function (req, res) {
+  let body = ''
+  req.setEncoding('utf8')
+  req.on('data', function (chunk) {
+    body += chunk
+  })
+  req.on('end', function () {
+    res.setHeader('content-type', 'application/json; charset=utf-8')
+    res.end(JSON.stringify(JSON.parse(body)))
+  })
+})
+
+server.listen(3000)

benchmarks/post/restify.cjs

@@ -0,0 +1,12 @@
+'use strict'
+
+const restify = require('restify')
+
+const server = restify.createServer()
+server.use(restify.plugins.bodyParser())
+server.post('/', function (req, res, next) {
+  res.send(req.body)
+  return next()
+})
+
+server.listen(3000, function () {})

lib/autocannon.js

@@ -10,8 +10,6 @@ const mkdir = promisify(_mkdir)
 const access = promisify(_access)
 const require = createRequire(import.meta.url)
 
-const resultsDirectory = join(process.cwd(), 'results')
-
 const run = (opts = {}) => new Promise((resolve, reject) => {
   opts.url = 'http://127.0.0.1:3000'
   autocannon(opts, (err, result) => {
@@ -23,11 +21,11 @@ const run = (opts = {}) => new Promise((resolve, reject) => {
   })
 })
 
-const writeResult = async (handler, result) => {
+const writeResult = async (resultsDirectory, handler, result) => {
   try {
     await access(resultsDirectory)
   } catch {
-    await mkdir(resultsDirectory)
+    await mkdir(resultsDirectory, { recursive: true })
   }
 
   result.server = handler
@@ -36,12 +34,12 @@ const writeResult = async (handler, result) => {
   return writeFile(dest, JSON.stringify(result))
 }
 
-export async function fire (opts, handler, save) {
+export async function fire (opts, handler, resultsDirectory) {
   const result = await run(opts)
-  return save ? writeResult(handler, result) : null
+  return resultsDirectory ? writeResult(resultsDirectory, handler, result) : null
 }
 
-const _compare = (a, b) => {
+const _compare = (a, b, resultsDirectory) => {
   const resA = require(`${resultsDirectory}/${a}.json`)
   const resB = require(`${resultsDirectory}/${b}.json`)
   const comp = compare(resA, resB)