From c7af6434749d90e6e6a6c7ac1b1ebf895887cbb1 Mon Sep 17 00:00:00 2001 From: Hampton Lintorn-Catlin Date: Fri, 6 Nov 2026 21:05:12 -0600 Subject: [PATCH 1/2] Run local CI checks and RSpec in parallel --- README.md | 4 + bin/ci | 8 +- config/ci.rb | 30 ++---- config/database.yml | 2 +- docs/CI.md | 59 +++++++++++ lib/local_ci.rb | 207 ++++++++++++++++++++++++++++++++++++++ spec/lib/local_ci_spec.rb | 145 ++++++++++++++++++++++++++ 7 files changed, 428 insertions(+), 27 deletions(-) create mode 100644 lib/local_ci.rb create mode 100644 spec/lib/local_ci_spec.rb diff --git a/README.md b/README.md index 749900fa..ffb1ea57 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,10 @@ bin/rails runner 'puts CoPlan::Ai.available?' bundle exec rspec ``` +Run the full local checks in parallel with `bin/ci`. It uses isolated databases +and a CPU-based worker count; use `bin/ci --jobs 12 --browser-jobs 6` to choose +your own limits. See [the CI guide](docs/CI.md) for setup, reports, and options. + See [CONTRIBUTING.md](./CONTRIBUTING.md) for pull request, spec, and visual testing requirements. Interactive features are tested with JavaScript-enabled browser specs in `spec/system/`. diff --git a/bin/ci b/bin/ci index 4137ad5b..af9d61d5 100755 --- a/bin/ci +++ b/bin/ci @@ -1,6 +1,8 @@ #!/usr/bin/env ruby require_relative "../config/boot" -require "active_support/continuous_integration" +require_relative "../lib/local_ci" -CI = ActiveSupport::ContinuousIntegration -require_relative "../config/ci.rb" +$stdout.sync = true +Dir.chdir(File.expand_path("..", __dir__)) do + LocalCi.start(ARGV) +end diff --git a/config/ci.rb b/config/ci.rb index e56a92e4..279885e2 100644 --- a/config/ci.rb +++ b/config/ci.rb @@ -1,23 +1,7 @@ -# Run using bin/ci - -CI.run do - step "Setup", "bin/setup --skip-server" - - step "Style: Ruby", "bin/rubocop" - - step "Security: Gem audit", "bin/bundler-audit" - step "Security: Importmap vulnerability audit", "bin/importmap audit" - step "Security: Brakeman code analysis", "bin/brakeman --quiet --no-pager --exit-on-warn --exit-on-error" - - step "Tests: Rails", "bin/rails test" - step "Tests: System", "bin/rails test:system" - step "Tests: Seeds", "env RAILS_ENV=test bin/rails db:seed:replant" - - # Optional: set a green GitHub commit status to unblock PR merge. - # Requires the `gh` CLI and `gh extension install basecamp/gh-signoff`. - # if success? - # step "Signoff: All systems go. Ready for merge and deploy.", "gh signoff" - # else - # failure "Signoff: CI failed. Do not merge or deploy.", "Fix the issues and try again." - # end -end +# Independent checks share the same worker budget as the RSpec shards. +LOCAL_CI_CHECKS = { + "rubocop" => [ "bin/rubocop", "--no-parallel" ], + "gem-audit" => [ "bin/bundler-audit" ], + "importmap-audit" => [ "bin/importmap", "audit" ], + "brakeman" => [ "bin/brakeman", "--quiet", "--no-pager", "--exit-on-warn", "--exit-on-error" ] +}.freeze diff --git a/config/database.yml b/config/database.yml index cb8c2fda..83cfb6f9 100644 --- a/config/database.yml +++ b/config/database.yml @@ -26,7 +26,7 @@ development: # Do not set this db to the same as development or production. test: <<: *default - database: coplan_test + database: <%= ENV.fetch("COPLAN_TEST_DATABASE", "coplan_test") %> # As with config/credentials.yml, you never want to store sensitive information, # like your database password, in your source code. If your source code is diff --git a/docs/CI.md b/docs/CI.md index 34269e94..2031808f 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -1,5 +1,64 @@ # GitHub CI +## Local checks in parallel + +Run `bundle install` and start MySQL, then use: + +```sh +bin/ci +``` + +This runs every RSpec file, including real-browser system specs, plus RuboCop, +the gem and importmap audits, Brakeman, and the demo seeds twice. It does not +run `bin/setup`, alter the development database, or replant your usual test +database. Dependencies must already be installed. Chrome for Testing is +provisioned by Selenium using the same local configuration as ordinary system specs. + +The default worker budget is half the detected CPU count, capped at eight and +at least one. Up to four workers run browser shards; the remaining slots can +run other specs or checks at the same time. RuboCop's own parallelism is disabled +so it shares this budget. To use more cores or less memory: + +```sh +bin/ci --jobs 12 --browser-jobs 6 +bin/ci --jobs 2 --browser-jobs 1 +bin/ci --jobs 1 +``` + +Each RSpec shard and the seed check get a newly named database, unique to this +checkout and invocation, and a separate schema file. The database account needs +permission to create and drop databases. Databases are dropped after each check, +including after failures and Ctrl+C. If the coordinator is forcibly killed or +the database server becomes unavailable, cleanup may need to be done manually; +the database names are in the check logs. + +Without `DATABASE_URL`, workers use the test connection in `config/database.yml`. +With it, workers retain its connection settings and replace the database name. +For example, PostgreSQL checks migrate each new database instead of loading the +MySQL schema: + +```sh +DATABASE_URL=postgresql://postgres@localhost/coplan_local_test bin/ci +``` + +Preview the selected worker counts and file counts without running commands: + +```sh +bin/ci --list +``` + +Files are balanced by source size, separately for browser and non-browser specs. +Every discovered spec runs once. The coordinator prints each check's start, +finish, duration, and log path. Logs, RSpec JSON reports, private schema files, +and `summary.json` remain under the printed `tmp/local-ci//` directory. +An empty suite, a missing report, a failed audit, or a cleanup failure makes +`bin/ci` exit nonzero. Other checks finish so one failure does not hide their results. +Repeat a test order with `bin/ci --seed `; keep the same worker +counts to preserve the same groups. GitHub's extra repeated regression cohorts +remain available separately through `bin/rspec-stress`. + +## GitHub Actions + The workflow in `.github/workflows/ci.yml` runs these jobs independently: | Job | Database | Runner | Coverage | diff --git a/lib/local_ci.rb b/lib/local_ci.rb new file mode 100644 index 00000000..729ffede --- /dev/null +++ b/lib/local_ci.rb @@ -0,0 +1,207 @@ +require "digest" +require "erb" +require "etc" +require "fileutils" +require "json" +require "optparse" +require "securerandom" +require "thread" +require "uri" +require "yaml" +require_relative "../config/ci" + +# Local tooling only: never boot Rails in the coordinator or share a test DB. +class LocalCi + Task = Struct.new(:name, :commands, :env, :database, :report, keyword_init: true) + + def self.start(arguments) + options = { jobs: [ [ Etc.nprocessors / 2, 1 ].max, 8 ].min, seed: Random.new_seed % 65536 } + parser = OptionParser.new do |args| + args.banner = "Usage: bin/ci [--jobs N] [--browser-jobs N] [--seed N] [--list]" + args.on("--jobs N", Integer, "Maximum simultaneous checks (default: half the CPUs, up to 8)") { |n| options[:jobs] = n } + args.on("--browser-jobs N", Integer, "Browser shards (default: up to 4)") { |n| options[:browser_jobs] = n } + args.on("--seed N", Integer, "RSpec random-order seed") { |n| options[:seed] = n } + args.on("--list", "Show the plan without running commands or creating databases") { options[:list] = true } + args.on("-h", "--help") { puts args; return } + end + parser.parse!(arguments) + raise OptionParser::InvalidArgument, arguments.join(" ") unless arguments.empty? + options[:browser_jobs] ||= [ options[:jobs], 4 ].min + unless options[:jobs].positive? && options[:browser_jobs].between?(1, options[:jobs]) + raise OptionParser::InvalidArgument, "jobs must be positive; browser-jobs must be between 1 and jobs" + end + list = options.delete(:list) + runner = new(**options) + if list + runner.describe + else + exit(runner.run ? 0 : 1) + end + rescue OptionParser::ParseError, ArgumentError => error + abort error.message + end + + # Greedy file-size balancing avoids putting every large browser spec together. + def self.shards(files, count) + groups = Array.new([ files.length, count ].min) { [] } + sizes = groups.to_h { |group| [ group.object_id, 0 ] } + files.sort_by { |file| [ -File.size(file), file ] }.each do |file| + group = groups.min_by { |candidate| sizes.fetch(candidate.object_id) } + group << file + sizes[group.object_id] += File.size(file) + end + groups + end + + attr_reader :tasks, :directory + + def initialize(jobs:, browser_jobs:, seed:) + @jobs, @seed = jobs, seed + @directory = File.expand_path("tmp/local-ci/#{Time.now.utc.strftime('%Y%m%dT%H%M%S')}-#{Process.pid}-#{SecureRandom.hex(3)}") + @mutex, @children, @results = Mutex.new, {}, {} + @interrupted = false + database_config = YAML.safe_load(ERB.new(File.read("config/database.yml")).result, aliases: true).fetch("test") + adapter = ENV["DATABASE_URL"] ? URI.parse(ENV.fetch("DATABASE_URL")).scheme : database_config.fetch("adapter") + raise ArgumentError, "Local CI supports mysql2 and PostgreSQL databases" unless %w[mysql2 postgres postgresql].include?(adapter) + @postgres = %w[postgres postgresql].include?(adapter) + browser = Dir["spec/system/**/*_spec.rb"] + other = Dir["spec/**/*_spec.rb"] - browser + raise ArgumentError, "No browser or non-browser specs found" if browser.empty? || other.empty? + @tasks = LOCAL_CI_CHECKS.map { |name, command| Task.new(name: name, commands: [ command ], env: {}) } + self.class.shards(browser, browser_jobs).each_with_index { |files, i| @tasks << spec_task("browser-#{i + 1}", files) } + self.class.shards(other, jobs).each_with_index { |files, i| @tasks << spec_task("specs-#{i + 1}", files) } + @tasks << database_task("seeds", [ [ "bin/rails", "db:seed" ], [ "bin/rails", "db:seed" ] ]) + end + + def describe + puts "Local CI: #{@jobs} workers, RSpec seed #{@seed}" + puts "Logs and reports: #{directory}" + tasks.each do |task| + count = task.commands.flatten.count { |arg| arg.end_with?("_spec.rb") } + puts " #{task.name}#{count.positive? ? " (#{count} spec files)" : ''}#{task.database ? ' — isolated database' : ''}" + end + end + + def run + describe + FileUtils.mkdir_p(directory) + queue = Queue.new + tasks.each { |task| queue << task } + threads = Array.new([ @jobs, tasks.length ].min) do + Thread.new do + loop do + task = queue.pop(true) + break if @interrupted + execute(task) + rescue ThreadError + break + end + end + end + previous_term = Signal.trap("TERM") { raise Interrupt } + begin + threads.each(&:join) + rescue Interrupt + pids = [] + @mutex.synchronize do + @interrupted = true + pids = @children.keys + pids.each { |pid| signal_group("TERM", pid) } + end + puts "Interrupted; stopping workers and removing their databases…" + killer = Thread.new do + sleep 5 + @mutex.synchronize { pids.each { |pid| signal_group("KILL", pid) if @children.key?(pid) } } + end + threads.each(&:join) + ensure + killer&.kill + killer&.join + Signal.trap("TERM", previous_term) + end + File.write(File.join(directory, "summary.json"), JSON.pretty_generate(@results)) + passed = !@interrupted && @results.length == tasks.length && @results.values.all? + puts "#{passed ? 'PASS' : 'FAIL'}: #{@results.count { |_, result| result }}/#{tasks.length} checks passed (#{directory})" + passed + end + + private + def signal_group(signal, pid) + Process.kill(signal, -pid) + rescue Errno::ESRCH + # The command completed between the snapshot and the signal. + end + + def spec_task(name, files) + report = File.join(directory, "#{name}.json") + task = database_task(name, [ [ "bundle", "exec", "rspec", *files, "--seed", @seed.to_s, + "--format", "progress", "--format", "json", "--out", report ] ]) + task.report = report + task + end + + def database_task(name, commands) + database = "coplan_ci_#{Digest::SHA256.hexdigest(directory)[0, 12]}_#{name.tr('-', '_')}" + env = { "RAILS_ENV" => "test", "COPLAN_TEST_DATABASE" => database, + "SCHEMA" => File.join(directory, "#{name}-schema.rb") } + if ENV["DATABASE_URL"] + url = URI.parse(ENV.fetch("DATABASE_URL")) + url.path = "/#{database}" + # Active Record query parameters take precedence over the URL path. + params = URI.decode_www_form(url.query || "").reject { |key, _| key == "database" } + params << [ "encoding", "unicode" ] if @postgres && params.none? { |key, _| key == "encoding" } + url.query = params.empty? ? nil : URI.encode_www_form(params) + env["DATABASE_URL"] = url.to_s + end + Task.new(name: name, commands: commands, env: env, database: database) + end + + def command(task, args, log, cleanup: false) + pid = @mutex.synchronize do + return false if @interrupted && !cleanup + Process.spawn(task.env, *args, out: log, err: log, pgroup: true).tap { |child| @children[child] = true } + end + _, status = Process.wait2(pid) + status.success? + ensure + @mutex.synchronize { @children.delete(pid) } if pid + end + + def execute(task) + started = Process.clock_gettime(Process::CLOCK_MONOTONIC) + path = File.join(directory, "#{task.name}.log") + puts "START #{task.name}" + passed = false + File.open(path, "w") do |log| + begin + if task.database + FileUtils.cp("db/schema.rb", task.env.fetch("SCHEMA")) unless @postgres + setup = [ "bin/rails", "db:create", @postgres ? "db:migrate" : "db:schema:load" ] + return unless command(task, setup, log) + end + passed = task.commands.all? { |args| command(task, args, log) } + if passed && task.report + summary = JSON.parse(File.read(task.report)).fetch("summary") + passed = summary.fetch("example_count").positive? && summary.fetch("failure_count").zero? + end + rescue StandardError => error + log.puts "#{error.class}: #{error.message}" + passed = false + ensure + if task.database + begin + dropped = command(task, [ "bin/rails", "db:drop" ], log, cleanup: true) + passed &&= dropped + rescue StandardError => error + log.puts "Database cleanup failed: #{error.class}: #{error.message}" + passed = false + end + end + end + end + ensure + @mutex.synchronize { @results[task.name] = passed } + elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started + puts "#{passed ? 'PASS' : 'FAIL'} #{task.name} (#{elapsed.round(1)}s) — #{path}" + end +end diff --git a/spec/lib/local_ci_spec.rb b/spec/lib/local_ci_spec.rb new file mode 100644 index 00000000..aaa15b19 --- /dev/null +++ b/spec/lib/local_ci_spec.rb @@ -0,0 +1,145 @@ +require "spec_helper" +require "tmpdir" +require "rbconfig" +require "open3" +require "timeout" +require_relative "../../lib/local_ci" + +RSpec.describe LocalCi do + around do |example| + original = ENV.to_h.slice("DATABASE_URL", "SCHEMA", "COPLAN_TEST_DATABASE") + ENV.delete("DATABASE_URL") + Dir.mktmpdir do |directory| + @directory = directory + example.run + end + ensure + %w[DATABASE_URL SCHEMA COPLAN_TEST_DATABASE].each { |key| original.key?(key) ? ENV[key] = original[key] : ENV.delete(key) } + end + + def runner(jobs: 2) + described_class.new(jobs: jobs, browser_jobs: 1, seed: 123).tap do |ci| + ci.instance_variable_set(:@directory, @directory) + end + end + + def task(name, source, report: nil) + described_class::Task.new(name: name, env: {}, report: report, + commands: [ [ RbConfig.ruby, "-e", source ] ]) + end + + it "includes every spec exactly once and keeps browser shards separate" do + ci = runner(jobs: 4) + commands = ci.tasks.flat_map(&:commands) + selected = commands.flatten.select { |arg| arg.end_with?("_spec.rb") } + expect(selected.sort).to eq(Dir["spec/**/*_spec.rb"].sort) + expect(ci.tasks.select { |item| item.name.start_with?("browser-") }.flat_map(&:commands).flatten.grep(/_spec\.rb$/)).to all(start_with("spec/system/")) + expect(ci.tasks.select { |item| item.name.start_with?("specs-") }.flat_map(&:commands).flatten.grep(/_spec\.rb$/)).not_to include(start_with("spec/system/")) + end + + it "uses different databases and schema files for every worker and invocation" do + first = runner.tasks.select(&:database) + second = runner.tasks.select(&:database) + databases = (first + second).map(&:database) + expect(databases.uniq).to eq(databases) + expect(first.map { |item| item.env.fetch("SCHEMA") }.uniq.length).to eq(first.length) + expect(first.map { |item| item.env.fetch("RAILS_ENV") }).to all(eq("test")) + expect(first.map(&:database)).not_to include("coplan_test", "coplan_development") + end + + it "preserves URL connection options while replacing every database selector" do + ENV["DATABASE_URL"] = "postgresql://test-user@localhost:5433/shared?encoding=unicode&database=shared&sslmode=require" + ci = runner + ci.tasks.select(&:database).each do |item| + url = URI.parse(item.env.fetch("DATABASE_URL")) + expect(url.path).to eq("/#{item.database}") + expect(url.userinfo).to eq("test-user") + expect(url.port).to eq(5433) + expect(URI.decode_www_form(url.query).to_h).to eq("encoding" => "unicode", "sslmode" => "require") + end + end + + it "runs workers concurrently and waits for every result after a failure" do + ci = runner + # Each process must see its sibling before it can finish. A sequential + # implementation fails the deadline instead of silently passing this test. + source = <<~RUBY + require 'timeout' + File.write(File.join(#{@directory.dump}, ARGV.fetch(0)), '') + Timeout.timeout(5) do + sleep 0.01 until Dir[File.join(#{@directory.dump}, 'ready-*')].length == 2 + end + RUBY + first = task("first", source) + first.commands.first.concat([ "ready-first" ]) + second = task("second", source) + second.commands.first.concat([ "ready-second" ]) + ci.tasks.replace([ first, second, task("failed", "exit 7"), task("last", "puts 'still ran'") ]) + expect(ci.run).to be false + expect(JSON.parse(File.read(File.join(@directory, "summary.json")))).to eq( + "first" => true, "second" => true, "failed" => false, "last" => true) + expect(File.read(File.join(@directory, "last.log"))).to include("still ran") + end + + it "rejects an empty suite or missing report even when RSpec exits zero" do + ci = runner + report = File.join(@directory, "empty.json") + File.write(report, JSON.generate(summary: { example_count: 0, failure_count: 0 })) + ci.tasks.replace([ task("empty", "exit 0", report: report), task("missing", "exit 0", report: "#{report}.missing") ]) + expect(ci.run).to be false + expect(JSON.parse(File.read(File.join(@directory, "summary.json"))).values).to eq([ false, false ]) + end + + it "cleans up a database after setup fails and never runs its specs" do + ci = runner(jobs: 1) + item = ci.tasks.find(&:database) + item.env["SCHEMA"] = File.join(@directory, "schema.rb") + ci.tasks.replace([ item ]) + commands = [] + allow(ci).to receive(:command) do |_task, args, _log, **_options| + commands << args + args.include?("db:drop") + end + expect(ci.run).to be false + expect(commands.map(&:last)).to eq([ "db:schema:load", "db:drop" ]) + end + + it "fails when database cleanup fails" do + ci = runner(jobs: 1) + item = ci.tasks.last # The seed check has no RSpec report to fake. + item.env["SCHEMA"] = File.join(@directory, "schema.rb") + ci.tasks.replace([ item ]) + allow(ci).to receive(:command) { |_task, args, _log, **_options| !args.include?("db:drop") } + expect(ci.run).to be false + end + + it "rejects invalid worker counts before starting commands" do + expect { described_class.start([ "--jobs", "0" ]) }.to raise_error(SystemExit) + expect { described_class.start([ "--jobs", "2", "--browser-jobs", "3" ]) }.to raise_error(SystemExit) + end + + it "stops child processes and exits unsuccessfully on interruption" do + started = File.join(@directory, "child.pid") + source = <<~RUBY + require #{File.expand_path('../../lib/local_ci', __dir__).dump} + ci = LocalCi.new(jobs: 1, browser_jobs: 1, seed: 123) + ci.instance_variable_set(:@directory, #{@directory.dump}) + child = "File.write(ARGV.fetch(0), Process.pid); sleep 60" + ci.tasks.replace([LocalCi::Task.new(name: 'interrupted', env: {}, commands: [[#{RbConfig.ruby.dump}, '-e', child, #{started.dump}]])]) + exit(ci.run ? 0 : 1) + RUBY + # Keep the signal inside a subprocess so RSpec's own handlers are untouched. + Open3.popen2e(RbConfig.ruby, "-e", source) do |input, output, wait| + input.close + Timeout.timeout(10) { sleep 0.01 until File.exist?(started) || !wait.alive? } + expect(wait).to be_alive, output.read unless wait.alive? + Process.kill("INT", wait.pid) + status = Timeout.timeout(15) { wait.value } + expect(status.exitstatus).to eq(1), output.read + expect { Process.kill(0, Integer(File.read(started))) }.to raise_error(Errno::ESRCH) + expect(File.read(File.join(@directory, "summary.json"))).to include('"interrupted"') + ensure + Process.kill("KILL", wait.pid) if wait.alive? + end + end +end From be0bd19bb1c874582a5223ca65cea6d9968e43cf Mon Sep 17 00:00:00 2001 From: Hampton Lintorn-Catlin Date: Wed, 11 Nov 2026 14:43:39 -0500 Subject: [PATCH 2/2] Use parallel_tests for local CI --- Gemfile | 3 + Gemfile.lock | 4 + bin/ci | 24 +++- bin/ci-db | 12 ++ config/ci.rb | 87 ++++++++++++++- docs/CI.md | 32 +++--- lib/ci_database.rb | 40 +++++++ lib/local_ci.rb | 207 ---------------------------------- spec/lib/ci_database_spec.rb | 31 ++++++ spec/lib/local_ci_spec.rb | 209 ++++++++++++++++------------------- 10 files changed, 310 insertions(+), 339 deletions(-) create mode 100755 bin/ci-db create mode 100644 lib/ci_database.rb delete mode 100644 lib/local_ci.rb create mode 100644 spec/lib/ci_database_spec.rb diff --git a/Gemfile b/Gemfile index cd3099f5..199d11f6 100644 --- a/Gemfile +++ b/Gemfile @@ -72,6 +72,9 @@ group :development, :test do # Omakase Ruby styling [https://github.com/rails/rubocop-rails-omakase/] gem "rubocop-rails-omakase", require: false + + # Parallel RSpec processes and runtime-based grouping for local CI. + gem "parallel_tests", "~> 5.9", require: false end group :development do diff --git a/Gemfile.lock b/Gemfile.lock index 3e0cf31a..d0e45897 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -306,6 +306,8 @@ GEM openssl (4.0.1) ostruct (0.6.3) parallel (1.27.0) + parallel_tests (5.9.0) + parallel parser (3.3.10.2) ast (~> 2.4.1) racc @@ -549,6 +551,7 @@ DEPENDENCIES jbuilder kamal mysql2 (~> 0.5) + parallel_tests (~> 5.9) pg (~> 1.5) propshaft puma (>= 5.0) @@ -686,6 +689,7 @@ CHECKSUMS openssl (4.0.1) sha256=e27974136b7b02894a1bce46c5397ee889afafe704a839446b54dc81cb9c5f7d ostruct (0.6.3) sha256=95a2ed4a4bd1d190784e666b47b2d3f078e4a9efda2fccf18f84ddc6538ed912 parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130 + parallel_tests (5.9.0) sha256=593169873cfeefe54f233103238cb3b078320e6eac0155d0f1107b39233f98f8 parser (3.3.10.2) sha256=6f60c84aa4bdcedb6d1a2434b738fe8a8136807b6adc8f7f53b97da9bc4e9357 pg (1.6.3) sha256=1388d0563e13d2758c1089e35e973a3249e955c659592d10e5b77c468f628a99 pg (1.6.3-aarch64-linux) sha256=0698ad563e02383c27510b76bf7d4cd2de19cd1d16a5013f375dd473e4be72ea diff --git a/bin/ci b/bin/ci index af9d61d5..92d4e617 100755 --- a/bin/ci +++ b/bin/ci @@ -1,8 +1,26 @@ #!/usr/bin/env ruby require_relative "../config/boot" -require_relative "../lib/local_ci" +require "etc" +require "optparse" +require "securerandom" -$stdout.sync = true Dir.chdir(File.expand_path("..", __dir__)) do - LocalCi.start(ARGV) + jobs = [ [ Etc.nprocessors / 2, 1 ].max, 8 ].min + browsers, seed, list = nil, Random.new_seed % 65536, false + OptionParser.new do |args| + args.banner = "Usage: bin/ci [--jobs N] [--browser-jobs N] [--seed N] [--list]" + args.on("--jobs N", Integer, "Parallel test processes (default: half the CPUs, up to 8)") { |n| jobs = n } + args.on("--browser-jobs N", Integer, "Parallel browser processes (default: up to 4)") { |n| browsers = n } + args.on("--seed N", Integer, "RSpec random-order seed") { |n| seed = n } + args.on("--list", "Preview the checks without creating databases") { list = true } + args.on("-h", "--help") { puts args; exit } + end.parse! + browsers ||= [ jobs, 4 ].min + abort "jobs must be positive; browser-jobs must be between 1 and jobs" unless jobs.positive? && browsers.between?(1, jobs) + abort "Unexpected arguments: #{ARGV.join(' ')}" unless ARGV.empty? + ENV["COPLAN_CI_JOBS"], ENV["COPLAN_CI_BROWSER_JOBS"], ENV["COPLAN_CI_SEED"] = jobs.to_s, browsers.to_s, seed.to_s + ENV["COPLAN_CI_DIRECTORY"] = File.expand_path("tmp/local-ci/#{Time.now.utc.strftime('%Y%m%dT%H%M%S')}-#{Process.pid}-#{SecureRandom.hex(3)}") + exec "bundle", "exec", "rake", "--rakefile", "config/ci.rb", "--jobs", jobs.to_s, list ? "preview" : "ci" +rescue OptionParser::ParseError => error + abort error.message end diff --git a/bin/ci-db b/bin/ci-db new file mode 100755 index 00000000..393511d4 --- /dev/null +++ b/bin/ci-db @@ -0,0 +1,12 @@ +#!/usr/bin/env ruby +require_relative "../config/boot" +require_relative "../lib/ci_database" + +database = CiDatabase.new +arguments = case ARGV +when [ "prepare" ] then database.prepare +when [ "drop" ] then [ "db:drop" ] +when [ "seed" ] then [ "db:seed" ] +else abort "Usage: bin/ci-db prepare|drop|seed (called by bin/ci)" +end +exec "bin/rails", *arguments diff --git a/config/ci.rb b/config/ci.rb index 279885e2..e8dcc0bd 100644 --- a/config/ci.rb +++ b/config/ci.rb @@ -1,7 +1,88 @@ -# Independent checks share the same worker budget as the RSpec shards. -LOCAL_CI_CHECKS = { +# Standalone Rakefile: the coordinator never boots Rails or opens a database. +require "rake" +require "fileutils" +require "json" +require "shellwords" + +jobs = Integer(ENV.fetch("COPLAN_CI_JOBS")) +browsers = Integer(ENV.fetch("COPLAN_CI_BROWSER_JOBS")) +seed = ENV.fetch("COPLAN_CI_SEED") +directory = ENV.fetch("COPLAN_CI_DIRECTORY") +results = {} +checks = { "rubocop" => [ "bin/rubocop", "--no-parallel" ], "gem-audit" => [ "bin/bundler-audit" ], "importmap-audit" => [ "bin/importmap", "audit" ], "brakeman" => [ "bin/brakeman", "--quiet", "--no-pager", "--exit-on-warn", "--exit-on-error" ] -}.freeze +} + +run = lambda do |name, env, *command| + path = File.join(directory, "#{name}.log") + puts "START #{name} — #{path}" + result = system(env, *command, out: path, err: [ :child, :out ]) + puts "#{result ? 'PASS' : 'FAIL'} #{name} — #{path}" + result +end + +parallel = lambda do |env, count, *args| + [ env, "bundle", "exec", "parallel_test", "-n", count.to_s, "--first-is-1", *args ] +end + +pool = lambda do |name, count, files| + raise "No #{name} specs found" if files.empty? + count = [ count, files.length ].min + env = { "COPLAN_CI_GROUP" => name } + runtime = File.expand_path("tmp/parallel_runtime_#{name}.log") + options = Shellwords.join([ "--require", "./lib/ci_database", "--seed", seed, "--format", "progress", + "--format", "ParallelTests::RSpec::RuntimeLogger", "--out", runtime, + "--format", "json", "--out", File.join(directory, "#{name}-$TEST_ENV_NUMBER.json") ]) + begin + passed = run.call("#{name}-prepare", *parallel.call(env, count, "--exec", "bin/ci-db prepare")) + if passed + passed = run.call(name, env, "bundle", "exec", "parallel_rspec", "-n", count.to_s, "--first-is-1", + "--runtime-log", runtime, "--test-options", options, *files) + reports = Dir[File.join(directory, "#{name}-*.json")] + passed &&= reports.length == count && reports.all? do |path| + summary = JSON.parse(File.read(path)).fetch("summary") + summary.fetch("example_count").positive? && summary.fetch("failure_count").zero? + end + end + ensure + cleaned = run.call("#{name}-drop", *parallel.call(env, count, "--exec", "bin/ci-db drop")) + results[name] = passed && cleaned + end +end + +checks.each do |name, command| + task name do + results[name] = run.call(name, {}, *command) + end +end +multitask checks: checks.keys + +task :preview do + puts "Local CI: parallel_tests, #{jobs} non-browser workers, #{browsers} browser workers, seed #{seed}" + puts "Rake checks: #{checks.keys.join(', ')}" + puts "RSpec pools: non-browser, then browser; seeds twice on a separate database" + puts "Logs and reports: #{directory}" +end + +task :ci do + $stdout.sync = true + FileUtils.mkdir_p(directory) + Rake::Task[:preview].invoke + Rake::Task[:checks].invoke + browser_files = Dir["spec/system/**/*_spec.rb"] + pool.call("specs", jobs, Dir["spec/**/*_spec.rb"] - browser_files) + pool.call("browser", browsers, browser_files) + env = { "COPLAN_CI_GROUP" => "seeds", "TEST_ENV_NUMBER" => "1" } + begin + results["seeds"] = run.call("seeds", env, "bin/ci-db", "prepare") && + run.call("seeds-first", env, "bin/ci-db", "seed") && run.call("seeds-second", env, "bin/ci-db", "seed") + ensure + cleaned = run.call("seeds-drop", env, "bin/ci-db", "drop") + results["seeds"] &&= cleaned + end + File.write(File.join(directory, "summary.json"), JSON.pretty_generate(results)) + abort "Local CI failed; see #{directory}" unless results.values.all? +end diff --git a/docs/CI.md b/docs/CI.md index 2031808f..04d33807 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -14,10 +14,12 @@ run `bin/setup`, alter the development database, or replant your usual test database. Dependencies must already be installed. Chrome for Testing is provisioned by Selenium using the same local configuration as ordinary system specs. -The default worker budget is half the detected CPU count, capped at eight and -at least one. Up to four workers run browser shards; the remaining slots can -run other specs or checks at the same time. RuboCop's own parallelism is disabled -so it shares this budget. To use more cores or less memory: +[parallel_tests](https://github.com/grosser/parallel_tests) runs RSpec in separate +processes. The default is half the detected CPU count, capped at eight and at +least one. The non-browser pool runs first, followed by a browser pool capped at +four. These pools do not overlap. Rake runs the independent lint/security checks +concurrently in an earlier phase; RuboCop's nested parallelism is disabled. +To use more cores or less memory: ```sh bin/ci --jobs 12 --browser-jobs 6 @@ -25,8 +27,10 @@ bin/ci --jobs 2 --browser-jobs 1 bin/ci --jobs 1 ``` -Each RSpec shard and the seed check get a newly named database, unique to this -checkout and invocation, and a separate schema file. The database account needs +Each RSpec process and the seed check get a newly named database, unique to +this checkout and invocation, and a separate schema file. The gem assigns +`TEST_ENV_NUMBER` starting at 1; `CiDatabase` maps that number and the pool to +private database/schema names before Rails loads. The database account needs permission to create and drop databases. Databases are dropped after each check, including after failures and Ctrl+C. If the coordinator is forcibly killed or the database server becomes unavailable, cleanup may need to be done manually; @@ -41,21 +45,23 @@ MySQL schema: DATABASE_URL=postgresql://postgres@localhost/coplan_local_test bin/ci ``` -Preview the selected worker counts and file counts without running commands: +Preview the worker limits and checks without running commands: ```sh bin/ci --list ``` -Files are balanced by source size, separately for browser and non-browser specs. -Every discovered spec runs once. The coordinator prints each check's start, -finish, duration, and log path. Logs, RSpec JSON reports, private schema files, -and `summary.json` remain under the printed `tmp/local-ci//` directory. +The gem balances files by recorded runtime when enough measurements exist, +falling back to file size. Browser and non-browser runtime logs live at +`tmp/parallel_runtime_browser.log` and `tmp/parallel_runtime_specs.log`. +Every discovered spec runs once. Rake prints each check's start, result and log +path. Pool logs, numbered RSpec JSON reports, private schema files, and +`summary.json` remain under the printed `tmp/local-ci//` directory. An empty suite, a missing report, a failed audit, or a cleanup failure makes `bin/ci` exit nonzero. Other checks finish so one failure does not hide their results. Repeat a test order with `bin/ci --seed `; keep the same worker -counts to preserve the same groups. GitHub's extra repeated regression cohorts -remain available separately through `bin/rspec-stress`. +counts and runtime logs to preserve the same groups. GitHub's extra repeated +regression cohorts remain available separately through `bin/rspec-stress`. ## GitHub Actions diff --git a/lib/ci_database.rb b/lib/ci_database.rb new file mode 100644 index 00000000..bd3e8e5e --- /dev/null +++ b/lib/ci_database.rb @@ -0,0 +1,40 @@ +require "digest" +require "erb" +require "fileutils" +require "uri" +require "yaml" + +# The only project-specific parallel_tests setup: fresh database/schema names. +class CiDatabase + attr_reader :env, :postgres + + def initialize(source = ENV) + directory = source.fetch("COPLAN_CI_DIRECTORY") + group = source.fetch("COPLAN_CI_GROUP") + number = source.fetch("TEST_ENV_NUMBER") + raise ArgumentError, "Invalid CI worker" unless group.match?(/\A(?:specs|browser|seeds)\z/) && number.match?(/\A[1-9]\d*\z/) + database = "coplan_ci_#{Digest::SHA256.hexdigest(directory)[0, 12]}_#{group}_#{number}" + config = YAML.safe_load(ERB.new(File.read("config/database.yml")).result, aliases: true).fetch("test") + url = URI.parse(source.fetch("DATABASE_URL")) if source["DATABASE_URL"] + adapter = url ? url.scheme : config.fetch("adapter") + raise ArgumentError, "Local CI supports mysql2 and PostgreSQL databases" unless %w[mysql2 postgres postgresql].include?(adapter) + @postgres = %w[postgres postgresql].include?(adapter) + @env = { "RAILS_ENV" => "test", "COPLAN_TEST_DATABASE" => database, "SCHEMA" => File.join(directory, "#{group}-#{number}-schema.rb") } + if url + url.path = "/#{database}" + params = URI.decode_www_form(url.query || "").reject { |key, _| key == "database" } + params << [ "encoding", "unicode" ] if postgres && params.none? { |key, _| key == "encoding" } + url.query = params.empty? ? nil : URI.encode_www_form(params) + @env["DATABASE_URL"] = url.to_s + end + end + + def prepare + FileUtils.mkdir_p(File.dirname(env.fetch("SCHEMA"))) + FileUtils.cp("db/schema.rb", env.fetch("SCHEMA")) unless postgres + [ "db:create", postgres ? "db:migrate" : "db:schema:load" ] + end +end + +# parallel_rspec loads this before rails_helper, using its TEST_ENV_NUMBER. +ENV.update(CiDatabase.new.env) if ENV["COPLAN_CI_GROUP"] diff --git a/lib/local_ci.rb b/lib/local_ci.rb deleted file mode 100644 index 729ffede..00000000 --- a/lib/local_ci.rb +++ /dev/null @@ -1,207 +0,0 @@ -require "digest" -require "erb" -require "etc" -require "fileutils" -require "json" -require "optparse" -require "securerandom" -require "thread" -require "uri" -require "yaml" -require_relative "../config/ci" - -# Local tooling only: never boot Rails in the coordinator or share a test DB. -class LocalCi - Task = Struct.new(:name, :commands, :env, :database, :report, keyword_init: true) - - def self.start(arguments) - options = { jobs: [ [ Etc.nprocessors / 2, 1 ].max, 8 ].min, seed: Random.new_seed % 65536 } - parser = OptionParser.new do |args| - args.banner = "Usage: bin/ci [--jobs N] [--browser-jobs N] [--seed N] [--list]" - args.on("--jobs N", Integer, "Maximum simultaneous checks (default: half the CPUs, up to 8)") { |n| options[:jobs] = n } - args.on("--browser-jobs N", Integer, "Browser shards (default: up to 4)") { |n| options[:browser_jobs] = n } - args.on("--seed N", Integer, "RSpec random-order seed") { |n| options[:seed] = n } - args.on("--list", "Show the plan without running commands or creating databases") { options[:list] = true } - args.on("-h", "--help") { puts args; return } - end - parser.parse!(arguments) - raise OptionParser::InvalidArgument, arguments.join(" ") unless arguments.empty? - options[:browser_jobs] ||= [ options[:jobs], 4 ].min - unless options[:jobs].positive? && options[:browser_jobs].between?(1, options[:jobs]) - raise OptionParser::InvalidArgument, "jobs must be positive; browser-jobs must be between 1 and jobs" - end - list = options.delete(:list) - runner = new(**options) - if list - runner.describe - else - exit(runner.run ? 0 : 1) - end - rescue OptionParser::ParseError, ArgumentError => error - abort error.message - end - - # Greedy file-size balancing avoids putting every large browser spec together. - def self.shards(files, count) - groups = Array.new([ files.length, count ].min) { [] } - sizes = groups.to_h { |group| [ group.object_id, 0 ] } - files.sort_by { |file| [ -File.size(file), file ] }.each do |file| - group = groups.min_by { |candidate| sizes.fetch(candidate.object_id) } - group << file - sizes[group.object_id] += File.size(file) - end - groups - end - - attr_reader :tasks, :directory - - def initialize(jobs:, browser_jobs:, seed:) - @jobs, @seed = jobs, seed - @directory = File.expand_path("tmp/local-ci/#{Time.now.utc.strftime('%Y%m%dT%H%M%S')}-#{Process.pid}-#{SecureRandom.hex(3)}") - @mutex, @children, @results = Mutex.new, {}, {} - @interrupted = false - database_config = YAML.safe_load(ERB.new(File.read("config/database.yml")).result, aliases: true).fetch("test") - adapter = ENV["DATABASE_URL"] ? URI.parse(ENV.fetch("DATABASE_URL")).scheme : database_config.fetch("adapter") - raise ArgumentError, "Local CI supports mysql2 and PostgreSQL databases" unless %w[mysql2 postgres postgresql].include?(adapter) - @postgres = %w[postgres postgresql].include?(adapter) - browser = Dir["spec/system/**/*_spec.rb"] - other = Dir["spec/**/*_spec.rb"] - browser - raise ArgumentError, "No browser or non-browser specs found" if browser.empty? || other.empty? - @tasks = LOCAL_CI_CHECKS.map { |name, command| Task.new(name: name, commands: [ command ], env: {}) } - self.class.shards(browser, browser_jobs).each_with_index { |files, i| @tasks << spec_task("browser-#{i + 1}", files) } - self.class.shards(other, jobs).each_with_index { |files, i| @tasks << spec_task("specs-#{i + 1}", files) } - @tasks << database_task("seeds", [ [ "bin/rails", "db:seed" ], [ "bin/rails", "db:seed" ] ]) - end - - def describe - puts "Local CI: #{@jobs} workers, RSpec seed #{@seed}" - puts "Logs and reports: #{directory}" - tasks.each do |task| - count = task.commands.flatten.count { |arg| arg.end_with?("_spec.rb") } - puts " #{task.name}#{count.positive? ? " (#{count} spec files)" : ''}#{task.database ? ' — isolated database' : ''}" - end - end - - def run - describe - FileUtils.mkdir_p(directory) - queue = Queue.new - tasks.each { |task| queue << task } - threads = Array.new([ @jobs, tasks.length ].min) do - Thread.new do - loop do - task = queue.pop(true) - break if @interrupted - execute(task) - rescue ThreadError - break - end - end - end - previous_term = Signal.trap("TERM") { raise Interrupt } - begin - threads.each(&:join) - rescue Interrupt - pids = [] - @mutex.synchronize do - @interrupted = true - pids = @children.keys - pids.each { |pid| signal_group("TERM", pid) } - end - puts "Interrupted; stopping workers and removing their databases…" - killer = Thread.new do - sleep 5 - @mutex.synchronize { pids.each { |pid| signal_group("KILL", pid) if @children.key?(pid) } } - end - threads.each(&:join) - ensure - killer&.kill - killer&.join - Signal.trap("TERM", previous_term) - end - File.write(File.join(directory, "summary.json"), JSON.pretty_generate(@results)) - passed = !@interrupted && @results.length == tasks.length && @results.values.all? - puts "#{passed ? 'PASS' : 'FAIL'}: #{@results.count { |_, result| result }}/#{tasks.length} checks passed (#{directory})" - passed - end - - private - def signal_group(signal, pid) - Process.kill(signal, -pid) - rescue Errno::ESRCH - # The command completed between the snapshot and the signal. - end - - def spec_task(name, files) - report = File.join(directory, "#{name}.json") - task = database_task(name, [ [ "bundle", "exec", "rspec", *files, "--seed", @seed.to_s, - "--format", "progress", "--format", "json", "--out", report ] ]) - task.report = report - task - end - - def database_task(name, commands) - database = "coplan_ci_#{Digest::SHA256.hexdigest(directory)[0, 12]}_#{name.tr('-', '_')}" - env = { "RAILS_ENV" => "test", "COPLAN_TEST_DATABASE" => database, - "SCHEMA" => File.join(directory, "#{name}-schema.rb") } - if ENV["DATABASE_URL"] - url = URI.parse(ENV.fetch("DATABASE_URL")) - url.path = "/#{database}" - # Active Record query parameters take precedence over the URL path. - params = URI.decode_www_form(url.query || "").reject { |key, _| key == "database" } - params << [ "encoding", "unicode" ] if @postgres && params.none? { |key, _| key == "encoding" } - url.query = params.empty? ? nil : URI.encode_www_form(params) - env["DATABASE_URL"] = url.to_s - end - Task.new(name: name, commands: commands, env: env, database: database) - end - - def command(task, args, log, cleanup: false) - pid = @mutex.synchronize do - return false if @interrupted && !cleanup - Process.spawn(task.env, *args, out: log, err: log, pgroup: true).tap { |child| @children[child] = true } - end - _, status = Process.wait2(pid) - status.success? - ensure - @mutex.synchronize { @children.delete(pid) } if pid - end - - def execute(task) - started = Process.clock_gettime(Process::CLOCK_MONOTONIC) - path = File.join(directory, "#{task.name}.log") - puts "START #{task.name}" - passed = false - File.open(path, "w") do |log| - begin - if task.database - FileUtils.cp("db/schema.rb", task.env.fetch("SCHEMA")) unless @postgres - setup = [ "bin/rails", "db:create", @postgres ? "db:migrate" : "db:schema:load" ] - return unless command(task, setup, log) - end - passed = task.commands.all? { |args| command(task, args, log) } - if passed && task.report - summary = JSON.parse(File.read(task.report)).fetch("summary") - passed = summary.fetch("example_count").positive? && summary.fetch("failure_count").zero? - end - rescue StandardError => error - log.puts "#{error.class}: #{error.message}" - passed = false - ensure - if task.database - begin - dropped = command(task, [ "bin/rails", "db:drop" ], log, cleanup: true) - passed &&= dropped - rescue StandardError => error - log.puts "Database cleanup failed: #{error.class}: #{error.message}" - passed = false - end - end - end - end - ensure - @mutex.synchronize { @results[task.name] = passed } - elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started - puts "#{passed ? 'PASS' : 'FAIL'} #{task.name} (#{elapsed.round(1)}s) — #{path}" - end -end diff --git a/spec/lib/ci_database_spec.rb b/spec/lib/ci_database_spec.rb new file mode 100644 index 00000000..16546a39 --- /dev/null +++ b/spec/lib/ci_database_spec.rb @@ -0,0 +1,31 @@ +require "spec_helper" +require_relative "../../lib/ci_database" + +RSpec.describe CiDatabase do + let(:source) { { "COPLAN_CI_DIRECTORY" => "/tmp/coplan-ci-example", "COPLAN_CI_GROUP" => "specs", "TEST_ENV_NUMBER" => "1" } } + + it "keeps workers, cohorts and concurrent invocations in different databases" do + inputs = [ source, source.merge("TEST_ENV_NUMBER" => "2"), source.merge("COPLAN_CI_GROUP" => "browser"), + source.merge("COPLAN_CI_DIRECTORY" => "/tmp/coplan-ci-other") ] + configs = inputs.map { |input| described_class.new(input).env } + expect(configs.map { |env| env.fetch("COPLAN_TEST_DATABASE") }.uniq.length).to eq(4) + expect(configs.map { |env| env.fetch("SCHEMA") }.uniq.length).to eq(4) + expect(configs.map { |env| env.fetch("RAILS_ENV") }).to all(eq("test")) + end + + it "replaces URL database selectors without changing the connection" do + config = described_class.new(source.merge("DATABASE_URL" => "postgresql://test-user@localhost:5433/shared?database=shared&sslmode=require")) + url = URI.parse(config.env.fetch("DATABASE_URL")) + expect(url.path).to eq("/#{config.env.fetch('COPLAN_TEST_DATABASE')}") + expect([ url.user, url.host, url.port ]).to eq([ "test-user", "localhost", 5433 ]) + expect(URI.decode_www_form(url.query).to_h).to eq("sslmode" => "require", "encoding" => "unicode") + expect(config.postgres).to be true + end + + it "rejects missing or invalid worker numbers rather than using a shared database" do + [ "", "0", "../shared" ].each do |number| + expect { described_class.new(source.merge("TEST_ENV_NUMBER" => number)) }.to raise_error(ArgumentError) + end + expect { described_class.new(source.merge("COPLAN_CI_GROUP" => "development")) }.to raise_error(ArgumentError) + end +end diff --git a/spec/lib/local_ci_spec.rb b/spec/lib/local_ci_spec.rb index aaa15b19..82dd8bfd 100644 --- a/spec/lib/local_ci_spec.rb +++ b/spec/lib/local_ci_spec.rb @@ -1,145 +1,128 @@ require "spec_helper" -require "tmpdir" -require "rbconfig" +require "fileutils" +require "json" require "open3" +require "rbconfig" +require "tmpdir" require "timeout" -require_relative "../../lib/local_ci" -RSpec.describe LocalCi do +RSpec.describe "Local CI with parallel_tests" do + ROOT = File.expand_path("../..", __dir__) + around do |example| - original = ENV.to_h.slice("DATABASE_URL", "SCHEMA", "COPLAN_TEST_DATABASE") - ENV.delete("DATABASE_URL") Dir.mktmpdir do |directory| @directory = directory + %w[bin config lib db].each { |name| FileUtils.mkdir_p(File.join(directory, name)) } + %w[bin/ci bin/ci-db config/ci.rb lib/ci_database.rb].each { |path| FileUtils.cp(File.join(ROOT, path), File.join(directory, path)) } + File.write(File.join(directory, "config/boot.rb"), "require 'bundler/setup'\n") + File.write(File.join(directory, "config/database.yml"), "test:\n adapter: mysql2\n") + File.write(File.join(directory, "db/schema.rb"), "# fake schema\n") + %w[rubocop bundler-audit importmap brakeman].each { |name| executable("bin/#{name}", "exit(ENV['FAIL_AUDIT'] && #{name.dump} == 'bundler-audit' ? 7 : 0)") } + executable("bin/rails", <<~'RUBY') + require 'json' + name = "#{ENV.fetch('COPLAN_CI_GROUP')}-#{ENV.fetch('TEST_ENV_NUMBER')}" + File.open("#{name}-database.jsonl", 'a') do |file| + file.puts(JSON.generate(command:ARGV, database:ENV.fetch('COPLAN_TEST_DATABASE'), schema:ENV.fetch('SCHEMA'))) + end + exit 3 if ENV['FAIL_SEED_DROP'] && ENV['COPLAN_CI_GROUP'] == 'seeds' && ARGV == ['db:drop'] + RUBY + %w[models/a models/b system/a system/b].each do |name| + path = File.join(directory, "spec/#{name}_spec.rb") + FileUtils.mkdir_p(File.dirname(path)) + File.write(path, <<~'RUBY') + require 'timeout' + RSpec.describe 'CI integration' do + it 'uses the worker environment before loading Rails' do + group, number = ENV.values_at('COPLAN_CI_GROUP', 'TEST_ENV_NUMBER') + expect(ENV.fetch('COPLAN_TEST_DATABASE')).to end_with("_#{group}_#{number}") + File.write("ready-#{group}-#{number}", Process.pid) + Timeout.timeout(10) { sleep 0.01 until Dir["ready-#{group}-*"].length == 2 } + sleep 60 if ENV['HOLD_SPECS'] + expect(ENV['FAIL_SPEC']).to be_nil + end + end unless ENV['EMPTY_SPECS'] + RUBY + end example.run end - ensure - %w[DATABASE_URL SCHEMA COPLAN_TEST_DATABASE].each { |key| original.key?(key) ? ENV[key] = original[key] : ENV.delete(key) } end - def runner(jobs: 2) - described_class.new(jobs: jobs, browser_jobs: 1, seed: 123).tap do |ci| - ci.instance_variable_set(:@directory, @directory) - end + def executable(path, source) + file = File.join(@directory, path) + File.write(file, "#!#{RbConfig.ruby}\n#{source}\n") + File.chmod(0o755, file) end - def task(name, source, report: nil) - described_class::Task.new(name: name, env: {}, report: report, - commands: [ [ RbConfig.ruby, "-e", source ] ]) + def run_ci(extra_env = {}, *args) + Open3.capture3({ "BUNDLE_GEMFILE" => File.join(ROOT, "Gemfile"), "COPLAN_CI_GROUP" => nil }.merge(extra_env), + File.join(@directory, "bin/ci"), "--jobs", "2", "--browser-jobs", "2", *args, chdir: @directory) end - it "includes every spec exactly once and keeps browser shards separate" do - ci = runner(jobs: 4) - commands = ci.tasks.flat_map(&:commands) - selected = commands.flatten.select { |arg| arg.end_with?("_spec.rb") } - expect(selected.sort).to eq(Dir["spec/**/*_spec.rb"].sort) - expect(ci.tasks.select { |item| item.name.start_with?("browser-") }.flat_map(&:commands).flatten.grep(/_spec\.rb$/)).to all(start_with("spec/system/")) - expect(ci.tasks.select { |item| item.name.start_with?("specs-") }.flat_map(&:commands).flatten.grep(/_spec\.rb$/)).not_to include(start_with("spec/system/")) - end - - it "uses different databases and schema files for every worker and invocation" do - first = runner.tasks.select(&:database) - second = runner.tasks.select(&:database) - databases = (first + second).map(&:database) - expect(databases.uniq).to eq(databases) - expect(first.map { |item| item.env.fetch("SCHEMA") }.uniq.length).to eq(first.length) - expect(first.map { |item| item.env.fetch("RAILS_ENV") }).to all(eq("test")) - expect(first.map(&:database)).not_to include("coplan_test", "coplan_development") - end - - it "preserves URL connection options while replacing every database selector" do - ENV["DATABASE_URL"] = "postgresql://test-user@localhost:5433/shared?encoding=unicode&database=shared&sslmode=require" - ci = runner - ci.tasks.select(&:database).each do |item| - url = URI.parse(item.env.fetch("DATABASE_URL")) - expect(url.path).to eq("/#{item.database}") - expect(url.userinfo).to eq("test-user") - expect(url.port).to eq(5433) - expect(URI.decode_www_form(url.query).to_h).to eq("encoding" => "unicode", "sslmode" => "require") + def database_commands + Dir[File.join(@directory, "*-database.jsonl")].to_h do |path| + [ File.basename(path), File.readlines(path).map { |line| JSON.parse(line) } ] end end - it "runs workers concurrently and waits for every result after a failure" do - ci = runner - # Each process must see its sibling before it can finish. A sequential - # implementation fails the deadline instead of silently passing this test. - source = <<~RUBY - require 'timeout' - File.write(File.join(#{@directory.dump}, ARGV.fetch(0)), '') - Timeout.timeout(5) do - sleep 0.01 until Dir[File.join(#{@directory.dump}, 'ready-*')].length == 2 - end - RUBY - first = task("first", source) - first.commands.first.concat([ "ready-first" ]) - second = task("second", source) - second.commands.first.concat([ "ready-second" ]) - ci.tasks.replace([ first, second, task("failed", "exit 7"), task("last", "puts 'still ran'") ]) - expect(ci.run).to be false - expect(JSON.parse(File.read(File.join(@directory, "summary.json")))).to eq( - "first" => true, "second" => true, "failed" => false, "last" => true) - expect(File.read(File.join(@directory, "last.log"))).to include("still ran") + it "runs real parallel_tests workers, writes numbered reports and cleans up owned databases" do + output, error, status = run_ci + expect(status).to be_success, "#{output}\n#{error}" + directory = Dir[File.join(@directory, "tmp/local-ci/*")].fetch(0) + reports = Dir[File.join(directory, "{specs,browser}-*.json")] + expect(reports.length).to eq(4) + expect(reports.map { |path| JSON.parse(File.read(path)).fetch("summary").fetch("example_count") }).to eq([ 1, 1, 1, 1 ]) + rows = database_commands.values + expect(rows.length).to eq(5) + expect(rows.map { |entries| entries.first.fetch("database") }.uniq.length).to eq(5) + expect(rows.map { |entries| entries.last.fetch("command") }).to all(eq([ "db:drop" ])) + expect(File.read(File.join(@directory, "tmp/parallel_runtime_browser.log"))).to include("spec/system/") + expect(JSON.parse(File.read(File.join(directory, "summary.json"))).values).to all(be true) end - it "rejects an empty suite or missing report even when RSpec exits zero" do - ci = runner - report = File.join(@directory, "empty.json") - File.write(report, JSON.generate(summary: { example_count: 0, failure_count: 0 })) - ci.tasks.replace([ task("empty", "exit 0", report: report), task("missing", "exit 0", report: "#{report}.missing") ]) - expect(ci.run).to be false - expect(JSON.parse(File.read(File.join(@directory, "summary.json"))).values).to eq([ false, false ]) - end - - it "cleans up a database after setup fails and never runs its specs" do - ci = runner(jobs: 1) - item = ci.tasks.find(&:database) - item.env["SCHEMA"] = File.join(@directory, "schema.rb") - ci.tasks.replace([ item ]) - commands = [] - allow(ci).to receive(:command) do |_task, args, _log, **_options| - commands << args - args.include?("db:drop") - end - expect(ci.run).to be false - expect(commands.map(&:last)).to eq([ "db:schema:load", "db:drop" ]) + it "keeps cleanup and other pools when an audit or RSpec process fails" do + output, error, status = run_ci({ "FAIL_SPEC" => "1", "FAIL_AUDIT" => "1" }) + expect(status).not_to be_success, "#{output}\n#{error}" + rows = database_commands.values + expect(rows.length).to eq(5) + expect(rows.map { |entries| entries.last.fetch("command") }).to all(eq([ "db:drop" ])) + directory = Dir[File.join(@directory, "tmp/local-ci/*")].fetch(0) + summary = JSON.parse(File.read(File.join(directory, "summary.json"))) + expect(summary).to include("gem-audit" => false, "specs" => false, "browser" => false, "seeds" => true) end - it "fails when database cleanup fails" do - ci = runner(jobs: 1) - item = ci.tasks.last # The seed check has no RSpec report to fake. - item.env["SCHEMA"] = File.join(@directory, "schema.rb") - ci.tasks.replace([ item ]) - allow(ci).to receive(:command) { |_task, args, _log, **_options| !args.include?("db:drop") } - expect(ci.run).to be false + it "previews without starting workers or creating databases" do + output, error, status = run_ci({}, "--list") + expect(status).to be_success, error + expect(output).to include("parallel_tests", "2 non-browser workers", "2 browser workers") + expect(database_commands).to be_empty + expect(Dir[File.join(@directory, "tmp/local-ci/*")]).to be_empty end - it "rejects invalid worker counts before starting commands" do - expect { described_class.start([ "--jobs", "0" ]) }.to raise_error(SystemExit) - expect { described_class.start([ "--jobs", "2", "--browser-jobs", "3" ]) }.to raise_error(SystemExit) + it "rejects empty suites and reports a failed seed database cleanup" do + output, error, status = run_ci({ "EMPTY_SPECS" => "1", "FAIL_SEED_DROP" => "1" }) + expect(status).not_to be_success, "#{output}\n#{error}" + directory = Dir[File.join(@directory, "tmp/local-ci/*")].fetch(0) + reports = Dir[File.join(directory, "{specs,browser}-*.json")] + expect(reports.length).to eq(4) + expect(reports.map { |path| JSON.parse(File.read(path)).fetch("summary").fetch("example_count") }).to eq([ 0, 0, 0, 0 ]) + summary = JSON.parse(File.read(File.join(directory, "summary.json"))) + expect(summary).to include("specs" => false, "browser" => false, "seeds" => false) end - it "stops child processes and exits unsuccessfully on interruption" do - started = File.join(@directory, "child.pid") - source = <<~RUBY - require #{File.expand_path('../../lib/local_ci', __dir__).dump} - ci = LocalCi.new(jobs: 1, browser_jobs: 1, seed: 123) - ci.instance_variable_set(:@directory, #{@directory.dump}) - child = "File.write(ARGV.fetch(0), Process.pid); sleep 60" - ci.tasks.replace([LocalCi::Task.new(name: 'interrupted', env: {}, commands: [[#{RbConfig.ruby.dump}, '-e', child, #{started.dump}]])]) - exit(ci.run ? 0 : 1) - RUBY - # Keep the signal inside a subprocess so RSpec's own handlers are untouched. - Open3.popen2e(RbConfig.ruby, "-e", source) do |input, output, wait| + it "stops after Ctrl+C and drops the interrupted pool's databases" do + env = { "BUNDLE_GEMFILE" => File.join(ROOT, "Gemfile"), "COPLAN_CI_GROUP" => nil, "HOLD_SPECS" => "1" } + Open3.popen2e(env, File.join(@directory, "bin/ci"), "--jobs", "2", "--browser-jobs", "2", chdir: @directory, pgroup: true) do |input, output, wait| input.close - Timeout.timeout(10) { sleep 0.01 until File.exist?(started) || !wait.alive? } + Timeout.timeout(30) { sleep 0.01 until Dir[File.join(@directory, "ready-specs-*")].length == 2 || !wait.alive? } expect(wait).to be_alive, output.read unless wait.alive? - Process.kill("INT", wait.pid) - status = Timeout.timeout(15) { wait.value } - expect(status.exitstatus).to eq(1), output.read - expect { Process.kill(0, Integer(File.read(started))) }.to raise_error(Errno::ESRCH) - expect(File.read(File.join(@directory, "summary.json"))).to include('"interrupted"') + Process.kill("INT", -wait.pid) + status = Timeout.timeout(20) { wait.value } + expect(status).not_to be_success + rows = database_commands.values + expect(rows.length).to eq(2) + expect(rows.map { |entries| entries.last.fetch("command") }).to all(eq([ "db:drop" ])) ensure - Process.kill("KILL", wait.pid) if wait.alive? + Process.kill("KILL", -wait.pid) if wait.alive? end end end