diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index ae001dd..916bd09 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -12,11 +12,19 @@
 
 ENV['RAILS_ENV'] = 'test'
 require 'rails'
-ENV['RAILS'] = Rails.version
-ENV['RAILS_ROOT'] = File.expand_path("../rails/rails-#{ENV['RAILS']}", __FILE__)
-
-# Create the test app if it doesn't exist
-system 'rake setup' unless File.exist?(ENV['RAILS_ROOT'])
+require 'fileutils'
+require_relative 'support/test_app_paths'
+ENV['RAILS_ROOT'] = TestAppPaths.app_root
+
+# Create the test app if it doesn't exist. rails new creates the target
+# directory before the template runs, so a template that fails halfway leaves
+# the directory behind and every later run would skip regeneration and die on
+# config/environment.rb instead of reporting the real error. Check the file the
+# suite actually loads, and refuse to continue if the build did not succeed.
+unless File.exist?(File.join(ENV['RAILS_ROOT'], 'config', 'environment.rb'))
+  FileUtils.rm_rf(ENV['RAILS_ROOT'])
+  abort 'rake setup failed; dummy app not built' unless system('rake setup')
+end
 
 require 'active_model'
 # require ActiveRecord to ensure that Ransack loads correctly
diff --git a/spec/support/test_app_paths.rb b/spec/support/test_app_paths.rb
new file mode 100644
index 0000000..d77fe5d
--- /dev/null
+++ b/spec/support/test_app_paths.rb
@@ -0,0 +1,19 @@
+require 'rails/version'
+
+# Where the generated dummy app lives, and under what name.
+#
+# The name has to carry every input that changes what gets generated.
+# Keying on the Rails version alone meant `AA=3.2.0 rspec` silently reused an
+# app whose active_admin:install had been run by a different Active Admin, so
+# the leg reported on a build it never produced.
+module TestAppPaths
+  module_function
+
+  def app_dir_name
+    "rails-#{Rails::VERSION::STRING}-aa#{ENV['AA'] || 'default'}"
+  end
+
+  def app_root
+    File.expand_path("../rails/#{app_dir_name}", __dir__)
+  end
+end
diff --git a/tasks/test.rake b/tasks/test.rake
index e7d8dab..689b7fa 100644
--- a/tasks/test.rake
+++ b/tasks/test.rake
@@ -1,6 +1,7 @@
+require_relative '../spec/support/test_app_paths'
+
 desc 'Creates a test rails app for the specs to run against'
 task :setup do
-  require 'rails/version'
 
   rails_new_args = %w[
     --skip-turbolinks
@@ -10,5 +11,8 @@ task :setup do
     spec/support/rails_template.rb
   ].join(' ')
 
-  system "bundle exec rails new spec/rails/rails-#{Rails::VERSION::STRING} #{rails_new_args}"
+  puts "[setup] Rails #{Rails::VERSION::STRING} / Active Admin #{ENV['AA'] || '(Gemfile default)'}"
+
+  abort 'rails new failed' unless
+    system("bundle exec rails new spec/rails/#{TestAppPaths.app_dir_name} #{rails_new_args}")
 end