diff --git a/Rakefile b/Rakefile
index d21efc97a..2216b4a31 100644
--- a/Rakefile
+++ b/Rakefile
@@ -19,6 +19,7 @@ end
 Bundler::GemHelper.install_tasks
 
 require 'solr_wrapper'
+require 'open3'
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)
@@ -30,16 +31,42 @@ require 'engine_cart/rake_task'
 
 require 'spotlight/version'
 
+def system_with_error_handling(*args)
+  Open3.popen3(*args) do |_stdin, stdout, stderr, thread|
+    puts stdout.read
+    raise "Unable to run #{args.inspect}: #{stderr.read}" unless thread.value.success?
+  end
+end
+
+def with_solr(&block) # rubocop:disable Metrics/MethodLength
+  # We're being invoked by the app entrypoint script and solr is already up via docker compose
+  if ENV['SOLR_ENV'] == 'docker-compose'
+    yield
+  elsif system('docker compose version')
+    # We're not running `docker compose up' but still want to use a docker instance of solr.
+    begin
+      puts 'Starting Solr'
+      system_with_error_handling 'docker compose up -d solr'
+      yield
+    ensure
+      puts 'Stopping Solr'
+      system_with_error_handling 'docker compose stop solr'
+    end
+  else
+    SolrWrapper.wrap do |solr|
+      solr.with_collection(&block)
+    end
+  end
+end
+
 task ci: ['engine_cart:generate'] do
   ENV['environment'] = 'test'
 
-  SolrWrapper.wrap(port: '8983') do |solr|
-    solr.with_collection(name: 'blacklight-core', dir: 'lib/generators/spotlight/templates/solr/conf') do
-      Rake::Task['spotlight:fixtures'].invoke
+  with_solr do
+    Rake::Task['spotlight:fixtures'].invoke
 
-      # run the tests
-      Rake::Task['spec'].invoke
-    end
+    # run the tests
+    Rake::Task['spec'].invoke
   end
 end