Merge pull request #190 from SamSaffron/performance-tests

Performance tests

Rakefile

@@ -21,8 +21,6 @@ end
 
 Bundler.require(:default, :test)
 
-task default: :spec
-
 module CustomBuild
   def build_gem
     `cp assets/message-bus* vendor/assets/javascripts`
@@ -36,27 +34,36 @@ module Bundler
   end
 end
 
-run_spec = proc do |backend|
-  begin
-    ENV['MESSAGE_BUS_BACKEND'] = backend
-    sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/**/*_spec.rb'].to_a.join(' ')}"
-  ensure
-    ENV.delete('MESSAGE_BUS_BACKEND')
-  end
-end
-
-task spec: [:spec_memory, :spec_redis, :spec_postgres, :spec_client_js, :rubocop, :test_doc]
-
 task spec_client_js: 'jasmine:ci'
 
-task :spec_redis do
-  run_spec.call('redis')
-end
+backends = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?<backend>.*).rb})[:backend] } - ["base"]
 
-task :spec_memory do
-  run_spec.call('memory')
+namespace :spec do
+  backends.each do |backend|
+    desc "Run tests on the #{backend} backend"
+    task backend do
+      begin
+        ENV['MESSAGE_BUS_BACKEND'] = backend
+        sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/**/*_spec.rb'].to_a.join(' ')}"
+      ensure
+        ENV.delete('MESSAGE_BUS_BACKEND')
+      end
+    end
+  end
 end
 
-task :spec_postgres do
-  run_spec.call('postgres')
+desc "Run tests on all backends, plus client JS tests"
+task spec: backends.map { |backend| "spec:#{backend}" } + [:spec_client_js]
+
+desc "Run performance benchmarks on all backends"
+task :performance do
+  begin
+    ENV['MESSAGE_BUS_BACKENDS'] = backends.join(",")
+    sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/performance/*.rb'].to_a.join(' ')}"
+  ensure
+    ENV.delete('MESSAGE_BUS_BACKENDS')
+  end
 end
+
+desc "Run all tests, link checks and confirms documentation compiles without error"
+task default: [:spec, :rubocop, :test_doc]

spec/helpers.rb

@@ -0,0 +1,19 @@
+def wait_for(timeout_milliseconds = 2000)
+  timeout = (timeout_milliseconds + 0.0) / 1000
+  finish = Time.now + timeout
+
+  Thread.new do
+    sleep(0.001) while Time.now < finish && !yield
+  end.join
+end
+
+def test_config_for_backend(backend)
+  config = { backend: backend }
+  case backend
+  when :redis
+    config[:url] = ENV['REDISURL']
+  when :postgres
+    config[:backend_options] = { host: ENV['PGHOST'], user: ENV['PGUSER'] || ENV['USER'], password: ENV['PGPASSWORD'], dbname: ENV['PGDATABASE'] || 'message_bus_test' }
+  end
+  config
+end

spec/performance/publish.rb

@@ -0,0 +1,102 @@
+$LOAD_PATH << File.join(File.dirname(__FILE__), '..', '..', 'lib')
+require 'logger'
+require 'benchmark'
+require 'message_bus'
+
+require_relative "../helpers"
+
+backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym)
+channel = "/foo"
+iterations = 10_000
+results = []
+
+puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}"
+
+benchmark_publication_only = lambda do |bm, backend|
+  bus = MessageBus::Instance.new
+  bus.configure(test_config_for_backend(backend))
+
+  bm.report("#{backend} - publication only") do
+    iterations.times { bus.publish(channel, "Hello world") }
+  end
+
+  bus.reset!
+  bus.destroy
+end
+
+benchmark_subscription_no_trimming = lambda do |bm, backend|
+  test_title = "#{backend} - subscription no trimming"
+
+  bus = MessageBus::Instance.new
+  bus.configure(test_config_for_backend(backend))
+
+  bus.reliable_pub_sub.max_backlog_size = iterations
+  bus.reliable_pub_sub.max_global_backlog_size = iterations
+
+  messages_received = 0
+  bus.after_fork
+  bus.subscribe(channel) do |_message|
+    messages_received += 1
+  end
+
+  bm.report(test_title) do
+    iterations.times { bus.publish(channel, "Hello world") }
+    wait_for(60000) { messages_received == iterations }
+  end
+
+  results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%"
+
+  bus.reset!
+  bus.destroy
+end
+
+benchmark_subscription_with_trimming = lambda do |bm, backend|
+  test_title = "#{backend} - subscription with trimming"
+
+  bus = MessageBus::Instance.new
+  bus.configure(test_config_for_backend(backend))
+
+  bus.reliable_pub_sub.max_backlog_size = (iterations / 10)
+  bus.reliable_pub_sub.max_global_backlog_size = (iterations / 10)
+
+  messages_received = 0
+  bus.after_fork
+  bus.subscribe(channel) do |_message|
+    messages_received += 1
+  end
+
+  bm.report(test_title) do
+    iterations.times { bus.publish(channel, "Hello world") }
+    wait_for(60000) { messages_received == iterations }
+  end
+
+  results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%"
+
+  bus.reset!
+  bus.destroy
+end
+
+puts
+Benchmark.bm(60) do |bm|
+  backends.each do |backend|
+    benchmark_publication_only.call(bm, backend)
+  end
+
+  puts
+
+  backends.each do |backend|
+    benchmark_subscription_no_trimming.call(bm, backend)
+  end
+
+  results << nil
+  puts
+
+  backends.each do |backend|
+    benchmark_subscription_with_trimming.call(bm, backend)
+  end
+end
+puts
+
+results.each do |result|
+  puts result
+end

spec/spec_helper.rb

@@ -7,29 +7,14 @@
 require 'minitest/autorun'
 require 'minitest/spec'
 
+require_relative "helpers"
+
 backend = (ENV['MESSAGE_BUS_BACKEND'] || :redis).to_sym
-MESSAGE_BUS_CONFIG = { backend: backend }
+MESSAGE_BUS_CONFIG = test_config_for_backend(backend)
 require "message_bus/backends/#{backend}"
 PUB_SUB_CLASS = MessageBus::BACKENDS.fetch(backend)
-case backend
-when :redis
-  MESSAGE_BUS_CONFIG.merge!(url: ENV['REDISURL'])
-when :postgres
-  MESSAGE_BUS_CONFIG.merge!(backend_options: { host: ENV['PGHOST'], user: ENV['PGUSER'] || ENV['USER'], password: ENV['PGPASSWORD'], dbname: ENV['PGDATABASE'] || 'message_bus_test' })
-end
 puts "Running with backend: #{backend}"
 
-def wait_for(timeout_milliseconds = 2000)
-  timeout = (timeout_milliseconds + 0.0) / 1000
-  finish = Time.now + timeout
-
-  Thread.new do
-    while Time.now < finish && !yield
-      sleep(0.001)
-    end
-  end.join
-end
-
 def test_only(*backends)
   backend = MESSAGE_BUS_CONFIG[:backend]
   skip "Test doesn't apply to #{backend}" unless backends.include?(backend)