From 1ea744ab6aef6dafa55f8e393c052e78b2d8d940 Mon Sep 17 00:00:00 2001 From: Ben Langfeld Date: Wed, 5 Dec 2018 11:38:58 -0200 Subject: [PATCH 1/3] Generate backend-specific spec tasks automatically One less place to keep a list of backends. Also moves non-spec checks out of `rake spec` and into the `rake` default task, as well as renaming `spec_BACKEND` to `spec:BACKEND` (eg `rake spec:memory`) and giving tasks descriptions so they show up in `rake -T`. --- Rakefile | 39 ++++++++++++++++++--------------------- 1 file changed, 18 insertions(+), 21 deletions(-) diff --git a/Rakefile b/Rakefile index 0885c2e0..c1b61e8d 100644 --- a/Rakefile +++ b/Rakefile @@ -21,8 +21,6 @@ end Bundler.require(:default, :test) -task default: :spec - module CustomBuild def build_gem `cp assets/message-bus* vendor/assets/javascripts` @@ -36,27 +34,26 @@ module Bundler end end -run_spec = proc do |backend| - begin - ENV['MESSAGE_BUS_BACKEND'] = backend - sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/**/*_spec.rb'].to_a.join(' ')}" - ensure - ENV.delete('MESSAGE_BUS_BACKEND') - end -end - -task spec: [:spec_memory, :spec_redis, :spec_postgres, :spec_client_js, :rubocop, :test_doc] - task spec_client_js: 'jasmine:ci' -task :spec_redis do - run_spec.call('redis') +backends = Dir["lib/message_bus/backends/*.rb"].map { |file| file.match(%r{backends/(?.*).rb})[:backend] } - ["base"] + +namespace :spec do + backends.each do |backend| + desc "Run tests on the #{backend} backend" + task backend do + begin + ENV['MESSAGE_BUS_BACKEND'] = backend + sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/**/*_spec.rb'].to_a.join(' ')}" + ensure + ENV.delete('MESSAGE_BUS_BACKEND') + end + end + end end -task :spec_memory do - run_spec.call('memory') -end +desc "Run tests on all backends, plus client JS tests" +task spec: backends.map { |backend| "spec:#{backend}" } + [:spec_client_js] -task :spec_postgres do - run_spec.call('postgres') -end +desc "Run all tests, link checks and confirm documentation compiles without error" +task default: [:spec, :rubocop, :test_doc] From 79cba6d0f97fd85050aa7ac6eb27da10e95e2681 Mon Sep 17 00:00:00 2001 From: Ben Langfeld Date: Wed, 5 Dec 2018 12:08:56 -0200 Subject: [PATCH 2/3] Benchmark publication performance --- Rakefile | 12 ++++++++++- spec/helpers.rb | 19 ++++++++++++++++ spec/performance/publish.rb | 43 +++++++++++++++++++++++++++++++++++++ spec/spec_helper.rb | 21 +++--------------- 4 files changed, 76 insertions(+), 19 deletions(-) create mode 100644 spec/helpers.rb create mode 100755 spec/performance/publish.rb diff --git a/Rakefile b/Rakefile index c1b61e8d..30a3747d 100644 --- a/Rakefile +++ b/Rakefile @@ -55,5 +55,15 @@ end desc "Run tests on all backends, plus client JS tests" task spec: backends.map { |backend| "spec:#{backend}" } + [:spec_client_js] -desc "Run all tests, link checks and confirm documentation compiles without error" +desc "Run performance benchmarks on all backends" +task :performance do + begin + ENV['MESSAGE_BUS_BACKENDS'] = backends.join(",") + sh "#{FileUtils::RUBY} -e \"ARGV.each{|f| load f}\" #{Dir['spec/performance/*.rb'].to_a.join(' ')}" + ensure + ENV.delete('MESSAGE_BUS_BACKENDS') + end +end + +desc "Run all tests, link checks and confirms documentation compiles without error" task default: [:spec, :rubocop, :test_doc] diff --git a/spec/helpers.rb b/spec/helpers.rb new file mode 100644 index 00000000..c948aaae --- /dev/null +++ b/spec/helpers.rb @@ -0,0 +1,19 @@ +def wait_for(timeout_milliseconds = 2000) + timeout = (timeout_milliseconds + 0.0) / 1000 + finish = Time.now + timeout + + Thread.new do + sleep(0.001) while Time.now < finish && !yield + end.join +end + +def test_config_for_backend(backend) + config = { backend: backend } + case backend + when :redis + config[:url] = ENV['REDISURL'] + when :postgres + config[:backend_options] = { host: ENV['PGHOST'], user: ENV['PGUSER'] || ENV['USER'], password: ENV['PGPASSWORD'], dbname: ENV['PGDATABASE'] || 'message_bus_test' } + end + config +end diff --git a/spec/performance/publish.rb b/spec/performance/publish.rb new file mode 100755 index 00000000..b3511311 --- /dev/null +++ b/spec/performance/publish.rb @@ -0,0 +1,43 @@ +$LOAD_PATH << File.join(File.dirname(__FILE__), '..', '..', 'lib') +require 'logger' +require 'benchmark' +require 'message_bus' + +require_relative "../helpers" + +backends = ENV['MESSAGE_BUS_BACKENDS'].split(",").map(&:to_sym) +channel = "/foo" +iterations = 10_000 +results = [] + +puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}" + +puts +Benchmark.bm(10) do |bm| + backends.each do |backend| + messages_received = 0 + + bus = MessageBus::Instance.new + bus.configure(test_config_for_backend(backend)) + + bus.after_fork + bus.subscribe(channel) do |_message| + messages_received += 1 + end + + bm.report(backend) do + iterations.times { bus.publish(channel, "Hello world") } + wait_for(2000) { messages_received == iterations } + end + + results << "[#{backend}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%" + + bus.reset! + bus.destroy + end +end +puts + +results.each do |result| + puts result +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index bf65d319..bdec5395 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -7,29 +7,14 @@ require 'minitest/autorun' require 'minitest/spec' +require_relative "helpers" + backend = (ENV['MESSAGE_BUS_BACKEND'] || :redis).to_sym -MESSAGE_BUS_CONFIG = { backend: backend } +MESSAGE_BUS_CONFIG = test_config_for_backend(backend) require "message_bus/backends/#{backend}" PUB_SUB_CLASS = MessageBus::BACKENDS.fetch(backend) -case backend -when :redis - MESSAGE_BUS_CONFIG.merge!(url: ENV['REDISURL']) -when :postgres - MESSAGE_BUS_CONFIG.merge!(backend_options: { host: ENV['PGHOST'], user: ENV['PGUSER'] || ENV['USER'], password: ENV['PGPASSWORD'], dbname: ENV['PGDATABASE'] || 'message_bus_test' }) -end puts "Running with backend: #{backend}" -def wait_for(timeout_milliseconds = 2000) - timeout = (timeout_milliseconds + 0.0) / 1000 - finish = Time.now + timeout - - Thread.new do - while Time.now < finish && !yield - sleep(0.001) - end - end.join -end - def test_only(*backends) backend = MESSAGE_BUS_CONFIG[:backend] skip "Test doesn't apply to #{backend}" unless backends.include?(backend) From 255c50bf40fbd96de753208639e0e20fbfa193ab Mon Sep 17 00:00:00 2001 From: Ben Langfeld Date: Wed, 5 Dec 2018 16:39:39 -0200 Subject: [PATCH 3/3] Run variations on publish benchmark * Strictly publishing only, without subscribing * Publishing and subscribing with max backlog length large enough to not have to trim backlogs during the benchmark * Publishing and subscribing backlogs with approximately 10% of the capacity of the benchmark --- spec/performance/publish.rb | 91 ++++++++++++++++++++++++++++++------- 1 file changed, 75 insertions(+), 16 deletions(-) diff --git a/spec/performance/publish.rb b/spec/performance/publish.rb index b3511311..1cc19dc8 100755 --- a/spec/performance/publish.rb +++ b/spec/performance/publish.rb @@ -12,28 +12,87 @@ puts "Running publication benchmark with #{iterations} iterations on backends: #{backends.inspect}" +benchmark_publication_only = lambda do |bm, backend| + bus = MessageBus::Instance.new + bus.configure(test_config_for_backend(backend)) + + bm.report("#{backend} - publication only") do + iterations.times { bus.publish(channel, "Hello world") } + end + + bus.reset! + bus.destroy +end + +benchmark_subscription_no_trimming = lambda do |bm, backend| + test_title = "#{backend} - subscription no trimming" + + bus = MessageBus::Instance.new + bus.configure(test_config_for_backend(backend)) + + bus.reliable_pub_sub.max_backlog_size = iterations + bus.reliable_pub_sub.max_global_backlog_size = iterations + + messages_received = 0 + bus.after_fork + bus.subscribe(channel) do |_message| + messages_received += 1 + end + + bm.report(test_title) do + iterations.times { bus.publish(channel, "Hello world") } + wait_for(60000) { messages_received == iterations } + end + + results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%" + + bus.reset! + bus.destroy +end + +benchmark_subscription_with_trimming = lambda do |bm, backend| + test_title = "#{backend} - subscription with trimming" + + bus = MessageBus::Instance.new + bus.configure(test_config_for_backend(backend)) + + bus.reliable_pub_sub.max_backlog_size = (iterations / 10) + bus.reliable_pub_sub.max_global_backlog_size = (iterations / 10) + + messages_received = 0 + bus.after_fork + bus.subscribe(channel) do |_message| + messages_received += 1 + end + + bm.report(test_title) do + iterations.times { bus.publish(channel, "Hello world") } + wait_for(60000) { messages_received == iterations } + end + + results << "[#{test_title}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%" + + bus.reset! + bus.destroy +end + puts -Benchmark.bm(10) do |bm| +Benchmark.bm(60) do |bm| backends.each do |backend| - messages_received = 0 - - bus = MessageBus::Instance.new - bus.configure(test_config_for_backend(backend)) + benchmark_publication_only.call(bm, backend) + end - bus.after_fork - bus.subscribe(channel) do |_message| - messages_received += 1 - end + puts - bm.report(backend) do - iterations.times { bus.publish(channel, "Hello world") } - wait_for(2000) { messages_received == iterations } - end + backends.each do |backend| + benchmark_subscription_no_trimming.call(bm, backend) + end - results << "[#{backend}]: #{iterations} messages sent, #{messages_received} received, rate of #{(messages_received.to_f / iterations.to_f) * 100}%" + results << nil + puts - bus.reset! - bus.destroy + backends.each do |backend| + benchmark_subscription_with_trimming.call(bm, backend) end end puts