Add Transport#serialize_envelope to control envelope serialization

Because we now need to check if a serialized envelope item is oversized
(see
#1603 (comment)),
envelope serialization is better performed by Transport to have more
control to filter oversized items.

sentry-ruby/lib/sentry/envelope.rb

@@ -34,10 +34,6 @@ def add_item(headers, payload)
       @items << Item.new(headers, payload)
     end
 
-    def to_s
-      [JSON.generate(@headers), *@items.map(&:to_s)].join("\n")
-    end
-
     def item_types
       @items.map(&:type)
     end

sentry-ruby/lib/sentry/event.rb

@@ -23,6 +23,7 @@ class Event
     WRITER_ATTRIBUTES = SERIALIZEABLE_ATTRIBUTES - %i(type timestamp level)
 
     MAX_MESSAGE_SIZE_IN_BYTES = 1024 * 8
+    MAX_SERIALIZED_PAYLOAD_SIZE = 1024 * 200
 
     SKIP_INSPECTION_ATTRIBUTES = [:@modules, :@stacktrace_builder, :@send_default_pii, :@trusted_proxies, :@rack_env_whitelist]
 

sentry-ruby/lib/sentry/transport.rb

@@ -57,8 +57,42 @@ def send_envelope(envelope)
 
       return if envelope.items.empty?
 
-      log_info("[Transport] Sending envelope with items [#{envelope.item_types.join(', ')}] #{envelope.event_id} to Sentry")
-      send_data(envelope.to_s)
+      data = serialize_envelope(envelope)
+
+      if data
+        log_info("[Transport] Sending envelope with items [#{envelope.item_types.join(', ')}] #{envelope.event_id} to Sentry")
+        send_data(data)
+      end
+    end
+
+    def serialize_envelope(envelope)
+      serialized_items = envelope.items.map do |item|
+        result = item.to_s
+
+        if result.bytesize > Event::MAX_SERIALIZED_PAYLOAD_SIZE
+          item.payload.delete(:breadcrumbs)
+          result = item.to_s
+        end
+
+        if result.bytesize > Event::MAX_SERIALIZED_PAYLOAD_SIZE
+          size_breakdown = item.payload.map do |key, value|
+            "#{key}: #{JSON.generate(value).bytesize}"
+          end.join(", ")
+
+          log_debug("Envelope item [#{item.type}] is still oversized without breadcrumbs: {#{size_breakdown}}")
+          result = nil
+        end
+
+        result
+      end
+
+      serialized_items.compact!
+
+      if serialized_items.empty?
+        nil
+      else
+        [JSON.generate(envelope.headers), *serialized_items].join("\n")
+      end
     end
 
     def is_rate_limited?(item_type)

sentry-ruby/spec/sentry/transport/http_transport_spec.rb

@@ -13,7 +13,7 @@
   let(:client) { Sentry::Client.new(configuration) }
   let(:event) { client.event_from_message("foobarbaz") }
   let(:data) do
-    subject.envelope_from_event(event.to_hash).to_s
+    subject.serialize_envelope(subject.envelope_from_event(event.to_hash))
   end
 
   subject { client.transport }

sentry-ruby/spec/sentry/transport_spec.rb

@@ -18,18 +18,13 @@
 
   subject { client.transport }
 
-  describe "#envelope_from_event" do
-
-    before do
-      Sentry.init do |config|
-        config.dsn = DUMMY_DSN
-      end
-    end
-
+  describe "#serialize_envelope" do
     context "normal event" do
       let(:event) { client.event_from_exception(ZeroDivisionError.new("divided by 0")) }
+      let(:envelope) { subject.envelope_from_event(event) }
+
       it "generates correct envelope content" do
-        result = subject.envelope_from_event(event.to_hash).to_s
+        result = subject.serialize_envelope(envelope)
 
         envelope_header, item_header, item = result.split("\n")
 
@@ -51,12 +46,11 @@
       let(:transaction) do
         Sentry::Transaction.new(name: "test transaction", op: "rack.request", hub: hub)
       end
-      let(:event) do
-        client.event_from_transaction(transaction)
-      end
+      let(:event) { client.event_from_transaction(transaction) }
+      let(:envelope) { subject.envelope_from_event(event) }
 
       it "generates correct envelope content" do
-        result = subject.envelope_from_event(event.to_hash).to_s
+        result = subject.serialize_envelope(envelope)
 
         envelope_header, item_header, item = result.split("\n")
 
@@ -76,14 +70,15 @@
 
     context "client report" do
       let(:event) { client.event_from_exception(ZeroDivisionError.new("divided by 0")) }
+      let(:envelope) { subject.envelope_from_event(event) }
       before do
         5.times { subject.record_lost_event(:ratelimit_backoff, 'error') }
         3.times { subject.record_lost_event(:queue_overflow, 'transaction') }
       end
 
       it "incudes client report in envelope" do
         Timecop.travel(Time.now + 90) do
-          result = subject.envelope_from_event(event.to_hash).to_s
+          result = subject.serialize_envelope(envelope)
 
           client_report_header, client_report_payload = result.split("\n").last(2)
 
@@ -103,6 +98,151 @@
         end
       end
     end
+
+    context "oversized event" do
+      let(:event) { client.event_from_message("foo") }
+      let(:envelope) { subject.envelope_from_event(event) }
+
+      before do
+        event.breadcrumbs = Sentry::BreadcrumbBuffer.new(100)
+        100.times do |i|
+          event.breadcrumbs.record Sentry::Breadcrumb.new(category: i.to_s, message: "x" * Sentry::Event::MAX_MESSAGE_SIZE_IN_BYTES)
+        end
+        serialized_result = JSON.generate(event.to_hash)
+        expect(serialized_result.bytesize).to be > Sentry::Event::MAX_SERIALIZED_PAYLOAD_SIZE
+      end
+
+      it "removes breadcrumbs and carry on" do
+        data = subject.serialize_envelope(envelope)
+        expect(data.bytesize).to be < Sentry::Event::MAX_SERIALIZED_PAYLOAD_SIZE
+
+        expect(envelope.items.count).to eq(1)
+
+        event_item = envelope.items.first
+        expect(event_item.payload[:breadcrumbs]).to be_nil
+      end
+
+      context "if it's still oversized" do
+        before do
+          100.times do |i|
+            event.contexts["context_#{i}"] = "s" * Sentry::Event::MAX_MESSAGE_SIZE_IN_BYTES
+          end
+        end
+
+        it "rejects the item and logs attributes size breakdown" do
+          data = subject.serialize_envelope(envelope)
+          expect(data).to be_nil
+          expect(io.string).not_to match(/Sending envelope with items \[event\]/)
+          expect(io.string).to match(/tags: 2, contexts: 820791, extra: 2/)
+        end
+      end
+    end
+  end
+
+  describe "#send_envelope" do
+    context "normal event" do
+      let(:event) { client.event_from_exception(ZeroDivisionError.new("divided by 0")) }
+      let(:envelope) { subject.envelope_from_event(event) }
+
+      it "sends the event and logs the action" do
+        expect(subject).to receive(:send_data)
+
+        subject.send_envelope(envelope)
+
+        expect(io.string).to match(/Sending envelope with items \[event\]/)
+      end
+    end
+
+    context "transaction event" do
+      let(:transaction) do
+        Sentry::Transaction.new(name: "test transaction", op: "rack.request", hub: hub)
+      end
+      let(:event) { client.event_from_transaction(transaction) }
+      let(:envelope) { subject.envelope_from_event(event) }
+
+      it "sends the event and logs the action" do
+        expect(subject).to receive(:send_data)
+
+        subject.send_envelope(envelope)
+
+        expect(io.string).to match(/Sending envelope with items \[transaction\]/)
+      end
+    end
+
+    context "client report" do
+      let(:event) { client.event_from_exception(ZeroDivisionError.new("divided by 0")) }
+      let(:envelope) { subject.envelope_from_event(event) }
+      before do
+        5.times { subject.record_lost_event(:ratelimit_backoff, 'error') }
+        3.times { subject.record_lost_event(:queue_overflow, 'transaction') }
+      end
+
+      it "sends the event and logs the action" do
+        Timecop.travel(Time.now + 90) do
+          expect(subject).to receive(:send_data)
+
+          subject.send_envelope(envelope)
+
+          expect(io.string).to match(/Sending envelope with items \[event, client_report\]/)
+        end
+      end
+    end
+
+    context "oversized event" do
+      let(:event) { client.event_from_message("foo") }
+      let(:envelope) { subject.envelope_from_event(event) }
+
+      before do
+        event.breadcrumbs = Sentry::BreadcrumbBuffer.new(100)
+        100.times do |i|
+          event.breadcrumbs.record Sentry::Breadcrumb.new(category: i.to_s, message: "x" * Sentry::Event::MAX_MESSAGE_SIZE_IN_BYTES)
+        end
+        serialized_result = JSON.generate(event.to_hash)
+        expect(serialized_result.bytesize).to be > Sentry::Event::MAX_SERIALIZED_PAYLOAD_SIZE
+      end
+
+      it "sends the event and logs the action" do
+        expect(subject).to receive(:send_data)
+
+        subject.send_envelope(envelope)
+
+        expect(io.string).to match(/Sending envelope with items \[event\]/)
+      end
+
+      context "if it's still oversized" do
+        before do
+          100.times do |i|
+            event.contexts["context_#{i}"] = "s" * Sentry::Event::MAX_MESSAGE_SIZE_IN_BYTES
+          end
+        end
+
+        it "rejects the event item and doesn't send the envelope" do
+          expect(subject).not_to receive(:send_data)
+
+          subject.send_envelope(envelope)
+
+          expect(io.string).to match(/tags: 2, contexts: 820791, extra: 2/)
+          expect(io.string).not_to match(/Sending envelope with items \[event\]/)
+        end
+
+        context "with other types of items" do
+          before do
+            5.times { subject.record_lost_event(:ratelimit_backoff, 'error') }
+            3.times { subject.record_lost_event(:queue_overflow, 'transaction') }
+          end
+
+          it "excludes oversized event and sends the rest" do
+            Timecop.travel(Time.now + 90) do
+              expect(subject).to receive(:send_data)
+
+              subject.send_envelope(envelope)
+
+              expect(io.string).to match(/Sending envelope with items \[client_report\]/)
+            end
+          end
+        end
+      end
+    end
   end
 
   describe "#send_event" do