From c1ad04ac06da183aee97fcb3705d8a34ced4db1d Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 24 Aug 2016 00:15:09 -0400 Subject: [PATCH 01/19] Start adding more robust protocol tests --- src/kafka-net/Common/BigEndianBinaryReader.cs | 12 +- src/kafka-tests/Unit/ProtocolMessageTests.cs | 238 ++++++++++++++++++ 2 files changed, 249 insertions(+), 1 deletion(-) diff --git a/src/kafka-net/Common/BigEndianBinaryReader.cs b/src/kafka-net/Common/BigEndianBinaryReader.cs index 7cc0c7a7..8bdb9604 100644 --- a/src/kafka-net/Common/BigEndianBinaryReader.cs +++ b/src/kafka-net/Common/BigEndianBinaryReader.cs @@ -31,7 +31,7 @@ public BigEndianBinaryReader(IEnumerable payload) : base(new MemoryStream( } public long Length{get{return base.BaseStream.Length;}} - public long Position { get { return base.BaseStream.Position; } set { base.BaseStream.Position = 0; } } + public long Position { get { return base.BaseStream.Position; } set { base.BaseStream.Position = value; } } public bool HasData { get { return base.BaseStream.Position < base.BaseStream.Length; } } public bool Available(int dataSize) @@ -104,6 +104,16 @@ public override UInt64 ReadUInt64() return EndianAwareRead(8, BitConverter.ToUInt64); } + public override string ReadString() + { + return ReadInt16String(); + } + + public byte[] ReadBytes() + { + return ReadIntPrefixedBytes(); + } + public string ReadInt16String() { var size = ReadInt16(); diff --git a/src/kafka-tests/Unit/ProtocolMessageTests.cs b/src/kafka-tests/Unit/ProtocolMessageTests.cs index a0088765..17ac46d3 100644 --- a/src/kafka-tests/Unit/ProtocolMessageTests.cs +++ b/src/kafka-tests/Unit/ProtocolMessageTests.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.IO; using System.Linq; +using System.Security.Cryptography; using System.Text; using kafka_tests.Helpers; using KafkaNet.Common; @@ -9,10 +11,246 @@ namespace kafka_tests.Unit { + /// + /// From http://kafka.apache.org/protocol.html#protocol_types + /// The protocol is built out of the following primitive types. + /// + /// Fixed Width Primitives: + /// int8, int16, int32, int64 - Signed integers with the given precision (in bits) stored in big endian order. + /// + /// Variable Length Primitives: + /// bytes, string - These types consist of a signed integer giving a length N followed by N bytes of content. + /// A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32. + /// + /// Arrays: + /// This is a notation for handling repeated structures. These will always be encoded as an int32 size containing + /// the length N followed by N repetitions of the structure which can itself be made up of other primitive types. + /// In the BNF grammars below we will show an array of a structure foo as [foo]. + /// + /// Message formats are from http://kafka.apache.org/protocol.html#protocol_messages + /// + /// Request Header => api_key api_version correlation_id client_id + /// api_key => INT16 -- The id of the request type. + /// api_version => INT16 -- The version of the API. + /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. + /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. + /// + /// Response Header => correlation_id + /// correlation_id => INT32 -- The user-supplied value passed in with the request + /// [TestFixture] [Category("Unit")] public class ProtocolMessageTests { + /// + /// Produce Request (Version: 0,1,2) => acks timeout [topic_data] + /// acks => INT16 -- The number of nodes that should replicate the produce before returning. -1 indicates the full ISR. + /// timeout => INT32 -- The time to await a response in ms. + /// topic_data => topic [data] + /// topic => STRING + /// data => partition record_set + /// partition => INT32 + /// record_set => BYTES + /// + /// where: + /// record_set => MessageSetSize MessageSet + /// MessageSetSize => int32 + /// + /// NB. MessageSets are not preceded by an int32 like other array elements in the protocol: + /// MessageSet => [Offset MessageSize Message] + /// Offset => int64 -- This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, + /// it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, + /// each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. + /// MessageSize => int32 + /// + /// Message => Crc MagicByte Attributes Timestamp Key Value + /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. + /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. + /// Attributes => int8 -- This byte holds metadata attributes about the message. + /// The lowest 3 bits contain the compression codec used for the message. + /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) + /// All other bits should be set to 0. + /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. + /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. + /// + [Test] + public void ProduceApiRequestBytes( + [Values(0)] short version, // currently only supported version + [Values(0, 1, 2, -1)] short acks, + [Values(0, 1, 1000)] int timeoutMilliseconds, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1)] int partitionsPerTopic, // client only supports 1 + [Values(1, 5)] int totalPartitions, + [Values(1, 2, 3)] int messagesPerSet) + { + var randomizer = new Randomizer(); + var clientId = nameof(ProduceApiRequestBytes); + + var request = new ProduceRequest { + Acks = acks, + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + TimeoutMS = timeoutMilliseconds, + Payload = new List() + }; + + for (var t = 0; t < topicsPerRequest; t++) { + var payload = new Payload { + Topic = topic + t, + Partition = t % totalPartitions, + Codec = MessageCodec.CodecNone, + Messages = new List() + }; + for (var m = 0; m < messagesPerSet; m++) { + var message = new Message { + MagicNumber = 1, + Key = m > 0 ? new byte[8] : null, + Value = new byte[8*(m + 1)] + }; + if (message.Key != null) { + randomizer.NextBytes(message.Key); + } + randomizer.NextBytes(message.Value); + payload.Messages.Add(message); + } + request.Payload.Add(payload); + } + + var data = request.Encode(); + + AssertProtocolBytes(data.Buffer, + reader => { + AssertRequestHeader(request.ApiKey, version, request.CorrelationId, request.ClientId, reader); + + Assert.That(reader.ReadInt16(), Is.EqualTo(request.Acks), "acks"); + Assert.That(reader.ReadInt32(), Is.EqualTo(request.TimeoutMS), "timeout"); + + Assert.That(reader.ReadInt32(), Is.EqualTo(request.Payload.Count), "[topic_data]"); + foreach (var payload in request.Payload) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(partitionsPerTopic), "[Partition]"); + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partition), "Partition"); + + var finalPosition = reader.ReadInt32() + reader.Position; + AssertMessageSet(payload.Messages.Select(m => + new Tuple(m.Attribute, version == 2 ? DateTime.MinValue : (DateTime?) null, m.Key, m.Value)), reader); + Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSetSize"); + } + }); + } + + private static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + /// NB. MessageSets are not preceded by an int32 like other array elements in the protocol: + /// MessageSet => [Offset MessageSize Message] + /// Offset => int64 -- This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, + /// it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, + /// each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. + /// MessageSize => int32 + /// + private static void AssertMessageSet(IEnumerable> messageValues, BigEndianBinaryReader reader) + { + foreach (var value in messageValues) { + var offset = reader.ReadInt64(); + if (value.Item1 != (byte)MessageCodec.CodecNone) { + // TODO: assert offset? + } + var finalPosition = reader.ReadInt32() + reader.Position; + AssertMessage(value.Item1, value.Item2, value.Item3, value.Item4, reader); + Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSize"); + } + } + + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + /// Message => Crc MagicByte Attributes Timestamp Key Value + /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. + /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. + /// Attributes => int8 -- This byte holds metadata attributes about the message. + /// The lowest 3 bits contain the compression codec used for the message. + /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) + /// All other bits should be set to 0. + /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. + /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. + /// + private static void AssertMessage(byte attributes, DateTime? timestamp, byte[] key, byte[] value, BigEndianBinaryReader reader) + { + var crc = (uint)reader.ReadInt32(); + var positionAfterCrc = reader.Position; + Assert.That(reader.ReadByte(), Is.EqualTo((byte)1), "MagicByte"); + Assert.That(reader.ReadByte(), Is.EqualTo((byte)attributes), "Attributes"); + if (timestamp.HasValue) { + Assert.That(reader.ReadInt64(), Is.EqualTo((timestamp.Value - UnixEpoch).TotalMilliseconds), "Timestamp"); + } + Assert.That(reader.ReadBytes(), Is.EqualTo(key), "Key"); + Assert.That(reader.ReadBytes(), Is.EqualTo(value), "Value"); + + var positionAfterMessage = reader.Position; + reader.Position = positionAfterCrc; + var messageBytes = reader.ReadBytes((int) (positionAfterMessage - positionAfterCrc)); + Assert.That(Crc32Provider.Compute(messageBytes), Is.EqualTo(crc)); + reader.Position = positionAfterMessage; + } + + /// + /// From http://kafka.apache.org/protocol.html#protocol_messages + /// + /// Request Header => api_key api_version correlation_id client_id + /// api_key => INT16 -- The id of the request type. + /// api_version => INT16 -- The version of the API. + /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. + /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. + /// + private static void AssertRequestHeader(ApiKeyRequestType apiKey, short version, int correlationId, string clientId, BigEndianBinaryReader reader) + { + // Request Header + Assert.That(reader.ReadInt16(), Is.EqualTo((short) apiKey), "api_key"); + Assert.That(reader.ReadInt16(), Is.EqualTo(version), "api_version"); + Assert.That(reader.ReadInt32(), Is.EqualTo(correlationId), "correlation_id"); + Assert.That(reader.ReadString(), Is.EqualTo(clientId), "client_id"); + } + + /// + /// From http://kafka.apache.org/protocol.html#protocol_messages + /// + /// Response Header => correlation_id + /// correlation_id => INT32 -- The user-supplied value passed in with the request + /// + private void AssertResponseMessage(byte[] bytes, int correlationId, Action assertResponse) + { + AssertProtocolBytes(bytes, reader => { + // Response Header + Assert.That(reader.ReadInt32(), Is.EqualTo(correlationId)); + + // Response + assertResponse(reader); + }); + } + + /// + /// From http://kafka.apache.org/protocol.html#protocol_common + /// + /// RequestOrResponse => Size (RequestMessage | ResponseMessage) + /// Size => int32 -- The message_size field gives the size of the subsequent request or response message in bytes. + /// The client can read requests by first reading this 4 byte size as an integer N, and then reading + /// and parsing the subsequent N bytes of the request. + /// + private void AssertProtocolBytes(byte[] bytes, Action assert) + { + using (var reader = new BigEndianBinaryReader(bytes)) { + Assert.That(reader.ReadInt32(), Is.EqualTo(reader.Length - 4), "Size"); + assert(reader); + Assert.That(reader.HasData, Is.False); + } + } + [Test] [ExpectedException(typeof(FailCrcCheckException))] public void DecodeMessageShouldThrowWhenCrcFails() From 88ed3d056b8e17527c7871d562e9a736ebb9aa76 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 24 Aug 2016 00:44:14 -0400 Subject: [PATCH 02/19] cleanup approach for tests --- .../Unit/ProtocolAssertionExtensions.cs | 173 ++++++++++++++++++ src/kafka-tests/Unit/ProtocolMessageTests.cs | 118 +----------- src/kafka-tests/kafka-tests.csproj | 1 + 3 files changed, 177 insertions(+), 115 deletions(-) create mode 100644 src/kafka-tests/Unit/ProtocolAssertionExtensions.cs diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs new file mode 100644 index 00000000..14564be7 --- /dev/null +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -0,0 +1,173 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using KafkaNet; +using KafkaNet.Common; +using KafkaNet.Protocol; +using NUnit.Framework; + +namespace kafka_tests.Unit +{ + public static class ProtocolAssertionExtensions + { + private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + /// + /// MessageSet => [Offset MessageSize Message] + /// Offset => int64 -- This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, + /// it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, + /// each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. + /// MessageSize => int32 + /// + /// NB. MessageSets are not preceded by an int32 like other array elements in the protocol: + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + public static void AssertMessageSet(this BigEndianBinaryReader reader, int version, IEnumerable messages) + { + foreach (var message in messages) { + var offset = reader.ReadInt64(); + if (message.Attribute != (byte)MessageCodec.CodecNone) { + // TODO: assert offset? + } + var finalPosition = reader.ReadInt32() + reader.Position; + reader.AssertMessage(version, message); + Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSize"); + } + } + + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + /// Message => Crc MagicByte Attributes Timestamp Key Value + /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. + /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. + /// Attributes => int8 -- This byte holds metadata attributes about the message. + /// The lowest 3 bits contain the compression codec used for the message. + /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) + /// All other bits should be set to 0. + /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. + /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. + /// + public static void AssertMessage(this BigEndianBinaryReader reader, int version, Message message) + { + var crc = (uint)reader.ReadInt32(); + var positionAfterCrc = reader.Position; + Assert.That(reader.ReadByte(), Is.EqualTo(message.MagicNumber), "MagicByte"); + Assert.That(reader.ReadByte(), Is.EqualTo(message.Attribute), "Attributes"); + if (version == 2) { + Assert.That(reader.ReadInt64(), Is.EqualTo((message.Timestamp - UnixEpoch).TotalMilliseconds), "Timestamp"); + } + Assert.That(reader.ReadBytes(), Is.EqualTo(message.Key), "Key"); + Assert.That(reader.ReadBytes(), Is.EqualTo(message.Value), "Value"); + + var positionAfterMessage = reader.Position; + reader.Position = positionAfterCrc; + var crcBytes = reader.ReadBytes((int) (positionAfterMessage - positionAfterCrc)); + Assert.That(Crc32Provider.Compute(crcBytes), Is.EqualTo(crc)); + reader.Position = positionAfterMessage; + } + + /// + /// From http://kafka.apache.org/protocol.html#protocol_messages + /// + /// Request Header => api_key api_version correlation_id client_id + /// api_key => INT16 -- The id of the request type. + /// api_version => INT16 -- The version of the API. + /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. + /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. + /// + public static void AssertRequestHeader(this BigEndianBinaryReader reader, short version, IKafkaRequest request) + { + reader.AssertRequestHeader(request.ApiKey, version, request.CorrelationId, request.ClientId); + } + + /// + /// MessageSet => [Offset MessageSize Message] + /// Offset => int64 -- This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, + /// it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, + /// each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. + /// MessageSize => int32 + /// + /// NB. MessageSets are not preceded by an int32 like other array elements in the protocol: + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + public static void AssertMessageSet(this BigEndianBinaryReader reader, int version, IEnumerable> messageValues) + { + var messages = messageValues.Select( + t => + new Message { + Attribute = t.Item1, + Timestamp = t.Item2, + Key = t.Item3, + Value = t.Item4, + MagicNumber = 1 + }); + reader.AssertMessageSet(version, messages); + } + + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + /// Message => Crc MagicByte Attributes Timestamp Key Value + /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. + /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. + /// Attributes => int8 -- This byte holds metadata attributes about the message. + /// The lowest 3 bits contain the compression codec used for the message. + /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) + /// All other bits should be set to 0. + /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. + /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. + /// + public static void AssertMessage(this BigEndianBinaryReader reader, int version, byte attributes, DateTime timestamp, byte[] key, byte[] value) + { + reader.AssertMessage(version, new Message { Attribute = attributes, Timestamp = timestamp, Key = key, Value = value, MagicNumber = 1}); + } + + /// + /// From http://kafka.apache.org/protocol.html#protocol_messages + /// + /// Request Header => api_key api_version correlation_id client_id + /// api_key => INT16 -- The id of the request type. + /// api_version => INT16 -- The version of the API. + /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. + /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. + /// + public static void AssertRequestHeader(this BigEndianBinaryReader reader, ApiKeyRequestType apiKey, short version, int correlationId, string clientId) + { + Assert.That(reader.ReadInt16(), Is.EqualTo((short) apiKey), "api_key"); + Assert.That(reader.ReadInt16(), Is.EqualTo(version), "api_version"); + Assert.That(reader.ReadInt32(), Is.EqualTo(correlationId), "correlation_id"); + Assert.That(reader.ReadString(), Is.EqualTo(clientId), "client_id"); + } + + /// + /// From http://kafka.apache.org/protocol.html#protocol_messages + /// + /// Response Header => correlation_id + /// correlation_id => INT32 -- The user-supplied value passed in with the request + /// + public static void AssertResponseHeader(this BigEndianBinaryReader reader, int correlationId) + { + Assert.That(reader.ReadInt32(), Is.EqualTo(correlationId)); + } + + /// + /// From http://kafka.apache.org/protocol.html#protocol_common + /// + /// RequestOrResponse => Size (RequestMessage | ResponseMessage) + /// Size => int32 -- The message_size field gives the size of the subsequent request or response message in bytes. + /// The client can read requests by first reading this 4 byte size as an integer N, and then reading + /// and parsing the subsequent N bytes of the request. + /// + public static void AssertProtocol(this byte[] bytes, Action assertions) + { + using (var reader = new BigEndianBinaryReader(bytes)) { + Assert.That(reader.ReadInt32(), Is.EqualTo(reader.Length - 4), "Size"); + assertions(reader); + Assert.That(reader.HasData, Is.False); + } + } + } +} \ No newline at end of file diff --git a/src/kafka-tests/Unit/ProtocolMessageTests.cs b/src/kafka-tests/Unit/ProtocolMessageTests.cs index 17ac46d3..4b4e4fea 100644 --- a/src/kafka-tests/Unit/ProtocolMessageTests.cs +++ b/src/kafka-tests/Unit/ProtocolMessageTests.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Security.Cryptography; using System.Text; using kafka_tests.Helpers; using KafkaNet.Common; @@ -120,9 +119,9 @@ public void ProduceApiRequestBytes( var data = request.Encode(); - AssertProtocolBytes(data.Buffer, + data.Buffer.AssertProtocol( reader => { - AssertRequestHeader(request.ApiKey, version, request.CorrelationId, request.ClientId, reader); + reader.AssertRequestHeader(version, request); Assert.That(reader.ReadInt16(), Is.EqualTo(request.Acks), "acks"); Assert.That(reader.ReadInt32(), Is.EqualTo(request.TimeoutMS), "timeout"); @@ -134,123 +133,12 @@ public void ProduceApiRequestBytes( Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partition), "Partition"); var finalPosition = reader.ReadInt32() + reader.Position; - AssertMessageSet(payload.Messages.Select(m => - new Tuple(m.Attribute, version == 2 ? DateTime.MinValue : (DateTime?) null, m.Key, m.Value)), reader); + reader.AssertMessageSet(version, payload.Messages); Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSetSize"); } }); } - private static DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - - /// - /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets - /// - /// NB. MessageSets are not preceded by an int32 like other array elements in the protocol: - /// MessageSet => [Offset MessageSize Message] - /// Offset => int64 -- This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, - /// it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, - /// each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. - /// MessageSize => int32 - /// - private static void AssertMessageSet(IEnumerable> messageValues, BigEndianBinaryReader reader) - { - foreach (var value in messageValues) { - var offset = reader.ReadInt64(); - if (value.Item1 != (byte)MessageCodec.CodecNone) { - // TODO: assert offset? - } - var finalPosition = reader.ReadInt32() + reader.Position; - AssertMessage(value.Item1, value.Item2, value.Item3, value.Item4, reader); - Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSize"); - } - } - - /// - /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets - /// - /// Message => Crc MagicByte Attributes Timestamp Key Value - /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. - /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. - /// Attributes => int8 -- This byte holds metadata attributes about the message. - /// The lowest 3 bits contain the compression codec used for the message. - /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) - /// All other bits should be set to 0. - /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). - /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. - /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. - /// - private static void AssertMessage(byte attributes, DateTime? timestamp, byte[] key, byte[] value, BigEndianBinaryReader reader) - { - var crc = (uint)reader.ReadInt32(); - var positionAfterCrc = reader.Position; - Assert.That(reader.ReadByte(), Is.EqualTo((byte)1), "MagicByte"); - Assert.That(reader.ReadByte(), Is.EqualTo((byte)attributes), "Attributes"); - if (timestamp.HasValue) { - Assert.That(reader.ReadInt64(), Is.EqualTo((timestamp.Value - UnixEpoch).TotalMilliseconds), "Timestamp"); - } - Assert.That(reader.ReadBytes(), Is.EqualTo(key), "Key"); - Assert.That(reader.ReadBytes(), Is.EqualTo(value), "Value"); - - var positionAfterMessage = reader.Position; - reader.Position = positionAfterCrc; - var messageBytes = reader.ReadBytes((int) (positionAfterMessage - positionAfterCrc)); - Assert.That(Crc32Provider.Compute(messageBytes), Is.EqualTo(crc)); - reader.Position = positionAfterMessage; - } - - /// - /// From http://kafka.apache.org/protocol.html#protocol_messages - /// - /// Request Header => api_key api_version correlation_id client_id - /// api_key => INT16 -- The id of the request type. - /// api_version => INT16 -- The version of the API. - /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. - /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. - /// - private static void AssertRequestHeader(ApiKeyRequestType apiKey, short version, int correlationId, string clientId, BigEndianBinaryReader reader) - { - // Request Header - Assert.That(reader.ReadInt16(), Is.EqualTo((short) apiKey), "api_key"); - Assert.That(reader.ReadInt16(), Is.EqualTo(version), "api_version"); - Assert.That(reader.ReadInt32(), Is.EqualTo(correlationId), "correlation_id"); - Assert.That(reader.ReadString(), Is.EqualTo(clientId), "client_id"); - } - - /// - /// From http://kafka.apache.org/protocol.html#protocol_messages - /// - /// Response Header => correlation_id - /// correlation_id => INT32 -- The user-supplied value passed in with the request - /// - private void AssertResponseMessage(byte[] bytes, int correlationId, Action assertResponse) - { - AssertProtocolBytes(bytes, reader => { - // Response Header - Assert.That(reader.ReadInt32(), Is.EqualTo(correlationId)); - - // Response - assertResponse(reader); - }); - } - - /// - /// From http://kafka.apache.org/protocol.html#protocol_common - /// - /// RequestOrResponse => Size (RequestMessage | ResponseMessage) - /// Size => int32 -- The message_size field gives the size of the subsequent request or response message in bytes. - /// The client can read requests by first reading this 4 byte size as an integer N, and then reading - /// and parsing the subsequent N bytes of the request. - /// - private void AssertProtocolBytes(byte[] bytes, Action assert) - { - using (var reader = new BigEndianBinaryReader(bytes)) { - Assert.That(reader.ReadInt32(), Is.EqualTo(reader.Length - 4), "Size"); - assert(reader); - Assert.That(reader.HasData, Is.False); - } - } - [Test] [ExpectedException(typeof(FailCrcCheckException))] public void DecodeMessageShouldThrowWhenCrcFails() diff --git a/src/kafka-tests/kafka-tests.csproj b/src/kafka-tests/kafka-tests.csproj index 5665176b..e8d7848b 100644 --- a/src/kafka-tests/kafka-tests.csproj +++ b/src/kafka-tests/kafka-tests.csproj @@ -90,6 +90,7 @@ + From cdb8e3ba528313b9b8e148794b550f7b2dca3dcc Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 24 Aug 2016 10:38:31 -0400 Subject: [PATCH 03/19] more protocol byte tests --- src/kafka-net/Protocol/Message.cs | 5 + .../Unit/ProtocolAssertionExtensions.cs | 56 ++++++-- src/kafka-tests/Unit/ProtocolByteTests.cs | 111 +++++++++++++++ src/kafka-tests/Unit/ProtocolMessageTests.cs | 126 ------------------ src/kafka-tests/kafka-tests.csproj | 1 + 5 files changed, 164 insertions(+), 135 deletions(-) create mode 100644 src/kafka-tests/Unit/ProtocolByteTests.cs diff --git a/src/kafka-net/Protocol/Message.cs b/src/kafka-net/Protocol/Message.cs index c325b650..25381fe4 100644 --- a/src/kafka-net/Protocol/Message.cs +++ b/src/kafka-net/Protocol/Message.cs @@ -51,6 +51,11 @@ public class Message /// public byte[] Value { get; set; } + /// + /// This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// + public DateTime Timestamp { get; set; } + /// /// Construct an empty message. /// diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index 14564be7..fa103cf4 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -12,6 +12,41 @@ public static class ProtocolAssertionExtensions { private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + /// + /// ProduceRequest => RequiredAcks Timeout [TopicName [Partition MessageSetSize MessageSet]] + /// RequiredAcks => int16 -- This field indicates how many acknowledgements the servers should receive before responding to the request. + /// If it is 0 the server will not send any response (this is the only case where the server will not reply to + /// a request). If it is 1, the server will wait the data is written to the local log before sending a response. + /// If it is -1 the server will block until the message is committed by all in sync replicas before sending a response. + /// Timeout => int32 -- This provides a maximum time in milliseconds the server can await the receipt of the number of acknowledgements + /// in RequiredAcks. The timeout is not an exact limit on the request time for a few reasons: (1) it does not include + /// network latency, (2) the timer begins at the beginning of the processing of this request so if many requests are + /// queued due to server overload that wait time will not be included, (3) we will not terminate a local write so if + /// the local write time exceeds this timeout it will not be respected. To get a hard timeout of this type the client + /// should use the socket timeout. + /// TopicName => string -- The topic that data is being published to. + /// Partition => int32 -- The partition that data is being published to. + /// MessageSetSize => int32 -- The size, in bytes, of the message set that follows. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + public static void AssertProduceRequest(this BigEndianBinaryReader reader, int version, ProduceRequest request) + { + Assert.That(reader.ReadInt16(), Is.EqualTo(request.Acks), "acks"); + Assert.That(reader.ReadInt32(), Is.EqualTo(request.TimeoutMS), "timeout"); + + Assert.That(reader.ReadInt32(), Is.EqualTo(request.Payload.Count), "[topic_data]"); + foreach (var payload in request.Payload) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partition), "Partition"); + + var finalPosition = reader.ReadInt32() + reader.Position; + reader.AssertMessageSet(version, payload.Messages); + Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSetSize"); + } + } + /// /// MessageSet => [Offset MessageSize Message] /// Offset => int64 -- This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, @@ -109,6 +144,9 @@ public static void AssertMessageSet(this BigEndianBinaryReader reader, int versi /// /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets /// + /// version 0: + /// Message => Crc MagicByte Attributes Key Value + /// version 1: /// Message => Crc MagicByte Attributes Timestamp Key Value /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. @@ -116,7 +154,7 @@ public static void AssertMessageSet(this BigEndianBinaryReader reader, int versi /// The lowest 3 bits contain the compression codec used for the message. /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) /// All other bits should be set to 0. - /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// Timestamp => int64 -- This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. /// @@ -146,7 +184,7 @@ public static void AssertRequestHeader(this BigEndianBinaryReader reader, ApiKey /// From http://kafka.apache.org/protocol.html#protocol_messages /// /// Response Header => correlation_id - /// correlation_id => INT32 -- The user-supplied value passed in with the request + /// correlation_id => INT32 -- The user-supplied value passed in with the request /// public static void AssertResponseHeader(this BigEndianBinaryReader reader, int correlationId) { @@ -154,19 +192,19 @@ public static void AssertResponseHeader(this BigEndianBinaryReader reader, int c } /// - /// From http://kafka.apache.org/protocol.html#protocol_common - /// /// RequestOrResponse => Size (RequestMessage | ResponseMessage) - /// Size => int32 -- The message_size field gives the size of the subsequent request or response message in bytes. - /// The client can read requests by first reading this 4 byte size as an integer N, and then reading - /// and parsing the subsequent N bytes of the request. + /// Size => int32 -- The Size field gives the size of the subsequent request or response message in bytes. + /// The client can read requests by first reading this 4 byte size as an integer N, and + /// then reading and parsing the subsequent N bytes of the request. + /// + /// From: https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-CommonRequestandResponseStructure /// public static void AssertProtocol(this byte[] bytes, Action assertions) { using (var reader = new BigEndianBinaryReader(bytes)) { - Assert.That(reader.ReadInt32(), Is.EqualTo(reader.Length - 4), "Size"); + var finalPosition = reader.ReadInt32() + reader.Position; assertions(reader); - Assert.That(reader.HasData, Is.False); + Assert.That(reader.Position, Is.EqualTo(finalPosition), "Size"); } } } diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs new file mode 100644 index 00000000..e4e96ac6 --- /dev/null +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -0,0 +1,111 @@ +using System; +using System.Collections.Generic; +using KafkaNet.Protocol; +using NUnit.Framework; + +namespace kafka_tests.Unit +{ + /// + /// From http://kafka.apache.org/protocol.html#protocol_types + /// The protocol is built out of the following primitive types. + /// + /// Fixed Width Primitives: + /// int8, int16, int32, int64 - Signed integers with the given precision (in bits) stored in big endian order. + /// + /// Variable Length Primitives: + /// bytes, string - These types consist of a signed integer giving a length N followed by N bytes of content. + /// A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32. + /// + /// Arrays: + /// This is a notation for handling repeated structures. These will always be encoded as an int32 size containing + /// the length N followed by N repetitions of the structure which can itself be made up of other primitive types. + /// In the BNF grammars below we will show an array of a structure foo as [foo]. + /// + /// Message formats are from https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-CommonRequestandResponseStructure + /// + /// RequestOrResponse => Size (RequestMessage | ResponseMessage) + /// Size => int32 : The Size field gives the size of the subsequent request or response message in bytes. + /// The client can read requests by first reading this 4 byte size as an integer N, and + /// then reading and parsing the subsequent N bytes of the request. + /// + /// Request Header => api_key api_version correlation_id client_id + /// api_key => INT16 -- The id of the request type. + /// api_version => INT16 -- The version of the API. + /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. + /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. + /// + /// Response Header => correlation_id + /// correlation_id => INT32 -- The user-supplied value passed in with the request + /// + [TestFixture] + [Category("Unit")] + public class ProtocolByteTests + { + /// + /// Produce Request (Version: 0,1,2) => acks timeout [topic_data] + /// acks => INT16 -- The number of nodes that should replicate the produce before returning. -1 indicates the full ISR. + /// timeout => INT32 -- The time to await a response in ms. + /// topic_data => topic [data] + /// topic => STRING + /// data => partition record_set + /// partition => INT32 + /// record_set => BYTES + /// + /// where: + /// record_set => MessageSetSize MessageSet + /// MessageSetSize => int32 + /// + [Test] + public void ProduceApiRequest( + [Values(0)] short version, // currently only supported version + [Values(0, 1, 2, -1)] short acks, + [Values(0, 1, 1000)] int timeoutMilliseconds, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int totalPartitions, + [Values(1, 2, 3)] int messagesPerSet) + { + var randomizer = new Randomizer(); + var clientId = nameof(ProduceApiRequest); + + var request = new ProduceRequest { + Acks = acks, + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + TimeoutMS = timeoutMilliseconds, + Payload = new List() + }; + + for (var t = 0; t < topicsPerRequest; t++) { + var payload = new Payload { + Topic = topic + t, + Partition = t % totalPartitions, + Codec = MessageCodec.CodecNone, + Messages = new List() + }; + for (var m = 0; m < messagesPerSet; m++) { + var message = new Message { + MagicNumber = 1, + Timestamp = DateTime.UtcNow, + Key = m > 0 ? new byte[8] : null, + Value = new byte[8*(m + 1)] + }; + if (message.Key != null) { + randomizer.NextBytes(message.Key); + } + randomizer.NextBytes(message.Value); + payload.Messages.Add(message); + } + request.Payload.Add(payload); + } + + var data = request.Encode(); + + data.Buffer.AssertProtocol( + reader => { + reader.AssertRequestHeader(version, request); + reader.AssertProduceRequest(version, request); + }); + } + } +} \ No newline at end of file diff --git a/src/kafka-tests/Unit/ProtocolMessageTests.cs b/src/kafka-tests/Unit/ProtocolMessageTests.cs index 4b4e4fea..a0088765 100644 --- a/src/kafka-tests/Unit/ProtocolMessageTests.cs +++ b/src/kafka-tests/Unit/ProtocolMessageTests.cs @@ -1,5 +1,4 @@ using System; -using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; @@ -10,135 +9,10 @@ namespace kafka_tests.Unit { - /// - /// From http://kafka.apache.org/protocol.html#protocol_types - /// The protocol is built out of the following primitive types. - /// - /// Fixed Width Primitives: - /// int8, int16, int32, int64 - Signed integers with the given precision (in bits) stored in big endian order. - /// - /// Variable Length Primitives: - /// bytes, string - These types consist of a signed integer giving a length N followed by N bytes of content. - /// A length of -1 indicates null. string uses an int16 for its size, and bytes uses an int32. - /// - /// Arrays: - /// This is a notation for handling repeated structures. These will always be encoded as an int32 size containing - /// the length N followed by N repetitions of the structure which can itself be made up of other primitive types. - /// In the BNF grammars below we will show an array of a structure foo as [foo]. - /// - /// Message formats are from http://kafka.apache.org/protocol.html#protocol_messages - /// - /// Request Header => api_key api_version correlation_id client_id - /// api_key => INT16 -- The id of the request type. - /// api_version => INT16 -- The version of the API. - /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. - /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. - /// - /// Response Header => correlation_id - /// correlation_id => INT32 -- The user-supplied value passed in with the request - /// [TestFixture] [Category("Unit")] public class ProtocolMessageTests { - /// - /// Produce Request (Version: 0,1,2) => acks timeout [topic_data] - /// acks => INT16 -- The number of nodes that should replicate the produce before returning. -1 indicates the full ISR. - /// timeout => INT32 -- The time to await a response in ms. - /// topic_data => topic [data] - /// topic => STRING - /// data => partition record_set - /// partition => INT32 - /// record_set => BYTES - /// - /// where: - /// record_set => MessageSetSize MessageSet - /// MessageSetSize => int32 - /// - /// NB. MessageSets are not preceded by an int32 like other array elements in the protocol: - /// MessageSet => [Offset MessageSize Message] - /// Offset => int64 -- This is the offset used in kafka as the log sequence number. When the producer is sending non compressed messages, - /// it can set the offsets to anything. When the producer is sending compressed messages, to avoid server side recompression, - /// each compressed message should have offset starting from 0 and increasing by one for each inner message in the compressed message. - /// MessageSize => int32 - /// - /// Message => Crc MagicByte Attributes Timestamp Key Value - /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. - /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. - /// Attributes => int8 -- This byte holds metadata attributes about the message. - /// The lowest 3 bits contain the compression codec used for the message. - /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) - /// All other bits should be set to 0. - /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). - /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. - /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. - /// - [Test] - public void ProduceApiRequestBytes( - [Values(0)] short version, // currently only supported version - [Values(0, 1, 2, -1)] short acks, - [Values(0, 1, 1000)] int timeoutMilliseconds, - [Values("test", "a really long name, with spaces and punctuation!")] string topic, - [Values(1, 10)] int topicsPerRequest, - [Values(1)] int partitionsPerTopic, // client only supports 1 - [Values(1, 5)] int totalPartitions, - [Values(1, 2, 3)] int messagesPerSet) - { - var randomizer = new Randomizer(); - var clientId = nameof(ProduceApiRequestBytes); - - var request = new ProduceRequest { - Acks = acks, - ClientId = clientId, - CorrelationId = clientId.GetHashCode(), - TimeoutMS = timeoutMilliseconds, - Payload = new List() - }; - - for (var t = 0; t < topicsPerRequest; t++) { - var payload = new Payload { - Topic = topic + t, - Partition = t % totalPartitions, - Codec = MessageCodec.CodecNone, - Messages = new List() - }; - for (var m = 0; m < messagesPerSet; m++) { - var message = new Message { - MagicNumber = 1, - Key = m > 0 ? new byte[8] : null, - Value = new byte[8*(m + 1)] - }; - if (message.Key != null) { - randomizer.NextBytes(message.Key); - } - randomizer.NextBytes(message.Value); - payload.Messages.Add(message); - } - request.Payload.Add(payload); - } - - var data = request.Encode(); - - data.Buffer.AssertProtocol( - reader => { - reader.AssertRequestHeader(version, request); - - Assert.That(reader.ReadInt16(), Is.EqualTo(request.Acks), "acks"); - Assert.That(reader.ReadInt32(), Is.EqualTo(request.TimeoutMS), "timeout"); - - Assert.That(reader.ReadInt32(), Is.EqualTo(request.Payload.Count), "[topic_data]"); - foreach (var payload in request.Payload) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); - Assert.That(reader.ReadInt32(), Is.EqualTo(partitionsPerTopic), "[Partition]"); - Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partition), "Partition"); - - var finalPosition = reader.ReadInt32() + reader.Position; - reader.AssertMessageSet(version, payload.Messages); - Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSetSize"); - } - }); - } - [Test] [ExpectedException(typeof(FailCrcCheckException))] public void DecodeMessageShouldThrowWhenCrcFails() diff --git a/src/kafka-tests/kafka-tests.csproj b/src/kafka-tests/kafka-tests.csproj index e8d7848b..cf701291 100644 --- a/src/kafka-tests/kafka-tests.csproj +++ b/src/kafka-tests/kafka-tests.csproj @@ -91,6 +91,7 @@ + From 4c5ca9ea16a14131613f3b69e2082032613b5724 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 24 Aug 2016 11:56:32 -0400 Subject: [PATCH 04/19] Add version 1 and 2 support for ProduceApiRequest --- src/kafka-net/Interfaces/IKafkaRequest.cs | 4 +++ src/kafka-net/Protocol/BaseRequest.cs | 8 +++-- src/kafka-net/Protocol/Message.cs | 34 ++++++++++++++----- src/kafka-net/Protocol/ProduceRequest.cs | 14 ++++---- .../Unit/ProtocolAssertionExtensions.cs | 2 +- src/kafka-tests/Unit/ProtocolByteTests.cs | 5 +-- 6 files changed, 46 insertions(+), 21 deletions(-) diff --git a/src/kafka-net/Interfaces/IKafkaRequest.cs b/src/kafka-net/Interfaces/IKafkaRequest.cs index cfec5deb..b616cb30 100644 --- a/src/kafka-net/Interfaces/IKafkaRequest.cs +++ b/src/kafka-net/Interfaces/IKafkaRequest.cs @@ -27,6 +27,10 @@ public interface IKafkaRequest /// ApiKeyRequestType ApiKey { get; } /// + /// This is a numeric version number for the api request. It allows the server to properly interpret the request as the protocol evolves. Responses will always be in the format corresponding to the request version. + /// + short ApiVersion { get; set; } + /// /// Encode this request into the Kafka wire protocol. /// /// Byte[] representing the binary wire protocol of this request. diff --git a/src/kafka-net/Protocol/BaseRequest.cs b/src/kafka-net/Protocol/BaseRequest.cs index 7ffde936..7332c0af 100644 --- a/src/kafka-net/Protocol/BaseRequest.cs +++ b/src/kafka-net/Protocol/BaseRequest.cs @@ -14,7 +14,6 @@ public abstract class BaseRequest /// https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol /// protected const int ReplicaId = -1; - protected const Int16 ApiVersion = 0; private string _clientId = "Kafka-Net"; private int _correlationId = 1; @@ -29,6 +28,11 @@ public abstract class BaseRequest /// public int CorrelationId { get { return _correlationId; } set { _correlationId = value; } } + /// + /// This is a numeric version number for the api request. It allows the server to properly interpret the request as the protocol evolves. Responses will always be in the format corresponding to the request version. + /// + public short ApiVersion { get; set; } = 0; + /// /// Flag which tells the broker call to expect a response for this request. /// @@ -43,7 +47,7 @@ public static KafkaMessagePacker EncodeHeader(IKafkaRequest request) { return new KafkaMessagePacker() .Pack(((Int16)request.ApiKey)) - .Pack(ApiVersion) + .Pack(request.ApiVersion) .Pack(request.CorrelationId) .Pack(request.ClientId, StringPrefixEncoding.Int16); } diff --git a/src/kafka-net/Protocol/Message.cs b/src/kafka-net/Protocol/Message.cs index 25381fe4..ab4497a8 100644 --- a/src/kafka-net/Protocol/Message.cs +++ b/src/kafka-net/Protocol/Message.cs @@ -40,6 +40,10 @@ public class Message public byte MagicNumber { get; set; } /// /// Attribute value outside message body used for added codec/compression info. + /// + /// The lowest 3 bits contain the compression codec used for the message. + /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) + /// All other bits should be set to 0. /// public byte Attribute { get; set; } /// @@ -50,7 +54,6 @@ public class Message /// The message body contents. Can contain compress message set. /// public byte[] Value { get; set; } - /// /// This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). /// @@ -77,15 +80,16 @@ public Message(string value, string key = null) /// Encodes a collection of messages into one byte[]. Encoded in order of list. /// /// The collection of messages to encode together. + /// Message version /// Encoded byte[] representing the collection of messages. - public static byte[] EncodeMessageSet(IEnumerable messages) + public static byte[] EncodeMessageSet(IEnumerable messages, short version = 0) { using (var stream = new KafkaMessagePacker()) { foreach (var message in messages) { stream.Pack(InitialMessageOffset) - .Pack(EncodeMessage(message)); + .Pack(EncodeMessage(message, version)); } return stream.PayloadNoLength(); @@ -126,24 +130,36 @@ public static IEnumerable DecodeMessageSet(byte[] messageSet) } } + private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + private long TimestampMilliseconds + { + get { return Timestamp > UnixEpoch ? (long)(Timestamp - UnixEpoch).TotalMilliseconds : 0L; } + set { Timestamp = UnixEpoch.AddMilliseconds(value); } + } + /// /// Encodes a message object to byte[] /// /// Message data to encode. + /// Message version /// Encoded byte[] representation of the message object. /// /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// - public static byte[] EncodeMessage(Message message) + public static byte[] EncodeMessage(Message message, short version = 0) { using(var stream = new KafkaMessagePacker()) { - return stream.Pack(message.MagicNumber) - .Pack(message.Attribute) - .Pack(message.Key) - .Pack(message.Value) - .CrcPayload(); + stream.Pack(message.MagicNumber) + .Pack(message.Attribute); + if (version > 1) { + stream.Pack(message.TimestampMilliseconds); + } + return stream.Pack(message.Key) + .Pack(message.Value) + .CrcPayload(); } } diff --git a/src/kafka-net/Protocol/ProduceRequest.cs b/src/kafka-net/Protocol/ProduceRequest.cs index 63a94a22..4c5306ca 100644 --- a/src/kafka-net/Protocol/ProduceRequest.cs +++ b/src/kafka-net/Protocol/ProduceRequest.cs @@ -42,7 +42,7 @@ public IEnumerable Decode(byte[] payload) } #region Protocol... - private KafkaDataPayload EncodeProduceRequest(ProduceRequest request) + private static KafkaDataPayload EncodeProduceRequest(ProduceRequest request) { int totalCompressedBytes = 0; if (request.Payload == null) request.Payload = new List(); @@ -72,12 +72,12 @@ private KafkaDataPayload EncodeProduceRequest(ProduceRequest request) { case MessageCodec.CodecNone: - message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages))); + message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages), request.ApiVersion)); break; case MessageCodec.CodecGzip: - var compressedBytes = CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages)); + var compressedBytes = CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages), request.ApiVersion); Interlocked.Add(ref totalCompressedBytes, compressedBytes.CompressedAmount); - message.Pack(Message.EncodeMessageSet(new[] { compressedBytes.CompressedMessage })); + message.Pack(Message.EncodeMessageSet(new[] { compressedBytes.CompressedMessage }, request.ApiVersion)); break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec)); @@ -95,9 +95,9 @@ private KafkaDataPayload EncodeProduceRequest(ProduceRequest request) } } - private CompressedMessageResult CreateGzipCompressedMessage(IEnumerable messages) + private static CompressedMessageResult CreateGzipCompressedMessage(IEnumerable messages, short version) { - var messageSet = Message.EncodeMessageSet(messages); + var messageSet = Message.EncodeMessageSet(messages, version); var gZipBytes = Compression.Zip(messageSet); @@ -114,7 +114,7 @@ private CompressedMessageResult CreateGzipCompressedMessage(IEnumerable }; } - private IEnumerable DecodeProduceResponse(byte[] data) + private static IEnumerable DecodeProduceResponse(byte[] data) { using (var stream = new BigEndianBinaryReader(data)) { diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index fa103cf4..fee18add 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -91,7 +91,7 @@ public static void AssertMessage(this BigEndianBinaryReader reader, int version, Assert.That(reader.ReadByte(), Is.EqualTo(message.MagicNumber), "MagicByte"); Assert.That(reader.ReadByte(), Is.EqualTo(message.Attribute), "Attributes"); if (version == 2) { - Assert.That(reader.ReadInt64(), Is.EqualTo((message.Timestamp - UnixEpoch).TotalMilliseconds), "Timestamp"); + Assert.That(reader.ReadInt64(), Is.EqualTo((long)(message.Timestamp - UnixEpoch).TotalMilliseconds), "Timestamp"); } Assert.That(reader.ReadBytes(), Is.EqualTo(message.Key), "Key"); Assert.That(reader.ReadBytes(), Is.EqualTo(message.Value), "Value"); diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index e4e96ac6..72142318 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -57,7 +57,7 @@ public class ProtocolByteTests /// [Test] public void ProduceApiRequest( - [Values(0)] short version, // currently only supported version + [Values(0, 1, 2)] short version, [Values(0, 1, 2, -1)] short acks, [Values(0, 1, 1000)] int timeoutMilliseconds, [Values("test", "a really long name, with spaces and punctuation!")] string topic, @@ -73,7 +73,8 @@ public void ProduceApiRequest( ClientId = clientId, CorrelationId = clientId.GetHashCode(), TimeoutMS = timeoutMilliseconds, - Payload = new List() + Payload = new List(), + ApiVersion = version }; for (var t = 0; t < topicsPerRequest; t++) { From 6c56381aa500b1763c97d68f6c39c90480691820 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 24 Aug 2016 15:31:39 -0400 Subject: [PATCH 05/19] add tests and support for ProduceResponse version 1 and 2 --- src/kafka-net/Common/BigEndianBinaryWriter.cs | 2 +- src/kafka-net/Common/Extensions.cs | 12 ++ src/kafka-net/Protocol/Message.cs | 10 +- src/kafka-net/Protocol/ProduceRequest.cs | 18 ++- .../Unit/ProtocolAssertionExtensions.cs | 88 +++++++++++-- src/kafka-tests/Unit/ProtocolByteTests.cs | 122 ++++++++++++++++-- 6 files changed, 218 insertions(+), 34 deletions(-) diff --git a/src/kafka-net/Common/BigEndianBinaryWriter.cs b/src/kafka-net/Common/BigEndianBinaryWriter.cs index 1a5a4664..edc2dab2 100644 --- a/src/kafka-net/Common/BigEndianBinaryWriter.cs +++ b/src/kafka-net/Common/BigEndianBinaryWriter.cs @@ -101,7 +101,7 @@ public override void Write(UInt64 value) public override void Write(string value) { - throw new NotSupportedException("Kafka requires specific string length prefix encoding."); + Write(value, StringPrefixEncoding.Int16); } public void Write(byte[] value, StringPrefixEncoding encoding) diff --git a/src/kafka-net/Common/Extensions.cs b/src/kafka-net/Common/Extensions.cs index 096275c2..3c4ddd17 100644 --- a/src/kafka-net/Common/Extensions.cs +++ b/src/kafka-net/Common/Extensions.cs @@ -234,5 +234,17 @@ public static Exception ExtractException(this Task task) return new ApplicationException("Unknown exception occured."); } + + private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + + public static long ToUnixEpochMilliseconds(this DateTime pointInTime) + { + return pointInTime > UnixEpoch ? (long)(pointInTime - UnixEpoch).TotalMilliseconds : 0L; + } + + public static DateTime FromUnixEpochMilliseconds(this long milliseconds) + { + return UnixEpoch.AddMilliseconds(milliseconds); + } } } diff --git a/src/kafka-net/Protocol/Message.cs b/src/kafka-net/Protocol/Message.cs index ab4497a8..8991764c 100644 --- a/src/kafka-net/Protocol/Message.cs +++ b/src/kafka-net/Protocol/Message.cs @@ -130,14 +130,6 @@ public static IEnumerable DecodeMessageSet(byte[] messageSet) } } - private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); - - private long TimestampMilliseconds - { - get { return Timestamp > UnixEpoch ? (long)(Timestamp - UnixEpoch).TotalMilliseconds : 0L; } - set { Timestamp = UnixEpoch.AddMilliseconds(value); } - } - /// /// Encodes a message object to byte[] /// @@ -155,7 +147,7 @@ public static byte[] EncodeMessage(Message message, short version = 0) stream.Pack(message.MagicNumber) .Pack(message.Attribute); if (version > 1) { - stream.Pack(message.TimestampMilliseconds); + stream.Pack(message.Timestamp.ToUnixEpochMilliseconds()); } return stream.Pack(message.Key) .Pack(message.Value) diff --git a/src/kafka-net/Protocol/ProduceRequest.cs b/src/kafka-net/Protocol/ProduceRequest.cs index 4c5306ca..19ebe4db 100644 --- a/src/kafka-net/Protocol/ProduceRequest.cs +++ b/src/kafka-net/Protocol/ProduceRequest.cs @@ -38,7 +38,7 @@ public KafkaDataPayload Encode() public IEnumerable Decode(byte[] payload) { - return DecodeProduceResponse(payload); + return DecodeProduceResponse(ApiVersion, payload); } #region Protocol... @@ -114,7 +114,7 @@ private static CompressedMessageResult CreateGzipCompressedMessage(IEnumerable DecodeProduceResponse(byte[] data) + private static IEnumerable DecodeProduceResponse(int version, byte[] data) { using (var stream = new BigEndianBinaryReader(data)) { @@ -136,6 +136,13 @@ private static IEnumerable DecodeProduceResponse(byte[] data) Offset = stream.ReadInt64() }; + if (version >= 2) { + var milliseconds = stream.ReadInt64(); + if (milliseconds >= 0) { + response.Timestamp = milliseconds.FromUnixEpochMilliseconds(); + } + } + yield return response; } } @@ -168,6 +175,13 @@ public class ProduceResponse /// The offset number to commit as completed. /// public long Offset { get; set; } + /// + /// If LogAppendTime is used for the topic, this is the timestamp assigned by the broker to the message set. + /// All the messages in the message set have the same timestamp. + /// If CreateTime is used, this field is always -1. The producer can assume the timestamp of the messages in the + /// produce request has been accepted by the broker if there is no error code returned. + /// + public DateTime? Timestamp { get; set; } public override bool Equals(object obj) { diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index fee18add..0780e0e2 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.IO; using System.Linq; using KafkaNet; using KafkaNet.Common; @@ -12,6 +13,50 @@ public static class ProtocolAssertionExtensions { private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + /// + /// ProduceResponse => [TopicName [Partition ErrorCode Offset *Timestamp]] *ThrottleTime + /// *ThrottleTime is only version 1 (0.9.0) and above + /// *Timestamp is only version 2 (0.10.0) and above + /// TopicName => string -- The topic this response entry corresponds to. + /// Partition => int32 -- The partition this response entry corresponds to. + /// ErrorCode => int16 -- The error from this partition, if any. Errors are given on a per-partition basis because a given partition may be + /// unavailable or maintained on a different host, while others may have successfully accepted the produce request. + /// Offset => int64 -- The offset assigned to the first message in the message set appended to this partition. + /// Timestamp => int64 -- If LogAppendTime is used for the topic, this is the timestamp assigned by the broker to the message set. + /// All the messages in the message set have the same timestamp. + /// If CreateTime is used, this field is always -1. The producer can assume the timestamp of the messages in the + /// produce request has been accepted by the broker if there is no error code returned. + /// Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// ThrottleTime => int32 -- Duration in milliseconds for which the request was throttled due to quota violation. + /// (Zero if the request did not violate any quota). + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + public static void AssertProduceResponse(this BigEndianBinaryReader reader, int version, int throttleTime, IEnumerable response) + { + var responses = response.ToList(); + Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); + foreach (var payload in responses) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); + Assert.That(reader.ReadInt16(), Is.EqualTo(payload.Error), "Error"); + Assert.That(reader.ReadInt64(), Is.EqualTo(payload.Offset), "Offset"); + if (version >= 2) { + var timestamp = reader.ReadInt64(); + if (timestamp == -1L) { + Assert.That(payload.Timestamp, Is.Null, "Timestamp"); + } else { + Assert.That(payload.Timestamp, Is.Not.Null, "Timestamp"); + Assert.That(payload.Timestamp.Value, Is.EqualTo(UnixEpoch.AddMilliseconds(timestamp)), "Timestamp"); + } + } + } + if (version >= 1) { + Assert.That(reader.ReadInt32(), Is.EqualTo(throttleTime), "ThrottleTime"); + } + } + /// /// ProduceRequest => RequiredAcks Timeout [TopicName [Partition MessageSetSize MessageSet]] /// RequiredAcks => int16 -- This field indicates how many acknowledgements the servers should receive before responding to the request. @@ -30,7 +75,7 @@ public static class ProtocolAssertionExtensions /// /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets /// - public static void AssertProduceRequest(this BigEndianBinaryReader reader, int version, ProduceRequest request) + public static void AssertProduceRequest(this BigEndianBinaryReader reader, ProduceRequest request) { Assert.That(reader.ReadInt16(), Is.EqualTo(request.Acks), "acks"); Assert.That(reader.ReadInt32(), Is.EqualTo(request.TimeoutMS), "timeout"); @@ -42,8 +87,8 @@ public static void AssertProduceRequest(this BigEndianBinaryReader reader, int v Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partition), "Partition"); var finalPosition = reader.ReadInt32() + reader.Position; - reader.AssertMessageSet(version, payload.Messages); - Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSetSize"); + reader.AssertMessageSet(request.ApiVersion, payload.Messages); + Assert.That(reader.Position, Is.EqualTo(finalPosition), $"MessageSetSize was {finalPosition - 4} but ended in a different spot."); } } @@ -66,21 +111,22 @@ public static void AssertMessageSet(this BigEndianBinaryReader reader, int versi } var finalPosition = reader.ReadInt32() + reader.Position; reader.AssertMessage(version, message); - Assert.That(reader.Position, Is.EqualTo(finalPosition), "MessageSize"); + Assert.That(reader.Position, Is.EqualTo(finalPosition), $"MessageSize was {finalPosition - 4} but ended in a different spot."); } } /// /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets /// - /// Message => Crc MagicByte Attributes Timestamp Key Value + /// Message => Crc MagicByte Attributes *Timestamp Key Value + /// *Timestamp is only version 2 (0.10) and above /// Crc => int32 -- The CRC is the CRC32 of the remainder of the message bytes. This is used to check the integrity of the message on the broker and consumer. /// MagicByte => int8 -- This is a version id used to allow backwards compatible evolution of the message binary format. The current value is 1. /// Attributes => int8 -- This byte holds metadata attributes about the message. /// The lowest 3 bits contain the compression codec used for the message. /// The fourth lowest bit represents the timestamp type. 0 stands for CreateTime and 1 stands for LogAppendTime. The producer should always set this bit to 0. (since 0.10.0) /// All other bits should be set to 0. - /// Timestamp => int64 -- ONLY version 1! This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// Timestamp => int64 -- This is the timestamp of the message. The timestamp type is indicated in the attributes. Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). /// Key => bytes -- The key is an optional message key that was used for partition assignment. The key can be null. /// Value => bytes -- The value is the actual message contents as an opaque byte array. Kafka supports recursive messages in which case this may itself contain a message set. The message can be null. /// @@ -112,9 +158,9 @@ public static void AssertMessage(this BigEndianBinaryReader reader, int version, /// correlation_id => INT32 -- A user-supplied integer value that will be passed back with the response. /// client_id => NULLABLE_STRING -- A user specified identifier for the client making the request. /// - public static void AssertRequestHeader(this BigEndianBinaryReader reader, short version, IKafkaRequest request) + public static void AssertRequestHeader(this BigEndianBinaryReader reader, IKafkaRequest request) { - reader.AssertRequestHeader(request.ApiKey, version, request.CorrelationId, request.ClientId); + reader.AssertRequestHeader(request.ApiKey, request.ApiVersion, request.CorrelationId, request.ClientId); } /// @@ -204,8 +250,32 @@ public static void AssertProtocol(this byte[] bytes, Action + /// From http://kafka.apache.org/protocol.html#protocol_messages + /// + /// Response Header => correlation_id + /// correlation_id => INT32 -- The user-supplied value passed in with the request + /// + public static void WriteResponseHeader(this BigEndianBinaryWriter writer, int correlationId) + { + writer.Write(correlationId); + } + } } \ No newline at end of file diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 72142318..0cadcaf2 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -1,5 +1,7 @@ using System; using System.Collections.Generic; +using System.IO; +using KafkaNet.Common; using KafkaNet.Protocol; using NUnit.Framework; @@ -42,18 +44,22 @@ namespace kafka_tests.Unit public class ProtocolByteTests { /// - /// Produce Request (Version: 0,1,2) => acks timeout [topic_data] - /// acks => INT16 -- The number of nodes that should replicate the produce before returning. -1 indicates the full ISR. - /// timeout => INT32 -- The time to await a response in ms. - /// topic_data => topic [data] - /// topic => STRING - /// data => partition record_set - /// partition => INT32 - /// record_set => BYTES + /// ProduceRequest => RequiredAcks Timeout [TopicName [Partition MessageSetSize MessageSet]] + /// RequiredAcks => int16 -- This field indicates how many acknowledgements the servers should receive before responding to the request. + /// If it is 0 the server will not send any response (this is the only case where the server will not reply to + /// a request). If it is 1, the server will wait the data is written to the local log before sending a response. + /// If it is -1 the server will block until the message is committed by all in sync replicas before sending a response. + /// Timeout => int32 -- This provides a maximum time in milliseconds the server can await the receipt of the number of acknowledgements + /// in RequiredAcks. The timeout is not an exact limit on the request time for a few reasons: (1) it does not include + /// network latency, (2) the timer begins at the beginning of the processing of this request so if many requests are + /// queued due to server overload that wait time will not be included, (3) we will not terminate a local write so if + /// the local write time exceeds this timeout it will not be respected. To get a hard timeout of this type the client + /// should use the socket timeout. + /// TopicName => string -- The topic that data is being published to. + /// Partition => int32 -- The partition that data is being published to. + /// MessageSetSize => int32 -- The size, in bytes, of the message set that follows. /// - /// where: - /// record_set => MessageSetSize MessageSet - /// MessageSetSize => int32 + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets /// [Test] public void ProduceApiRequest( @@ -104,9 +110,99 @@ public void ProduceApiRequest( data.Buffer.AssertProtocol( reader => { - reader.AssertRequestHeader(version, request); - reader.AssertProduceRequest(version, request); + reader.AssertRequestHeader(request); + reader.AssertProduceRequest(request); }); } + + /// + /// ProduceResponse => [TopicName [Partition ErrorCode Offset *Timestamp]] *ThrottleTime + /// *ThrottleTime is only version 1 (0.9.0) and above + /// *Timestamp is only version 2 (0.10.0) and above + /// TopicName => string -- The topic this response entry corresponds to. + /// Partition => int32 -- The partition this response entry corresponds to. + /// ErrorCode => int16 -- The error from this partition, if any. Errors are given on a per-partition basis because a given partition may be + /// unavailable or maintained on a different host, while others may have successfully accepted the produce request. + /// Offset => int64 -- The offset assigned to the first message in the message set appended to this partition. + /// Timestamp => int64 -- If LogAppendTime is used for the topic, this is the timestamp assigned by the broker to the message set. + /// All the messages in the message set have the same timestamp. + /// If CreateTime is used, this field is always -1. The producer can assume the timestamp of the messages in the + /// produce request has been accepted by the broker if there is no error code returned. + /// Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). + /// ThrottleTime => int32 -- Duration in milliseconds for which the request was throttled due to quota violation. + /// (Zero if the request did not violate any quota). + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets + /// + [Test] + public void InterpretApiResponse( + [Values(0, 1, 2)] short version, + [Values(-1, 0, 123456, 10000000)] long timestampMilliseconds, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int totalPartitions, + [Values( + ErrorResponseCode.NoError, + ErrorResponseCode.InvalidMessage, + ErrorResponseCode.NotCoordinatorForConsumerCode + )] ErrorResponseCode errorCode, + [Values(0, 1234, 100000)] int throttleTime) + { + var randomizer = new Randomizer(); + var clientId = nameof(InterpretApiResponse); + var correlationId = clientId.GetHashCode(); + + byte[] data = null; + using (var stream = new MemoryStream()) { + var writer = new BigEndianBinaryWriter(stream); + writer.WriteResponseHeader(correlationId); + + writer.Write(topicsPerRequest); + for (var t = 0; t < topicsPerRequest; t++) { + writer.Write(topic + t); + writer.Write(1); // partitionsPerTopic + writer.Write(t % totalPartitions); + writer.Write((short)errorCode); + writer.Write((long)randomizer.Next()); + if (version >= 2) { + writer.Write(timestampMilliseconds); + } + } + if (version >= 1) { + writer.Write(throttleTime); + } + data = new byte[stream.Position]; + Buffer.BlockCopy(stream.GetBuffer(), 0, data, 0, data.Length); + } + + var request = new ProduceRequest { ApiVersion = version }; + var responses = request.Decode(data); // doesn't include the size in the decode -- the framework deals with it, I'd assume + data.PrefixWithInt32Length().AssertProtocol( + reader => + { + reader.AssertResponseHeader(correlationId); + reader.AssertProduceResponse(version, throttleTime, responses); + }); + } + + public static IEnumerable ErrorResponseCodes => new[] { + ErrorResponseCode.NoError, + ErrorResponseCode.Unknown, + ErrorResponseCode.OffsetOutOfRange, + ErrorResponseCode.InvalidMessage, + ErrorResponseCode.UnknownTopicOrPartition, + ErrorResponseCode.InvalidMessageSize, + ErrorResponseCode.LeaderNotAvailable, + ErrorResponseCode.NotLeaderForPartition, + ErrorResponseCode.RequestTimedOut, + ErrorResponseCode.BrokerNotAvailable, + ErrorResponseCode.ReplicaNotAvailable, + ErrorResponseCode.MessageSizeTooLarge, + //ErrorResponseCode.StaleControllerEpochCode, + ErrorResponseCode.OffsetMetadataTooLargeCode, + ErrorResponseCode.OffsetsLoadInProgressCode, + ErrorResponseCode.ConsumerCoordinatorNotAvailableCode, + ErrorResponseCode.NotCoordinatorForConsumerCode + }; } } \ No newline at end of file From 3e707700c79eb8478f073b8bb40d4e58d916afe1 Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 24 Aug 2016 16:17:27 -0400 Subject: [PATCH 06/19] add tests for FetchRequest --- .../Unit/ProtocolAssertionExtensions.cs | 42 ++++++++++++- src/kafka-tests/Unit/ProtocolByteTests.cs | 60 +++++++++++++++++++ 2 files changed, 99 insertions(+), 3 deletions(-) diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index 0780e0e2..4ac86e28 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,7 +11,43 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { - private static readonly DateTime UnixEpoch = new DateTime(1970, 1, 1, 0, 0, 0, DateTimeKind.Utc); + /// + /// FetchRequest => ReplicaId MaxWaitTime MinBytes [TopicName [Partition FetchOffset MaxBytes]] + /// ReplicaId => int32 -- The replica id indicates the node id of the replica initiating this request. Normal client consumers should always + /// specify this as -1 as they have no node id. Other brokers set this to be their own node id. The value -2 is accepted + /// to allow a non-broker to issue fetch requests as if it were a replica broker for debugging purposes. + /// MaxWaitTime => int32 -- The max wait time is the maximum amount of time in milliseconds to block waiting if insufficient data is available + /// at the time the request is issued. + /// MinBytes => int32 -- This is the minimum number of bytes of messages that must be available to give a response. If the client sets this + /// to 0 the server will always respond immediately, however if there is no new data since their last request they will + /// just get back empty message sets. If this is set to 1, the server will respond as soon as at least one partition has + /// at least 1 byte of data or the specified timeout occurs. By setting higher values in combination with the timeout the + /// consumer can tune for throughput and trade a little additional latency for reading only large chunks of data (e.g. + /// setting MaxWaitTime to 100 ms and setting MinBytes to 64k would allow the server to wait up to 100ms to try to accumulate + /// 64k of data before responding). + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition the fetch is for. + /// FetchOffset => int64 -- The offset to begin this fetch from. + /// MaxBytes => int32 -- The maximum bytes to include in the message set for this partition. This helps bound the size of the response. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchAPI + /// + public static void AssertFetchRequest(this BigEndianBinaryReader reader, FetchRequest request) + { + Assert.That(reader.ReadInt32(), Is.EqualTo(-1), "ReplicaId"); + Assert.That(reader.ReadInt32(), Is.EqualTo(request.MaxWaitTime), "MaxWaitTime"); + Assert.That(reader.ReadInt32(), Is.EqualTo(request.MinBytes), "MinBytes"); + + Assert.That(reader.ReadInt32(), Is.EqualTo(request.Fetches.Count), "[TopicName]"); + foreach (var payload in request.Fetches) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); + + Assert.That(reader.ReadInt64(), Is.EqualTo(payload.Offset), "FetchOffset"); + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.MaxBytes), "MaxBytes"); + } + } /// /// ProduceResponse => [TopicName [Partition ErrorCode Offset *Timestamp]] *ThrottleTime @@ -48,7 +84,7 @@ public static void AssertProduceResponse(this BigEndianBinaryReader reader, int Assert.That(payload.Timestamp, Is.Null, "Timestamp"); } else { Assert.That(payload.Timestamp, Is.Not.Null, "Timestamp"); - Assert.That(payload.Timestamp.Value, Is.EqualTo(UnixEpoch.AddMilliseconds(timestamp)), "Timestamp"); + Assert.That(payload.Timestamp.Value, Is.EqualTo(timestamp.FromUnixEpochMilliseconds()), "Timestamp"); } } } @@ -137,7 +173,7 @@ public static void AssertMessage(this BigEndianBinaryReader reader, int version, Assert.That(reader.ReadByte(), Is.EqualTo(message.MagicNumber), "MagicByte"); Assert.That(reader.ReadByte(), Is.EqualTo(message.Attribute), "Attributes"); if (version == 2) { - Assert.That(reader.ReadInt64(), Is.EqualTo((long)(message.Timestamp - UnixEpoch).TotalMilliseconds), "Timestamp"); + Assert.That(reader.ReadInt64(), Is.EqualTo(message.Timestamp.ToUnixEpochMilliseconds()), "Timestamp"); } Assert.That(reader.ReadBytes(), Is.EqualTo(message.Key), "Key"); Assert.That(reader.ReadBytes(), Is.EqualTo(message.Value), "Value"); diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 0cadcaf2..91d626da 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -185,6 +185,66 @@ public void InterpretApiResponse( }); } + /// + /// FetchRequest => ReplicaId MaxWaitTime MinBytes [TopicName [Partition FetchOffset MaxBytes]] + /// ReplicaId => int32 -- The replica id indicates the node id of the replica initiating this request. Normal client consumers should always + /// specify this as -1 as they have no node id. Other brokers set this to be their own node id. The value -2 is accepted + /// to allow a non-broker to issue fetch requests as if it were a replica broker for debugging purposes. + /// MaxWaitTime => int32 -- The max wait time is the maximum amount of time in milliseconds to block waiting if insufficient data is available + /// at the time the request is issued. + /// MinBytes => int32 -- This is the minimum number of bytes of messages that must be available to give a response. If the client sets this + /// to 0 the server will always respond immediately, however if there is no new data since their last request they will + /// just get back empty message sets. If this is set to 1, the server will respond as soon as at least one partition has + /// at least 1 byte of data or the specified timeout occurs. By setting higher values in combination with the timeout the + /// consumer can tune for throughput and trade a little additional latency for reading only large chunks of data (e.g. + /// setting MaxWaitTime to 100 ms and setting MinBytes to 64k would allow the server to wait up to 100ms to try to accumulate + /// 64k of data before responding). + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition the fetch is for. + /// FetchOffset => int64 -- The offset to begin this fetch from. + /// MaxBytes => int32 -- The maximum bytes to include in the message set for this partition. This helps bound the size of the response. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchAPI + /// + [Test] + public void FetchApiRequest( + [Values(0, 1, 2)] short version, + [Values(0, 10, 100)] int timeoutMilliseconds, + [Values(0, 64000)] int minBytes, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int totalPartitions, + [Values(64000, 25600000)] int maxBytes) + { + var randomizer = new Randomizer(); + var clientId = nameof(FetchApiRequest); + + var request = new FetchRequest { + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + Fetches = new List(), + ApiVersion = version + }; + + for (var t = 0; t < topicsPerRequest; t++) { + var payload = new Fetch { + Topic = topic + t, + PartitionId = t % totalPartitions, + Offset = randomizer.Next(0, int.MaxValue), + MaxBytes = maxBytes + }; + request.Fetches.Add(payload); + } + + var data = request.Encode(); + + data.Buffer.AssertProtocol( + reader => { + reader.AssertRequestHeader(request); + reader.AssertFetchRequest(request); + }); + } + public static IEnumerable ErrorResponseCodes => new[] { ErrorResponseCode.NoError, ErrorResponseCode.Unknown, From 62abb16321d197eb1572d53f9c8852f7d1cd286a Mon Sep 17 00:00:00 2001 From: Andrew Date: Wed, 24 Aug 2016 17:44:20 -0400 Subject: [PATCH 07/19] Fix message versioning (using MagicNumber), add tests and support for FetchApiResponse --- src/kafka-net/Protocol/FetchRequest.cs | 8 +- src/kafka-net/Protocol/Message.cs | 19 ++- src/kafka-net/Protocol/ProduceRequest.cs | 14 +- .../Unit/ProtocolAssertionExtensions.cs | 35 +++++ src/kafka-tests/Unit/ProtocolByteTests.cs | 140 ++++++++++++------ 5 files changed, 160 insertions(+), 56 deletions(-) diff --git a/src/kafka-net/Protocol/FetchRequest.cs b/src/kafka-net/Protocol/FetchRequest.cs index c7e87540..1078ef45 100644 --- a/src/kafka-net/Protocol/FetchRequest.cs +++ b/src/kafka-net/Protocol/FetchRequest.cs @@ -37,7 +37,7 @@ public KafkaDataPayload Encode() public IEnumerable Decode(byte[] payload) { - return DecodeFetchResponse(payload); + return DecodeFetchResponse(ApiVersion, payload); } private KafkaDataPayload EncodeFetchRequest(FetchRequest request) @@ -78,12 +78,16 @@ private KafkaDataPayload EncodeFetchRequest(FetchRequest request) } } - private IEnumerable DecodeFetchResponse(byte[] data) + private IEnumerable DecodeFetchResponse(int version, byte[] data) { using (var stream = new BigEndianBinaryReader(data)) { var correlationId = stream.ReadInt32(); + if (version >= 1) { + var throttleTime = stream.ReadInt32(); + } + var topicCount = stream.ReadInt32(); for (int i = 0; i < topicCount; i++) { diff --git a/src/kafka-net/Protocol/Message.cs b/src/kafka-net/Protocol/Message.cs index 8991764c..793d09db 100644 --- a/src/kafka-net/Protocol/Message.cs +++ b/src/kafka-net/Protocol/Message.cs @@ -80,16 +80,15 @@ public Message(string value, string key = null) /// Encodes a collection of messages into one byte[]. Encoded in order of list. /// /// The collection of messages to encode together. - /// Message version /// Encoded byte[] representing the collection of messages. - public static byte[] EncodeMessageSet(IEnumerable messages, short version = 0) + public static byte[] EncodeMessageSet(IEnumerable messages) { using (var stream = new KafkaMessagePacker()) { foreach (var message in messages) { stream.Pack(InitialMessageOffset) - .Pack(EncodeMessage(message, version)); + .Pack(EncodeMessage(message)); } return stream.PayloadNoLength(); @@ -134,19 +133,18 @@ public static IEnumerable DecodeMessageSet(byte[] messageSet) /// Encodes a message object to byte[] /// /// Message data to encode. - /// Message version /// Encoded byte[] representation of the message object. /// /// Format: /// Crc (Int32), MagicByte (Byte), Attribute (Byte), Key (Byte[]), Value (Byte[]) /// - public static byte[] EncodeMessage(Message message, short version = 0) + public static byte[] EncodeMessage(Message message) { using(var stream = new KafkaMessagePacker()) { stream.Pack(message.MagicNumber) .Pack(message.Attribute); - if (version > 1) { + if (message.MagicNumber >= 1) { stream.Pack(message.Timestamp.ToUnixEpochMilliseconds()); } return stream.Pack(message.Key) @@ -175,9 +173,16 @@ public static IEnumerable DecodeMessage(long offset, byte[] payload) Meta = new MessageMetadata { Offset = offset }, MagicNumber = stream.ReadByte(), Attribute = stream.ReadByte(), - Key = stream.ReadIntPrefixedBytes() }; + if (message.MagicNumber >= 1) { + var timestamp = stream.ReadInt64(); + if (timestamp >= 0) { + message.Timestamp = timestamp.FromUnixEpochMilliseconds(); + } + } + message.Key = stream.ReadIntPrefixedBytes(); + var codec = (MessageCodec)(ProtocolConstants.AttributeCodeMask & message.Attribute); switch (codec) { diff --git a/src/kafka-net/Protocol/ProduceRequest.cs b/src/kafka-net/Protocol/ProduceRequest.cs index 19ebe4db..69015160 100644 --- a/src/kafka-net/Protocol/ProduceRequest.cs +++ b/src/kafka-net/Protocol/ProduceRequest.cs @@ -72,12 +72,12 @@ private static KafkaDataPayload EncodeProduceRequest(ProduceRequest request) { case MessageCodec.CodecNone: - message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages), request.ApiVersion)); + message.Pack(Message.EncodeMessageSet(payloads.SelectMany(x => x.Messages))); break; case MessageCodec.CodecGzip: - var compressedBytes = CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages), request.ApiVersion); + var compressedBytes = CreateGzipCompressedMessage(payloads.SelectMany(x => x.Messages)); Interlocked.Add(ref totalCompressedBytes, compressedBytes.CompressedAmount); - message.Pack(Message.EncodeMessageSet(new[] { compressedBytes.CompressedMessage }, request.ApiVersion)); + message.Pack(Message.EncodeMessageSet(new[] { compressedBytes.CompressedMessage })); break; default: throw new NotSupportedException(string.Format("Codec type of {0} is not supported.", groupedPayload.Key.Codec)); @@ -95,9 +95,9 @@ private static KafkaDataPayload EncodeProduceRequest(ProduceRequest request) } } - private static CompressedMessageResult CreateGzipCompressedMessage(IEnumerable messages, short version) + private static CompressedMessageResult CreateGzipCompressedMessage(IEnumerable messages) { - var messageSet = Message.EncodeMessageSet(messages, version); + var messageSet = Message.EncodeMessageSet(messages); var gZipBytes = Compression.Zip(messageSet); @@ -146,6 +146,10 @@ private static IEnumerable DecodeProduceResponse(int version, b yield return response; } } + + if (version >= 2) { + var throttleTime = stream.ReadInt32(); + } } } #endregion diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index 4ac86e28..bc4f6f07 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,6 +11,41 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { + /// + /// FetchResponse => *ThrottleTime [TopicName [Partition ErrorCode HighwaterMarkOffset MessageSetSize MessageSet]] + /// *ThrottleTime is only version 1 (0.9.0) and above + /// ThrottleTime => int32 -- Duration in milliseconds for which the request was throttled due to quota violation. (Zero if the request did not + /// violate any quota.) + /// TopicName => string -- The topic this response entry corresponds to. + /// Partition => int32 -- The partition this response entry corresponds to. + /// ErrorCode => int16 -- The error from this partition, if any. Errors are given on a per-partition basis because a given partition may + /// be unavailable or maintained on a different host, while others may have successfully accepted the produce request. + /// HighwaterMarkOffset => int64 -- The offset at the end of the log for this partition. This can be used by the client to determine how many messages + /// behind the end of the log they are. + /// MessageSetSize => int32 -- The size in bytes of the message set for this partition + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchResponse + /// + public static void AssertFetchResponse(this BigEndianBinaryReader reader, int version, int throttleTime, IEnumerable response) + { + var responses = response.ToList(); + if (version >= 1) { + Assert.That(reader.ReadInt32(), Is.EqualTo(throttleTime), "ThrottleTime"); + } + Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); + foreach (var payload in responses) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); + Assert.That(reader.ReadInt16(), Is.EqualTo(payload.Error), "Error"); + Assert.That(reader.ReadInt64(), Is.EqualTo(payload.HighWaterMark), "HighwaterMarkOffset"); + + var finalPosition = reader.ReadInt32() + reader.Position; + reader.AssertMessageSet(version, payload.Messages); + Assert.That(reader.Position, Is.EqualTo(finalPosition), $"MessageSetSize was {finalPosition - 4} but ended in a different spot."); + } + } + /// /// FetchRequest => ReplicaId MaxWaitTime MinBytes [TopicName [Partition FetchOffset MaxBytes]] /// ReplicaId => int32 -- The replica id indicates the node id of the replica initiating this request. Normal client consumers should always diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 91d626da..b10661f2 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -71,7 +71,6 @@ public void ProduceApiRequest( [Values(1, 5)] int totalPartitions, [Values(1, 2, 3)] int messagesPerSet) { - var randomizer = new Randomizer(); var clientId = nameof(ProduceApiRequest); var request = new ProduceRequest { @@ -84,26 +83,13 @@ public void ProduceApiRequest( }; for (var t = 0; t < topicsPerRequest; t++) { - var payload = new Payload { - Topic = topic + t, - Partition = t % totalPartitions, - Codec = MessageCodec.CodecNone, - Messages = new List() - }; - for (var m = 0; m < messagesPerSet; m++) { - var message = new Message { - MagicNumber = 1, - Timestamp = DateTime.UtcNow, - Key = m > 0 ? new byte[8] : null, - Value = new byte[8*(m + 1)] - }; - if (message.Key != null) { - randomizer.NextBytes(message.Key); - } - randomizer.NextBytes(message.Value); - payload.Messages.Add(message); - } - request.Payload.Add(payload); + request.Payload.Add( + new Payload { + Topic = topic + t, + Partition = t % totalPartitions, + Codec = MessageCodec.CodecNone, + Messages = GenerateMessages(messagesPerSet, (byte) (version >= 2 ? 1 : 0)) + }); } var data = request.Encode(); @@ -135,7 +121,7 @@ public void ProduceApiRequest( /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-Messagesets /// [Test] - public void InterpretApiResponse( + public void ProduceApiResponse( [Values(0, 1, 2)] short version, [Values(-1, 0, 123456, 10000000)] long timestampMilliseconds, [Values("test", "a really long name, with spaces and punctuation!")] string topic, @@ -149,7 +135,7 @@ public void InterpretApiResponse( [Values(0, 1234, 100000)] int throttleTime) { var randomizer = new Randomizer(); - var clientId = nameof(InterpretApiResponse); + var clientId = nameof(ProduceApiResponse); var correlationId = clientId.GetHashCode(); byte[] data = null; @@ -245,24 +231,94 @@ public void FetchApiRequest( }); } - public static IEnumerable ErrorResponseCodes => new[] { - ErrorResponseCode.NoError, - ErrorResponseCode.Unknown, - ErrorResponseCode.OffsetOutOfRange, - ErrorResponseCode.InvalidMessage, - ErrorResponseCode.UnknownTopicOrPartition, - ErrorResponseCode.InvalidMessageSize, - ErrorResponseCode.LeaderNotAvailable, - ErrorResponseCode.NotLeaderForPartition, - ErrorResponseCode.RequestTimedOut, - ErrorResponseCode.BrokerNotAvailable, - ErrorResponseCode.ReplicaNotAvailable, - ErrorResponseCode.MessageSizeTooLarge, - //ErrorResponseCode.StaleControllerEpochCode, - ErrorResponseCode.OffsetMetadataTooLargeCode, - ErrorResponseCode.OffsetsLoadInProgressCode, - ErrorResponseCode.ConsumerCoordinatorNotAvailableCode, - ErrorResponseCode.NotCoordinatorForConsumerCode - }; + /// + /// FetchResponse => *ThrottleTime [TopicName [Partition ErrorCode HighwaterMarkOffset MessageSetSize MessageSet]] + /// *ThrottleTime is only version 1 (0.9.0) and above + /// ThrottleTime => int32 -- Duration in milliseconds for which the request was throttled due to quota violation. (Zero if the request did not + /// violate any quota.) + /// TopicName => string -- The topic this response entry corresponds to. + /// Partition => int32 -- The partition this response entry corresponds to. + /// ErrorCode => int16 -- The error from this partition, if any. Errors are given on a per-partition basis because a given partition may + /// be unavailable or maintained on a different host, while others may have successfully accepted the produce request. + /// HighwaterMarkOffset => int64 -- The offset at the end of the log for this partition. This can be used by the client to determine how many messages + /// behind the end of the log they are. + /// MessageSetSize => int32 -- The size in bytes of the message set for this partition + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-FetchResponse + /// + [Test] + public void FetchApiResponse( + [Values(0, 1, 2)] short version, + [Values(0, 1234)] int throttleTime, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int totalPartitions, + [Values( + ErrorResponseCode.NoError, + ErrorResponseCode.OffsetOutOfRange, + ErrorResponseCode.NotLeaderForPartition, + ErrorResponseCode.ReplicaNotAvailable, + ErrorResponseCode.Unknown + )] ErrorResponseCode errorCode, + [Values(2, 3)] int messagesPerSet + ) + { + var randomizer = new Randomizer(); + var clientId = nameof(FetchApiResponse); + var correlationId = clientId.GetHashCode(); + + byte[] data = null; + using (var stream = new MemoryStream()) { + var writer = new BigEndianBinaryWriter(stream); + writer.WriteResponseHeader(correlationId); + + if (version >= 1) { + writer.Write(throttleTime); + } + writer.Write(topicsPerRequest); + for (var t = 0; t < topicsPerRequest; t++) { + writer.Write(topic + t); + writer.Write(1); // partitionsPerTopic + writer.Write(t % totalPartitions); + writer.Write((short)errorCode); + writer.Write((long)randomizer.Next()); + + var messageSet = Message.EncodeMessageSet(GenerateMessages(messagesPerSet, (byte) (version >= 2 ? 1 : 0))); + writer.Write(messageSet.Length); + writer.Write(messageSet); + } + data = new byte[stream.Position]; + Buffer.BlockCopy(stream.GetBuffer(), 0, data, 0, data.Length); + } + + var request = new FetchRequest { ApiVersion = version }; + var responses = request.Decode(data); // doesn't include the size in the decode -- the framework deals with it, I'd assume + data.PrefixWithInt32Length().AssertProtocol( + reader => + { + reader.AssertResponseHeader(correlationId); + reader.AssertFetchResponse(version, throttleTime, responses); + }); + } + + private List GenerateMessages(int count, byte version) + { + var randomizer = new Randomizer(); + var messages = new List(); + for (var m = 0; m < count; m++) { + var message = new Message { + MagicNumber = version, + Timestamp = DateTime.UtcNow, + Key = m > 0 ? new byte[8] : null, + Value = new byte[8*(m + 1)] + }; + if (message.Key != null) { + randomizer.NextBytes(message.Key); + } + randomizer.NextBytes(message.Value); + messages.Add(message); + } + return messages; + } } } \ No newline at end of file From 95af6abdcf3cba08659e37b0fc6462e49a5e3f39 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 26 Aug 2016 00:01:45 -0400 Subject: [PATCH 08/19] Add tests for OffsetsApi --- .../Unit/ProtocolAssertionExtensions.cs | 57 ++++++++++ src/kafka-tests/Unit/ProtocolByteTests.cs | 107 ++++++++++++++++++ 2 files changed, 164 insertions(+) diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index bc4f6f07..244113c3 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,6 +11,63 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { + /// + /// OffsetResponse => [TopicName [PartitionOffsets]] + /// PartitionOffsets => Partition ErrorCode [Offset] + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition the fetch is for. + /// ErrorCode => int16 -- The error from this partition, if any. Errors are given on a per-partition basis because a given partition may + /// be unavailable or maintained on a different host, while others may have successfully accepted the produce request. + /// Offset => int64 + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetAPI(AKAListOffset) + /// + public static void AssertOffsetResponse(this BigEndianBinaryReader reader, IEnumerable response) + { + var responses = response.ToList(); + Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); + foreach (var payload in responses) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); + Assert.That(reader.ReadInt16(), Is.EqualTo(payload.Error), "ErrorCode"); + + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Offsets.Count), "[Offset]"); + foreach (var offset in payload.Offsets) { + Assert.That(reader.ReadInt64(), Is.EqualTo(offset), "Offset"); + } + } + } + + /// + /// OffsetRequest => ReplicaId [TopicName [Partition Time MaxNumberOfOffsets]] + /// ReplicaId => int32 -- The replica id indicates the node id of the replica initiating this request. Normal client consumers should always + /// specify this as -1 as they have no node id. Other brokers set this to be their own node id. The value -2 is accepted + /// to allow a non-broker to issue fetch requests as if it were a replica broker for debugging purposes. + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition the fetch is for. + /// Time => int64 -- Used to ask for all messages before a certain time (ms). There are two special values. Specify -1 to receive the + /// latest offset (i.e. the offset of the next coming message) and -2 to receive the earliest available offset. Note + /// that because offsets are pulled in descending order, asking for the earliest offset will always return you a single element. + /// MaxNumberOfOffsets => int32 + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetAPI(AKAListOffset) + /// + public static void AssertOffsetRequest(this BigEndianBinaryReader reader, OffsetRequest request) + { + Assert.That(reader.ReadInt32(), Is.EqualTo(-1), "ReplicaId"); + + Assert.That(reader.ReadInt32(), Is.EqualTo(request.Offsets.Count), "[TopicName]"); + foreach (var payload in request.Offsets) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); + + Assert.That(reader.ReadInt64(), Is.EqualTo(payload.Time), "Time"); + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.MaxOffsets), "MaxNumberOfOffsets"); + } + } + /// /// FetchResponse => *ThrottleTime [TopicName [Partition ErrorCode HighwaterMarkOffset MessageSetSize MessageSet]] /// *ThrottleTime is only version 1 (0.9.0) and above diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index b10661f2..0b7b3aa0 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -301,6 +301,113 @@ public void FetchApiResponse( }); } + /// + /// OffsetRequest => ReplicaId [TopicName [Partition Time MaxNumberOfOffsets]] + /// ReplicaId => int32 -- The replica id indicates the node id of the replica initiating this request. Normal client consumers should always + /// specify this as -1 as they have no node id. Other brokers set this to be their own node id. The value -2 is accepted + /// to allow a non-broker to issue fetch requests as if it were a replica broker for debugging purposes. + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition the fetch is for. + /// Time => int64 -- Used to ask for all messages before a certain time (ms). There are two special values. Specify -1 to receive the + /// latest offset (i.e. the offset of the next coming message) and -2 to receive the earliest available offset. Note + /// that because offsets are pulled in descending order, asking for the earliest offset will always return you a single element. + /// MaxNumberOfOffsets => int32 + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetAPI(AKAListOffset) + /// + [Test] + public void OffsetsApiRequest( + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int totalPartitions, + [Values(-2, -1, 123456, 10000000)] long time, + [Values(1, 10)] int maxOffsets) + { + var clientId = nameof(OffsetsApiRequest); + + var request = new OffsetRequest { + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + Offsets = new List(), + ApiVersion = 0 + }; + + for (var t = 0; t < topicsPerRequest; t++) { + var payload = new Offset { + Topic = topic + t, + PartitionId = t % totalPartitions, + Time = time, + MaxOffsets = maxOffsets + }; + request.Offsets.Add(payload); + } + + var data = request.Encode(); + + data.Buffer.AssertProtocol( + reader => { + reader.AssertRequestHeader(request); + reader.AssertOffsetRequest(request); + }); + } + + /// + /// OffsetResponse => [TopicName [PartitionOffsets]] + /// PartitionOffsets => Partition ErrorCode [Offset] + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition the fetch is for. + /// ErrorCode => int16 -- The error from this partition, if any. Errors are given on a per-partition basis because a given partition may + /// be unavailable or maintained on a different host, while others may have successfully accepted the produce request. + /// Offset => int64 + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetAPI(AKAListOffset) + /// + [Test] + public void OffsetsApiResponse( + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(5)] int totalPartitions, + [Values( + ErrorResponseCode.UnknownTopicOrPartition, + ErrorResponseCode.NotLeaderForPartition, + ErrorResponseCode.Unknown + )] ErrorResponseCode errorCode, + [Values(1, 5)] int offsetsPerPartition) + { + var randomizer = new Randomizer(); + var clientId = nameof(OffsetsApiResponse); + var correlationId = clientId.GetHashCode(); + + byte[] data = null; + using (var stream = new MemoryStream()) { + var writer = new BigEndianBinaryWriter(stream); + writer.WriteResponseHeader(correlationId); + + writer.Write(topicsPerRequest); + for (var t = 0; t < topicsPerRequest; t++) { + writer.Write(topic + t); + writer.Write(1); // partitionsPerTopic + writer.Write(t % totalPartitions); + writer.Write((short)errorCode); + writer.Write(offsetsPerPartition); + for (var o = 0; o < offsetsPerPartition; o++) { + writer.Write((long)randomizer.Next()); + } + } + data = new byte[stream.Position]; + Buffer.BlockCopy(stream.GetBuffer(), 0, data, 0, data.Length); + } + + var request = new OffsetRequest { ApiVersion = 0 }; + var responses = request.Decode(data); // doesn't include the size in the decode -- the framework deals with it, I'd assume + data.PrefixWithInt32Length().AssertProtocol( + reader => + { + reader.AssertResponseHeader(correlationId); + reader.AssertOffsetResponse(responses); + }); + } + private List GenerateMessages(int count, byte version) { var randomizer = new Randomizer(); From f13ada5f87e4c702748a658ac6721f51c9f07f79 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 12:27:04 -0400 Subject: [PATCH 09/19] Add protocol tests for MetadataApiRequest & Response --- src/kafka-net/Protocol/Protocol.cs | 103 ++++++++++++++- .../Unit/ProtocolAssertionExtensions.cs | 62 +++++++++ src/kafka-tests/Unit/ProtocolByteTests.cs | 119 ++++++++++++++++++ 3 files changed, 283 insertions(+), 1 deletion(-) diff --git a/src/kafka-net/Protocol/Protocol.cs b/src/kafka-net/Protocol/Protocol.cs index 84b913fe..dd1b752e 100644 --- a/src/kafka-net/Protocol/Protocol.cs +++ b/src/kafka-net/Protocol/Protocol.cs @@ -124,6 +124,11 @@ public enum ErrorResponseCode : short /// OffsetMetadataTooLargeCode = 12, + /// + /// The server disconnected before a response was received. + /// + NetworkException = 13, + /// /// The broker returns this error code for an offset fetch request if it is still loading offsets (after a leader change for that offsets topic partition). /// @@ -137,7 +142,103 @@ public enum ErrorResponseCode : short /// /// The broker returns this error code if it receives an offset fetch or commit request for a consumer group that it is not a coordinator for. /// - NotCoordinatorForConsumerCode = 16 + NotCoordinatorForConsumerCode = 16, + + /// + /// The request attempted to perform an operation on an invalid topic. + /// + InvalidTopic = 17, + + /// + /// The request included message batch larger than the configured segment size on the server. + /// + RecordListTooLarge = 18, + + /// + /// Messages are rejected since there are fewer in-sync replicas than required. + /// + NotEnoughReplicas = 19, + + /// + /// Messages are written to the log, but to fewer in-sync replicas than required. + /// + NotEnoughReplicasAfterAppend = 20, + + /// + /// Produce request specified an invalid value for required acks. + /// + InvalidRequiredAcks = 21, + + /// + /// Specified group generation id is not valid. + /// + IllegalGeneration = 22, + + /// + /// The group member's supported protocols are incompatible with those of existing members. + /// + InconsistentGroupProtocol = 23, + + /// + /// The configured groupId is invalid. + /// + InvalidGroupId = 24, + + /// + /// The coordinator is not aware of this member. + /// + UnknownMemberId = 25, + + /// + /// The session timeout is not within the range allowed by the broker (as configured + /// by group.min.session.timeout.ms and group.max.session.timeout.ms). + /// + InvalidSessionTimeout = 26, + + /// + /// The group is rebalancing, so a rejoin is needed. + /// + RebalanceInProgress = 27, + + /// + /// The committing offset data size is not valid + /// + InvalidCommitOffsetSize = 28, + + /// + /// Not authorized to access topic. + /// + TopicAuthorizationFailed = 29, + + /// + /// Not authorized to access group. + /// + GroupAuthorizationFailed = 30, + + /// + /// Cluster authorization failed. + /// + ClusterAuthorizationFailed = 31, + + /// + /// The timestamp of the message is out of acceptable range. + /// + InvalidTimestamp = 32, + + /// + /// The broker does not support the requested SASL mechanism. + /// + UnsupportedSaslMechanism = 33, + + /// + /// Request is not valid given the current SASL state. + /// + IllegalSaslState = 34, + + /// + /// The version of API is not supported. + /// + UnsupportedVersion = 35 } /// diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index 244113c3..d3228411 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,6 +11,68 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { + /// + /// MetadataResponse => [Broker][TopicMetadata] + /// Broker => NodeId Host Port (any number of brokers may be returned) + /// -- The node id, hostname, and port information for a kafka broker + /// NodeId => int32 + /// Host => string + /// Port => int32 + /// TopicMetadata => TopicErrorCode TopicName [PartitionMetadata] + /// TopicErrorCode => int16 + /// PartitionMetadata => PartitionErrorCode PartitionId Leader Replicas Isr + /// PartitionErrorCode => int16 + /// PartitionId => int32 + /// Leader => int32 -- The node id for the kafka broker currently acting as leader for this partition. + /// If no leader exists because we are in the middle of a leader election this id will be -1. + /// Replicas => [int32] -- The set of alive nodes that currently acts as slaves for the leader for this partition. + /// Isr => [int32] -- The set subset of the replicas that are "caught up" to the leader + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI + /// + public static void AssertMetadataResponse(this BigEndianBinaryReader reader, MetadataResponse response) + { + Assert.That(reader.ReadInt32(), Is.EqualTo(response.Brokers.Count), "[Broker]"); + foreach (var payload in response.Brokers) { + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.BrokerId), "NodeId"); + Assert.That(reader.ReadString(), Is.EqualTo(payload.Host), "Host"); + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Port), "Port"); + } + Assert.That(reader.ReadInt32(), Is.EqualTo(response.Topics.Count), "[TopicMetadata]"); + foreach (var payload in response.Topics) { + Assert.That(reader.ReadInt16(), Is.EqualTo((short)payload.ErrorCode), "TopicErrorCode"); + Assert.That(reader.ReadString(), Is.EqualTo(payload.Name), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partitions.Count), "[PartitionMetadata]"); + foreach (var partition in payload.Partitions) { + Assert.That(reader.ReadInt16(), Is.EqualTo((short) partition.ErrorCode), "PartitionErrorCode"); + Assert.That(reader.ReadInt32(), Is.EqualTo(partition.PartitionId), "PartitionId"); + Assert.That(reader.ReadInt32(), Is.EqualTo(partition.LeaderId), "Leader"); + Assert.That(reader.ReadInt32(), Is.EqualTo(partition.Replicas.Count), "[Replicas]"); + foreach (var replica in partition.Replicas) { + Assert.That(reader.ReadInt32(), Is.EqualTo(replica), "Replicas"); + } + Assert.That(reader.ReadInt32(), Is.EqualTo(partition.Isrs.Count), "[Isr]"); + foreach (var isr in partition.Isrs) { + Assert.That(reader.ReadInt32(), Is.EqualTo(isr), "Isr"); + } + } + } + } + + /// + /// TopicMetadataRequest => [TopicName] + /// TopicName => string -- The topics to produce metadata for. If empty the request will yield metadata for all topics. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI + /// + public static void AssertMetadataRequest(this BigEndianBinaryReader reader, MetadataRequest request) + { + Assert.That(reader.ReadInt32(), Is.EqualTo(request.Topics.Count), "[TopicName]"); + foreach (var payload in request.Topics) { + Assert.That(reader.ReadString(), Is.EqualTo(payload), "TopicName"); + } + } + /// /// OffsetResponse => [TopicName [PartitionOffsets]] /// PartitionOffsets => Partition ErrorCode [Offset] diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 0b7b3aa0..55308122 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; using KafkaNet.Common; using KafkaNet.Protocol; using NUnit.Framework; @@ -408,6 +409,124 @@ public void OffsetsApiResponse( }); } + /// + /// TopicMetadataRequest => [TopicName] + /// TopicName => string -- The topics to produce metadata for. If empty the request will yield metadata for all topics. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI + /// + [Test] + public void MetadataApiRequest( + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(0, 1, 10)] int topicsPerRequest) + { + var clientId = nameof(MetadataApiRequest); + + var request = new MetadataRequest { + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + Topics = new List(), + ApiVersion = 0 + }; + + for (var t = 0; t < topicsPerRequest; t++) { + request.Topics.Add(topic + t); + } + + var data = request.Encode(); + + data.Buffer.AssertProtocol( + reader => { + reader.AssertRequestHeader(request); + reader.AssertMetadataRequest(request); + }); + } + + /// + /// MetadataResponse => [Broker][TopicMetadata] + /// Broker => NodeId Host Port (any number of brokers may be returned) + /// -- The node id, hostname, and port information for a kafka broker + /// NodeId => int32 + /// Host => string + /// Port => int32 + /// TopicMetadata => TopicErrorCode TopicName [PartitionMetadata] + /// TopicErrorCode => int16 + /// PartitionMetadata => PartitionErrorCode PartitionId Leader Replicas Isr + /// PartitionErrorCode => int16 + /// PartitionId => int32 + /// Leader => int32 -- The node id for the kafka broker currently acting as leader for this partition. + /// If no leader exists because we are in the middle of a leader election this id will be -1. + /// Replicas => [int32] -- The set of alive nodes that currently acts as slaves for the leader for this partition. + /// Isr => [int32] -- The set subset of the replicas that are "caught up" to the leader + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI + /// + [Test] + public void MetadataApiResponse( + [Values(1, 5)] int brokersPerRequest, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int partitionsPerTopic, + [Values( + ErrorResponseCode.NoError, + ErrorResponseCode.UnknownTopicOrPartition, + ErrorResponseCode.LeaderNotAvailable, + ErrorResponseCode.InvalidTopic, + ErrorResponseCode.TopicAuthorizationFailed + )] ErrorResponseCode errorCode) + { + var randomizer = new Randomizer(); + var clientId = nameof(MetadataApiResponse); + var correlationId = clientId.GetHashCode(); + + byte[] data = null; + using (var stream = new MemoryStream()) { + var writer = new BigEndianBinaryWriter(stream); + writer.WriteResponseHeader(correlationId); + + writer.Write(brokersPerRequest); + for (var b = 0; b < brokersPerRequest; b++) { + writer.Write(b); + writer.Write("http://broker-" + b); + writer.Write(9092 + b); + } + + writer.Write(topicsPerRequest); + for (var t = 0; t < topicsPerRequest; t++) { + writer.Write((short) errorCode); + writer.Write(topic + t); + writer.Write(partitionsPerTopic); + for (var p = 0; p < partitionsPerTopic; p++) { + writer.Write((short) errorCode); + writer.Write(p); + var leader = randomizer.Next(0, brokersPerRequest - 1); + writer.Write(leader); + var replicas = randomizer.Next(0, brokersPerRequest - 1); + writer.Write(replicas); + for (var r = 0; r < replicas; r++) { + writer.Write(r); + } + var isr = randomizer.Next(0, replicas); + writer.Write(isr); + for (var i = 0; i < isr; i++) { + writer.Write(i); + } + } + } + data = new byte[stream.Position]; + Buffer.BlockCopy(stream.GetBuffer(), 0, data, 0, data.Length); + } + + var request = new MetadataRequest {ApiVersion = 0}; + var responses = request.Decode(data).Single(); // note that this is a bit weird + // doesn't include the size in the decode -- the framework deals with it, I'd assume + data.PrefixWithInt32Length().AssertProtocol( + reader => { + reader.AssertResponseHeader(correlationId); + reader.AssertMetadataResponse(responses); + }); + } + private List GenerateMessages(int count, byte version) { var randomizer = new Randomizer(); From fe6bffcb37aca91ec3c8f57a2ee470697e0cd0b0 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 16:08:54 -0400 Subject: [PATCH 10/19] Add protocol tests for OffsetCommitRequest support for version 1 & 2 --- src/kafka-net/Protocol/OffsetCommitRequest.cs | 23 ++++- .../Unit/ProtocolAssertionExtensions.cs | 46 ++++++++++ src/kafka-tests/Unit/ProtocolByteTests.cs | 91 ++++++++++++++++--- 3 files changed, 145 insertions(+), 15 deletions(-) diff --git a/src/kafka-net/Protocol/OffsetCommitRequest.cs b/src/kafka-net/Protocol/OffsetCommitRequest.cs index f758c5ac..dd412c22 100644 --- a/src/kafka-net/Protocol/OffsetCommitRequest.cs +++ b/src/kafka-net/Protocol/OffsetCommitRequest.cs @@ -12,7 +12,10 @@ namespace KafkaNet.Protocol public class OffsetCommitRequest : BaseRequest, IKafkaRequest { public ApiKeyRequestType ApiKey { get { return ApiKeyRequestType.OffsetCommit; } } + public int GenerationId { get; set; } + public string MemberId { get; set; } public string ConsumerGroup { get; set; } + public TimeSpan? OffsetRetention { get; set; } public List OffsetCommits { get; set; } public KafkaDataPayload Encode() @@ -31,6 +34,18 @@ private KafkaDataPayload EncodeOffsetCommitRequest(OffsetCommitRequest request) using (var message = EncodeHeader(request).Pack(request.ConsumerGroup, StringPrefixEncoding.Int16)) { + if (request.ApiVersion >= 1) { + message.Pack(request.GenerationId) + .Pack(request.MemberId, StringPrefixEncoding.Int16); + } + if (request.ApiVersion >= 2) { + if (request.OffsetRetention.HasValue) { + message.Pack((long) request.OffsetRetention.Value.TotalMilliseconds); + } else { + message.Pack(-1L); + } + } + var topicGroups = request.OffsetCommits.GroupBy(x => x.Topic).ToList(); message.Pack(topicGroups.Count); @@ -45,9 +60,11 @@ private KafkaDataPayload EncodeOffsetCommitRequest(OffsetCommitRequest request) foreach (var commit in partition) { message.Pack(partition.Key) - .Pack(commit.Offset) - .Pack(commit.TimeStamp) - .Pack(commit.Metadata, StringPrefixEncoding.Int16); + .Pack(commit.Offset); + if (ApiVersion == 1) { + message.Pack(commit.TimeStamp); + } + message.Pack(commit.Metadata, StringPrefixEncoding.Int16); } } } diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index d3228411..5324b0ea 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,6 +11,52 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { + /// + /// OffsetCommitRequest => ConsumerGroup *ConsumerGroupGenerationId *MemberId *RetentionTime [TopicName [Partition Offset *TimeStamp Metadata]] + /// *ConsumerGroupGenerationId, MemberId is only version 1 (0.8.2) and above + /// *TimeStamp is only version 1 (0.8.2) + /// *RetentionTime is only version 2 (0.9.0) and above + /// ConsumerGroupId => string -- The consumer group id. + /// ConsumerGroupGenerationId => int32 -- The generation of the consumer group. + /// MemberId => string -- The consumer id assigned by the group coordinator. + /// RetentionTime => int64 -- Time period in ms to retain the offset. + /// TopicName => string -- The topic to commit. + /// Partition => int32 -- The partition id. + /// Offset => int64 -- message offset to be committed. + /// Timestamp => int64 -- Commit timestamp. + /// Metadata => string -- Any associated metadata the client wants to keep + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + public static void AssertOffsetCommitRequest(this BigEndianBinaryReader reader, OffsetCommitRequest request) + { + Assert.That(reader.ReadString(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); + + if (request.ApiVersion >= 1) { + Assert.That(reader.ReadInt32(), Is.EqualTo(request.GenerationId), "ConsumerGroupGenerationId"); + Assert.That(reader.ReadString(), Is.EqualTo(request.MemberId), "MemberId"); + } + if (request.ApiVersion >= 2) { + var expectedRetention = request.OffsetRetention.HasValue + ? (long) request.OffsetRetention.Value.TotalMilliseconds + : -1L; + Assert.That(reader.ReadInt64(), Is.EqualTo(expectedRetention), "RetentionTime"); + } + + Assert.That(reader.ReadInt32(), Is.EqualTo(request.OffsetCommits.Count), "[TopicName]"); + foreach (var payload in request.OffsetCommits) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); + Assert.That(reader.ReadInt64(), Is.EqualTo(payload.Offset), "Offset"); + + if (request.ApiVersion == 1) { + Assert.That(reader.ReadInt64(), Is.EqualTo(payload.TimeStamp), "TimeStamp"); + } + Assert.That(reader.ReadString(), Is.EqualTo(payload.Metadata), "Metadata"); + } + } + /// /// MetadataResponse => [Broker][TopicMetadata] /// Broker => NodeId Host Port (any number of brokers may be returned) diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 55308122..6aee1aad 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -411,7 +411,7 @@ public void OffsetsApiResponse( /// /// TopicMetadataRequest => [TopicName] - /// TopicName => string -- The topics to produce metadata for. If empty the request will yield metadata for all topics. + /// TopicName => string -- The topics to produce metadata for. If no topics are specified fetch metadata for all topics. /// /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI /// @@ -445,19 +445,20 @@ public void MetadataApiRequest( /// /// MetadataResponse => [Broker][TopicMetadata] /// Broker => NodeId Host Port (any number of brokers may be returned) - /// -- The node id, hostname, and port information for a kafka broker - /// NodeId => int32 - /// Host => string - /// Port => int32 + /// -- The node id, hostname, and port information for a kafka broker + /// NodeId => int32 -- The broker id. + /// Host => string -- The hostname of the broker. + /// Port => int32 -- The port on which the broker accepts requests. /// TopicMetadata => TopicErrorCode TopicName [PartitionMetadata] - /// TopicErrorCode => int16 + /// TopicErrorCode => int16 -- The error code for the given topic. + /// TopicName => string -- The name of the topic. /// PartitionMetadata => PartitionErrorCode PartitionId Leader Replicas Isr - /// PartitionErrorCode => int16 - /// PartitionId => int32 - /// Leader => int32 -- The node id for the kafka broker currently acting as leader for this partition. + /// PartitionErrorCode => int16 -- The error code for the partition, if any. + /// PartitionId => int32 -- The id of the partition. + /// Leader => int32 -- The id of the broker acting as leader for this partition. /// If no leader exists because we are in the middle of a leader election this id will be -1. - /// Replicas => [int32] -- The set of alive nodes that currently acts as slaves for the leader for this partition. - /// Isr => [int32] -- The set subset of the replicas that are "caught up" to the leader + /// Replicas => [int32] -- The set of all nodes that host this partition. + /// Isr => [int32] -- The set of nodes that are in sync with the leader for this partition. /// /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-MetadataAPI /// @@ -487,7 +488,7 @@ public void MetadataApiResponse( writer.Write(brokersPerRequest); for (var b = 0; b < brokersPerRequest; b++) { writer.Write(b); - writer.Write("http://broker-" + b); + writer.Write("broker-" + b); writer.Write(9092 + b); } @@ -527,6 +528,72 @@ public void MetadataApiResponse( }); } + /// + /// OffsetCommitRequest => ConsumerGroup *ConsumerGroupGenerationId *MemberId *RetentionTime [TopicName [Partition Offset *TimeStamp Metadata]] + /// *ConsumerGroupGenerationId, MemberId is only version 1 (0.8.2) and above + /// *TimeStamp is only version 1 (0.8.2) + /// *RetentionTime is only version 2 (0.9.0) and above + /// ConsumerGroupId => string -- The consumer group id. + /// ConsumerGroupGenerationId => int32 -- The generation of the consumer group. + /// MemberId => string -- The consumer id assigned by the group coordinator. + /// RetentionTime => int64 -- Time period in ms to retain the offset. + /// TopicName => string -- The topic to commit. + /// Partition => int32 -- The partition id. + /// Offset => int64 -- message offset to be committed. + /// Timestamp => int64 -- Commit timestamp. + /// Metadata => string -- Any associated metadata the client wants to keep + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + [Test] + public void OffsetCommitApiRequest( + [Values(0, 1, 2)] short version, + [Values("group1", "group2")] string groupId, + [Values(0, 1, 2)] int generation, + [Values(-1, 1024, 20000)] int retentionTime, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(5)] int maxPartitions, + [Values(1, 10)] int maxOffsets, + [Values(null, "something useful for the client")] string metadata) + { + var clientId = nameof(OffsetCommitApiRequest); + var randomizer = new Randomizer(); + + var request = new OffsetCommitRequest { + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + ConsumerGroup = groupId, + OffsetCommits = new List(), + ApiVersion = version, + GenerationId = generation, + MemberId = "member" + generation + }; + + if (retentionTime >= 0) { + request.OffsetRetention = TimeSpan.FromMilliseconds(retentionTime); + } + for (var t = 0; t < topicsPerRequest; t++) { + var payload = new OffsetCommit { + Topic = topic + t, + PartitionId = t % maxPartitions, + Offset = randomizer.Next(0, int.MaxValue), + Metadata = metadata + }; + payload.TimeStamp = retentionTime; + request.OffsetCommits.Add(payload); + } + + var data = request.Encode(); + + data.Buffer.AssertProtocol( + reader => + { + reader.AssertRequestHeader(request); + reader.AssertOffsetCommitRequest(request); + }); + } + private List GenerateMessages(int count, byte version) { var randomizer = new Randomizer(); From 0f3e7cbb7027317682ca6f6ae14f6bc2ae919ad0 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 16:23:11 -0400 Subject: [PATCH 11/19] Add protocol tests for OffsetCommitResponse --- .../Unit/ProtocolAssertionExtensions.cs | 23 ++++++++ src/kafka-tests/Unit/ProtocolByteTests.cs | 57 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index 5324b0ea..4d48370a 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,6 +11,29 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { + /// + /// OffsetCommitResponse => [TopicName [Partition ErrorCode]]] + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition. + /// ErrorCode => int16 -- The error code for the partition, if any. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + public static void AssertOffsetCommitResponse(this BigEndianBinaryReader reader, IEnumerable response) + { + var responses = response.GroupBy(r => r.Topic).ToList(); + Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); + foreach (var payload in responses) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Key), "TopicName"); + var partitions = payload.ToList(); + Assert.That(reader.ReadInt32(), Is.EqualTo(partitions.Count), "[Partition]"); + foreach (var partition in partitions) { + Assert.That(reader.ReadInt32(), Is.EqualTo(partition.PartitionId), "Partition"); + Assert.That(reader.ReadInt16(), Is.EqualTo((short)partition.Error), "ErrorCode"); + } + } + } + /// /// OffsetCommitRequest => ConsumerGroup *ConsumerGroupGenerationId *MemberId *RetentionTime [TopicName [Partition Offset *TimeStamp Metadata]] /// *ConsumerGroupGenerationId, MemberId is only version 1 (0.8.2) and above diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 6aee1aad..f7346167 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -594,6 +594,63 @@ public void OffsetCommitApiRequest( }); } + /// + /// OffsetCommitResponse => [TopicName [Partition ErrorCode]]] + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition. + /// ErrorCode => int16 -- The error code for the partition, if any. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + [Test] + public void OffsetCommitApiResponse( + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int partitionsPerTopic, + [Values( + ErrorResponseCode.NoError, + ErrorResponseCode.OffsetMetadataTooLargeCode, + ErrorResponseCode.OffsetsLoadInProgressCode, + ErrorResponseCode.NotCoordinatorForConsumerCode, + ErrorResponseCode.IllegalGeneration, + ErrorResponseCode.UnknownMemberId, + ErrorResponseCode.RebalanceInProgress, + ErrorResponseCode.InvalidCommitOffsetSize, + ErrorResponseCode.TopicAuthorizationFailed, + ErrorResponseCode.GroupAuthorizationFailed + )] ErrorResponseCode errorCode) + { + var clientId = nameof(OffsetCommitApiResponse); + var correlationId = clientId.GetHashCode(); + + byte[] data = null; + using (var stream = new MemoryStream()) { + var writer = new BigEndianBinaryWriter(stream); + writer.WriteResponseHeader(correlationId); + + writer.Write(topicsPerRequest); + for (var t = 0; t < topicsPerRequest; t++) { + writer.Write(topic + t); + writer.Write(partitionsPerTopic); + for (var p = 0; p < partitionsPerTopic; p++) { + writer.Write(p); + writer.Write((short) errorCode); + } + } + data = new byte[stream.Position]; + Buffer.BlockCopy(stream.GetBuffer(), 0, data, 0, data.Length); + } + + var request = new OffsetCommitRequest {ApiVersion = 0}; + var responses = request.Decode(data); + // doesn't include the size in the decode -- the framework deals with it, I'd assume + data.PrefixWithInt32Length().AssertProtocol( + reader => { + reader.AssertResponseHeader(correlationId); + reader.AssertOffsetCommitResponse(responses); + }); + } + private List GenerateMessages(int count, byte version) { var randomizer = new Randomizer(); From 3f6eb3aa272cffb09f20ef767999db00de443196 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 16:48:12 -0400 Subject: [PATCH 12/19] Add protocol tests for OffsetFetch --- .../Unit/ProtocolAssertionExtensions.cs | 48 ++++++++ src/kafka-tests/Unit/ProtocolByteTests.cs | 104 ++++++++++++++++++ 2 files changed, 152 insertions(+) diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index 4d48370a..cbd8291d 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,6 +11,54 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { + /// + /// OffsetFetchResponse => [TopicName [Partition Offset Metadata ErrorCode]] + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition. + /// Offset => int64 -- The offset, or -1 if none exists. + /// Metadata => string -- The metadata associated with the topic and partition. + /// ErrorCode => int16 -- The error code for the partition, if any. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + public static void AssertOffsetFetchResponse(this BigEndianBinaryReader reader, IEnumerable response) + { + var responses = response.GroupBy(r => r.Topic).ToList(); + Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); + foreach (var payload in responses) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Key), "TopicName"); + var partitions = payload.ToList(); + Assert.That(reader.ReadInt32(), Is.EqualTo(partitions.Count), "[Partition]"); + foreach (var partition in partitions) { + Assert.That(reader.ReadInt32(), Is.EqualTo(partition.PartitionId), "Partition"); + Assert.That(reader.ReadInt64(), Is.EqualTo(partition.Offset), "Offset"); + Assert.That(reader.ReadString(), Is.EqualTo(partition.MetaData), "Metadata"); + Assert.That(reader.ReadInt16(), Is.EqualTo((short)partition.Error), "ErrorCode"); + } + } + } + + + /// + /// OffsetFetchRequest => ConsumerGroup [TopicName [Partition]] + /// ConsumerGroup => string -- The consumer group id. + /// TopicName => string -- The topic to commit. + /// Partition => int32 -- The partition id. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + public static void AssertOffsetFetchRequest(this BigEndianBinaryReader reader, OffsetFetchRequest request) + { + Assert.That(reader.ReadString(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); + + Assert.That(reader.ReadInt32(), Is.EqualTo(request.Topics.Count), "[TopicName]"); + foreach (var payload in request.Topics) { + Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model + Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); + } + } + /// /// OffsetCommitResponse => [TopicName [Partition ErrorCode]]] /// TopicName => string -- The name of the topic. diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index f7346167..b28f4374 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -651,6 +651,110 @@ public void OffsetCommitApiResponse( }); } + /// + /// OffsetFetchRequest => ConsumerGroup [TopicName [Partition]] + /// ConsumerGroup => string -- The consumer group id. + /// TopicName => string -- The topic to commit. + /// Partition => int32 -- The partition id. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + [Test] + public void OffsetFetchApiRequest( + [Values("group1", "group2")] string groupId, + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(5)] int maxPartitions) + { + var clientId = nameof(OffsetFetchApiRequest); + + var request = new OffsetFetchRequest { + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + ConsumerGroup = groupId, + Topics = new List(), + ApiVersion = 0 + }; + + for (var t = 0; t < topicsPerRequest; t++) { + var payload = new OffsetFetch { + Topic = topic + t, + PartitionId = t % maxPartitions + }; + request.Topics.Add(payload); + } + + var data = request.Encode(); + + data.Buffer.AssertProtocol( + reader => + { + reader.AssertRequestHeader(request); + reader.AssertOffsetFetchRequest(request); + }); + } + + /// + /// OffsetFetchResponse => [TopicName [Partition Offset Metadata ErrorCode]] + /// TopicName => string -- The name of the topic. + /// Partition => int32 -- The id of the partition. + /// Offset => int64 -- The offset, or -1 if none exists. + /// Metadata => string -- The metadata associated with the topic and partition. + /// ErrorCode => int16 -- The error code for the partition, if any. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + [Test] + public void OffsetFetchApiResponse( + [Values("test", "a really long name, with spaces and punctuation!")] string topic, + [Values(1, 10)] int topicsPerRequest, + [Values(1, 5)] int partitionsPerTopic, + [Values( + ErrorResponseCode.NoError, + ErrorResponseCode.UnknownTopicOrPartition, + ErrorResponseCode.OffsetsLoadInProgressCode, + ErrorResponseCode.NotCoordinatorForConsumerCode, + ErrorResponseCode.IllegalGeneration, + ErrorResponseCode.UnknownMemberId, + ErrorResponseCode.TopicAuthorizationFailed, + ErrorResponseCode.GroupAuthorizationFailed + )] ErrorResponseCode errorCode) + { + var clientId = nameof(OffsetFetchApiResponse); + var correlationId = clientId.GetHashCode(); + var randomizer = new Randomizer(); + + byte[] data = null; + using (var stream = new MemoryStream()) { + var writer = new BigEndianBinaryWriter(stream); + writer.WriteResponseHeader(correlationId); + + writer.Write(topicsPerRequest); + for (var t = 0; t < topicsPerRequest; t++) { + writer.Write(topic + t); + writer.Write(partitionsPerTopic); + for (var p = 0; p < partitionsPerTopic; p++) { + writer.Write(p); + var offset = (long)randomizer.Next(int.MinValue, int.MaxValue); + writer.Write(offset); + writer.Write(offset >= 0 ? topic : string.Empty); + writer.Write((short) errorCode); + } + } + data = new byte[stream.Position]; + Buffer.BlockCopy(stream.GetBuffer(), 0, data, 0, data.Length); + } + + var request = new OffsetFetchRequest {ApiVersion = 0}; + var responses = request.Decode(data); + // doesn't include the size in the decode -- the framework deals with it, I'd assume + data.PrefixWithInt32Length().AssertProtocol( + reader => { + reader.AssertResponseHeader(correlationId); + reader.AssertOffsetFetchResponse(responses); + }); + } + private List GenerateMessages(int count, byte version) { var randomizer = new Randomizer(); From ec33c848ac70dce4c6f86716c18d9c74a01eb291 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 20:56:36 -0400 Subject: [PATCH 13/19] Add protocol tests for GroupCoordinatorApi --- .../Unit/ProtocolAssertionExtensions.cs | 29 +++++++- src/kafka-tests/Unit/ProtocolByteTests.cs | 73 +++++++++++++++++++ 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index cbd8291d..d9bc71f7 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -11,6 +11,34 @@ namespace kafka_tests.Unit { public static class ProtocolAssertionExtensions { + /// + /// GroupCoordinatorResponse => ErrorCode CoordinatorId CoordinatorHost CoordinatorPort + /// ErrorCode => int16 -- The error code. + /// CoordinatorId => int32 -- The broker id. + /// CoordinatorHost => string -- The hostname of the broker. + /// CoordinatorPort => int32 -- The port on which the broker accepts requests. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + public static void AssertGroupCoordinatorResponse(this BigEndianBinaryReader reader, ConsumerMetadataResponse response) + { + Assert.That(reader.ReadInt16(), Is.EqualTo(response.Error), "ErrorCode"); + Assert.That(reader.ReadInt32(), Is.EqualTo(response.CoordinatorId), "CoordinatorId"); + Assert.That(reader.ReadString(), Is.EqualTo(response.CoordinatorHost), "CoordinatorHost"); + Assert.That(reader.ReadInt32(), Is.EqualTo(response.CoordinatorPort), "CoordinatorPort"); + } + + /// + /// GroupCoordinatorRequest => GroupId + /// GroupId => string -- The consumer group id. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + public static void AssertGroupCoordinatorRequest(this BigEndianBinaryReader reader, ConsumerMetadataRequest request) + { + Assert.That(reader.ReadString(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); + } + /// /// OffsetFetchResponse => [TopicName [Partition Offset Metadata ErrorCode]] /// TopicName => string -- The name of the topic. @@ -38,7 +66,6 @@ public static void AssertOffsetFetchResponse(this BigEndianBinaryReader reader, } } - /// /// OffsetFetchRequest => ConsumerGroup [TopicName [Partition]] /// ConsumerGroup => string -- The consumer group id. diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index b28f4374..79fce095 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -755,6 +755,79 @@ public void OffsetFetchApiResponse( }); } + /// + /// GroupCoordinatorRequest => GroupId + /// GroupId => string -- The consumer group id. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + [Test] + public void GroupCoordinatorApiRequest([Values("group1", "group2")] string groupId) + { + var clientId = nameof(GroupCoordinatorApiRequest); + + var request = new ConsumerMetadataRequest { + ClientId = clientId, + CorrelationId = clientId.GetHashCode(), + ConsumerGroup = groupId, + ApiVersion = 0 + }; + + var data = request.Encode(); + + data.Buffer.AssertProtocol( + reader => + { + reader.AssertRequestHeader(request); + reader.AssertGroupCoordinatorRequest(request); + }); + } + + /// + /// GroupCoordinatorResponse => ErrorCode CoordinatorId CoordinatorHost CoordinatorPort + /// ErrorCode => int16 -- The error code. + /// CoordinatorId => int32 -- The broker id. + /// CoordinatorHost => string -- The hostname of the broker. + /// CoordinatorPort => int32 -- The port on which the broker accepts requests. + /// + /// From https://cwiki.apache.org/confluence/display/KAFKA/A+Guide+To+The+Kafka+Protocol#AGuideToTheKafkaProtocol-OffsetCommit/FetchAPI + /// + [Test] + public void GroupCoordinatorApiResponse( + [Values( + ErrorResponseCode.NoError, + ErrorResponseCode.ConsumerCoordinatorNotAvailableCode, + ErrorResponseCode.GroupAuthorizationFailed + )] ErrorResponseCode errorCode, + [Values(0, 1)] int coordinatorId + ) + { + var clientId = nameof(GroupCoordinatorApiResponse); + var correlationId = clientId.GetHashCode(); + + byte[] data = null; + using (var stream = new MemoryStream()) { + var writer = new BigEndianBinaryWriter(stream); + writer.WriteResponseHeader(correlationId); + writer.Write((short)errorCode); + writer.Write(coordinatorId); + writer.Write("broker-" + coordinatorId); + writer.Write(9092 + coordinatorId); + + data = new byte[stream.Position]; + Buffer.BlockCopy(stream.GetBuffer(), 0, data, 0, data.Length); + } + + var request = new ConsumerMetadataRequest {ApiVersion = 0}; + var responses = request.Decode(data).Single(); + // doesn't include the size in the decode -- the framework deals with it, I'd assume + data.PrefixWithInt32Length().AssertProtocol( + reader => { + reader.AssertResponseHeader(correlationId); + reader.AssertGroupCoordinatorResponse(responses); + }); + } + private List GenerateMessages(int count, byte version) { var randomizer = new Randomizer(); From 673d5f63d00193322f4cc1203abfc41a9145137a Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 21:18:20 -0400 Subject: [PATCH 14/19] remove c#6 auto-property initializer --- src/kafka-net/Protocol/BaseRequest.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/kafka-net/Protocol/BaseRequest.cs b/src/kafka-net/Protocol/BaseRequest.cs index 7332c0af..be611e92 100644 --- a/src/kafka-net/Protocol/BaseRequest.cs +++ b/src/kafka-net/Protocol/BaseRequest.cs @@ -16,6 +16,7 @@ public abstract class BaseRequest protected const int ReplicaId = -1; private string _clientId = "Kafka-Net"; private int _correlationId = 1; + private short _apiVersion = 0; /// /// Descriptive name of the source of the messages sent to kafka @@ -31,7 +32,7 @@ public abstract class BaseRequest /// /// This is a numeric version number for the api request. It allows the server to properly interpret the request as the protocol evolves. Responses will always be in the format corresponding to the request version. /// - public short ApiVersion { get; set; } = 0; + public short ApiVersion { get { return _apiVersion; } set { _apiVersion = value; } } /// /// Flag which tells the broker call to expect a response for this request. From 87fb33f5912353b9b90f632590c63d03b777675d Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 21:20:40 -0400 Subject: [PATCH 15/19] use string.format rather than string interpolation --- src/kafka-tests/Unit/ProtocolAssertionExtensions.cs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index d9bc71f7..feee08b6 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -305,7 +305,8 @@ public static void AssertFetchResponse(this BigEndianBinaryReader reader, int ve var finalPosition = reader.ReadInt32() + reader.Position; reader.AssertMessageSet(version, payload.Messages); - Assert.That(reader.Position, Is.EqualTo(finalPosition), $"MessageSetSize was {finalPosition - 4} but ended in a different spot."); + Assert.That(reader.Position, Is.EqualTo(finalPosition), + string.Format("MessageSetSize was {0} but ended in a different spot.", finalPosition - 4)); } } @@ -422,7 +423,8 @@ public static void AssertProduceRequest(this BigEndianBinaryReader reader, Produ var finalPosition = reader.ReadInt32() + reader.Position; reader.AssertMessageSet(request.ApiVersion, payload.Messages); - Assert.That(reader.Position, Is.EqualTo(finalPosition), $"MessageSetSize was {finalPosition - 4} but ended in a different spot."); + Assert.That(reader.Position, Is.EqualTo(finalPosition), + string.Format("MessageSetSize was {0} but ended in a different spot.", finalPosition - 4)); } } @@ -445,7 +447,8 @@ public static void AssertMessageSet(this BigEndianBinaryReader reader, int versi } var finalPosition = reader.ReadInt32() + reader.Position; reader.AssertMessage(version, message); - Assert.That(reader.Position, Is.EqualTo(finalPosition), $"MessageSize was {finalPosition - 4} but ended in a different spot."); + Assert.That(reader.Position, Is.EqualTo(finalPosition), + string.Format("MessageSize was {0} but ended in a different spot.", finalPosition - 4)); } } @@ -584,7 +587,8 @@ public static void AssertProtocol(this byte[] bytes, Action Date: Mon, 29 Aug 2016 21:24:12 -0400 Subject: [PATCH 16/19] remove use of nameof expression --- src/kafka-tests/Unit/ProtocolByteTests.cs | 28 +++++++++++------------ 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 79fce095..af8f9470 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -72,7 +72,7 @@ public void ProduceApiRequest( [Values(1, 5)] int totalPartitions, [Values(1, 2, 3)] int messagesPerSet) { - var clientId = nameof(ProduceApiRequest); + var clientId = "ProduceApiRequest"; var request = new ProduceRequest { Acks = acks, @@ -136,7 +136,7 @@ public void ProduceApiResponse( [Values(0, 1234, 100000)] int throttleTime) { var randomizer = new Randomizer(); - var clientId = nameof(ProduceApiResponse); + var clientId = "ProduceApiResponse"; var correlationId = clientId.GetHashCode(); byte[] data = null; @@ -204,7 +204,7 @@ public void FetchApiRequest( [Values(64000, 25600000)] int maxBytes) { var randomizer = new Randomizer(); - var clientId = nameof(FetchApiRequest); + var clientId = "FetchApiRequest"; var request = new FetchRequest { ClientId = clientId, @@ -265,7 +265,7 @@ public void FetchApiResponse( ) { var randomizer = new Randomizer(); - var clientId = nameof(FetchApiResponse); + var clientId = "FetchApiResponse"; var correlationId = clientId.GetHashCode(); byte[] data = null; @@ -324,7 +324,7 @@ public void OffsetsApiRequest( [Values(-2, -1, 123456, 10000000)] long time, [Values(1, 10)] int maxOffsets) { - var clientId = nameof(OffsetsApiRequest); + var clientId = "OffsetsApiRequest"; var request = new OffsetRequest { ClientId = clientId, @@ -376,7 +376,7 @@ public void OffsetsApiResponse( [Values(1, 5)] int offsetsPerPartition) { var randomizer = new Randomizer(); - var clientId = nameof(OffsetsApiResponse); + var clientId = "OffsetsApiResponse"; var correlationId = clientId.GetHashCode(); byte[] data = null; @@ -420,7 +420,7 @@ public void MetadataApiRequest( [Values("test", "a really long name, with spaces and punctuation!")] string topic, [Values(0, 1, 10)] int topicsPerRequest) { - var clientId = nameof(MetadataApiRequest); + var clientId = "MetadataApiRequest"; var request = new MetadataRequest { ClientId = clientId, @@ -477,7 +477,7 @@ public void MetadataApiResponse( )] ErrorResponseCode errorCode) { var randomizer = new Randomizer(); - var clientId = nameof(MetadataApiResponse); + var clientId = "MetadataApiResponse"; var correlationId = clientId.GetHashCode(); byte[] data = null; @@ -557,7 +557,7 @@ public void OffsetCommitApiRequest( [Values(1, 10)] int maxOffsets, [Values(null, "something useful for the client")] string metadata) { - var clientId = nameof(OffsetCommitApiRequest); + var clientId = "OffsetCommitApiRequest"; var randomizer = new Randomizer(); var request = new OffsetCommitRequest { @@ -620,7 +620,7 @@ public void OffsetCommitApiResponse( ErrorResponseCode.GroupAuthorizationFailed )] ErrorResponseCode errorCode) { - var clientId = nameof(OffsetCommitApiResponse); + var clientId = "OffsetCommitApiResponse"; var correlationId = clientId.GetHashCode(); byte[] data = null; @@ -666,7 +666,7 @@ public void OffsetFetchApiRequest( [Values(1, 10)] int topicsPerRequest, [Values(5)] int maxPartitions) { - var clientId = nameof(OffsetFetchApiRequest); + var clientId = "OffsetFetchApiRequest"; var request = new OffsetFetchRequest { ClientId = clientId, @@ -720,7 +720,7 @@ public void OffsetFetchApiResponse( ErrorResponseCode.GroupAuthorizationFailed )] ErrorResponseCode errorCode) { - var clientId = nameof(OffsetFetchApiResponse); + var clientId = "OffsetFetchApiResponse"; var correlationId = clientId.GetHashCode(); var randomizer = new Randomizer(); @@ -764,7 +764,7 @@ public void OffsetFetchApiResponse( [Test] public void GroupCoordinatorApiRequest([Values("group1", "group2")] string groupId) { - var clientId = nameof(GroupCoordinatorApiRequest); + var clientId = "GroupCoordinatorApiRequest"; var request = new ConsumerMetadataRequest { ClientId = clientId, @@ -802,7 +802,7 @@ public void GroupCoordinatorApiResponse( [Values(0, 1)] int coordinatorId ) { - var clientId = nameof(GroupCoordinatorApiResponse); + var clientId = "GroupCoordinatorApiResponse"; var correlationId = clientId.GetHashCode(); byte[] data = null; From 10c7f18fcedea54009f47594deb5b4ef7e9394d9 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 21:46:53 -0400 Subject: [PATCH 17/19] revert some changes to BigEndianBinaryReader/Writer --- src/kafka-net/Common/BigEndianBinaryReader.cs | 5 ----- src/kafka-net/Common/BigEndianBinaryWriter.cs | 2 +- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/src/kafka-net/Common/BigEndianBinaryReader.cs b/src/kafka-net/Common/BigEndianBinaryReader.cs index 8bdb9604..6d105cf2 100644 --- a/src/kafka-net/Common/BigEndianBinaryReader.cs +++ b/src/kafka-net/Common/BigEndianBinaryReader.cs @@ -104,11 +104,6 @@ public override UInt64 ReadUInt64() return EndianAwareRead(8, BitConverter.ToUInt64); } - public override string ReadString() - { - return ReadInt16String(); - } - public byte[] ReadBytes() { return ReadIntPrefixedBytes(); diff --git a/src/kafka-net/Common/BigEndianBinaryWriter.cs b/src/kafka-net/Common/BigEndianBinaryWriter.cs index edc2dab2..1a5a4664 100644 --- a/src/kafka-net/Common/BigEndianBinaryWriter.cs +++ b/src/kafka-net/Common/BigEndianBinaryWriter.cs @@ -101,7 +101,7 @@ public override void Write(UInt64 value) public override void Write(string value) { - Write(value, StringPrefixEncoding.Int16); + throw new NotSupportedException("Kafka requires specific string length prefix encoding."); } public void Write(byte[] value, StringPrefixEncoding encoding) From 252d47fe2f9dc96ba4a83db545f66dc7faf0e545 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 21:50:46 -0400 Subject: [PATCH 18/19] Use explicit string encoding functions in tests --- .../Unit/ProtocolAssertionExtensions.cs | 42 +++++++++---------- src/kafka-tests/Unit/ProtocolByteTests.cs | 18 ++++---- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs index feee08b6..8deafa35 100644 --- a/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs +++ b/src/kafka-tests/Unit/ProtocolAssertionExtensions.cs @@ -24,7 +24,7 @@ public static void AssertGroupCoordinatorResponse(this BigEndianBinaryReader rea { Assert.That(reader.ReadInt16(), Is.EqualTo(response.Error), "ErrorCode"); Assert.That(reader.ReadInt32(), Is.EqualTo(response.CoordinatorId), "CoordinatorId"); - Assert.That(reader.ReadString(), Is.EqualTo(response.CoordinatorHost), "CoordinatorHost"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(response.CoordinatorHost), "CoordinatorHost"); Assert.That(reader.ReadInt32(), Is.EqualTo(response.CoordinatorPort), "CoordinatorPort"); } @@ -36,7 +36,7 @@ public static void AssertGroupCoordinatorResponse(this BigEndianBinaryReader rea /// public static void AssertGroupCoordinatorRequest(this BigEndianBinaryReader reader, ConsumerMetadataRequest request) { - Assert.That(reader.ReadString(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); } /// @@ -54,13 +54,13 @@ public static void AssertOffsetFetchResponse(this BigEndianBinaryReader reader, var responses = response.GroupBy(r => r.Topic).ToList(); Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); foreach (var payload in responses) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Key), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Key), "TopicName"); var partitions = payload.ToList(); Assert.That(reader.ReadInt32(), Is.EqualTo(partitions.Count), "[Partition]"); foreach (var partition in partitions) { Assert.That(reader.ReadInt32(), Is.EqualTo(partition.PartitionId), "Partition"); Assert.That(reader.ReadInt64(), Is.EqualTo(partition.Offset), "Offset"); - Assert.That(reader.ReadString(), Is.EqualTo(partition.MetaData), "Metadata"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(partition.MetaData), "Metadata"); Assert.That(reader.ReadInt16(), Is.EqualTo((short)partition.Error), "ErrorCode"); } } @@ -76,11 +76,11 @@ public static void AssertOffsetFetchResponse(this BigEndianBinaryReader reader, /// public static void AssertOffsetFetchRequest(this BigEndianBinaryReader reader, OffsetFetchRequest request) { - Assert.That(reader.ReadString(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); Assert.That(reader.ReadInt32(), Is.EqualTo(request.Topics.Count), "[TopicName]"); foreach (var payload in request.Topics) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); } @@ -99,7 +99,7 @@ public static void AssertOffsetCommitResponse(this BigEndianBinaryReader reader, var responses = response.GroupBy(r => r.Topic).ToList(); Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); foreach (var payload in responses) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Key), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Key), "TopicName"); var partitions = payload.ToList(); Assert.That(reader.ReadInt32(), Is.EqualTo(partitions.Count), "[Partition]"); foreach (var partition in partitions) { @@ -128,11 +128,11 @@ public static void AssertOffsetCommitResponse(this BigEndianBinaryReader reader, /// public static void AssertOffsetCommitRequest(this BigEndianBinaryReader reader, OffsetCommitRequest request) { - Assert.That(reader.ReadString(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(request.ConsumerGroup), "ConsumerGroup"); if (request.ApiVersion >= 1) { Assert.That(reader.ReadInt32(), Is.EqualTo(request.GenerationId), "ConsumerGroupGenerationId"); - Assert.That(reader.ReadString(), Is.EqualTo(request.MemberId), "MemberId"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(request.MemberId), "MemberId"); } if (request.ApiVersion >= 2) { var expectedRetention = request.OffsetRetention.HasValue @@ -143,7 +143,7 @@ public static void AssertOffsetCommitRequest(this BigEndianBinaryReader reader, Assert.That(reader.ReadInt32(), Is.EqualTo(request.OffsetCommits.Count), "[TopicName]"); foreach (var payload in request.OffsetCommits) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); Assert.That(reader.ReadInt64(), Is.EqualTo(payload.Offset), "Offset"); @@ -151,7 +151,7 @@ public static void AssertOffsetCommitRequest(this BigEndianBinaryReader reader, if (request.ApiVersion == 1) { Assert.That(reader.ReadInt64(), Is.EqualTo(payload.TimeStamp), "TimeStamp"); } - Assert.That(reader.ReadString(), Is.EqualTo(payload.Metadata), "Metadata"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Metadata), "Metadata"); } } @@ -179,13 +179,13 @@ public static void AssertMetadataResponse(this BigEndianBinaryReader reader, Met Assert.That(reader.ReadInt32(), Is.EqualTo(response.Brokers.Count), "[Broker]"); foreach (var payload in response.Brokers) { Assert.That(reader.ReadInt32(), Is.EqualTo(payload.BrokerId), "NodeId"); - Assert.That(reader.ReadString(), Is.EqualTo(payload.Host), "Host"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Host), "Host"); Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Port), "Port"); } Assert.That(reader.ReadInt32(), Is.EqualTo(response.Topics.Count), "[TopicMetadata]"); foreach (var payload in response.Topics) { Assert.That(reader.ReadInt16(), Is.EqualTo((short)payload.ErrorCode), "TopicErrorCode"); - Assert.That(reader.ReadString(), Is.EqualTo(payload.Name), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Name), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partitions.Count), "[PartitionMetadata]"); foreach (var partition in payload.Partitions) { Assert.That(reader.ReadInt16(), Is.EqualTo((short) partition.ErrorCode), "PartitionErrorCode"); @@ -213,7 +213,7 @@ public static void AssertMetadataRequest(this BigEndianBinaryReader reader, Meta { Assert.That(reader.ReadInt32(), Is.EqualTo(request.Topics.Count), "[TopicName]"); foreach (var payload in request.Topics) { - Assert.That(reader.ReadString(), Is.EqualTo(payload), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload), "TopicName"); } } @@ -233,7 +233,7 @@ public static void AssertOffsetResponse(this BigEndianBinaryReader reader, IEnum var responses = response.ToList(); Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); foreach (var payload in responses) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); Assert.That(reader.ReadInt16(), Is.EqualTo(payload.Error), "ErrorCode"); @@ -265,7 +265,7 @@ public static void AssertOffsetRequest(this BigEndianBinaryReader reader, Offset Assert.That(reader.ReadInt32(), Is.EqualTo(request.Offsets.Count), "[TopicName]"); foreach (var payload in request.Offsets) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); @@ -297,7 +297,7 @@ public static void AssertFetchResponse(this BigEndianBinaryReader reader, int ve } Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); foreach (var payload in responses) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); Assert.That(reader.ReadInt16(), Is.EqualTo(payload.Error), "Error"); @@ -339,7 +339,7 @@ public static void AssertFetchRequest(this BigEndianBinaryReader reader, FetchRe Assert.That(reader.ReadInt32(), Is.EqualTo(request.Fetches.Count), "[TopicName]"); foreach (var payload in request.Fetches) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); @@ -372,7 +372,7 @@ public static void AssertProduceResponse(this BigEndianBinaryReader reader, int var responses = response.ToList(); Assert.That(reader.ReadInt32(), Is.EqualTo(responses.Count), "[TopicName]"); foreach (var payload in responses) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.PartitionId), "Partition"); Assert.That(reader.ReadInt16(), Is.EqualTo(payload.Error), "Error"); @@ -417,7 +417,7 @@ public static void AssertProduceRequest(this BigEndianBinaryReader reader, Produ Assert.That(reader.ReadInt32(), Is.EqualTo(request.Payload.Count), "[topic_data]"); foreach (var payload in request.Payload) { - Assert.That(reader.ReadString(), Is.EqualTo(payload.Topic), "TopicName"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(payload.Topic), "TopicName"); Assert.That(reader.ReadInt32(), Is.EqualTo(1), "[Partition]"); // this is a mismatch between the protocol and the object model Assert.That(reader.ReadInt32(), Is.EqualTo(payload.Partition), "Partition"); @@ -560,7 +560,7 @@ public static void AssertRequestHeader(this BigEndianBinaryReader reader, ApiKey Assert.That(reader.ReadInt16(), Is.EqualTo((short) apiKey), "api_key"); Assert.That(reader.ReadInt16(), Is.EqualTo(version), "api_version"); Assert.That(reader.ReadInt32(), Is.EqualTo(correlationId), "correlation_id"); - Assert.That(reader.ReadString(), Is.EqualTo(clientId), "client_id"); + Assert.That(reader.ReadInt16String(), Is.EqualTo(clientId), "client_id"); } /// diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index af8f9470..982ad819 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -146,7 +146,7 @@ public void ProduceApiResponse( writer.Write(topicsPerRequest); for (var t = 0; t < topicsPerRequest; t++) { - writer.Write(topic + t); + writer.Write(topic + t, StringPrefixEncoding.Int16); writer.Write(1); // partitionsPerTopic writer.Write(t % totalPartitions); writer.Write((short)errorCode); @@ -278,7 +278,7 @@ public void FetchApiResponse( } writer.Write(topicsPerRequest); for (var t = 0; t < topicsPerRequest; t++) { - writer.Write(topic + t); + writer.Write(topic + t, StringPrefixEncoding.Int16); writer.Write(1); // partitionsPerTopic writer.Write(t % totalPartitions); writer.Write((short)errorCode); @@ -386,7 +386,7 @@ public void OffsetsApiResponse( writer.Write(topicsPerRequest); for (var t = 0; t < topicsPerRequest; t++) { - writer.Write(topic + t); + writer.Write(topic + t, StringPrefixEncoding.Int16); writer.Write(1); // partitionsPerTopic writer.Write(t % totalPartitions); writer.Write((short)errorCode); @@ -488,14 +488,14 @@ public void MetadataApiResponse( writer.Write(brokersPerRequest); for (var b = 0; b < brokersPerRequest; b++) { writer.Write(b); - writer.Write("broker-" + b); + writer.Write("broker-" + b, StringPrefixEncoding.Int16); writer.Write(9092 + b); } writer.Write(topicsPerRequest); for (var t = 0; t < topicsPerRequest; t++) { writer.Write((short) errorCode); - writer.Write(topic + t); + writer.Write(topic + t, StringPrefixEncoding.Int16); writer.Write(partitionsPerTopic); for (var p = 0; p < partitionsPerTopic; p++) { writer.Write((short) errorCode); @@ -630,7 +630,7 @@ public void OffsetCommitApiResponse( writer.Write(topicsPerRequest); for (var t = 0; t < topicsPerRequest; t++) { - writer.Write(topic + t); + writer.Write(topic + t, StringPrefixEncoding.Int16); writer.Write(partitionsPerTopic); for (var p = 0; p < partitionsPerTopic; p++) { writer.Write(p); @@ -731,13 +731,13 @@ public void OffsetFetchApiResponse( writer.Write(topicsPerRequest); for (var t = 0; t < topicsPerRequest; t++) { - writer.Write(topic + t); + writer.Write(topic + t, StringPrefixEncoding.Int16); writer.Write(partitionsPerTopic); for (var p = 0; p < partitionsPerTopic; p++) { writer.Write(p); var offset = (long)randomizer.Next(int.MinValue, int.MaxValue); writer.Write(offset); - writer.Write(offset >= 0 ? topic : string.Empty); + writer.Write(offset >= 0 ? topic : string.Empty, StringPrefixEncoding.Int16); writer.Write((short) errorCode); } } @@ -811,7 +811,7 @@ public void GroupCoordinatorApiResponse( writer.WriteResponseHeader(correlationId); writer.Write((short)errorCode); writer.Write(coordinatorId); - writer.Write("broker-" + coordinatorId); + writer.Write("broker-" + coordinatorId, StringPrefixEncoding.Int16); writer.Write(9092 + coordinatorId); data = new byte[stream.Position]; From caacda12354bdb2783d8e07885b0885214be4e44 Mon Sep 17 00:00:00 2001 From: Andrew Date: Mon, 29 Aug 2016 21:56:31 -0400 Subject: [PATCH 19/19] more reasonable number of tests --- src/kafka-tests/Unit/ProtocolByteTests.cs | 45 ++++++++--------------- 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/src/kafka-tests/Unit/ProtocolByteTests.cs b/src/kafka-tests/Unit/ProtocolByteTests.cs index 982ad819..bfc6db7c 100644 --- a/src/kafka-tests/Unit/ProtocolByteTests.cs +++ b/src/kafka-tests/Unit/ProtocolByteTests.cs @@ -65,12 +65,12 @@ public class ProtocolByteTests [Test] public void ProduceApiRequest( [Values(0, 1, 2)] short version, - [Values(0, 1, 2, -1)] short acks, - [Values(0, 1, 1000)] int timeoutMilliseconds, + [Values(0, 2, -1)] short acks, + [Values(0, 1000)] int timeoutMilliseconds, [Values("test", "a really long name, with spaces and punctuation!")] string topic, [Values(1, 10)] int topicsPerRequest, [Values(1, 5)] int totalPartitions, - [Values(1, 2, 3)] int messagesPerSet) + [Values(3)] int messagesPerSet) { var clientId = "ProduceApiRequest"; @@ -124,16 +124,15 @@ public void ProduceApiRequest( [Test] public void ProduceApiResponse( [Values(0, 1, 2)] short version, - [Values(-1, 0, 123456, 10000000)] long timestampMilliseconds, + [Values(-1, 0, 10000000)] long timestampMilliseconds, [Values("test", "a really long name, with spaces and punctuation!")] string topic, [Values(1, 10)] int topicsPerRequest, [Values(1, 5)] int totalPartitions, [Values( ErrorResponseCode.NoError, - ErrorResponseCode.InvalidMessage, - ErrorResponseCode.NotCoordinatorForConsumerCode + ErrorResponseCode.InvalidMessage )] ErrorResponseCode errorCode, - [Values(0, 1234, 100000)] int throttleTime) + [Values(0, 100000)] int throttleTime) { var randomizer = new Randomizer(); var clientId = "ProduceApiResponse"; @@ -196,12 +195,12 @@ public void ProduceApiResponse( [Test] public void FetchApiRequest( [Values(0, 1, 2)] short version, - [Values(0, 10, 100)] int timeoutMilliseconds, + [Values(0, 100)] int timeoutMilliseconds, [Values(0, 64000)] int minBytes, [Values("test", "a really long name, with spaces and punctuation!")] string topic, [Values(1, 10)] int topicsPerRequest, [Values(1, 5)] int totalPartitions, - [Values(64000, 25600000)] int maxBytes) + [Values(25600000)] int maxBytes) { var randomizer = new Randomizer(); var clientId = "FetchApiRequest"; @@ -256,12 +255,9 @@ public void FetchApiResponse( [Values(1, 5)] int totalPartitions, [Values( ErrorResponseCode.NoError, - ErrorResponseCode.OffsetOutOfRange, - ErrorResponseCode.NotLeaderForPartition, - ErrorResponseCode.ReplicaNotAvailable, - ErrorResponseCode.Unknown + ErrorResponseCode.OffsetOutOfRange )] ErrorResponseCode errorCode, - [Values(2, 3)] int messagesPerSet + [Values(3)] int messagesPerSet ) { var randomizer = new Randomizer(); @@ -470,10 +466,7 @@ public void MetadataApiResponse( [Values(1, 5)] int partitionsPerTopic, [Values( ErrorResponseCode.NoError, - ErrorResponseCode.UnknownTopicOrPartition, - ErrorResponseCode.LeaderNotAvailable, - ErrorResponseCode.InvalidTopic, - ErrorResponseCode.TopicAuthorizationFailed + ErrorResponseCode.UnknownTopicOrPartition )] ErrorResponseCode errorCode) { var randomizer = new Randomizer(); @@ -549,12 +542,12 @@ public void MetadataApiResponse( public void OffsetCommitApiRequest( [Values(0, 1, 2)] short version, [Values("group1", "group2")] string groupId, - [Values(0, 1, 2)] int generation, - [Values(-1, 1024, 20000)] int retentionTime, + [Values(0, 5)] int generation, + [Values(-1, 20000)] int retentionTime, [Values("test", "a really long name, with spaces and punctuation!")] string topic, [Values(1, 10)] int topicsPerRequest, [Values(5)] int maxPartitions, - [Values(1, 10)] int maxOffsets, + [Values(10)] int maxOffsets, [Values(null, "something useful for the client")] string metadata) { var clientId = "OffsetCommitApiRequest"; @@ -609,15 +602,7 @@ public void OffsetCommitApiResponse( [Values(1, 5)] int partitionsPerTopic, [Values( ErrorResponseCode.NoError, - ErrorResponseCode.OffsetMetadataTooLargeCode, - ErrorResponseCode.OffsetsLoadInProgressCode, - ErrorResponseCode.NotCoordinatorForConsumerCode, - ErrorResponseCode.IllegalGeneration, - ErrorResponseCode.UnknownMemberId, - ErrorResponseCode.RebalanceInProgress, - ErrorResponseCode.InvalidCommitOffsetSize, - ErrorResponseCode.TopicAuthorizationFailed, - ErrorResponseCode.GroupAuthorizationFailed + ErrorResponseCode.OffsetMetadataTooLargeCode )] ErrorResponseCode errorCode) { var clientId = "OffsetCommitApiResponse";