Parse Hashes out of Movie Files (#2031)

* proof of concept for our movie parser to return hash data.  This sketches out the API, still need to actually parse the hashes out of various file formats

* bk2 hash parsing

* more bk2 tests

* ltm hash parsing

* oops

* fm2 hash parsing

* another test

TASVideos.Parsers/Parsers/Bk2.cs

@@ -1,4 +1,6 @@
-namespace TASVideos.MovieParsers.Parsers;
+using System.Security.Cryptography;
+
+namespace TASVideos.MovieParsers.Parsers;
 
 [FileExtension("bk2")]
 internal class Bk2 : Parser, IParser
@@ -69,6 +71,23 @@ public async Task<IParseResult> Parse(Stream file, long length)
 				return Error("Could not determine the System Code");
 			}
 
+			string romHash = header.GetValueFor("SHA1");
+			if (string.IsNullOrEmpty(romHash))
+			{
+				romHash = header.GetValueFor("MD5");
+			}
+
+			HashType? hashType = romHash.Length switch {
+				2 * SHA1.HashSizeInBytes => HashType.Sha1,
+				2 * MD5.HashSizeInBytes => HashType.Md5,
+				8/* 2 * Crc32.HashLengthInBytes w/ System.IO.Hashing */ => HashType.Crc32,
+				_ => null
+			};
+			if (hashType is not null)
+			{
+				result.Hashes[hashType.Value] = romHash.ToLower();
+			}
+
 			int? rerecordVal = header.GetPositiveIntFor(Keys.RerecordCount);
 			if (rerecordVal.HasValue)
 			{

TASVideos.Parsers/Parsers/Fm2.cs

@@ -1,4 +1,6 @@
-namespace TASVideos.MovieParsers.Parsers;
+using System.Text;
+
+namespace TASVideos.MovieParsers.Parsers;
 
 [FileExtension("fm2")]
 internal class Fm2 : Parser, IParser
@@ -55,9 +57,43 @@ public async Task<IParseResult> Parse(Stream file, long length)
 			result.StartType = MovieStartType.Savestate;
 		}
 
+		var hashLine = header.GetValueFor(Keys.RomChecksum);
+		if (!string.IsNullOrWhiteSpace(hashLine))
+		{
+			var hashSplit = hashLine.Split(':');
+			var base64Line = hashSplit.Length == 2 ? hashSplit[1] : "";
+			if (!string.IsNullOrWhiteSpace(base64Line))
+			{
+				try
+				{
+					byte[] data = Convert.FromBase64String(base64Line);
+					string hash = BytesToHexString(data.AsSpan());
+					if (hash.Length == 32)
+					{
+						result.Hashes.Add(HashType.Md5, hash.ToLower());
+					}
+				}
+				catch
+				{
+					// Treat an invalid base64 hash as a missing hash
+				}
+			}
+		}
+
 		return result;
 	}
 
+	private static string BytesToHexString(ReadOnlySpan<byte> bytes)
+	{
+		StringBuilder sb = new(capacity: 2 * bytes.Length, maxCapacity: 2 * bytes.Length);
+		foreach (var b in bytes)
+		{
+			sb.Append($"{b:X2}");
+		}
+
+		return sb.ToString();
+	}
+
 	private static class Keys
 	{
 		public const string RerecordCount = "rerecordcount";
@@ -66,5 +102,6 @@ private static class Keys
 		public const string Length = "length";
 		public const string Fds = "fds";
 		public const string StartsFromSavestate = "savestate";
+		public const string RomChecksum = "romChecksum";
 	}
 }

TASVideos.Parsers/Parsers/Ltm.cs

@@ -17,7 +17,7 @@ internal class Ltm : Parser, IParser
 	private const string VariableFramerateHeader = "variable_framerate=";
 	private const string LengthSecondsHeader = "length_sec=";
 	private const string LengthNanosecondsHeader = "length_nsec=";
-
+	private const string Md5 = "md5=";
 	public async Task<IParseResult> Parse(Stream file, long length)
 	{
 		var result = new SuccessResult(FileExtension)
@@ -94,6 +94,14 @@ public async Task<IParseResult> Parse(Stream file, long length)
 						{
 							lengthNanoseconds = ParseDoubleFromConfig(s);
 						}
+						else if (s.StartsWith(Md5))
+						{
+							var md5 = ParseStringFromConfig(s);
+							if (md5.Length == 32)
+							{
+								result.Hashes.Add(HashType.Md5, md5.ToLower());
+							}
+						}
 					}
 
 					break;

TASVideos.Parsers/Result/ErrorResult.cs

@@ -18,4 +18,5 @@ internal class ErrorResult(string errorMsg) : IParseResult
 	public double? FrameRateOverride => null;
 	public long? CycleCount => null;
 	public string? Annotations => null;
+	public Dictionary<HashType, string> Hashes => [];
 }

TASVideos.Parsers/Result/IParseResult.cs

@@ -73,4 +73,11 @@ public interface IParseResult
 	/// Gets the annotations. These can be general comments, or other user entered descriptions supported by the file format.
 	/// </summary>
 	string? Annotations { get; }
+
+	Dictionary<HashType, string> Hashes { get; }
+}
+
+public enum HashType
+{
+	Md5, Sha1, Sha256, Crc32
 }

TASVideos.Parsers/Result/SuccessResult.cs

@@ -20,6 +20,8 @@ internal class SuccessResult(string fileExtension) : IParseResult
 	public string? Annotations { get; internal set; }
 
 	internal List<ParseWarnings> WarningList { get; } = [];
+
+	public Dictionary<HashType, string> Hashes { get; } = [];
 }
 
 internal static class ParseResultExtensions

tests/TASVideos.Core.Tests/Services/TestParseResult.cs

@@ -16,4 +16,5 @@ internal class TestParseResult : IParseResult
 	public double? FrameRateOverride { get; init; }
 	public long? CycleCount { get; init; }
 	public string? Annotations { get; init; }
+	public Dictionary<HashType, string> Hashes { get; init; } = new Dictionary<HashType, string>();
 }

tests/TASVideos.Core.Tests/Services/UserFilesTests.cs

@@ -256,5 +256,6 @@ private class TestParseResult : IParseResult
 		public double? FrameRateOverride => null;
 		public long? CycleCount => null;
 		public string? Annotations => null;
+		public Dictionary<HashType, string> Hashes { get; init; } = new Dictionary<HashType, string>();
 	}
 }

tests/TASVideos.MovieParsers.Tests/Bk2ParserTests.cs

@@ -331,4 +331,28 @@ public async Task Comments_ParseAsAnnotations()
 		var lines = result.Annotations.SplitWithEmpty("\n");
 		Assert.AreEqual(2, lines.Length);
 	}
+
+	[TestMethod]
+	[DataRow("hash-crc32-as-sha1", HashType.Crc32, "26b9ba0c")]
+	[DataRow("hash-crc32-as-md5", HashType.Crc32, "26b9ba0c")]
+	[DataRow("hash-md5-as-sha1", HashType.Md5, "811b027eaf99c2def7b933c5208636de")]
+	[DataRow("hash-md5", HashType.Md5, "811b027eaf99c2def7b933c5208636de")]
+	[DataRow("hash-sha1", HashType.Sha1, "ea343f4e445a9050d4b4fbac2c77d0693b1d0922")]
+	[DataRow("hash-sha1-as-md5", HashType.Sha1, "ea343f4e445a9050d4b4fbac2c77d0693b1d0922")]
+	public async Task Hashes(string filename, HashType hashType, string hash)
+	{
+		var result = await _bk2Parser.Parse(Embedded(filename + ".bk2"), EmbeddedLength(filename + ".bk2"));
+		Assert.AreEqual(1, result.Hashes.Count);
+		Assert.AreEqual(hashType, result.Hashes.First().Key);
+		Assert.AreEqual(hash, result.Hashes.First().Value);
+	}
+
+	[TestMethod]
+	[DataRow("hash-missing")]
+	[DataRow("hash-na")]
+	public async Task HashesMissing(string filename)
+	{
+		var result = await _bk2Parser.Parse(Embedded(filename + ".bk2"), EmbeddedLength(filename + ".bk2"));
+		Assert.AreEqual(0, result.Hashes.Count);
+	}
 }

tests/TASVideos.MovieParsers.Tests/Fm2ParserTests.cs

@@ -97,4 +97,27 @@ public async Task BinaryWithoutFrameCount()
 		AssertNoWarnings(result);
 		Assert.AreEqual(1, result.Errors.Count());
 	}
+
+	[TestMethod]
+	public async Task Hash()
+	{
+		var result = await _fm2Parser.Parse(Embedded("hash.fm2"), EmbeddedLength("hash.fm2"));
+		Assert.AreEqual(1, result.Hashes.Count);
+		Assert.AreEqual(HashType.Md5, result.Hashes.First().Key);
+		Assert.AreEqual("e9d82f825725c616b0be66ac85dc1b7a", result.Hashes.First().Value);
+	}
+
+	[TestMethod]
+	public async Task InvalidHash()
+	{
+		var result = await _fm2Parser.Parse(Embedded("hash-invalid.fm2"), EmbeddedLength("hash-invalid.fm2"));
+		Assert.AreEqual(0, result.Hashes.Count);
+	}
+
+	[TestMethod]
+	public async Task MissingHash()
+	{
+		var result = await _fm2Parser.Parse(Embedded("hash-missing.fm2"), EmbeddedLength("hash-missing.fm2"));
+		Assert.AreEqual(0, result.Hashes.Count);
+	}
 }

tests/TASVideos.MovieParsers.Tests/Fm2SampleFiles/hash-invalid.fm2

@@ -0,0 +1,2 @@
+romChecksum base64:ThisIsNotBase64
+

tests/TASVideos.MovieParsers.Tests/Fm2SampleFiles/hash.fm2

@@ -0,0 +1 @@
+romChecksum base64:6DgvglcLxhawvMAshDwbeQ==

tests/TASVideos.MovieParsers.Tests/LtmTests.cs

@@ -125,4 +125,34 @@ public async Task VariableFramerate()
 		Assert.AreEqual(30.002721239119342, result.FrameRateOverride);
 		AssertNoWarningsOrErrors(result);
 	}
+
+	[TestMethod]
+	public async Task Hash()
+	{
+		var result = await _ltmParser.Parse(Embedded("hash.ltm"), EmbeddedLength("hash.ltm"));
+		Assert.AreEqual(1, result.Hashes.Count);
+		Assert.AreEqual(HashType.Md5, result.Hashes.First().Key);
+		Assert.AreEqual("7d66e47fdc0807927c40ce1491c68ad3", result.Hashes.First().Value);
+	}
+
+	[TestMethod]
+	public async Task NoHash()
+	{
+		var result = await _ltmParser.Parse(Embedded("no-hash.ltm"), EmbeddedLength("no-hash.ltm"));
+		Assert.AreEqual(0, result.Hashes.Count);
+	}
+
+	[TestMethod]
+	public async Task MissingHash()
+	{
+		var result = await _ltmParser.Parse(Embedded("missing-hash.ltm"), EmbeddedLength("missing-hash.ltm"));
+		Assert.AreEqual(0, result.Hashes.Count);
+	}
+
+	[TestMethod]
+	public async Task InvalidHash()
+	{
+		var result = await _ltmParser.Parse(Embedded("invalid-hash.ltm"), EmbeddedLength("invalid-hash.ltm"));
+		Assert.AreEqual(0, result.Hashes.Count);
+	}
 }