Countless fixes to TFE conversion but apparently not enough to fix th…

…e crashes (#2423) #patch

#2202

---------

Co-authored-by: codefactor-io <[email protected]>

ImperatorToCK3.UnitTests/CommonUtils/HistoryTests.cs

@@ -122,6 +122,35 @@ public void NullIsReturnedForNonExistingField() {
 		Assert.Null(provHistory.GetFieldValue("title", new Date(1000, 1, 1)));
 	}
 
+	[Fact]
+	public void NegativeDatesAreSupported() {
+		var reader = new BufferedReader(
+			@"= {		#Sarkel
+					culture = mykenian
+					-750.1.2 = {
+						culture = macedonian
+					}
+					-100.1.2 = {
+						culture = greek
+					}
+					50.3.4 = {
+						culture = roman
+					}
+				}");
+
+		var provHistoryFactory = new HistoryFactory.HistoryFactoryBuilder()
+			.WithSimpleField("culture", "culture", null)
+			.Build();
+
+		var provHistory = provHistoryFactory.GetHistory(reader);
+
+		Assert.Equal("mykenian", provHistory.GetFieldValue("culture", new Date(-800, 1, 1))!.ToString());
+		Assert.Equal("macedonian", provHistory.GetFieldValue("culture", new Date(-750, 1, 2))!.ToString());
+		Assert.Equal("macedonian", provHistory.GetFieldValue("culture", new Date(-600, 1, 2))!.ToString());
+		Assert.Equal("greek", provHistory.GetFieldValue("culture", new Date(-100, 1, 2))!.ToString());
+		Assert.Equal("roman", provHistory.GetFieldValue("culture", new Date(50, 3, 4))!.ToString());
+	}
+
 	[Fact]
 	public void HistoryCanBeReadFromMultipleItems() {
 		var reader1 = new BufferedReader(

ImperatorToCK3.UnitTests/TestFiles/LandedTitlesTests/CK3/game/common/landed_titles/extra_landed_titles.txt

@@ -3,6 +3,8 @@ e_mongolia = {
 
 	k_mongolia = {
 		capital = c_karakorum
+
+		c_karakorum = {} # Just for the capital entry to be accepted by the converter.
 	}
 
 	k_jubu = {}

ImperatorToCK3/CK3/Characters/CharacterCollection.cs

@@ -846,4 +846,27 @@ public void RemoveUndefinedTraits(TraitMapper traitMapper) {
 			}
 		}
 	}
+
+	public void RemoveInvalidDynastiesFromHistory(DynastyCollection dynasties) {
+		Logger.Info("Removing invalid dynasties from CK3 character history...");
+
+		var ck3Characters = this.Where(c => !c.FromImperator).ToArray();
+		var validDynastyIds = dynasties.Select(d => d.Id).ToHashSet();
+
+		foreach (var character in ck3Characters) {
+			if (!character.History.Fields.TryGetValue("dynasty", out var dynastyField)) {
+				continue;
+			}
+
+			dynastyField.RemoveAllEntries(value => {
+				var dynastyId = value.ToString()?.RemQuotes();
+
+				if (string.IsNullOrWhiteSpace(dynastyId)) {
+					return true;
+				}
+
+				return !validDynastyIds.Contains(dynastyId);
+			});
+		}
+	}
 }

ImperatorToCK3/CK3/Characters/CharactersLoader.cs

@@ -1,6 +1,7 @@
 using commonItems;
 using commonItems.Mods;
 using Open.Collections.Synchronized;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace ImperatorToCK3.CK3.Characters;
@@ -33,9 +34,21 @@ public void LoadCK3Characters(ModFilesystem ck3ModFS, Date bookmarkDate) {
 			"set_relation_ward", "set_relation_mentor",
 			"add_opinion", "make_concubine",
 		];
-		string[] fieldsToClear = ["friends", "best_friends", "lovers", "rivals", "nemesis", "primary_title", "dna"];
+		string[] fieldsToClear = [
+			"friends", "best_friends", "lovers", "rivals", "nemesis",
+			"primary_title", "dna", "spawn_army", "add_character_modifier", "languages",
+			"claims",
+		];
 
+		var femaleCharacterIds = loadedCharacters.Where(c => c.Female).Select(c => c.Id).ToHashSet();
+		var maleCharacterIds = loadedCharacters.Select(c => c.Id).Except(femaleCharacterIds).ToHashSet();
+
 		foreach (var character in loadedCharacters) {
+			// Clear some fields we don't need.
+			foreach (var fieldName in fieldsToClear) {
+				character.History.Fields[fieldName].RemoveAllEntries();
+			}
+
 			// Remove post-bookmark history except for births and deaths.
 			foreach (var field in character.History.Fields) {
 				if (field.Id == "birth" || field.Id == "death") {
@@ -58,7 +71,9 @@ public void LoadCK3Characters(ModFilesystem ck3ModFS, Date bookmarkDate) {
 				deathField.RemoveAllEntries();
 				deathField.AddEntryToHistory(deathDate, "death", value: true);
 			}
-
+
+			RemoveInvalidMotherAndFatherEntries(character, femaleCharacterIds, maleCharacterIds);
+
 			// Remove dated name changes like 64.10.13 = { name = "Linus" }
 			var nameField = character.History.Fields["name"];
 			nameField.RemoveHistoryPastDate(birthDate);
@@ -67,14 +82,34 @@ public void LoadCK3Characters(ModFilesystem ck3ModFS, Date bookmarkDate) {
 			character.History.Fields["effects"].RemoveAllEntries(
 				entry => irrelevantEffects.Any(effect => entry.ToString()?.Contains(effect) ?? false));
 
-			// Clear some fields we don't need.
-			foreach (var fieldName in fieldsToClear) {
-				character.History.Fields[fieldName].RemoveAllEntries();
-			}
-
 			character.InitSpousesCache();
 			character.InitConcubinesCache();
 			character.UpdateChildrenCacheOfParents();
 		}
 	}
+
+	private static void RemoveInvalidMotherAndFatherEntries(Character character, HashSet<string> femaleCharacterIds, HashSet<string> maleCharacterIds) {
+		// Remove wrong sex mother and father references (male mothers, female fathers).
+		var motherField = character.History.Fields["mother"];
+		motherField.RemoveAllEntries(value => {
+			string? motherId = value.ToString()?.RemQuotes();
+			if (motherId is null || !femaleCharacterIds.Contains(motherId)) {
+				Logger.Debug($"Removing invalid mother {motherId} from character {character.Id}");
+				return true;
+			}
+
+			return false;
+		});
+
+		var fatherField = character.History.Fields["father"];
+		fatherField.RemoveAllEntries(value => {
+			string? fatherId = value.ToString()?.RemQuotes();
+			if (fatherId is null || !maleCharacterIds.Contains(fatherId)) {
+				Logger.Debug($"Removing invalid father {fatherId} from character {character.Id}");
+				return true;
+			}
+
+			return false;
+		});
+	}
 }

ImperatorToCK3/CK3/Characters/DNAFactory.cs

@@ -395,7 +395,10 @@ AccessoryGene ck3Gene
 			Logger.Warn($"No object mappings found for {geneInfo.ObjectName} in gene {irGeneName}!");
 			return null;
 		}
+
+		// Prefer using the smallest template that contains the object.
 		var ck3GeneTemplate = ck3Gene.GeneTemplates
+			.OrderBy(t => t.ObjectCountForAgeSex(irCharacter.AgeSex))
 			.FirstOrDefault(t => t.ContainsObjectForAgeSex(irCharacter.AgeSex, convertedSetEntry));
 		if (ck3GeneTemplate is null) {
 			Logger.Warn($"No template found for {convertedSetEntry} in CK3 gene {ck3Gene.Id}!");

ImperatorToCK3/CK3/Cultures/CultureCollection.cs

@@ -34,6 +34,11 @@ private void InitCultureDataParser(ColorFactory colorFactory, OrderedDictionary<
 		});
 		cultureDataParser.RegisterKeyword("parents", reader => {
 			cultureData.ParentCultureIds = reader.GetStrings().ToOrderedSet();
+
+			if (cultureData.ParentCultureIds.Count > 2) {
+				Logger.Warn("Found a culture that has more than 2 parents! Only the first 2 will be used.");
+				cultureData.ParentCultureIds = cultureData.ParentCultureIds.Take(2).ToOrderedSet();
+			}
 		});
 		cultureDataParser.RegisterKeyword("heritage", reader => {
 			var heritageId = reader.GetString();

ImperatorToCK3/CK3/Dynasties/Dynasty.cs

@@ -11,6 +11,8 @@
 using System.Diagnostics.CodeAnalysis;
 using System.Linq;
 
+using ImperatorCharacter = ImperatorToCK3.Imperator.Characters.Character;
+
 namespace ImperatorToCK3.CK3.Dynasties;
 
 [SerializationByProperties]
@@ -32,10 +34,10 @@ public Dynasty(Family irFamily, CharacterCollection irCharacters, CulturesDB irC
 			ck3Member?.SetDynastyId(Id, date: null);
 		}
 
-		SetLocFromImperatorFamilyName(irFamily.GetMaleForm(irCulturesDB), irLocDB, ck3LocDB);
+		SetLocFromImperatorFamilyName(irFamily.GetMaleForm(irCulturesDB), imperatorMembers, irLocDB, ck3LocDB);
 	}
 
-	public Dynasty(CK3.Characters.Character character, string irFamilyName, CulturesDB irCulturesDB, LocDB irLocDB, CK3LocDB ck3LocDB, Date date) {
+	public Dynasty(CK3.Characters.Character character, string irFamilyName, ImperatorCharacter[] irMembers, CulturesDB irCulturesDB, LocDB irLocDB, CK3LocDB ck3LocDB, Date date) {
 		FromImperator = true;
 		Id = $"dynn_irtock3_from_{character.Id}";
 		Name = Id;
@@ -47,7 +49,7 @@ public Dynasty(CK3.Characters.Character character, string irFamilyName, Cultures
 
 		character.SetDynastyId(Id, null);
 
-		SetLocFromImperatorFamilyName(Family.GetMaleForm(irFamilyName, irCulturesDB), irLocDB, ck3LocDB);
+		SetLocFromImperatorFamilyName(Family.GetMaleForm(irFamilyName, irCulturesDB), irMembers, irLocDB, ck3LocDB);
 	}
 
 	public Dynasty(string dynastyId, BufferedReader dynastyReader) {
@@ -125,8 +127,9 @@ private void SetCultureFromImperator(Family irFamily, IReadOnlyList<Character> i
 		Logger.Warn($"Couldn't determine culture for dynasty {Id}, needs manual setting!");
 	}
 
-	private void SetLocFromImperatorFamilyName(string irFamilyLocKey, LocDB irLocDB, CK3LocDB ck3LocDB) {
+	private void SetLocFromImperatorFamilyName(string irFamilyLocKey, ImperatorCharacter[] irMembers, LocDB irLocDB, CK3LocDB ck3LocDB) {
 		var irFamilyLoc = irLocDB.GetLocBlockForKey(irFamilyLocKey);
+
 		var ck3NameLoc = ck3LocDB.GetOrCreateLocBlock(Name);
 		if (irFamilyLoc is not null) {
 			ck3NameLoc.CopyFrom(irFamilyLoc);
@@ -137,6 +140,19 @@ private void SetLocFromImperatorFamilyName(string irFamilyLocKey, LocDB irLocDB,
 				return !string.IsNullOrEmpty(other) ? other : irFamilyLoc.Id;
 			});
 		} else { // fallback: use unlocalized Imperator family key
+			// If the loc key is an empty string, try using a family name from the family's members.
+			if (string.IsNullOrEmpty(irFamilyLocKey)) {
+				foreach (var irMember in irMembers) {
+					if (irMember.FamilyName is null) {
+						continue;
+					}
+
+					Logger.Debug($"Dynasty {Id} has an empty loc key! Using family name from member \"{irMember.FamilyName}\".");
+					ck3NameLoc[ConverterGlobals.PrimaryLanguage] = irMember.FamilyName;
+					return;
+				}
+			}
+
 			Logger.Debug($"Dynasty {Id} has no localization for name \"{irFamilyLocKey}\"! Using unlocalized name.");
 			ck3NameLoc[ConverterGlobals.PrimaryLanguage] = irFamilyLocKey;
 		}

ImperatorToCK3/CK3/Dynasties/DynastyCollection.cs

@@ -77,7 +77,8 @@ private void CreateDynastiesForCharactersFromMinorFamilies(Imperator.World irWor
 			}
 
 			// Neither character nor their father have a dynasty, so we need to create a new one.
-			var newDynasty = new Dynasty(ck3Character, irFamilyName, irWorld.CulturesDB, irLocDB, ck3LocDB, date);
+			Imperator.Characters.Character[] irFamilyMembers = [irCharacter];
+			var newDynasty = new Dynasty(ck3Character, irFamilyName, irFamilyMembers, irWorld.CulturesDB, irLocDB, ck3LocDB, date);
 			AddOrReplace(newDynasty);
 			++createdDynastiesCount;
 		}

ImperatorToCK3/CK3/Religions/DoctrineCategory.cs

@@ -8,6 +8,8 @@ namespace ImperatorToCK3.CK3.Religions;
 public sealed class DoctrineCategory : IIdentifiable<string> {
 	public string Id { get; }
 	public string? GroupId { get; private set; }
+	public int NumberOfPicks { get; private set; } = 1;
+
 	private readonly OrderedSet<string> doctrineIds = new();
 	public IReadOnlyCollection<string> DoctrineIds => doctrineIds.ToImmutableArray();
 
@@ -16,6 +18,7 @@ public DoctrineCategory(string id, BufferedReader categoryReader) {
 
 		var parser = new Parser();
 		parser.RegisterKeyword("group", reader => GroupId = reader.GetString());
+		parser.RegisterKeyword("number_of_picks", reader => NumberOfPicks = reader.GetInt());
 		parser.RegisterRegex(CommonRegexes.String, (reader, doctrineId) => {
 			doctrineIds.Add(doctrineId);
 			ParserHelpers.IgnoreItem(reader);

ImperatorToCK3/CK3/Religions/Faith.cs

@@ -26,11 +26,25 @@ public Faith(string id, FaithData faithData, Religion religion) {
 		attributes = [.. faithData.Attributes];
 
 		// Fixup for issue found in TFE: add reformed_icon if faith has unreformed_faith_doctrine.
-		if (DoctrineIds.Contains("unreformed_faith_doctrine") && !attributes.Any(pair => pair.Key == "reformed_icon")) {
+		if (DoctrineIds.Contains("unreformed_faith_doctrine") && attributes.All(pair => pair.Key != "reformed_icon")) {
 			// Use the icon attribute.
 			var icon = attributes.FirstOrDefault(pair => pair.Key == "icon");
 			attributes = [.. attributes, new KeyValuePair<string, StringOfItem>("reformed_icon", icon.Value)];
 		}
+
+		// Fix a faith having more doctrines in the same category than allowed.
+		foreach (var category in religion.ReligionCollection.DoctrineCategories) {
+			var doctrinesInCategory = DoctrineIds.Where(d => category.DoctrineIds.Contains(d)).ToArray();
+			if (doctrinesInCategory.Length > category.NumberOfPicks) {
+				Logger.Warn($"Faith {Id} has too many doctrines in category {category.Id}: " +
+				            $"{string.Join(", ", doctrinesInCategory)}. Keeping the last {category.NumberOfPicks} of them.");
+
+				DoctrineIds.ExceptWith(doctrinesInCategory);
+				foreach (var doctrine in doctrinesInCategory.Reverse().Take(category.NumberOfPicks)) {
+					DoctrineIds.Add(doctrine);
+				}
+			}
+		}
 	}
 
 	private readonly OrderedSet<string> holySiteIds;
@@ -78,17 +92,21 @@ public string Serialize(string indent, bool withBraces) {
 		return sb.ToString();
 	}
 
-	public string? GetDoctrineIdForDoctrineCategoryId(string doctrineCategoryId) {
+	public OrderedSet<string> GetDoctrineIdsForDoctrineCategoryId(string doctrineCategoryId) {
 		var category = Religion.ReligionCollection.DoctrineCategories[doctrineCategoryId];
-		return GetDoctrineIdForDoctrineCategory(category);
+		return GetDoctrineIdsForDoctrineCategory(category);
 	}
 
-	private string? GetDoctrineIdForDoctrineCategory(DoctrineCategory category) {
+	private OrderedSet<string> GetDoctrineIdsForDoctrineCategory(DoctrineCategory category) {
 		var potentialDoctrineIds = category.DoctrineIds;
 
 		// Look in faith first. If not found, look in religion.
-		var matchingInFaith = DoctrineIds.Intersect(potentialDoctrineIds).LastOrDefault();
-		return matchingInFaith ?? Religion.DoctrineIds.Intersect(potentialDoctrineIds).LastOrDefault();
+		var matchingInFaith = DoctrineIds.Intersect(potentialDoctrineIds).ToOrderedSet();
+		if (matchingInFaith.Any()) {
+			return matchingInFaith;
+		} else {
+			return Religion.DoctrineIds.Intersect(potentialDoctrineIds).ToOrderedSet();
+		}
 	}
 
 	public bool HasDoctrine(string doctrineId) {
@@ -98,6 +116,6 @@ public bool HasDoctrine(string doctrineId) {
 			return false;
 		}
 
-		return GetDoctrineIdForDoctrineCategory(category) == doctrineId;
+		return GetDoctrineIdsForDoctrineCategory(category).Contains(doctrineId);
 	}
 }

ImperatorToCK3/CK3/Religions/HolySite.cs

@@ -54,6 +54,18 @@ public HolySite(string id, BufferedReader holySiteReader, Title.LandedTitles lan
 		if (parsedBaronyId is not null) {
 			Barony = landedTitles[parsedBaronyId];
 		}
+
+		// Fix "barony not in specified county" errors reported by ck3-tiger.
+		if (Barony is not null && County is not null && Barony.DeJureLiege != County) {
+			string baseMessage = $"Holy site {Id} has barony {Barony.Id} not in specified county {County.Id}.";
+			var correctCounty = Barony.DeJureLiege;
+			if (correctCounty is not null) {
+				Logger.Debug($"{baseMessage} Setting county to {correctCounty.Id}.");
+				County = correctCounty;
+			} else {
+				Logger.Warn($"{baseMessage} Cannot find correct county.");
+			}
+		}
 	}
 
 	private static string GenerateHolySiteId(Title barony, Faith faith) {

ImperatorToCK3/CK3/Religions/Religion.cs

@@ -34,6 +34,20 @@ public Religion(string id, BufferedReader religionReader, ReligionCollection rel
 			attributes.Add(new KeyValuePair<string, StringOfItem>(keyword, reader.GetStringOfItem()));
 		});
 		religionParser.ParseStream(religionReader);
+
+		// Fix a religion having more doctrines in the same category than allowed.
+		foreach (var category in religions.DoctrineCategories) {
+			var doctrinesInCategory = DoctrineIds.Where(d => category.DoctrineIds.Contains(d)).ToArray();
+			if (doctrinesInCategory.Length > category.NumberOfPicks) {
+				Logger.Warn($"Religion {Id} has too many doctrines in category {category.Id}: " +
+				            $"{string.Join(", ", doctrinesInCategory)}. Keeping the last {category.NumberOfPicks} of them.");
+
+				DoctrineIds.ExceptWith(doctrinesInCategory);
+				foreach (var doctrine in doctrinesInCategory.Reverse().Take(category.NumberOfPicks)) {
+					DoctrineIds.Add(doctrine);
+				}
+			}
+		}
 	}
 	private void LoadFaith(string faithId, BufferedReader faithReader) {
 		faithData = new FaithData();

ImperatorToCK3/CK3/Religions/ReligionCollection.cs

@@ -300,7 +300,7 @@ Date date
 
 		var aliveFaithsWithSpiritualHeadDoctrine = Faiths
 			.Where(f => aliveCharacterFaithIds.Contains(f.Id) || provinceFaithIds.Contains(f.Id))
-			.Where(f => f.GetDoctrineIdForDoctrineCategoryId("doctrine_head_of_faith") == "doctrine_spiritual_head")
+			.Where(f => f.GetDoctrineIdsForDoctrineCategoryId("doctrine_head_of_faith").Contains("doctrine_spiritual_head"))
 			.ToImmutableList();
 
 		foreach (var faith in aliveFaithsWithSpiritualHeadDoctrine) {