diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4e300574fa53..7146a97195eb 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -75,6 +75,8 @@ Optimizations
 * GITHUB#13943: Removed shared `HitsThresholdChecker`, which reduces overhead
   but may delay a bit when dynamic pruning kicks in. (Adrien Grand)
 
+* GITHUB#13961: Replace Map<String,Object> with IntObjectHashMap for DV producer. (Pan Guixin)
+
 Bug Fixes
 ---------------------
 * GITHUB#13832: Fixed an issue where the DefaultPassageFormatter.format method did not format passages as intended
diff --git a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java
index 51930c243ddc..69d557d270ae 100644
--- a/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java
+++ b/lucene/core/src/java/org/apache/lucene/index/SlowCompositeCodecReaderWrapper.java
@@ -20,8 +20,10 @@
 import java.io.UncheckedIOException;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 import org.apache.lucene.codecs.DocValuesProducer;
 import org.apache.lucene.codecs.FieldsProducer;
@@ -32,7 +34,6 @@
 import org.apache.lucene.codecs.TermVectorsReader;
 import org.apache.lucene.index.MultiDocValues.MultiSortedDocValues;
 import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues;
-import org.apache.lucene.internal.hppc.IntObjectHashMap;
 import org.apache.lucene.search.KnnCollector;
 import org.apache.lucene.util.ArrayUtil;
 import org.apache.lucene.util.Bits;
@@ -376,7 +377,7 @@ private static class SlowCompositeDocValuesProducerWrapper extends DocValuesProd
     private final CodecReader[] codecReaders;
     private final DocValuesProducer[] producers;
     private final int[] docStarts;
-    private final IntObjectHashMap<OrdinalMap> cachedOrdMaps = new IntObjectHashMap<>();
+    private final Map<String, OrdinalMap> cachedOrdMaps = new HashMap<>();
 
     SlowCompositeDocValuesProducerWrapper(CodecReader[] codecReaders, int[] docStarts) {
       this.codecReaders = codecReaders;
@@ -415,14 +416,14 @@ public BinaryDocValues getBinary(FieldInfo field) throws IOException {
     public SortedDocValues getSorted(FieldInfo field) throws IOException {
       OrdinalMap map = null;
       synchronized (cachedOrdMaps) {
-        map = cachedOrdMaps.get(field.number);
+        map = cachedOrdMaps.get(field.name);
         if (map == null) {
           // uncached, or not a multi dv
           SortedDocValues dv =
               MultiDocValues.getSortedValues(new MultiReader(codecReaders), field.name);
           if (dv instanceof MultiSortedDocValues) {
             map = ((MultiSortedDocValues) dv).mapping;
-            cachedOrdMaps.put(field.number, map);
+            cachedOrdMaps.put(field.name, map);
           }
           return dv;
         }
@@ -451,14 +452,14 @@ public SortedNumericDocValues getSortedNumeric(FieldInfo field) throws IOExcepti
     public SortedSetDocValues getSortedSet(FieldInfo field) throws IOException {
       OrdinalMap map = null;
       synchronized (cachedOrdMaps) {
-        map = cachedOrdMaps.get(field.number);
+        map = cachedOrdMaps.get(field.name);
         if (map == null) {
           // uncached, or not a multi dv
           SortedSetDocValues dv =
               MultiDocValues.getSortedSetValues(new MultiReader(codecReaders), field.name);
           if (dv instanceof MultiSortedSetDocValues) {
             map = ((MultiSortedSetDocValues) dv).mapping;
-            cachedOrdMaps.put(field.number, map);
+            cachedOrdMaps.put(field.name, map);
           }
           return dv;
         }