diff --git a/cocos/2d/assembler/label/font-utils.ts b/cocos/2d/assembler/label/font-utils.ts
index 1afe602c179..52ec3ae6afd 100644
--- a/cocos/2d/assembler/label/font-utils.ts
+++ b/cocos/2d/assembler/label/font-utils.ts
@@ -30,6 +30,7 @@ import { BufferTextureCopy } from '../../../gfx';
 import { safeMeasureText, BASELINE_RATIO, MIDDLE_RATIO, getBaselineOffset, getSymbolCodeAt } from '../../utils/text-utils';
 import { director, DirectorEvent } from '../../../game/director';
 import { ccwindow } from '../../../core/global-exports';
+import { TextureBase } from '../../../asset/assets/texture-base';
 
 export interface ISharedLabelData {
     canvas: HTMLCanvasElement;
@@ -366,12 +367,11 @@ export class LetterAtlas {
         this.reset();
         const dict = this.fontDefDictionary;
         if (dict && dict.texture) {
-            dict.texture.destroy();
             dict.texture = null;
         }
     }
 
-    getTexture (): any {
+    getTexture (): TextureBase | null {
         return this.fontDefDictionary.getTexture();
     }
 
diff --git a/cocos/2d/assets/bitmap-font.ts b/cocos/2d/assets/bitmap-font.ts
index 0d3655e318c..587aeec1500 100644
--- a/cocos/2d/assets/bitmap-font.ts
+++ b/cocos/2d/assets/bitmap-font.ts
@@ -52,12 +52,31 @@ export interface ILetterDefinition {
 
 export class FontAtlas {
     public letterDefinitions: ILetterDefinition = {};
-    public declare texture: TextureBase | null;
+    public _texture: TextureBase | null = null;
 
     constructor (texture: TextureBase | null) {
         this.texture = texture;
     }
 
+    public set texture (texture: TextureBase | null) {
+        const oldTexture = this._texture;
+        if (oldTexture === texture) return;
+        if (oldTexture) {
+            oldTexture.decRef(false);
+            if (oldTexture.refCount <= 0) {
+                oldTexture.destroy();
+            }
+        }
+        if (texture) {
+            texture.addRef();
+        }
+        this._texture = texture;
+    }
+
+    public get texture (): TextureBase | null {
+        return this._texture;
+    }
+
     public addLetterDefinitions (letter: string, letterDefinition: FontLetterDefinition): void {
         this.letterDefinitions[letter] = letterDefinition;
     }
@@ -73,7 +92,7 @@ export class FontAtlas {
     }
 
     public getTexture (): TextureBase | null {
-        return this.texture;
+        return this._texture;
     }
 
     public getLetter (key: string): FontLetterDefinition {
diff --git a/cocos/2d/components/label.ts b/cocos/2d/components/label.ts
index fbee4f14d19..bc3cb940889 100644
--- a/cocos/2d/components/label.ts
+++ b/cocos/2d/components/label.ts
@@ -505,6 +505,7 @@ export class Label extends UIRenderer {
         }
         if (this._cacheMode === CacheMode.CHAR) {
             this._ttfSpriteFrame = null;
+            this.destroyLetterTexture();
         }
 
         this._cacheMode = value;
@@ -932,12 +933,22 @@ export class Label extends UIRenderer {
         // this._textRenderData = null;
         // this._textLayoutData = null;
 
-        // texture cannot be destroyed in here, lettertexture image source is public.
-        this._letterTexture = null;
+        this.destroyLetterTexture();
 
         super.onDestroy();
     }
 
+    private destroyLetterTexture (): void {
+        const letterTexture = this._letterTexture;
+        if (letterTexture) {
+            letterTexture.decRef(false);
+            if (letterTexture.refCount <= 0) {
+                letterTexture.destroy();
+            }
+        }
+        this._letterTexture = null;
+    }
+
     /**
      * @en update render data.
      * @zh 更新渲染相关数据。
@@ -1035,8 +1046,13 @@ export class Label extends UIRenderer {
             }
         } else {
             if (this.cacheMode === CacheMode.CHAR) {
-                this._letterTexture = this._assembler!.getAssemblerData();
-                this._texture = this._letterTexture;
+                const oldLetterTexture = this._letterTexture;
+                const letterTexture = this._assembler!.getAssemblerData();
+                if (letterTexture !== oldLetterTexture) {
+                    this.destroyLetterTexture();
+                    letterTexture.addRef();
+                }
+                this._texture = this._letterTexture = letterTexture;
             } else if (!this._ttfSpriteFrame) {
                 this._ttfSpriteFrame = new SpriteFrame();
                 this._assemblerData = this._assembler!.getAssemblerData();