From c27365a0dfc393cdd1cf1e3c9e08e3019db3f211 Mon Sep 17 00:00:00 2001 From: ale-rt Date: Tue, 30 Apr 2024 16:29:51 +0200 Subject: [PATCH] When calling the html serializer pass an encoding Fixes #238 --- news/238.bugfix | 1 + src/plone/app/theming/tests/test_transform.py | 27 ++++++++++++++++++- src/plone/app/theming/transform.py | 5 +++- 3 files changed, 31 insertions(+), 2 deletions(-) create mode 100644 news/238.bugfix diff --git a/news/238.bugfix b/news/238.bugfix new file mode 100644 index 00000000..c90b2a4b --- /dev/null +++ b/news/238.bugfix @@ -0,0 +1 @@ +Fix an issue with unicode characters happening with lxml 5 [ale-rt] diff --git a/src/plone/app/theming/tests/test_transform.py b/src/plone/app/theming/tests/test_transform.py index ddb57c9d..faafa34c 100644 --- a/src/plone/app/theming/tests/test_transform.py +++ b/src/plone/app/theming/tests/test_transform.py @@ -1,11 +1,13 @@ from App.config import getConfiguration from diazo.compiler import compile_theme +from html import unescape from lxml import etree from os import environ from plone.app.testing import setRoles from plone.app.testing import TEST_USER_ID from plone.app.theming.interfaces import IThemeSettings from plone.app.theming.testing import THEMING_FUNCTIONAL_TESTING +from plone.app.theming.testing import THEMING_INTEGRATION_TESTING from plone.app.theming.transform import ThemeTransform from plone.app.theming.utils import applyTheme from plone.app.theming.utils import getTheme @@ -25,7 +27,30 @@ import unittest -class TestCase(unittest.TestCase): +class IntegrationTestCase(unittest.TestCase): + + layer = THEMING_INTEGRATION_TESTING + + def test_transform_parseTree_with_unicode(self): + request = self.layer["request"] + request.response.setHeader("Content-Type", "text/html; charset=utf-8") + transform = ThemeTransform(None, request) + snippet = "\n".join( + ( + "", + "", + "", + "
à
", + "", + "", + ) + ) + parsed = transform.parseTree([snippet.encode()]) + serialized = unescape(parsed.serialize().decode()) + self.assertEqual(snippet, serialized) + + +class FunctionalTestCase(unittest.TestCase): layer = THEMING_FUNCTIONAL_TESTING def setUp(self): diff --git a/src/plone/app/theming/transform.py b/src/plone/app/theming/transform.py index eb388d5c..3f74c015 100644 --- a/src/plone/app/theming/transform.py +++ b/src/plone/app/theming/transform.py @@ -13,6 +13,7 @@ from zope.component import adapter from zope.interface import implementer from zope.interface import Interface +from ZPublisher.HTTPRequest import default_encoding import logging @@ -120,7 +121,9 @@ def parseTree(self, result): return None try: - return getHTMLSerializer(result, pretty_print=False) + return getHTMLSerializer( + result, pretty_print=False, encoding=default_encoding + ) except (AttributeError, TypeError, etree.ParseError): return None