From ec9899e1df47d0dd04775cb5aa9a9caf1861ec77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= <stephane@openfoodfacts.org>
Date: Thu, 20 Apr 2023 20:14:18 +0200
Subject: [PATCH 1/7] fix: escape issue in additives knowledge panel - remove
 latex formulas

---
 lib/ProductOpener/KnowledgePanels.pm | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/lib/ProductOpener/KnowledgePanels.pm b/lib/ProductOpener/KnowledgePanels.pm
index 5c64b5f7fcbcd..c441640df762d 100644
--- a/lib/ProductOpener/KnowledgePanels.pm
+++ b/lib/ProductOpener/KnowledgePanels.pm
@@ -221,6 +221,8 @@ sub convert_multiline_string_to_singleline ($line) {
 	# Escape quotes unless they have been escaped already
 	# negative look behind to not convert \" to \\"
 	$line =~ s/(?<!\\)"/\\"/g;
+	# Escape \ to \\
+	$line =~ s/(?<!\\)\\/\\\\/g;
 	return '"' . $line . '"';
 }
 
@@ -1290,6 +1292,26 @@ sub create_ingredients_analysis_panel ($product_ref, $target_lc, $target_cc, $op
 	return;
 }
 
+sub remove_latex_sequences ($string) {
+
+	# Some wikipedia abstracts have chemical formulas like {\\displaystyle \\mathrm {NaNO_{3}} +\\mathrm {Pb} \\to \\mathrm {NaNO_{2}} +\\mathrm {PbO} }
+	# In practice, we remove everything between { }
+
+	$string =~ s/
+        (                   
+        {                   # match an opening {
+            (?:
+                [^{}]++     # one or more non angle brackets, non backtracking
+                  |
+                (?1)        # found { or }, so recurse to capture group 1
+            )*
+        }                   # match a closing }
+        )                   
+        //xg;
+
+	return $string;
+}
+
 =head2 add_taxonomy_properties_in_target_languages_to_object ( $object_ref, $tagtype, $tagid, $properties_ref, $target_lcs_ref )
 
 This function adds to the hash ref $object_ref (for instance a data structure passed to a template) the values
@@ -1332,7 +1354,7 @@ sub add_taxonomy_properties_in_target_languages_to_object ($object_ref, $tagtype
 			}
 		}
 		if (defined $property_value) {
-			$object_ref->{$property} = $property_value;
+			$object_ref->{$property} = remove_latex_sequences($property_value);
 			$object_ref->{$property . "_lc"} = $property_lc;
 			$object_ref->{$property . "_language"}
 				= display_taxonomy_tag($target_lcs_ref->[0], "languages", $language_codes{$property_lc});

From 84e281b93645a7b4b3daf73f96c78d78cdce8c65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= <stephane@openfoodfacts.org>
Date: Fri, 21 Apr 2023 09:02:07 +0200
Subject: [PATCH 2/7] fix escaping

---
 lib/ProductOpener/KnowledgePanels.pm | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/lib/ProductOpener/KnowledgePanels.pm b/lib/ProductOpener/KnowledgePanels.pm
index c441640df762d..5558ded9d7127 100644
--- a/lib/ProductOpener/KnowledgePanels.pm
+++ b/lib/ProductOpener/KnowledgePanels.pm
@@ -218,11 +218,9 @@ sub convert_multiline_string_to_singleline ($line) {
 
 	# \R will match all Unicode newline sequence
 	$line =~ s/\R/\\n/sg;
-	# Escape quotes unless they have been escaped already
-	# negative look behind to not convert \" to \\"
-	$line =~ s/(?<!\\)"/\\"/g;
-	# Escape \ to \\
-	$line =~ s/(?<!\\)\\/\\\\/g;
+	# Escape " and \ unless they have been escaped already
+	# negative look behind to not convert \" to \\" or \\ to \\\\
+	$line =~ s/(?<!\\)("|\\)/\\$1/g;
 	return '"' . $line . '"';
 }
 

From e3902b37b3454d6d94b42681d8dbcf6252a04dc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= <stephane@openfoodfacts.org>
Date: Fri, 21 Apr 2023 13:42:41 +0200
Subject: [PATCH 3/7] do not double escape \n

---
 lib/ProductOpener/KnowledgePanels.pm | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/ProductOpener/KnowledgePanels.pm b/lib/ProductOpener/KnowledgePanels.pm
index 5558ded9d7127..e4e4ff15ebf5c 100644
--- a/lib/ProductOpener/KnowledgePanels.pm
+++ b/lib/ProductOpener/KnowledgePanels.pm
@@ -219,8 +219,8 @@ sub convert_multiline_string_to_singleline ($line) {
 	# \R will match all Unicode newline sequence
 	$line =~ s/\R/\\n/sg;
 	# Escape " and \ unless they have been escaped already
-	# negative look behind to not convert \" to \\" or \\ to \\\\
-	$line =~ s/(?<!\\)("|\\)/\\$1/g;
+	# negative look behind to not convert \n to \\n or \" to \\" or \\ to \\\\
+	$line =~ s/(?<!\\)(n|"|\\)/\\$1/g;
 	return '"' . $line . '"';
 }
 

From 78db5b7eacc6862f79f9a17b23d7b491f5a17789 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= <stephane@openfoodfacts.org>
Date: Fri, 21 Apr 2023 14:55:19 +0200
Subject: [PATCH 4/7] do not double escape \n

---
 lib/ProductOpener/KnowledgePanels.pm | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/lib/ProductOpener/KnowledgePanels.pm b/lib/ProductOpener/KnowledgePanels.pm
index e4e4ff15ebf5c..c3b80ff28cd90 100644
--- a/lib/ProductOpener/KnowledgePanels.pm
+++ b/lib/ProductOpener/KnowledgePanels.pm
@@ -216,11 +216,13 @@ The function converts the multiline string into a single line string.
 
 sub convert_multiline_string_to_singleline ($line) {
 
-	# \R will match all Unicode newline sequence
-	$line =~ s/\R/\\n/sg;
 	# Escape " and \ unless they have been escaped already
 	# negative look behind to not convert \n to \\n or \" to \\" or \\ to \\\\
-	$line =~ s/(?<!\\)(n|"|\\)/\\$1/g;
+	$line =~ s/(?<!\\)("|\\)/\\$1/g;
+
+	# \R will match all Unicode newline sequence
+	$line =~ s/\R/\\n/sg;
+
 	return '"' . $line . '"';
 }
 

From 8b517368d0444fe9d672751abfca6a386e0015a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= <stephane@openfoodfacts.org>
Date: Thu, 4 May 2023 10:52:34 +0200
Subject: [PATCH 5/7] remove unneeded escapes now handled in
 convert_multiline_string_to_singleline

---
 .../environment/carbon_footprint.tt.json      |  2 +-
 .../environment/ecoscore/agribalyse.tt.json   |  2 +-
 tests/unit/knowledge_panels.t                 | 22 +++++++++++++++++++
 3 files changed, 24 insertions(+), 2 deletions(-)
 create mode 100644 tests/unit/knowledge_panels.t

diff --git a/templates/api/knowledge-panels/environment/carbon_footprint.tt.json b/templates/api/knowledge-panels/environment/carbon_footprint.tt.json
index e49b9a8e4f14a..43c0b308d834e 100644
--- a/templates/api/knowledge-panels/environment/carbon_footprint.tt.json
+++ b/templates/api/knowledge-panels/environment/carbon_footprint.tt.json
@@ -29,7 +29,7 @@
                 "type": "summary",
                 "html": `
                     <p>[% lang('source_ademe_agribalyse_for_category') %] 
-                    <a href=\"https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]\">[% panel.agribalyse_category_name.dquote %]</a>
+                    <a href="https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]">[% panel.agribalyse_category_name.dquote %]</a>
                     ([% lang('source_ademe_agribalyse') %])
                     </p>
                     `
diff --git a/templates/api/knowledge-panels/environment/ecoscore/agribalyse.tt.json b/templates/api/knowledge-panels/environment/ecoscore/agribalyse.tt.json
index da0fd37150d38..d64541ade95b6 100644
--- a/templates/api/knowledge-panels/environment/ecoscore/agribalyse.tt.json
+++ b/templates/api/knowledge-panels/environment/ecoscore/agribalyse.tt.json
@@ -16,7 +16,7 @@
                 "type": "summary",
                 "html": `
                     <p>[% lang('categories_s') FILTER ucfirst %][% sep %]: 
-                    <a href=\"https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]\">[% panel.agribalyse_category_name %]</a>
+                    <a href="https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]">[% panel.agribalyse_category_name %]</a>
                     </p>
                     <ul>
                         <li>
diff --git a/tests/unit/knowledge_panels.t b/tests/unit/knowledge_panels.t
new file mode 100644
index 0000000000000..6292c90c1a859
--- /dev/null
+++ b/tests/unit/knowledge_panels.t
@@ -0,0 +1,22 @@
+#!/usr/bin/perl -w
+
+use Modern::Perl '2017';
+use utf8;
+
+use Test::More;
+use Log::Any::Adapter 'TAP';
+
+use ProductOpener::KnowledgePanels qw/:all/;
+
+is(ProductOpener::KnowledgePanels::convert_multiline_string_to_singleline('
+test
+muti-line string
+a slash A / B and anti-slash \
+HTML <a href="https://url.com">test</a>
+'),
+'"\ntest\nmuti-line string\na slash A / B and anti-slash \\\nHTML <a href=\"https://url.com\">test</a>\n"');
+
+is(ProductOpener::KnowledgePanels::convert_multiline_string_to_singleline('<a href="https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]">'),
+'"<a href=\"https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]\">"');
+
+done_testing();

From d7861a5c7acd5c56eef33acb071dc43f6e90d5de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= <stephane@openfoodfacts.org>
Date: Thu, 4 May 2023 12:18:26 +0200
Subject: [PATCH 6/7] lint

---
 tests/unit/knowledge_panels.t | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tests/unit/knowledge_panels.t b/tests/unit/knowledge_panels.t
index 6292c90c1a859..cfd4c2fc66c76 100644
--- a/tests/unit/knowledge_panels.t
+++ b/tests/unit/knowledge_panels.t
@@ -8,15 +8,20 @@ use Log::Any::Adapter 'TAP';
 
 use ProductOpener::KnowledgePanels qw/:all/;
 
-is(ProductOpener::KnowledgePanels::convert_multiline_string_to_singleline('
+is(
+	ProductOpener::KnowledgePanels::convert_multiline_string_to_singleline('
 test
 muti-line string
 a slash A / B and anti-slash \
 HTML <a href="https://url.com">test</a>
 '),
-'"\ntest\nmuti-line string\na slash A / B and anti-slash \\\nHTML <a href=\"https://url.com\">test</a>\n"');
+	'"\ntest\nmuti-line string\na slash A / B and anti-slash \\\nHTML <a href=\"https://url.com\">test</a>\n"'
+);
 
-is(ProductOpener::KnowledgePanels::convert_multiline_string_to_singleline('<a href="https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]">'),
-'"<a href=\"https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]\">"');
+is(
+	ProductOpener::KnowledgePanels::convert_multiline_string_to_singleline(
+		'<a href="https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]">'),
+	'"<a href=\"https://agribalyse.ademe.fr/app/aliments/[% product.ecoscore_data.agribalyse.code %]\">"'
+);
 
 done_testing();

From bd6f92da1401915c09b0780cfbc9a9c6eb5800d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?St=C3=A9phane=20Gigandet?= <stephane@openfoodfacts.org>
Date: Thu, 4 May 2023 12:29:20 +0200
Subject: [PATCH 7/7] update test

---
 tests/unit/knowledge_panels.t | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/unit/knowledge_panels.t b/tests/unit/knowledge_panels.t
index cfd4c2fc66c76..9839b49dad95c 100644
--- a/tests/unit/knowledge_panels.t
+++ b/tests/unit/knowledge_panels.t
@@ -15,7 +15,7 @@ muti-line string
 a slash A / B and anti-slash \
 HTML <a href="https://url.com">test</a>
 '),
-	'"\ntest\nmuti-line string\na slash A / B and anti-slash \\\nHTML <a href=\"https://url.com\">test</a>\n"'
+	'"\ntest\nmuti-line string\na slash A / B and anti-slash \\\\\nHTML <a href=\"https://url.com\">test</a>\n"'
 );
 
 is(