From a570590a727a96e2550a2a0cf376a39297741fd1 Mon Sep 17 00:00:00 2001 From: nascosto <2653283+nascosto@users.noreply.github.com> Date: Wed, 30 Oct 2024 10:54:34 -0600 Subject: [PATCH 1/3] Flatten hierarchy_radio fields during record transformation --- scraper/src/typesense_helper.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scraper/src/typesense_helper.py b/scraper/src/typesense_helper.py index 9a2d1e2..c29bf87 100644 --- a/scraper/src/typesense_helper.py +++ b/scraper/src/typesense_helper.py @@ -207,11 +207,12 @@ def transform_record(record): + transformed_record['weight']['position_descending'] ) - # Flatten nested hierarchy field + # Flatten nested hierarchy fields for x in range(0, 7): - if record['hierarchy'][f'lvl{x}'] is None: - continue - transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}'] + if record['hierarchy'][f'lvl{x}'] is not None: + transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}'] + if record['hierarchy_radio'][f'lvl{x}'] is not None: + transformed_record[f'hierarchy_radio.lvl{x}'] = record['hierarchy_radio'][f'lvl{x}'] # Convert version to array if 'version' in record and type(record['version']) == str: From 15a50077a10e109342abb8fad3c307521ce2f12f Mon Sep 17 00:00:00 2001 From: nascosto <2653283+nascosto@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:06:17 -0600 Subject: [PATCH 2/3] Get tests passing with hierarchy_radio --- .../tests/typesense_helper/commit_tmp_test.py | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/scraper/src/tests/typesense_helper/commit_tmp_test.py b/scraper/src/tests/typesense_helper/commit_tmp_test.py index cb53117..ba699ba 100644 --- a/scraper/src/tests/typesense_helper/commit_tmp_test.py +++ b/scraper/src/tests/typesense_helper/commit_tmp_test.py @@ -354,6 +354,15 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'anchor': 'See More', 'content': 'Explore our range of electronics', @@ -374,6 +383,15 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0', 'anchor': 'Learn More', 'content': 'Discover more about our company values', @@ -406,7 +424,17 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'hierarchy.lvl0': 'About Us', + 'hierarchy_radio.lvl0': 'About Us', 'id': '1', 'item_priority': 2000001001, 'language': 'en', @@ -429,9 +457,19 @@ def test_add_records(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'hierarchy.lvl0': 'Home', 'hierarchy.lvl1': 'Products', 'hierarchy.lvl2': 'Electronics', + 'hierarchy_radio.lvl2': 'Electronics', 'id': '0', 'item_priority': 1000002003, 'language': 'en', @@ -473,6 +511,15 @@ def test_commit_tmp_collection(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'anchor': 'See More', 'content': 'Explore our range of electronics', @@ -493,6 +540,15 @@ def test_commit_tmp_collection(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0', 'anchor': 'Learn More', 'content': 'Discover more about our company values', @@ -568,6 +624,15 @@ def test_commit_tmp_collection_with_curation_rules(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'anchor': 'See More', 'content': 'Explore our range of electronics', @@ -588,6 +653,15 @@ def test_commit_tmp_collection_with_curation_rules(typesense_client): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': 'About Us', + 'lvl1': None, + 'lvl2': None, + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0', 'anchor': 'Learn More', 'content': 'Discover more about our company values', From 8f410b37ae0340752e428d3b8555ee65b533ed3a Mon Sep 17 00:00:00 2001 From: nascosto <2653283+nascosto@users.noreply.github.com> Date: Wed, 30 Oct 2024 11:16:16 -0600 Subject: [PATCH 3/3] Fix additional test --- .../tests/typesense_helper/commit_tmp_test.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/scraper/src/tests/typesense_helper/commit_tmp_test.py b/scraper/src/tests/typesense_helper/commit_tmp_test.py index ba699ba..9061dbd 100644 --- a/scraper/src/tests/typesense_helper/commit_tmp_test.py +++ b/scraper/src/tests/typesense_helper/commit_tmp_test.py @@ -307,6 +307,15 @@ def test_transform_record(): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, 'version': '1.0,2.0', 'extra_field': None, } @@ -317,6 +326,7 @@ def test_transform_record(): 'hierarchy.lvl0': 'Home', 'hierarchy.lvl1': 'Products', 'hierarchy.lvl2': 'Electronics', + 'hierarchy_radio.lvl2': 'Electronics', 'version': ['1.0', '2.0'], 'hierarchy': { 'lvl0': 'Home', @@ -327,6 +337,15 @@ def test_transform_record(): 'lvl5': None, 'lvl6': None, }, + 'hierarchy_radio': { + 'lvl0': None, + 'lvl1': None, + 'lvl2': 'Electronics', + 'lvl3': None, + 'lvl4': None, + 'lvl5': None, + 'lvl6': None, + }, } # Act