Merge pull request #70 from nascosto/feature/radio

Flatten hierarchy_radio fields during record transformation
typesense · Oct 31, 2024 · edd508e · edd508e
2 parents b00e656 + 8f410b3
commit edd508e
Show file tree

Hide file tree

Showing 2 changed files with 98 additions and 4 deletions.
diff --git a/scraper/src/tests/typesense_helper/commit_tmp_test.py b/scraper/src/tests/typesense_helper/commit_tmp_test.py
@@ -307,6 +307,15 @@ def test_transform_record():
             'lvl5': None,
             'lvl6': None,
         },
+        'hierarchy_radio': {
+            'lvl0': None,
+            'lvl1': None,
+            'lvl2': 'Electronics',
+            'lvl3': None,
+            'lvl4': None,
+            'lvl5': None,
+            'lvl6': None,
+        },
         'version': '1.0,2.0',
         'extra_field': None,
     }
@@ -317,6 +326,7 @@ def test_transform_record():
         'hierarchy.lvl0': 'Home',
         'hierarchy.lvl1': 'Products',
         'hierarchy.lvl2': 'Electronics',
+        'hierarchy_radio.lvl2': 'Electronics',
         'version': ['1.0', '2.0'],
         'hierarchy': {
             'lvl0': 'Home',
@@ -327,6 +337,15 @@ def test_transform_record():
             'lvl5': None,
             'lvl6': None,
         },
+        'hierarchy_radio': {
+            'lvl0': None,
+            'lvl1': None,
+            'lvl2': 'Electronics',
+            'lvl3': None,
+            'lvl4': None,
+            'lvl5': None,
+            'lvl6': None,
+        },
     }
 
     # Act
@@ -354,6 +373,15 @@ def test_add_records(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': None,
+                'lvl1': None,
+                'lvl2': 'Electronics',
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'version': '1.0,2.0',
             'anchor': 'See More',
             'content': 'Explore our range of electronics',
@@ -374,6 +402,15 @@ def test_add_records(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': 'About Us',
+                'lvl1': None,
+                'lvl2': None,
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'version': '1.0',
             'anchor': 'Learn More',
             'content': 'Discover more about our company values',
@@ -406,7 +443,17 @@ def test_add_records(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': 'About Us',
+                'lvl1': None,
+                'lvl2': None,
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'hierarchy.lvl0': 'About Us',
+            'hierarchy_radio.lvl0': 'About Us',
             'id': '1',
             'item_priority': 2000001001,
             'language': 'en',
@@ -429,9 +476,19 @@ def test_add_records(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': None,
+                'lvl1': None,
+                'lvl2': 'Electronics',
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'hierarchy.lvl0': 'Home',
             'hierarchy.lvl1': 'Products',
             'hierarchy.lvl2': 'Electronics',
+            'hierarchy_radio.lvl2': 'Electronics',
             'id': '0',
             'item_priority': 1000002003,
             'language': 'en',
@@ -473,6 +530,15 @@ def test_commit_tmp_collection(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': None,
+                'lvl1': None,
+                'lvl2': 'Electronics',
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'version': '1.0,2.0',
             'anchor': 'See More',
             'content': 'Explore our range of electronics',
@@ -493,6 +559,15 @@ def test_commit_tmp_collection(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': 'About Us',
+                'lvl1': None,
+                'lvl2': None,
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'version': '1.0',
             'anchor': 'Learn More',
             'content': 'Discover more about our company values',
@@ -568,6 +643,15 @@ def test_commit_tmp_collection_with_curation_rules(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': None,
+                'lvl1': None,
+                'lvl2': 'Electronics',
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'version': '1.0,2.0',
             'anchor': 'See More',
             'content': 'Explore our range of electronics',
@@ -588,6 +672,15 @@ def test_commit_tmp_collection_with_curation_rules(typesense_client):
                 'lvl5': None,
                 'lvl6': None,
             },
+            'hierarchy_radio': {
+                'lvl0': 'About Us',
+                'lvl1': None,
+                'lvl2': None,
+                'lvl3': None,
+                'lvl4': None,
+                'lvl5': None,
+                'lvl6': None,
+            },
             'version': '1.0',
             'anchor': 'Learn More',
             'content': 'Discover more about our company values',

diff --git a/scraper/src/typesense_helper.py b/scraper/src/typesense_helper.py
@@ -207,11 +207,12 @@ def transform_record(record):
             + transformed_record['weight']['position_descending']
         )
 
-        # Flatten nested hierarchy field
+        # Flatten nested hierarchy fields
         for x in range(0, 7):
-            if record['hierarchy'][f'lvl{x}'] is None:
-                continue
-            transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}']
+            if record['hierarchy'][f'lvl{x}'] is not None:
+                transformed_record[f'hierarchy.lvl{x}'] = record['hierarchy'][f'lvl{x}']
+            if record['hierarchy_radio'][f'lvl{x}'] is not None:
+                transformed_record[f'hierarchy_radio.lvl{x}'] = record['hierarchy_radio'][f'lvl{x}']
 
         # Convert version to array
         if 'version' in record and type(record['version']) == str: