Merge pull request #2 from wrobstory/PR_support-value-arrays

PR_support-value-arrays

malort/stats.py

@@ -158,8 +158,8 @@ def updated_entry_stats(value, current_stats, parse_timestamps=True):
 def recur_dict(value, stats, parent=None, **kwargs):
     """
     Recurse through a dict `value` and update `stats` for each field.
-    Can handle nested dicts and lists of dicts, but will raise exception
-    for list of values
+    Can handle nested dicts, lists of dicts, and lists of values (must be
+    JSON parsable)
 
     Parameters
     ----------
@@ -172,26 +172,31 @@ def recur_dict(value, stats, parent=None, **kwargs):
 
     parent = parent or ''
 
+    def update_stats(current_val, nested_path, base_key):
+        "Updater function"
+        if nested_path not in stats:
+            stats[nested_path] = {}
+        current_stats = stats.get(nested_path)
+        value_type, new_stats = updated_entry_stats(current_val, current_stats,
+                                                    **kwargs)
+        current_stats[value_type] = new_stats
+        current_stats['base_key'] = base_key
+
     if isinstance(value, dict):
         for k, v in value.items():
             parent_path = '.'.join([parent, k]) if parent != '' else k
             if isinstance(v, (list, dict)):
                 recur_dict(v, stats, parent_path)
             else:
-                if parent_path not in stats:
-                    stats[parent_path] = {}
-                current_stats = stats.get(parent_path)
-                value_type, new_stats = updated_entry_stats(v, current_stats,
-                                                            **kwargs)
-                current_stats[value_type] = new_stats
-                current_stats['base_key'] = k
+                update_stats(v, parent_path, k)
 
     elif isinstance(value, list):
         for v in value:
             if isinstance(v, (list, dict)):
                 recur_dict(v, stats, parent)
             else:
-                raise ValueError('List of values found. Malort can only pro'
-                                 'cess key: value pairs!')
+                base_key = parent.split(".")[-1]
+                update_stats(json.dumps(value), parent, base_key)
+                return stats
 
     return stats

malort/tests/test_malort_stats.py

@@ -106,7 +106,8 @@ def test_stats_number(self):
 class TestRecurDict(TestHelpers):
 
     def test_recur_simple(self):
-        simple1 = {'key1': 1, 'key2': 'Foo', 'key3': 4.0, 'key4': True}
+        simple1 = {'key1': 1, 'key2': 'Foo', 'key3': 4.0, 'key4': True,
+                   'key5': ['one', 'two', 'three']}
         expected = {
             'key1': {'int': {'count': 1, 'max': 1, 'mean': 1.0, 'min': 1},
                      'base_key': 'key1'},
@@ -117,7 +118,10 @@ def test_recur_simple(self):
                                'min': 4.0, 'max_precision': 2,
                                'max_scale': 1, 'fixed_length': True},
                      'base_key': 'key3'},
-            'key4': {'bool': {'count': 1}, 'base_key': 'key4'}
+            'key4': {'bool': {'count': 1}, 'base_key': 'key4'},
+            'key5': {'str': {'count': 1, 'max': 23, 'mean': 23.0, 'min': 23,
+                     'sample': ['["one", "two", "three"]']},
+                     'base_key': 'key5'}
         }
 
         stats = mt.stats.recur_dict(simple1, {})
@@ -263,11 +267,36 @@ def test_recur_with_array(self):
         stats = mt.stats.recur_dict(with_list, {})
         self.assert_stats(stats, self.depth_two_expected)
 
-    def test_raises_with_list_of_values(self):
+    def test_recur_with_val_array(self):
+        with_list = {
+          "key1": 1,
+          "key2": ["foo", "bar", "baz"],
+          "key3": [{"key2": ["foo", "bar"]}]
+        }
+
+        stats = mt.stats.recur_dict(with_list, {})
+        expected = {'key1': {'base_key': 'key1',
+                             'int': {'count': 1, 'max': 1,
+                                     'mean': 1.0, 'min': 1}},
+                    'key2': {'base_key': 'key2',
+                             'str': {'count': 1,
+                                     'max': 21,
+                                     'mean': 21.0,
+                                     'min': 21,
+                                     'sample': ['["foo", "bar", "baz"]']}},
+                    'key3.key2': {'base_key': 'key2',
+                                  'str': {'count': 1,
+                                          'max': 14,
+                                          'mean': 14.0,
+                                          'min': 14,
+                                          'sample': ['["foo", "bar"]']}}}
+        self.assert_stats(stats, expected)
+
+    def test_raises_with_list_of_unknown(self):
         with_values = {
             'key1': 'Foo',
-            'key2': ['Foo', 'Bar']
+            'key2': [set('This is a set')]
         }
 
-        with pytest.raises(ValueError):
+        with pytest.raises(TypeError):
             mt.stats.recur_dict(with_values, {})

setup.py

@@ -7,7 +7,7 @@
 
 setup(
     name='malort',
-    version='0.0.1',
+    version='0.0.2',
     description='JSON to Postgres Column Types',
     author='Rob Story',
     author_email='[email protected]',