Merge pull request #2553 from tkalir/bug-2547-memory-issue-in-fill-ca…

…che-for-streets

Bug 2547 memory issue in fill cache for streets

anyway/parsers/infographics_data_cache_updater.py

@@ -21,6 +21,7 @@
 from anyway.request_params import RequestParams
 import anyway.infographics_utils
 from anyway.widgets.widget import widgets_dict
+from anyway.utilities import chunked_generator
 import logging
 import json
 
@@ -287,18 +288,19 @@ def build_street_cache_into_temp():
     start = datetime.now()
     db.session.query(InfographicsStreetDataCacheTemp).delete()
     db.session.commit()
-    db.get_engine().execute(
-        InfographicsStreetDataCacheTemp.__table__.insert(),  # pylint: disable=no-member
-        [
-            {
-                "yishuv_symbol": d["yishuv_symbol"],
-                "street": d["street1"],
-                "years_ago": d["years_ago"],
-                "data": anyway.infographics_utils.create_infographics_data_for_location(d),
-            }
-            for d in get_street_infographic_keys()
-        ],
-    )
+    for chunk in chunked_generator(get_street_infographic_keys(), 4960):
+        db.get_engine().execute(
+            InfographicsStreetDataCacheTemp.__table__.insert(),  # pylint: disable=no-member
+            [
+                {
+                    "yishuv_symbol": d["yishuv_symbol"],
+                    "street": d["street1"],
+                    "years_ago": d["years_ago"],
+                    "data": anyway.infographics_utils.create_infographics_data_for_location(d),
+                }
+                for d in chunk
+            ],
+        )
     db.session.commit()
     logging.info(f"cache rebuild took:{str(datetime.now() - start)}")
 

anyway/utilities.py

@@ -259,6 +259,18 @@ def chunks(l, n):
         yield l[i : i + n]
 
 
+def chunked_generator(input_generator, chunk_size):
+    chunk = []
+    for item in input_generator:
+        chunk.append(item)
+        if len(chunk) == chunk_size:
+            yield chunk
+            chunk = []
+
+    if chunk:
+        yield chunk
+
+
 def parse_age_from_range(age_range: int) -> typing.Optional[typing.Tuple[int, int]]:
     # Convert from 'age_group' field in the table 'involved_markers_hebrew' to age range numbers
     ret_age_code_to_age_range = {