Log Consumables That are Assumed to Be Used (#1510)

* additional logger entry in consumables.py to record items used

* update test_consumables.py with optional argument

* fix failing test

* rollback analysis script

* Update src/tlo/methods/consumables.py

Co-authored-by: Tim Hallett <[email protected]>

* Update src/tlo/methods/consumables.py

Co-authored-by: Tim Hallett <[email protected]>

* Update src/tlo/methods/consumables.py

Co-authored-by: Tim Hallett <[email protected]>

* add test for new consumables logger

* fix failing test test_two_loggers_in_healthsystem

---------

Co-authored-by: Tim Hallett <[email protected]>

src/tlo/methods/consumables.py

@@ -60,7 +60,7 @@ def __init__(self,
         self._item_code_designations = item_code_designations
 
         # Save all item_codes that are defined and pd.Series with probs of availability from ResourceFile
-        self.item_codes,  self._processed_consumables_data = \
+        self.item_codes, self._processed_consumables_data = \
             self._process_consumables_data(availability_data=availability_data)
 
         # Set the availability based on the argument provided (this can be updated later after the class is initialised)
@@ -199,7 +199,8 @@ def _determine_default_return_value(cons_availability, default_return_value):
 
     def _request_consumables(self,
                              facility_info: 'FacilityInfo',  # noqa: F821
-                             item_codes: dict,
+                             essential_item_codes: dict,
+                             optional_item_codes: Optional[dict] = None,
                              to_log: bool = True,
                              treatment_id: Optional[str] = None
                              ) -> dict:
@@ -208,41 +209,52 @@ def _request_consumables(self,
 
         :param facility_info: The facility_info from which the request for consumables originates
         :param item_codes: dict of the form {<item_code>: <quantity>} for the items requested
+        :param optional_item_codes: dict of the form {<item_code>: <quantity>} for the optional items requested
         :param to_log: whether the request is logged.
         :param treatment_id: the TREATMENT_ID of the HSI (which is entered to the log, if provided).
         :return: dict of the form {<item_code>: <bool>} indicating the availability of each item requested.
         """
+        # If optional_item_codes is None, treat it as an empty dictionary
+        optional_item_codes = optional_item_codes or {}
+        _all_item_codes = {**essential_item_codes, **optional_item_codes}
 
         # Issue warning if any item_code is not recognised.
-        not_recognised_item_codes = item_codes.keys() - self.item_codes
+        not_recognised_item_codes = _all_item_codes.keys() - self.item_codes
         if len(not_recognised_item_codes) > 0:
             self._not_recognised_item_codes[treatment_id] |= not_recognised_item_codes
 
         # Look-up whether each of these items is available in this facility currently:
-        available = self._lookup_availability_of_consumables(item_codes=item_codes, facility_info=facility_info)
+        available = self._lookup_availability_of_consumables(item_codes=_all_item_codes, facility_info=facility_info)
 
         # Log the request and the outcome:
         if to_log:
-            items_available = {k: v for k, v in item_codes.items() if available[k]}
-            items_not_available = {k: v for k, v in item_codes.items() if not available[k]}
-            logger.info(key='Consumables',
-                        data={
-                            'TREATMENT_ID': (treatment_id if treatment_id is not None else ""),
-                            'Item_Available': str(items_available),
-                            'Item_NotAvailable': str(items_not_available),
-                        },
-                        # NB. Casting the data to strings because logger complains with dict of varying sizes/keys
-                        description="Record of each consumable item that is requested."
-                        )
-
-            self._summary_counter.record_availability(items_available=items_available,
-                                                      items_not_available=items_not_available)
+            items_available = {k: v for k, v in _all_item_codes.items() if available[k]}
+            items_not_available = {k: v for k, v in _all_item_codes.items() if not available[k]}
+
+            # Log items used if all essential items are available
+            items_used = items_available if all(available.get(k, False) for k in essential_item_codes) else {}
+
+            logger.info(
+                key='Consumables',
+                data={
+                    'TREATMENT_ID': treatment_id or "",
+                    'Item_Available': str(items_available),
+                    'Item_NotAvailable': str(items_not_available),
+                    'Item_Used': str(items_used),
+                },
+                description="Record of requested and used consumable items."
+            )
+            self._summary_counter.record_availability(
+                items_available=items_available,
+                items_not_available=items_not_available,
+                items_used=items_used,
+            )
 
         # Return the result of the check on availability
         return available
 
     def _lookup_availability_of_consumables(self,
-                                            facility_info: 'FacilityInfo',   # noqa: F821
+                                            facility_info: 'FacilityInfo',  # noqa: F821
                                             item_codes: dict
                                             ) -> dict:
         """Lookup whether a particular item_code is in the set of available items for that facility (in
@@ -267,12 +279,12 @@ def _lookup_availability_of_consumables(self,
 
     def on_simulation_end(self):
         """Do tasks at the end of the simulation.
-         
+
         Raise warnings and enter to log about item_codes not recognised.
         """
         if len(self._not_recognised_item_codes) > 0:
             not_recognised_item_codes = {
-                treatment_id if treatment_id is not None else "": sorted(codes) 
+                treatment_id if treatment_id is not None else "": sorted(codes)
                 for treatment_id, codes in self._not_recognised_item_codes.items()
             }
             warnings.warn(
@@ -363,10 +375,11 @@ def _reset_internal_stores(self) -> None:
 
         self._items = {
             'Available': defaultdict(int),
-            'NotAvailable': defaultdict(int)
+            'NotAvailable': defaultdict(int),
+            'Used': defaultdict(int),
         }
 
-    def record_availability(self, items_available: dict, items_not_available: dict) -> None:
+    def record_availability(self, items_available: dict, items_not_available: dict, items_used: dict) -> None:
         """Add information about the availability of requested items to the running summaries."""
 
         # Record items that were available
@@ -377,6 +390,10 @@ def record_availability(self, items_available: dict, items_not_available: dict)
         for _item, _num in items_not_available.items():
             self._items['NotAvailable'][_item] += _num
 
+        # Record items that were used
+        for _item, _num in items_used.items():
+            self._items['Used'][_item] += _num
+
     def write_to_log_and_reset_counters(self):
         """Log summary statistics and reset the data structures."""
 
@@ -387,6 +404,7 @@ def write_to_log_and_reset_counters(self):
             data={
                 "Item_Available": self._items['Available'],
                 "Item_NotAvailable": self._items['NotAvailable'],
+                "Item_Used": self._items['Used'],
             },
         )
 

src/tlo/methods/hsi_event.py

@@ -225,7 +225,8 @@ def get_consumables(
 
         # Checking the availability and logging:
         rtn = self.healthcare_system.consumables._request_consumables(
-            item_codes={**_item_codes, **_optional_item_codes},
+            essential_item_codes=_item_codes,
+            optional_item_codes=_optional_item_codes,
             to_log=_to_log,
             facility_info=self.facility_info,
             treatment_id=self.TREATMENT_ID,

tests/test_consumables.py

@@ -61,7 +61,7 @@ def test_using_recognised_item_codes(seed):
 
     # Make requests for consumables (which would normally come from an instance of `HSI_Event`).
     rtn = cons._request_consumables(
-        item_codes={0: 1, 1: 1},
+        essential_item_codes={0: 1, 1: 1},
         facility_info=facility_info_0
     )
 
@@ -88,7 +88,7 @@ def test_unrecognised_item_code_is_recorded(seed):
 
     # Make requests for consumables (which would normally come from an instance of `HSI_Event`).
     rtn = cons._request_consumables(
-        item_codes={99: 1},
+        essential_item_codes={99: 1},
         facility_info=facility_info_0
     )
 
@@ -128,7 +128,8 @@ def test_consumables_availability_options(seed):
         cons.on_start_of_day(date=date)
 
         assert _expected_result == cons._request_consumables(
-            item_codes={_item_code: 1 for _item_code in all_items_request}, to_log=False, facility_info=facility_info_0
+            essential_item_codes={_item_code: 1 for _item_code in all_items_request},
+            to_log=False, facility_info=facility_info_0
         )
 
 
@@ -153,7 +154,8 @@ def request_item(cons, item_code: Union[list, int]):
             item_code = [item_code]
 
         return all(cons._request_consumables(
-            item_codes={_i: 1 for _i in item_code}, to_log=False, facility_info=facility_info_0
+            essential_item_codes={_i: 1 for _i in item_code},
+            to_log=False, facility_info=facility_info_0
         ).values())
 
     rng = get_rng(seed)
@@ -250,7 +252,7 @@ def test_consumables_available_at_right_frequency(seed):
     for _ in range(n_trials):
         cons.on_start_of_day(date=date)
         rtn = cons._request_consumables(
-            item_codes=requested_items,
+            essential_item_codes=requested_items,
             facility_info=facility_info_0,
         )
         for _i in requested_items:
@@ -273,6 +275,47 @@ def is_obs_frequency_consistent_with_expected_probability(n_obs, n_trials, p):
                                                                  p=average_availability_of_known_items)
 
 
+@pytest.mark.parametrize("p_known_items, expected_items_used", [
+    # Test 1
+    ({0: 0.0, 1: 1.0, 2: 1.0, 3: 1.0}, {}),
+    # Test 2
+    ({0: 1.0, 1: 1.0, 2: 0.0, 3: 1.0}, {0: 5, 1: 10, 3: 2})
+])
+def test_items_used_includes_only_available_items(seed, p_known_items, expected_items_used):
+    """
+    Test that 'items_used' includes only items that are available.
+    Items should only be logged if the essential items are ALL available
+    If essential items are available, then optional items can be logged as items_used if available
+    Test 1: should not have any items_used as essential item 0 is not available
+    Test 2: should have essential items logged as items_used, but optional item 2 is not available
+    """
+
+    data = create_dummy_data_for_cons_availability(
+        intrinsic_availability=p_known_items,
+        months=[1],
+        facility_ids=[0]
+    )
+    rng = get_rng(seed)
+    date = datetime.datetime(2010, 1, 1)
+
+    cons = Consumables(availability_data=data, rng=rng)
+
+    # Define essential and optional item codes
+    essential_item_codes = {0: 5, 1: 10}  # these must match parameters above
+    optional_item_codes = {2: 7, 3: 2}
+
+    cons.on_start_of_day(date=date)
+    cons._request_consumables(
+        essential_item_codes=essential_item_codes,
+        optional_item_codes=optional_item_codes,
+        facility_info=facility_info_0,
+    )
+
+    # Access items used from the Consumables summary counter
+    items_used = getattr(cons._summary_counter, '_items', {}).get('Used')
+    assert items_used == expected_items_used, f"Expected items_used to be {expected_items_used}, but got {items_used}"
+
+
 def get_sim_with_dummy_module_registered(tmpdir=None, run=True, data=None):
     """Return an initialised simulation object with a Dummy Module registered. If the `data` argument is provided,
     the parameter in HealthSystem that holds the data on consumables availability is over-written."""

tests/test_healthsystem.py

@@ -952,7 +952,7 @@ def apply(self, person_id, squeeze_factor):
             } == set(detailed_hsi_event.columns)
     assert {'date', 'Frac_Time_Used_Overall', 'Frac_Time_Used_By_Facility_ID', 'Frac_Time_Used_By_OfficerType',
             } == set(detailed_capacity.columns)
-    assert {'date', 'TREATMENT_ID', 'Item_Available', 'Item_NotAvailable'
+    assert {'date', 'TREATMENT_ID', 'Item_Available', 'Item_NotAvailable', 'Item_Used'
             } == set(detailed_consumables.columns)
 
     bed_types = sim.modules['HealthSystem'].bed_days.bed_types
@@ -1019,6 +1019,9 @@ def dict_all_close(dict_1, dict_2):
     assert summary_consumables['Item_NotAvailable'].apply(pd.Series).sum().to_dict() == \
            detailed_consumables['Item_NotAvailable'].apply(
                lambda x: {f'{k}': v for k, v in eval(x).items()}).apply(pd.Series).sum().to_dict()
+    assert summary_consumables['Item_Used'].apply(pd.Series).sum().to_dict() == \
+           detailed_consumables['Item_Used'].apply(
+               lambda x: {f'{k}': v for k, v in eval(x).items()}).apply(pd.Series).sum().to_dict()
 
     #  - Bed-Days (bed-type by bed-type and year by year)
     for _bed_type in bed_types: