diff --git a/Changelog.rst b/Changelog.rst index 1bbaf8583d..cbc00cf8d3 100644 --- a/Changelog.rst +++ b/Changelog.rst @@ -1,10 +1,22 @@ Version NEXTVERSION ------------------- -**2025-12-??** - +**2026-01-??** + +* New function to control the creation of cached elements during data + display: `cf.display_data` + (https://github.com/NCAS-CMS/cf-python/issues/913) +* New methods: `cf.Data.get_cached_elements`, + `cf.Data.cache_elements` + (https://github.com/NCAS-CMS/cf-python/issues/913) +* Set cached elements during `cf.Data.__init__` + (https://github.com/NCAS-CMS/cf-python/issues/913) +* Removed the `cf.constants.CONSTANTS` dictionary, replacing it + with `cf.ConstantAccess.constants` + (https://github.com/NCAS-CMS/cf-python/issues/902) * Reduce the time taken to import `cf` (https://github.com/NCAS-CMS/cf-python/issues/902) +* Changed dependency: ``cfdm>=1.12.4.0, <1.12.5.0`` ---- diff --git a/cf/functions.py b/cf/functions.py index f63b037305..7fa8a3d08a 100644 --- a/cf/functions.py +++ b/cf/functions.py @@ -156,6 +156,7 @@ def configuration( tempdir=None, chunksize=None, log_level=None, + display_data=None, regrid_logging=None, relaxed_identities=None, bounds_combination_mode=None, @@ -177,6 +178,7 @@ def configuration( * `tempdir` * `chunksize` * `log_level` + * `display_data` * `regrid_logging` * `relaxed_identities` * `bounds_combination_mode` @@ -200,10 +202,10 @@ def configuration( .. versionadded:: 3.6.0 .. seealso:: `atol`, `rtol`, `tempdir`, `chunksize`, - `total_memory`, `log_level`, `regrid_logging`, - `relaxed_identities`, `bounds_combination_mode`, - `active_storage`, `active_storage_url`, - `active_storage_max_requests` + `total_memory`, `log_level`, `display_data`, + `regrid_logging`, `relaxed_identities`, + `bounds_combination_mode`, `active_storage`, + `active_storage_url`, `active_storage_max_requests` :Parameters: @@ -245,6 +247,12 @@ def configuration( * ``'DETAIL'`` (``3``); * ``'DEBUG'`` (``-1``). + display_data `bool` or `Constant`, optional + The new display data option. The default is to not change + the current behaviour. + + .. versionadded:: NEXTVERSION + regrid_logging: `bool` or `Constant`, optional The new value (either True to enable logging or False to disable it). The default is to not change the current @@ -303,6 +311,7 @@ def configuration( 'log_level': 'WARNING', 'bounds_combination_mode': 'AND', 'chunksize': 82873466.88000001, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -320,6 +329,7 @@ def configuration( 'log_level': 'WARNING', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -347,6 +357,7 @@ def configuration( 'log_level': 'INFO', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None} >>> with cf.configuration(atol=9, rtol=10): @@ -360,6 +371,7 @@ def configuration( 'log_level': 'INFO', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -372,6 +384,7 @@ def configuration( 'log_level': 'INFO', 'bounds_combination_mode': 'AND', 'chunksize': 75000000.0, + 'display_data': True, 'active_storage': False, 'active_storage_url': None, 'active_storage_max_requests': 100} @@ -402,6 +415,7 @@ def configuration( new_tempdir=tempdir, new_chunksize=chunksize, new_log_level=log_level, + new_display_data=display_data, new_regrid_logging=regrid_logging, new_relaxed_identities=relaxed_identities, bounds_combination_mode=bounds_combination_mode, @@ -445,6 +459,7 @@ def _configuration(_Configuration, **kwargs): "new_tempdir": tempdir, "new_chunksize": chunksize, "new_log_level": log_level, + "new_display_data": display_data, "new_regrid_logging": regrid_logging, "new_relaxed_identities": relaxed_identities, "bounds_combination_mode": bounds_combination_mode, @@ -459,10 +474,6 @@ def _configuration(_Configuration, **kwargs): old = ConstantAccess.constants(copy=True) - # old = {name.lower(): val for name, val in CONSTANTS.items()} - # - # old.pop("total_memory", None) - # Filter out 'None' kwargs from configuration() defaults. Note that this # does not filter out '0' or 'True' values, which is important as the user # might be trying to set those, as opposed to None emerging as default. @@ -552,7 +563,6 @@ def FREE_MEMORY(): # Functions inherited from cfdm # -------------------------------------------------------------------- class ConstantAccess(cfdm.ConstantAccess): - _constants = {} _Constant = Constant def __docstring_substitutions__(self): @@ -576,6 +586,10 @@ class log_level(ConstantAccess, cfdm.log_level): _reset_log_emergence_level = _reset_log_emergence_level +class display_data(ConstantAccess, cfdm.display_data): + pass + + class regrid_logging(ConstantAccess): """Whether or not to enable `esmpy` regridding logging. diff --git a/cf/mixin/properties.py b/cf/mixin/properties.py index dd7d80fa86..326c286678 100644 --- a/cf/mixin/properties.py +++ b/cf/mixin/properties.py @@ -7,8 +7,6 @@ _DEPRECATION_ERROR_KWARGS, _DEPRECATION_ERROR_METHOD, ) -from ..functions import atol as cf_atol -from ..functions import rtol as cf_rtol from ..mixin_container import Container from ..query import Query from ..units import Units @@ -34,32 +32,6 @@ def __new__(cls, *args, **kwargs): instance._Data = Data return instance - # ---------------------------------------------------------------- - # Private attributes - # ---------------------------------------------------------------- - @property - def _atol(self): - """Return the tolerance on absolute differences between real - numbers, as returned by the `cf.atol` function. - - This is used by, for example, the `_equals` method. - - """ - return cf_atol().value - - @property - def _rtol(self): - """Return the tolerance on relative differences between real - numbers, as returned by the `cf.rtol` function. - - This is used by, for example, the `_equals` method. - - """ - return cf_rtol().value - - # ---------------------------------------------------------------- - # Private methods - # ---------------------------------------------------------------- def _matching_values(self, value0, value1, units=False, basic=False): """Whether two values match. @@ -100,9 +72,6 @@ def _matching_values(self, value0, value1, units=False, basic=False): return self._equals(value1, value0, basic=basic) - # ---------------------------------------------------------------- - # Attributes - # ---------------------------------------------------------------- @property def id(self): """An identity for the {{class}} object. @@ -150,9 +119,6 @@ def id(self): f"{self.__class__.__name__} doesn't have attribute 'id'" ) - # ---------------------------------------------------------------- - # CF properties - # ---------------------------------------------------------------- @property def calendar(self): """The calendar CF property. @@ -554,9 +520,6 @@ def valid_range(self, value): def valid_range(self): self.del_property("valid_range", default=AttributeError()) - # ---------------------------------------------------------------- - # Methods - # ---------------------------------------------------------------- def get_property(self, prop, default=ValueError()): """Get a CF property. diff --git a/cf/mixin2/container.py b/cf/mixin2/container.py index c5f0081462..44397301f4 100644 --- a/cf/mixin2/container.py +++ b/cf/mixin2/container.py @@ -6,7 +6,6 @@ """ from ..docstring import _docstring_substitution_definitions -from ..functions import atol, rtol class Container: @@ -55,23 +54,3 @@ def __docstring_package_depth__(self): """ return 0 - - @property - def _atol(self): - """Internal alias for `{{package}}.atol`. - - An alias is necessary to avoid a name clash with the keyword - argument of identical name (`atol`) in calling functions. - - """ - return atol().value - - @property - def _rtol(self): - """Internal alias for `{{package}}.rtol`. - - An alias is necessary to avoid a name clash with the keyword - argument of identical name (`rtol`) in calling functions. - - """ - return rtol().value diff --git a/cf/read_write/um/umread.py b/cf/read_write/um/umread.py index 8f21984094..9daf1c4612 100644 --- a/cf/read_write/um/umread.py +++ b/cf/read_write/um/umread.py @@ -3575,7 +3575,7 @@ def read( # Return now if there are valid file types return [] - f = self.file_open(filename, parse=True) + f = self.dataset_open(filename, parse=True) info = is_log_level_info(logger) @@ -3598,7 +3598,7 @@ def read( for var in f.vars ] - self.file_close() + self.dataset_close() return [field for x in um for field in x.fields if field] @@ -3632,7 +3632,7 @@ def _open_um_file( The open PP or FF file object. """ - self.file_close() + self.dataset_close() try: f = File( filename, @@ -3678,15 +3678,15 @@ def is_um_file(self, filename): try: # Note: No need to completely parse the file to ascertain # if it's PP or FF. - self.file_open(filename, parse=False) + self.dataset_open(filename, parse=False) except Exception: - self.file_close() + self.dataset_close() return False else: - self.file_close() + self.dataset_close() return True - def file_close(self): + def dataset_close(self): """Close the file that has been read. :Returns: @@ -3700,7 +3700,7 @@ def file_close(self): self._um_file = None - def file_open(self, filename, parse=True): + def dataset_open(self, filename, parse=True): """Open the file for reading. :Paramters: diff --git a/cf/test/test_Data.py b/cf/test/test_Data.py index 2300a62e47..21bdff254d 100644 --- a/cf/test/test_Data.py +++ b/cf/test/test_Data.py @@ -1165,7 +1165,7 @@ def test_Data_concatenate(self): str(d) str(e) f = cf.Data.concatenate([d, e], axis=1) - cached = f._get_cached_elements() + cached = f.get_cached_elements() self.assertEqual(cached[0], d.first_element()) self.assertEqual(cached[-1], e.last_element()) @@ -1205,7 +1205,7 @@ def test_Data_concatenate(self): repr(e) f = cf.Data.concatenate([d, e], axis=0) self.assertEqual( - f._get_cached_elements(), + f.get_cached_elements(), {0: d.first_element(), -1: e.last_element()}, ) @@ -1709,6 +1709,12 @@ def test_Data_array(self): d = cf.Data([["2000-12-3 12:00"]], "days since 2000-12-01", dt=True) self.assertEqual(d.array, 2.5) + # Cached values + d = cf.Data([1, 2]) + d._del_cached_elements() + d.array + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 2}) + def test_Data_binary_mask(self): """Test the `binary_mask` Data property.""" d = cf.Data([[0, 1, 2, 3.0]], "m") @@ -3221,6 +3227,12 @@ def test_Data_compute(self): d = cf.Data([["2000-12-3 12:00"]], "days since 2000-12-01", dt=True) self.assertEqual(d.compute(), 2.5) + # Cached values + d = cf.Data([1, 2]) + d._del_cached_elements() + d.compute() + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 2}) + def test_Data_persist(self): """Test Data.persist.""" d = cf.Data(9, "km") @@ -4376,9 +4388,8 @@ def test_Data_Units(self): # Adjusted cached values d = cf.Data([1000, 2000, 3000], "m") - repr(d) d.Units = cf.Units("km") - self.assertEqual(d._get_cached_elements(), {0: 1.0, 1: 2.0, -1: 3.0}) + self.assertEqual(d.get_cached_elements(), {0: 1.0, 1: 2.0, -1: 3.0}) def test_Data_get_data(self): """Test the `get_data` Data method.""" @@ -4450,53 +4461,24 @@ def test_Data__str__(self): """Test `Data.__str__`""" elements0 = (0, -1, 1) for array in ([1], [1, 2], [1, 2, 3]): - elements = elements0[: len(array)] - d = cf.Data(array) - cache = d._get_cached_elements() - for element in elements: - self.assertNotIn(element, cache) - - self.assertEqual(str(d), str(array)) - cache = d._get_cached_elements() - for element in elements: - self.assertIn(element, cache) - d[0] = 1 - cache = d._get_cached_elements() - for element in elements: - self.assertNotIn(element, cache) - self.assertEqual(str(d), str(array)) - cache = d._get_cached_elements() - for element in elements: - self.assertIn(element, cache) - d += 0 - cache = d._get_cached_elements() - for element in elements: - self.assertNotIn(element, cache) - self.assertEqual(str(d), str(array)) - cache = d._get_cached_elements() - for element in elements: - self.assertIn(element, cache) # Test when size > 3, i.e. second element is not there. d = cf.Data([1, 2, 3, 4]) - cache = d._get_cached_elements() - for element in elements0: - self.assertNotIn(element, cache) self.assertEqual(str(d), "[1, ..., 4]") - cache = d._get_cached_elements() + cache = d.get_cached_elements() self.assertNotIn(1, cache) for element in elements0[:2]: self.assertIn(element, cache) d[0] = 1 for element in elements0: - self.assertNotIn(element, d._get_cached_elements()) + self.assertNotIn(element, d.get_cached_elements()) def test_Data_cull_graph(self): """Test Data.cull_graph.""" @@ -4703,6 +4685,135 @@ def test_Data_to_units(self): with self.assertRaises(ValueError): e.to_units("degC") + def test_Data_cache_elements(self): + """Test setting of cached elements.""" + d = cf.Data(1) + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 1}) + self.assertIsNone(d._del_cached_elements()) + self.assertFalse(d.get_cached_elements()) + + # Test via __init__, which calls `cache_elements` + for array in (np.ma.masked, True, "x"): + d = cf.Data(array) + for i in range(2): + self.assertEqual( + d.get_cached_elements(), {0: array, -1: array} + ) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in ( + 1, + 1.0, + np.array(1), + np.array([1]), + [1], + (1,), + ): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 1}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([1, 2]), [1, 2]): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 2}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([1, 2, 3]), (1, 2, 3)): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 3}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([1, 2, 3, 4]), (1, 2, 3, 4)): + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 4}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in (np.array([[1, 2]]), [[1, 2]]): + d = cf.Data(array) + for i in range(2): + self.assertEqual( + d.get_cached_elements(), {0: 1, 1: 2, -2: 1, -1: 2} + ) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + for array in ( + np.array([[1, 2], [7, 8]]), + ([1, 2], [7, 8]), + np.array([[1, 2], [3, 4], [7, 8]]), + [[1, 2], [3, 4], [5, 6], [7, 8]], + ): + d = cf.Data(array) + for i in range(2): + self.assertEqual( + d.get_cached_elements(), {0: 1, 1: 2, -2: 7, -1: 8} + ) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + # Sparse array + from scipy.sparse import csr_array + + indptr = np.array([0, 2, 3, 6]) + indices = np.array([0, 2, 2, 0, 1, 2]) + data = np.array([1, 2, 3, 4, 5, 6]) + array = csr_array((data, indices, indptr), shape=(3, 3)) + d = cf.Data(array) + for i in range(2): + self.assertEqual(d.get_cached_elements(), {0: 1, -1: 6}) + # Check that getting the array doesn't change the + # cached elements + if i: + d.array + + # Check set_cached_elements + for array in ([1, 2, 3], [[1, 2, 3]]): + d = cf.Data(array) + d._del_cached_elements() + d.cache_elements() + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 3}) + + # Check that __str__ sets missing cached elements + d = cf.Data([[1, 2, 3]]) + d._del_cached_elements() + str(d) + self.assertEqual(d.get_cached_elements(), {0: 1, 1: 2, -1: 3}) + + # Interaction with `cf.display_data` + d = cf.Data([[1, 2, 3]]) + d._del_cached_elements() + with cf.display_data(False): + self.assertEqual(repr(d), "") + + with cf.display_data(True): + self.assertEqual(repr(d), "") + + with cf.display_data(False): + self.assertEqual(repr(d), "") + if __name__ == "__main__": print("Run date:", datetime.datetime.now()) diff --git a/cf/test/test_functions.py b/cf/test/test_functions.py index 1a26a1afdc..4ca7750e5f 100644 --- a/cf/test/test_functions.py +++ b/cf/test/test_functions.py @@ -55,7 +55,7 @@ def test_configuration(self): self.assertIsInstance(org, dict) # Check all keys that should be there are, with correct value type: - self.assertEqual(len(org), 11) # update expected len if add new key(s) + self.assertEqual(len(org), 12) # update expected len if add new key(s) # Types expected: self.assertIsInstance(org["atol"], float) @@ -70,6 +70,7 @@ def test_configuration(self): # Log level may be input as an int but always given as # equiv. string self.assertIsInstance(org["log_level"], str) + self.assertIsInstance(org["display_data"], bool) # Store some sensible values to reset items to for testing, ensuring: # 1) they are kept different to the defaults (i.e. org values); and