diff --git a/hamilton/plugins/.cph/.pandas_extensions.py_c745125af4be0661754c2da2281c6cd1.prob b/hamilton/plugins/.cph/.pandas_extensions.py_c745125af4be0661754c2da2281c6cd1.prob new file mode 100644 index 000000000..9343a622e --- /dev/null +++ b/hamilton/plugins/.cph/.pandas_extensions.py_c745125af4be0661754c2da2281c6cd1.prob @@ -0,0 +1 @@ +{"name":"Local: pandas_extensions","url":"d:\\avinash_files\\hamiltoon\\hamilton\\hamilton\\plugins\\pandas_extensions.py","tests":[{"id":1768885818931,"input":"","output":""}],"interactive":false,"memoryLimit":1024,"timeLimit":3000,"srcPath":"d:\\avinash_files\\hamiltoon\\hamilton\\hamilton\\plugins\\pandas_extensions.py","group":"local","local":true} \ No newline at end of file diff --git a/hamilton/plugins/pandas_extensions.py b/hamilton/plugins/pandas_extensions.py index b4ce2786d..6acc157e4 100644 --- a/hamilton/plugins/pandas_extensions.py +++ b/hamilton/plugins/pandas_extensions.py @@ -23,6 +23,7 @@ from io import BufferedReader, BytesIO, StringIO from pathlib import Path from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, Union +from packaging.version import Version try: import pandas as pd @@ -171,6 +172,7 @@ def applicable_types(cls) -> Collection[Type]: def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs = {} + pandas_version = Version(pd.__version__) if self.sep is not None: kwargs["sep"] = self.sep if self.delimiter is not None: @@ -205,14 +207,10 @@ def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs["keep_default_na"] = self.keep_default_na if self.na_filter is not None: kwargs["na_filter"] = self.na_filter - if self.verbose is not None: - kwargs["verbose"] = self.verbose if self.skip_blank_lines is not None: kwargs["skip_blank_lines"] = self.skip_blank_lines if self.parse_dates is not None: kwargs["parse_dates"] = self.parse_dates - if self.keep_date_col is not None: - kwargs["keep_date_col"] = self.keep_date_col if self.date_format is not None: kwargs["date_format"] = self.date_format if self.dayfirst is not None: @@ -247,8 +245,6 @@ def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs["dialect"] = self.dialect if self.on_bad_lines is not None: kwargs["on_bad_lines"] = self.on_bad_lines - if self.delim_whitespace is not None: - kwargs["delim_whitespace"] = self.delim_whitespace if self.low_memory is not None: kwargs["low_memory"] = self.low_memory if self.memory_map is not None: @@ -257,8 +253,17 @@ def _get_loading_kwargs(self) -> Dict[str, Any]: kwargs["float_precision"] = self.float_precision if self.storage_options is not None: kwargs["storage_options"] = self.storage_options - if pd.__version__ >= "2.0" and self.dtype_backend is not None: - kwargs["dtype_backend"] = self.dtype_backend + if pandas_version < Version("2.0"): + if self.keep_date_col is not None: + kwargs["keep_date_col"] = self.keep_date_col + if self.verbose is not None: + kwargs["verbose"] = self.verbose + if self.delim_whitespace is not None: + kwargs["delim_whitespace"] = self.delim_whitespace + + if pandas_version >= Version("2.0"): + if self.dtype_backend is not None: + kwargs["dtype_backend"] = self.dtype_backend return kwargs