Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
{"name":"Local: pandas_extensions","url":"d:\\avinash_files\\hamiltoon\\hamilton\\hamilton\\plugins\\pandas_extensions.py","tests":[{"id":1768885818931,"input":"","output":""}],"interactive":false,"memoryLimit":1024,"timeLimit":3000,"srcPath":"d:\\avinash_files\\hamiltoon\\hamilton\\hamilton\\plugins\\pandas_extensions.py","group":"local","local":true}
21 changes: 13 additions & 8 deletions hamilton/plugins/pandas_extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from io import BufferedReader, BytesIO, StringIO
from pathlib import Path
from typing import Any, Callable, Collection, Dict, Iterator, List, Optional, Tuple, Type, Union
from packaging.version import Version

try:
import pandas as pd
Expand Down Expand Up @@ -171,6 +172,7 @@ def applicable_types(cls) -> Collection[Type]:

def _get_loading_kwargs(self) -> Dict[str, Any]:
kwargs = {}
pandas_version = Version(pd.__version__)
if self.sep is not None:
kwargs["sep"] = self.sep
if self.delimiter is not None:
Expand Down Expand Up @@ -205,14 +207,10 @@ def _get_loading_kwargs(self) -> Dict[str, Any]:
kwargs["keep_default_na"] = self.keep_default_na
if self.na_filter is not None:
kwargs["na_filter"] = self.na_filter
if self.verbose is not None:
kwargs["verbose"] = self.verbose
if self.skip_blank_lines is not None:
kwargs["skip_blank_lines"] = self.skip_blank_lines
if self.parse_dates is not None:
kwargs["parse_dates"] = self.parse_dates
if self.keep_date_col is not None:
kwargs["keep_date_col"] = self.keep_date_col
if self.date_format is not None:
kwargs["date_format"] = self.date_format
if self.dayfirst is not None:
Expand Down Expand Up @@ -247,8 +245,6 @@ def _get_loading_kwargs(self) -> Dict[str, Any]:
kwargs["dialect"] = self.dialect
if self.on_bad_lines is not None:
kwargs["on_bad_lines"] = self.on_bad_lines
if self.delim_whitespace is not None:
kwargs["delim_whitespace"] = self.delim_whitespace
if self.low_memory is not None:
kwargs["low_memory"] = self.low_memory
if self.memory_map is not None:
Expand All @@ -257,8 +253,17 @@ def _get_loading_kwargs(self) -> Dict[str, Any]:
kwargs["float_precision"] = self.float_precision
if self.storage_options is not None:
kwargs["storage_options"] = self.storage_options
if pd.__version__ >= "2.0" and self.dtype_backend is not None:
kwargs["dtype_backend"] = self.dtype_backend
if pandas_version < Version("2.0"):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You need to import Version at the top of the file -- then tests should pass...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can you check now @skrawcz

if self.keep_date_col is not None:
kwargs["keep_date_col"] = self.keep_date_col
if self.verbose is not None:
kwargs["verbose"] = self.verbose
if self.delim_whitespace is not None:
kwargs["delim_whitespace"] = self.delim_whitespace

if pandas_version >= Version("2.0"):
if self.dtype_backend is not None:
kwargs["dtype_backend"] = self.dtype_backend

return kwargs

Expand Down