Skip to content

Commit e3e94c2

Browse files
committed
support cloud versioning
1 parent 290bcaf commit e3e94c2

File tree

2 files changed

+57
-0
lines changed

2 files changed

+57
-0
lines changed

dvc_gs/__init__.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,21 @@
44
from dvc_objects.fs.base import ObjectFileSystem
55
from funcy import cached_property, wrap_prop
66

7+
from .path import GSPath
8+
79

810
class GSFileSystem(ObjectFileSystem):
911
protocol = "gs"
1012
REQUIRES = {"gcsfs": "gcsfs"}
1113
PARAM_CHECKSUM = "etag"
1214

15+
@cached_property
16+
def path(self) -> GSPath:
17+
def _getcwd():
18+
return self.fs.root_marker
19+
20+
return GSPath(self.sep, getcwd=_getcwd)
21+
1322
def _prepare_credentials(self, **config):
1423
login_info = {"consistency": None}
1524
login_info["project"] = config.get("projectname")
@@ -33,3 +42,15 @@ def _strip_protocol(cls, path: str) -> str:
3342

3443
def unstrip_protocol(self, path: str) -> str:
3544
return "gs://" + path.lstrip("/")
45+
46+
@staticmethod
47+
def _get_kwargs_from_urls(urlpath: str):
48+
from gcsfs import GCSFileSystem
49+
50+
parts = GCSFileSystem._split_path( # pylint: disable=protected-access
51+
urlpath, version_aware=True
52+
)
53+
_, _, generation = parts
54+
if generation is not None:
55+
return {"version_aware": True}
56+
return {}

dvc_gs/path.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
from typing import Optional, Tuple
2+
3+
from dvc_objects.fs.base import AnyFSPath
4+
from dvc_objects.fs.path import Path
5+
6+
7+
class GSPath(Path):
8+
def split_version(self, path: AnyFSPath) -> Tuple[str, Optional[str]]:
9+
from gcsfs import GCSFileSystem
10+
11+
parts = GCSFileSystem._split_path( # pylint: disable=protected-access
12+
path, version_aware=True
13+
)
14+
bucket, key, generation = parts
15+
return f"gs://{bucket}/{key}", generation
16+
17+
def join_version(self, path: AnyFSPath, version_id: Optional[str]) -> str:
18+
path, path_version = self.split_version(path)
19+
if path_version:
20+
raise ValueError("path already includes an object generation")
21+
return f"{path}#{version_id}"
22+
23+
def version_path(self, path: AnyFSPath, version_id: Optional[str]) -> str:
24+
path, _ = self.split_version(path)
25+
return self.join_version(path, version_id)
26+
27+
def coalesce_version(
28+
self, path: AnyFSPath, version_id: Optional[str]
29+
) -> Tuple[AnyFSPath, Optional[str]]:
30+
path, path_version_id = self.split_version(path)
31+
versions = {ver for ver in (version_id, path_version_id) if ver}
32+
if len(versions) > 1:
33+
raise ValueError(
34+
f"Path version mismatch: '{path}', '{version_id}'"
35+
)
36+
return path, (versions.pop() if versions else None)

0 commit comments

Comments
 (0)