docs: configuration options

Author: roeap

Cargo.lock

@@ -1,2 +1,10 @@
 [workspace]
 members = ["object-store"]
+
+[patch.crates-io]
+object_store = { git = "https://github.com/roeap/arrow-rs/", rev = "6b7b34e7953a8aabd4d1b7cb068e3d61e27aed4a", features = [
+  "azure",
+  "aws",
+  "gcp",
+  "aws_profile",
+] }

Cargo.toml

@@ -67,7 +67,9 @@ assert copied == data
 ### Configuration
 
 As much as possible we aim to make access to various storage backends dependent
-only on runtime configuration.
+only on runtime configuration. The kind of service is always derived from the
+url used to specifiy the storage location. Some basic configuration can also be
+derived from the url string, dependent on the chosen url format.
 
 ```py
 from object_store import ObjectStore
@@ -96,6 +98,57 @@ os.environ["AZURE_TENANT_ID"] = "<my-tenant-id>"
 store = ObjectStore("az://<container-name>")
 ```
 
+#### Azure
+
+The recommended url format is `az://<container>/<path>` and Azure always requieres
+`azure_storage_account_name` to be configured.
+
+- master key
+  - `azure_storage_account_key`
+- service principal
+  - `azure_client_id`
+  - `azure_client_secret`
+  - `azure_tenant_id`
+- shared access signature
+  - `azure_storage_sas_key` (as provided by StorageExplorer)
+- bearer token
+  - `azure_storage_token`
+- managed identity (with user assigned identity)
+  - if using user assigned identity one of `azure_client_id`, `azure_object_id`, `azure_msi_resource_id`
+  - `use_managed_identity`
+- workload identity
+  - `azure_client_id`
+  - `azure_tenant_id`
+  - `azure_federated_token_file`
+
+#### S3
+
+The recommended url format is `s3://<bucket>/<path>` S3 storage always requires a
+region to be specified via one of `aws_region` or `aws_default_region`.
+
+- access key
+  - `aws_access_key_id`
+  - `aws_secret_access_key`
+- session token
+  - `aws_session_token`
+- imds instance metadata
+  - `aws_metadata_endpoint`
+- profile
+  - `aws_profile`
+
+AWS supports [virtual hosting of buckets][aws-virtual], which can be configured by setting
+`aws_virtual_hosted_style_request` to "true".
+
+When an alternative implementation or a mocked service like localstack is used, the service
+endpoint needs to be explicitly specified via `aws_endpoint`.
+
+#### GCS
+
+The recommended url format is `gs://<bucket>/<path>`.
+
+- service account
+  - `google_service_account`
+
 ### with `pyarrow`
 
 ```py
@@ -154,3 +207,4 @@ just test
 [ci-link]: https://github.com/roeap/object-store-python/actions/workflows/ci.yaml
 [black-img]: https://img.shields.io/badge/code%20style-black-000000.svg
 [black-link]: https://github.com/psf/black
+[aws-virtual]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html

README.md

@@ -17,6 +17,7 @@ classifiers = [
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
+  "Programming Language :: Rust",
   "Intended Audience :: Developers",
   "License :: OSI Approved :: Apache Software License",
 ]

object-store/pyproject.toml

@@ -160,61 +160,55 @@ impl ObjectStoreBuilder {
             ObjectStoreKind::Local => ObjectStoreImpl::Local(LocalFileSystem::new()),
             ObjectStoreKind::InMemory => ObjectStoreImpl::InMemory(InMemory::new()),
             ObjectStoreKind::Azure => {
-                let maybe_store = MicrosoftAzureBuilder::new()
+                let store = MicrosoftAzureBuilder::new()
                     .with_url(url.clone())
                     .try_with_options(&self.options)?
                     .with_client_options(self.client_options.clone().unwrap_or_default())
                     .with_retry(self.retry_config.clone().unwrap_or_default())
-                    .build();
-                if let Ok(store) = maybe_store {
-                    ObjectStoreImpl::Azrue(store)
-                } else {
-                    let store = MicrosoftAzureBuilder::from_env()
-                        .with_url(url.clone())
-                        .try_with_options(&self.options)?
-                        .with_client_options(self.client_options.unwrap_or_default())
-                        .with_retry(self.retry_config.unwrap_or_default())
-                        .build()?;
-                    ObjectStoreImpl::Azrue(store)
-                }
+                    .build()
+                    .or_else(|_| {
+                        MicrosoftAzureBuilder::from_env()
+                            .with_url(url.clone())
+                            .try_with_options(&self.options)?
+                            .with_client_options(self.client_options.clone().unwrap_or_default())
+                            .with_retry(self.retry_config.clone().unwrap_or_default())
+                            .build()
+                    })?;
+                ObjectStoreImpl::Azrue(store)
             }
             ObjectStoreKind::S3 => {
-                let maybe_store = AmazonS3Builder::new()
+                let store = AmazonS3Builder::new()
                     .with_url(url.clone())
                     .try_with_options(&self.options)?
                     .with_client_options(self.client_options.clone().unwrap_or_default())
                     .with_retry(self.retry_config.clone().unwrap_or_default())
-                    .build();
-                if let Ok(store) = maybe_store {
-                    ObjectStoreImpl::S3(store)
-                } else {
-                    let store = AmazonS3Builder::from_env()
-                        .with_url(url.clone())
-                        .try_with_options(&self.options)?
-                        .with_client_options(self.client_options.unwrap_or_default())
-                        .with_retry(self.retry_config.unwrap_or_default())
-                        .build()?;
-                    ObjectStoreImpl::S3(store)
-                }
+                    .build()
+                    .or_else(|_| {
+                        AmazonS3Builder::from_env()
+                            .with_url(url.clone())
+                            .try_with_options(&self.options)?
+                            .with_client_options(self.client_options.unwrap_or_default())
+                            .with_retry(self.retry_config.unwrap_or_default())
+                            .build()
+                    })?;
+                ObjectStoreImpl::S3(store)
             }
             ObjectStoreKind::Google => {
-                let maybe_store = GoogleCloudStorageBuilder::new()
+                let store = GoogleCloudStorageBuilder::new()
                     .with_url(url.clone())
                     .try_with_options(&self.options)?
                     .with_client_options(self.client_options.clone().unwrap_or_default())
                     .with_retry(self.retry_config.clone().unwrap_or_default())
-                    .build();
-                if let Ok(store) = maybe_store {
-                    ObjectStoreImpl::Gcp(store)
-                } else {
-                    let store = GoogleCloudStorageBuilder::from_env()
-                        .with_url(url.clone())
-                        .try_with_options(&self.options)?
-                        .with_client_options(self.client_options.unwrap_or_default())
-                        .with_retry(self.retry_config.unwrap_or_default())
-                        .build()?;
-                    ObjectStoreImpl::Gcp(store)
-                }
+                    .build()
+                    .or_else(|_| {
+                        GoogleCloudStorageBuilder::from_env()
+                            .with_url(url.clone())
+                            .try_with_options(&self.options)?
+                            .with_client_options(self.client_options.unwrap_or_default())
+                            .with_retry(self.retry_config.unwrap_or_default())
+                            .build()
+                    })?;
+                ObjectStoreImpl::Gcp(store)
             }
         };