Skip to content

Commit 3f3faea

Browse files
sfc-gh-okalacisfc-gh-okalaci
committed
Change iceberg table and catalog locations
This change does 2 changes: - When a table is created with `catalog=obeject_store`, the catalog file used to be in `defaultPrefix/_catalog/frompg/pg_db.json` after this commit: `defaultPrefix/frompg/catalog/pg_db.json` - When any iceberg table is created, it used to be located in `defaultPrefix/dbname/schema/table_name`. After this commit: `defaultPrefix/frompg/tables/dbname/schema/table_name` The goal is to be able to differentiate the tables written by Postgres, and other systems. The `frompg` allows this. Also, we can control the access of `/catalog` folder such that only priviledged users can access that.
1 parent aa687ad commit 3f3faea

File tree

7 files changed

+39
-25
lines changed

7 files changed

+39
-25
lines changed

‎pg_lake_iceberg/include/pg_lake/iceberg/catalog.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
extern char *IcebergDefaultLocationPrefix;
2929

3030

31+
extern PGDLLEXPORT char *ExternalIcebergStoragePrefix;
32+
extern PGDLLEXPORT char *InternalIcebergStoragePrefix;
33+
3134
/*
3235
* From the external view perspective, the pg_catalog.iceberg_tables
3336
* (and pg_lake_iceberg.tables) includes both the internal and

‎pg_lake_iceberg/include/pg_lake/object_store_catalog/object_store_catalog.h‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,6 @@
88
extern PGDLLEXPORT bool EnableObjectStoreCatalog;
99

1010
extern PGDLLEXPORT char *ObjectStoreCatalogLocationPrefix;
11-
extern PGDLLEXPORT char *ExternalObjectStorePrefix;
12-
extern PGDLLEXPORT char *InternalObjectStorePrefix;
1311

1412
extern PGDLLEXPORT void InitObjectStoreCatalog(void);
1513
extern PGDLLEXPORT void ExportIcebergCatalogIfChanged(void);

‎pg_lake_iceberg/src/iceberg/catalog.c‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@
3333

3434
char *IcebergDefaultLocationPrefix = NULL;
3535

36+
char *ExternalIcebergStoragePrefix = "fromsf";
37+
char *InternalIcebergStoragePrefix = "frompg";
38+
3639
static char *GetIcebergCatalogMetadataLocation(Oid relationId, bool forUpdate);
3740
static char *GetIcebergExternalMetadataLocation(Oid relationId);
3841
static char *GetIcebergCatalogMetadataLocationInternal(Oid relationId, bool isPrevMetadata, bool forUpdate);
@@ -739,6 +742,8 @@ UpdateAllInternalIcebergTablesToReadOnly(void)
739742
/*
740743
* GetIcebergDefaultLocationPrefix returns the default location prefix
741744
* for iceberg tables. Trailing slash is removed, if present.
745+
* We always append InternalIcebergStoragePrefix/tables to the returned value as that's
746+
* distinctive for tables created by Postgres/pg_lake itself.
742747
*/
743748
const char *
744749
GetIcebergDefaultLocationPrefix(void)
@@ -748,6 +753,7 @@ GetIcebergDefaultLocationPrefix(void)
748753
return NULL;
749754
}
750755

756+
char *locationPrefix = IcebergDefaultLocationPrefix;
751757
size_t len = strlen(IcebergDefaultLocationPrefix);
752758

753759
if (len > 0 && IcebergDefaultLocationPrefix[len - 1] == '/')
@@ -757,10 +763,19 @@ GetIcebergDefaultLocationPrefix(void)
757763

758764
locationPrefixRemovedTrailingSlash[len - 1] = '\0';
759765

760-
return locationPrefixRemovedTrailingSlash;
766+
locationPrefix = locationPrefixRemovedTrailingSlash;
767+
}
768+
769+
if (InternalIcebergStoragePrefix == NULL)
770+
{
771+
ereport(ERROR,
772+
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
773+
errmsg("pg_lake_iceberg.internal_iceberg_storage_prefix is not set"),
774+
errdetail("Set the GUC to use catalog=object_store.")));
761775
}
762776

763-
return IcebergDefaultLocationPrefix;
777+
/* always append InternalIcebergStoragePrefix/tables */
778+
return psprintf("%s/%s/tables", locationPrefix, InternalIcebergStoragePrefix);
764779
}
765780

766781

‎pg_lake_iceberg/src/init.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -98,19 +98,19 @@ _PG_init(void)
9898
0,
9999
NULL, NULL, NULL);
100100

101-
DefineCustomStringVariable("pg_lake_iceberg.external_object_store_catalog_prefix",
101+
DefineCustomStringVariable("pg_lake_iceberg.external_iceberg_storage_prefix",
102102
gettext_noop("Specifies the prefix used for the object store catalog files for external tables."),
103103
NULL,
104-
&ExternalObjectStorePrefix,
104+
&ExternalIcebergStoragePrefix,
105105
"fromsf",
106106
PGC_SIGHUP,
107107
GUC_NO_SHOW_ALL | GUC_SUPERUSER_ONLY | GUC_NOT_IN_SAMPLE,
108108
NULL, NULL, NULL);
109109

110-
DefineCustomStringVariable("pg_lake_iceberg.internal_object_store_catalog_prefix",
110+
DefineCustomStringVariable("pg_lake_iceberg.internal_iceberg_storage_prefix",
111111
gettext_noop("Specifies the prefix used for the object store catalog files for internal tables."),
112112
NULL,
113-
&InternalObjectStorePrefix,
113+
&InternalIcebergStoragePrefix,
114114
"frompg",
115115
PGC_SIGHUP,
116116
GUC_NO_SHOW_ALL | GUC_SUPERUSER_ONLY | GUC_NOT_IN_SAMPLE,

‎pg_lake_iceberg/src/object_store_catalog/object_store_catalog.c‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@
2222
#include "pg_lake/storage/local_storage.h"
2323

2424
char *ObjectStoreCatalogLocationPrefix = NULL;
25-
char *ExternalObjectStorePrefix = "fromsf";
26-
char *InternalObjectStorePrefix = "frompg";
2725

2826
PG_FUNCTION_INFO_V1(list_object_store_tables);
2927
PG_FUNCTION_INFO_V1(trigger_object_store_catalog_generation);
@@ -496,16 +494,16 @@ GetExternalObjectStoreCatalogFilePath(const char *catalogName)
496494
errdetail("Set the GUC to use catalog=object_store.")));
497495
}
498496

499-
if (ExternalObjectStorePrefix == NULL)
497+
if (ExternalIcebergStoragePrefix == NULL)
500498
{
501499
ereport(ERROR,
502500
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
503-
errmsg("pg_lake_iceberg.external_object_store_prefix is not set"),
501+
errmsg("pg_lake_iceberg.external_iceberg_storage_prefix is not set"),
504502
errdetail("Set the GUC to use catalog=object_store.")));
505503
}
506504

507-
return psprintf("%s/_catalog/%s/%s.json", defaultPrefix,
508-
ExternalObjectStorePrefix, URLEncodePath(catalogName));
505+
return psprintf("%s/%s/catalog/%s.json", defaultPrefix,
506+
ExternalIcebergStoragePrefix, URLEncodePath(catalogName));
509507
}
510508

511509
static char *
@@ -521,16 +519,16 @@ GetInternalObjectStoreCatalogFilePath(const char *catalogName)
521519
errdetail("Set the GUC to use catalog=object_store.")));
522520
}
523521

524-
if (InternalObjectStorePrefix == NULL)
522+
if (InternalIcebergStoragePrefix == NULL)
525523
{
526524
ereport(ERROR,
527525
(errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE),
528-
errmsg("pg_lake_iceberg.internal_object_store_prefix is not set"),
526+
errmsg("pg_lake_iceberg.internal_iceberg_storage_prefix is not set"),
529527
errdetail("Set the GUC to use catalog=object_store.")));
530528
}
531529

532-
return psprintf("%s/_catalog/%s/%s.json", defaultPrefix,
533-
InternalObjectStorePrefix, URLEncodePath(catalogName));
530+
return psprintf("%s/%s/catalog/%s.json", defaultPrefix,
531+
InternalIcebergStoragePrefix, URLEncodePath(catalogName));
534532
}
535533

536534
/*

‎pg_lake_table/tests/pytests/test_create_table.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -219,7 +219,7 @@ def test_create_table_with_default_location(
219219
)[0][0]
220220

221221
assert (
222-
f"s3://{TEST_BUCKET}/{dbname}/public/test_create_table_with_default_location/{table_oid}"
222+
f"s3://{TEST_BUCKET}/frompg/tables/{dbname}/public/test_create_table_with_default_location/{table_oid}"
223223
in first_table_metadata_location
224224
)
225225

@@ -275,7 +275,7 @@ def test_create_table_with_default_location(
275275
)[0][0]
276276

277277
assert (
278-
f"s3://{TEST_BUCKET}/{dbname}/public/test_create_table_with_default_location/{table_oid}"
278+
f"s3://{TEST_BUCKET}/frompg/tables/{dbname}/public/test_create_table_with_default_location/{table_oid}"
279279
in second_table_metadata_location
280280
)
281281

@@ -333,7 +333,7 @@ def test_create_table_with_default_location_override(
333333
pg_conn,
334334
)
335335
assert (
336-
f"s3://{TEST_BUCKET}/test_create_table_with_default_location_override/{dbname}/public/test_create_table_with_default_location"
336+
f"s3://{TEST_BUCKET}/test_create_table_with_default_location_override/frompg/tables/{dbname}/public/test_create_table_with_default_location"
337337
in result[0][0]
338338
)
339339

‎pg_lake_table/tests/pytests/test_object_store_catalog.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -978,14 +978,14 @@ def adjust_object_store_settings(superuser_conn):
978978
# to be able to read the same tables that we write, use the same prefix
979979
run_command(
980980
f"""
981-
ALTER SYSTEM SET pg_lake_iceberg.internal_object_store_catalog_prefix = 'tmp';
981+
ALTER SYSTEM SET pg_lake_iceberg.internal_iceberg_storage_prefix = 'tmp';
982982
""",
983983
superuser_conn,
984984
)
985985

986986
run_command(
987987
f"""
988-
ALTER SYSTEM SET pg_lake_iceberg.external_object_store_catalog_prefix = 'tmp';
988+
ALTER SYSTEM SET pg_lake_iceberg.external_iceberg_storage_prefix = 'tmp';
989989
""",
990990
superuser_conn,
991991
)
@@ -1009,13 +1009,13 @@ def adjust_object_store_settings(superuser_conn):
10091009
)
10101010
run_command(
10111011
f"""
1012-
ALTER SYSTEM RESET pg_lake_iceberg.internal_object_store_catalog_prefix;
1012+
ALTER SYSTEM RESET pg_lake_iceberg.internal_iceberg_storage_prefix;
10131013
""",
10141014
superuser_conn,
10151015
)
10161016
run_command(
10171017
f"""
1018-
ALTER SYSTEM RESET pg_lake_iceberg.external_object_store_catalog_prefix;
1018+
ALTER SYSTEM RESET pg_lake_iceberg.external_iceberg_storage_prefix;
10191019
""",
10201020
superuser_conn,
10211021
)

0 commit comments

Comments
 (0)