Skip to content

Commit 0d01cfc

Browse files
sfc-gh-abozkurtsfc-gh-abozkurt
committed
Refactor table types
Signed-off-by: Aykut Bozkurt <[email protected]>
1 parent c5684c6 commit 0d01cfc

File tree

37 files changed

+665
-630
lines changed

37 files changed

+665
-630
lines changed

‎pg_lake_copy/src/copy/copy.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -634,7 +634,7 @@ IsCopyFromPushdownable(Relation relation, List *columnNameList,
634634
/*
635635
* All of this only works for writable pg_lake tables.
636636
*/
637-
if (!IsAnyWritableLakeTable(relationId))
637+
if (!IsWritablePgLakeTable(relationId) && !IsWritableIcebergTable(relationId))
638638
return false;
639639

640640
bool allowDefaultConsts = false;

‎pg_lake_engine/include/pg_lake/copy/copy_format.h‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
#define PG_LAKE_COPY_FORMAT_H
2020

2121
#include "commands/copy.h"
22-
#include "pg_lake/util/rel_utils.h"
22+
#include "pg_lake/util/catalog_type.h"
23+
#include "pg_lake/util/table_type.h"
2324

2425
#define S3_URL_PREFIX "s3://"
2526
#define GCS_URL_PREFIX "gs://"

‎pg_lake_engine/include/pg_lake/pgduck/remote_storage.h‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ typedef struct RemoteFileDesc
3838

3939
extern PGDLLEXPORT int64 GetRemoteFileSize(char *path);
4040
extern PGDLLEXPORT int64 GetRemoteParquetFileRowCount(char *path);
41-
extern PGDLLEXPORT List *GetRemoteParquetColumnStats(char *path, List *leafFields);
4241
extern PGDLLEXPORT List *ListRemoteFileDescriptions(char *pattern);
4342
extern PGDLLEXPORT List *ListRemoteFileNames(char *pattern);
4443
extern PGDLLEXPORT bool RemoteFileExists(char *path);

‎pg_lake_engine/include/pg_lake/util/catalog_type.h‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
/*
2+
* Copyright 2025 Snowflake Inc.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
#pragma once
19+
20+
21+
typedef enum IcebergCatalogType
22+
{
23+
NONE_CATALOG = 0,
24+
25+
/* catalog='postgres' */
26+
POSTGRES_CATALOG = 1,
27+
28+
/*
29+
* catalog='rest', read_only=True Always treat like external iceberg
30+
* table, read the metadata location from the external catalog and never
31+
* modify.
32+
*/
33+
REST_CATALOG_READ_ONLY = 2,
34+
35+
/*
36+
* catalog='rest', read_only=False Treat like internal iceberg table, use
37+
* all the catalog tables like lake_table.files.
38+
*/
39+
REST_CATALOG_READ_WRITE = 3,
40+
41+
/*
42+
* Similar to REST_CATALOG_READ_ONLY, but using an object store compatible
43+
* API instead of a REST catalog server.
44+
*/
45+
OBJECT_STORE_READ_ONLY = 4,
46+
47+
/*
48+
* Similar to REST_CATALOG_READ_WRITE, but using an object store
49+
* compatible API instead of a REST catalog server.
50+
*/
51+
OBJECT_STORE_READ_WRITE = 5
52+
} IcebergCatalogType;
53+
54+
extern PGDLLEXPORT IcebergCatalogType GetIcebergCatalogType(Oid relationId);
55+
extern PGDLLEXPORT bool HasRestCatalogTableOption(List *options);
56+
extern PGDLLEXPORT bool HasObjectStoreCatalogTableOption(List *options);
57+
extern PGDLLEXPORT bool HasReadOnlyOption(List *options);

‎pg_lake_engine/include/pg_lake/util/rel_utils.h‎

Lines changed: 9 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -19,57 +19,16 @@
1919

2020
#include "nodes/primnodes.h"
2121

22-
/* distinguish tables with "server pg_lake" vs "server pg_lake_iceberg" */
23-
typedef enum PgLakeTableType
24-
{
25-
PG_LAKE_INVALID_TABLE_TYPE,
26-
PG_LAKE_TABLE_TYPE,
27-
PG_LAKE_ICEBERG_TABLE_TYPE
28-
} PgLakeTableType;
22+
#include "pg_lake/copy/copy_format.h"
23+
#include "pg_lake/util/catalog_type.h"
24+
#include "pg_lake/util/table_type.h"
2925

3026

31-
typedef enum IcebergCatalogType
32-
{
33-
NOT_ICEBERG_TABLE = 0,
34-
35-
/* catalog='postgres' */
36-
POSTGRES_CATALOG = 1,
37-
38-
/*
39-
* catalog='rest', read_only=True Always treat like external iceberg
40-
* table, read the metadata location from the external catalog and never
41-
* modify.
42-
*/
43-
REST_CATALOG_READ_ONLY = 2,
44-
45-
/*
46-
* catalog='rest', read_only=False Treat like internal iceberg table, use
47-
* all the catalog tables like lake_table.files.
48-
*/
49-
REST_CATALOG_READ_WRITE = 3,
50-
51-
/*
52-
* Similar to REST_CATALOG_READ_ONLY, but using an object store compatible
53-
* API instead of a REST catalog server.
54-
*/
55-
OBJECT_STORE_READ_ONLY = 4,
56-
57-
/*
58-
* Similar to REST_CATALOG_READ_WRITE, but using an object store
59-
* compatible API instead of a REST catalog server.
60-
*/
61-
OBJECT_STORE_READ_WRITE = 5
62-
} IcebergCatalogType;
63-
64-
struct PgLakeTableProperties;
65-
6627
#define PG_LAKE_SERVER_NAME "pg_lake"
6728
#define PG_LAKE_ICEBERG_SERVER_NAME "pg_lake_iceberg"
6829

6930
extern PGDLLEXPORT bool IsAnyLakeForeignTableById(Oid foreignTableId);
7031
extern PGDLLEXPORT char *GetQualifiedRelationName(Oid relationId);
71-
extern PGDLLEXPORT const char *PgLakeTableTypeToName(PgLakeTableType tableType);
72-
extern PGDLLEXPORT PgLakeTableType GetPgLakeTableType(Oid foreignTableId);
7332
extern PGDLLEXPORT char *GetPgLakeForeignServerName(Oid foreignTableId);
7433
extern PGDLLEXPORT PgLakeTableType GetPgLakeTableTypeViaServerName(char *serverName);
7534
extern PGDLLEXPORT bool IsPgLakeForeignTableById(Oid foreignTableId);
@@ -79,8 +38,12 @@ extern PGDLLEXPORT bool IsAnyWritableLakeTable(Oid foreignTableId);
7938
extern PGDLLEXPORT bool IsPgLakeIcebergServerName(const char *serverName);
8039
extern PGDLLEXPORT char *GetWritableTableLocation(Oid relationId, char **queryArguments);
8140
extern PGDLLEXPORT void EnsureTableOwner(Oid relationId);
82-
extern PGDLLEXPORT struct PgLakeTableProperties GetPgLakeTableProperties(Oid relationId);
83-
extern PGDLLEXPORT bool IsInternalOrExternalIcebergTable(struct PgLakeTableProperties properties);
41+
extern PGDLLEXPORT bool IsAnyLakeForeignTable(RangeTblEntry *rte);
42+
extern PGDLLEXPORT CopyDataFormat GetForeignTableFormat(Oid foreignTableId);
43+
extern PGDLLEXPORT char *GetForeignTablePath(Oid foreignTableId);
44+
extern PGDLLEXPORT void ErrorIfTypeUnsupportedForIcebergTables(Oid typeOid, int32 typmod, char *columnName);
45+
extern PGDLLEXPORT void ErrorIfTypeUnsupportedNumericForIcebergTables(int32 typmod, char *columnName);
46+
extern PGDLLEXPORT PgLakeTableProperties GetPgLakeTableProperties(Oid relationId);
8447

8548
/* range var help */
8649
extern PGDLLEXPORT List *MakeNameListFromRangeVar(const RangeVar *rel);

‎pg_lake_engine/include/pg_lake/util/table_type.h‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/*
2+
* Copyright 2025 Snowflake Inc.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
#pragma once
19+
20+
21+
/* distinguish tables with "server pg_lake" vs "server pg_lake_iceberg" */
22+
typedef enum PgLakeTableType
23+
{
24+
PG_LAKE_INVALID_TABLE_TYPE,
25+
PG_LAKE_TABLE_TYPE,
26+
PG_LAKE_ICEBERG_TABLE_TYPE
27+
} PgLakeTableType;
28+
29+
30+
extern PGDLLEXPORT const char *PgLakeTableTypeToName(PgLakeTableType tableType);
31+
extern PGDLLEXPORT PgLakeTableType GetPgLakeTableType(Oid foreignTableId);
32+
extern PGDLLEXPORT bool IsWritablePgLakeTable(Oid relationId);
33+
extern PGDLLEXPORT bool IsIcebergTable(Oid relationId);
34+
extern PGDLLEXPORT bool IsInternalIcebergTable(Oid relationId);
35+
extern PGDLLEXPORT bool IsExternalIcebergTable(Oid relationId);

‎pg_lake_engine/src/utils/catalog_type.c‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Copyright 2025 Snowflake Inc.
3+
* SPDX-License-Identifier: Apache-2.0
4+
*
5+
* Licensed under the Apache License, Version 2.0 (the "License");
6+
* you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* https://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
#include "postgres.h"
19+
#include "miscadmin.h"
20+
#include "foreign/foreign.h"
21+
#include "catalog/pg_foreign_table.h"
22+
23+
24+
#include "pg_lake/copy/copy_format.h"
25+
#include "pg_lake/parsetree/options.h"
26+
#include "pg_lake/util/catalog_type.h"
27+
#include "pg_lake/util/rel_utils.h"
28+
29+
30+
/*
31+
* GetIcebergCatalogType returns the IcebergCatalogType for the given
32+
* relation ID.
33+
*/
34+
IcebergCatalogType
35+
GetIcebergCatalogType(Oid relationId)
36+
{
37+
if (!IsPgLakeIcebergForeignTableById(relationId))
38+
return NONE_CATALOG;
39+
40+
ForeignTable *foreignTable = GetForeignTable(relationId);
41+
List *options = foreignTable->options;
42+
43+
bool hasRestCatalogOption = HasRestCatalogTableOption(options);
44+
bool hasObjectStoreCatalogOption = HasObjectStoreCatalogTableOption(options);
45+
bool hasReadOnlyOption = HasReadOnlyOption(options);
46+
47+
if (hasRestCatalogOption && hasReadOnlyOption)
48+
{
49+
return REST_CATALOG_READ_ONLY;
50+
}
51+
else if (hasRestCatalogOption && !hasReadOnlyOption)
52+
{
53+
return REST_CATALOG_READ_WRITE;
54+
}
55+
else if (hasObjectStoreCatalogOption && hasReadOnlyOption)
56+
{
57+
return OBJECT_STORE_READ_ONLY;
58+
}
59+
else if (hasObjectStoreCatalogOption && !hasReadOnlyOption)
60+
{
61+
return OBJECT_STORE_READ_WRITE;
62+
}
63+
else
64+
{
65+
return POSTGRES_CATALOG;
66+
}
67+
}
68+
69+
70+
/*
71+
* HasRestCatalogTableOption returns true if the options contain
72+
* catalog='rest'.
73+
*/
74+
bool
75+
HasRestCatalogTableOption(List *options)
76+
{
77+
char *catalog = GetStringOption(options, "catalog", false);
78+
79+
return catalog ? strncasecmp(catalog, "rest", strlen("rest")) == 0 : false;
80+
}
81+
82+
83+
/*
84+
* HasObjectStoreCatalogTableOption returns true if the options contain
85+
* catalog='object_store'.
86+
*/
87+
bool
88+
HasObjectStoreCatalogTableOption(List *options)
89+
{
90+
char *catalog = GetStringOption(options, "catalog", false);
91+
92+
return catalog ? strncasecmp(catalog, "object_store", strlen("object_store")) == 0 : false;
93+
}
94+
95+
96+
/*
97+
* HasReadOnlyOption returns true if the options contain
98+
* catalog='read_only'.
99+
*/
100+
bool
101+
HasReadOnlyOption(List *options)
102+
{
103+
char *readOnly = GetStringOption(options, "read_only", false);
104+
105+
return readOnly ? strncasecmp(readOnly, "true", strlen("true")) == 0 : false;
106+
}

0 commit comments

Comments
 (0)