Merge branch 'develop' into sdp-sep-wallet-registration

# Conflicts:
#	helmchart/sdp/README.md

Author: marwen-abid

CHANGELOG.md

@@ -6,10 +6,19 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
 
 ## [Unreleased]
 
+### Added
+
+- Add configurable database connection pool settings to prevent idle connection buildup in multi-tenant deployments [#932](https://github.com/stellar/stellar-disbursement-platform-backend/pull/932)
+
+### Changed
+
+- Default Max Open Connections per pool changed from 30 to 20 to prevent idle connection buildup in multi-tenant deployments [#932](https://github.com/stellar/stellar-disbursement-platform-backend/pull/932)
+
 ### Fixed
 
 - Fix HTML validation to allow apostrophes in invitation messages while maintaining security against XSS attacks [#930](https://github.com/stellar/stellar-disbursement-platform-backend/pull/930)
 
+
 ## [5.0.0](https://github.com/stellar/stellar-disbursement-platform-backend/releases/tag/5.0.0) ([diff](https://github.com/stellar/stellar-disbursement-platform-backend/compare/4.1.0...5.0.0))
 
 ### Added

README.md

@@ -386,6 +386,23 @@ The following environment variables can be used to configure the intervals of th
 >Prior to version 3.7.0, background jobs were configured using ENABLE_SCHEDULER=true and EVENT_BROKER_TYPE=NONE.
 >This configuration has been deprecated in favor of using EVENT_BROKER_TYPE=SCHEDULER.
 
+### Database connection pool
+
+Tune the per-tenant PostgreSQL connection pool with env vars (defaults shown):
+
+```sh
+# Maximum open connections per pool (default: 10)
+DB_MAX_OPEN_CONNS=10
+# Maximum idle connections retained (default: 0)
+DB_MAX_IDLE_CONNS=0
+# Close idle connections after N seconds (default: 2)
+DB_CONN_MAX_IDLE_TIME_SECONDS=2
+# Recycle connections after N seconds (default: 300 = 5 minutes)
+DB_CONN_MAX_LIFETIME_SECONDS=300
+```
+
+These settings help prevent idle connection buildup across multi-tenant scheduler cycles, especially on constrained databases.
+
 ## Wallets
 
 Please check the [Making Your Wallet SDP-Ready](https://docs.stellar.org/stellar-disbursement-platform/making-your-wallet-sdp-ready) section of the Stellar Docs for more information on how to integrate your wallet with the SDP.

cmd/channel_accounts.go

@@ -133,7 +133,14 @@ func (c *ChannelAccountsCommand) Command(cmdService ChAccCmdServiceInterface) *c
 			metricsServeOpts.MonitorService = &tssMonitorSvc
 
 			// Setup the TSSDBConnectionPool
-			dbcpOptions := di.DBConnectionPoolOptions{DatabaseURL: globalOptions.DatabaseURL, MonitorService: &tssMonitorSvc}
+			dbcpOptions := di.DBConnectionPoolOptions{
+				DatabaseURL:            globalOptions.DatabaseURL,
+				MonitorService:         &tssMonitorSvc,
+				MaxOpenConns:           globalOptions.DBPool.DBMaxOpenConns,
+				MaxIdleConns:           globalOptions.DBPool.DBMaxIdleConns,
+				ConnMaxIdleTimeSeconds: globalOptions.DBPool.DBConnMaxIdleTimeSeconds,
+				ConnMaxLifetimeSeconds: globalOptions.DBPool.DBConnMaxLifetimeSeconds,
+			}
 			c.TSSDBConnectionPool, err = di.NewTSSDBConnectionPool(ctx, dbcpOptions)
 			if err != nil {
 				log.Ctx(ctx).Fatalf("Error creating TSS DB connection pool: %v", err)

cmd/db/db.go

@@ -33,7 +33,13 @@ func (c *DatabaseCommand) Command(globalOptions *utils.GlobalOptionsType) *cobra
 		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			utils.PropagatePersistentPreRun(cmd, args)
 
-			adminDBConnectionPool, err := di.NewAdminDBConnectionPool(cmd.Context(), di.DBConnectionPoolOptions{DatabaseURL: globalOptions.DatabaseURL})
+			adminDBConnectionPool, err := di.NewAdminDBConnectionPool(cmd.Context(), di.DBConnectionPoolOptions{
+				DatabaseURL:            globalOptions.DatabaseURL,
+				MaxOpenConns:           globalOptions.DBPool.DBMaxOpenConns,
+				MaxIdleConns:           globalOptions.DBPool.DBMaxIdleConns,
+				ConnMaxIdleTimeSeconds: globalOptions.DBPool.DBConnMaxIdleTimeSeconds,
+				ConnMaxLifetimeSeconds: globalOptions.DBPool.DBConnMaxLifetimeSeconds,
+			})
 			if err != nil {
 				log.Ctx(cmd.Context()).Fatalf("getting Admin database connection pool: %v", err)
 			}

cmd/distribution_accounts.go

@@ -52,7 +52,13 @@ func (c *DistributionAccountCommand) Command(cmdService DistAccCmdServiceInterfa
 			}
 
 			// Setup the TSSDBConnectionPool
-			dbcpOptions := di.DBConnectionPoolOptions{DatabaseURL: globalOptions.DatabaseURL}
+			dbcpOptions := di.DBConnectionPoolOptions{
+				DatabaseURL:            globalOptions.DatabaseURL,
+				MaxOpenConns:           globalOptions.DBPool.DBMaxOpenConns,
+				MaxIdleConns:           globalOptions.DBPool.DBMaxIdleConns,
+				ConnMaxIdleTimeSeconds: globalOptions.DBPool.DBConnMaxIdleTimeSeconds,
+				ConnMaxLifetimeSeconds: globalOptions.DBPool.DBConnMaxLifetimeSeconds,
+			}
 			c.TSSDBConnectionPool, err = di.NewTSSDBConnectionPool(ctx, dbcpOptions)
 			if err != nil {
 				log.Ctx(ctx).Fatalf("Error creating TSS DB connection pool: %v", err)

cmd/serve.go

@@ -377,6 +377,12 @@ func (c *ServeCommand) Command(serverService ServerServiceInterface, monitorServ
 		cmdUtils.SchedulerConfigOptions(&schedulerOpts)...,
 	)
 
+	// DB pool tuning options (serve)
+	configOpts = append(
+		configOpts,
+		cmdUtils.DBPoolConfigOptions(&globalOptions.DBPool)...,
+	)
+
 	// bridge integration options
 	bridgeIntegrationOpts := cmdUtils.BridgeIntegrationOptions{}
 	configOpts = append(
@@ -436,7 +442,14 @@ func (c *ServeCommand) Command(serverService ServerServiceInterface, monitorServ
 			ctx := cmd.Context()
 
 			// Setup the Admin DB connection pool
-			dbcpOptions := di.DBConnectionPoolOptions{DatabaseURL: globalOptions.DatabaseURL, MonitorService: monitorService}
+			dbcpOptions := di.DBConnectionPoolOptions{
+				DatabaseURL:            globalOptions.DatabaseURL,
+				MonitorService:         monitorService,
+				MaxOpenConns:           globalOptions.DBPool.DBMaxOpenConns,
+				MaxIdleConns:           globalOptions.DBPool.DBMaxIdleConns,
+				ConnMaxIdleTimeSeconds: globalOptions.DBPool.DBConnMaxIdleTimeSeconds,
+				ConnMaxLifetimeSeconds: globalOptions.DBPool.DBConnMaxLifetimeSeconds,
+			}
 			adminDBConnectionPool, err := di.NewAdminDBConnectionPool(ctx, dbcpOptions)
 			if err != nil {
 				log.Ctx(ctx).Fatalf("error getting Admin DB connection pool: %v", err)

cmd/tenants.go

@@ -196,7 +196,13 @@ func (cmd *TenantsCommand) Command() *cobra.Command {
 }
 
 func initDBPools(ctx context.Context, dbURL string) (admin, mtn, tss dbpkg.DBConnectionPool, err error) {
-	opts := di.DBConnectionPoolOptions{DatabaseURL: dbURL}
+	opts := di.DBConnectionPoolOptions{
+		DatabaseURL:            dbURL,
+		MaxOpenConns:           globalOptions.DBPool.DBMaxOpenConns,
+		MaxIdleConns:           globalOptions.DBPool.DBMaxIdleConns,
+		ConnMaxIdleTimeSeconds: globalOptions.DBPool.DBConnMaxIdleTimeSeconds,
+		ConnMaxLifetimeSeconds: globalOptions.DBPool.DBConnMaxLifetimeSeconds,
+	}
 	if admin, err = di.NewAdminDBConnectionPool(ctx, opts); err != nil {
 		return
 	}

cmd/transaction_submitter.go

@@ -123,6 +123,12 @@ func (c *TxSubmitterCommand) Command(submitterService TxSubmitterServiceInterfac
 		cmdUtils.TransactionSubmitterEngineConfigOptions(&txSubmitterOpts)...,
 	)
 
+	// DB pool tuning options (tss)
+	configOpts = append(
+		configOpts,
+		cmdUtils.DBPoolConfigOptions(&globalOptions.DBPool)...,
+	)
+
 	cmd := &cobra.Command{
 		Use:   "tss",
 		Short: "Run the Transaction Submission Service",
@@ -153,7 +159,14 @@ func (c *TxSubmitterCommand) Command(submitterService TxSubmitterServiceInterfac
 			tssOpts.MonitorService = tssMonitorSvc
 
 			// Initializing the TSSDBConnectionPool
-			dbcpOptions := di.DBConnectionPoolOptions{DatabaseURL: globalOptions.DatabaseURL, MonitorService: &tssMonitorSvc}
+			dbcpOptions := di.DBConnectionPoolOptions{
+				DatabaseURL:            globalOptions.DatabaseURL,
+				MonitorService:         &tssMonitorSvc,
+				MaxOpenConns:           globalOptions.DBPool.DBMaxOpenConns,
+				MaxIdleConns:           globalOptions.DBPool.DBMaxIdleConns,
+				ConnMaxIdleTimeSeconds: globalOptions.DBPool.DBConnMaxIdleTimeSeconds,
+				ConnMaxLifetimeSeconds: globalOptions.DBPool.DBConnMaxLifetimeSeconds,
+			}
 			tssDBConnectionPool, err := di.NewTSSDBConnectionPool(ctx, dbcpOptions)
 			if err != nil {
 				log.Ctx(ctx).Fatalf("error getting TSS DB connection pool: %v", err)

cmd/utils/global_options.go

@@ -13,6 +13,7 @@ type GlobalOptionsType struct {
 	Version           string
 	GitCommit         string
 	DatabaseURL       string
+	DBPool            DBPoolOptions
 	BaseURL           string
 	SDPUIBaseURL      string
 	NetworkPassphrase string

cmd/utils/shared_config_options.go

@@ -10,6 +10,7 @@ import (
 	"github.com/stellar/go/support/config"
 	"github.com/stellar/go/txnbuild"
 
+	"github.com/stellar/stellar-disbursement-platform-backend/db"
 	"github.com/stellar/stellar-disbursement-platform-backend/internal/crashtracker"
 	di "github.com/stellar/stellar-disbursement-platform-backend/internal/dependencyinjection"
 	"github.com/stellar/stellar-disbursement-platform-backend/internal/message"
@@ -19,6 +20,52 @@ import (
 	"github.com/stellar/stellar-disbursement-platform-backend/stellar-multitenant/pkg/tenant"
 )
 
+// DBPoolOptions contains tunables for the PostgreSQL connection pool.
+type DBPoolOptions struct {
+	DBMaxOpenConns           int
+	DBMaxIdleConns           int
+	DBConnMaxIdleTimeSeconds int
+	DBConnMaxLifetimeSeconds int
+}
+
+// DBPoolConfigOptions returns config options for tuning the DB connection pool.
+func DBPoolConfigOptions(opts *DBPoolOptions) []*config.ConfigOption {
+	return []*config.ConfigOption{
+		{
+			Name:        "db-max-open-conns",
+			Usage:       "Maximum number of open DB connections per pool",
+			OptType:     types.Int,
+			ConfigKey:   &opts.DBMaxOpenConns,
+			FlagDefault: db.DefaultDBPoolConfig.MaxOpenConns,
+			Required:    false,
+		},
+		{
+			Name:        "db-max-idle-conns",
+			Usage:       "Maximum number of idle DB connections retained per pool",
+			OptType:     types.Int,
+			ConfigKey:   &opts.DBMaxIdleConns,
+			FlagDefault: db.DefaultDBPoolConfig.MaxIdleConns,
+			Required:    false,
+		},
+		{
+			Name:        "db-conn-max-idle-time-seconds",
+			Usage:       "Maximum idle time in seconds before a connection is closed",
+			OptType:     types.Int,
+			ConfigKey:   &opts.DBConnMaxIdleTimeSeconds,
+			FlagDefault: db.DefaultConnMaxIdleTimeSeconds,
+			Required:    false,
+		},
+		{
+			Name:        "db-conn-max-lifetime-seconds",
+			Usage:       "Maximum lifetime in seconds for a single connection",
+			OptType:     types.Int,
+			ConfigKey:   &opts.DBConnMaxLifetimeSeconds,
+			FlagDefault: db.DefaultConnMaxLifetimeSeconds,
+			Required:    false,
+		},
+	}
+}
+
 // TwilioConfigOptions returns the config options for Twilio. Relevant for loading configs needed for the messenger type(s): `TWILIO_*`.
 func TwilioConfigOptions(opts *message.MessengerOptions) []*config.ConfigOption {
 	return []*config.ConfigOption{