feat(catalog): add discovery block for custom module paths

Add experimental catalog-discovery feature allowing custom module
directory paths via discovery blocks in catalog configuration.

Previously, catalog only searched the hardcoded "modules/" directory.
This change allows repositories with non-standard layouts (e.g.,
"tf-modules/", "infrastructure/") to be used in the catalog.

Changes:
- Add discovery block support to catalog configuration
- Add ModulePaths field to config.Discovery struct
- Add Repo.WithModulePaths() functional option for custom paths
- Support multiple discovery blocks with different configurations
- Maintain backward compatibility (naked URLs use default "modules/")
- Update tests and documentation

Closes #4632

Author: shuv0id

cli/commands/catalog/catalog_test.go

@@ -25,7 +25,7 @@ func TestCatalogCommandInitialization(t *testing.T) {
 	require.NoError(t, err)
 
 	// Create mock repository function for testing
-	mockNewRepo := func(ctx context.Context, logger log.Logger, repoURL, path string, walkWithSymlinks, allowCAS bool) (*module.Repo, error) {
+	mockNewRepo := func(ctx context.Context, logger log.Logger, repoURL, path string, walkWithSymlinks, allowCAS bool, opts ...module.RepoOpt) (*module.Repo, error) {
 		// Create a temporary directory structure for testing
 		dummyRepoDir := filepath.Join(t.TempDir(), strings.ReplaceAll(repoURL, "github.com/gruntwork-io/", ""))
 		os.MkdirAll(filepath.Join(dummyRepoDir, ".git"), 0755)

cli/commands/catalog/tui/model_test.go

@@ -31,7 +31,7 @@ import (
 func createMockCatalogService(t *testing.T, opts *options.TerragruntOptions) catalog.CatalogService {
 	t.Helper()
 
-	mockNewRepo := func(ctx context.Context, logger log.Logger, repoURL, path string, walkWithSymlinks, allowCAS bool) (*module.Repo, error) {
+	mockNewRepo := func(ctx context.Context, logger log.Logger, repoURL, path string, walkWithSymlinks, allowCAS bool, opts ...module.RepoOpt) (*module.Repo, error) {
 		// Create a temporary directory structure for testing
 		dummyRepoDir := filepath.Join(t.TempDir(), strings.ReplaceAll(repoURL, "github.com/gruntwork-io/", ""))
 

config/catalog.go

@@ -38,27 +38,42 @@ var (
 	catalogBlockReg = regexp.MustCompile(fmt.Sprintf(hclBlockRegExprFmt, MetadataCatalog))
 )
 
+type Discovery struct {
+	URLs        []string `hcl:"urls,attr" cty:"urls"`
+	ModulePaths []string `hcl:"module_paths" cty:"module_paths"`
+}
+
 type CatalogConfig struct {
-	NoShell         *bool    `hcl:"no_shell,optional" cty:"no_shell"`
-	NoHooks         *bool    `hcl:"no_hooks,optional" cty:"no_hooks"`
-	DefaultTemplate string   `hcl:"default_template,optional" cty:"default_template"`
-	URLs            []string `hcl:"urls,attr" cty:"urls"`
+	NoShell         *bool       `hcl:"no_shell,optional" cty:"no_shell"`
+	NoHooks         *bool       `hcl:"no_hooks,optional" cty:"no_hooks"`
+	DefaultTemplate string      `hcl:"default_template,optional" cty:"default_template"`
+	URLs            []string    `hcl:"urls,attr" cty:"urls"`
+	Discovery       []Discovery `hcl:"discovery,block" cty:"discovery"`
 }
 
 func (cfg *CatalogConfig) String() string {
-	return fmt.Sprintf("Catalog{URLs = %v, DefaultTemplate = %v, NoShell = %v, NoHooks = %v}", cfg.URLs, cfg.DefaultTemplate, cfg.NoShell, cfg.NoHooks)
+	discoveryInfo := "none"
+
+	if len(cfg.Discovery) > 0 {
+		totalURLs := 0
+		for _, discoveryBlock := range cfg.Discovery {
+			totalURLs += len(discoveryBlock.URLs)
+		}
+
+		discoveryInfo = fmt.Sprintf("%d discovery block(s) with %d URLs", len(cfg.Discovery), totalURLs)
+	}
+
+	return fmt.Sprintf("Catalog{URLs = %v, DefaultTemplate = %v,  NoShell = %v, NoHooks = %v, Discovery = %s}", cfg.URLs, cfg.DefaultTemplate, cfg.NoShell, cfg.NoHooks, discoveryInfo)
 }
 
 func (cfg *CatalogConfig) normalize(configPath string) {
 	configDir := filepath.Dir(configPath)
 
 	// transform relative paths to absolute ones
-	for i, url := range cfg.URLs {
-		url := filepath.Join(configDir, url)
+	cfg.URLs = normalizeURLs(configDir, cfg.URLs)
 
-		if files.FileExists(url) {
-			cfg.URLs[i] = url
-		}
+	for i := range cfg.Discovery {
+		cfg.Discovery[i].URLs = normalizeURLs(configDir, cfg.Discovery[i].URLs)
 	}
 
 	if cfg.DefaultTemplate != "" {
@@ -69,6 +84,27 @@ func (cfg *CatalogConfig) normalize(configPath string) {
 	}
 }
 
+func normalizeURLs(baseDir string, urls []string) []string {
+	if len(urls) == 0 {
+		return nil
+	}
+
+	normalized := make([]string, 0, len(urls))
+
+	for _, url := range urls {
+		absolutePath := filepath.Join(baseDir, url)
+
+		if !files.FileExists(absolutePath) {
+			normalized = append(normalized, url)
+			continue
+		}
+
+		normalized = append(normalized, absolutePath)
+	}
+
+	return normalized
+}
+
 // ReadCatalogConfig reads the `catalog` block from the nearest `terragrunt.hcl` file in the parent directories.
 //
 // We want users to be able to browse to any folder in an `infra-live` repo, run `terragrunt catalog` (with no URL) arg.

config/catalog_test.go

@@ -121,6 +121,25 @@ func TestCatalogParseConfigFile(t *testing.T) {
 				DefaultTemplate: "/test/fixtures/scaffold/external-template",
 			},
 		},
+		{
+			configPath: filepath.Join(basePath, "config5.hcl"),
+			expectedConfig: &config.CatalogConfig{
+				URLs: []string{
+					"github.com/gruntwork-io/terraform-aws-eks",
+					"github.com/gruntwork-io/terraform-aws-vpc",
+				},
+				Discovery: []config.Discovery{
+					{
+						URLs:        []string{"github.com/acme-corp/infrastructure"},
+						ModulePaths: []string{"infra"},
+					},
+					{
+						URLs:        []string{"github.com/acme-corp/platform"},
+						ModulePaths: []string{"terraform"},
+					},
+				},
+			},
+		},
 	}
 
 	for i, tt := range testCases {

docs-starlight/src/content/docs/04-reference/04-experiments.md

@@ -70,6 +70,7 @@ The following experiments are available:
 - [symlinks](#symlinks)
 - [cas](#cas)
 - [filter-flag](#filter-flag)
+- [catalog-discovery](#catalog-discovery)
 
 ### symlinks
 
@@ -186,6 +187,47 @@ When this experiment stabilizes, the following queue control flags will be depre
 
 The current plan is to continue to support the flags as aliases for particular `--filter` patterns.
 
+#### `catalog-discovery`
+
+Support for configurable module discovery paths in Terragrunt Catalog.
+
+#### `catalog-discovery` - What it does
+
+By default, Terragrunt Catalog searches for modules only in the `modules/` directory of catalog repositories. This experiment enables the `discovery` block in catalog configuration, allowing you to specify custom directories where modules should be discovered. This is useful when your catalog repositories organize modules in non-standard directories (e.g., `tf-modules/`, `infrastructure/`, or organization-specific paths).
+
+**Example usage:**
+```hcl
+catalog {
+  # Standard URLs use default "modules/" path
+  urls = [
+    "github.com/gruntwork-io/repo-a.git"
+  ]
+  
+  # Discovery block with custom paths
+  discovery {
+    urls = [
+      "github.com/gruntwork-io/repo-b.git",
+      "github.com/gruntwork-io/repo-c.git"
+    ]
+    module_paths = ["tf-modules", "infrastructure", "terraform"]
+  }
+  
+  # Multiple discovery blocks with different configurations
+  discovery {
+    urls = ["github.com/acme/repo-d.git"]
+    module_paths = ["infra-modules"]
+  }
+}
+```
+
+**Key features:**
+
+- Configure custom module directory paths per repository or group of repositories
+- Multiple `discovery` blocks allow different path configurations for different repos
+- URLs in `catalog.urls` continue to use the default `modules/` path
+- The repository root directory is always checked for modules, regardless of configured paths
+
+
 ## Completed Experiments
 
 - [cli-redesign](#cli-redesign)

internal/experiment/experiment.go

@@ -34,6 +34,9 @@ const (
 	AutoProviderCacheDir = "auto-provider-cache-dir"
 	// FilterFlag is the experiment that enables usage of the filter flag for filtering components
 	FilterFlag = "filter-flag"
+	// CatalogDiscovery is the experiment that enables custom module discovery paths
+	// via the discovery block in catalog configuration.
+	CatalogDiscovery = "catalog-discovery"
 )
 
 const (
@@ -79,6 +82,9 @@ func NewExperiments() Experiments {
 		{
 			Name: FilterFlag,
 		},
+		{
+			Name: CatalogDiscovery,
+		},
 	}
 }
 

internal/services/catalog/catalog.go

@@ -25,7 +25,7 @@ import (
 
 // NewRepoFunc defines the signature for a function that creates a new repository.
 // This allows for mocking in tests.
-type NewRepoFunc func(ctx context.Context, l log.Logger, cloneURL, path string, walkWithSymlinks, allowCAS bool) (*module.Repo, error)
+type NewRepoFunc func(ctx context.Context, l log.Logger, cloneURL, path string, walkWithSymlinks, allowCAS bool, opts ...module.RepoOpt) (*module.Repo, error)
 
 const (
 	// tempDirFormat is used to create unique temporary directory names for catalog repositories.
@@ -90,10 +90,13 @@ func (s *catalogServiceImpl) WithRepoURL(repoURL string) CatalogService {
 	return s
 }
 
-// Load implements the CatalogService interface.
+// oad implements the CatalogService interface.
 // It contains the core logic for cloning/updating repositories and finding Terragrunt modules within them.
 func (s *catalogServiceImpl) Load(ctx context.Context, l log.Logger) error {
-	repoURLs := []string{s.repoURL}
+	var discoveries []config.Discovery
+
+	//  Evaluate experimental feature for CatalogDiscovery
+	catalogDiscovery := s.opts.Experiments.Evaluate(experiment.CatalogDiscovery)
 
 	// If no specific repoURL was provided to the service, try to read from catalog config.
 	if s.repoURL == "" {
@@ -102,65 +105,44 @@ func (s *catalogServiceImpl) Load(ctx context.Context, l log.Logger) error {
 			return errors.Errorf("failed to read catalog configuration: %w", err)
 		}
 
-		if catalogCfg != nil && len(catalogCfg.URLs) > 0 {
-			repoURLs = catalogCfg.URLs
-		} else {
+		if catalogCfg == nil {
 			return errors.Errorf("no catalog URLs provided")
 		}
-	}
-
-	// Remove duplicates
-	repoURLs = util.RemoveDuplicatesFromList(repoURLs)
-	if len(repoURLs) == 0 || (len(repoURLs) == 1 && repoURLs[0] == "") {
-		return errors.Errorf("no valid repository URLs specified after configuration and flag processing")
-	}
 
-	var allModules module.Modules
+		// Check if we have any valid configuration
+		hasNakedURLs := len(catalogCfg.URLs) > 0
+		hasDiscoveryBlocks := len(catalogCfg.Discovery) > 0
 
-	// Evaluate experimental features for symlinks and content-addressable storage.
-	walkWithSymlinks := s.opts.Experiments.Evaluate(experiment.Symlinks)
-	allowCAS := s.opts.Experiments.Evaluate(experiment.CAS)
-
-	var errs []error
-
-	for _, currentRepoURL := range repoURLs {
-		if currentRepoURL == "" {
-			l.Warnf("Empty repository URL encountered, skipping.")
-			continue
+		// Warn if discovery blocks exist but feature is disabled
+		if !catalogDiscovery && hasDiscoveryBlocks {
+			l.Warn("Skipping catalog discovery — discovery block detected, but the CatalogDiscovery experiment is disabled")
 		}
 
-		// Create a unique path in the system's temporary directory for this repository.
-		// The path is based on a SHA1 hash of the repository URL to ensure uniqueness and idempotency.
-		encodedRepoURL := util.EncodeBase64Sha1(currentRepoURL)
-		tempPath := filepath.Join(os.TempDir(), fmt.Sprintf(tempDirFormat, encodedRepoURL))
-
-		l.Debugf("Processing repository %s in temporary path %s", currentRepoURL, tempPath)
-
-		// Initialize the repository. This might involve cloning or updating.
-		// Use the newRepo function stored in the service instance.
-		repo, err := s.newRepo(ctx, l, currentRepoURL, tempPath, walkWithSymlinks, allowCAS)
-		if err != nil {
-			l.Errorf("Failed to initialize repository %s: %v", currentRepoURL, err)
-
-			errs = append(errs, err)
-
-			continue
+		if !hasNakedURLs && (!catalogDiscovery || !hasDiscoveryBlocks) {
+			return errors.Errorf("no catalog URLs provided")
 		}
 
-		// Find modules within the initialized repository.
-		repoModules, err := repo.FindModules(ctx)
-		if err != nil {
-			l.Errorf("Failed to find modules in repository %s: %v", currentRepoURL, err)
-
-			errs = append(errs, err)
+		if hasNakedURLs {
+			discoveries = append(discoveries, config.Discovery{URLs: catalogCfg.URLs})
+		}
 
-			continue
+		if catalogDiscovery && hasDiscoveryBlocks {
+			discoveries = append(discoveries, catalogCfg.Discovery...)
+		}
+	} else {
+		discoveries = []config.Discovery{
+			{
+				URLs: []string{s.repoURL},
+			},
 		}
+	}
 
-		l.Infof("Found %d module(s) in repository %q", len(repoModules), currentRepoURL)
-		allModules = append(allModules, repoModules...)
+	// De-duplicate URLs within each discovery
+	for i, discovery := range discoveries {
+		discoveries[i].URLs = util.RemoveDuplicatesFromList(discovery.URLs)
 	}
 
+	allModules, errs := s.loadModulesFromDiscoveries(ctx, l, discoveries)
 	s.modules = allModules
 
 	if len(errs) > 0 {
@@ -174,6 +156,65 @@ func (s *catalogServiceImpl) Load(ctx context.Context, l log.Logger) error {
 	return nil
 }
 
+func (s *catalogServiceImpl) loadModulesFromDiscoveries(ctx context.Context, l log.Logger, discoveries []config.Discovery) (module.Modules, []error) {
+	var (
+		errs       []error
+		allModules module.Modules
+	)
+
+	// Evaluate experimental features for symlinks and content-addressable storage.
+	walkWithSymlinks := s.opts.Experiments.Evaluate(experiment.Symlinks)
+	allowCAS := s.opts.Experiments.Evaluate(experiment.CAS)
+
+	for _, discovery := range discoveries {
+		for _, currentRepoURL := range discovery.URLs {
+			if currentRepoURL == "" {
+				l.Warnf("Empty repository URL encountered, skipping.")
+				continue
+			}
+
+			// Create a unique path in the system's temporary directory for this repository.
+			// The path is based on a SHA1 hash of the repository URL to ensure uniqueness and idempotency.
+			encodedRepoURL := util.EncodeBase64Sha1(currentRepoURL)
+			tempPath := filepath.Join(os.TempDir(), fmt.Sprintf(tempDirFormat, encodedRepoURL))
+
+			l.Debugf("Processing repository %s in temporary path %s", currentRepoURL, tempPath)
+
+			var repoOpts []module.RepoOpt
+
+			if len(discovery.ModulePaths) > 0 {
+				repoOpts = append(repoOpts, module.WithModulePaths(discovery.ModulePaths))
+			}
+
+			// Initialize the repository. This might involve cloning or updating.
+			// Use the newRepo function stored in the service instance.
+			repo, err := s.newRepo(ctx, l, currentRepoURL, tempPath, walkWithSymlinks, allowCAS, repoOpts...)
+			if err != nil {
+				l.Errorf("Failed to initialize repository %s: %v", currentRepoURL, err)
+
+				errs = append(errs, err)
+
+				continue
+			}
+
+			// Find modules within the initialized repository.
+			repoModules, err := repo.FindModules(ctx)
+			if err != nil {
+				l.Errorf("Failed to find modules in repository %s: %v", currentRepoURL, err)
+
+				errs = append(errs, err)
+
+				continue
+			}
+
+			l.Infof("Found %d module(s) in repository %q", len(repoModules), currentRepoURL)
+			allModules = append(allModules, repoModules...)
+		}
+	}
+
+	return allModules, errs
+}
+
 func (s *catalogServiceImpl) Modules() module.Modules {
 	return s.modules
 }