make config imports more testable

Author: fstab

config/v3/configV3.go

@@ -36,12 +36,17 @@ const (
 )
 
 func Unmarshal(config []byte) (*Config, error) {
+	return unmarshal(config, NewFileLoader())
+}
+
+// For testing, allow injection of mock file loader.
+func unmarshal(config []byte, fileLoader FileLoader) (*Config, error) {
 	cfg := &Config{}
 	err := yaml.Unmarshal(config, cfg)
 	if err != nil {
-		return nil, fmt.Errorf("invalid configuration: %v. make sure to use 'single quotes' around strings with special characters (like match patterns or label templates), and make sure to use '-' only for lists (metrics) but not for maps (labels).", err.Error())
+		return nil, fmt.Errorf("invalid configuration: %v. make sure to use 'single quotes' around strings with special characters (like match patterns or label templates), and make sure to use '-' only for lists (metrics) but not for maps (labels)", err.Error())
 	}
-	importedMetrics, err := importMetrics(cfg.Imports)
+	importedMetrics, err := importMetrics(cfg.Imports, fileLoader)
 	if err != nil {
 		return nil, err
 	}
@@ -154,6 +159,68 @@ type ServerConfig struct {
 	Key      string `yaml:",omitempty"`
 }
 
+func importMetrics(importsConfig ImportsConfig, fileLoader FileLoader) (MetricsConfig, error) {
+	var (
+		importConfig ImportConfig
+		result       MetricsConfig
+		err          error
+		files        []*ConfigFile
+	)
+	for _, importConfig = range importsConfig {
+		if importConfig.Type != importMetricsType {
+			continue
+		}
+		err = importConfig.validate()
+		if err != nil {
+			return nil, err
+		}
+		if len(importConfig.Dir) > 0 {
+			files, err = fileLoader.LoadDir(importConfig.Dir)
+		} else {
+			files, err = fileLoader.LoadGlob(importConfig.File)
+		}
+		if err != nil {
+			return nil, err
+		}
+		for _, file := range files {
+			var metricsConfig MetricsConfig
+			err := yaml.Unmarshal([]byte(file.Contents), &metricsConfig)
+			if err != nil {
+				return nil, fmt.Errorf("%v: %v", file.Path, err)
+			}
+			for i := range metricsConfig {
+				applyImportDefaults(&metricsConfig[i], importConfig.Defaults)
+				result = append(result, metricsConfig[i])
+			}
+		}
+	}
+	return result, nil
+}
+
+func applyImportDefaults(metricConfig *MetricConfig, defaults DefaultConfig) {
+	for key, value := range defaults.Labels {
+		if _, exists := metricConfig.Labels[key]; !exists {
+			if metricConfig.Labels == nil {
+				metricConfig.Labels = make(map[string]string)
+			}
+			metricConfig.Labels[key] = value
+		}
+	}
+	if len(metricConfig.Quantiles) == 0 {
+		metricConfig.Quantiles = defaults.Quantiles
+	}
+	if len(metricConfig.Buckets) == 0 {
+		metricConfig.Buckets = defaults.Buckets
+	}
+	if metricConfig.Retention == 0 {
+		metricConfig.Retention = defaults.Retention
+	}
+	if len(metricConfig.Path) == 0 && len(metricConfig.Paths) == 0 {
+		metricConfig.Path = defaults.Path
+		metricConfig.Paths = defaults.Paths
+	}
+}
+
 func (cfg *Config) addDefaults() {
 	cfg.Global.addDefaults()
 	cfg.Input.addDefaults()

config/v3/configV3_test.go

@@ -176,11 +176,18 @@ server:
     port: 9144
 `
 
-const additional_config_file = `
+const config_with_imports = `
 global:
     config_version: 3
 input:
     type: stdin
+imports:
+    - type: metrics
+      file: /etc/grok/metrics.d/*.yaml
+      defaults:
+          path: /var/log/syslog/*
+          labels:
+              logfile: '{{base .logfile}}'
 grok_patterns:
     - WARN WARN
     - ERROR ERROR
@@ -189,22 +196,36 @@ metrics:
       name: errors_total
       help: Dummy help message.
       match: ERROR
-imports:
-    - type: metrics
-      file: /etc/grok/metrics.d/*.yaml
-      defaults:
-          path: /var/log/syslog/*
-          labels:
-              logfile: '{{base .logfile}}'
 server:
     protocol: http
     port: 9144
 `
 
-//func TestAdditionalConfigFiles(t *testing.T) {
-//	cfg := loadOrFail(t, additional_config_file)
-//	fmt.Printf("cfg: %#v\n", cfg.Imports)
-//}
+const import_1 = `
+    - type: counter
+      name: errors_total_1
+      help: Dummy help message.
+      match: ERROR
+`
+
+const import_2 = `
+    - type: counter
+      name: errors_total_2
+      help: Dummy help message.
+      match: ERROR
+`
+
+type mockLoader struct {
+	files []*ConfigFile
+}
+
+func (f *mockLoader) LoadDir(dir string) ([]*ConfigFile, error) {
+	return f.files, nil
+}
+
+func (f *mockLoader) LoadGlob(globString string) ([]*ConfigFile, error) {
+	return f.files, nil
+}
 
 func TestCounterValidConfig(t *testing.T) {
 	loadOrFail(t, counter_config)
@@ -365,6 +386,26 @@ func TestEmptyGrokSection(t *testing.T) {
 	loadOrFail(t, empty_grok_section)
 }
 
+func TestConfigWithImports(t *testing.T) {
+	fileLoader := &mockLoader{
+		files: []*ConfigFile{
+			{Path: "file1.yaml", Contents: import_1},
+			{Path: "file2.yaml", Contents: import_2},
+		},
+	}
+	cfg, err := unmarshal([]byte(config_with_imports), fileLoader)
+	if err != nil {
+		t.Fatalf("Failed to read config: %v", err.Error())
+	}
+	err = equalsIgnoreIndentation(cfg.String(), config_with_imports)
+	if err != nil {
+		t.Fatalf("Expected:\n%v\nActual:\n%v\n%v", config_with_imports, cfg, err)
+	}
+	if len(cfg.AllMetrics) != 3 {
+		t.Fatalf("expected 3 metrics, but found %v", len(cfg.AllMetrics))
+	}
+}
+
 func loadOrFail(t *testing.T, cfgString string) *Config {
 	cfg, err := Unmarshal([]byte(cfgString))
 	if err != nil {

config/v3/fileLoader.go

@@ -0,0 +1,71 @@
+// Copyright 2020 The grok_exporter Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package v3
+
+import (
+	"fmt"
+	"github.com/fstab/grok_exporter/tailer/glob"
+	"io/ioutil"
+	"path"
+)
+
+type ConfigFile struct {
+	Path     string
+	Contents string
+}
+
+type FileLoader interface {
+	LoadDir(dir string) ([]*ConfigFile, error)
+	LoadGlob(globString string) ([]*ConfigFile, error)
+}
+
+type fileLoader struct{}
+
+func NewFileLoader() FileLoader {
+	return &fileLoader{}
+}
+
+func (f *fileLoader) LoadDir(dir string) ([]*ConfigFile, error) {
+	return f.LoadGlob(path.Join(dir, "*"))
+}
+
+func (f *fileLoader) LoadGlob(globString string) ([]*ConfigFile, error) {
+	result := make([]*ConfigFile, 0, 0)
+	g, err := glob.Parse(globString)
+	if err != nil {
+		return nil, err
+	}
+	fileInfos, err := ioutil.ReadDir(g.Dir())
+	if err != nil {
+		return nil, err
+	}
+	for _, fileInfo := range fileInfos {
+		filePath := path.Join(g.Dir(), fileInfo.Name())
+		if g.Match(filePath) {
+			contents, err := ioutil.ReadFile(filePath)
+			if err != nil {
+				return nil, err
+			}
+			result = append(result, &ConfigFile{
+				Path:     filePath,
+				Contents: string(contents),
+			})
+		}
+	}
+	if len(result) == 0 {
+		return nil, fmt.Errorf("%v: file(s) not found", globString)
+	}
+	return result, nil
+}