Implement file stub generator

Author: mblaschke

examples/gosync.yml

@@ -33,6 +33,12 @@ sync:
         - .git
         - node_modules/
 
+    ## Create file stubs for /tmp/remote/other/path to ./other/
+    - path: /tmp/remote/other/path-big-files
+      local: ./other-stubs/
+      options:
+        generate-stubs: true
+
     #########################
     ## Database
     #########################

sync/config.go

@@ -19,6 +19,9 @@ type Filesystem struct {
 	Path string
 	Local string
 	Filter Filter
+	Options struct {
+		GenerateStubs bool `yaml:"generate-stubs"`
+	}
 }
 
 

sync/filter.go

@@ -20,6 +20,30 @@ func (filter *Filter) compile() {
 	}
 }
 
+func (filter *Filter) ApplyFilter(lines []string) []string {
+	filter.compile()
+
+	if len(filter.Include) > 0 {
+		lines = filter.calculateMatching(filter.includeRegexp, lines)
+	}
+
+	if len(filter.Exclude) > 0 {
+		excludes := filter.calculateMatching(filter.excludeRegexp, lines)
+
+		tmp := []string{}
+		for _, line := range lines {
+			if ! stringInSlice(line, excludes) {
+				tmp = append(tmp, line)
+			}
+		}
+
+		lines = tmp
+	}
+
+	return lines
+}
+
+
 func (filter *Filter) CalcExcludes(lines []string) []string {
 	filter.compile()
 	return filter.calculateMatching(filter.excludeRegexp, lines)
@@ -43,3 +67,12 @@ func (filter *Filter) calculateMatching(regexpList []*regexp.Regexp, lines []str
 
 	return ret
 }
+
+func stringInSlice(a string, list []string) bool {
+	for _, b := range list {
+		if b == a {
+			return true
+		}
+	}
+	return false
+}

sync/server_deploy.go

@@ -15,6 +15,13 @@ func (server *Server) Deploy() {
 }
 
 func (server *Server) DeployFilesystem() {
+	// check for generate-stubs option (not allowed)
+	for _, filesystem := range server.Filesystem {
+		if filesystem.Options.GenerateStubs {
+			Logger.FatalExit(2, "Generate Stubs is not allowed for deployment")
+		}
+	}
+
 	for _, filesystem := range server.Filesystem {
 		Logger.Main("Starting deploy of %s", filesystem.String(server, "deploy"))
 		filesystem.Deploy(server)

sync/server_sync.go

@@ -16,8 +16,13 @@ func (server *Server) Sync() {
 
 func (server *Server) SyncFilesystem() {
 	for _, filesystem := range server.Filesystem {
-		Logger.Main("Starting sync of %s", filesystem.String(server, "sync"))
-		filesystem.Sync(server)
+		if filesystem.Options.GenerateStubs {
+			Logger.Main("Starting stub generator for %s", filesystem.String(server, "sync"))
+			filesystem.SyncStubs(server)
+		} else {
+			Logger.Main("Starting sync of %s", filesystem.String(server, "sync"))
+			filesystem.Sync(server)
+		}
 	}
 }
 

sync/server_sync_filesystem_stubs.go

@@ -0,0 +1,49 @@
+package sync
+
+import (
+	"errors"
+	"bufio"
+	"strings"
+	"path"
+	"path/filepath"
+	"github.com/webdevops/go-shell"
+	"github.com/webdevops/go-stubfilegenerator"
+)
+
+// General sync
+func (filesystem *Filesystem) SyncStubs(server *Server) {
+	switch server.Connection.GetType() {
+	case "ssh":
+		filesystem.generateStubs(server)
+	case "docker":
+		errors.New("Docker not supported")
+	}
+}
+
+// Sync filesystem using rsync
+func (filesystem *Filesystem) generateStubs(server *Server) {
+	cmd := shell.Cmd(server.Connection.CommandBuilder("find", filesystem.Path, "-type f")...)
+	output := cmd.Run().Stdout.String()
+
+	removeBasePath := filesystem.Path
+	localBasePath := filesystem.localPath(server)
+
+	// build list and filter it by user filter list
+	fileList := []string{}
+	scanner := bufio.NewScanner(strings.NewReader(output))
+	for scanner.Scan() {
+		fileList = append(fileList, strings.TrimPrefix(scanner.Text(), removeBasePath))
+	}
+	fileList = filesystem.Filter.ApplyFilter(fileList)
+
+
+	// generate stubs
+	stubGen := stubfilegenerator.StubGenerator()
+	for _, filePath := range fileList {
+		localPath := path.Join(localBasePath, filePath)
+
+		stubGen.TemplateVariables["PATH"] = localPath
+		localAbsPath, _ := filepath.Abs(localPath)
+		stubGen.GenerateStub(localAbsPath)
+	}
+}