Dev 2320/include exclude filter for sync (#48)

cmd/filemanager.go

@@ -33,6 +33,8 @@ const (
 	fileManagerDeleteOption      = "delete"
 	fileManagerRecursiveOption   = "recursive"
 	fileManagerOverwriteOption   = "overwrite"
+	fileManagerExcludeOption     = "exclude"
+	fileManagerIncludeOption     = "include"
 )
 
 var filemanagerCommand = Command{
@@ -71,6 +73,14 @@ var fileManagerSyncCommand = Command{
 			Help: "Delete files in the destination that are not in the source",
 			Flag: "true",
 		},
+		{
+			Name: fileManagerExcludeOption,
+			Help: "Comma-separated path patterns to exclude relative to the source directory",
+		},
+		{
+			Name: fileManagerIncludeOption,
+			Help: "Comma-separated path patterns to include relative to the source directory",
+		},
 	},
 	Target: func(opts Options) error {
 		ctx := context.Background()
@@ -89,6 +99,17 @@ var fileManagerSyncCommand = Command{
 		remotePath := path.Clean(opts[fileManagerDestinationOption])
 		localPath := filepath.Clean(opts[fileManagerSourceOption])
 
+		exludes := opts[fileManagerExcludeOption]
+		includes := opts[fileManagerIncludeOption]
+
+		if exludes != "" {
+			fileManager.WithExcludes(exludes)
+		}
+
+		if includes != "" {
+			fileManager.WithIncludes(includes)
+		}
+
 		fmt.Printf("Syncing directory %s to %s\n", localPath, remotePath)
 
 		startSync := time.Now()

filemanager.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 )
@@ -36,6 +37,10 @@ type FileManager struct {
 	deleteMissing bool
 	// dryRun does not perform any changes.
 	dryRun bool
+	// excludes removes local paths from being considered
+	excludes []regexp.Regexp
+	// includes includes only local paths that match the pattern
+	includes []regexp.Regexp
 	// remoteFiles is a map of the remote files.
 	remoteFiles map[string]File
 	// localFiles is a map of the local files.
@@ -78,6 +83,36 @@ func (m *FileManager) GetRemoteSnapshot() map[string]File {
 	return m.remoteFiles
 }
 
+// WithExcludes sets the excludes
+func (m *FileManager) WithExcludes(excludes string) *FileManager {
+	m.excludes = preparePathMatchList(excludes)
+	return m
+}
+
+// WithIncludes sets the includes
+func (m *FileManager) WithIncludes(includes string) *FileManager {
+	m.includes = preparePathMatchList(includes)
+	return m
+}
+
+// preparePathMatchList prepares a list of regular expressions from a comma-separated list of patterns.
+func preparePathMatchList(patterns string) []regexp.Regexp {
+	patternsList := strings.Split(patterns, ",")
+
+	pathMatchList := make([]regexp.Regexp, 0)
+
+	for _, pathPattern := range patternsList {
+		pathPattern = strings.TrimLeft(pathPattern, "/")
+		pathPattern = regexp.QuoteMeta(pathPattern)
+
+		pathRE := regexp.MustCompile(pathPattern)
+
+		pathMatchList = append(pathMatchList, *pathRE)
+	}
+
+	return pathMatchList
+}
+
 // Sync synchronizes the local directory with the FileManager.
 func (m *FileManager) Sync(ctx context.Context, source, dest string) error {
 
@@ -236,6 +271,10 @@ func (m *FileManager) SnapshotLocal(localPath string) error {
 			return err
 		}
 
+		if m.matchExcludes(fileName) && !m.matchIncludes(fileName) {
+			return nil
+		}
+
 		fileName = filepath.ToSlash(fileName)
 		m.localFiles[fileName] = File{
 			Name:  fileName,
@@ -254,6 +293,36 @@ func (m *FileManager) SnapshotLocal(localPath string) error {
 	return nil
 }
 
+// matchExcludes determines if a path matches any of the exclude filters.
+func (m *FileManager) matchExcludes(path string) bool {
+	if len(m.excludes) == 0 {
+		return false
+	}
+	return matchRElist(path, m.excludes)
+}
+
+// matchIncludes determines if a path matches any of the include filters.
+func (m *FileManager) matchIncludes(path string) bool {
+	if len(m.includes) == 0 {
+		return true
+	}
+	return matchRElist(path, m.includes)
+}
+
+// matchRElist determines if a pattern matches a list of regular expressions.
+func matchRElist(pattern string, list []regexp.Regexp) bool {
+	if len(list) == 0 {
+		return false
+	}
+
+	for _, re := range list {
+		if re.MatchString(pattern) {
+			return true
+		}
+	}
+	return false
+}
+
 // SnapshotRemote populates the remoteFiles map with the files in the FileManager.
 func (m *FileManager) SnapshotRemote(ctx context.Context, remotePath string) error {
 

filemanager_test.go

@@ -29,6 +29,38 @@ func Test_FileManager_Token_Refresh(t *testing.T) {
 
 }
 
+func Test_FileManager_Exclude_Include(t *testing.T) {
+
+	if !testingHasCredentials {
+		t.Skip("Skipping test because QBEE_EMAIL and QBEE_PASSWORD are not set")
+	}
+
+	ctx := context.Background()
+
+	cli, err := LoginGetAuthenticatedClient(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m := NewFileManager().WithClient(cli).WithDryRun(true).WithExcludes("cmd/,.git").WithIncludes("cmd/filemanager.go")
+
+	if err := m.SnapshotLocal("."); err != nil {
+		t.Fatal(err)
+	}
+
+	list := m.GetLocalSnapshot()
+	if len(list) == 0 {
+		t.Fatal("should have 1 file")
+	}
+
+	for _, f := range list {
+		if f.Path == "cmd/filemanager.go" {
+			return
+		}
+	}
+	t.Fatal("file not found")
+}
+
 func Test_FileManager_Upload_Download(t *testing.T) {
 	if !testingHasCredentials {
 		t.Skip("Skipping test because QBEE_EMAIL and QBEE_PASSWORD are not set")