diff --git a/slug.go b/slug.go index 8dd4078..059d267 100644 --- a/slug.go +++ b/slug.go @@ -33,11 +33,15 @@ func Pack(src string, w io.Writer, dereference bool) (*Meta, error) { // Tar the file contents. tarW := tar.NewWriter(gzipW) + // Load the ignore rule configuration, which will use + // defaults if no .terraformignore is configured + ignoreRules := parseIgnoreFile(src) + // Track the metadata details as we go. meta := &Meta{} // Walk the tree of files. - err := filepath.Walk(src, packWalkFn(src, src, src, tarW, meta, dereference)) + err := filepath.Walk(src, packWalkFn(src, src, src, tarW, meta, dereference, ignoreRules)) if err != nil { return nil, err } @@ -55,17 +59,12 @@ func Pack(src string, w io.Writer, dereference bool) (*Meta, error) { return meta, nil } -func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference bool) filepath.WalkFunc { +func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference bool, ignoreRules []rule) filepath.WalkFunc { return func(path string, info os.FileInfo, err error) error { if err != nil { return err } - // Skip the .git directory. - if info.IsDir() && info.Name() == ".git" { - return filepath.SkipDir - } - // Get the relative path from the current src directory. subpath, err := filepath.Rel(src, path) if err != nil { @@ -75,20 +74,16 @@ func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference return nil } - // Ignore the .terraform directory itself. - if info.IsDir() && info.Name() == ".terraform" { - return nil - } - - // Ignore any files in the .terraform directory. - if !info.IsDir() && filepath.Dir(subpath) == ".terraform" { + if m := matchIgnoreRule(subpath, ignoreRules); m { return nil } - // Skip .terraform subdirectories, except for the modules subdirectory. - if strings.HasPrefix(subpath, ".terraform"+string(filepath.Separator)) && - !strings.HasPrefix(subpath, filepath.Clean(".terraform/modules")) { - return filepath.SkipDir + // Catch directories so we don't end up with empty directories, + // the files are ignored correctly + if info.IsDir() { + if m := matchIgnoreRule(subpath+string(os.PathSeparator), ignoreRules); m { + return nil + } } // Get the relative path from the initial root directory. @@ -159,7 +154,7 @@ func packWalkFn(root, src, dst string, tarW *tar.Writer, meta *Meta, dereference // If the target is a directory we can recurse into the target // directory by calling the packWalkFn with updated arguments. if info.IsDir() { - return filepath.Walk(target, packWalkFn(root, target, path, tarW, meta, dereference)) + return filepath.Walk(target, packWalkFn(root, target, path, tarW, meta, dereference, ignoreRules)) } // Dereference this symlink by updating the header with the target file diff --git a/slug_test.go b/slug_test.go index e8cdf61..b72c617 100644 --- a/slug_test.go +++ b/slug_test.go @@ -115,6 +115,18 @@ func TestPack(t *testing.T) { t.Fatal("expected to include foo.terraform/bar.txt") } + // Make sure baz.txt is excluded. + bazTxt := false + for _, file := range fileList { + if file == filepath.Clean("baz.txt") { + bazTxt = true + break + } + } + if bazTxt { + t.Fatal("should not include baz.txt") + } + // Check the metadata expect := &Meta{ Files: fileList, diff --git a/terraformignore.go b/terraformignore.go new file mode 100644 index 0000000..864dc92 --- /dev/null +++ b/terraformignore.go @@ -0,0 +1,225 @@ +package slug + +import ( + "bufio" + "fmt" + "io" + "os" + "path/filepath" + "regexp" + "strings" + "text/scanner" +) + +func parseIgnoreFile(rootPath string) []rule { + // Look for .terraformignore at our root path/src + file, err := os.Open(filepath.Join(rootPath, ".terraformignore")) + defer file.Close() + + // If there's any kind of file error, punt and use the default ignore patterns + if err != nil { + // Only show the error debug if an error *other* than IsNotExist + if !os.IsNotExist(err) { + fmt.Fprintf(os.Stderr, "Error reading .terraformignore, default exclusions will apply: %v \n", err) + } + return defaultExclusions + } + return readRules(file) +} + +func readRules(input io.Reader) []rule { + rules := defaultExclusions + scanner := bufio.NewScanner(input) + scanner.Split(bufio.ScanLines) + + for scanner.Scan() { + pattern := scanner.Text() + // Ignore blank lines + if len(pattern) == 0 { + continue + } + // Trim spaces + pattern = strings.TrimSpace(pattern) + // Ignore comments + if pattern[0] == '#' { + continue + } + // New rule structure + rule := rule{} + // Exclusions + if pattern[0] == '!' { + rule.excluded = true + pattern = pattern[1:] + } + // If it is a directory, add ** so we catch descendants + if pattern[len(pattern)-1] == os.PathSeparator { + pattern = pattern + "**" + } + // If it starts with /, it is absolute + if pattern[0] == os.PathSeparator { + pattern = pattern[1:] + } else { + // Otherwise prepend **/ + pattern = "**" + string(os.PathSeparator) + pattern + } + rule.val = pattern + rule.dirs = strings.Split(pattern, string(os.PathSeparator)) + rules = append(rules, rule) + } + + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Error reading .terraformignore, default exclusions will apply: %v \n", err) + return defaultExclusions + } + return rules +} + +func matchIgnoreRule(path string, rules []rule) bool { + matched := false + path = filepath.FromSlash(path) + for _, rule := range rules { + match, _ := rule.match(path) + + if match { + matched = !rule.excluded + } + } + + if matched { + debug(true, path, "Skipping excluded path:", path) + } + + return matched +} + +type rule struct { + val string // the value of the rule itself + excluded bool // ! is present, an exclusion rule + dirs []string // directories of the rule + regex *regexp.Regexp // regular expression to match for the rule +} + +func (r *rule) match(path string) (bool, error) { + if r.regex == nil { + if err := r.compile(); err != nil { + return false, filepath.ErrBadPattern + } + } + + b := r.regex.MatchString(path) + debug(false, path, path, r.regex, b) + return b, nil +} + +func (r *rule) compile() error { + regStr := "^" + pattern := r.val + // Go through the pattern and convert it to a regexp. + // Use a scanner to support utf-8 chars. + var scan scanner.Scanner + scan.Init(strings.NewReader(pattern)) + + sl := string(os.PathSeparator) + escSL := sl + if sl == `\` { + escSL += `\` + } + + for scan.Peek() != scanner.EOF { + ch := scan.Next() + if ch == '*' { + if scan.Peek() == '*' { + // is some flavor of "**" + scan.Next() + + // Treat **/ as ** so eat the "/" + if string(scan.Peek()) == sl { + scan.Next() + } + + if scan.Peek() == scanner.EOF { + // is "**EOF" - to align with .gitignore just accept all + regStr += ".*" + } else { + // is "**" + // Note that this allows for any # of /'s (even 0) because + // the .* will eat everything, even /'s + regStr += "(.*" + escSL + ")?" + } + } else { + // is "*" so map it to anything but "/" + regStr += "[^" + escSL + "]*" + } + } else if ch == '?' { + // "?" is any char except "/" + regStr += "[^" + escSL + "]" + } else if ch == '.' || ch == '$' { + // Escape some regexp special chars that have no meaning + // in golang's filepath.Match + regStr += `\` + string(ch) + } else if ch == '\\' { + // escape next char. Note that a trailing \ in the pattern + // will be left alone (but need to escape it) + if sl == `\` { + // On windows map "\" to "\\", meaning an escaped backslash, + // and then just continue because filepath.Match on + // Windows doesn't allow escaping at all + regStr += escSL + continue + } + if scan.Peek() != scanner.EOF { + regStr += `\` + string(scan.Next()) + } else { + regStr += `\` + } + } else { + regStr += string(ch) + } + } + + regStr += "$" + re, err := regexp.Compile(regStr) + if err != nil { + return err + } + + r.regex = re + return nil +} + +/* + Default rules as they would appear in .terraformignore: + .git/ + .terraform/ + !.terraform/modules/ +*/ + +var defaultExclusions = []rule{ + { + val: "**/.git/**", + excluded: false, + }, + { + val: "**/.terraform/**", + excluded: false, + }, + { + val: "**/.terraform/modules/**", + excluded: true, + }, +} + +func debug(printAll bool, path string, message ...interface{}) { + logLevel := os.Getenv("TF_IGNORE") == "trace" + debugPath := os.Getenv("TF_IGNORE_DEBUG") + isPath := debugPath != "" + if isPath { + isPath = strings.Contains(path, debugPath) + } + + if logLevel { + if printAll || isPath { + fmt.Println(message...) + } + } +} diff --git a/terraformignore_test.go b/terraformignore_test.go new file mode 100644 index 0000000..a5a13ac --- /dev/null +++ b/terraformignore_test.go @@ -0,0 +1,114 @@ +package slug + +import ( + "testing" +) + +func TestTerraformIgnore(t *testing.T) { + // path to directory without .terraformignore + p := parseIgnoreFile("testdata/external-dir") + if len(p) != 3 { + t.Fatal("A directory without .terraformignore should get the default patterns") + } + + // load the .terraformignore file's patterns + ignoreRules := parseIgnoreFile("testdata/archive-dir") + type file struct { + // the actual path, should be file path format /dir/subdir/file.extension + path string + // should match + match bool + } + paths := []file{ + { + path: ".terraform/", + match: true, + }, + { + path: "included.txt", + match: false, + }, + { + path: ".terraform/foo/bar", + match: true, + }, + { + path: ".terraform/foo/bar/more/directories/so/many", + match: true, + }, + { + path: ".terraform/foo/ignored-subdirectory/", + match: true, + }, + { + path: "baz.txt", + match: true, + }, + { + path: "parent/foo/baz.txt", + match: true, + }, + { + path: "parent/foo/bar.tf", + match: true, + }, + { + path: "parent/bar/bar.tf", + match: false, + }, + // baz.txt is ignored, but a file name including it should not be + { + path: "something/with-baz.txt", + match: false, + }, + { + path: "something/baz.x", + match: false, + }, + // Getting into * patterns + { + path: "foo/ignored-doc.md", + match: true, + }, + // Should match [a-z] group + { + path: "bar/something-a.txt", + match: true, + }, + // ignore sub- terraform.d paths + { + path: "some-module/terraform.d/x", + match: true, + }, + // but not the root one + { + path: "terraform.d/", + match: false, + }, + { + path: "terraform.d/foo", + match: false, + }, + // We ignore the directory, but a file of the same name could exist + { + path: "terraform.d", + match: false, + }, + // boop.text is ignored everywhere + { + path: "baz/boop.txt", + match: true, + }, + // except at current directory + { + path: "boop.txt", + match: false, + }, + } + for i, p := range paths { + match := matchIgnoreRule(p.path, ignoreRules) + if match != p.match { + t.Fatalf("%s at index %d should be %t", p.path, i, p.match) + } + } +} diff --git a/testdata/.DS_Store b/testdata/.DS_Store new file mode 100644 index 0000000..d281a79 Binary files /dev/null and b/testdata/.DS_Store differ diff --git a/testdata/archive-dir/.terraformignore b/testdata/archive-dir/.terraformignore new file mode 100644 index 0000000..3503ae9 --- /dev/null +++ b/testdata/archive-dir/.terraformignore @@ -0,0 +1,20 @@ +# comments are ignored + # extra spaces are irrelevant +# ignore a file + baz.txt +# below is an empty line + +# ignore a directory +terraform.d/ +# exclude ignoring a directory at the root +!/terraform.d/ +# ignore a file at a subpath +**/foo/bar.tf +# ignore files with specific endings +foo/*.md +# character groups +bar/something-[a-z].txt +# ignore a file +boop.txt +# but not one at the current directory +!/boop.txt \ No newline at end of file diff --git a/testdata/archive-dir/baz.txt b/testdata/archive-dir/baz.txt new file mode 100644 index 0000000..3f95386 --- /dev/null +++ b/testdata/archive-dir/baz.txt @@ -0,0 +1 @@ +baz \ No newline at end of file