From 6824eb41ea6a99df67a080d8a2dae87d122fbbd3 Mon Sep 17 00:00:00 2001 From: Miccah Date: Tue, 30 Jan 2024 12:21:37 -0800 Subject: [PATCH 1/2] Fix filesystem enumeration ignore paths bug (#2355) --- pkg/sources/filesystem/filesystem.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go index 9a1c4ceb9975..ca4edd947bd3 100644 --- a/pkg/sources/filesystem/filesystem.go +++ b/pkg/sources/filesystem/filesystem.go @@ -240,6 +240,9 @@ func (s *Source) Enumerate(ctx context.Context, reporter sources.UnitReporter) e return nil } fullPath := filepath.Join(path, relativePath) + if s.filter != nil && !s.filter.Pass(fullPath) { + return nil + } item := sources.CommonSourceUnit{ID: fullPath} return reporter.UnitOk(ctx, item) }) From 7c59ff95d5c77326ce83c21d6b09d0f9f4be7d5d Mon Sep 17 00:00:00 2001 From: ahrav Date: Tue, 30 Jan 2024 12:30:51 -0800 Subject: [PATCH 2/2] [feat] - tmp file diffs (#2306) * Write large diffs to tmp files * address comments * Move bufferedfilewriter to own pkg * update test * swallow write err * use buffer pool * use size vs len * use interface * fix test * update comments * fix test * remove unused * remove * remove unused * move parser and commit struct closer to where they are used * linter change * add more kvp pairs to error * fix test * update * address comments * remove bufferedfile writer * address comments * adjust interface * fix finalize * address comments * lint * remove guard * fix * add TODO --- pkg/gitparse/gitparse.go | 298 +++++++++++++++++++++++++--------- pkg/gitparse/gitparse_test.go | 286 ++++++++++++++++++-------------- pkg/sources/git/git.go | 96 +++++++---- 3 files changed, 455 insertions(+), 225 deletions(-) diff --git a/pkg/gitparse/gitparse.go b/pkg/gitparse/gitparse.go index dcd544c85531..14aecc6bafa5 100644 --- a/pkg/gitparse/gitparse.go +++ b/pkg/gitparse/gitparse.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "fmt" - "github.com/go-logr/logr" "io" "os" "os/exec" @@ -13,6 +12,8 @@ import ( "strings" "time" + "github.com/go-logr/logr" + "github.com/trufflesecurity/trufflehog/v3/pkg/common" "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) @@ -22,12 +23,122 @@ const ( defaultDateFormat = "Mon Jan 02 15:04:05 2006 -0700" // defaultMaxDiffSize is the maximum size for a diff. Larger diffs will be cut off. - defaultMaxDiffSize = 1 * 1024 * 1024 * 1024 // 1GB + defaultMaxDiffSize = 2 * 1024 * 1024 * 1024 // 2GB // defaultMaxCommitSize is the maximum size for a commit. Larger commits will be cut off. - defaultMaxCommitSize = 1 * 1024 * 1024 * 1024 // 1GB + defaultMaxCommitSize = 2 * 1024 * 1024 * 1024 // 2GB +) + +// contentWriter defines a common interface for writing, reading, and managing diff content. +// It abstracts the underlying storage mechanism, allowing flexibility in how content is handled. +// This interface enables the use of different content storage strategies (e.g., in-memory buffer, file-based storage) +// based on performance needs or resource constraints, providing a unified way to interact with different content types. +type contentWriter interface { // Write appends data to the content storage. + // Write appends data to the content storage. + Write(ctx context.Context, data []byte) (int, error) + // ReadCloser provides a reader for accessing stored content. + ReadCloser() (io.ReadCloser, error) + // CloseForWriting closes the content storage for writing. + CloseForWriting() error + // Len returns the current size of the content. + Len() int + // String returns the content as a string or an error if the content cannot be converted to a string. + String() (string, error) +} + +// state represents the current mode of buffer. +type state uint8 + +const ( + // writeOnly indicates the buffer is in write-only mode. + writeOnly state = iota + // readOnly indicates the buffer has been closed and is in read-only mode. + readOnly ) +// buffer is a wrapper around bytes.Buffer, implementing the contentWriter interface. +// This allows bytes.Buffer to be used wherever a contentWriter is required, ensuring compatibility +// with the contentWriter interface while leveraging the existing implementation of bytes.Buffer. +type buffer struct { + state state // current state of the buffer (writeOnly or readOnly) + bytes.Buffer +} + +func newBuffer() *buffer { return &buffer{state: writeOnly} } + +// Write delegates the writing operation to the underlying bytes.Buffer, ignoring the context. +// The context is included to satisfy the contentWriter interface, allowing for future extensions +// where context handling might be necessary (e.g., for timeouts or cancellation). +func (b *buffer) Write(_ context.Context, data []byte) (int, error) { + if b.state == readOnly { + return 0, fmt.Errorf("buffer is in read-only mode") + } + return b.Buffer.Write(data) +} + +// ReadCloser provides a read-closer for the buffer's content. +// It wraps the buffer's content in a NopCloser to provide a ReadCloser without additional closing behavior, +// as closing a bytes.Buffer is a no-op. +func (b *buffer) ReadCloser() (io.ReadCloser, error) { + if b.state == writeOnly { + return nil, fmt.Errorf("buffer is in write-only mode") + } + return io.NopCloser(bytes.NewReader(b.Bytes())), nil +} + +// CloseForWriting is a no-op for buffer, as there is no resource cleanup needed for bytes.Buffer. +func (b *buffer) CloseForWriting() error { + b.state = readOnly + return nil +} + +// String returns the buffer's content as a string. +func (b *buffer) String() (string, error) { return b.Buffer.String(), nil } + +// Diff contains the information about a file diff in a commit. +// It abstracts the underlying content representation, allowing for flexible handling of diff content. +// The use of contentWriter enables the management of diff data either in memory or on disk, +// based on its size, optimizing resource usage and performance. +type Diff struct { + PathB string + LineStart int + contentWriter contentWriter + IsBinary bool +} + +type diffOption func(*Diff) + +// withPathB sets the PathB option. +func withPathB(pathB string) diffOption { return func(d *Diff) { d.PathB = pathB } } + +// NewDiff creates a new Diff with a threshold. +func NewDiff(opts ...diffOption) *Diff { + diff := new(Diff) + diff.contentWriter = newBuffer() + for _, opt := range opts { + opt(diff) + } + + return diff +} + +// Len returns the length of the storage. +func (d *Diff) Len() int { return d.contentWriter.Len() } + +// ReadCloser returns a ReadCloser for the contentWriter. +func (d *Diff) ReadCloser() (io.ReadCloser, error) { return d.contentWriter.ReadCloser() } + +// write delegates to the contentWriter. +func (d *Diff) write(ctx context.Context, p []byte) error { + _, err := d.contentWriter.Write(ctx, p) + return err +} + +// finalize ensures proper closure of resources associated with the Diff. +// handle the final flush in the finalize method, in case there's data remaining in the buffer. +// This method should be called to release resources, especially when writing to a file. +func (d *Diff) finalize() error { return d.contentWriter.CloseForWriting() } + // Commit contains commit header info and diffs. type Commit struct { Hash string @@ -38,12 +149,53 @@ type Commit struct { Size int // in bytes } -// Diff contains the info about a file diff in a commit. -type Diff struct { - PathB string - LineStart int - Content bytes.Buffer - IsBinary bool +// Equal compares the content of two Commits to determine if they are the same. +func (c1 *Commit) Equal(ctx context.Context, c2 *Commit) bool { + switch { + case c1.Hash != c2.Hash: + return false + case c1.Author != c2.Author: + return false + case !c1.Date.Equal(c2.Date): + return false + case c1.Message.String() != c2.Message.String(): + return false + case len(c1.Diffs) != len(c2.Diffs): + return false + } + + // isEqualString handles the error-prone String() method calls and compares the results. + isEqualContentString := func(s1, s2 contentWriter) (bool, error) { + str1, err := s1.String() + if err != nil { + return false, err + } + str2, err := s2.String() + if err != nil { + return false, err + } + return str1 == str2, nil + } + + for i := range c1.Diffs { + d1 := c1.Diffs[i] + d2 := c2.Diffs[i] + switch { + case d1.PathB != d2.PathB: + return false + case d1.LineStart != d2.LineStart: + return false + case d1.IsBinary != d2.IsBinary: + return false + default: + equal, err := isEqualContentString(d1.contentWriter, d2.contentWriter) + if err != nil || !equal { + ctx.Logger().Error(err, "failed to compare diff content") + return false + } + } + } + return true } // Parser sets values used in GitParse. @@ -51,6 +203,7 @@ type Parser struct { maxDiffSize int maxCommitSize int dateFormat string + contentWriter contentWriter } type ParseState int @@ -123,6 +276,7 @@ func NewParser(options ...Option) *Parser { dateFormat: defaultDateFormat, maxDiffSize: defaultMaxDiffSize, maxCommitSize: defaultMaxCommitSize, + contentWriter: newBuffer(), } for _, option := range options { option(parser) @@ -130,38 +284,6 @@ func NewParser(options ...Option) *Parser { return parser } -// Equal compares the content of two Commits to determine if they are the same. -func (c1 *Commit) Equal(c2 *Commit) bool { - switch { - case c1.Hash != c2.Hash: - return false - case c1.Author != c2.Author: - return false - case !c1.Date.Equal(c2.Date): - return false - case c1.Message.String() != c2.Message.String(): - return false - case len(c1.Diffs) != len(c2.Diffs): - return false - } - for i := range c1.Diffs { - d1 := c1.Diffs[i] - d2 := c2.Diffs[i] - switch { - case d1.PathB != d2.PathB: - return false - case d1.LineStart != d2.LineStart: - return false - case d1.Content.String() != d2.Content.String(): - return false - case d1.IsBinary != d2.IsBinary: - return false - } - } - return true - -} - // RepoPath parses the output of the `git log` command for the `source` path. func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool, excludedGlobs []string, isBare bool) (chan Commit, error) { args := []string{"-C", source, "log", "-p", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"} @@ -254,11 +376,11 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch outReader := bufio.NewReader(stdOut) var ( currentCommit *Commit - currentDiff Diff totalLogSize int ) var latestState = Initial + currentDiff := NewDiff() defer common.RecoverWithExit(ctx) defer close(commitChan) @@ -277,9 +399,23 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch latestState = CommitLine // If there is a currentDiff, add it to currentCommit. - if currentDiff.Content.Len() > 0 || currentDiff.IsBinary { - currentCommit.Diffs = append(currentCommit.Diffs, currentDiff) - currentCommit.Size += currentDiff.Content.Len() + if currentDiff.Len() > 0 || currentDiff.IsBinary { + // TODO: Consider modifying the diffs field in the Commit struct to be a []*Diff. + // Otherwise, we end up with this temporal coupling where we have to finalize the diff + // before we can add it to the commit. I found this out the hard way when I tried to + // test this. + if err := currentDiff.finalize(); err != nil { + ctx.Logger().Error( + err, + "failed to finalize diff", + "commit", currentCommit.Hash, + "diff", currentDiff.PathB, + "size", currentDiff.Len(), + "latest_state", latestState.String(), + ) + } + currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff) + currentCommit.Size += currentDiff.Len() } // If there is a currentCommit, send it to the channel. if currentCommit != nil { @@ -287,10 +423,8 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch totalLogSize += currentCommit.Size } // Create a new currentDiff and currentCommit - currentDiff = Diff{} - currentCommit = &Commit{ - Message: strings.Builder{}, - } + currentDiff = NewDiff() + currentCommit = &Commit{Message: strings.Builder{}} // Check that the commit line contains a hash and set it. if len(line) >= 47 { currentCommit.Hash = string(line[7:47]) @@ -326,12 +460,21 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch if currentCommit == nil { currentCommit = &Commit{} } - if currentDiff.Content.Len() > 0 || currentDiff.IsBinary { - currentCommit.Diffs = append(currentCommit.Diffs, currentDiff) + if currentDiff.Len() > 0 || currentDiff.IsBinary { + if err := currentDiff.finalize(); err != nil { + ctx.Logger().Error(err, + "failed to finalize diff", + "commit", currentCommit.Hash, + "diff", currentDiff.PathB, + "size", currentDiff.Len(), + "latest_state", latestState.String(), + ) + } + currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff) // If the currentDiff is over 1GB, drop it into the channel so it isn't held in memory waiting for more commits. totalSize := 0 for _, diff := range currentCommit.Diffs { - totalSize += diff.Content.Len() + totalSize += diff.Len() } if totalSize > c.maxCommitSize { oldCommit := currentCommit @@ -348,7 +491,7 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch currentCommit.Message.WriteString(oldCommit.Message.String()) } } - currentDiff = Diff{} + currentDiff = NewDiff() case isModeLine(isStaged, latestState, line): latestState = ModeLine // NoOp @@ -375,12 +518,20 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch case isHunkLineNumberLine(isStaged, latestState, line): latestState = HunkLineNumberLine - if currentDiff.Content.Len() > 0 || currentDiff.IsBinary { - currentCommit.Diffs = append(currentCommit.Diffs, currentDiff) - } - currentDiff = Diff{ - PathB: currentDiff.PathB, + if currentDiff.Len() > 0 || currentDiff.IsBinary { + if err := currentDiff.finalize(); err != nil { + ctx.Logger().Error( + err, + "failed to finalize diff", + "commit", currentCommit.Hash, + "diff", currentDiff.PathB, + "size", currentDiff.Len(), + "latest_state", latestState.String(), + ) + } + currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff) } + currentDiff = NewDiff(withPathB(currentDiff.PathB)) words := bytes.Split(line, []byte(" ")) if len(words) >= 3 { @@ -395,24 +546,21 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch latestState = HunkContentLine } // TODO: Why do we care about this? It creates empty lines in the diff. If there are no plusLines, it's just newlines. - currentDiff.Content.Write([]byte("\n")) + if err := currentDiff.write(ctx, []byte("\n")); err != nil { + ctx.Logger().Error(err, "failed to write to diff") + } case isHunkPlusLine(isStaged, latestState, line): if latestState != HunkContentLine { latestState = HunkContentLine } - currentDiff.Content.Write(line[1:]) - case isHunkMinusLine(isStaged, latestState, line): - if latestState != HunkContentLine { - latestState = HunkContentLine + if err := currentDiff.write(ctx, line[1:]); err != nil { + ctx.Logger().Error(err, "failed to write to diff") } // NoOp. We only care about additions. - case isHunkNewlineWarningLine(isStaged, latestState, line): - if latestState != HunkContentLine { - latestState = HunkContentLine - } - // NoOp - case isHunkEmptyLine(isStaged, latestState, line): + case isHunkMinusLine(isStaged, latestState, line), + isHunkNewlineWarningLine(isStaged, latestState, line), + isHunkEmptyLine(isStaged, latestState, line): if latestState != HunkContentLine { latestState = HunkContentLine } @@ -439,14 +587,14 @@ func (c *Parser) FromReader(ctx context.Context, stdOut io.Reader, commitChan ch latestState = ParseFailure } - if currentDiff.Content.Len() > c.maxDiffSize { + if currentDiff.Len() > c.maxDiffSize { ctx.Logger().V(2).Info(fmt.Sprintf( "Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize, )) break } } - cleanupParse(currentCommit, ¤tDiff, commitChan, &totalLogSize) + cleanupParse(ctx, currentCommit, currentDiff, commitChan, &totalLogSize) ctx.Logger().V(2).Info("finished parsing git log.", "total_log_size", totalLogSize) } @@ -716,9 +864,13 @@ func isCommitSeparatorLine(isStaged bool, latestState ParseState, line []byte) b return false } -func cleanupParse(currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) { +func cleanupParse(ctx context.Context, currentCommit *Commit, currentDiff *Diff, commitChan chan Commit, totalLogSize *int) { + if err := currentDiff.finalize(); err != nil { + ctx.Logger().Error(err, "failed to finalize diff") + return + } // Ignore empty or binary diffs (this condition may be redundant). - if currentDiff != nil && (currentDiff.Content.Len() > 0 || currentDiff.IsBinary) { + if currentDiff != nil && (currentDiff.Len() > 0 || currentDiff.IsBinary) { currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff) } if currentCommit != nil { diff --git a/pkg/gitparse/gitparse_test.go b/pkg/gitparse/gitparse_test.go index fa35ebf711da..9f1e03abc3fe 100644 --- a/pkg/gitparse/gitparse_test.go +++ b/pkg/gitparse/gitparse_test.go @@ -6,6 +6,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/assert" + "github.com/trufflesecurity/trufflehog/v3/pkg/context" ) @@ -703,7 +705,7 @@ func TestCommitParsing(t *testing.T) { break } - if !commit.Equal(&expected[i]) { + if !commit.Equal(context.Background(), &expected[i]) { t.Errorf("Commit does not match.\nexpected: %+v\n%s\nactual : %+v\n%s", expected[i], expected[i].Message.String(), commit, commit.Message.String()) } i++ @@ -736,7 +738,7 @@ func TestIndividualCommitParsing(t *testing.T) { } // Assert - if !commit.Equal(&expected[i]) { + if !commit.Equal(context.Background(), &expected[i]) { t.Errorf("Commit does not match.\nexpected: %+v\n%s\nactual : %+v\n%s", expected[i], expected[j].Message.String(), commit, commit.Message.String()) } j++ @@ -744,6 +746,12 @@ func TestIndividualCommitParsing(t *testing.T) { } } +func newBufferWithContent(content []byte) *buffer { + var b buffer + _, _ = b.Write(context.Background(), content) // Using Write method to add content + return &b +} + func TestStagedDiffParsing(t *testing.T) { expected := []Commit{ { @@ -753,44 +761,45 @@ func TestStagedDiffParsing(t *testing.T) { Message: strings.Builder{}, Diffs: []Diff{ { - PathB: "aws", - LineStart: 1, - Content: *bytes.NewBuffer([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")), - IsBinary: false, + PathB: "aws", + LineStart: 1, + contentWriter: newBufferWithContent([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")), + IsBinary: false, }, { - PathB: "aws2", - LineStart: 1, - Content: *bytes.NewBuffer([]byte("\n\nthis is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n\nokay thank you bye\n")), - IsBinary: false, + PathB: "aws2", + LineStart: 1, + contentWriter: newBufferWithContent([]byte("\n\nthis is the secret: [Default]\nAccess key Id: AKIAILE3JG6KMS3HZGCA\nSecret Access Key: 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\n\nokay thank you bye\n")), + IsBinary: false, }, { - PathB: "core/runtime/src/main/java/io/quarkus/runtime/QuarkusApplication.java", - LineStart: 3, - Content: *bytes.NewBuffer([]byte("/**\n * This is usually used for command mode applications with a startup logic. The logic is executed inside\n * {@link QuarkusApplication#run} method before the main application exits.\n */\n")), - IsBinary: false, + PathB: "core/runtime/src/main/java/io/quarkus/runtime/QuarkusApplication.java", + LineStart: 3, + contentWriter: newBufferWithContent([]byte("/**\n * This is usually used for command mode applications with a startup logic. The logic is executed inside\n * {@link QuarkusApplication#run} method before the main application exits.\n */\n")), + IsBinary: false, }, { - PathB: "trufflehog_3.42.0_linux_arm64.tar.gz", - IsBinary: true, + PathB: "trufflehog_3.42.0_linux_arm64.tar.gz", + IsBinary: true, + contentWriter: newBufferWithContent(nil), }, { - PathB: "tzu", - LineStart: 11, - Content: *bytes.NewBuffer([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")), - IsBinary: false, + PathB: "tzu", + LineStart: 11, + contentWriter: newBufferWithContent([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")), + IsBinary: false, }, { - PathB: "lao", - LineStart: 1, - Content: *bytes.NewBuffer([]byte("The Way that can be told of is not the eternal Way;\nThe name that can be named is not the eternal name.\nThe Nameless is the origin of Heaven and Earth;\nThe Named is the mother of all things.\nTherefore let there always be non-being,\n so we may see their subtlety,\nAnd let there always be being,\n so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n they have different names.\n")), - IsBinary: false, + PathB: "lao", + LineStart: 1, + contentWriter: newBufferWithContent([]byte("The Way that can be told of is not the eternal Way;\nThe name that can be named is not the eternal name.\nThe Nameless is the origin of Heaven and Earth;\nThe Named is the mother of all things.\nTherefore let there always be non-being,\n so we may see their subtlety,\nAnd let there always be being,\n so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n they have different names.\n")), + IsBinary: false, }, { - PathB: "tzu", - LineStart: 1, - Content: *bytes.NewBuffer([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n so we may see their subtlety,\nAnd let there always be being,\n so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")), - IsBinary: false, + PathB: "tzu", + LineStart: 1, + contentWriter: newBufferWithContent([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n so we may see their subtlety,\nAnd let there always be being,\n so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")), + IsBinary: false, }, }, }, @@ -809,7 +818,7 @@ func TestStagedDiffParsing(t *testing.T) { break } - if !commit.Equal(&expected[i]) { + if !commit.Equal(context.Background(), &expected[i]) { t.Errorf("Commit does not match.\nexpected:\n%+v\n\nactual:\n%+v\n", expected[i], commit) } i++ @@ -825,10 +834,10 @@ func TestCommitParseFailureRecovery(t *testing.T) { Message: newStringBuilderValue("Add travis testing\n"), Diffs: []Diff{ { - PathB: ".travis.yml", - LineStart: 1, - Content: *bytes.NewBuffer([]byte("language: python\npython:\n - \"2.6\"\n - \"2.7\"\n - \"3.2\"\n - \"3.3\"\n - \"3.4\"\n - \"3.5\"\n - \"3.5-dev\" # 3.5 development branch\n - \"3.6\"\n - \"3.6-dev\" # 3.6 development branch\n - \"3.7-dev\" # 3.7 development branch\n - \"nightly\"\n")), - IsBinary: false, + PathB: ".travis.yml", + LineStart: 1, + contentWriter: newBufferWithContent([]byte("language: python\npython:\n - \"2.6\"\n - \"2.7\"\n - \"3.2\"\n - \"3.3\"\n - \"3.4\"\n - \"3.5\"\n - \"3.5-dev\" # 3.5 development branch\n - \"3.6\"\n - \"3.6-dev\" # 3.6 development branch\n - \"3.7-dev\" # 3.7 development branch\n - \"nightly\"\n")), + IsBinary: false, }, }, }, @@ -846,10 +855,10 @@ func TestCommitParseFailureRecovery(t *testing.T) { Message: newStringBuilderValue("Change file\n"), Diffs: []Diff{ { - PathB: "tzu", - LineStart: 11, - Content: *bytes.NewBuffer([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")), - IsBinary: false, + PathB: "tzu", + LineStart: 11, + contentWriter: newBufferWithContent([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")), + IsBinary: false, }, }, }, @@ -868,7 +877,7 @@ func TestCommitParseFailureRecovery(t *testing.T) { break } - if !commit.Equal(&expected[i]) { + if !commit.Equal(context.Background(), &expected[i]) { t.Errorf("Commit does not match.\nexpected: %+v\n\nactual : %+v\n", expected[i], commit) } i++ @@ -954,22 +963,22 @@ func TestDiffParseFailureRecovery(t *testing.T) { Message: strings.Builder{}, Diffs: []Diff{ { - PathB: "aws", - LineStart: 1, - Content: *bytes.NewBuffer([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")), - IsBinary: false, + PathB: "aws", + LineStart: 1, + contentWriter: newBufferWithContent([]byte("[default]\naws_access_key_id = AKIAXYZDQCEN4B6JSJQI\naws_secret_access_key = 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\noutput = json\nregion = us-east-2\n")), + IsBinary: false, }, { - PathB: "tzu", - LineStart: 11, - Content: *bytes.NewBuffer([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")), - IsBinary: false, + PathB: "tzu", + LineStart: 11, + contentWriter: newBufferWithContent([]byte("\n\n\n\nSource: https://www.gnu.org/software/diffutils/manual/diffutils.html#An-Example-of-Unified-Format\n")), + IsBinary: false, }, { - PathB: "tzu", - LineStart: 1, - Content: *bytes.NewBuffer([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n so we may see their subtlety,\nAnd let there always be being,\n so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")), - IsBinary: false, + PathB: "tzu", + LineStart: 1, + contentWriter: newBufferWithContent([]byte("The Nameless is the origin of Heaven and Earth;\nThe named is the mother of all things.\n\nTherefore let there always be non-being,\n so we may see their subtlety,\nAnd let there always be being,\n so we may see their outcome.\nThe two are the same,\nBut after they are produced,\n they have different names.\nThey both may be called deep and profound.\nDeeper and more profound,\nThe door of all subtleties!\n")), + IsBinary: false, }, }, }, @@ -988,7 +997,7 @@ func TestDiffParseFailureRecovery(t *testing.T) { break } - if !commit.Equal(&expected[i]) { + if !commit.Equal(context.Background(), &expected[i]) { t.Errorf("Commit does not match.\nexpected: %+v\n\nactual : %+v\n", expected[i], commit) } i++ @@ -1103,8 +1112,8 @@ func TestMaxDiffSize(t *testing.T) { select { case commit := <-commitChan: - if commit.Diffs[0].Content.Len() > parser.maxDiffSize+1024 { - t.Errorf("diff did not match MaxDiffSize. Got: %d, expected (max): %d", commit.Diffs[0].Content.Len(), parser.maxDiffSize+1024) + if commit.Diffs[0].Len() > parser.maxDiffSize+1024 { + t.Errorf("diff did not match MaxDiffSize. Got: %d, expected (max): %d", commit.Diffs[0].Len(), parser.maxDiffSize+1024) } case <-ctx.Done(): t.Fatal("Test timed out") @@ -1568,22 +1577,22 @@ func expectedCommits() []Commit { Message: newStringBuilderValue("Added Unusable coloring\n"), Diffs: []Diff{ { - PathB: "components/item.lua", - LineStart: 9, - Content: *bytes.NewBuffer([]byte("\n\nlocal Unfit = LibStub('Unfit-1.0')\n\n\n")), - IsBinary: false, + PathB: "components/item.lua", + LineStart: 9, + contentWriter: newBufferWithContent([]byte("\n\nlocal Unfit = LibStub('Unfit-1.0')\n\n\n")), + IsBinary: false, }, { - PathB: "embeds.xml", - LineStart: 6, - Content: *bytes.NewBuffer([]byte("\n\n