[feat] - Remove go-git dependency (#2174)

* remove use of go-git for binary files

* fix it

* use limit reader

* fix comment

* fix test

* address comments

* address comments

* address comments

pkg/engine/git_test.go

@@ -107,11 +107,6 @@ func TestGitEngine(t *testing.T) {
 func BenchmarkGitEngine(b *testing.B) {
 	ctx := context.Background()
 	repoUrl := "https://github.com/dustin-decker/secretsandstuff.git"
-	path, _, err := git.PrepareRepo(ctx, repoUrl)
-	if err != nil {
-		b.Error(err)
-	}
-	defer os.RemoveAll(path)
 
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -132,11 +127,9 @@ func BenchmarkGitEngine(b *testing.B) {
 		}
 	}()
 
+	cfg := sources.GitConfig{URI: repoUrl}
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
-		// TODO: this is measuring the time it takes to initialize the source
-		// and not to do the full scan
-		cfg := sources.GitConfig{URI: path}
 		if err := e.ScanGit(ctx, cfg); err != nil {
 			return
 		}

pkg/sources/git/git.go

@@ -5,6 +5,7 @@ import (
 	"bytes"
 	"errors"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
 	"os/exec"
@@ -456,6 +457,8 @@ func (s *Git) CommitsScanned() uint64 {
 	return atomic.LoadUint64(&s.metrics.commitsScanned)
 }
 
+const gitDirName = ".git"
+
 func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, reporter sources.ChunkReporter) error {
 	if err := GitCmdCheck(); err != nil {
 		return err
@@ -474,6 +477,8 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 
 	var depth int64
 
+	gitDir := filepath.Join(path, gitDirName)
+
 	logger := ctx.Logger().WithValues("repo", urlMetadata)
 	logger.V(1).Info("scanning repo", "base", scanOptions.BaseHash, "head", scanOptions.HeadHash)
 	for commit := range commitChan {
@@ -516,7 +521,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 					SourceMetadata: metadata,
 					Verify:         s.verify,
 				}
-				if err := handleBinary(ctx, repo, reporter, chunkSkel, commitHash, fileName); err != nil {
+				if err := handleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName); err != nil {
 					logger.V(1).Info("error handling binary file", "error", err, "filename", fileName, "commit", commitHash, "file", diff.PathB)
 				}
 				continue
@@ -632,6 +637,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 
 	var depth int64
 	reachedBase := false
+	gitDir := filepath.Join(path, gitDirName)
 
 	ctx.Logger().V(1).Info("scanning staged changes", "path", path)
 	for commit := range commitChan {
@@ -679,7 +685,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,
 					SourceMetadata: metadata,
 					Verify:         s.verify,
 				}
-				if err := handleBinary(ctx, repo, reporter, chunkSkel, commitHash, fileName); err != nil {
+				if err := handleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName); err != nil {
 					logger.V(1).Info("error handling binary file", "error", err, "filename", fileName)
 				}
 				continue
@@ -964,30 +970,60 @@ func getSafeRemoteURL(repo *git.Repository, preferred string) string {
 	return safeURL
 }
 
-func handleBinary(ctx context.Context, repo *git.Repository, reporter sources.ChunkReporter, chunkSkel *sources.Chunk, commitHash plumbing.Hash, path string) error {
+func handleBinary(ctx context.Context, gitDir string, reporter sources.ChunkReporter, chunkSkel *sources.Chunk, commitHash plumbing.Hash, path string) error {
 	ctx.Logger().V(5).Info("handling binary file", "path", path)
-	commit, err := repo.CommitObject(commitHash)
-	if err != nil {
-		return err
-	}
 
-	file, err := commit.File(path)
-	if err != nil {
-		return err
-	}
-
-	if common.SkipFile(file.Name) {
-		ctx.Logger().V(5).Info("skipping binary file", "path", path, "file_name", file.Name)
+	if common.SkipFile(path) {
+		ctx.Logger().V(5).Info("skipping binary file", "path", path)
 		return nil
 	}
 
-	fileReader, err := file.Reader()
+	const maxSize = 1 * 1024 * 1024 * 1024 // 1GB
+	cmd := exec.Command("git", "-C", gitDir, "cat-file", "blob", commitHash.String()+":"+path)
+
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+
+	fileReader, err := cmd.StdoutPipe()
 	if err != nil {
 		return err
 	}
-	defer fileReader.Close()
 
-	reader, err := diskbufferreader.New(fileReader)
+	if err := cmd.Start(); err != nil {
+		return err
+	}
+	defer func() {
+		if err := fileReader.Close(); err != nil {
+			ctx.Logger().Error(err, "error closing fileReader")
+		}
+		if err := cmd.Wait(); err != nil {
+			ctx.Logger().Error(
+				err, "error waiting for command",
+				"command", cmd.String(),
+				"stderr", stderr.String(),
+				"commit", commitHash,
+			)
+		}
+	}()
+
+	var fileContent bytes.Buffer
+	// Create a limited reader to ensure we don't read more than the max size.
+	lr := io.LimitReader(fileReader, int64(maxSize))
+
+	// Using io.CopyBuffer for performance advantages. Though buf is mandatory
+	// for the method, due to the internal implementation of io.CopyBuffer, when
+	// *bytes.Buffer implements io.WriterTo or io.ReaderFrom, the provided buf
+	// is simply ignored. Thus, we can pass nil for the buf parameter.
+	_, err = io.CopyBuffer(&fileContent, lr, nil)
+	if err != nil && !errors.Is(err, io.EOF) {
+		return err
+	}
+
+	if fileContent.Len() == maxSize {
+		ctx.Logger().V(2).Info("Max archive size reached.", "path", path)
+	}
+
+	reader, err := diskbufferreader.New(&fileContent)
 	if err != nil {
 		return err
 	}