Testing

* Adjust types to use DetectorID struct

* Parse versions with detector include and exclude input

* Update detectors filter to use version

Co-authored-by: steeeve <steve@trufflesec.com>

* Implement Versioner for github, gitlab, and npm detectors

Co-authored-by: steeeve <steve@trufflesec.com>

---------

Co-authored-by: steeeve <steve@trufflesec.com>

.github/workflows/release.yml

@@ -1,16 +1,9 @@
 name: Release
 
-# on:
-#   push:
-#     tags:
-#       - 'v*'
-
 on:
   push:
     tags:
       - v*
-    branches:
-      - main
 
 permissions:
   contents: write

.gitignore

@@ -2,3 +2,6 @@
 dist
 .env
 *.test
+
+# binary
+trufflehog

go.mod

@@ -120,6 +120,7 @@ require (
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.23.0 // indirect
+	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
 	github.com/pierrec/lz4/v4 v4.1.14 // indirect
 	github.com/pjbgf/sha1cd v0.2.3 // indirect
 	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect

go.sum

@@ -281,6 +281,8 @@ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=

main.go

@@ -216,30 +216,52 @@ func run(state overseer.State) {
 	}
 
 	// Build include and exclude detector filter sets.
-	var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]struct{}
+	var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID
 	{
 		includeList, err := config.ParseDetectors(*includeDetectors)
 		if err != nil {
 			// Exit if there was an error to inform the user of the misconfiguration.
-			logger.Error(err, "invalid include list detector configuration")
+			logFatal(err, "invalid include list detector configuration")
-			os.Exit(1)
 		}
 		excludeList, err := config.ParseDetectors(*excludeDetectors)
 		if err != nil {
 			// Exit if there was an error to inform the user of the misconfiguration.
-			logger.Error(err, "invalid exclude list detector configuration")
+			logFatal(err, "invalid exclude list detector configuration")
-			os.Exit(1)
 		}
-		includeDetectorTypes = detectorTypeToSet(includeList)
+		includeDetectorTypes = detectorTypeToMap(includeList)
-		excludeDetectorTypes = detectorTypeToSet(excludeList)
+		excludeDetectorTypes = detectorTypeToMap(excludeList)
 	}
 	includeFilter := func(d detectors.Detector) bool {
-		_, ok := includeDetectorTypes[d.Type()]
+		id, ok := includeDetectorTypes[d.Type()]
-		return ok
+		if id.Version == 0 {
+			return ok
+		}
+		versionD, ok := d.(detectors.Versioner)
+		if !ok {
+			// Error: version provided but not a detectors.Versioner
+			logFatal(
+				fmt.Errorf("version provided but detector does not have a version"),
+				"invalid include list detector configuration",
+				"detector", id,
+			)
+		}
+		return versionD.Version() == id.Version
 	}
 	excludeFilter := func(d detectors.Detector) bool {
-		_, ok := excludeDetectorTypes[d.Type()]
+		id, ok := excludeDetectorTypes[d.Type()]
-		return !ok
+		if id.Version == 0 {
+			return !ok
+		}
+		versionD, ok := d.(detectors.Versioner)
+		if !ok {
+			// Error: version provided but not a detectors.Versioner
+			logFatal(
+				fmt.Errorf("version provided but detector does not have a version"),
+				"invalid exclude list detector configuration",
+				"detector", id,
+			)
+		}
+		return versionD.Version() != id.Version
 	}
 
 	e := engine.Start(ctx,
@@ -431,10 +453,10 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
 	}
 }
 
-func detectorTypeToSet(detectors []detectorspb.DetectorType) map[detectorspb.DetectorType]struct{} {
+func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {
-	output := make(map[detectorspb.DetectorType]struct{}, len(detectors))
+	output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))
 	for _, d := range detectors {
-		output[d] = struct{}{}
+		output[d.ID] = d
 	}
 	return output
 }

pkg/config/detectors.go

@@ -2,15 +2,15 @@ package config
 
 import (
 	"fmt"
+	"sort"
 	"strconv"
 	"strings"
 
 	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-	"golang.org/x/exp/slices"
 )
 
 var (
-	specialGroups = map[string][]dpb.DetectorType{
+	specialGroups = map[string][]DetectorID{
 		"all": allDetectors(),
 	}
 
@@ -31,14 +31,19 @@ func init() {
 	}
 }
 
+type DetectorID struct {
+	ID      dpb.DetectorType
+	Version int
+}
+
 // ParseDetectors parses user supplied string into a list of detectors types.
 // "all" will return the list of all available detectors. The input is comma
 // separated and may use the case-insensitive detector name defined in the
 // protobuf, or the protobuf enum number. A range may be used as well in the
 // form "start-end". Order is preserved and duplicates are ignored.
-func ParseDetectors(input string) ([]dpb.DetectorType, error) {
+func ParseDetectors(input string) ([]DetectorID, error) {
-	var output []dpb.DetectorType
+	var output []DetectorID
-	seenDetector := map[dpb.DetectorType]struct{}{}
+	seenDetector := map[DetectorID]struct{}{}
 	for _, item := range strings.Split(input, ",") {
 		item = strings.TrimSpace(item)
 		if item == "" {
@@ -63,24 +68,35 @@ func ParseDetectors(input string) ([]dpb.DetectorType, error) {
 	return output, nil
 }
 
-// allDetectors reutrns an ordered slice of all detector types.
+func (id DetectorID) String() string {
-func allDetectors() []dpb.DetectorType {
+	name := dpb.DetectorType_name[int32(id.ID)]
-	all := make([]dpb.DetectorType, 0, len(dpb.DetectorType_name))
+	if name == "" {
-	for id := range dpb.DetectorType_name {
+		name = "<invalid ID>"
-		all = append(all, dpb.DetectorType(id))
 	}
-	slices.Sort(all)
+	if id.Version == 0 {
+		return name
+	}
+	return fmt.Sprintf("%s.v%d", name, id.Version)
+}
+
+// allDetectors reutrns an ordered slice of all detector types.
+func allDetectors() []DetectorID {
+	all := make([]DetectorID, 0, len(dpb.DetectorType_name))
+	for id := range dpb.DetectorType_name {
+		all = append(all, DetectorID{ID: dpb.DetectorType(id)})
+	}
+	sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
 	return all
 }
 
 // asRange converts a single input into a slice of detector types. If the input
 // is not in range format, a slice of length 1 is returned. Unbounded ranges
 // are allowed.
-func asRange(input string) ([]dpb.DetectorType, error) {
+func asRange(input string) ([]DetectorID, error) {
 	// Check if it's a single detector type.
-	dt, err := asDetectorType(input)
+	dt, err := asDetectorID(input)
 	if err == nil {
-		return []dpb.DetectorType{dt}, nil
+		return []DetectorID{dt}, nil
 	}
 
 	// Check if it's a range; if not return the error from above.
@@ -91,50 +107,77 @@ func asRange(input string) ([]dpb.DetectorType, error) {
 	start, end = strings.TrimSpace(start), strings.TrimSpace(end)
 
 	// Convert the range start and end to a DetectorType.
-	dtStart, err := asDetectorType(start)
+	dtStart, err := asDetectorID(start)
 	if err != nil {
 		return nil, err
 	}
-	dtEnd, err := asDetectorType(end)
+	dtEnd, err := asDetectorID(end)
 	// If end is empty it's an unbounded range.
 	if err != nil && end != "" {
 		return nil, err
 	}
 	if end == "" {
-		dtEnd = maxDetectorType
+		dtEnd.ID = maxDetectorType
+	}
+
+	// Ensure these ranges don't have versions.
+	if dtEnd.Version != 0 || dtStart.Version != 0 {
+		return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
 	}
 
 	step := dpb.DetectorType(1)
-	if dtStart > dtEnd {
+	if dtStart.ID > dtEnd.ID {
 		step = -1
 	}
-	var output []dpb.DetectorType
+	var output []DetectorID
-	for dt := dtStart; dt != dtEnd; dt += step {
+	for dt := dtStart.ID; dt != dtEnd.ID; dt += step {
 		if _, ok := validDetectors[dt]; !ok {
 			continue
 		}
-		output = append(output, dt)
+		output = append(output, DetectorID{ID: dt})
 	}
 	return append(output, dtEnd), nil
 }
 
-// asDetectorType converts the case-insensitive input into a detector type.
+// asDetectorID converts the case-insensitive input into a DetectorID.
 // Name or ID may be used.
-func asDetectorType(input string) (dpb.DetectorType, error) {
+func asDetectorID(input string) (DetectorID, error) {
 	if input == "" {
-		return 0, fmt.Errorf("empty detector")
+		return DetectorID{}, fmt.Errorf("empty detector")
 	}
+	var detectorID DetectorID
+	// Separate the version if there is one.
+	if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {
+		parsedVersion, err := parseVersion(version)
+		if err != nil {
+			return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)
+		}
+		detectorID.Version = parsedVersion
+		// Because there was a version, the detector type input is the part before the '.'
+		input = detector
+	}
+
 	// Check if it's a named detector.
 	if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
-		return dt, nil
+		detectorID.ID = dt
+		return detectorID, nil
 	}
 	// Check if it's a detector ID.
 	if i, err := strconv.ParseInt(input, 10, 32); err == nil {
 		dt := dpb.DetectorType(i)
 		if _, ok := validDetectors[dt]; !ok {
-			return 0, fmt.Errorf("invalid detector ID: %s", input)
+			return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
 		}
-		return dt, nil
+		detectorID.ID = dt
+		return detectorID, nil
 	}
-	return 0, fmt.Errorf("unrecognized detector type: %s", input)
+	return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)
+}
+
+func parseVersion(v string) (int, error) {
+	if !strings.HasPrefix(strings.ToLower(v), "v") {
+		return 0, fmt.Errorf("version must start with 'v'")
+	}
+	version := strings.TrimLeft(v, "vV")
+	return strconv.Atoi(version)
 }

pkg/config/detectors_test.go

@@ -10,21 +10,28 @@ import (
 func TestDetectorParsing(t *testing.T) {
 	tests := map[string]struct {
 		input    string
-		expected []dpb.DetectorType
+		expected []DetectorID
 	}{
 		"all":                       {"AlL", allDetectors()},
 		"trailing range":            {"0-", allDetectors()},
 		"all after 1":               {"1-", allDetectors()[1:]},
-		"named and valid range":     {"aWs,8-9", []dpb.DetectorType{dpb.DetectorType_AWS, dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
+		"named and valid range":     {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
-		"duplicate order preserved": {"9, 8, 9", []dpb.DetectorType{9, 8}},
+		"duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
-		"named range":               {"github - gitlab", []dpb.DetectorType{dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
+		"named range":               {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
-		"range preserved":           {"8-9, 7-10", []dpb.DetectorType{8, 9, 7, 10}},
+		"range preserved":           {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
-		"reverse range":             {"9-8", []dpb.DetectorType{9, 8}},
+		"reverse range":             {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
 		"range preserved with all":  {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
-		"empty list item":           {"8, ,9", []dpb.DetectorType{8, 9}},
+		"empty list item":           {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
 		"invalid end range":         {"0-1337", nil},
 		"invalid name":              {"foo", nil},
 		"negative":                  {"-1", nil},
+		"github.v1":                 {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
+		"gitlab.v100":               {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
+		"range with versions":       {"github.v2 - gitlab.v1", nil},
+		"invalid version no v":      {"gitlab.2", nil},
+		"invalid version no number": {"gitlab.github", nil},
+		"capital V is fine":         {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
+		"id number with version":    {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
 	}
 
 	for name, tt := range tests {

pkg/detectors/detectors.go

@@ -25,6 +25,12 @@ type Detector interface {
 	Type() detectorspb.DetectorType
 }
 
+// Versioner is an optional interface that a detector can implement to
+// differentiate instances of the same detector type.
+type Versioner interface {
+	Version() int
+}
+
 type Result struct {
 	// DetectorType is the type of Detector.
 	DetectorType detectorspb.DetectorType

pkg/detectors/github/github.go

@@ -15,8 +15,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 2 }
 
 var (
 	// Oauth token

pkg/detectors/github_old/github_old.go

@@ -14,8 +14,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 1 }
 
 var (
 	// Oauth token
@@ -38,7 +41,7 @@ type userRes struct {
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"github","gh","pat"}
+	return []string{"github", "gh", "pat"}
 }
 
 // FromData will find and optionally verify GitHub secrets in a given set of bytes.

pkg/detectors/gitlab/gitlab.go

@@ -14,8 +14,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 1 }
 
 var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)

pkg/detectors/gitlabv2/gitlab.go

@@ -13,8 +13,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (*Scanner) Version() int { return 2 }
 
 var (
 	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)

pkg/detectors/npmtoken/npmtoken.go

@@ -14,8 +14,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 1 }
 
 var (
 	client = common.SaneHttpClient()
@@ -34,7 +37,7 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
 
-  	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue

pkg/detectors/npmtokenv2/npmtokenv2.go

@@ -13,8 +13,11 @@ import (
 
 type Scanner struct{}
 
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (s Scanner) Version() int { return 2 }
 
 var (
 	client = common.SaneHttpClient()

pkg/engine/engine.go

@@ -9,6 +9,7 @@ import (
 	"sync/atomic"
 	"time"
 
+	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
 	"google.golang.org/protobuf/proto"
 
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -36,6 +37,10 @@ type Engine struct {
 	// If there are multiple unverified results for the same chunk for the same detector,
 	// only the first one will be kept.
 	filterUnverified bool
+
+	// prefilter is a ahocorasick struct used for doing efficient string
+	// matching given a set of words (keywords from the rules in the config)
+	prefilter ahocorasick.AhoCorasick
 }
 
 type EngineOption func(*Engine)
@@ -110,7 +115,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 
 	// Set defaults.
-
 	if e.concurrency == 0 {
 		numCPU := runtime.NumCPU()
 		ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
@@ -128,6 +132,23 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		e.detectors[false] = []detectors.Detector{}
 	}
 
+	// build ahocorasick prefilter for efficient string matching
+	// on keywords
+	keywords := []string{}
+	for _, d := range e.detectors[false] {
+		keywords = append(keywords, d.Keywords()...)
+	}
+	for _, d := range e.detectors[true] {
+		keywords = append(keywords, d.Keywords()...)
+	}
+	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
+		AsciiCaseInsensitive: true,
+		MatchOnlyWholeWords:  false,
+		MatchKind:            ahocorasick.LeftMostLongestMatch,
+		DFA:                  true,
+	})
+	e.prefilter = builder.Build(keywords)
+
 	ctx.Logger().V(2).Info("loaded decoders", "count", len(e.decoders))
 	ctx.Logger().V(2).Info("loaded detectors",
 		"total", len(e.detectors[true])+len(e.detectors[false]),
@@ -208,6 +229,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 func (e *Engine) detectorWorker(ctx context.Context) {
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
+			matchedKeywords := make(map[string]struct{})
 			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType
@@ -224,21 +246,29 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 				if decoded == nil {
 					continue
 				}
+
 				dataLower := strings.ToLower(string(decoded.Data))
+				matches := e.prefilter.FindAll(dataLower)
+
+				for _, m := range matches {
+					matchedKeywords[dataLower[m.Start():m.End()]] = struct{}{}
+				}
+
 				for verify, detectorsSet := range e.detectors {
 					for _, detector := range detectorsSet {
-						start := time.Now()
+						chunkContainsKeyword := false
-						foundKeyword := false
 						for _, kw := range detector.Keywords() {
-							if strings.Contains(dataLower, strings.ToLower(kw)) {
+							if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
-								foundKeyword = true
+								chunkContainsKeyword = true
-								break
 							}
 						}
-						if !foundKeyword {
+
+						if !chunkContainsKeyword {
 							continue
 						}
 
+						start := time.Now()
+
 						results, err := func() ([]detectors.Result, error) {
 							ctx, cancel := context.WithTimeout(ctx, time.Second*10)
 							defer cancel()

pkg/gitparse/gitparse.go

@@ -117,10 +117,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
 
 // RepoPath parses the output of the `git log` command for the `source` path.
 func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
-	args := []string{"-C", source, "log", "-p", "-U5", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
+	args := []string{"-C", source, "log", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
-	if abbreviatedLog {
-		args = append(args, "--diff-filter=AM")
-	}
 	if head != "" {
 		args = append(args, head)
 	} else {
@@ -134,12 +131,13 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd)
+	return c.executeCommand(ctx, source, cmd)
 }
 
 // Unstaged parses the output of the `git diff` command for the `source` path.
 func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, error) {
-	args := []string{"-C", source, "diff", "-p", "-U5", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
+	// Provide the --cached flag to diff to get the diff of the staged changes.
+	args := []string{"-C", source, "diff", "-p", "-U5", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
 
 	cmd := exec.Command("git", args...)
 
@@ -148,11 +146,11 @@ func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, erro
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
 
-	return c.executeCommand(ctx, cmd)
+	return c.executeCommand(ctx, source, cmd)
 }
 
 // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
-func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
+func (c *Parser) executeCommand(ctx context.Context, source string, cmd *exec.Cmd) (chan Commit, error) {
 	commitChan := make(chan Commit, 64)
 
 	stdOut, err := cmd.StdoutPipe()
@@ -177,7 +175,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
 	}()
 
 	go func() {
-		c.fromReader(ctx, stdOut, commitChan)
+		c.fromReader(ctx, source, stdOut, commitChan)
 		if err := cmd.Wait(); err != nil {
 			ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
 		}
@@ -186,11 +184,12 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
 	return commitChan, nil
 }
 
-func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
+func (c *Parser) fromReader(ctx context.Context, source string, stdOut io.Reader, commitChan chan Commit) {
 	outReader := bufio.NewReader(stdOut)
 	var currentCommit *Commit
 	var currentDiff *Diff
 
+	lastCommit := ""
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
 	for {
@@ -201,6 +200,143 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		if err != nil && len(line) == 0 {
 			break
 		}
+		// for i := 0; i < len(line); i++ {
+		// 	b := make([]byte, 1)
+		// 	n, err := stdOut.Read(b)
+		// 	if errors.Is(err, io.EOF) || n == 0 {
+		// 		return
+		// 	}
+		// 	line[i] = b[0]
+		// 	if b[0] == byte('\n') {
+		// 		line = line[:0]
+		// 		break
+		// 	}
+		// }
+
+		switch {
+		case isCommitLine(line):
+			// If there is a currentDiff, add it to currentCommit.
+			if currentDiff != nil && currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+			}
+			// If there is a currentCommit, send it to the channel.
+			if currentCommit != nil {
+				args := []string{"-C", source, "diff", "--date=format:%a %b %d %H:%M:%S %Y %z", currentCommit.Hash}
+				if lastCommit != "" {
+					args = append(args, lastCommit)
+				}
+				lastCommit = currentCommit.Hash
+
+				cmd := exec.Command("git", args...)
+
+				absPath, err := filepath.Abs(source)
+				if err == nil {
+					cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
+				}
+
+				// stdnOut, err := cmd.StdoutPipe()
+				// if err != nil {
+				// 	return
+				// }
+				err = cmd.Start()
+				if err != nil {
+					return
+				}
+
+				_ = cmd.Wait()
+				//c.funkReader(ctx, stdnOut, commitChan, currentCommit)
+			}
+			// Create a new currentDiff and currentCommit
+			currentDiff = &Diff{}
+			currentCommit = &Commit{
+				Message: strings.Builder{},
+			}
+			// Check that the commit line contains a hash and set it.
+			if len(line) >= 47 {
+				currentCommit.Hash = string(line[7:47])
+			}
+		case isAuthorLine(line):
+			currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
+		case isDateLine(line):
+			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
+			if err != nil {
+				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
+			}
+			currentCommit.Date = date
+		case isModeLine(line):
+			// NoOp
+		case isIndexLine(line):
+			// NoOp
+		case isPlusFileLine(line):
+			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
+		case isMinusFileLine(line):
+			// NoOp
+		case isPlusDiffLine(line):
+			currentDiff.Content.Write(line[1:])
+		case isMinusDiffLine(line):
+			// NoOp. We only care about additions.
+		case isMessageLine(line):
+			currentCommit.Message.Write(line[4:])
+		case isContextDiffLine(line):
+			currentDiff.Content.Write([]byte("\n"))
+		case isBinaryLine(line):
+			currentDiff.IsBinary = true
+			currentDiff.PathB = pathFromBinaryLine(line)
+		case isLineNumberDiffLine(line):
+			if currentDiff != nil && currentDiff.Content.Len() > 0 {
+				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
+			}
+			newDiff := &Diff{
+				PathB: currentDiff.PathB,
+			}
+
+			currentDiff = newDiff
+
+			words := bytes.Split(line, []byte(" "))
+			if len(words) >= 3 {
+				startSlice := bytes.Split(words[2], []byte(","))
+				lineStart, err := strconv.Atoi(string(startSlice[0]))
+				if err == nil {
+					currentDiff.LineStart = lineStart
+				}
+			}
+		}
+		if currentDiff.Content.Len() > c.maxDiffSize {
+			ctx.Logger().V(2).Info(fmt.Sprintf(
+				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
+			))
+			break
+		}
+	}
+	cleanupParse(currentCommit, currentDiff, commitChan)
+}
+
+func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, currentCommit *Commit) {
+	outReader := bufio.NewReader(stdOut)
+	var currentDiff *Diff
+
+	defer common.RecoverWithExit(ctx)
+	for {
+		if common.IsDone(ctx) {
+			break
+		}
+		line, err := outReader.ReadBytes([]byte("\n")[0])
+		if err != nil && len(line) == 0 {
+			break
+		}
+		// for i := 0; i < len(line); i++ {
+		// 	b := make([]byte, 1)
+		// 	n, err := stdOut.Read(b)
+		// 	if errors.Is(err, io.EOF) || n == 0 {
+		// 		return
+		// 	}
+		// 	line[i] = b[0]
+		// 	if b[0] == byte('\n') {
+		// 		line = line[:0]
+		// 		break
+		// 	}
+		// }
+
 		switch {
 		case isCommitLine(line):
 			// If there is a currentDiff, add it to currentCommit.

pkg/sources/git/git.go

@@ -472,9 +472,9 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 	}
 }
 
-// ScanUnstaged chunks unstaged changes.
+// ScanStaged chunks staged changes.
-func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
-	// get the URL metadata for reporting (may be empty)
+	// Get the URL metadata for reporting (may be empty).
 	urlMetadata := getSafeRemoteURL(repo, "origin")
 
 	commitChan, err := gitparse.NewParser().Unstaged(ctx, path)
@@ -488,11 +488,11 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 	var depth int64
 	var reachedBase = false
 
-	ctx.Logger().V(1).Info("scanning unstaged changes", "path", path)
+	ctx.Logger().V(1).Info("scanning staged changes", "path", path)
 	for commit := range commitChan {
 		for _, diff := range commit.Diffs {
 			logger := ctx.Logger().WithValues("filename", diff.PathB, "commit", commit.Hash, "file", diff.PathB)
-			logger.V(2).Info("scanning unstaged changes from git")
+			logger.V(2).Info("scanning staged changes from git")
 
 			if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
 				logger.V(1).Info("reached max depth")
@@ -525,7 +525,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 			// Handle binary files by reading the entire file rather than using the diff.
 			if diff.IsBinary {
 				commitHash := plumbing.NewHash(hash)
-				metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, 0)
+				metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, 0)
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
@@ -539,7 +539,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 				continue
 			}
 
-			metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, int64(diff.LineStart))
+			metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, int64(diff.LineStart))
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
@@ -564,7 +564,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
 	if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		return err
 	}
-	if err := s.ScanUnstaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
+	if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
 	}