Compare commits

..

2 Commits

Author SHA1 Message Date
bill-rich 1bbc35e5be Testing 2023-03-07 08:39:18 -08:00
Miccah e6846ede54 Support filtering detectors by version (#1150)
* Adjust types to use DetectorID struct

* Parse versions with detector include and exclude input

* Update detectors filter to use version

Co-authored-by: steeeve <steve@trufflesec.com>

* Implement Versioner for github, gitlab, and npm detectors

Co-authored-by: steeeve <steve@trufflesec.com>

---------

Co-authored-by: steeeve <steve@trufflesec.com>
2023-03-02 16:33:56 -06:00
11 changed files with 297 additions and 66 deletions
+34 -12
View File
@@ -216,31 +216,53 @@ func run(state overseer.State) {
} }
// Build include and exclude detector filter sets. // Build include and exclude detector filter sets.
var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]struct{} var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID
{ {
includeList, err := config.ParseDetectors(*includeDetectors) includeList, err := config.ParseDetectors(*includeDetectors)
if err != nil { if err != nil {
// Exit if there was an error to inform the user of the misconfiguration. // Exit if there was an error to inform the user of the misconfiguration.
logger.Error(err, "invalid include list detector configuration") logFatal(err, "invalid include list detector configuration")
os.Exit(1)
} }
excludeList, err := config.ParseDetectors(*excludeDetectors) excludeList, err := config.ParseDetectors(*excludeDetectors)
if err != nil { if err != nil {
// Exit if there was an error to inform the user of the misconfiguration. // Exit if there was an error to inform the user of the misconfiguration.
logger.Error(err, "invalid exclude list detector configuration") logFatal(err, "invalid exclude list detector configuration")
os.Exit(1)
} }
includeDetectorTypes = detectorTypeToSet(includeList) includeDetectorTypes = detectorTypeToMap(includeList)
excludeDetectorTypes = detectorTypeToSet(excludeList) excludeDetectorTypes = detectorTypeToMap(excludeList)
} }
includeFilter := func(d detectors.Detector) bool { includeFilter := func(d detectors.Detector) bool {
_, ok := includeDetectorTypes[d.Type()] id, ok := includeDetectorTypes[d.Type()]
if id.Version == 0 {
return ok return ok
} }
versionD, ok := d.(detectors.Versioner)
if !ok {
// Error: version provided but not a detectors.Versioner
logFatal(
fmt.Errorf("version provided but detector does not have a version"),
"invalid include list detector configuration",
"detector", id,
)
}
return versionD.Version() == id.Version
}
excludeFilter := func(d detectors.Detector) bool { excludeFilter := func(d detectors.Detector) bool {
_, ok := excludeDetectorTypes[d.Type()] id, ok := excludeDetectorTypes[d.Type()]
if id.Version == 0 {
return !ok return !ok
} }
versionD, ok := d.(detectors.Versioner)
if !ok {
// Error: version provided but not a detectors.Versioner
logFatal(
fmt.Errorf("version provided but detector does not have a version"),
"invalid exclude list detector configuration",
"detector", id,
)
}
return versionD.Version() != id.Version
}
e := engine.Start(ctx, e := engine.Start(ctx,
engine.WithConcurrency(*concurrency), engine.WithConcurrency(*concurrency),
@@ -431,10 +453,10 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
} }
} }
func detectorTypeToSet(detectors []detectorspb.DetectorType) map[detectorspb.DetectorType]struct{} { func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {
output := make(map[detectorspb.DetectorType]struct{}, len(detectors)) output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))
for _, d := range detectors { for _, d := range detectors {
output[d] = struct{}{} output[d.ID] = d
} }
return output return output
} }
+71 -28
View File
@@ -2,15 +2,15 @@ package config
import ( import (
"fmt" "fmt"
"sort"
"strconv" "strconv"
"strings" "strings"
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb" dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"golang.org/x/exp/slices"
) )
var ( var (
specialGroups = map[string][]dpb.DetectorType{ specialGroups = map[string][]DetectorID{
"all": allDetectors(), "all": allDetectors(),
} }
@@ -31,14 +31,19 @@ func init() {
} }
} }
type DetectorID struct {
ID dpb.DetectorType
Version int
}
// ParseDetectors parses user supplied string into a list of detectors types. // ParseDetectors parses user supplied string into a list of detectors types.
// "all" will return the list of all available detectors. The input is comma // "all" will return the list of all available detectors. The input is comma
// separated and may use the case-insensitive detector name defined in the // separated and may use the case-insensitive detector name defined in the
// protobuf, or the protobuf enum number. A range may be used as well in the // protobuf, or the protobuf enum number. A range may be used as well in the
// form "start-end". Order is preserved and duplicates are ignored. // form "start-end". Order is preserved and duplicates are ignored.
func ParseDetectors(input string) ([]dpb.DetectorType, error) { func ParseDetectors(input string) ([]DetectorID, error) {
var output []dpb.DetectorType var output []DetectorID
seenDetector := map[dpb.DetectorType]struct{}{} seenDetector := map[DetectorID]struct{}{}
for _, item := range strings.Split(input, ",") { for _, item := range strings.Split(input, ",") {
item = strings.TrimSpace(item) item = strings.TrimSpace(item)
if item == "" { if item == "" {
@@ -63,24 +68,35 @@ func ParseDetectors(input string) ([]dpb.DetectorType, error) {
return output, nil return output, nil
} }
// allDetectors reutrns an ordered slice of all detector types. func (id DetectorID) String() string {
func allDetectors() []dpb.DetectorType { name := dpb.DetectorType_name[int32(id.ID)]
all := make([]dpb.DetectorType, 0, len(dpb.DetectorType_name)) if name == "" {
for id := range dpb.DetectorType_name { name = "<invalid ID>"
all = append(all, dpb.DetectorType(id))
} }
slices.Sort(all) if id.Version == 0 {
return name
}
return fmt.Sprintf("%s.v%d", name, id.Version)
}
// allDetectors reutrns an ordered slice of all detector types.
func allDetectors() []DetectorID {
all := make([]DetectorID, 0, len(dpb.DetectorType_name))
for id := range dpb.DetectorType_name {
all = append(all, DetectorID{ID: dpb.DetectorType(id)})
}
sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
return all return all
} }
// asRange converts a single input into a slice of detector types. If the input // asRange converts a single input into a slice of detector types. If the input
// is not in range format, a slice of length 1 is returned. Unbounded ranges // is not in range format, a slice of length 1 is returned. Unbounded ranges
// are allowed. // are allowed.
func asRange(input string) ([]dpb.DetectorType, error) { func asRange(input string) ([]DetectorID, error) {
// Check if it's a single detector type. // Check if it's a single detector type.
dt, err := asDetectorType(input) dt, err := asDetectorID(input)
if err == nil { if err == nil {
return []dpb.DetectorType{dt}, nil return []DetectorID{dt}, nil
} }
// Check if it's a range; if not return the error from above. // Check if it's a range; if not return the error from above.
@@ -91,50 +107,77 @@ func asRange(input string) ([]dpb.DetectorType, error) {
start, end = strings.TrimSpace(start), strings.TrimSpace(end) start, end = strings.TrimSpace(start), strings.TrimSpace(end)
// Convert the range start and end to a DetectorType. // Convert the range start and end to a DetectorType.
dtStart, err := asDetectorType(start) dtStart, err := asDetectorID(start)
if err != nil { if err != nil {
return nil, err return nil, err
} }
dtEnd, err := asDetectorType(end) dtEnd, err := asDetectorID(end)
// If end is empty it's an unbounded range. // If end is empty it's an unbounded range.
if err != nil && end != "" { if err != nil && end != "" {
return nil, err return nil, err
} }
if end == "" { if end == "" {
dtEnd = maxDetectorType dtEnd.ID = maxDetectorType
}
// Ensure these ranges don't have versions.
if dtEnd.Version != 0 || dtStart.Version != 0 {
return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
} }
step := dpb.DetectorType(1) step := dpb.DetectorType(1)
if dtStart > dtEnd { if dtStart.ID > dtEnd.ID {
step = -1 step = -1
} }
var output []dpb.DetectorType var output []DetectorID
for dt := dtStart; dt != dtEnd; dt += step { for dt := dtStart.ID; dt != dtEnd.ID; dt += step {
if _, ok := validDetectors[dt]; !ok { if _, ok := validDetectors[dt]; !ok {
continue continue
} }
output = append(output, dt) output = append(output, DetectorID{ID: dt})
} }
return append(output, dtEnd), nil return append(output, dtEnd), nil
} }
// asDetectorType converts the case-insensitive input into a detector type. // asDetectorID converts the case-insensitive input into a DetectorID.
// Name or ID may be used. // Name or ID may be used.
func asDetectorType(input string) (dpb.DetectorType, error) { func asDetectorID(input string) (DetectorID, error) {
if input == "" { if input == "" {
return 0, fmt.Errorf("empty detector") return DetectorID{}, fmt.Errorf("empty detector")
} }
var detectorID DetectorID
// Separate the version if there is one.
if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {
parsedVersion, err := parseVersion(version)
if err != nil {
return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)
}
detectorID.Version = parsedVersion
// Because there was a version, the detector type input is the part before the '.'
input = detector
}
// Check if it's a named detector. // Check if it's a named detector.
if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok { if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
return dt, nil detectorID.ID = dt
return detectorID, nil
} }
// Check if it's a detector ID. // Check if it's a detector ID.
if i, err := strconv.ParseInt(input, 10, 32); err == nil { if i, err := strconv.ParseInt(input, 10, 32); err == nil {
dt := dpb.DetectorType(i) dt := dpb.DetectorType(i)
if _, ok := validDetectors[dt]; !ok { if _, ok := validDetectors[dt]; !ok {
return 0, fmt.Errorf("invalid detector ID: %s", input) return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
} }
return dt, nil detectorID.ID = dt
return detectorID, nil
} }
return 0, fmt.Errorf("unrecognized detector type: %s", input) return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)
}
func parseVersion(v string) (int, error) {
if !strings.HasPrefix(strings.ToLower(v), "v") {
return 0, fmt.Errorf("version must start with 'v'")
}
version := strings.TrimLeft(v, "vV")
return strconv.Atoi(version)
} }
+14 -7
View File
@@ -10,21 +10,28 @@ import (
func TestDetectorParsing(t *testing.T) { func TestDetectorParsing(t *testing.T) {
tests := map[string]struct { tests := map[string]struct {
input string input string
expected []dpb.DetectorType expected []DetectorID
}{ }{
"all": {"AlL", allDetectors()}, "all": {"AlL", allDetectors()},
"trailing range": {"0-", allDetectors()}, "trailing range": {"0-", allDetectors()},
"all after 1": {"1-", allDetectors()[1:]}, "all after 1": {"1-", allDetectors()[1:]},
"named and valid range": {"aWs,8-9", []dpb.DetectorType{dpb.DetectorType_AWS, dpb.DetectorType_Github, dpb.DetectorType_Gitlab}}, "named and valid range": {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
"duplicate order preserved": {"9, 8, 9", []dpb.DetectorType{9, 8}}, "duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
"named range": {"github - gitlab", []dpb.DetectorType{dpb.DetectorType_Github, dpb.DetectorType_Gitlab}}, "named range": {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
"range preserved": {"8-9, 7-10", []dpb.DetectorType{8, 9, 7, 10}}, "range preserved": {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
"reverse range": {"9-8", []dpb.DetectorType{9, 8}}, "reverse range": {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
"range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)}, "range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
"empty list item": {"8, ,9", []dpb.DetectorType{8, 9}}, "empty list item": {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
"invalid end range": {"0-1337", nil}, "invalid end range": {"0-1337", nil},
"invalid name": {"foo", nil}, "invalid name": {"foo", nil},
"negative": {"-1", nil}, "negative": {"-1", nil},
"github.v1": {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
"gitlab.v100": {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
"range with versions": {"github.v2 - gitlab.v1", nil},
"invalid version no v": {"gitlab.2", nil},
"invalid version no number": {"gitlab.github", nil},
"capital V is fine": {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
"id number with version": {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
} }
for name, tt := range tests { for name, tt := range tests {
+6
View File
@@ -25,6 +25,12 @@ type Detector interface {
Type() detectorspb.DetectorType Type() detectorspb.DetectorType
} }
// Versioner is an optional interface that a detector can implement to
// differentiate instances of the same detector type.
type Versioner interface {
Version() int
}
type Result struct { type Result struct {
// DetectorType is the type of Detector. // DetectorType is the type of Detector.
DetectorType detectorspb.DetectorType DetectorType detectorspb.DetectorType
+4 -1
View File
@@ -15,8 +15,11 @@ import (
type Scanner struct{} type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time. // Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (s Scanner) Version() int { return 2 }
var ( var (
// Oauth token // Oauth token
+4 -1
View File
@@ -14,8 +14,11 @@ import (
type Scanner struct{} type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time. // Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (s Scanner) Version() int { return 1 }
var ( var (
// Oauth token // Oauth token
+4 -1
View File
@@ -14,8 +14,11 @@ import (
type Scanner struct{} type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time. // Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (s Scanner) Version() int { return 1 }
var ( var (
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`) keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
+4 -1
View File
@@ -13,8 +13,11 @@ import (
type Scanner struct{} type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time. // Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (*Scanner) Version() int { return 2 }
var ( var (
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`) keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
+4 -1
View File
@@ -14,8 +14,11 @@ import (
type Scanner struct{} type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time. // Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (s Scanner) Version() int { return 1 }
var ( var (
client = common.SaneHttpClient() client = common.SaneHttpClient()
+4 -1
View File
@@ -13,8 +13,11 @@ import (
type Scanner struct{} type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time. // Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil) var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.Versioner = (*Scanner)(nil)
func (s Scanner) Version() int { return 2 }
var ( var (
client = common.SaneHttpClient() client = common.SaneHttpClient()
+144 -9
View File
@@ -117,10 +117,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
// RepoPath parses the output of the `git log` command for the `source` path. // RepoPath parses the output of the `git log` command for the `source` path.
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) { func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
args := []string{"-C", source, "log", "-p", "-U5", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"} args := []string{"-C", source, "log", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
if abbreviatedLog {
args = append(args, "--diff-filter=AM")
}
if head != "" { if head != "" {
args = append(args, head) args = append(args, head)
} else { } else {
@@ -134,7 +131,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git"))) cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
} }
return c.executeCommand(ctx, cmd) return c.executeCommand(ctx, source, cmd)
} }
// Unstaged parses the output of the `git diff` command for the `source` path. // Unstaged parses the output of the `git diff` command for the `source` path.
@@ -149,11 +146,11 @@ func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, erro
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git"))) cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
} }
return c.executeCommand(ctx, cmd) return c.executeCommand(ctx, source, cmd)
} }
// executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete. // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) { func (c *Parser) executeCommand(ctx context.Context, source string, cmd *exec.Cmd) (chan Commit, error) {
commitChan := make(chan Commit, 64) commitChan := make(chan Commit, 64)
stdOut, err := cmd.StdoutPipe() stdOut, err := cmd.StdoutPipe()
@@ -178,7 +175,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
}() }()
go func() { go func() {
c.fromReader(ctx, stdOut, commitChan) c.fromReader(ctx, source, stdOut, commitChan)
if err := cmd.Wait(); err != nil { if err := cmd.Wait(); err != nil {
ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err) ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
} }
@@ -187,11 +184,12 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
return commitChan, nil return commitChan, nil
} }
func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) { func (c *Parser) fromReader(ctx context.Context, source string, stdOut io.Reader, commitChan chan Commit) {
outReader := bufio.NewReader(stdOut) outReader := bufio.NewReader(stdOut)
var currentCommit *Commit var currentCommit *Commit
var currentDiff *Diff var currentDiff *Diff
lastCommit := ""
defer common.RecoverWithExit(ctx) defer common.RecoverWithExit(ctx)
defer close(commitChan) defer close(commitChan)
for { for {
@@ -202,6 +200,143 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
if err != nil && len(line) == 0 { if err != nil && len(line) == 0 {
break break
} }
// for i := 0; i < len(line); i++ {
// b := make([]byte, 1)
// n, err := stdOut.Read(b)
// if errors.Is(err, io.EOF) || n == 0 {
// return
// }
// line[i] = b[0]
// if b[0] == byte('\n') {
// line = line[:0]
// break
// }
// }
switch {
case isCommitLine(line):
// If there is a currentDiff, add it to currentCommit.
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
}
// If there is a currentCommit, send it to the channel.
if currentCommit != nil {
args := []string{"-C", source, "diff", "--date=format:%a %b %d %H:%M:%S %Y %z", currentCommit.Hash}
if lastCommit != "" {
args = append(args, lastCommit)
}
lastCommit = currentCommit.Hash
cmd := exec.Command("git", args...)
absPath, err := filepath.Abs(source)
if err == nil {
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
}
// stdnOut, err := cmd.StdoutPipe()
// if err != nil {
// return
// }
err = cmd.Start()
if err != nil {
return
}
_ = cmd.Wait()
//c.funkReader(ctx, stdnOut, commitChan, currentCommit)
}
// Create a new currentDiff and currentCommit
currentDiff = &Diff{}
currentCommit = &Commit{
Message: strings.Builder{},
}
// Check that the commit line contains a hash and set it.
if len(line) >= 47 {
currentCommit.Hash = string(line[7:47])
}
case isAuthorLine(line):
currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
case isDateLine(line):
date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
if err != nil {
ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
}
currentCommit.Date = date
case isModeLine(line):
// NoOp
case isIndexLine(line):
// NoOp
case isPlusFileLine(line):
currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
case isMinusFileLine(line):
// NoOp
case isPlusDiffLine(line):
currentDiff.Content.Write(line[1:])
case isMinusDiffLine(line):
// NoOp. We only care about additions.
case isMessageLine(line):
currentCommit.Message.Write(line[4:])
case isContextDiffLine(line):
currentDiff.Content.Write([]byte("\n"))
case isBinaryLine(line):
currentDiff.IsBinary = true
currentDiff.PathB = pathFromBinaryLine(line)
case isLineNumberDiffLine(line):
if currentDiff != nil && currentDiff.Content.Len() > 0 {
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
}
newDiff := &Diff{
PathB: currentDiff.PathB,
}
currentDiff = newDiff
words := bytes.Split(line, []byte(" "))
if len(words) >= 3 {
startSlice := bytes.Split(words[2], []byte(","))
lineStart, err := strconv.Atoi(string(startSlice[0]))
if err == nil {
currentDiff.LineStart = lineStart
}
}
}
if currentDiff.Content.Len() > c.maxDiffSize {
ctx.Logger().V(2).Info(fmt.Sprintf(
"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
))
break
}
}
cleanupParse(currentCommit, currentDiff, commitChan)
}
func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, currentCommit *Commit) {
outReader := bufio.NewReader(stdOut)
var currentDiff *Diff
defer common.RecoverWithExit(ctx)
for {
if common.IsDone(ctx) {
break
}
line, err := outReader.ReadBytes([]byte("\n")[0])
if err != nil && len(line) == 0 {
break
}
// for i := 0; i < len(line); i++ {
// b := make([]byte, 1)
// n, err := stdOut.Read(b)
// if errors.Is(err, io.EOF) || n == 0 {
// return
// }
// line[i] = b[0]
// if b[0] == byte('\n') {
// line = line[:0]
// break
// }
// }
switch { switch {
case isCommitLine(line): case isCommitLine(line):
// If there is a currentDiff, add it to currentCommit. // If there is a currentDiff, add it to currentCommit.