Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 1bbc35e5be | |||
| e6846ede54 |
@@ -216,31 +216,53 @@ func run(state overseer.State) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Build include and exclude detector filter sets.
|
// Build include and exclude detector filter sets.
|
||||||
var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]struct{}
|
var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID
|
||||||
{
|
{
|
||||||
includeList, err := config.ParseDetectors(*includeDetectors)
|
includeList, err := config.ParseDetectors(*includeDetectors)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Exit if there was an error to inform the user of the misconfiguration.
|
// Exit if there was an error to inform the user of the misconfiguration.
|
||||||
logger.Error(err, "invalid include list detector configuration")
|
logFatal(err, "invalid include list detector configuration")
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
excludeList, err := config.ParseDetectors(*excludeDetectors)
|
excludeList, err := config.ParseDetectors(*excludeDetectors)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
// Exit if there was an error to inform the user of the misconfiguration.
|
// Exit if there was an error to inform the user of the misconfiguration.
|
||||||
logger.Error(err, "invalid exclude list detector configuration")
|
logFatal(err, "invalid exclude list detector configuration")
|
||||||
os.Exit(1)
|
|
||||||
}
|
}
|
||||||
includeDetectorTypes = detectorTypeToSet(includeList)
|
includeDetectorTypes = detectorTypeToMap(includeList)
|
||||||
excludeDetectorTypes = detectorTypeToSet(excludeList)
|
excludeDetectorTypes = detectorTypeToMap(excludeList)
|
||||||
}
|
}
|
||||||
includeFilter := func(d detectors.Detector) bool {
|
includeFilter := func(d detectors.Detector) bool {
|
||||||
_, ok := includeDetectorTypes[d.Type()]
|
id, ok := includeDetectorTypes[d.Type()]
|
||||||
|
if id.Version == 0 {
|
||||||
return ok
|
return ok
|
||||||
}
|
}
|
||||||
|
versionD, ok := d.(detectors.Versioner)
|
||||||
|
if !ok {
|
||||||
|
// Error: version provided but not a detectors.Versioner
|
||||||
|
logFatal(
|
||||||
|
fmt.Errorf("version provided but detector does not have a version"),
|
||||||
|
"invalid include list detector configuration",
|
||||||
|
"detector", id,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return versionD.Version() == id.Version
|
||||||
|
}
|
||||||
excludeFilter := func(d detectors.Detector) bool {
|
excludeFilter := func(d detectors.Detector) bool {
|
||||||
_, ok := excludeDetectorTypes[d.Type()]
|
id, ok := excludeDetectorTypes[d.Type()]
|
||||||
|
if id.Version == 0 {
|
||||||
return !ok
|
return !ok
|
||||||
}
|
}
|
||||||
|
versionD, ok := d.(detectors.Versioner)
|
||||||
|
if !ok {
|
||||||
|
// Error: version provided but not a detectors.Versioner
|
||||||
|
logFatal(
|
||||||
|
fmt.Errorf("version provided but detector does not have a version"),
|
||||||
|
"invalid exclude list detector configuration",
|
||||||
|
"detector", id,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
return versionD.Version() != id.Version
|
||||||
|
}
|
||||||
|
|
||||||
e := engine.Start(ctx,
|
e := engine.Start(ctx,
|
||||||
engine.WithConcurrency(*concurrency),
|
engine.WithConcurrency(*concurrency),
|
||||||
@@ -431,10 +453,10 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func detectorTypeToSet(detectors []detectorspb.DetectorType) map[detectorspb.DetectorType]struct{} {
|
func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {
|
||||||
output := make(map[detectorspb.DetectorType]struct{}, len(detectors))
|
output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))
|
||||||
for _, d := range detectors {
|
for _, d := range detectors {
|
||||||
output[d] = struct{}{}
|
output[d.ID] = d
|
||||||
}
|
}
|
||||||
return output
|
return output
|
||||||
}
|
}
|
||||||
|
|||||||
+71
-28
@@ -2,15 +2,15 @@ package config
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
specialGroups = map[string][]dpb.DetectorType{
|
specialGroups = map[string][]DetectorID{
|
||||||
"all": allDetectors(),
|
"all": allDetectors(),
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,14 +31,19 @@ func init() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type DetectorID struct {
|
||||||
|
ID dpb.DetectorType
|
||||||
|
Version int
|
||||||
|
}
|
||||||
|
|
||||||
// ParseDetectors parses user supplied string into a list of detectors types.
|
// ParseDetectors parses user supplied string into a list of detectors types.
|
||||||
// "all" will return the list of all available detectors. The input is comma
|
// "all" will return the list of all available detectors. The input is comma
|
||||||
// separated and may use the case-insensitive detector name defined in the
|
// separated and may use the case-insensitive detector name defined in the
|
||||||
// protobuf, or the protobuf enum number. A range may be used as well in the
|
// protobuf, or the protobuf enum number. A range may be used as well in the
|
||||||
// form "start-end". Order is preserved and duplicates are ignored.
|
// form "start-end". Order is preserved and duplicates are ignored.
|
||||||
func ParseDetectors(input string) ([]dpb.DetectorType, error) {
|
func ParseDetectors(input string) ([]DetectorID, error) {
|
||||||
var output []dpb.DetectorType
|
var output []DetectorID
|
||||||
seenDetector := map[dpb.DetectorType]struct{}{}
|
seenDetector := map[DetectorID]struct{}{}
|
||||||
for _, item := range strings.Split(input, ",") {
|
for _, item := range strings.Split(input, ",") {
|
||||||
item = strings.TrimSpace(item)
|
item = strings.TrimSpace(item)
|
||||||
if item == "" {
|
if item == "" {
|
||||||
@@ -63,24 +68,35 @@ func ParseDetectors(input string) ([]dpb.DetectorType, error) {
|
|||||||
return output, nil
|
return output, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// allDetectors reutrns an ordered slice of all detector types.
|
func (id DetectorID) String() string {
|
||||||
func allDetectors() []dpb.DetectorType {
|
name := dpb.DetectorType_name[int32(id.ID)]
|
||||||
all := make([]dpb.DetectorType, 0, len(dpb.DetectorType_name))
|
if name == "" {
|
||||||
for id := range dpb.DetectorType_name {
|
name = "<invalid ID>"
|
||||||
all = append(all, dpb.DetectorType(id))
|
|
||||||
}
|
}
|
||||||
slices.Sort(all)
|
if id.Version == 0 {
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
return fmt.Sprintf("%s.v%d", name, id.Version)
|
||||||
|
}
|
||||||
|
|
||||||
|
// allDetectors reutrns an ordered slice of all detector types.
|
||||||
|
func allDetectors() []DetectorID {
|
||||||
|
all := make([]DetectorID, 0, len(dpb.DetectorType_name))
|
||||||
|
for id := range dpb.DetectorType_name {
|
||||||
|
all = append(all, DetectorID{ID: dpb.DetectorType(id)})
|
||||||
|
}
|
||||||
|
sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
|
||||||
return all
|
return all
|
||||||
}
|
}
|
||||||
|
|
||||||
// asRange converts a single input into a slice of detector types. If the input
|
// asRange converts a single input into a slice of detector types. If the input
|
||||||
// is not in range format, a slice of length 1 is returned. Unbounded ranges
|
// is not in range format, a slice of length 1 is returned. Unbounded ranges
|
||||||
// are allowed.
|
// are allowed.
|
||||||
func asRange(input string) ([]dpb.DetectorType, error) {
|
func asRange(input string) ([]DetectorID, error) {
|
||||||
// Check if it's a single detector type.
|
// Check if it's a single detector type.
|
||||||
dt, err := asDetectorType(input)
|
dt, err := asDetectorID(input)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
return []dpb.DetectorType{dt}, nil
|
return []DetectorID{dt}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check if it's a range; if not return the error from above.
|
// Check if it's a range; if not return the error from above.
|
||||||
@@ -91,50 +107,77 @@ func asRange(input string) ([]dpb.DetectorType, error) {
|
|||||||
start, end = strings.TrimSpace(start), strings.TrimSpace(end)
|
start, end = strings.TrimSpace(start), strings.TrimSpace(end)
|
||||||
|
|
||||||
// Convert the range start and end to a DetectorType.
|
// Convert the range start and end to a DetectorType.
|
||||||
dtStart, err := asDetectorType(start)
|
dtStart, err := asDetectorID(start)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
dtEnd, err := asDetectorType(end)
|
dtEnd, err := asDetectorID(end)
|
||||||
// If end is empty it's an unbounded range.
|
// If end is empty it's an unbounded range.
|
||||||
if err != nil && end != "" {
|
if err != nil && end != "" {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
if end == "" {
|
if end == "" {
|
||||||
dtEnd = maxDetectorType
|
dtEnd.ID = maxDetectorType
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure these ranges don't have versions.
|
||||||
|
if dtEnd.Version != 0 || dtStart.Version != 0 {
|
||||||
|
return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
|
||||||
}
|
}
|
||||||
|
|
||||||
step := dpb.DetectorType(1)
|
step := dpb.DetectorType(1)
|
||||||
if dtStart > dtEnd {
|
if dtStart.ID > dtEnd.ID {
|
||||||
step = -1
|
step = -1
|
||||||
}
|
}
|
||||||
var output []dpb.DetectorType
|
var output []DetectorID
|
||||||
for dt := dtStart; dt != dtEnd; dt += step {
|
for dt := dtStart.ID; dt != dtEnd.ID; dt += step {
|
||||||
if _, ok := validDetectors[dt]; !ok {
|
if _, ok := validDetectors[dt]; !ok {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
output = append(output, dt)
|
output = append(output, DetectorID{ID: dt})
|
||||||
}
|
}
|
||||||
return append(output, dtEnd), nil
|
return append(output, dtEnd), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// asDetectorType converts the case-insensitive input into a detector type.
|
// asDetectorID converts the case-insensitive input into a DetectorID.
|
||||||
// Name or ID may be used.
|
// Name or ID may be used.
|
||||||
func asDetectorType(input string) (dpb.DetectorType, error) {
|
func asDetectorID(input string) (DetectorID, error) {
|
||||||
if input == "" {
|
if input == "" {
|
||||||
return 0, fmt.Errorf("empty detector")
|
return DetectorID{}, fmt.Errorf("empty detector")
|
||||||
}
|
}
|
||||||
|
var detectorID DetectorID
|
||||||
|
// Separate the version if there is one.
|
||||||
|
if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {
|
||||||
|
parsedVersion, err := parseVersion(version)
|
||||||
|
if err != nil {
|
||||||
|
return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)
|
||||||
|
}
|
||||||
|
detectorID.Version = parsedVersion
|
||||||
|
// Because there was a version, the detector type input is the part before the '.'
|
||||||
|
input = detector
|
||||||
|
}
|
||||||
|
|
||||||
// Check if it's a named detector.
|
// Check if it's a named detector.
|
||||||
if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
|
if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
|
||||||
return dt, nil
|
detectorID.ID = dt
|
||||||
|
return detectorID, nil
|
||||||
}
|
}
|
||||||
// Check if it's a detector ID.
|
// Check if it's a detector ID.
|
||||||
if i, err := strconv.ParseInt(input, 10, 32); err == nil {
|
if i, err := strconv.ParseInt(input, 10, 32); err == nil {
|
||||||
dt := dpb.DetectorType(i)
|
dt := dpb.DetectorType(i)
|
||||||
if _, ok := validDetectors[dt]; !ok {
|
if _, ok := validDetectors[dt]; !ok {
|
||||||
return 0, fmt.Errorf("invalid detector ID: %s", input)
|
return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
|
||||||
}
|
}
|
||||||
return dt, nil
|
detectorID.ID = dt
|
||||||
|
return detectorID, nil
|
||||||
}
|
}
|
||||||
return 0, fmt.Errorf("unrecognized detector type: %s", input)
|
return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseVersion(v string) (int, error) {
|
||||||
|
if !strings.HasPrefix(strings.ToLower(v), "v") {
|
||||||
|
return 0, fmt.Errorf("version must start with 'v'")
|
||||||
|
}
|
||||||
|
version := strings.TrimLeft(v, "vV")
|
||||||
|
return strconv.Atoi(version)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -10,21 +10,28 @@ import (
|
|||||||
func TestDetectorParsing(t *testing.T) {
|
func TestDetectorParsing(t *testing.T) {
|
||||||
tests := map[string]struct {
|
tests := map[string]struct {
|
||||||
input string
|
input string
|
||||||
expected []dpb.DetectorType
|
expected []DetectorID
|
||||||
}{
|
}{
|
||||||
"all": {"AlL", allDetectors()},
|
"all": {"AlL", allDetectors()},
|
||||||
"trailing range": {"0-", allDetectors()},
|
"trailing range": {"0-", allDetectors()},
|
||||||
"all after 1": {"1-", allDetectors()[1:]},
|
"all after 1": {"1-", allDetectors()[1:]},
|
||||||
"named and valid range": {"aWs,8-9", []dpb.DetectorType{dpb.DetectorType_AWS, dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
|
"named and valid range": {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
|
||||||
"duplicate order preserved": {"9, 8, 9", []dpb.DetectorType{9, 8}},
|
"duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
|
||||||
"named range": {"github - gitlab", []dpb.DetectorType{dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
|
"named range": {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
|
||||||
"range preserved": {"8-9, 7-10", []dpb.DetectorType{8, 9, 7, 10}},
|
"range preserved": {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
|
||||||
"reverse range": {"9-8", []dpb.DetectorType{9, 8}},
|
"reverse range": {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
|
||||||
"range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
|
"range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
|
||||||
"empty list item": {"8, ,9", []dpb.DetectorType{8, 9}},
|
"empty list item": {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
|
||||||
"invalid end range": {"0-1337", nil},
|
"invalid end range": {"0-1337", nil},
|
||||||
"invalid name": {"foo", nil},
|
"invalid name": {"foo", nil},
|
||||||
"negative": {"-1", nil},
|
"negative": {"-1", nil},
|
||||||
|
"github.v1": {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
|
||||||
|
"gitlab.v100": {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
|
||||||
|
"range with versions": {"github.v2 - gitlab.v1", nil},
|
||||||
|
"invalid version no v": {"gitlab.2", nil},
|
||||||
|
"invalid version no number": {"gitlab.github", nil},
|
||||||
|
"capital V is fine": {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
|
||||||
|
"id number with version": {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
|
||||||
}
|
}
|
||||||
|
|
||||||
for name, tt := range tests {
|
for name, tt := range tests {
|
||||||
|
|||||||
@@ -25,6 +25,12 @@ type Detector interface {
|
|||||||
Type() detectorspb.DetectorType
|
Type() detectorspb.DetectorType
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Versioner is an optional interface that a detector can implement to
|
||||||
|
// differentiate instances of the same detector type.
|
||||||
|
type Versioner interface {
|
||||||
|
Version() int
|
||||||
|
}
|
||||||
|
|
||||||
type Result struct {
|
type Result struct {
|
||||||
// DetectorType is the type of Detector.
|
// DetectorType is the type of Detector.
|
||||||
DetectorType detectorspb.DetectorType
|
DetectorType detectorspb.DetectorType
|
||||||
|
|||||||
@@ -15,8 +15,11 @@ import (
|
|||||||
|
|
||||||
type Scanner struct{}
|
type Scanner struct{}
|
||||||
|
|
||||||
// Ensure the Scanner satisfies the interface at compile time.
|
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||||
var _ detectors.Detector = (*Scanner)(nil)
|
var _ detectors.Detector = (*Scanner)(nil)
|
||||||
|
var _ detectors.Versioner = (*Scanner)(nil)
|
||||||
|
|
||||||
|
func (s Scanner) Version() int { return 2 }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// Oauth token
|
// Oauth token
|
||||||
|
|||||||
@@ -14,8 +14,11 @@ import (
|
|||||||
|
|
||||||
type Scanner struct{}
|
type Scanner struct{}
|
||||||
|
|
||||||
// Ensure the Scanner satisfies the interface at compile time.
|
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||||
var _ detectors.Detector = (*Scanner)(nil)
|
var _ detectors.Detector = (*Scanner)(nil)
|
||||||
|
var _ detectors.Versioner = (*Scanner)(nil)
|
||||||
|
|
||||||
|
func (s Scanner) Version() int { return 1 }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
// Oauth token
|
// Oauth token
|
||||||
|
|||||||
@@ -14,8 +14,11 @@ import (
|
|||||||
|
|
||||||
type Scanner struct{}
|
type Scanner struct{}
|
||||||
|
|
||||||
// Ensure the Scanner satisfies the interface at compile time.
|
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||||
var _ detectors.Detector = (*Scanner)(nil)
|
var _ detectors.Detector = (*Scanner)(nil)
|
||||||
|
var _ detectors.Versioner = (*Scanner)(nil)
|
||||||
|
|
||||||
|
func (s Scanner) Version() int { return 1 }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||||
|
|||||||
@@ -13,8 +13,11 @@ import (
|
|||||||
|
|
||||||
type Scanner struct{}
|
type Scanner struct{}
|
||||||
|
|
||||||
// Ensure the Scanner satisfies the interface at compile time.
|
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||||
var _ detectors.Detector = (*Scanner)(nil)
|
var _ detectors.Detector = (*Scanner)(nil)
|
||||||
|
var _ detectors.Versioner = (*Scanner)(nil)
|
||||||
|
|
||||||
|
func (*Scanner) Version() int { return 2 }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
|
keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
|
||||||
|
|||||||
@@ -14,8 +14,11 @@ import (
|
|||||||
|
|
||||||
type Scanner struct{}
|
type Scanner struct{}
|
||||||
|
|
||||||
// Ensure the Scanner satisfies the interface at compile time.
|
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||||
var _ detectors.Detector = (*Scanner)(nil)
|
var _ detectors.Detector = (*Scanner)(nil)
|
||||||
|
var _ detectors.Versioner = (*Scanner)(nil)
|
||||||
|
|
||||||
|
func (s Scanner) Version() int { return 1 }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
client = common.SaneHttpClient()
|
client = common.SaneHttpClient()
|
||||||
|
|||||||
@@ -13,8 +13,11 @@ import (
|
|||||||
|
|
||||||
type Scanner struct{}
|
type Scanner struct{}
|
||||||
|
|
||||||
// Ensure the Scanner satisfies the interface at compile time.
|
// Ensure the Scanner satisfies the interfaces at compile time.
|
||||||
var _ detectors.Detector = (*Scanner)(nil)
|
var _ detectors.Detector = (*Scanner)(nil)
|
||||||
|
var _ detectors.Versioner = (*Scanner)(nil)
|
||||||
|
|
||||||
|
func (s Scanner) Version() int { return 2 }
|
||||||
|
|
||||||
var (
|
var (
|
||||||
client = common.SaneHttpClient()
|
client = common.SaneHttpClient()
|
||||||
|
|||||||
+144
-9
@@ -117,10 +117,7 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
|
|||||||
|
|
||||||
// RepoPath parses the output of the `git log` command for the `source` path.
|
// RepoPath parses the output of the `git log` command for the `source` path.
|
||||||
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
|
func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
|
||||||
args := []string{"-C", source, "log", "-p", "-U5", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
|
args := []string{"-C", source, "log", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
|
||||||
if abbreviatedLog {
|
|
||||||
args = append(args, "--diff-filter=AM")
|
|
||||||
}
|
|
||||||
if head != "" {
|
if head != "" {
|
||||||
args = append(args, head)
|
args = append(args, head)
|
||||||
} else {
|
} else {
|
||||||
@@ -134,7 +131,7 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
|
|||||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.executeCommand(ctx, cmd)
|
return c.executeCommand(ctx, source, cmd)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Unstaged parses the output of the `git diff` command for the `source` path.
|
// Unstaged parses the output of the `git diff` command for the `source` path.
|
||||||
@@ -149,11 +146,11 @@ func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, erro
|
|||||||
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||||
}
|
}
|
||||||
|
|
||||||
return c.executeCommand(ctx, cmd)
|
return c.executeCommand(ctx, source, cmd)
|
||||||
}
|
}
|
||||||
|
|
||||||
// executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
|
// executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
|
||||||
func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
|
func (c *Parser) executeCommand(ctx context.Context, source string, cmd *exec.Cmd) (chan Commit, error) {
|
||||||
commitChan := make(chan Commit, 64)
|
commitChan := make(chan Commit, 64)
|
||||||
|
|
||||||
stdOut, err := cmd.StdoutPipe()
|
stdOut, err := cmd.StdoutPipe()
|
||||||
@@ -178,7 +175,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
|
|||||||
}()
|
}()
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
c.fromReader(ctx, stdOut, commitChan)
|
c.fromReader(ctx, source, stdOut, commitChan)
|
||||||
if err := cmd.Wait(); err != nil {
|
if err := cmd.Wait(); err != nil {
|
||||||
ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
|
ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
|
||||||
}
|
}
|
||||||
@@ -187,11 +184,12 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
|
|||||||
return commitChan, nil
|
return commitChan, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
|
func (c *Parser) fromReader(ctx context.Context, source string, stdOut io.Reader, commitChan chan Commit) {
|
||||||
outReader := bufio.NewReader(stdOut)
|
outReader := bufio.NewReader(stdOut)
|
||||||
var currentCommit *Commit
|
var currentCommit *Commit
|
||||||
var currentDiff *Diff
|
var currentDiff *Diff
|
||||||
|
|
||||||
|
lastCommit := ""
|
||||||
defer common.RecoverWithExit(ctx)
|
defer common.RecoverWithExit(ctx)
|
||||||
defer close(commitChan)
|
defer close(commitChan)
|
||||||
for {
|
for {
|
||||||
@@ -202,6 +200,143 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
|||||||
if err != nil && len(line) == 0 {
|
if err != nil && len(line) == 0 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
|
// for i := 0; i < len(line); i++ {
|
||||||
|
// b := make([]byte, 1)
|
||||||
|
// n, err := stdOut.Read(b)
|
||||||
|
// if errors.Is(err, io.EOF) || n == 0 {
|
||||||
|
// return
|
||||||
|
// }
|
||||||
|
// line[i] = b[0]
|
||||||
|
// if b[0] == byte('\n') {
|
||||||
|
// line = line[:0]
|
||||||
|
// break
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case isCommitLine(line):
|
||||||
|
// If there is a currentDiff, add it to currentCommit.
|
||||||
|
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||||
|
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||||
|
}
|
||||||
|
// If there is a currentCommit, send it to the channel.
|
||||||
|
if currentCommit != nil {
|
||||||
|
args := []string{"-C", source, "diff", "--date=format:%a %b %d %H:%M:%S %Y %z", currentCommit.Hash}
|
||||||
|
if lastCommit != "" {
|
||||||
|
args = append(args, lastCommit)
|
||||||
|
}
|
||||||
|
lastCommit = currentCommit.Hash
|
||||||
|
|
||||||
|
cmd := exec.Command("git", args...)
|
||||||
|
|
||||||
|
absPath, err := filepath.Abs(source)
|
||||||
|
if err == nil {
|
||||||
|
cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
|
||||||
|
}
|
||||||
|
|
||||||
|
// stdnOut, err := cmd.StdoutPipe()
|
||||||
|
// if err != nil {
|
||||||
|
// return
|
||||||
|
// }
|
||||||
|
err = cmd.Start()
|
||||||
|
if err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = cmd.Wait()
|
||||||
|
//c.funkReader(ctx, stdnOut, commitChan, currentCommit)
|
||||||
|
}
|
||||||
|
// Create a new currentDiff and currentCommit
|
||||||
|
currentDiff = &Diff{}
|
||||||
|
currentCommit = &Commit{
|
||||||
|
Message: strings.Builder{},
|
||||||
|
}
|
||||||
|
// Check that the commit line contains a hash and set it.
|
||||||
|
if len(line) >= 47 {
|
||||||
|
currentCommit.Hash = string(line[7:47])
|
||||||
|
}
|
||||||
|
case isAuthorLine(line):
|
||||||
|
currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
|
||||||
|
case isDateLine(line):
|
||||||
|
date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
|
||||||
|
if err != nil {
|
||||||
|
ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
|
||||||
|
}
|
||||||
|
currentCommit.Date = date
|
||||||
|
case isModeLine(line):
|
||||||
|
// NoOp
|
||||||
|
case isIndexLine(line):
|
||||||
|
// NoOp
|
||||||
|
case isPlusFileLine(line):
|
||||||
|
currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
|
||||||
|
case isMinusFileLine(line):
|
||||||
|
// NoOp
|
||||||
|
case isPlusDiffLine(line):
|
||||||
|
currentDiff.Content.Write(line[1:])
|
||||||
|
case isMinusDiffLine(line):
|
||||||
|
// NoOp. We only care about additions.
|
||||||
|
case isMessageLine(line):
|
||||||
|
currentCommit.Message.Write(line[4:])
|
||||||
|
case isContextDiffLine(line):
|
||||||
|
currentDiff.Content.Write([]byte("\n"))
|
||||||
|
case isBinaryLine(line):
|
||||||
|
currentDiff.IsBinary = true
|
||||||
|
currentDiff.PathB = pathFromBinaryLine(line)
|
||||||
|
case isLineNumberDiffLine(line):
|
||||||
|
if currentDiff != nil && currentDiff.Content.Len() > 0 {
|
||||||
|
currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
|
||||||
|
}
|
||||||
|
newDiff := &Diff{
|
||||||
|
PathB: currentDiff.PathB,
|
||||||
|
}
|
||||||
|
|
||||||
|
currentDiff = newDiff
|
||||||
|
|
||||||
|
words := bytes.Split(line, []byte(" "))
|
||||||
|
if len(words) >= 3 {
|
||||||
|
startSlice := bytes.Split(words[2], []byte(","))
|
||||||
|
lineStart, err := strconv.Atoi(string(startSlice[0]))
|
||||||
|
if err == nil {
|
||||||
|
currentDiff.LineStart = lineStart
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if currentDiff.Content.Len() > c.maxDiffSize {
|
||||||
|
ctx.Logger().V(2).Info(fmt.Sprintf(
|
||||||
|
"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
|
||||||
|
))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cleanupParse(currentCommit, currentDiff, commitChan)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, currentCommit *Commit) {
|
||||||
|
outReader := bufio.NewReader(stdOut)
|
||||||
|
var currentDiff *Diff
|
||||||
|
|
||||||
|
defer common.RecoverWithExit(ctx)
|
||||||
|
for {
|
||||||
|
if common.IsDone(ctx) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
line, err := outReader.ReadBytes([]byte("\n")[0])
|
||||||
|
if err != nil && len(line) == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// for i := 0; i < len(line); i++ {
|
||||||
|
// b := make([]byte, 1)
|
||||||
|
// n, err := stdOut.Read(b)
|
||||||
|
// if errors.Is(err, io.EOF) || n == 0 {
|
||||||
|
// return
|
||||||
|
// }
|
||||||
|
// line[i] = b[0]
|
||||||
|
// if b[0] == byte('\n') {
|
||||||
|
// line = line[:0]
|
||||||
|
// break
|
||||||
|
// }
|
||||||
|
// }
|
||||||
|
|
||||||
switch {
|
switch {
|
||||||
case isCommitLine(line):
|
case isCommitLine(line):
|
||||||
// If there is a currentDiff, add it to currentCommit.
|
// If there is a currentDiff, add it to currentCommit.
|
||||||
|
|||||||
Reference in New Issue
Block a user