Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Dustin Decker
|
443175c71d | ||
|
raju-bs
|
a2e2174a12 | ||
|
raju-bs
|
69c4bdd427 | ||
|
raju-bs
|
4be186581a |
@@ -100,12 +100,6 @@ trufflehog s3 --bucket=<bucket name> --only-verified
|
||||
docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
|
||||
```
|
||||
|
||||
### Example 6: Scan individual files or directories
|
||||
|
||||
```bash
|
||||
trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
|
||||
```
|
||||
|
||||
# What's new in v3?
|
||||
|
||||
TruffleHog v3 is a complete rewrite in Go with many new powerful features.
|
||||
@@ -173,10 +167,10 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
|
||||
- github
|
||||
- gitlab
|
||||
- S3
|
||||
- filesystem (files and directories)
|
||||
- filesystem
|
||||
- syslog
|
||||
- circleci
|
||||
- stdin (coming soon)
|
||||
- file and stdin (coming soon)
|
||||
|
||||
Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
|
||||
|
||||
@@ -317,7 +311,7 @@ detectors:
|
||||
```
|
||||
|
||||
```
|
||||
$ trufflehog filesystem /tmp --config config.yaml --only-verified
|
||||
$ trufflehog filesystem --directory /tmp --config config.yaml --only-verified
|
||||
🐷🔑🐷 TruffleHog. Unearth your secrets. 🐷🔑🐷
|
||||
|
||||
Found verified result 🐷🔑
|
||||
|
||||
@@ -11,7 +11,7 @@ replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff
|
||||
require (
|
||||
cloud.google.com/go/secretmanager v1.10.0
|
||||
github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
|
||||
github.com/TheZeroSlave/zapsentry v1.14.0
|
||||
github.com/TheZeroSlave/zapsentry v1.12.0
|
||||
github.com/aws/aws-sdk-go v1.44.83
|
||||
github.com/bill-rich/disk-buffer-reader v0.1.7
|
||||
github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
|
||||
@@ -45,18 +45,19 @@ require (
|
||||
github.com/mholt/archiver/v4 v4.0.0-alpha.7
|
||||
github.com/paulbellamy/ratecounter v0.2.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rabbitmq/amqp091-go v1.7.0
|
||||
github.com/rabbitmq/amqp091-go v1.6.0
|
||||
github.com/sergi/go-diff v1.3.1
|
||||
github.com/stretchr/testify v1.8.1
|
||||
github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
|
||||
github.com/xanzy/go-gitlab v0.80.2
|
||||
go.mongodb.org/mongo-driver v1.11.2
|
||||
github.com/xanzy/go-gitlab v0.78.0
|
||||
go.mongodb.org/mongo-driver v1.11.1
|
||||
go.uber.org/zap v1.24.0
|
||||
golang.org/x/crypto v0.6.0
|
||||
golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
|
||||
golang.org/x/oauth2 v0.5.0
|
||||
golang.org/x/oauth2 v0.4.0
|
||||
golang.org/x/sync v0.1.0
|
||||
golang.org/x/text v0.7.0
|
||||
google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c
|
||||
google.golang.org/protobuf v1.28.1
|
||||
gopkg.in/alecthomas/kingpin.v2 v2.2.6
|
||||
gopkg.in/h2non/gock.v1 v1.1.2
|
||||
@@ -137,13 +138,12 @@ require (
|
||||
go.uber.org/atomic v1.7.0 // indirect
|
||||
go.uber.org/multierr v1.6.0 // indirect
|
||||
golang.org/x/mod v0.7.0 // indirect
|
||||
golang.org/x/net v0.7.0 // indirect
|
||||
golang.org/x/net v0.6.0 // indirect
|
||||
golang.org/x/sys v0.5.0 // indirect
|
||||
golang.org/x/time v0.3.0 // indirect
|
||||
golang.org/x/tools v0.3.0 // indirect
|
||||
google.golang.org/api v0.103.0 // indirect
|
||||
google.golang.org/appengine v1.6.7 // indirect
|
||||
google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect
|
||||
google.golang.org/grpc v1.51.0 // indirect
|
||||
gopkg.in/warnings.v0 v0.1.2 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
|
||||
@@ -37,8 +37,8 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
|
||||
github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
|
||||
github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
|
||||
github.com/TheZeroSlave/zapsentry v1.14.0 h1:z6H0lIyKVpdXsP5K8J11b4/+V+QCDLDslZmtXsnduMk=
|
||||
github.com/TheZeroSlave/zapsentry v1.14.0/go.mod h1:0hUR1Et5tSroEYhWOZcxE2iBqcFkuG7jGGlUmOA7XAs=
|
||||
github.com/TheZeroSlave/zapsentry v1.12.0 h1:hhfwvD7pQbnCIOvAEGM4NXLiaX4b6W6ZWNtL6awSBGc=
|
||||
github.com/TheZeroSlave/zapsentry v1.12.0/go.mod h1:00uO/VpPrSJG/XigAfTi0F4WMFIw2DmP/IDVUhPBvNw=
|
||||
github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
|
||||
github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
|
||||
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
|
||||
@@ -295,8 +295,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/rabbitmq/amqp091-go v1.7.0 h1:V5CF5qPem5OGSnEo8BoSbsDGwejg6VUJsKEdneaoTUo=
|
||||
github.com/rabbitmq/amqp091-go v1.7.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
|
||||
github.com/rabbitmq/amqp091-go v1.6.0 h1:fiFHcMcP+5/A+35MXniiagho1lEXX4+jpHGR+q+LeBU=
|
||||
github.com/rabbitmq/amqp091-go v1.6.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
|
||||
@@ -333,8 +333,8 @@ github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhN
|
||||
github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
|
||||
github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
|
||||
github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
|
||||
github.com/xanzy/go-gitlab v0.80.2 h1:CH1Q7NDklqZllox4ICVF4PwlhQGfPtE+w08Jsb74ZX0=
|
||||
github.com/xanzy/go-gitlab v0.80.2/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M=
|
||||
github.com/xanzy/go-gitlab v0.78.0 h1:8jUHfQVAprG04Av5g0PxVd3CNsZ5hCbojIax7Hba1mE=
|
||||
github.com/xanzy/go-gitlab v0.78.0/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M=
|
||||
github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
|
||||
github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
@@ -350,8 +350,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
|
||||
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
|
||||
go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
|
||||
go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
|
||||
go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
|
||||
go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
|
||||
go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
|
||||
go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
|
||||
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
|
||||
@@ -409,11 +409,11 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
|
||||
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
|
||||
golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
|
||||
golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
|
||||
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
|
||||
golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
|
||||
golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
|
||||
golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
|
||||
@@ -9,8 +9,6 @@ import (
|
||||
"text/template"
|
||||
|
||||
"github.com/go-errors/errors"
|
||||
"golang.org/x/text/cases"
|
||||
"golang.org/x/text/language"
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
)
|
||||
|
||||
@@ -26,7 +24,7 @@ func main() {
|
||||
log.SetPrefix("😲 [generate] ")
|
||||
|
||||
kingpin.MustParse(app.Parse(os.Args[1:]))
|
||||
nameTitle = cases.Title(language.AmericanEnglish).String(*name)
|
||||
nameTitle = strings.Title(*name)
|
||||
nameLower = strings.ToLower(*name)
|
||||
nameUpper = strings.ToUpper(*name)
|
||||
|
||||
@@ -67,7 +65,7 @@ type templateJob struct {
|
||||
}
|
||||
|
||||
func mustWriteTemplates(jobs []templateJob) {
|
||||
log.Printf("Generating %s %s\n", cases.Title(language.AmericanEnglish).String(*kind), nameTitle)
|
||||
log.Printf("Generating %s %s\n", strings.Title(*kind), nameTitle)
|
||||
|
||||
// Make the folder.
|
||||
log.Printf("Creating folder %s\n", folderPath())
|
||||
@@ -86,7 +84,7 @@ func mustWriteTemplates(jobs []templateJob) {
|
||||
|
||||
for _, rplString := range job.ReplaceString {
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToLower(rplString), "<<.NameLower>>")
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, cases.Title(language.AmericanEnglish).String(rplString), "<<.NameTitle>>")
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, strings.Title(rplString), "<<.NameTitle>>")
|
||||
tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToUpper(rplString), "<<.NameUpper>>")
|
||||
}
|
||||
|
||||
|
||||
@@ -173,9 +173,7 @@ func main() {
|
||||
}
|
||||
|
||||
for _, repo := range strings.Split(*scanCmdRepo, ",") {
|
||||
if err := sem.Acquire(ctx, 1); err != nil {
|
||||
logFatal(err, "timed out waiting for semaphore")
|
||||
}
|
||||
sem.Acquire(ctx, 1)
|
||||
wgChunkers.Add(1)
|
||||
go func(r string) {
|
||||
defer sem.Release(1)
|
||||
|
||||
@@ -21,12 +21,10 @@ import (
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
|
||||
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
|
||||
@@ -52,8 +50,6 @@ var (
|
||||
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
|
||||
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
|
||||
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
|
||||
includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
|
||||
excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
|
||||
|
||||
gitScan = cli.Command("git", "Find credentials in git repositories.")
|
||||
gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
|
||||
@@ -86,10 +82,8 @@ var (
|
||||
gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
|
||||
|
||||
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
|
||||
filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
|
||||
// DEPRECATED: --directory is deprecated in favor of arguments.
|
||||
filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
|
||||
filesystemScan = cli.Command("filesystem", "Find credentials in a filesystem.")
|
||||
filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
|
||||
// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
|
||||
// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
|
||||
filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
|
||||
@@ -214,40 +208,11 @@ func run(state overseer.State) {
|
||||
handlers.SetArchiveMaxTimeout(*archiveTimeout)
|
||||
}
|
||||
|
||||
// Build include and exclude detector filter sets.
|
||||
var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]struct{}
|
||||
{
|
||||
includeList, err := config.ParseDetectors(*includeDetectors)
|
||||
if err != nil {
|
||||
// Exit if there was an error to inform the user of the misconfiguration.
|
||||
logger.Error(err, "invalid include list detector configuration")
|
||||
os.Exit(1)
|
||||
}
|
||||
excludeList, err := config.ParseDetectors(*excludeDetectors)
|
||||
if err != nil {
|
||||
// Exit if there was an error to inform the user of the misconfiguration.
|
||||
logger.Error(err, "invalid exclude list detector configuration")
|
||||
os.Exit(1)
|
||||
}
|
||||
includeDetectorTypes = detectorTypeToSet(includeList)
|
||||
excludeDetectorTypes = detectorTypeToSet(excludeList)
|
||||
}
|
||||
includeFilter := func(d detectors.Detector) bool {
|
||||
_, ok := includeDetectorTypes[d.Type()]
|
||||
return ok
|
||||
}
|
||||
excludeFilter := func(d detectors.Detector) bool {
|
||||
_, ok := excludeDetectorTypes[d.Type()]
|
||||
return !ok
|
||||
}
|
||||
|
||||
e := engine.Start(ctx,
|
||||
engine.WithConcurrency(*concurrency),
|
||||
engine.WithDecoders(decoders.DefaultDecoders()...),
|
||||
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
|
||||
engine.WithDetectors(!*noVerification, conf.Detectors...),
|
||||
engine.WithFilterDetectors(includeFilter),
|
||||
engine.WithFilterDetectors(excludeFilter),
|
||||
engine.WithFilterUnverified(*filterUnverified),
|
||||
)
|
||||
|
||||
@@ -321,15 +286,10 @@ func run(state overseer.State) {
|
||||
if err != nil {
|
||||
logFatal(err, "could not create filter")
|
||||
}
|
||||
if len(*filesystemDirectories) > 0 {
|
||||
ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
|
||||
}
|
||||
paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories))
|
||||
paths = append(paths, *filesystemPaths...)
|
||||
paths = append(paths, *filesystemDirectories...)
|
||||
|
||||
cfg := sources.FilesystemConfig{
|
||||
Paths: paths,
|
||||
Filter: filter,
|
||||
Directories: *filesystemDirectories,
|
||||
Filter: filter,
|
||||
}
|
||||
if err = e.ScanFileSystem(ctx, cfg); err != nil {
|
||||
logFatal(err, "Failed to scan filesystem")
|
||||
@@ -429,11 +389,3 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
|
||||
os.Exit(0)
|
||||
}
|
||||
}
|
||||
|
||||
func detectorTypeToSet(detectors []detectorspb.DetectorType) map[detectorspb.DetectorType]struct{} {
|
||||
output := make(map[detectorspb.DetectorType]struct{}, len(detectors))
|
||||
for _, d := range detectors {
|
||||
output[d] = struct{}{}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
@@ -7,9 +7,9 @@ import (
|
||||
"time"
|
||||
|
||||
secretmanager "cloud.google.com/go/secretmanager/apiv1"
|
||||
"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/pkg/errors"
|
||||
secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
|
||||
)
|
||||
|
||||
type Secret struct{ kv map[string]string }
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
"golang.org/x/exp/slices"
|
||||
)
|
||||
|
||||
var (
|
||||
specialGroups = map[string][]dpb.DetectorType{
|
||||
"all": allDetectors(),
|
||||
}
|
||||
|
||||
detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))
|
||||
validDetectors = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))
|
||||
maxDetectorType dpb.DetectorType
|
||||
)
|
||||
|
||||
// Setup package local global variables.
|
||||
func init() {
|
||||
for k, v := range dpb.DetectorType_value {
|
||||
dt := dpb.DetectorType(v)
|
||||
detectorTypeValue[strings.ToLower(k)] = dt
|
||||
validDetectors[dt] = struct{}{}
|
||||
if dt > maxDetectorType {
|
||||
maxDetectorType = dt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ParseDetectors parses user supplied string into a list of detectors types.
|
||||
// "all" will return the list of all available detectors. The input is comma
|
||||
// separated and may use the case-insensitive detector name defined in the
|
||||
// protobuf, or the protobuf enum number. A range may be used as well in the
|
||||
// form "start-end". Order is preserved and duplicates are ignored.
|
||||
func ParseDetectors(input string) ([]dpb.DetectorType, error) {
|
||||
var output []dpb.DetectorType
|
||||
seenDetector := map[dpb.DetectorType]struct{}{}
|
||||
for _, item := range strings.Split(input, ",") {
|
||||
item = strings.TrimSpace(item)
|
||||
if item == "" {
|
||||
continue
|
||||
}
|
||||
detectors, ok := specialGroups[strings.ToLower(item)]
|
||||
if !ok {
|
||||
var err error
|
||||
detectors, err = asRange(item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, d := range detectors {
|
||||
if _, ok := seenDetector[d]; ok {
|
||||
continue
|
||||
}
|
||||
seenDetector[d] = struct{}{}
|
||||
output = append(output, d)
|
||||
}
|
||||
}
|
||||
return output, nil
|
||||
}
|
||||
|
||||
// allDetectors reutrns an ordered slice of all detector types.
|
||||
func allDetectors() []dpb.DetectorType {
|
||||
all := make([]dpb.DetectorType, 0, len(dpb.DetectorType_name))
|
||||
for id := range dpb.DetectorType_name {
|
||||
all = append(all, dpb.DetectorType(id))
|
||||
}
|
||||
slices.Sort(all)
|
||||
return all
|
||||
}
|
||||
|
||||
// asRange converts a single input into a slice of detector types. If the input
|
||||
// is not in range format, a slice of length 1 is returned. Unbounded ranges
|
||||
// are allowed.
|
||||
func asRange(input string) ([]dpb.DetectorType, error) {
|
||||
// Check if it's a single detector type.
|
||||
dt, err := asDetectorType(input)
|
||||
if err == nil {
|
||||
return []dpb.DetectorType{dt}, nil
|
||||
}
|
||||
|
||||
// Check if it's a range; if not return the error from above.
|
||||
start, end, found := strings.Cut(input, "-")
|
||||
if !found {
|
||||
return nil, err
|
||||
}
|
||||
start, end = strings.TrimSpace(start), strings.TrimSpace(end)
|
||||
|
||||
// Convert the range start and end to a DetectorType.
|
||||
dtStart, err := asDetectorType(start)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dtEnd, err := asDetectorType(end)
|
||||
// If end is empty it's an unbounded range.
|
||||
if err != nil && end != "" {
|
||||
return nil, err
|
||||
}
|
||||
if end == "" {
|
||||
dtEnd = maxDetectorType
|
||||
}
|
||||
|
||||
step := dpb.DetectorType(1)
|
||||
if dtStart > dtEnd {
|
||||
step = -1
|
||||
}
|
||||
var output []dpb.DetectorType
|
||||
for dt := dtStart; dt != dtEnd; dt += step {
|
||||
if _, ok := validDetectors[dt]; !ok {
|
||||
continue
|
||||
}
|
||||
output = append(output, dt)
|
||||
}
|
||||
return append(output, dtEnd), nil
|
||||
}
|
||||
|
||||
// asDetectorType converts the case-insensitive input into a detector type.
|
||||
// Name or ID may be used.
|
||||
func asDetectorType(input string) (dpb.DetectorType, error) {
|
||||
if input == "" {
|
||||
return 0, fmt.Errorf("empty detector")
|
||||
}
|
||||
// Check if it's a named detector.
|
||||
if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
|
||||
return dt, nil
|
||||
}
|
||||
// Check if it's a detector ID.
|
||||
if i, err := strconv.ParseInt(input, 10, 32); err == nil {
|
||||
dt := dpb.DetectorType(i)
|
||||
if _, ok := validDetectors[dt]; !ok {
|
||||
return 0, fmt.Errorf("invalid detector ID: %s", input)
|
||||
}
|
||||
return dt, nil
|
||||
}
|
||||
return 0, fmt.Errorf("unrecognized detector type: %s", input)
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
|
||||
)
|
||||
|
||||
func TestDetectorParsing(t *testing.T) {
|
||||
tests := map[string]struct {
|
||||
input string
|
||||
expected []dpb.DetectorType
|
||||
}{
|
||||
"all": {"AlL", allDetectors()},
|
||||
"trailing range": {"0-", allDetectors()},
|
||||
"all after 1": {"1-", allDetectors()[1:]},
|
||||
"named and valid range": {"aWs,8-9", []dpb.DetectorType{dpb.DetectorType_AWS, dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
|
||||
"duplicate order preserved": {"9, 8, 9", []dpb.DetectorType{9, 8}},
|
||||
"named range": {"github - gitlab", []dpb.DetectorType{dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
|
||||
"range preserved": {"8-9, 7-10", []dpb.DetectorType{8, 9, 7, 10}},
|
||||
"reverse range": {"9-8", []dpb.DetectorType{9, 8}},
|
||||
"range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
|
||||
"empty list item": {"8, ,9", []dpb.DetectorType{8, 9}},
|
||||
"invalid end range": {"0-1337", nil},
|
||||
"invalid name": {"foo", nil},
|
||||
"negative": {"-1", nil},
|
||||
}
|
||||
|
||||
for name, tt := range tests {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
got, gotErr := ParseDetectors(tt.input)
|
||||
if tt.expected == nil {
|
||||
assert.Error(t, gotErr)
|
||||
return
|
||||
}
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -20,8 +20,8 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
|
||||
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "key", "automate", "local"}) + `\b([0-9a-zA-Z]{20})\b`)
|
||||
userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "user", "automate", "local"}) + `\b([a-zA-Z\d]{3,18}[._-]?[a-zA-Z\d]{6})\b`)
|
||||
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "key", "automate", "local"}) + `\b([0-9a-zA-Z_]{20})\b`)
|
||||
userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "user", "automate", "local"}) + `\b([a-zA-Z\d]{3,18}[._-]+[a-zA-Z\d]{6})\b`)
|
||||
)
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
@@ -67,11 +67,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode >= 200 && res.StatusCode < 300 {
|
||||
s1.Verified = true
|
||||
} else {
|
||||
// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.
|
||||
if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -44,6 +44,8 @@ func New() Scanner {
|
||||
}
|
||||
|
||||
type Scanner struct {
|
||||
wordList []string
|
||||
badList []string
|
||||
programmingBookWords []string
|
||||
excludeMatchers []*regexp.Regexp
|
||||
}
|
||||
|
||||
@@ -26,6 +26,10 @@ var (
|
||||
client = common.SaneHttpClient()
|
||||
)
|
||||
|
||||
type proxyRes struct {
|
||||
Verified bool `json:"verified"`
|
||||
}
|
||||
|
||||
// Keywords are used for efficiently pre-filtering chunks.
|
||||
// Use identifiers in the secret preferably, or the provider name.
|
||||
func (s Scanner) Keywords() []string {
|
||||
|
||||
@@ -73,31 +73,6 @@ func WithFilterUnverified(filter bool) EngineOption {
|
||||
}
|
||||
}
|
||||
|
||||
// WithFilterDetectors applies a filter to the configured list of detectors. If
|
||||
// the filterFunc returns true, the detector will be included for scanning.
|
||||
// This option applies to the existing list of detectors configured, so the
|
||||
// order this option appears matters. All filtering happens before scanning.
|
||||
func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
|
||||
return func(e *Engine) {
|
||||
// If no detectors are configured, do nothing.
|
||||
if e.detectors == nil {
|
||||
return
|
||||
}
|
||||
e.detectors[true] = filterDetectors(filterFunc, e.detectors[true])
|
||||
e.detectors[false] = filterDetectors(filterFunc, e.detectors[false])
|
||||
}
|
||||
}
|
||||
|
||||
func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
|
||||
var output []detectors.Detector
|
||||
for _, detector := range input {
|
||||
if filterFunc(detector) {
|
||||
output = append(output, detector)
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
|
||||
func Start(ctx context.Context, options ...EngineOption) *Engine {
|
||||
e := &Engine{
|
||||
chunks: make(chan *sources.Chunk),
|
||||
|
||||
@@ -17,7 +17,7 @@ import (
|
||||
// ScanFileSystem scans a given file system.
|
||||
func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {
|
||||
connection := &sourcespb.Filesystem{
|
||||
Paths: c.Paths,
|
||||
Directories: c.Directories,
|
||||
}
|
||||
var conn anypb.Any
|
||||
err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
|
||||
|
||||
@@ -241,17 +241,14 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
|
||||
totalSize += diff.Content.Len()
|
||||
}
|
||||
if totalSize > c.maxCommitSize {
|
||||
oldCommit := currentCommit
|
||||
commitChan <- *currentCommit
|
||||
currentCommit = &Commit{
|
||||
Hash: currentCommit.Hash,
|
||||
Author: currentCommit.Author,
|
||||
Date: currentCommit.Date,
|
||||
Message: strings.Builder{},
|
||||
Message: currentCommit.Message,
|
||||
Diffs: []Diff{},
|
||||
}
|
||||
// Message needs to be recreated here otherwise writing to it again will result in a panic.
|
||||
currentCommit.Message.WriteString(oldCommit.Message.String())
|
||||
}
|
||||
}
|
||||
currentDiff = &Diff{}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1267,117 +1267,6 @@ var _ interface {
|
||||
ErrorName() string
|
||||
} = GitlabValidationError{}
|
||||
|
||||
// Validate checks the field values on GoogleDrive with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *GoogleDrive) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on GoogleDrive with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in GoogleDriveMultiError, or
|
||||
// nil if none found.
|
||||
func (m *GoogleDrive) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *GoogleDrive) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
// no validation rules for File
|
||||
|
||||
// no validation rules for Link
|
||||
|
||||
// no validation rules for Email
|
||||
|
||||
// no validation rules for Timestamp
|
||||
|
||||
// no validation rules for Shared
|
||||
|
||||
// no validation rules for LastModifiedBy
|
||||
|
||||
if len(errors) > 0 {
|
||||
return GoogleDriveMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GoogleDriveMultiError is an error wrapping multiple validation errors
|
||||
// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
|
||||
type GoogleDriveMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m GoogleDriveMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m GoogleDriveMultiError) AllErrors() []error { return m }
|
||||
|
||||
// GoogleDriveValidationError is the validation error returned by
|
||||
// GoogleDrive.Validate if the designated constraints aren't met.
|
||||
type GoogleDriveValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e GoogleDriveValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e GoogleDriveValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e GoogleDriveValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e GoogleDriveValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e GoogleDriveValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sGoogleDrive.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = GoogleDriveValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = GoogleDriveValidationError{}
|
||||
|
||||
// Validate checks the field values on GCS with the rules defined in the proto
|
||||
// definition for this message. If any rules are violated, the first error
|
||||
// encountered is returned, or nil if there are no violations.
|
||||
|
||||
+386
-489
File diff suppressed because it is too large
Load Diff
@@ -1887,112 +1887,6 @@ var _ interface {
|
||||
ErrorName() string
|
||||
} = GitHubValidationError{}
|
||||
|
||||
// Validate checks the field values on GoogleDrive with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the first
|
||||
// error encountered is returned, or nil if there are no violations.
|
||||
func (m *GoogleDrive) Validate() error {
|
||||
return m.validate(false)
|
||||
}
|
||||
|
||||
// ValidateAll checks the field values on GoogleDrive with the rules defined in
|
||||
// the proto definition for this message. If any rules are violated, the
|
||||
// result is a list of violation errors wrapped in GoogleDriveMultiError, or
|
||||
// nil if none found.
|
||||
func (m *GoogleDrive) ValidateAll() error {
|
||||
return m.validate(true)
|
||||
}
|
||||
|
||||
func (m *GoogleDrive) validate(all bool) error {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
var errors []error
|
||||
|
||||
switch m.Credential.(type) {
|
||||
|
||||
case *GoogleDrive_RefreshToken:
|
||||
// no validation rules for RefreshToken
|
||||
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
return GoogleDriveMultiError(errors)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// GoogleDriveMultiError is an error wrapping multiple validation errors
|
||||
// returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
|
||||
type GoogleDriveMultiError []error
|
||||
|
||||
// Error returns a concatenation of all the error messages it wraps.
|
||||
func (m GoogleDriveMultiError) Error() string {
|
||||
var msgs []string
|
||||
for _, err := range m {
|
||||
msgs = append(msgs, err.Error())
|
||||
}
|
||||
return strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// AllErrors returns a list of validation violation errors.
|
||||
func (m GoogleDriveMultiError) AllErrors() []error { return m }
|
||||
|
||||
// GoogleDriveValidationError is the validation error returned by
|
||||
// GoogleDrive.Validate if the designated constraints aren't met.
|
||||
type GoogleDriveValidationError struct {
|
||||
field string
|
||||
reason string
|
||||
cause error
|
||||
key bool
|
||||
}
|
||||
|
||||
// Field function returns field value.
|
||||
func (e GoogleDriveValidationError) Field() string { return e.field }
|
||||
|
||||
// Reason function returns reason value.
|
||||
func (e GoogleDriveValidationError) Reason() string { return e.reason }
|
||||
|
||||
// Cause function returns cause value.
|
||||
func (e GoogleDriveValidationError) Cause() error { return e.cause }
|
||||
|
||||
// Key function returns key value.
|
||||
func (e GoogleDriveValidationError) Key() bool { return e.key }
|
||||
|
||||
// ErrorName returns error name.
|
||||
func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
|
||||
|
||||
// Error satisfies the builtin error interface
|
||||
func (e GoogleDriveValidationError) Error() string {
|
||||
cause := ""
|
||||
if e.cause != nil {
|
||||
cause = fmt.Sprintf(" | caused by: %v", e.cause)
|
||||
}
|
||||
|
||||
key := ""
|
||||
if e.key {
|
||||
key = "key for "
|
||||
}
|
||||
|
||||
return fmt.Sprintf(
|
||||
"invalid %sGoogleDrive.%s: %s%s",
|
||||
key,
|
||||
e.field,
|
||||
e.reason,
|
||||
cause)
|
||||
}
|
||||
|
||||
var _ error = GoogleDriveValidationError{}
|
||||
|
||||
var _ interface {
|
||||
Field() string
|
||||
Reason() string
|
||||
Key() bool
|
||||
Cause() error
|
||||
ErrorName() string
|
||||
} = GoogleDriveValidationError{}
|
||||
|
||||
// Validate checks the field values on JIRA with the rules defined in the proto
|
||||
// definition for this message. If any rules are violated, the first error
|
||||
// encountered is returned, or nil if there are no violations.
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
@@ -30,7 +29,3 @@ func (s *ScanErrors) Count() uint64 {
|
||||
defer s.mu.RUnlock()
|
||||
return uint64(len(s.errors))
|
||||
}
|
||||
|
||||
func (s *ScanErrors) String() string {
|
||||
return fmt.Sprintf("%v", s.errors)
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
package sources
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
@@ -148,12 +147,3 @@ func TestScanErrorsCount(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestScanErrorsString(t *testing.T) {
|
||||
se := NewScanErrors()
|
||||
se.Add(nil)
|
||||
want := "[<nil>]"
|
||||
if got := fmt.Sprintf("%v", se); got != want {
|
||||
t.Errorf("got %q, want %q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -70,7 +70,8 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
|
||||
if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
|
||||
return errors.WrapPrefix(err, "error unmarshalling connection", 0)
|
||||
}
|
||||
s.paths = append(conn.Paths, conn.Directories...)
|
||||
|
||||
s.paths = conn.Directories
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -82,122 +83,100 @@ func (s *Source) WithFilter(filter *common.Filter) {
|
||||
// Chunks emits chunks of bytes over a channel.
|
||||
func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
|
||||
for i, path := range s.paths {
|
||||
logger := ctx.Logger().WithValues("path", path)
|
||||
if common.IsDone(ctx) {
|
||||
return nil
|
||||
}
|
||||
s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")
|
||||
|
||||
cleanPath := filepath.Clean(path)
|
||||
fileInfo, err := os.Stat(cleanPath)
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to get file info")
|
||||
continue
|
||||
}
|
||||
done := false
|
||||
go func() {
|
||||
<-ctx.Done()
|
||||
done = true
|
||||
}()
|
||||
|
||||
if fileInfo.IsDir() {
|
||||
err = s.scanDir(ctx, cleanPath, chunksChan)
|
||||
} else {
|
||||
err = s.scanFile(ctx, cleanPath, chunksChan)
|
||||
}
|
||||
err := fs.WalkDir(os.DirFS(cleanPath), ".", func(relativePath string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
path := filepath.Join(cleanPath, relativePath)
|
||||
logger := ctx.Logger().WithValues("file", path)
|
||||
|
||||
fileStat, err := os.Stat(path)
|
||||
if err != nil {
|
||||
logger.Error(err, "unable to stat file")
|
||||
return nil
|
||||
}
|
||||
if !fileStat.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.filter != nil && !s.filter.Pass(path) {
|
||||
return nil
|
||||
}
|
||||
|
||||
inputFile, err := os.Open(path)
|
||||
if err != nil {
|
||||
logger.V(2).Info("error opening file", "error", err)
|
||||
return nil
|
||||
}
|
||||
defer inputFile.Close()
|
||||
logger.V(5).Info("scanning file")
|
||||
|
||||
reReader, err := diskbufferreader.New(inputFile)
|
||||
if err != nil {
|
||||
logger.Error(err, "Could not create re-readable reader.")
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := reReader.Reset(); err != nil {
|
||||
return err
|
||||
}
|
||||
reReader.Stop()
|
||||
data, err := io.ReadAll(reReader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: data,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
if err != nil && err != io.EOF {
|
||||
logger.Info("error scanning filesystem", "error", err)
|
||||
return errors.New(err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
|
||||
return fs.WalkDir(os.DirFS(path), ".", func(relativePath string, d fs.DirEntry, err error) error {
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
fullPath := filepath.Join(path, relativePath)
|
||||
|
||||
// Skip over non-regular files. We do this check here to suppress noisy
|
||||
// logs for trying to scan directories and other non-regular files in
|
||||
// our traversal.
|
||||
fileStat, err := os.Stat(fullPath)
|
||||
if err != nil {
|
||||
ctx.Logger().Info("unable to stat file", "path", fullPath, "error", err)
|
||||
return nil
|
||||
}
|
||||
if !fileStat.Mode().IsRegular() {
|
||||
return nil
|
||||
}
|
||||
if s.filter != nil && !s.filter.Pass(fullPath) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {
|
||||
ctx.Logger().Info("error scanning file", "path", fullPath, "error", err)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
|
||||
logger := ctx.Logger().WithValues("path", path)
|
||||
fileStat, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to stat file: %w", err)
|
||||
}
|
||||
if !fileStat.Mode().IsRegular() {
|
||||
return fmt.Errorf("not a regular file")
|
||||
}
|
||||
|
||||
inputFile, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to open file: %w", err)
|
||||
}
|
||||
defer inputFile.Close()
|
||||
logger.V(3).Info("scanning file")
|
||||
|
||||
reReader, err := diskbufferreader.New(inputFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("could not create re-readable reader: %w", err)
|
||||
}
|
||||
defer reReader.Close()
|
||||
|
||||
chunkSkel := &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
}
|
||||
if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if err := reReader.Reset(); err != nil {
|
||||
return err
|
||||
}
|
||||
reReader.Stop()
|
||||
data, err := io.ReadAll(reReader)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to read file: %w", err)
|
||||
}
|
||||
chunksChan <- &sources.Chunk{
|
||||
SourceType: s.Type(),
|
||||
SourceName: s.name,
|
||||
SourceID: s.SourceID(),
|
||||
Data: data,
|
||||
SourceMetadata: &source_metadatapb.MetaData{
|
||||
Data: &source_metadatapb.MetaData_Filesystem{
|
||||
Filesystem: &source_metadatapb.Filesystem{
|
||||
File: sanitizer.UTF8(path),
|
||||
},
|
||||
},
|
||||
},
|
||||
Verify: s.verify,
|
||||
|
||||
if done {
|
||||
return nil
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -33,7 +33,7 @@ func TestSource_Scan(t *testing.T) {
|
||||
init: init{
|
||||
name: "this repo",
|
||||
connection: &sourcespb.Filesystem{
|
||||
Paths: []string{"."},
|
||||
Directories: []string{"."},
|
||||
},
|
||||
verify: true,
|
||||
},
|
||||
|
||||
@@ -137,7 +137,7 @@ func TestSource_Scan(t *testing.T) {
|
||||
}
|
||||
chunksCh := make(chan *sources.Chunk, 1)
|
||||
go func() {
|
||||
assert.NoError(t, s.Chunks(ctx, chunksCh))
|
||||
s.Chunks(ctx, chunksCh)
|
||||
}()
|
||||
gotChunk := <-chunksCh
|
||||
gotChunk.Data = nil
|
||||
|
||||
@@ -96,8 +96,8 @@ type GitlabConfig struct {
|
||||
|
||||
// FilesystemConfig defines the optional configuration for a filesystem source.
|
||||
type FilesystemConfig struct {
|
||||
// Paths is the list of files and directories to scan.
|
||||
Paths []string
|
||||
// Directories is the list of directories to scan.
|
||||
Directories []string
|
||||
// Filter is the filter to use to scan the source.
|
||||
Filter *common.Filter
|
||||
}
|
||||
|
||||
@@ -111,15 +111,6 @@ message Gitlab {
|
||||
int64 line = 7;
|
||||
}
|
||||
|
||||
message GoogleDrive {
|
||||
string file = 1;
|
||||
string link = 2;
|
||||
string email = 3;
|
||||
string timestamp = 4;
|
||||
bool shared = 5;
|
||||
string last_modified_by = 6;
|
||||
}
|
||||
|
||||
message GCS {
|
||||
string bucket = 1;
|
||||
string file = 2;
|
||||
|
||||
+1
-11
@@ -40,7 +40,6 @@ enum SourceType {
|
||||
SOURCE_TYPE_SYSLOG = 25;
|
||||
SOURCE_TYPE_PUBLIC_EVENT_MONITORING = 26;
|
||||
SOURCE_TYPE_SLACK_REALTIME = 27;
|
||||
SOURCE_TYPE_GOOGLE_DRIVE = 28;
|
||||
}
|
||||
|
||||
message LocalSource {
|
||||
@@ -116,11 +115,8 @@ message ECR {
|
||||
repeated string registries = 2;
|
||||
}
|
||||
|
||||
message Filesystem {
|
||||
// DEPRECATED: directories is deprecated and can be removed / renamed to
|
||||
// paths when we no longer depend on the name in enterprise configs.
|
||||
message Filesystem{
|
||||
repeated string directories = 1;
|
||||
repeated string paths = 2;
|
||||
}
|
||||
|
||||
message GCS {
|
||||
@@ -168,12 +164,6 @@ message GitHub {
|
||||
repeated string includeRepos = 12;
|
||||
}
|
||||
|
||||
message GoogleDrive {
|
||||
oneof credential {
|
||||
string refresh_token = 1;
|
||||
}
|
||||
}
|
||||
|
||||
message JIRA {
|
||||
string endpoint = 1 [(validate.rules).string.uri_ref = true];
|
||||
oneof credential {
|
||||
|
||||
Reference in New Issue
Block a user