Testing

Support filtering detectors by version (#1150 )
* Adjust types to use DetectorID struct * Parse versions with detector include and exclude input * Update detectors filter to use version Co-authored-by: steeeve <steve@trufflesec.com> * Implement Versioner for github, gitlab, and npm detectors Co-authored-by: steeeve <steve@trufflesec.com> --------- Co-authored-by: steeeve <steve@trufflesec.com>
2023-03-07 08:39:18 -08:00 · 2023-03-02 16:33:56 -06:00 · 2023-03-02 09:57:38 -08:00 · 2023-03-02 11:32:37 -06:00 · 2023-03-01 09:00:12 -08:00 · 2023-03-01 08:58:36 -08:00
74 changed files with 1992 additions and 1014 deletions
@@ -1,16 +1,9 @@
 name: Release
 # on:
 #   push:
 #     tags:
 #       - 'v*'
 on:
  push:
    tags:
      - v*
    branches:
      - main
 permissions:
  contents: write
@@ -1,4 +1,7 @@
 .idea
 dist
 .env
-*.test
+*.test
 # binary
 trufflehog
@@ -0,0 +1,6 @@
 - id: trufflehog
  name: TruffleHog
  description: Detect secrets in your data with TruffleHog.
  entry: trufflehog git file://. --since-commit HEAD --only-verified --fail
  language: golang
  pass_filenames: false
@@ -100,6 +100,12 @@ trufflehog s3 --bucket=<bucket name> --only-verified
 docker run --rm -v "$HOME/.ssh:/root/.ssh:ro" trufflesecurity/trufflehog:latest git ssh://github.com/trufflesecurity/test_keys
 ```
 ### Example 6: Scan individual files or directories
 ```bash
 trufflehog filesystem path/to/file1.txt path/to/file2.txt path/to/dir
 ```
 # What's new in v3?
 TruffleHog v3 is a complete rewrite in Go with many new powerful features.
@@ -167,10 +173,10 @@ TruffleHog has a sub-command for each source of data that you may want to scan:
 - github
 - gitlab
 - S3
- filesystem
+- filesystem (files and directories)
 - syslog
 - circleci
- file and stdin (coming soon)
+- stdin (coming soon)
 Each subcommand can have options that you can see with the `--help` flag provided to the sub command:
@@ -184,6 +190,7 @@ Flags:
      --help                     Show context-sensitive help (also try --help-long and --help-man).
      --debug                    Run in debug mode.
      --trace                    Run in trace mode.
      --profile                  Enables profiling and sets a pprof and fgprof server on :18066.
  -j, --json                     Output in JSON format.
      --json-legacy              Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.
      --concurrency=10           Number of concurrent workers.
@@ -311,7 +318,7 @@ detectors:
 ```
 ```
-$ trufflehog filesystem --directory /tmp --config config.yaml --only-verified
+$ trufflehog filesystem /tmp --config config.yaml --only-verified
 🐷🔑🐷  TruffleHog. Unearth your secrets. 🐷🔑🐷
 Found verified result 🐷🔑
@@ -11,7 +11,7 @@ replace github.com/gitleaks/go-gitdiff => github.com/trufflesecurity/go-gitdiff
 require (
 	cloud.google.com/go/secretmanager v1.10.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11
-	github.com/TheZeroSlave/zapsentry v1.12.0
+	github.com/TheZeroSlave/zapsentry v1.14.0
 	github.com/aws/aws-sdk-go v1.44.83
 	github.com/bill-rich/disk-buffer-reader v0.1.7
 	github.com/bill-rich/go-syslog v0.0.0-20220413021637-49edb52a574c
@@ -45,19 +45,18 @@ require (
 	github.com/mholt/archiver/v4 v4.0.0-alpha.7
 	github.com/paulbellamy/ratecounter v0.2.0
 	github.com/pkg/errors v0.9.1
-	github.com/rabbitmq/amqp091-go v1.6.0
+	github.com/rabbitmq/amqp091-go v1.7.0
 	github.com/sergi/go-diff v1.3.1
-	github.com/stretchr/testify v1.8.1
+	github.com/stretchr/testify v1.8.2
 	github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502
-	github.com/xanzy/go-gitlab v0.78.0
+	github.com/xanzy/go-gitlab v0.80.2
-	go.mongodb.org/mongo-driver v1.11.1
+	go.mongodb.org/mongo-driver v1.11.2
 	go.uber.org/zap v1.24.0
 	golang.org/x/crypto v0.6.0
 	golang.org/x/exp v0.0.0-20221018205818-5c77f4b2bbd7
-	golang.org/x/oauth2 v0.4.0
+	golang.org/x/oauth2 v0.5.0
 	golang.org/x/sync v0.1.0
 	golang.org/x/text v0.7.0
 	google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/alecthomas/kingpin.v2 v2.2.6
 	gopkg.in/h2non/gock.v1 v1.1.2
@@ -121,6 +120,7 @@ require (
 	github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect
 	github.com/onsi/ginkgo v1.16.5 // indirect
 	github.com/onsi/gomega v1.23.0 // indirect
 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
 	github.com/pierrec/lz4/v4 v4.1.14 // indirect
 	github.com/pjbgf/sha1cd v0.2.3 // indirect
 	github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirect
@@ -138,12 +138,13 @@ require (
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
-	golang.org/x/net v0.6.0 // indirect
+	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/sys v0.5.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	golang.org/x/tools v0.3.0 // indirect
 	google.golang.org/api v0.103.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect
 	google.golang.org/grpc v1.51.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -37,8 +37,8 @@ github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VM
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4 h1:ra2OtmuW0AE5csawV4YXMNGNQQXvLRps3z2Z59OPO+I=
 github.com/ProtonMail/go-crypto v0.0.0-20221026131551-cf6655e29de4/go.mod h1:UBYPn8k0D56RtnR8RFQMjmh4KrZzWJ5o7Z9SYjossQ8=
-github.com/TheZeroSlave/zapsentry v1.12.0 h1:hhfwvD7pQbnCIOvAEGM4NXLiaX4b6W6ZWNtL6awSBGc=
+github.com/TheZeroSlave/zapsentry v1.14.0 h1:z6H0lIyKVpdXsP5K8J11b4/+V+QCDLDslZmtXsnduMk=
-github.com/TheZeroSlave/zapsentry v1.12.0/go.mod h1:00uO/VpPrSJG/XigAfTi0F4WMFIw2DmP/IDVUhPBvNw=
+github.com/TheZeroSlave/zapsentry v1.14.0/go.mod h1:0hUR1Et5tSroEYhWOZcxE2iBqcFkuG7jGGlUmOA7XAs=
 github.com/acomagu/bufpipe v1.0.3 h1:fxAGrHZTgQ9w5QqVItgzwj235/uYZYgbXitB+dLupOk=
 github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
@@ -281,6 +281,8 @@ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=
 github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=
 github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
 github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
 github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
 github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
@@ -295,8 +297,8 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/rabbitmq/amqp091-go v1.6.0 h1:fiFHcMcP+5/A+35MXniiagho1lEXX4+jpHGR+q+LeBU=
+github.com/rabbitmq/amqp091-go v1.7.0 h1:V5CF5qPem5OGSnEo8BoSbsDGwejg6VUJsKEdneaoTUo=
-github.com/rabbitmq/amqp091-go v1.6.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
+github.com/rabbitmq/amqp091-go v1.7.0/go.mod h1:wfClAtY0C7bOHxd3GjmF26jEHn+rR/0B3+YV+Vn9/NI=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
@@ -320,8 +322,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502 h1:34icjjmqJ2HPjrSuJYEkdZ+0ItmGQAQ75cRHIiftIyE=
 github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8=
 github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw=
@@ -333,8 +336,8 @@ github.com/trufflesecurity/overseer v1.1.7-custom5/go.mod h1:nT9w37AiO1Nop2VhVhN
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8=
 github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
-github.com/xanzy/go-gitlab v0.78.0 h1:8jUHfQVAprG04Av5g0PxVd3CNsZ5hCbojIax7Hba1mE=
+github.com/xanzy/go-gitlab v0.80.2 h1:CH1Q7NDklqZllox4ICVF4PwlhQGfPtE+w08Jsb74ZX0=
-github.com/xanzy/go-gitlab v0.78.0/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M=
+github.com/xanzy/go-gitlab v0.80.2/go.mod h1:DlByVTSXhPsJMYL6+cm8e8fTJjeBmhrXdC/yvkKKt6M=
 github.com/xanzy/ssh-agent v0.3.3 h1:+/15pJfg/RsTxqYcX6fHqOXZwwMP+2VyYWJeWM2qQFM=
 github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI1Bc68Uw=
 github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
@@ -350,8 +353,8 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
-go.mongodb.org/mongo-driver v1.11.1 h1:QP0znIRTuL0jf1oBQoAoM0C6ZJfBK4kx0Uumtv1A7w8=
+go.mongodb.org/mongo-driver v1.11.2 h1:+1v2rDQUWNcGW7/7E0Jvdz51V38XXxJfhzbV17aNHCw=
-go.mongodb.org/mongo-driver v1.11.1/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
+go.mongodb.org/mongo-driver v1.11.2/go.mod h1:s7p5vEtfbeR1gYi6pnj3c3/urpbLv2T5Sfd6Rp2HBB8=
 go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
 go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
@@ -409,11 +412,11 @@ golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q=
+golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
+golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
-golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M=
+golang.org/x/oauth2 v0.5.0 h1:HuArIo48skDwlrvM3sEdHXElYslAMsf3KwRkkW4MC4s=
-golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec=
+golang.org/x/oauth2 v0.5.0/go.mod h1:9/XBHVqLaWO3/BRHs5jbpYCnOZVjj5V0ndyaAM7KB4I=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -9,6 +9,8 @@ import (
 	"text/template"
 	"github.com/go-errors/errors"
 	"golang.org/x/text/cases"
 	"golang.org/x/text/language"
 	"gopkg.in/alecthomas/kingpin.v2"
 )
@@ -24,7 +26,7 @@ func main() {
 	log.SetPrefix("😲 [generate] ")
 	kingpin.MustParse(app.Parse(os.Args[1:]))
-	nameTitle = strings.Title(*name)
+	nameTitle = cases.Title(language.AmericanEnglish).String(*name)
 	nameLower = strings.ToLower(*name)
 	nameUpper = strings.ToUpper(*name)
@@ -65,7 +67,7 @@ type templateJob struct {
 }
 func mustWriteTemplates(jobs []templateJob) {
-	log.Printf("Generating %s %s\n", strings.Title(*kind), nameTitle)
+	log.Printf("Generating %s %s\n", cases.Title(language.AmericanEnglish).String(*kind), nameTitle)
 	// Make the folder.
 	log.Printf("Creating folder %s\n", folderPath())
@@ -84,7 +86,7 @@ func mustWriteTemplates(jobs []templateJob) {
 		for _, rplString := range job.ReplaceString {
 			tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToLower(rplString), "<<.NameLower>>")
-			tmplRaw = strings.ReplaceAll(tmplRaw, strings.Title(rplString), "<<.NameTitle>>")
+			tmplRaw = strings.ReplaceAll(tmplRaw, cases.Title(language.AmericanEnglish).String(rplString), "<<.NameTitle>>")
 			tmplRaw = strings.ReplaceAll(tmplRaw, strings.ToUpper(rplString), "<<.NameUpper>>")
 		}
@@ -173,7 +173,9 @@ func main() {
 		}
 		for _, repo := range strings.Split(*scanCmdRepo, ",") {
-			sem.Acquire(ctx, 1)
+			if err := sem.Acquire(ctx, 1); err != nil {
 				logFatal(err, "timed out waiting for semaphore")
 			}
 			wgChunkers.Add(1)
 			go func(r string) {
 				defer sem.Release(1)
@@ -21,10 +21,12 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/config"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/context"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/log"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/output"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
@@ -36,6 +38,7 @@ var (
 	cmd              string
 	debug            = cli.Flag("debug", "Run in debug mode.").Bool()
 	trace            = cli.Flag("trace", "Run in trace mode.").Bool()
 	profile          = cli.Flag("profile", "Enables profiling and sets a pprof and fgprof server on :18066.").Bool()
 	jsonOut          = cli.Flag("json", "Output in JSON format.").Short('j').Bool()
 	jsonLegacy       = cli.Flag("json-legacy", "Use the pre-v3.0 JSON format. Only works with git, gitlab, and github sources.").Bool()
 	concurrency      = cli.Flag("concurrency", "Number of concurrent workers.").Default(strconv.Itoa(runtime.NumCPU())).Int()
@@ -50,6 +53,8 @@ var (
 	archiveMaxSize       = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
 	archiveMaxDepth      = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
 	archiveTimeout       = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
 	includeDetectors     = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
 	excludeDetectors     = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
 	gitScan             = cli.Command("git", "Find credentials in git repositories.")
 	gitScanURI          = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
@@ -82,8 +87,10 @@ var (
 	gitlabScanIncludePaths = gitlabScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
 	gitlabScanExcludePaths = gitlabScan.Flag("exclude-paths", "Path to file with newline separated regexes for files to exclude in scan.").Short('x').String()
-	filesystemScan        = cli.Command("filesystem", "Find credentials in a filesystem.")
+	filesystemScan  = cli.Command("filesystem", "Find credentials in a filesystem.")
-	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Required().Strings()
+	filesystemPaths = filesystemScan.Arg("path", "Path to file or directory to scan.").Strings()
 	// DEPRECATED: --directory is deprecated in favor of arguments.
 	filesystemDirectories = filesystemScan.Flag("directory", "Path to directory to scan. You can repeat this flag.").Strings()
 	// TODO: Add more filesystem scan options. Currently only supports scanning a list of directories.
 	// filesystemScanRecursive = filesystemScan.Flag("recursive", "Scan recursively.").Short('r').Bool()
 	filesystemScanIncludePaths = filesystemScan.Flag("include-paths", "Path to file with newline separated regexes for files to include in scan.").Short('i').String()
@@ -177,7 +184,7 @@ func run(state overseer.State) {
 		*concurrency = 1
 	}
-	if *debug {
+	if *profile {
 		go func() {
 			router := mux.NewRouter()
 			router.PathPrefix("/debug/pprof").Handler(http.DefaultServeMux)
@@ -208,11 +215,62 @@ func run(state overseer.State) {
 		handlers.SetArchiveMaxTimeout(*archiveTimeout)
 	}
 	// Build include and exclude detector filter sets.
 	var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID
 	{
 		includeList, err := config.ParseDetectors(*includeDetectors)
 		if err != nil {
 			// Exit if there was an error to inform the user of the misconfiguration.
 			logFatal(err, "invalid include list detector configuration")
 		}
 		excludeList, err := config.ParseDetectors(*excludeDetectors)
 		if err != nil {
 			// Exit if there was an error to inform the user of the misconfiguration.
 			logFatal(err, "invalid exclude list detector configuration")
 		}
 		includeDetectorTypes = detectorTypeToMap(includeList)
 		excludeDetectorTypes = detectorTypeToMap(excludeList)
 	}
 	includeFilter := func(d detectors.Detector) bool {
 		id, ok := includeDetectorTypes[d.Type()]
 		if id.Version == 0 {
 			return ok
 		}
 		versionD, ok := d.(detectors.Versioner)
 		if !ok {
 			// Error: version provided but not a detectors.Versioner
 			logFatal(
 				fmt.Errorf("version provided but detector does not have a version"),
 				"invalid include list detector configuration",
 				"detector", id,
 			)
 		}
 		return versionD.Version() == id.Version
 	}
 	excludeFilter := func(d detectors.Detector) bool {
 		id, ok := excludeDetectorTypes[d.Type()]
 		if id.Version == 0 {
 			return !ok
 		}
 		versionD, ok := d.(detectors.Versioner)
 		if !ok {
 			// Error: version provided but not a detectors.Versioner
 			logFatal(
 				fmt.Errorf("version provided but detector does not have a version"),
 				"invalid exclude list detector configuration",
 				"detector", id,
 			)
 		}
 		return versionD.Version() != id.Version
 	}
 	e := engine.Start(ctx,
 		engine.WithConcurrency(*concurrency),
 		engine.WithDecoders(decoders.DefaultDecoders()...),
 		engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
 		engine.WithDetectors(!*noVerification, conf.Detectors...),
 		engine.WithFilterDetectors(includeFilter),
 		engine.WithFilterDetectors(excludeFilter),
 		engine.WithFilterUnverified(*filterUnverified),
 	)
@@ -286,10 +344,15 @@ func run(state overseer.State) {
 		if err != nil {
 			logFatal(err, "could not create filter")
 		}
-
+		if len(*filesystemDirectories) > 0 {
 			ctx.Logger().Info("--directory flag is deprecated, please pass directories as arguments")
 		}
 		paths := make([]string, 0, len(*filesystemPaths)+len(*filesystemDirectories))
 		paths = append(paths, *filesystemPaths...)
 		paths = append(paths, *filesystemDirectories...)
 		cfg := sources.FilesystemConfig{
-			Directories: *filesystemDirectories,
+			Paths:  paths,
-			Filter:      filter,
+			Filter: filter,
 		}
 		if err = e.ScanFileSystem(ctx, cfg); err != nil {
 			logFatal(err, "Failed to scan filesystem")
@@ -389,3 +452,11 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
 		os.Exit(0)
 	}
 }
 func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {
 	output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))
 	for _, d := range detectors {
 		output[d.ID] = d
 	}
 	return output
 }
@@ -7,9 +7,9 @@ import (
 	"time"
 	secretmanager "cloud.google.com/go/secretmanager/apiv1"
 	"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
 	"github.com/joho/godotenv"
 	"github.com/pkg/errors"
 	secretmanagerpb "google.golang.org/genproto/googleapis/cloud/secretmanager/v1"
 )
 type Secret struct{ kv map[string]string }
@@ -0,0 +1,183 @@
 package config
 import (
 	"fmt"
 	"sort"
 	"strconv"
 	"strings"
 	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 var (
 	specialGroups = map[string][]DetectorID{
 		"all": allDetectors(),
 	}
 	detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))
 	validDetectors    = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))
 	maxDetectorType   dpb.DetectorType
 )
 // Setup package local global variables.
 func init() {
 	for k, v := range dpb.DetectorType_value {
 		dt := dpb.DetectorType(v)
 		detectorTypeValue[strings.ToLower(k)] = dt
 		validDetectors[dt] = struct{}{}
 		if dt > maxDetectorType {
 			maxDetectorType = dt
 		}
 	}
 }
 type DetectorID struct {
 	ID      dpb.DetectorType
 	Version int
 }
 // ParseDetectors parses user supplied string into a list of detectors types.
 // "all" will return the list of all available detectors. The input is comma
 // separated and may use the case-insensitive detector name defined in the
 // protobuf, or the protobuf enum number. A range may be used as well in the
 // form "start-end". Order is preserved and duplicates are ignored.
 func ParseDetectors(input string) ([]DetectorID, error) {
 	var output []DetectorID
 	seenDetector := map[DetectorID]struct{}{}
 	for _, item := range strings.Split(input, ",") {
 		item = strings.TrimSpace(item)
 		if item == "" {
 			continue
 		}
 		detectors, ok := specialGroups[strings.ToLower(item)]
 		if !ok {
 			var err error
 			detectors, err = asRange(item)
 			if err != nil {
 				return nil, err
 			}
 		}
 		for _, d := range detectors {
 			if _, ok := seenDetector[d]; ok {
 				continue
 			}
 			seenDetector[d] = struct{}{}
 			output = append(output, d)
 		}
 	}
 	return output, nil
 }
 func (id DetectorID) String() string {
 	name := dpb.DetectorType_name[int32(id.ID)]
 	if name == "" {
 		name = "<invalid ID>"
 	}
 	if id.Version == 0 {
 		return name
 	}
 	return fmt.Sprintf("%s.v%d", name, id.Version)
 }
 // allDetectors reutrns an ordered slice of all detector types.
 func allDetectors() []DetectorID {
 	all := make([]DetectorID, 0, len(dpb.DetectorType_name))
 	for id := range dpb.DetectorType_name {
 		all = append(all, DetectorID{ID: dpb.DetectorType(id)})
 	}
 	sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })
 	return all
 }
 // asRange converts a single input into a slice of detector types. If the input
 // is not in range format, a slice of length 1 is returned. Unbounded ranges
 // are allowed.
 func asRange(input string) ([]DetectorID, error) {
 	// Check if it's a single detector type.
 	dt, err := asDetectorID(input)
 	if err == nil {
 		return []DetectorID{dt}, nil
 	}
 	// Check if it's a range; if not return the error from above.
 	start, end, found := strings.Cut(input, "-")
 	if !found {
 		return nil, err
 	}
 	start, end = strings.TrimSpace(start), strings.TrimSpace(end)
 	// Convert the range start and end to a DetectorType.
 	dtStart, err := asDetectorID(start)
 	if err != nil {
 		return nil, err
 	}
 	dtEnd, err := asDetectorID(end)
 	// If end is empty it's an unbounded range.
 	if err != nil && end != "" {
 		return nil, err
 	}
 	if end == "" {
 		dtEnd.ID = maxDetectorType
 	}
 	// Ensure these ranges don't have versions.
 	if dtEnd.Version != 0 || dtStart.Version != 0 {
 		return nil, fmt.Errorf("versions within ranges are not supported: %s", input)
 	}
 	step := dpb.DetectorType(1)
 	if dtStart.ID > dtEnd.ID {
 		step = -1
 	}
 	var output []DetectorID
 	for dt := dtStart.ID; dt != dtEnd.ID; dt += step {
 		if _, ok := validDetectors[dt]; !ok {
 			continue
 		}
 		output = append(output, DetectorID{ID: dt})
 	}
 	return append(output, dtEnd), nil
 }
 // asDetectorID converts the case-insensitive input into a DetectorID.
 // Name or ID may be used.
 func asDetectorID(input string) (DetectorID, error) {
 	if input == "" {
 		return DetectorID{}, fmt.Errorf("empty detector")
 	}
 	var detectorID DetectorID
 	// Separate the version if there is one.
 	if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {
 		parsedVersion, err := parseVersion(version)
 		if err != nil {
 			return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)
 		}
 		detectorID.Version = parsedVersion
 		// Because there was a version, the detector type input is the part before the '.'
 		input = detector
 	}
 	// Check if it's a named detector.
 	if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
 		detectorID.ID = dt
 		return detectorID, nil
 	}
 	// Check if it's a detector ID.
 	if i, err := strconv.ParseInt(input, 10, 32); err == nil {
 		dt := dpb.DetectorType(i)
 		if _, ok := validDetectors[dt]; !ok {
 			return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)
 		}
 		detectorID.ID = dt
 		return detectorID, nil
 	}
 	return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)
 }
 func parseVersion(v string) (int, error) {
 	if !strings.HasPrefix(strings.ToLower(v), "v") {
 		return 0, fmt.Errorf("version must start with 'v'")
 	}
 	version := strings.TrimLeft(v, "vV")
 	return strconv.Atoi(version)
 }
@@ -0,0 +1,47 @@
 package config
 import (
 	"testing"
 	"github.com/stretchr/testify/assert"
 	dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 )
 func TestDetectorParsing(t *testing.T) {
 	tests := map[string]struct {
 		input    string
 		expected []DetectorID
 	}{
 		"all":                       {"AlL", allDetectors()},
 		"trailing range":            {"0-", allDetectors()},
 		"all after 1":               {"1-", allDetectors()[1:]},
 		"named and valid range":     {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
 		"duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},
 		"named range":               {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},
 		"range preserved":           {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},
 		"reverse range":             {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},
 		"range preserved with all":  {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
 		"empty list item":           {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},
 		"invalid end range":         {"0-1337", nil},
 		"invalid name":              {"foo", nil},
 		"negative":                  {"-1", nil},
 		"github.v1":                 {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},
 		"gitlab.v100":               {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},
 		"range with versions":       {"github.v2 - gitlab.v1", nil},
 		"invalid version no v":      {"gitlab.2", nil},
 		"invalid version no number": {"gitlab.github", nil},
 		"capital V is fine":         {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},
 		"id number with version":    {"8.v2", []DetectorID{{ID: 8, Version: 2}}},
 	}
 	for name, tt := range tests {
 		t.Run(name, func(t *testing.T) {
 			got, gotErr := ParseDetectors(tt.input)
 			if tt.expected == nil {
 				assert.Error(t, gotErr)
 				return
 			}
 			assert.Equal(t, tt.expected, got)
 		})
 	}
 }
@@ -12,6 +12,7 @@ import (
 	"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
 	"golang.org/x/sync/errgroup"
 )
 // The maximum number of matches from one chunk. This const is used when
@@ -82,68 +83,99 @@ func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []b
 	// ]
 	matches := permutateMatches(regexMatches)
-	// Create result object and test for verification.
+	g := new(errgroup.Group)
 	for _, match := range matches {
 		if common.IsDone(ctx) {
 			// TODO: Log we're possibly leaving out results.
 			return results, nil
 		}
 		var raw string
 		for _, values := range match {
 			// values[0] contains the entire regex match.
 			raw += values[0]
 		}
 		result := detectors.Result{
 			DetectorType: detectorspb.DetectorType_CustomRegex,
 			Raw:          []byte(raw),
 		}
-		if !verify {
+	// Create result object and test for verification.
-			results = append(results, result)
+	resultsCh := make(chan detectors.Result, maxTotalMatches)
-			continue
+	for _, match := range matches {
-		}
+		match := match
-		// Verify via webhook.
+		g.Go(func() error {
-		jsonBody, err := json.Marshal(map[string]map[string][]string{
+			return c.createResults(ctx, match, verify, resultsCh)
 			c.GetName(): match,
 		})
-		if err != nil {
+	}
-			continue
+
-		}
+	if err := g.Wait(); err != nil {
-		// Try each config until we successfully verify.
+		return nil, err
-		for _, verifyConfig := range c.GetVerify() {
+	}
-			if common.IsDone(ctx) {
+	close(resultsCh)
-				// TODO: Log we're possibly leaving out results.
+
-				return results, nil
+	for result := range resultsCh {
 			}
 			req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
 			if err != nil {
 				continue
 			}
 			for _, header := range verifyConfig.GetHeaders() {
 				key, value, found := strings.Cut(header, ":")
 				if !found {
 					// Should be unreachable due to validation.
 					continue
 				}
 				req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
 			}
 			res, err := httpClient.Do(req)
 			if err != nil {
 				continue
 			}
 			// TODO: Read response body.
 			res.Body.Close()
 			if res.StatusCode == http.StatusOK {
 				result.Verified = true
 				break
 			}
 		}
 		results = append(results, result)
 	}
 	return results, nil
 }
 func (c *customRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
 	if common.IsDone(ctx) {
 		// TODO: Log we're possibly leaving out results.
 		return ctx.Err()
 	}
 	var raw string
 	for _, values := range match {
 		// values[0] contains the entire regex match.
 		raw += values[0]
 	}
 	result := detectors.Result{
 		DetectorType: detectorspb.DetectorType_CustomRegex,
 		Raw:          []byte(raw),
 	}
 	if !verify {
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		case results <- result:
 			return nil
 		}
 	}
 	// Verify via webhook.
 	jsonBody, err := json.Marshal(map[string]map[string][]string{
 		c.GetName(): match,
 	})
 	if err != nil {
 		// This should never happen, but if it does, return nil to not
 		// disrupt other verification.
 		return nil
 	}
 	// Try each config until we successfully verify.
 	for _, verifyConfig := range c.GetVerify() {
 		if common.IsDone(ctx) {
 			// TODO: Log we're possibly leaving out results.
 			return ctx.Err()
 		}
 		req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
 		if err != nil {
 			continue
 		}
 		for _, header := range verifyConfig.GetHeaders() {
 			key, value, found := strings.Cut(header, ":")
 			if !found {
 				// Should be unreachable due to validation.
 				continue
 			}
 			req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
 		}
 		res, err := httpClient.Do(req)
 		if err != nil {
 			continue
 		}
 		// TODO: Read response body.
 		res.Body.Close()
 		if res.StatusCode == http.StatusOK {
 			result.Verified = true
 			break
 		}
 	}
 	select {
 	case <-ctx.Done():
 		return ctx.Err()
 	case results <- result:
 		return nil
 	}
 }
 func (c *customRegexWebhook) Keywords() []string {
 	return c.GetKeywords()
 }
@@ -2,7 +2,6 @@ package abbysale
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -86,7 +86,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				s1.Verified = isValid
 				// If there is a valid one, we need to stop iterating now and return the valid result
-				if isValid == true {
+				if isValid {
 					break
 				}
 			}
@@ -2,7 +2,6 @@ package brandfetch
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -56,7 +55,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -21,7 +21,7 @@ var (
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
 	keyPat  = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "key", "automate", "local"}) + `\b([0-9a-zA-Z]{20})\b`)
-	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "user", "automate", "local"}) + `\b(^[a-zA-Z\d]+([._-]?[a-zA-Z\d]+)*[a-zA-Z\d]+$)\b`)
+	userPat = regexp.MustCompile(detectors.PrefixRegex([]string{"browserstack", "user", "automate", "local"}) + `\b([a-zA-Z\d]{3,18}[._-]?[a-zA-Z\d]{6})\b`)
 )
 // Keywords are used for efficiently pre-filtering chunks.
@@ -51,6 +51,7 @@ func TestBrowserStack_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_BrowserStack,
 					Verified:     true,
 					RawV2:        []byte(fmt.Sprintf("%s%s", secret, secretUser)),
 				},
 			},
 			wantErr: false,
@@ -67,6 +68,7 @@ func TestBrowserStack_FromChunk(t *testing.T) {
 				{
 					DetectorType: detectorspb.DetectorType_BrowserStack,
 					Verified:     false,
 					RawV2:        []byte(fmt.Sprintf("%s%s", inactiveSecret, secretUser)),
 				},
 			},
 			wantErr: false,
@@ -2,7 +2,6 @@ package clockify
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("content-type", "application/json")
-			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -66,7 +66,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				}
 			} else {
 			}
 		}
@@ -25,6 +25,12 @@ type Detector interface {
 	Type() detectorspb.DetectorType
 }
 // Versioner is an optional interface that a detector can implement to
 // differentiate instances of the same detector type.
 type Versioner interface {
 	Version() int
 }
 type Result struct {
 	// DetectorType is the type of Detector.
 	DetectorType detectorspb.DetectorType
@@ -2,7 +2,6 @@ package everhour
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -61,7 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Fastly-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Fastly-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				bodyBytes, err := io.ReadAll(res.Body)
@@ -2,7 +2,6 @@ package figmapersonalaccesstoken
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Figma-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Figma-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package fulcrum
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("X-ApiToken", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-ApiToken", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -44,10 +44,7 @@ func New() Scanner {
 }
 type Scanner struct {
-	wordList             []string
+	excludeMatchers []*regexp.Regexp
 	badList              []string
 	programmingBookWords []string
 	excludeMatchers      []*regexp.Regexp
 }
 // Ensure the Scanner satisfies the interface at compile time.
@@ -15,8 +15,11 @@ import (
 type Scanner struct{}
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 func (s Scanner) Version() int { return 2 }
 var (
 	// Oauth token
@@ -14,8 +14,11 @@ import (
 type Scanner struct{}
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 func (s Scanner) Version() int { return 1 }
 var (
 	// Oauth token
@@ -38,7 +41,7 @@ type userRes struct {
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
-	return []string{"github","gh","pat"}
+	return []string{"github", "gh", "pat"}
 }
 // FromData will find and optionally verify GitHub secrets in a given set of bytes.
@@ -14,8 +14,11 @@ import (
 type Scanner struct{}
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 func (s Scanner) Version() int { return 1 }
 var (
 	keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)
@@ -13,8 +13,11 @@ import (
 type Scanner struct{}
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 func (*Scanner) Version() int { return 2 }
 var (
 	keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)
@@ -2,7 +2,6 @@ package impala
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -56,11 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
 					// This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
 					// if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 					// 	continue
 					// }
 				}
 			}
 		}
@@ -2,7 +2,6 @@ package keenio
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -61,7 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 					continue
 				}
 				req.Header.Add("Content-Type", "application/json")
-				req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+				req.Header.Add("Authorization", resMatch)
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
@@ -2,7 +2,6 @@ package newrelicpersonalapikey
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Api-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Api-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package newscatcher
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package nftport
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Authorization", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -14,8 +14,11 @@ import (
 type Scanner struct{}
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 func (s Scanner) Version() int { return 1 }
 var (
 	client = common.SaneHttpClient()
@@ -34,7 +37,7 @@ func (s Scanner) Keywords() []string {
 func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
 	dataStr := string(data)
-  	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+	matches := keyPat.FindAllStringSubmatch(dataStr, -1)
 	for _, match := range matches {
 		if len(match) != 2 {
 			continue
@@ -13,8 +13,11 @@ import (
 type Scanner struct{}
-// Ensure the Scanner satisfies the interface at compile time.
+// Ensure the Scanner satisfies the interfaces at compile time.
 var _ detectors.Detector = (*Scanner)(nil)
 var _ detectors.Versioner = (*Scanner)(nil)
 func (s Scanner) Version() int { return 2 }
 var (
 	client = common.SaneHttpClient()
@@ -2,7 +2,6 @@ package omnisend
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-API-KEY", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-API-KEY", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package openuv
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-access-token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-access-token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package pepipost
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("api_key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("api_key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package postman
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package postmark
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Accept", "application/json")
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Postmark-Server-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Postmark-Server-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package prospectio
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Authorization", resMatch)
 			req.Header.Add("Content-Type", "application/vnd.api+json; charset=utf-8")
 			res, err := client.Do(req)
 			if err == nil {
@@ -2,7 +2,6 @@ package pushbulletapikey
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Access-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Access-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package qubole
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -54,7 +53,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			}
 			req.Header.Add("Content-Type", "application/json")
 			req.Header.Add("Accept", "application/json")
-			req.Header.Add("X-AUTH-TOKEN", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-AUTH-TOKEN", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package rapidapi
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("x-rapidapi-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-rapidapi-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -16,7 +16,6 @@ type Scanner struct{}
 var _ detectors.Detector = (*Scanner)(nil)
 var (
 	client    = common.SaneHttpClient()
 	verifyURL = "https://api.rechargeapps.com/token_information"
 	// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
@@ -2,7 +2,6 @@ package scalewaykey
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("X-Auth-Token", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Auth-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package sendinbluev2
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -51,7 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -65,19 +65,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				if strings.Contains(body, "app_url") {
 					s1.Verified = true
 				} else {
 					// if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
 					// 	continue
 					// }
 				}
 			}
 		}
 		results = append(results, s1)
 	}
 	return results, nil
 }
@@ -2,7 +2,6 @@ package skrappio
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Content-Type", "application/json")
-			req.Header.Add("X-Access-Key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("X-Access-Key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -56,6 +56,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			req.Header.Set("X-Sf-Token", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
 				if res.StatusCode >= 200 && res.StatusCode < 300 {
 					s1.Verified = true
 				} else {
@@ -2,7 +2,6 @@ package stormglass
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Authorization", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Authorization", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -2,7 +2,6 @@ package tatumio
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -53,7 +52,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				continue
 			}
 			req.Header.Add("Accept", "application/vnd.tatumio+json; version=3")
-			req.Header.Add("x-api-key", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("x-api-key", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -26,10 +26,6 @@ var (
 	client = common.SaneHttpClient()
 )
 type proxyRes struct {
 	Verified bool `json:"verified"`
 }
 // Keywords are used for efficiently pre-filtering chunks.
 // Use identifiers in the secret preferably, or the provider name.
 func (s Scanner) Keywords() []string {
@@ -61,6 +61,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 				}
 				res, err := client.Do(req)
 				if err == nil {
 					defer res.Body.Close()
 					bodyBytes, err := io.ReadAll(res.Body)
 					if err != nil {
 						continue
@@ -2,7 +2,6 @@ package zonkafeedback
 import (
 	"context"
 	"fmt"
 	"net/http"
 	"regexp"
 	"strings"
@@ -52,7 +51,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
 			if err != nil {
 				continue
 			}
-			req.Header.Add("Z-API-TOKEN", fmt.Sprintf("%s", resMatch))
+			req.Header.Add("Z-API-TOKEN", resMatch)
 			res, err := client.Do(req)
 			if err == nil {
 				defer res.Body.Close()
@@ -9,6 +9,7 @@ import (
 	"sync/atomic"
 	"time"
 	ahocorasick "github.com/petar-dambovaliev/aho-corasick"
 	"google.golang.org/protobuf/proto"
 	"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -36,6 +37,10 @@ type Engine struct {
 	// If there are multiple unverified results for the same chunk for the same detector,
 	// only the first one will be kept.
 	filterUnverified bool
 	// prefilter is a ahocorasick struct used for doing efficient string
 	// matching given a set of words (keywords from the rules in the config)
 	prefilter ahocorasick.AhoCorasick
 }
 type EngineOption func(*Engine)
@@ -73,6 +78,31 @@ func WithFilterUnverified(filter bool) EngineOption {
 	}
 }
 // WithFilterDetectors applies a filter to the configured list of detectors. If
 // the filterFunc returns true, the detector will be included for scanning.
 // This option applies to the existing list of detectors configured, so the
 // order this option appears matters. All filtering happens before scanning.
 func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
 	return func(e *Engine) {
 		// If no detectors are configured, do nothing.
 		if e.detectors == nil {
 			return
 		}
 		e.detectors[true] = filterDetectors(filterFunc, e.detectors[true])
 		e.detectors[false] = filterDetectors(filterFunc, e.detectors[false])
 	}
 }
 func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
 	var output []detectors.Detector
 	for _, detector := range input {
 		if filterFunc(detector) {
 			output = append(output, detector)
 		}
 	}
 	return output
 }
 func Start(ctx context.Context, options ...EngineOption) *Engine {
 	e := &Engine{
 		chunks:          make(chan *sources.Chunk),
@@ -85,7 +115,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 	}
 	// Set defaults.
 	if e.concurrency == 0 {
 		numCPU := runtime.NumCPU()
 		ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
@@ -103,6 +132,23 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
 		e.detectors[false] = []detectors.Detector{}
 	}
 	// build ahocorasick prefilter for efficient string matching
 	// on keywords
 	keywords := []string{}
 	for _, d := range e.detectors[false] {
 		keywords = append(keywords, d.Keywords()...)
 	}
 	for _, d := range e.detectors[true] {
 		keywords = append(keywords, d.Keywords()...)
 	}
 	builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{
 		AsciiCaseInsensitive: true,
 		MatchOnlyWholeWords:  false,
 		MatchKind:            ahocorasick.LeftMostLongestMatch,
 		DFA:                  true,
 	})
 	e.prefilter = builder.Build(keywords)
 	ctx.Logger().V(2).Info("loaded decoders", "count", len(e.decoders))
 	ctx.Logger().V(2).Info("loaded detectors",
 		"total", len(e.detectors[true])+len(e.detectors[false]),
@@ -183,6 +229,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
 func (e *Engine) detectorWorker(ctx context.Context) {
 	for originalChunk := range e.chunks {
 		for chunk := range sources.Chunker(originalChunk) {
 			matchedKeywords := make(map[string]struct{})
 			atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))
 			for _, decoder := range e.decoders {
 				var decoderType detectorspb.DecoderType
@@ -199,21 +246,29 @@ func (e *Engine) detectorWorker(ctx context.Context) {
 				if decoded == nil {
 					continue
 				}
 				dataLower := strings.ToLower(string(decoded.Data))
 				matches := e.prefilter.FindAll(dataLower)
 				for _, m := range matches {
 					matchedKeywords[dataLower[m.Start():m.End()]] = struct{}{}
 				}
 				for verify, detectorsSet := range e.detectors {
 					for _, detector := range detectorsSet {
-						start := time.Now()
+						chunkContainsKeyword := false
 						foundKeyword := false
 						for _, kw := range detector.Keywords() {
-							if strings.Contains(dataLower, strings.ToLower(kw)) {
+							if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
-								foundKeyword = true
+								chunkContainsKeyword = true
 								break
 							}
 						}
-						if !foundKeyword {
+
 						if !chunkContainsKeyword {
 							continue
 						}
 						start := time.Now()
 						results, err := func() ([]detectors.Result, error) {
 							ctx, cancel := context.WithTimeout(ctx, time.Second*10)
 							defer cancel()
@@ -17,7 +17,7 @@ import (
 // ScanFileSystem scans a given file system.
 func (e *Engine) ScanFileSystem(ctx context.Context, c sources.FilesystemConfig) error {
 	connection := &sourcespb.Filesystem{
-		Directories: c.Directories,
+		Paths: c.Paths,
 	}
 	var conn anypb.Any
 	err := anypb.MarshalFrom(&conn, connection, proto.MarshalOptions{})
@@ -115,43 +115,9 @@ func (c1 *Commit) Equal(c2 *Commit) bool {
 }
 func (c *Parser) BeepBoop(commitHash string) (chan Commit, error) {
 	dummy := make(chan Commit)
 	cocmd := exec.Command("git", "checkout", commitHash)
 	cocmd.Dir = "/home/hrich/go/src/github.com/trufflesecurity/thog"
 	cocmd.Run()
 	go func() {
 		defer close(dummy)
 		args := []string{"-C", "/home/hrich/go/src/github.com/trufflesecurity/thog", "log", "-p", "-U5", "-2", "--date=format:%a %b %d %H:%M:%S %Y %z"}
 		cmd := exec.Command("git", args...)
 		commitChan, err := c.executeCommand(context.Background(), cmd)
 		if err != nil {
 			fmt.Println(err)
 		}
 		for commit := range commitChan {
 			if commit.Hash == commitHash {
 				continue
 			}
 			coolchan, _ := c.BeepBoop(commit.Hash)
 			for commit2 := range coolchan {
 				dummy <- commit2
 			}
 		}
 	}()
 	return dummy, nil
 }
 // RepoPath parses the output of the `git log` command for the `source` path.
 func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbreviatedLog bool) (chan Commit, error) {
-	args := []string{"-C", source, "log", "-p", "-U5", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
+	args := []string{"-C", source, "log", "--full-history", "--date=format:%a %b %d %H:%M:%S %Y %z"}
 	if abbreviatedLog {
 		args = append(args, "--diff-filter=AM")
 	}
 	if head != "" {
 		args = append(args, head)
 	} else {
@@ -165,12 +131,13 @@ func (c *Parser) RepoPath(ctx context.Context, source string, head string, abbre
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
-	return c.executeCommand(ctx, cmd)
+	return c.executeCommand(ctx, source, cmd)
 }
 // Unstaged parses the output of the `git diff` command for the `source` path.
 func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, error) {
-	args := []string{"-C", source, "diff", "-p", "-U5", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
+	// Provide the --cached flag to diff to get the diff of the staged changes.
 	args := []string{"-C", source, "diff", "-p", "-U5", "--cached", "--full-history", "--diff-filter=AM", "--date=format:%a %b %d %H:%M:%S %Y %z", "HEAD"}
 	cmd := exec.Command("git", args...)
@@ -179,11 +146,11 @@ func (c *Parser) Unstaged(ctx context.Context, source string) (chan Commit, erro
 		cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 	}
-	return c.executeCommand(ctx, cmd)
+	return c.executeCommand(ctx, source, cmd)
 }
 // executeCommand runs an exec.Cmd, reads stdout and stderr, and waits for the Cmd to complete.
-func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit, error) {
+func (c *Parser) executeCommand(ctx context.Context, source string, cmd *exec.Cmd) (chan Commit, error) {
 	commitChan := make(chan Commit, 64)
 	stdOut, err := cmd.StdoutPipe()
@@ -208,7 +175,7 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
 	}()
 	go func() {
-		c.fromReader(ctx, stdOut, commitChan)
+		c.fromReader(ctx, source, stdOut, commitChan)
 		if err := cmd.Wait(); err != nil {
 			ctx.Logger().V(2).Info("Error waiting for git command to complete.", "error", err)
 		}
@@ -217,11 +184,12 @@ func (c *Parser) executeCommand(ctx context.Context, cmd *exec.Cmd) (chan Commit
 	return commitChan, nil
 }
-func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit) {
+func (c *Parser) fromReader(ctx context.Context, source string, stdOut io.Reader, commitChan chan Commit) {
 	outReader := bufio.NewReader(stdOut)
 	var currentCommit *Commit
 	var currentDiff *Diff
 	lastCommit := ""
 	defer common.RecoverWithExit(ctx)
 	defer close(commitChan)
 	for {
@@ -232,6 +200,143 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 		if err != nil && len(line) == 0 {
 			break
 		}
 		// for i := 0; i < len(line); i++ {
 		// 	b := make([]byte, 1)
 		// 	n, err := stdOut.Read(b)
 		// 	if errors.Is(err, io.EOF) || n == 0 {
 		// 		return
 		// 	}
 		// 	line[i] = b[0]
 		// 	if b[0] == byte('\n') {
 		// 		line = line[:0]
 		// 		break
 		// 	}
 		// }
 		switch {
 		case isCommitLine(line):
 			// If there is a currentDiff, add it to currentCommit.
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
 			// If there is a currentCommit, send it to the channel.
 			if currentCommit != nil {
 				args := []string{"-C", source, "diff", "--date=format:%a %b %d %H:%M:%S %Y %z", currentCommit.Hash}
 				if lastCommit != "" {
 					args = append(args, lastCommit)
 				}
 				lastCommit = currentCommit.Hash
 				cmd := exec.Command("git", args...)
 				absPath, err := filepath.Abs(source)
 				if err == nil {
 					cmd.Env = append(cmd.Env, fmt.Sprintf("GIT_DIR=%s", filepath.Join(absPath, ".git")))
 				}
 				// stdnOut, err := cmd.StdoutPipe()
 				// if err != nil {
 				// 	return
 				// }
 				err = cmd.Start()
 				if err != nil {
 					return
 				}
 				_ = cmd.Wait()
 				//c.funkReader(ctx, stdnOut, commitChan, currentCommit)
 			}
 			// Create a new currentDiff and currentCommit
 			currentDiff = &Diff{}
 			currentCommit = &Commit{
 				Message: strings.Builder{},
 			}
 			// Check that the commit line contains a hash and set it.
 			if len(line) >= 47 {
 				currentCommit.Hash = string(line[7:47])
 			}
 		case isAuthorLine(line):
 			currentCommit.Author = strings.TrimRight(string(line[8:]), "\n")
 		case isDateLine(line):
 			date, err := time.Parse(c.dateFormat, strings.TrimSpace(string(line[6:])))
 			if err != nil {
 				ctx.Logger().V(2).Info("Could not parse date from git stream.", "error", err)
 			}
 			currentCommit.Date = date
 		case isModeLine(line):
 			// NoOp
 		case isIndexLine(line):
 			// NoOp
 		case isPlusFileLine(line):
 			currentDiff.PathB = strings.TrimRight(strings.TrimRight(string(line[6:]), "\n"), "\t") // Trim the newline and tab characters. https://github.com/trufflesecurity/trufflehog/issues/1060
 		case isMinusFileLine(line):
 			// NoOp
 		case isPlusDiffLine(line):
 			currentDiff.Content.Write(line[1:])
 		case isMinusDiffLine(line):
 			// NoOp. We only care about additions.
 		case isMessageLine(line):
 			currentCommit.Message.Write(line[4:])
 		case isContextDiffLine(line):
 			currentDiff.Content.Write([]byte("\n"))
 		case isBinaryLine(line):
 			currentDiff.IsBinary = true
 			currentDiff.PathB = pathFromBinaryLine(line)
 		case isLineNumberDiffLine(line):
 			if currentDiff != nil && currentDiff.Content.Len() > 0 {
 				currentCommit.Diffs = append(currentCommit.Diffs, *currentDiff)
 			}
 			newDiff := &Diff{
 				PathB: currentDiff.PathB,
 			}
 			currentDiff = newDiff
 			words := bytes.Split(line, []byte(" "))
 			if len(words) >= 3 {
 				startSlice := bytes.Split(words[2], []byte(","))
 				lineStart, err := strconv.Atoi(string(startSlice[0]))
 				if err == nil {
 					currentDiff.LineStart = lineStart
 				}
 			}
 		}
 		if currentDiff.Content.Len() > c.maxDiffSize {
 			ctx.Logger().V(2).Info(fmt.Sprintf(
 				"Diff for %s exceeded MaxDiffSize(%d)", currentDiff.PathB, c.maxDiffSize,
 			))
 			break
 		}
 	}
 	cleanupParse(currentCommit, currentDiff, commitChan)
 }
 func (c *Parser) funkReader(ctx context.Context, stdOut io.Reader, commitChan chan Commit, currentCommit *Commit) {
 	outReader := bufio.NewReader(stdOut)
 	var currentDiff *Diff
 	defer common.RecoverWithExit(ctx)
 	for {
 		if common.IsDone(ctx) {
 			break
 		}
 		line, err := outReader.ReadBytes([]byte("\n")[0])
 		if err != nil && len(line) == 0 {
 			break
 		}
 		// for i := 0; i < len(line); i++ {
 		// 	b := make([]byte, 1)
 		// 	n, err := stdOut.Read(b)
 		// 	if errors.Is(err, io.EOF) || n == 0 {
 		// 		return
 		// 	}
 		// 	line[i] = b[0]
 		// 	if b[0] == byte('\n') {
 		// 		line = line[:0]
 		// 		break
 		// 	}
 		// }
 		switch {
 		case isCommitLine(line):
 			// If there is a currentDiff, add it to currentCommit.
@@ -272,14 +377,17 @@ func (c *Parser) fromReader(ctx context.Context, stdOut io.Reader, commitChan ch
 					totalSize += diff.Content.Len()
 				}
 				if totalSize > c.maxCommitSize {
 					oldCommit := currentCommit
 					commitChan <- *currentCommit
 					currentCommit = &Commit{
 						Hash:    currentCommit.Hash,
 						Author:  currentCommit.Author,
 						Date:    currentCommit.Date,
-						Message: currentCommit.Message,
+						Message: strings.Builder{},
 						Diffs:   []Diff{},
 					}
 					// Message needs to be recreated here otherwise writing to it again will result in a panic.
 					currentCommit.Message.WriteString(oldCommit.Message.String())
 				}
 			}
 			currentDiff = &Diff{}
@@ -1267,6 +1267,117 @@ var _ interface {
 	ErrorName() string
 } = GitlabValidationError{}
 // Validate checks the field values on GoogleDrive with the rules defined in
 // the proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.
 func (m *GoogleDrive) Validate() error {
 	return m.validate(false)
 }
 // ValidateAll checks the field values on GoogleDrive with the rules defined in
 // the proto definition for this message. If any rules are violated, the
 // result is a list of violation errors wrapped in GoogleDriveMultiError, or
 // nil if none found.
 func (m *GoogleDrive) ValidateAll() error {
 	return m.validate(true)
 }
 func (m *GoogleDrive) validate(all bool) error {
 	if m == nil {
 		return nil
 	}
 	var errors []error
 	// no validation rules for File
 	// no validation rules for Link
 	// no validation rules for Email
 	// no validation rules for Timestamp
 	// no validation rules for Shared
 	// no validation rules for LastModifiedBy
 	if len(errors) > 0 {
 		return GoogleDriveMultiError(errors)
 	}
 	return nil
 }
 // GoogleDriveMultiError is an error wrapping multiple validation errors
 // returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
 type GoogleDriveMultiError []error
 // Error returns a concatenation of all the error messages it wraps.
 func (m GoogleDriveMultiError) Error() string {
 	var msgs []string
 	for _, err := range m {
 		msgs = append(msgs, err.Error())
 	}
 	return strings.Join(msgs, "; ")
 }
 // AllErrors returns a list of validation violation errors.
 func (m GoogleDriveMultiError) AllErrors() []error { return m }
 // GoogleDriveValidationError is the validation error returned by
 // GoogleDrive.Validate if the designated constraints aren't met.
 type GoogleDriveValidationError struct {
 	field  string
 	reason string
 	cause  error
 	key    bool
 }
 // Field function returns field value.
 func (e GoogleDriveValidationError) Field() string { return e.field }
 // Reason function returns reason value.
 func (e GoogleDriveValidationError) Reason() string { return e.reason }
 // Cause function returns cause value.
 func (e GoogleDriveValidationError) Cause() error { return e.cause }
 // Key function returns key value.
 func (e GoogleDriveValidationError) Key() bool { return e.key }
 // ErrorName returns error name.
 func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
 // Error satisfies the builtin error interface
 func (e GoogleDriveValidationError) Error() string {
 	cause := ""
 	if e.cause != nil {
 		cause = fmt.Sprintf(" | caused by: %v", e.cause)
 	}
 	key := ""
 	if e.key {
 		key = "key for "
 	}
 	return fmt.Sprintf(
 		"invalid %sGoogleDrive.%s: %s%s",
 		key,
 		e.field,
 		e.reason,
 		cause)
 }
 var _ error = GoogleDriveValidationError{}
 var _ interface {
 	Field() string
 	Reason() string
 	Key() bool
 	Cause() error
 	ErrorName() string
 } = GoogleDriveValidationError{}
 // Validate checks the field values on GCS with the rules defined in the proto
 // definition for this message. If any rules are violated, the first error
 // encountered is returned, or nil if there are no violations.
@@ -1887,6 +1887,112 @@ var _ interface {
 	ErrorName() string
 } = GitHubValidationError{}
 // Validate checks the field values on GoogleDrive with the rules defined in
 // the proto definition for this message. If any rules are violated, the first
 // error encountered is returned, or nil if there are no violations.
 func (m *GoogleDrive) Validate() error {
 	return m.validate(false)
 }
 // ValidateAll checks the field values on GoogleDrive with the rules defined in
 // the proto definition for this message. If any rules are violated, the
 // result is a list of violation errors wrapped in GoogleDriveMultiError, or
 // nil if none found.
 func (m *GoogleDrive) ValidateAll() error {
 	return m.validate(true)
 }
 func (m *GoogleDrive) validate(all bool) error {
 	if m == nil {
 		return nil
 	}
 	var errors []error
 	switch m.Credential.(type) {
 	case *GoogleDrive_RefreshToken:
 		// no validation rules for RefreshToken
 	}
 	if len(errors) > 0 {
 		return GoogleDriveMultiError(errors)
 	}
 	return nil
 }
 // GoogleDriveMultiError is an error wrapping multiple validation errors
 // returned by GoogleDrive.ValidateAll() if the designated constraints aren't met.
 type GoogleDriveMultiError []error
 // Error returns a concatenation of all the error messages it wraps.
 func (m GoogleDriveMultiError) Error() string {
 	var msgs []string
 	for _, err := range m {
 		msgs = append(msgs, err.Error())
 	}
 	return strings.Join(msgs, "; ")
 }
 // AllErrors returns a list of validation violation errors.
 func (m GoogleDriveMultiError) AllErrors() []error { return m }
 // GoogleDriveValidationError is the validation error returned by
 // GoogleDrive.Validate if the designated constraints aren't met.
 type GoogleDriveValidationError struct {
 	field  string
 	reason string
 	cause  error
 	key    bool
 }
 // Field function returns field value.
 func (e GoogleDriveValidationError) Field() string { return e.field }
 // Reason function returns reason value.
 func (e GoogleDriveValidationError) Reason() string { return e.reason }
 // Cause function returns cause value.
 func (e GoogleDriveValidationError) Cause() error { return e.cause }
 // Key function returns key value.
 func (e GoogleDriveValidationError) Key() bool { return e.key }
 // ErrorName returns error name.
 func (e GoogleDriveValidationError) ErrorName() string { return "GoogleDriveValidationError" }
 // Error satisfies the builtin error interface
 func (e GoogleDriveValidationError) Error() string {
 	cause := ""
 	if e.cause != nil {
 		cause = fmt.Sprintf(" | caused by: %v", e.cause)
 	}
 	key := ""
 	if e.key {
 		key = "key for "
 	}
 	return fmt.Sprintf(
 		"invalid %sGoogleDrive.%s: %s%s",
 		key,
 		e.field,
 		e.reason,
 		cause)
 }
 var _ error = GoogleDriveValidationError{}
 var _ interface {
 	Field() string
 	Reason() string
 	Key() bool
 	Cause() error
 	ErrorName() string
 } = GoogleDriveValidationError{}
 // Validate checks the field values on JIRA with the rules defined in the proto
 // definition for this message. If any rules are violated, the first error
 // encountered is returned, or nil if there are no violations.
@@ -1,6 +1,7 @@
 package sources
 import (
 	"fmt"
 	"sync"
 )
@@ -29,3 +30,7 @@ func (s *ScanErrors) Count() uint64 {
 	defer s.mu.RUnlock()
 	return uint64(len(s.errors))
 }
 func (s *ScanErrors) String() string {
 	return fmt.Sprintf("%v", s.errors)
 }
@@ -1,6 +1,7 @@
 package sources
 import (
 	"fmt"
 	"reflect"
 	"sync"
 	"testing"
@@ -147,3 +148,12 @@ func TestScanErrorsCount(t *testing.T) {
 		})
 	}
 }
 func TestScanErrorsString(t *testing.T) {
 	se := NewScanErrors()
 	se.Add(nil)
 	want := "[<nil>]"
 	if got := fmt.Sprintf("%v", se); got != want {
 		t.Errorf("got %q, want %q", got, want)
 	}
 }
@@ -70,8 +70,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,
 	if err := anypb.UnmarshalTo(connection, &conn, proto.UnmarshalOptions{}); err != nil {
 		return errors.WrapPrefix(err, "error unmarshalling connection", 0)
 	}
-
+	s.paths = append(conn.Paths, conn.Directories...)
 	s.paths = conn.Directories
 	return nil
 }
@@ -83,100 +82,122 @@ func (s *Source) WithFilter(filter *common.Filter) {
 // Chunks emits chunks of bytes over a channel.
 func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {
 	for i, path := range s.paths {
 		logger := ctx.Logger().WithValues("path", path)
 		if common.IsDone(ctx) {
 			return nil
 		}
 		s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", path), "")
 		cleanPath := filepath.Clean(path)
-		done := false
+		fileInfo, err := os.Stat(cleanPath)
-		go func() {
+		if err != nil {
-			<-ctx.Done()
+			logger.Error(err, "unable to get file info")
-			done = true
+			continue
-		}()
+		}
-		err := fs.WalkDir(os.DirFS(cleanPath), ".", func(relativePath string, d fs.DirEntry, err error) error {
+		if fileInfo.IsDir() {
-			if err != nil {
+			err = s.scanDir(ctx, cleanPath, chunksChan)
-				return nil
+		} else {
-			}
+			err = s.scanFile(ctx, cleanPath, chunksChan)
-
+		}
 			path := filepath.Join(cleanPath, relativePath)
 			logger := ctx.Logger().WithValues("file", path)
 			fileStat, err := os.Stat(path)
 			if err != nil {
 				logger.Error(err, "unable to stat file")
 				return nil
 			}
 			if !fileStat.Mode().IsRegular() {
 				return nil
 			}
 			if s.filter != nil && !s.filter.Pass(path) {
 				return nil
 			}
 			inputFile, err := os.Open(path)
 			if err != nil {
 				logger.V(2).Info("error opening file", "error", err)
 				return nil
 			}
 			defer inputFile.Close()
 			logger.V(5).Info("scanning file")
 			reReader, err := diskbufferreader.New(inputFile)
 			if err != nil {
 				logger.Error(err, "Could not create re-readable reader.")
 			}
 			defer reReader.Close()
 			chunkSkel := &sources.Chunk{
 				SourceType: s.Type(),
 				SourceName: s.name,
 				SourceID:   s.SourceID(),
 				SourceMetadata: &source_metadatapb.MetaData{
 					Data: &source_metadatapb.MetaData_Filesystem{
 						Filesystem: &source_metadatapb.Filesystem{
 							File: sanitizer.UTF8(path),
 						},
 					},
 				},
 				Verify: s.verify,
 			}
 			if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
 				return nil
 			}
 			if err := reReader.Reset(); err != nil {
 				return err
 			}
 			reReader.Stop()
 			data, err := io.ReadAll(reReader)
 			if err != nil {
 				return err
 			}
 			chunksChan <- &sources.Chunk{
 				SourceType: s.Type(),
 				SourceName: s.name,
 				SourceID:   s.SourceID(),
 				Data:       data,
 				SourceMetadata: &source_metadatapb.MetaData{
 					Data: &source_metadatapb.MetaData_Filesystem{
 						Filesystem: &source_metadatapb.Filesystem{
 							File: sanitizer.UTF8(path),
 						},
 					},
 				},
 				Verify: s.verify,
 			}
 			return nil
 		})
 		if err != nil && err != io.EOF {
-			return errors.New(err)
+			logger.Info("error scanning filesystem", "error", err)
 		}
-
+	}
-		if done {
+	return nil
-			return nil
+}
-		}
+
-
+func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
 	return fs.WalkDir(os.DirFS(path), ".", func(relativePath string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return nil
 		}
 		fullPath := filepath.Join(path, relativePath)
 		// Skip over non-regular files. We do this check here to suppress noisy
 		// logs for trying to scan directories and other non-regular files in
 		// our traversal.
 		fileStat, err := os.Stat(fullPath)
 		if err != nil {
 			ctx.Logger().Info("unable to stat file", "path", fullPath, "error", err)
 			return nil
 		}
 		if !fileStat.Mode().IsRegular() {
 			return nil
 		}
 		if s.filter != nil && !s.filter.Pass(fullPath) {
 			return nil
 		}
 		if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {
 			ctx.Logger().Info("error scanning file", "path", fullPath, "error", err)
 		}
 		return nil
 	})
 }
 func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sources.Chunk) error {
 	logger := ctx.Logger().WithValues("path", path)
 	fileStat, err := os.Stat(path)
 	if err != nil {
 		return fmt.Errorf("unable to stat file: %w", err)
 	}
 	if !fileStat.Mode().IsRegular() {
 		return fmt.Errorf("not a regular file")
 	}
 	inputFile, err := os.Open(path)
 	if err != nil {
 		return fmt.Errorf("unable to open file: %w", err)
 	}
 	defer inputFile.Close()
 	logger.V(3).Info("scanning file")
 	reReader, err := diskbufferreader.New(inputFile)
 	if err != nil {
 		return fmt.Errorf("could not create re-readable reader: %w", err)
 	}
 	defer reReader.Close()
 	chunkSkel := &sources.Chunk{
 		SourceType: s.Type(),
 		SourceName: s.name,
 		SourceID:   s.SourceID(),
 		SourceMetadata: &source_metadatapb.MetaData{
 			Data: &source_metadatapb.MetaData_Filesystem{
 				Filesystem: &source_metadatapb.Filesystem{
 					File: sanitizer.UTF8(path),
 				},
 			},
 		},
 		Verify: s.verify,
 	}
 	if handlers.HandleFile(ctx, reReader, chunkSkel, chunksChan) {
 		return nil
 	}
 	if err := reReader.Reset(); err != nil {
 		return err
 	}
 	reReader.Stop()
 	data, err := io.ReadAll(reReader)
 	if err != nil {
 		return fmt.Errorf("unable to read file: %w", err)
 	}
 	chunksChan <- &sources.Chunk{
 		SourceType: s.Type(),
 		SourceName: s.name,
 		SourceID:   s.SourceID(),
 		Data:       data,
 		SourceMetadata: &source_metadatapb.MetaData{
 			Data: &source_metadatapb.MetaData_Filesystem{
 				Filesystem: &source_metadatapb.Filesystem{
 					File: sanitizer.UTF8(path),
 				},
 			},
 		},
 		Verify: s.verify,
 	}
 	return nil
 }
@@ -33,7 +33,7 @@ func TestSource_Scan(t *testing.T) {
 			init: init{
 				name: "this repo",
 				connection: &sourcespb.Filesystem{
-					Directories: []string{"."},
+					Paths: []string{"."},
 				},
 				verify: true,
 			},
@@ -5,7 +5,6 @@ import (
 	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"net/url"
 	"os"
 	"os/exec"
@@ -257,7 +256,7 @@ func CloneRepo(ctx context.Context, userInfo *url.Userinfo, gitUrl string, args
 	if err := gitCmdCheck(); err != nil {
 		return "", nil, err
 	}
-	clonePath, err := ioutil.TempDir(os.TempDir(), "trufflehog")
+	clonePath, err := os.MkdirTemp(os.TempDir(), "trufflehog")
 	if err != nil {
 		return "", nil, err
 	}
@@ -339,7 +338,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path string
 		return err
 	}
-	commitChan, err := gitparse.NewParser().BeepBoop("main")
+	commitChan, err := gitparse.NewParser().RepoPath(ctx, path, scanOptions.HeadHash, scanOptions.BaseHash == "")
 	if err != nil {
 		return err
 	}
@@ -473,9 +472,9 @@ func (s *Git) gitChunk(ctx context.Context, diff gitparse.Diff, fileName, email,
 	}
 }
-// ScanUnstaged chunks unstaged changes.
+// ScanStaged chunks staged changes.
-func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
+func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string, scanOptions *ScanOptions, chunksChan chan *sources.Chunk) error {
-	// get the URL metadata for reporting (may be empty)
+	// Get the URL metadata for reporting (may be empty).
 	urlMetadata := getSafeRemoteURL(repo, "origin")
 	commitChan, err := gitparse.NewParser().Unstaged(ctx, path)
@@ -489,11 +488,11 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 	var depth int64
 	var reachedBase = false
-	ctx.Logger().V(1).Info("scanning unstaged changes", "path", path)
+	ctx.Logger().V(1).Info("scanning staged changes", "path", path)
 	for commit := range commitChan {
 		for _, diff := range commit.Diffs {
 			logger := ctx.Logger().WithValues("filename", diff.PathB, "commit", commit.Hash, "file", diff.PathB)
-			logger.V(2).Info("scanning unstaged changes from git")
+			logger.V(2).Info("scanning staged changes from git")
 			if scanOptions.MaxDepth > 0 && depth >= scanOptions.MaxDepth {
 				logger.V(1).Info("reached max depth")
@@ -526,7 +525,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 			// Handle binary files by reading the entire file rather than using the diff.
 			if diff.IsBinary {
 				commitHash := plumbing.NewHash(hash)
-				metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, 0)
+				metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, 0)
 				chunkSkel := &sources.Chunk{
 					SourceName:     s.sourceName,
 					SourceID:       s.sourceID,
@@ -540,7 +539,7 @@ func (s *Git) ScanUnstaged(ctx context.Context, repo *git.Repository, path strin
 				continue
 			}
-			metadata := s.sourceMetadataFunc(fileName, email, "Unstaged", when, urlMetadata, int64(diff.LineStart))
+			metadata := s.sourceMetadataFunc(fileName, email, "Staged", when, urlMetadata, int64(diff.LineStart))
 			chunksChan <- &sources.Chunk{
 				SourceName:     s.sourceName,
 				SourceID:       s.sourceID,
@@ -565,7 +564,7 @@ func (s *Git) ScanRepo(ctx context.Context, repo *git.Repository, repoPath strin
 	if err := s.ScanCommits(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		return err
 	}
-	if err := s.ScanUnstaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
+	if err := s.ScanStaged(ctx, repo, repoPath, scanOptions, chunksChan); err != nil {
 		ctx.Logger().V(1).Info("error scanning unstaged changes", "error", err)
 	}
@@ -137,7 +137,7 @@ func TestSource_Scan(t *testing.T) {
 			}
 			chunksCh := make(chan *sources.Chunk, 1)
 			go func() {
-				s.Chunks(ctx, chunksCh)
+				assert.NoError(t, s.Chunks(ctx, chunksCh))
 			}()
 			gotChunk := <-chunksCh
 			gotChunk.Data = nil
@@ -96,8 +96,8 @@ type GitlabConfig struct {
 // FilesystemConfig defines the optional configuration for a filesystem source.
 type FilesystemConfig struct {
-	// Directories is the list of directories to scan.
+	// Paths is the list of files and directories to scan.
-	Directories []string
+	Paths []string
 	// Filter is the filter to use to scan the source.
 	Filter *common.Filter
 }
@@ -111,6 +111,15 @@ message Gitlab {
  int64 line = 7;
 }
 message GoogleDrive {
  string file = 1;
  string link = 2;
  string email = 3;
  string timestamp = 4;
  bool shared = 5;
  string last_modified_by = 6;
 }
 message GCS {
  string bucket = 1;
  string file = 2;
@@ -40,6 +40,7 @@ enum SourceType {
  SOURCE_TYPE_SYSLOG = 25;
  SOURCE_TYPE_PUBLIC_EVENT_MONITORING = 26;
  SOURCE_TYPE_SLACK_REALTIME = 27;
  SOURCE_TYPE_GOOGLE_DRIVE = 28;
 }
 message LocalSource {
@@ -115,8 +116,11 @@ message ECR {
  repeated string registries = 2;
 }
-message Filesystem{
+message Filesystem {
  // DEPRECATED: directories is deprecated and can be removed / renamed to
  // paths when we no longer depend on the name in enterprise configs.
  repeated string directories = 1;
  repeated string paths = 2;
 }
 message GCS {
@@ -164,6 +168,12 @@ message GitHub {
  repeated string includeRepos = 12;
 }
 message GoogleDrive {
  oneof credential {
    string refresh_token = 1;
  }
 }
 message JIRA {
  string endpoint = 1 [(validate.rules).string.uri_ref = true];
  oneof credential {
Author	SHA1	Message	Date
bill-rich	1bbc35e5be	Testing	2023-03-07 08:39:18 -08:00
Miccah	e6846ede54	Support filtering detectors by version (#1150 ) * Adjust types to use DetectorID struct * Parse versions with detector include and exclude input * Update detectors filter to use version Co-authored-by: steeeve <steve@trufflesec.com> * Implement Versioner for github, gitlab, and npm detectors Co-authored-by: steeeve <steve@trufflesec.com> --------- Co-authored-by: steeeve <steve@trufflesec.com>	2023-03-02 16:33:56 -06:00
Dustin Decker	4500ac3b10	Release should only run on tags (#1146 )	2023-03-02 09:57:38 -08:00
Zachary Rice	4777b77ec6	Keyword optimization (#1144 ) * init * ignore trufflehog binary and added comment * remove unused keywords in chunk, better comment * remove keywords from engine struct	2023-03-02 11:32:37 -06:00
zhuwenxing	c72840de67	Rename .pre-commit-hooks.yml to .pre-commit-hooks.yaml (#1141 ) Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>	2023-03-01 09:00:12 -08:00
ahrav	aa47e5e248	Only scanned staged git changes. (#1143 )	2023-03-01 08:58:36 -08:00
Yassine Ilmi	0cf9139df6	Disable profiler in debug mode and add profile switch (#1136 )	2023-02-28 12:49:54 -08:00
ahrav	86370333ec	Add pre-commit yml config (#1138 ) * Add pre-commit yml config. * Add --fail flag.	2023-02-28 11:31:24 -08:00
dependabot[bot]	64c163cd90	Bump github.com/stretchr/testify from 1.8.1 to 1.8.2 (#1130 ) Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.1 to 1.8.2. - [Release notes](https://github.com/stretchr/testify/releases) - [Commits](https://github.com/stretchr/testify/compare/v1.8.1...v1.8.2) --- updated-dependencies: - dependency-name: github.com/stretchr/testify dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-02-28 08:46:04 -08:00
Miccah	3870be256c	Close response bodies (#1137 )	2023-02-28 10:43:00 -06:00
ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	2315192fda	Custom regex parallel verify (#1127 ) * Refactor generating CustomRegex results into a helper function * Added errGroup for createResults, and ensure goroutines are non-blocking * clean return --------- Co-authored-by: Miccah Castorina <m.castorina93@gmail.com>	2023-02-28 11:12:24 -05:00
Miccah	6209a80ce1	[chore] Address more linter errors (#1134 ) * Address lint errors in detectors * Update deprecated ioutil call	2023-02-28 10:00:41 -06:00
Miccah	4efe5313f4	[chore] Address lint errors (#1133 ) * Update strings.Title to cases.Title * Migrate go-genproto to google-cloud-go See: https://github.com/googleapis/google-cloud-go/blob/main/migration.md * Check error in test * Check error from sem.Acquire * Remove unused code	2023-02-27 21:03:47 -06:00
Miccah	7373954ddf	[chore] Update docs for individual file scanning (#1132 )	2023-02-27 21:03:19 -06:00
Miccah	d2d03426ed	Implement String for ScanErrors (#1131 ) This will concatenate all errors together into a single string. When possible, it would be better to log the actual errors slice to take advantage of structured logging.	2023-02-27 21:02:59 -06:00
Miccah	dd39848709	Add ability to include and exclude detectors (#1106 ) * Add ability to include and exclude detectors * Trim space before checking for empty items * Explicitly check for integer overflow * Use strconv.ParseInt instead of strconv.Atoi * Address comments	2023-02-27 16:46:45 -06:00
Miccah	c5b4d6f28b	Support file scanning in filesystem source (#1030 ) * Rename directories to paths * Generate protos * Add file scanning support to filesystem source * Add directories back to filesystem proto * Generate protos * Combine paths and directories from in source * Add filesystem filter * Address comments	2023-02-27 12:15:05 -06:00
Bill Rich	ae2d510ced	Gitparse message fix (#1125 ) * Fix messages being reused * Add comment about change.	2023-02-23 15:20:54 -08:00
dependabot[bot]	05f6bd369f	Bump github.com/xanzy/go-gitlab from 0.78.0 to 0.80.2 (#1117 ) Bumps [github.com/xanzy/go-gitlab](https://github.com/xanzy/go-gitlab) from 0.78.0 to 0.80.2. - [Release notes](https://github.com/xanzy/go-gitlab/releases) - [Changelog](https://github.com/xanzy/go-gitlab/blob/master/releases_test.go) - [Commits](https://github.com/xanzy/go-gitlab/compare/v0.78.0...v0.80.2) --- updated-dependencies: - dependency-name: github.com/xanzy/go-gitlab dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-02-22 09:33:16 -08:00
raju-kamble	d151c1363e	fixing browserstack regex username detection (#1123 )	2023-02-22 08:17:48 -08:00
ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d	ee5b028c67	Adding initial protos for Google Drive scanner (#1121 )	2023-02-22 10:04:46 -05:00
dependabot[bot]	38562df0f6	Bump github.com/rabbitmq/amqp091-go from 1.6.0 to 1.7.0 (#1103 ) Bumps [github.com/rabbitmq/amqp091-go](https://github.com/rabbitmq/amqp091-go) from 1.6.0 to 1.7.0. - [Release notes](https://github.com/rabbitmq/amqp091-go/releases) - [Changelog](https://github.com/rabbitmq/amqp091-go/blob/main/CHANGELOG.md) - [Commits](https://github.com/rabbitmq/amqp091-go/compare/v1.6.0...v1.7.0) --- updated-dependencies: - dependency-name: github.com/rabbitmq/amqp091-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-02-21 15:36:46 -08:00
dependabot[bot]	4f0e66f064	Bump github.com/TheZeroSlave/zapsentry from 1.12.0 to 1.14.0 (#1118 ) Bumps [github.com/TheZeroSlave/zapsentry](https://github.com/TheZeroSlave/zapsentry) from 1.12.0 to 1.14.0. - [Release notes](https://github.com/TheZeroSlave/zapsentry/releases) - [Commits](https://github.com/TheZeroSlave/zapsentry/compare/v1.12.0...v1.14.0) --- updated-dependencies: - dependency-name: github.com/TheZeroSlave/zapsentry dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-02-21 15:36:16 -08:00
dependabot[bot]	0b4e3a66b1	Bump go.mongodb.org/mongo-driver from 1.11.1 to 1.11.2 (#1119 ) Bumps [go.mongodb.org/mongo-driver](https://github.com/mongodb/mongo-go-driver) from 1.11.1 to 1.11.2. - [Release notes](https://github.com/mongodb/mongo-go-driver/releases) - [Commits](https://github.com/mongodb/mongo-go-driver/compare/v1.11.1...v1.11.2) --- updated-dependencies: - dependency-name: go.mongodb.org/mongo-driver dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-02-21 15:35:53 -08:00
dependabot[bot]	a293033386	Bump golang.org/x/net from 0.6.0 to 0.7.0 (#1122 ) Bumps [golang.org/x/net](https://github.com/golang/net) from 0.6.0 to 0.7.0. - [Release notes](https://github.com/golang/net/releases) - [Commits](https://github.com/golang/net/compare/v0.6.0...v0.7.0) --- updated-dependencies: - dependency-name: golang.org/x/net dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-02-21 15:35:21 -08:00
raju-kamble	d20f43b5c6	fix browserstack detector (#1120 ) * fixing browserstack regex username detection * fixing browserstack regex username detection * fixing browserstack regex username detection * fix patterns * fix patterns --------- Co-authored-by: raju-bs <raju@browserstack.com> Co-authored-by: Dustin Decker <dustin@trufflesec.com>	2023-02-21 15:33:16 -08:00
dependabot[bot]	9ef9e9870d	Bump golang.org/x/oauth2 from 0.4.0 to 0.5.0 (#1116 ) Bumps [golang.org/x/oauth2](https://github.com/golang/oauth2) from 0.4.0 to 0.5.0. - [Release notes](https://github.com/golang/oauth2/releases) - [Commits](https://github.com/golang/oauth2/compare/v0.4.0...v0.5.0) --- updated-dependencies: - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>	2023-02-21 13:42:36 -08:00